Language fixes (#5384)

* Add some comments

* Decouple language settings

* Normalize on read/write

* Refactor

* Support device locale on app startup

* Cleanup, port to web

* Clean up comments

* Comment

* Try not to mutate

* Protect util handling, update test

* Dedupe array values

src/state/persisted/index.ts

@@ -8,6 +8,7 @@ import {
   tryStringify,
 } from '#/state/persisted/schema'
 import {PersistedApi} from './types'
+import {normalizeData} from './util'
 
 export type {PersistedAccount, Schema} from '#/state/persisted/schema'
 export {defaults} from '#/state/persisted/schema'
@@ -33,10 +34,10 @@ export async function write<K extends keyof Schema>(
   key: K,
   value: Schema[K],
 ): Promise<void> {
-  _state = {
+  _state = normalizeData({
     ..._state,
     [key]: value,
-  }
+  })
   await writeToStorage(_state)
 }
 write satisfies PersistedApi['write']
@@ -81,6 +82,9 @@ async function readFromStorage(): Promise<Schema | undefined> {
     })
   }
   if (rawData) {
-    return tryParse(rawData)
+    const parsed = tryParse(rawData)
+    if (parsed) {
+      return normalizeData(parsed)
+    }
   }
 }

src/state/persisted/index.web.ts

@@ -9,6 +9,7 @@ import {
   tryStringify,
 } from '#/state/persisted/schema'
 import {PersistedApi} from './types'
+import {normalizeData} from './util'
 
 export type {PersistedAccount, Schema} from '#/state/persisted/schema'
 export {defaults} from '#/state/persisted/schema'
@@ -56,10 +57,10 @@ export async function write<K extends keyof Schema>(
   } catch (e) {
     // Ignore and go through the normal path.
   }
-  _state = {
+  _state = normalizeData({
     ..._state,
     [key]: value,
-  }
+  })
   writeToStorage(_state)
   broadcast.postMessage({event: {type: UPDATE_EVENT, key}})
   broadcast.postMessage({event: UPDATE_EVENT}) // Backcompat while upgrading
@@ -140,9 +141,11 @@ function readFromStorage(): Schema | undefined {
       return lastResult
     } else {
       const result = tryParse(rawData)
-      lastRawData = rawData
-      lastResult = result
-      return result
+      if (result) {
+        lastRawData = rawData
+        lastResult = normalizeData(result)
+        return lastResult
+      }
     }
   }
 }

src/state/persisted/schema.ts

@@ -1,7 +1,8 @@
 import {z} from 'zod'
 
+import {deviceLanguageCodes, deviceLocales} from '#/locale/deviceLocales'
+import {findSupportedAppLanguage} from '#/locale/helpers'
 import {logger} from '#/logger'
-import {deviceLocales} from '#/platform/detection'
 import {PlatformInfo} from '../../../modules/expo-bluesky-swiss-army'
 
 const externalEmbedOptions = ['show', 'hide'] as const
@@ -55,10 +56,39 @@ const schema = z.object({
     lastEmailConfirm: z.string().optional(),
   }),
   languagePrefs: z.object({
-    primaryLanguage: z.string(), // should move to server
-    contentLanguages: z.array(z.string()), // should move to server
-    postLanguage: z.string(), // should move to server
+    /**
+     * The target language for translating posts.
+     *
+     * BCP-47 2-letter language code without region.
+     */
+    primaryLanguage: z.string(),
+    /**
+     * The languages the user can read, passed to feeds.
+     *
+     * BCP-47 2-letter language codes without region.
+     */
+    contentLanguages: z.array(z.string()),
+    /**
+     * The language(s) the user is currently posting in, configured within the
+     * composer. Multiple languages are psearate by commas.
+     *
+     * BCP-47 2-letter language code without region.
+     */
+    postLanguage: z.string(),
+    /**
+     * The user's post language history, used to pre-populate the post language
+     * selector in the composer. Within each value, multiple languages are
+     * separated by values.
+     *
+     * BCP-47 2-letter language codes without region.
+     */
     postLanguageHistory: z.array(z.string()),
+    /**
+     * The language for UI translations in the app.
+     *
+     * BCP-47 2-letter language code with or without region,
+     * to match with {@link AppLanguage}.
+     */
     appLanguage: z.string(),
   }),
   requireAltTextEnabled: z.boolean(), // should move to server
@@ -108,13 +138,17 @@ export const defaults: Schema = {
     lastEmailConfirm: undefined,
   },
   languagePrefs: {
-    primaryLanguage: deviceLocales[0] || 'en',
-    contentLanguages: deviceLocales || [],
-    postLanguage: deviceLocales[0] || 'en',
-    postLanguageHistory: (deviceLocales || [])
+    primaryLanguage: deviceLanguageCodes[0] || 'en',
+    contentLanguages: deviceLanguageCodes || [],
+    postLanguage: deviceLanguageCodes[0] || 'en',
+    postLanguageHistory: (deviceLanguageCodes || [])
       .concat(['en', 'ja', 'pt', 'de'])
       .slice(0, 6),
-    appLanguage: deviceLocales[0] || 'en',
+    // try full language tag first, then fallback to language code
+    appLanguage: findSupportedAppLanguage([
+      deviceLocales.at(0)?.languageTag,
+      deviceLanguageCodes[0],
+    ]),
   },
   requireAltTextEnabled: false,
   largeAltBadgeEnabled: false,

src/state/persisted/util.ts

@@ -0,0 +1,51 @@
+import {parse} from 'bcp-47'
+
+import {dedupArray} from '#/lib/functions'
+import {logger} from '#/logger'
+import {Schema} from '#/state/persisted/schema'
+
+export function normalizeData(data: Schema) {
+  const next = {...data}
+
+  /**
+   * Normalize language prefs to ensure that these values only contain 2-letter
+   * country codes without region.
+   */
+  try {
+    const langPrefs = {...next.languagePrefs}
+    langPrefs.primaryLanguage = normalizeLanguageTagToTwoLetterCode(
+      langPrefs.primaryLanguage,
+    )
+    langPrefs.contentLanguages = dedupArray(
+      langPrefs.contentLanguages.map(lang =>
+        normalizeLanguageTagToTwoLetterCode(lang),
+      ),
+    )
+    langPrefs.postLanguage = langPrefs.postLanguage
+      .split(',')
+      .map(lang => normalizeLanguageTagToTwoLetterCode(lang))
+      .filter(Boolean)
+      .join(',')
+    langPrefs.postLanguageHistory = dedupArray(
+      langPrefs.postLanguageHistory.map(postLanguage => {
+        return postLanguage
+          .split(',')
+          .map(lang => normalizeLanguageTagToTwoLetterCode(lang))
+          .filter(Boolean)
+          .join(',')
+      }),
+    )
+    next.languagePrefs = langPrefs
+  } catch (e: any) {
+    logger.error(`persisted state: failed to normalize language prefs`, {
+      safeMessage: e.message,
+    })
+  }
+
+  return next
+}
+
+export function normalizeLanguageTagToTwoLetterCode(lang: string) {
+  const result = parse(lang).language
+  return result ?? lang
+}