Language fixes (#5384)

* Add some comments

* Decouple language settings

* Normalize on read/write

* Refactor

* Support device locale on app startup

* Cleanup, port to web

* Clean up comments

* Comment

* Try not to mutate

* Protect util handling, update test

* Dedupe array values

package.json

@@ -110,6 +110,7 @@
     "await-lock": "^2.2.2",
     "babel-plugin-transform-remove-console": "^6.9.4",
     "base64-js": "^1.5.1",
+    "bcp-47": "^2.1.0",
     "bcp-47-match": "^2.0.3",
     "date-fns": "^2.30.0",
     "deprecated-react-native-prop-types": "^5.0.0",

src/components/AppLanguageDropdown.tsx

@@ -24,8 +24,6 @@ export function AppLanguageDropdown() {
       if (sanitizedLang !== value) {
         setLangPrefs.setAppLanguage(sanitizeAppLanguageSetting(value))
       }
-      setLangPrefs.setPrimaryLanguage(value)
-      setLangPrefs.setContentLanguage(value)
 
       // reset feeds to refetch content
       resetPostsFeedQueries(queryClient)

src/components/AppLanguageDropdown.web.tsx

@@ -27,8 +27,6 @@ export function AppLanguageDropdown() {
       if (sanitizedLang !== value) {
         setLangPrefs.setAppLanguage(sanitizeAppLanguageSetting(value))
       }
-      setLangPrefs.setPrimaryLanguage(value)
-      setLangPrefs.setContentLanguage(value)
 
       // reset feeds to refetch content
       resetPostsFeedQueries(queryClient)

src/locale/deviceLocales.ts

@@ -0,0 +1,53 @@
+import {getLocales as defaultGetLocales, Locale} from 'expo-localization'
+
+import {dedupArray} from '#/lib/functions'
+
+type LocalWithLanguageCode = Locale & {
+  languageCode: string
+}
+
+/**
+ * Normalized locales
+ *
+ * Handles legacy migration for Java devices.
+ *
+ * {@link https://github.com/bluesky-social/social-app/pull/4461}
+ * {@link https://xml.coverpages.org/iso639a.html}
+ */
+export function getLocales() {
+  const locales = defaultGetLocales?.() ?? []
+  const output: LocalWithLanguageCode[] = []
+
+  for (const locale of locales) {
+    if (typeof locale.languageCode === 'string') {
+      if (locale.languageCode === 'in') {
+        // indonesian
+        locale.languageCode = 'id'
+      }
+      if (locale.languageCode === 'iw') {
+        // hebrew
+        locale.languageCode = 'he'
+      }
+      if (locale.languageCode === 'ji') {
+        // yiddish
+        locale.languageCode = 'yi'
+      }
+
+      // @ts-ignore checked above
+      output.push(locale)
+    }
+  }
+
+  return output
+}
+
+export const deviceLocales = getLocales()
+
+/**
+ * BCP-47 language tag without region e.g. array of 2-char lang codes
+ *
+ * {@link https://docs.expo.dev/versions/latest/sdk/localization/#locale}
+ */
+export const deviceLanguageCodes = dedupArray(
+  deviceLocales.map(l => l.languageCode),
+)

src/locale/helpers.ts

@@ -160,8 +160,13 @@ export function sanitizeAppLanguageSetting(appLanguage: string): AppLanguage {
   return AppLanguage.en
 }
 
+/**
+ * Handles legacy migration for Java devices.
+ *
+ * {@link https://github.com/bluesky-social/social-app/pull/4461}
+ * {@link https://xml.coverpages.org/iso639a.html}
+ */
 export function fixLegacyLanguageCode(code: string | null): string | null {
-  // handle some legacy code conversions, see https://xml.coverpages.org/iso639a.html
   if (code === 'in') {
     // indonesian
     return 'id'
@@ -176,3 +181,20 @@ export function fixLegacyLanguageCode(code: string | null): string | null {
   }
   return code
 }
+
+/**
+ * Find the first language supported by our translation infra. Values should be
+ * in order of preference, and match the values of {@link AppLanguage}.
+ *
+ * If no match, returns `en`.
+ */
+export function findSupportedAppLanguage(languageTags: (string | undefined)[]) {
+  const supported = new Set(Object.values(AppLanguage))
+  for (const tag of languageTags) {
+    if (!tag) continue
+    if (supported.has(tag as AppLanguage)) {
+      return tag
+    }
+  }
+  return AppLanguage.en
+}

src/platform/detection.ts

@@ -1,8 +1,4 @@
 import {Platform} from 'react-native'
-import {getLocales} from 'expo-localization'
-
-import {fixLegacyLanguageCode} from '#/locale/helpers'
-import {dedupArray} from 'lib/functions'
 
 export const isIOS = Platform.OS === 'ios'
 export const isAndroid = Platform.OS === 'android'
@@ -15,9 +11,3 @@ export const isMobileWeb =
   // @ts-ignore we know window exists -prf
   global.window.matchMedia(isMobileWebMediaQuery)?.matches
 export const isIPhoneWeb = isWeb && /iPhone/.test(navigator.userAgent)
-
-export const deviceLocales = dedupArray(
-  getLocales?.()
-    .map?.(locale => fixLegacyLanguageCode(locale.languageCode))
-    .filter(code => typeof code === 'string'),
-) as string[]

src/state/persisted/index.ts

@@ -8,6 +8,7 @@ import {
   tryStringify,
 } from '#/state/persisted/schema'
 import {PersistedApi} from './types'
+import {normalizeData} from './util'
 
 export type {PersistedAccount, Schema} from '#/state/persisted/schema'
 export {defaults} from '#/state/persisted/schema'
@@ -33,10 +34,10 @@ export async function write<K extends keyof Schema>(
   key: K,
   value: Schema[K],
 ): Promise<void> {
-  _state = {
+  _state = normalizeData({
     ..._state,
     [key]: value,
-  }
+  })
   await writeToStorage(_state)
 }
 write satisfies PersistedApi['write']
@@ -81,6 +82,9 @@ async function readFromStorage(): Promise<Schema | undefined> {
     })
   }
   if (rawData) {
-    return tryParse(rawData)
+    const parsed = tryParse(rawData)
+    if (parsed) {
+      return normalizeData(parsed)
+    }
   }
 }

src/state/persisted/index.web.ts

@@ -9,6 +9,7 @@ import {
   tryStringify,
 } from '#/state/persisted/schema'
 import {PersistedApi} from './types'
+import {normalizeData} from './util'
 
 export type {PersistedAccount, Schema} from '#/state/persisted/schema'
 export {defaults} from '#/state/persisted/schema'
@@ -56,10 +57,10 @@ export async function write<K extends keyof Schema>(
   } catch (e) {
     // Ignore and go through the normal path.
   }
-  _state = {
+  _state = normalizeData({
     ..._state,
     [key]: value,
-  }
+  })
   writeToStorage(_state)
   broadcast.postMessage({event: {type: UPDATE_EVENT, key}})
   broadcast.postMessage({event: UPDATE_EVENT}) // Backcompat while upgrading
@@ -140,9 +141,11 @@ function readFromStorage(): Schema | undefined {
       return lastResult
     } else {
       const result = tryParse(rawData)
-      lastRawData = rawData
+      if (result) {
-      lastResult = result
+        lastRawData = rawData
-      return result
+        lastResult = normalizeData(result)
+        return lastResult
+      }
     }
   }
 }

src/state/persisted/schema.ts

@@ -1,7 +1,8 @@
 import {z} from 'zod'
 
+import {deviceLanguageCodes, deviceLocales} from '#/locale/deviceLocales'
+import {findSupportedAppLanguage} from '#/locale/helpers'
 import {logger} from '#/logger'
-import {deviceLocales} from '#/platform/detection'
 import {PlatformInfo} from '../../../modules/expo-bluesky-swiss-army'
 
 const externalEmbedOptions = ['show', 'hide'] as const
@@ -55,10 +56,39 @@ const schema = z.object({
     lastEmailConfirm: z.string().optional(),
   }),
   languagePrefs: z.object({
-    primaryLanguage: z.string(), // should move to server
+    /**
-    contentLanguages: z.array(z.string()), // should move to server
+     * The target language for translating posts.
-    postLanguage: z.string(), // should move to server
+     *
+     * BCP-47 2-letter language code without region.
+     */
+    primaryLanguage: z.string(),
+    /**
+     * The languages the user can read, passed to feeds.
+     *
+     * BCP-47 2-letter language codes without region.
+     */
+    contentLanguages: z.array(z.string()),
+    /**
+     * The language(s) the user is currently posting in, configured within the
+     * composer. Multiple languages are psearate by commas.
+     *
+     * BCP-47 2-letter language code without region.
+     */
+    postLanguage: z.string(),
+    /**
+     * The user's post language history, used to pre-populate the post language
+     * selector in the composer. Within each value, multiple languages are
+     * separated by values.
+     *
+     * BCP-47 2-letter language codes without region.
+     */
     postLanguageHistory: z.array(z.string()),
+    /**
+     * The language for UI translations in the app.
+     *
+     * BCP-47 2-letter language code with or without region,
+     * to match with {@link AppLanguage}.
+     */
     appLanguage: z.string(),
   }),
   requireAltTextEnabled: z.boolean(), // should move to server
@@ -108,13 +138,17 @@ export const defaults: Schema = {
     lastEmailConfirm: undefined,
   },
   languagePrefs: {
-    primaryLanguage: deviceLocales[0] || 'en',
+    primaryLanguage: deviceLanguageCodes[0] || 'en',
-    contentLanguages: deviceLocales || [],
+    contentLanguages: deviceLanguageCodes || [],
-    postLanguage: deviceLocales[0] || 'en',
+    postLanguage: deviceLanguageCodes[0] || 'en',
-    postLanguageHistory: (deviceLocales || [])
+    postLanguageHistory: (deviceLanguageCodes || [])
       .concat(['en', 'ja', 'pt', 'de'])
       .slice(0, 6),
-    appLanguage: deviceLocales[0] || 'en',
+    // try full language tag first, then fallback to language code
+    appLanguage: findSupportedAppLanguage([
+      deviceLocales.at(0)?.languageTag,
+      deviceLanguageCodes[0],
+    ]),
   },
   requireAltTextEnabled: false,
   largeAltBadgeEnabled: false,

src/state/persisted/util.ts

@@ -0,0 +1,51 @@
+import {parse} from 'bcp-47'
+
+import {dedupArray} from '#/lib/functions'
+import {logger} from '#/logger'
+import {Schema} from '#/state/persisted/schema'
+
+export function normalizeData(data: Schema) {
+  const next = {...data}
+
+  /**
+   * Normalize language prefs to ensure that these values only contain 2-letter
+   * country codes without region.
+   */
+  try {
+    const langPrefs = {...next.languagePrefs}
+    langPrefs.primaryLanguage = normalizeLanguageTagToTwoLetterCode(
+      langPrefs.primaryLanguage,
+    )
+    langPrefs.contentLanguages = dedupArray(
+      langPrefs.contentLanguages.map(lang =>
+        normalizeLanguageTagToTwoLetterCode(lang),
+      ),
+    )
+    langPrefs.postLanguage = langPrefs.postLanguage
+      .split(',')
+      .map(lang => normalizeLanguageTagToTwoLetterCode(lang))
+      .filter(Boolean)
+      .join(',')
+    langPrefs.postLanguageHistory = dedupArray(
+      langPrefs.postLanguageHistory.map(postLanguage => {
+        return postLanguage
+          .split(',')
+          .map(lang => normalizeLanguageTagToTwoLetterCode(lang))
+          .filter(Boolean)
+          .join(',')
+      }),
+    )
+    next.languagePrefs = langPrefs
+  } catch (e: any) {
+    logger.error(`persisted state: failed to normalize language prefs`, {
+      safeMessage: e.message,
+    })
+  }
+
+  return next
+}
+
+export function normalizeLanguageTagToTwoLetterCode(lang: string) {
+  const result = parse(lang).language
+  return result ?? lang
+}

src/state/session/__tests__/session-test.ts

@@ -10,6 +10,10 @@ jest.mock('jwt-decode', () => ({
   },
 }))
 
+jest.mock('expo-localization', () => ({
+  getLocales: () => [],
+}))
+
 describe('session', () => {
   it('can log in and out', () => {
     let state = getInitialState([])

src/view/com/modals/lang-settings/ContentLanguagesSettings.tsx

@@ -1,19 +1,20 @@
 import React from 'react'
 import {StyleSheet, View} from 'react-native'
-import {ScrollView} from '../util'
-import {Text} from '../../util/text/Text'
-import {usePalette} from 'lib/hooks/usePalette'
-import {useWebMediaQueries} from 'lib/hooks/useWebMediaQueries'
-import {deviceLocales} from 'platform/detection'
-import {LANGUAGES, LANGUAGES_MAP_CODE2} from '../../../../locale/languages'
-import {LanguageToggle} from './LanguageToggle'
-import {ConfirmLanguagesButton} from './ConfirmLanguagesButton'
 import {Trans} from '@lingui/macro'
+
+import {deviceLanguageCodes} from '#/locale/deviceLocales'
 import {useModalControls} from '#/state/modals'
 import {
   useLanguagePrefs,
   useLanguagePrefsApi,
 } from '#/state/preferences/languages'
+import {usePalette} from 'lib/hooks/usePalette'
+import {useWebMediaQueries} from 'lib/hooks/useWebMediaQueries'
+import {LANGUAGES, LANGUAGES_MAP_CODE2} from '../../../../locale/languages'
+import {Text} from '../../util/text/Text'
+import {ScrollView} from '../util'
+import {ConfirmLanguagesButton} from './ConfirmLanguagesButton'
+import {LanguageToggle} from './LanguageToggle'
 
 export const snapPoints = ['100%']
 
@@ -37,10 +38,10 @@ export function Component({}: {}) {
     langs.sort((a, b) => {
       const hasA =
         langPrefs.contentLanguages.includes(a.code2) ||
-        deviceLocales.includes(a.code2)
+        deviceLanguageCodes.includes(a.code2)
       const hasB =
         langPrefs.contentLanguages.includes(b.code2) ||
-        deviceLocales.includes(b.code2)
+        deviceLanguageCodes.includes(b.code2)
       if (hasA === hasB) return a.name.localeCompare(b.name)
       if (hasA) return -1
       return 1

src/view/com/modals/lang-settings/PostLanguagesSettings.tsx

@@ -1,20 +1,21 @@
 import React from 'react'
 import {StyleSheet, View} from 'react-native'
-import {ScrollView} from '../util'
-import {Text} from '../../util/text/Text'
-import {usePalette} from 'lib/hooks/usePalette'
-import {useWebMediaQueries} from 'lib/hooks/useWebMediaQueries'
-import {deviceLocales} from 'platform/detection'
-import {LANGUAGES, LANGUAGES_MAP_CODE2} from '../../../../locale/languages'
-import {ConfirmLanguagesButton} from './ConfirmLanguagesButton'
-import {ToggleButton} from 'view/com/util/forms/ToggleButton'
 import {Trans} from '@lingui/macro'
+
+import {deviceLanguageCodes} from '#/locale/deviceLocales'
 import {useModalControls} from '#/state/modals'
 import {
+  hasPostLanguage,
   useLanguagePrefs,
   useLanguagePrefsApi,
-  hasPostLanguage,
 } from '#/state/preferences/languages'
+import {usePalette} from 'lib/hooks/usePalette'
+import {useWebMediaQueries} from 'lib/hooks/useWebMediaQueries'
+import {ToggleButton} from 'view/com/util/forms/ToggleButton'
+import {LANGUAGES, LANGUAGES_MAP_CODE2} from '../../../../locale/languages'
+import {Text} from '../../util/text/Text'
+import {ScrollView} from '../util'
+import {ConfirmLanguagesButton} from './ConfirmLanguagesButton'
 
 export const snapPoints = ['100%']
 
@@ -38,10 +39,10 @@ export function Component() {
     langs.sort((a, b) => {
       const hasA =
         hasPostLanguage(langPrefs.postLanguage, a.code2) ||
-        deviceLocales.includes(a.code2)
+        deviceLanguageCodes.includes(a.code2)
       const hasB =
         hasPostLanguage(langPrefs.postLanguage, b.code2) ||
-        deviceLocales.includes(b.code2)
+        deviceLanguageCodes.includes(b.code2)
       if (hasA === hasB) return a.name.localeCompare(b.name)
       if (hasA) return -1
       return 1

yarn.lock

@@ -9572,6 +9572,15 @@ bcp-47-match@^2.0.3:
   resolved "https://registry.yarnpkg.com/bcp-47-match/-/bcp-47-match-2.0.3.tgz#603226f6e5d3914a581408be33b28a53144b09d0"
   integrity sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ==
 
+bcp-47@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/bcp-47/-/bcp-47-2.1.0.tgz#7e80734c3338fe8320894981dccf4968c3092df6"
+  integrity sha512-9IIS3UPrvIa1Ej+lVDdDwO7zLehjqsaByECw0bu2RRGP73jALm6FYbzI5gWbgHLvNdkvfXB5YrSbocZdOS0c0w==
+  dependencies:
+    is-alphabetical "^2.0.0"
+    is-alphanumerical "^2.0.0"
+    is-decimal "^2.0.0"
+
 better-opn@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/better-opn/-/better-opn-3.0.2.tgz#f96f35deaaf8f34144a4102651babcf00d1d8817"
@@ -13893,6 +13902,19 @@ ipaddr.js@^2.1.0:
   resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-2.2.0.tgz#d33fa7bac284f4de7af949638c9d68157c6b92e8"
   integrity sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==
 
+is-alphabetical@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-2.0.1.tgz#01072053ea7c1036df3c7d19a6daaec7f19e789b"
+  integrity sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==
+
+is-alphanumerical@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz#7c03fbe96e3e931113e57f964b0a368cc2dfd875"
+  integrity sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==
+  dependencies:
+    is-alphabetical "^2.0.0"
+    is-decimal "^2.0.0"
+
 is-arguments@^1.0.4:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b"
@@ -13987,6 +14009,11 @@ is-date-object@^1.0.1, is-date-object@^1.0.5:
   dependencies:
     has-tostringtag "^1.0.0"
 
+is-decimal@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-2.0.1.tgz#9469d2dc190d0214fd87d78b78caecc0cc14eef7"
+  integrity sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==
+
 is-directory@^0.3.1:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/is-directory/-/is-directory-0.3.1.tgz#61339b6f2475fc772fd9c9d83f5c8575dc154ae1"