acetheking987/bsky-social-app
1
0
Fork 0
Code Issues Actions Packages Projects Releases 1 Wiki Activity

Replace graphemer with unicode-segmenter (#9526)

Browse Source 
* replace graphemer with unicode-segmenter

* use grapheme entrypoint

* force resolution of unicode-segmenter
This commit is contained in:
Samuel Newman
2025-12-30 13:39:09 +02:00
committed by GitHub
parent 82877a096d
commit 9743149c26
11 changed files with 57 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+3 -1
View File
@@ -222,6 +222,7 @@
"tippy.js": "^6.3.7",
"tlds": "^1.234.0",
"tldts": "^6.1.46",
"unicode-segmenter": "^0.14.5",
"zod": "^3.20.2"
},
"devDependencies": {
@@ -286,7 +287,8 @@
"**/expo-constants": "18.0.8",
"**/expo-device": "7.1.4",
"**/zod": "3.23.8",
"**/multiformats": "9.9.0"
"**/multiformats": "9.9.0",
"unicode-segmenter": "0.14.5"
},
"jest": {
"preset": "jest-expo/ios",
src/components/dialogs/lists/CreateOrEditListDialog.tsx
+3 -3
View File
@@ -5,7 +5,7 @@ import {msg, Plural, Trans} from '@lingui/macro'
import {useLingui} from '@lingui/react'
import {cleanError} from '#/lib/strings/errors'
import {useWarnMaxGraphemeCount} from '#/lib/strings/helpers'
import {isOverMaxGraphemeCount} from '#/lib/strings/helpers'
import {richTextToString} from '#/lib/strings/rich-text-helpers'
import {shortenLinks, stripInvalidMentions} from '#/lib/strings/rich-text-manip'
import {logger} from '#/logger'
@@ -259,11 +259,11 @@ function DialogInner({
_,
])
const displayNameTooLong = useWarnMaxGraphemeCount({
const displayNameTooLong = isOverMaxGraphemeCount({
text: displayName,
maxCount: DISPLAY_NAME_MAX_GRAPHEMES,
})
const descriptionTooLong = useWarnMaxGraphemeCount({
const descriptionTooLong = isOverMaxGraphemeCount({
text: descriptionRt,
maxCount: DESCRIPTION_MAX_GRAPHEMES,
})
src/lib/strings/helpers.ts
+7 -27
View File
@@ -1,6 +1,5 @@
import {useCallback, useMemo} from 'react'
import {type RichText} from '@atproto/api'
import Graphemer from 'graphemer'
import {countGraphemes} from 'unicode-segmenter/grapheme'
import {shortenLinks} from './rich-text-manip'
@@ -29,37 +28,18 @@ export function enforceLen(
return str
}
export function useEnforceMaxGraphemeCount() {
const splitter = useMemo(() => new Graphemer(), [])
return useCallback(
(text: string, maxCount: number) => {
if (splitter.countGraphemes(text) > maxCount) {
return splitter.splitGraphemes(text).slice(0, maxCount).join('')
} else {
return text
}
},
[splitter],
)
}
export function useWarnMaxGraphemeCount({
export function isOverMaxGraphemeCount({
text,
maxCount,
}: {
text: string | RichText
maxCount: number
}) {
const splitter = useMemo(() => new Graphemer(), [])
return useMemo(() => {
if (typeof text === 'string') {
return splitter.countGraphemes(text) > maxCount
} else {
return shortenLinks(text).graphemeLength > maxCount
}
}, [splitter, maxCount, text])
if (typeof text === 'string') {
return countGraphemes(text) > maxCount
} else {
return shortenLinks(text).graphemeLength > maxCount
}
}
export function countLines(str: string | undefined): number {
src/screens/Messages/components/MessageInput.tsx
+2 -2
View File
@@ -14,7 +14,7 @@ import Animated, {
import {useSafeAreaInsets} from 'react-native-safe-area-context'
import {msg} from '@lingui/macro'
import {useLingui} from '@lingui/react'
import Graphemer from 'graphemer'
import {countGraphemes} from 'unicode-segmenter/grapheme'
import {HITSLOP_10, MAX_DM_GRAPHEME_LENGTH} from '#/lib/constants'
import {useHaptics} from '#/lib/haptics'
@@ -75,7 +75,7 @@ export function MessageInput({
if (!hasEmbed && message.trim() === '') {
return
}
if (new Graphemer().countGraphemes(message) > MAX_DM_GRAPHEME_LENGTH) {
if (countGraphemes(message) > MAX_DM_GRAPHEME_LENGTH) {
Toast.show(_(msg`Message is too long`), 'xmark')
return
}
src/screens/Messages/components/MessageInput.web.tsx
+2 -2
View File
@@ -2,9 +2,9 @@ import React from 'react'
import {Pressable, View} from 'react-native'
import {msg} from '@lingui/macro'
import {useLingui} from '@lingui/react'
import Graphemer from 'graphemer'
import {flushSync} from 'react-dom'
import TextareaAutosize from 'react-textarea-autosize'
import {countGraphemes} from 'unicode-segmenter/grapheme'
import {isSafari, isTouchDevice} from '#/lib/browser'
import {MAX_DM_GRAPHEME_LENGTH} from '#/lib/constants'
@@ -56,7 +56,7 @@ export function MessageInput({
if (!hasEmbed && message.trim() === '') {
return
}
if (new Graphemer().countGraphemes(message) > MAX_DM_GRAPHEME_LENGTH) {
if (countGraphemes(message) > MAX_DM_GRAPHEME_LENGTH) {
Toast.show(_(msg`Message is too long`), 'xmark')
return
}
src/screens/Profile/Header/EditProfileDialog.tsx
+3 -3
View File
@@ -6,7 +6,7 @@ import {useLingui} from '@lingui/react'
import {urls} from '#/lib/constants'
import {cleanError} from '#/lib/strings/errors'
import {useWarnMaxGraphemeCount} from '#/lib/strings/helpers'
import {isOverMaxGraphemeCount} from '#/lib/strings/helpers'
import {logger} from '#/logger'
import {type ImageMeta} from '#/state/gallery'
import {useProfileUpdateMutation} from '#/state/queries/profile'
@@ -203,11 +203,11 @@ function DialogInner({
_,
])
const displayNameTooLong = useWarnMaxGraphemeCount({
const displayNameTooLong = isOverMaxGraphemeCount({
text: displayName,
maxCount: DISPLAY_NAME_MAX_GRAPHEMES,
})
const descriptionTooLong = useWarnMaxGraphemeCount({
const descriptionTooLong = isOverMaxGraphemeCount({
text: description,
maxCount: DESCRIPTION_MAX_GRAPHEMES,
})
src/screens/Takendown.tsx
+9 -14
View File
@@ -1,4 +1,4 @@
import {useMemo, useState} from 'react'
import {useState} from 'react'
import {View} from 'react-native'
import {KeyboardAwareScrollView} from 'react-native-keyboard-controller'
import {useSafeAreaInsets} from 'react-native-safe-area-context'
@@ -6,7 +6,7 @@ import {type ComAtprotoAdminDefs, ToolsOzoneReportDefs} from '@atproto/api'
import {msg, Trans} from '@lingui/macro'
import {useLingui} from '@lingui/react'
import {useMutation} from '@tanstack/react-query'
import Graphemer from 'graphemer'
import {countGraphemes} from 'unicode-segmenter/grapheme'
import {
BLUESKY_MOD_SERVICE_HEADERS,
@@ -37,11 +37,10 @@ export function Takendown() {
const agent = useAgent()
const [isAppealling, setIsAppealling] = useState(false)
const [reason, setReason] = useState('')
const graphemer = useMemo(() => new Graphemer(), [])
const reasonGraphemeLength = useMemo(() => {
return graphemer.countGraphemes(reason)
}, [graphemer, reason])
const reasonGraphemeLength = countGraphemes(reason)
const isOverMaxLength =
reasonGraphemeLength > MAX_REPORT_REASON_GRAPHEME_LENGTH
const {
mutate: submitAppeal,
@@ -72,14 +71,11 @@ export function Takendown() {
const primaryBtn =
isAppealling && !isSuccess ? (
<Button
variant="solid"
color="primary"
size="large"
label={_(msg`Submit appeal`)}
onPress={() => submitAppeal(reason)}
disabled={
isPending || reasonGraphemeLength > MAX_REPORT_REASON_GRAPHEME_LENGTH
}>
disabled={isPending || isOverMaxLength}>
<ButtonText>
<Trans>Submit Appeal</Trans>
</ButtonText>
@@ -87,7 +83,6 @@ export function Takendown() {
</Button>
) : (
<Button
variant="solid"
size="large"
color="secondary_inverted"
label={_(msg`Sign out`)}
@@ -204,7 +199,7 @@ export function Takendown() {
<Text
style={[
a.text_md,
a.leading_normal,
a.leading_snug,
{color: t.palette.negative_500},
a.mt_lg,
]}>
@@ -213,13 +208,13 @@ export function Takendown() {
)}
</View>
) : (
<P style={[t.atoms.text_contrast_medium]}>
<P style={[t.atoms.text_contrast_medium, a.leading_snug]}>
<Trans>
Your account was found to be in violation of the{' '}
<SimpleInlineLinkText
label={_(msg`Bluesky Social Terms of Service`)}
to="https://bsky.social/about/support/tos"
style={[a.text_md, a.leading_normal]}>
style={[a.text_md, a.leading_snug]}>
Bluesky Social Terms of Service
</SimpleInlineLinkText>
. You have been sent an email outlining the specific violation
src/view/com/composer/text-input/TextInput.web.tsx
+2 -2
View File
@@ -20,7 +20,7 @@ import {Text as TiptapText} from '@tiptap/extension-text'
import {generateJSON} from '@tiptap/html'
import {Fragment, Node, Slice} from '@tiptap/pm/model'
import {EditorContent, type JSONContent, useEditor} from '@tiptap/react'
import Graphemer from 'graphemer'
import {splitGraphemes} from 'unicode-segmenter/grapheme'
import {useColorSchemeStyle} from '#/lib/hooks/useColorSchemeStyle'
import {blobToDataUri, isUriImage} from '#/lib/media/util'
@@ -218,7 +218,7 @@ export function TextInput({
// all the lines get mushed together -sfn
'\n',
)
const graphemes = new Graphemer().splitGraphemes(textBefore)
const graphemes = [...splitGraphemes(textBefore)]
if (graphemes.length > 0) {
const lastGrapheme = graphemes[graphemes.length - 1]
src/view/com/composer/text-input/hooks/useGrapheme.tsx
-36
View File
@@ -1,36 +0,0 @@
import {useCallback, useMemo} from 'react'
import Graphemer from 'graphemer'
export const useGrapheme = () => {
const splitter = useMemo(() => new Graphemer(), [])
const getGraphemeString = useCallback(
(name: string, length: number) => {
let remainingCharacters = 0
if (name.length > length) {
const graphemes = splitter.splitGraphemes(name)
if (graphemes.length > length) {
remainingCharacters = 0
name = `${graphemes.slice(0, length).join('')}…`
} else {
remainingCharacters = length - graphemes.length
name = graphemes.join('')
}
} else {
remainingCharacters = length - name.length
}
return {
name,
remainingCharacters,
}
},
[splitter],
)
return {
getGraphemeString,
}
}
src/view/com/composer/videos/SubtitleDialog.tsx
+22 -5
View File
@@ -4,7 +4,7 @@ import {msg, Trans} from '@lingui/macro'
import {useLingui} from '@lingui/react'
import {MAX_ALT_TEXT} from '#/lib/constants'
import {useEnforceMaxGraphemeCount} from '#/lib/strings/helpers'
import {isOverMaxGraphemeCount} from '#/lib/strings/helpers'
import {LANGUAGES} from '#/locale/languages'
import {isWeb} from '#/platform/detection'
import {useLanguagePrefs} from '#/state/preferences'
@@ -72,7 +72,6 @@ function SubtitleDialogInner({
const control = Dialog.useDialogContext()
const {_} = useLingui()
const t = useTheme()
const enforceLen = useEnforceMaxGraphemeCount()
const {primaryLanguage} = useLanguagePrefs()
const [altText, setAltText] = useState(defaultAltText)
@@ -94,18 +93,23 @@ function SubtitleDialogInner({
const subtitleMissingLanguage = captions.some(sub => sub.lang === '')
const isOverMaxLength = isOverMaxGraphemeCount({
text: altText,
maxCount: MAX_ALT_TEXT,
})
return (
<Dialog.ScrollableInner label={_(msg`Video settings`)}>
<View style={a.gap_md}>
<Text style={[a.text_xl, a.font_semi_bold, a.leading_tight]}>
<Trans>Alt text</Trans>
</Text>
<TextField.Root>
<TextField.Root isInvalid={isOverMaxLength}>
<Dialog.Input
label={_(msg`Alt text`)}
placeholder={_(msg`Add alt text (optional)`)}
value={altText}
onChangeText={evt => setAltText(enforceLen(evt, MAX_ALT_TEXT))}
onChangeText={setAltText}
maxLength={MAX_ALT_TEXT * 10}
multiline
style={{maxHeight: 300}}
@@ -118,6 +122,18 @@ function SubtitleDialogInner({
/>
</TextField.Root>
{isOverMaxLength && (
<Text
style={[
a.text_md,
{color: t.palette.negative_500},
a.leading_snug,
a.mt_md,
]}>
<Trans>Alt text must be less than {MAX_ALT_TEXT} characters.</Trans>
</Text>
)}
{isWeb && (
<>
<View
@@ -173,7 +189,8 @@ function SubtitleDialogInner({
saveAltText(altText)
control.close()
}}
style={a.mt_lg}>
style={a.mt_lg}
disabled={isOverMaxLength}>
<ButtonText>
<Trans>Done</Trans>
</ButtonText>
yarn.lock
+4 -4
View File
@@ -19158,10 +19158,10 @@ unicode-property-aliases-ecmascript@^2.0.0:
resolved "https://registry.yarnpkg.com/unicode-property-aliases-ecmascript/-/unicode-property-aliases-ecmascript-2.1.0.tgz#43d41e3be698bd493ef911077c9b131f827e8ccd"
integrity sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==
unicode-segmenter@^0.14.0:
version "0.14.0"
resolved "https://registry.yarnpkg.com/unicode-segmenter/-/unicode-segmenter-0.14.0.tgz#090128182bcc710327a1b7e4af4f5834444eaa61"
integrity sha512-AH4lhPCJANUnSLEKnM4byboctePJzltF4xj8b+NbNiYeAkAXGh7px2K/4NANFp7dnr6+zB3e6HLu8Jj8SKyvYg==
unicode-segmenter@0.14.5, unicode-segmenter@^0.14.0, unicode-segmenter@^0.14.5:
version "0.14.5"
resolved "https://registry.yarnpkg.com/unicode-segmenter/-/unicode-segmenter-0.14.5.tgz#c658f6dd30de172cdcd94542adc205ba43fb63c6"
integrity sha512-jHGmj2LUuqDcX3hqY12Ql+uhUTn8huuxNZGq7GvtF6bSybzH3aFgedYu/KTzQStEgt1Ra2F3HxadNXsNjb3m3g==
unimodules-app-loader@~6.0.8:
version "6.0.8"