Rough out new approach

(cherry picked from commit 9add225160e7e407befc73e9cdd9743a30cdf1cd)
This commit is contained in:
Eric Bailey
2025-08-12 15:06:30 -05:00
committed by Samuel Newman
parent 01687004ef
commit c6264508f7
2 changed files with 239 additions and 404 deletions
+56 -4
View File
@@ -78,7 +78,11 @@ import {logger} from '#/logger'
import {isAndroid, isIOS, isNative, isWeb} from '#/platform/detection'
import {useDialogStateControlContext} from '#/state/dialogs'
import {emitPostCreated} from '#/state/events'
import {type ComposerImage, pasteImage} from '#/state/gallery'
import {
type ComposerImage,
createComposerImage,
pasteImage,
} from '#/state/gallery'
import {useModalControls} from '#/state/modals'
import {useRequireAltTextEnabled} from '#/state/preferences'
import {
@@ -128,7 +132,10 @@ import * as Prompt from '#/components/Prompt'
import {toast} from '#/components/Toast'
import {Text as NewText} from '#/components/Typography'
import {BottomSheetPortalProvider} from '../../../../modules/bottom-sheet'
import {SelectMediaBtn} from './SelectMediaBtn'
import {
type Props as SelectMediaButtonProps,
SelectMediaBtn,
} from './SelectMediaBtn'
import {
type ComposerAction,
composerReducer,
@@ -1271,12 +1278,15 @@ function ComposerFooter({
const isMaxImages = images.length >= MAX_IMAGES
const isMaxVideos = !!video
let selectedAssetsCount = 0
let isMediaSelectionDisabled = false
if (media?.type === 'images') {
isMediaSelectionDisabled = isMaxImages
selectedAssetsCount = images.length
} else if (media?.type === 'video') {
isMediaSelectionDisabled = isMaxVideos
selectedAssetsCount = 1
} else {
isMediaSelectionDisabled = !!media
}
@@ -1298,6 +1308,48 @@ function ComposerFooter({
[dispatch],
)
const onSelectAssets = useCallback<SelectMediaButtonProps['onSelectAssets']>(
async ({type, assets, errors}) => {
if (assets.length) {
if (type === 'image') {
const images: ComposerImage[] = []
for (const image of assets) {
try {
images.push(
await createComposerImage({
path: image.uri,
width: image.width,
height: image.height,
mime: image.mimeType!,
}),
)
} catch (e: any) {
logger.error(`createComposerImage failed`, {
safeMessage: e.message,
})
}
}
onImageAdd(images)
} else if (type === 'video') {
onSelectVideo(post.id, assets[0])
} else if (type === 'gif') {
onSelectVideo(post.id, assets[0])
}
}
errors.map((error, i) => {
toast.show({
type: 'error',
content: error,
a11yLabel: error,
duration: 3e3 * (errors.length - i),
})
})
},
[post.id, onSelectVideo, onImageAdd],
)
return (
<View
style={[
@@ -1319,9 +1371,9 @@ function ComposerFooter({
<SelectMediaBtn
size={images.length}
disabled={isMediaSelectionDisabled}
onAdd={onImageAdd}
onSelectVideo={asset => onSelectVideo(post.id, asset)}
setError={onError}
selectedAssetsCount={selectedAssetsCount}
onSelectAssets={onSelectAssets}
/>
<OpenCameraBtn
disabled={media?.type === 'images' ? isMaxImages : !!media}
+183 -400
View File
@@ -7,260 +7,224 @@ import {
import {msg} from '@lingui/macro'
import {useLingui} from '@lingui/react'
import {
SUPPORTED_MIME_TYPES,
type SupportedMimeTypes,
VIDEO_MAX_DURATION_MS,
} from '#/lib/constants'
import {
usePhotoLibraryPermission,
useVideoLibraryPermission,
} from '#/lib/hooks/usePermissions'
import {getDataUriSize} from '#/lib/media/util'
import {isNative, isWeb} from '#/platform/detection'
import {type ComposerImage, createComposerImage} from '#/state/gallery'
import {extractDataUriMime} from '#/lib/media/util'
import {isIOS, isNative} from '#/platform/detection'
import {MAX_IMAGES} from '#/view/com/composer/state/composer'
import {getVideoMetadata} from '#/view/com/composer/videos/pickVideo'
// import {getVideoMetadata} from '#/view/com/composer/videos/pickVideo'
import {atoms as a, useTheme} from '#/alf'
import {Button} from '#/components/Button'
import {useSheetWrapper} from '#/components/Dialog/sheet-wrapper'
import {Image_Stroke2_Corner0_Rounded as Image} from '#/components/icons/Image'
import {toast} from '#/components/Toast'
type Props = {
export type Props = {
size: number
disabled?: boolean
onAdd: (next: ComposerImage[]) => void
onSelectVideo: (asset: ImagePickerAsset) => void
setError: (error: string) => void
selectedAssetsCount: number
onSelectAssets: (props: {
type: SelectedAsset['type']
assets: ImagePickerAsset[]
errors: string[]
}) => void
}
// (Note)APiligrim
// The SelectMediaBtn is responsible for supporting the selection of the following media in the Composer:
// up to 4 images
// up to 1 video
// up to 1 GIF (handled like a video - passed to onSelectVideo)
export type SelectedAsset = {
asset: ImagePickerAsset
type: 'video' | 'image' | 'gif'
}
export function SelectMediaBtn({disabled, onAdd, onSelectVideo}: Props) {
export enum SelectedAssetError {
Unsupported = 'Unsupported',
MixedTypes = 'MixedTypes',
MaxImages = 'MaxImages',
MaxVideos = 'MaxVideos',
MaxGIFs = 'MaxGIFs',
}
const SUPPORTED_VIDEO_MIME_TYPES = [
'video/mp4',
'video/mpeg',
'video/webm',
'video/quicktime',
] as const
export type SupportedVideoMimeType = (typeof SUPPORTED_VIDEO_MIME_TYPES)[number]
const SUPPORTED_IMAGE_MIME_TYPES = (
[
'image/gif',
'image/jpeg',
'image/png',
'image/svg+xml',
isIOS && 'image/heic',
] as const
).filter(Boolean)
export type SupportedImageMimeType = Exclude<
(typeof SUPPORTED_IMAGE_MIME_TYPES)[number],
boolean
>
const extensionToMimeType: Record<
string,
SupportedVideoMimeType | SupportedImageMimeType
> = {
mp4: 'video/mp4',
mov: 'video/quicktime',
webm: 'video/webm',
gif: 'image/gif',
jpg: 'image/jpeg',
jpeg: 'image/jpeg',
png: 'image/png',
svg: 'image/svg+xml',
heic: 'image/heic',
}
function getImagePickerAssetType(asset: ImagePickerAsset):
| {
success: true
type: SelectedAsset['type']
mimeType: string
}
| {
success: false
type: undefined
mimeType: undefined
} {
let mimeType = asset.mimeType
if (!mimeType) {
const maybeMimeType = extractDataUriMime(asset.uri)
if (
maybeMimeType.startsWith('image/') ||
maybeMimeType.startsWith('video/')
) {
mimeType = maybeMimeType
} else if (maybeMimeType.startsWith('file/')) {
const extension = asset.uri.split('.').pop()?.toLowerCase()
mimeType = extensionToMimeType[extension || '']
}
}
let type: SelectedAsset['type'] | undefined
if (mimeType === 'image/gif') {
type = 'gif'
} else if (
mimeType?.startsWith('video/') &&
SUPPORTED_VIDEO_MIME_TYPES.includes(mimeType as SupportedVideoMimeType)
) {
type = 'video'
} else if (
mimeType?.startsWith('image/') &&
SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType as SupportedImageMimeType)
) {
type = 'image'
}
if (!type || !mimeType) {
return {
success: false,
type: undefined,
mimeType: undefined,
}
}
return {
success: true,
type,
mimeType,
}
}
export function SelectMediaBtn({
disabled,
selectedAssetsCount,
onSelectAssets,
}: Props) {
const {_} = useLingui()
const {requestPhotoAccessIfNeeded} = usePhotoLibraryPermission()
const {requestVideoAccessIfNeeded} = useVideoLibraryPermission()
const sheetWrapper = useSheetWrapper()
const t = useTheme()
const selectionLimit = MAX_IMAGES - selectedAssetsCount
const processSelectedAssets = useCallback(
async (assets: ImagePickerAsset[]) => {
function performEarlyValidation(assetList: ImagePickerAsset[]) {
const images: ImagePickerAsset[] = []
const videosAndGifs: ImagePickerAsset[] = []
const invalids: ImagePickerAsset[] = []
const errors = new Set<SelectedAssetError>()
let supportedAssets: ImagePickerAsset[] = []
let primaryMediaType: SelectedAsset['type'] | undefined
// Categorize assets using basic checks (even without complete metadata)
for (const asset of assetList) {
const ext = asset.uri?.split('.').pop()?.toLowerCase()
const isGif = isGifAsset(asset, ext)
const isVideo = isVideoAsset(asset)
const isImage = isImageAsset(asset)
for (const asset of assets) {
const {success, type, mimeType} = getImagePickerAssetType(asset)
if (isGif || isVideo) {
// GIFs and videos are handled the same way (fixing regression)
videosAndGifs.push(asset)
} else if (isImage) {
images.push(asset)
} else {
invalids.push(asset)
console.error('Invalid asset detected:', asset)
}
if (!success) {
errors.add(SelectedAssetError.Unsupported)
continue
}
// Determine media type based on selected assets and preserve selection order
let mediaType: 'image' | 'video' | null = null
for (const asset of assetList) {
const ext = asset.uri?.split('.').pop()?.toLowerCase()
const isGif = isGifAsset(asset, ext)
const isVideo = isVideoAsset(asset)
const isImage = isImageAsset(asset)
// set the primary media type to the first valid asset type
primaryMediaType = primaryMediaType || type
if (isGif || isVideo) {
// GIFs are treated like videos
mediaType = 'video'
break
} else if (isImage) {
mediaType = 'image'
break
}
if (type !== primaryMediaType) {
// Selecting a mix of media types is not allowed
errors.add(SelectedAssetError.MixedTypes)
continue
}
let validAssets: ImagePickerAsset[] = []
let trimmed = false
if (mediaType === 'image') {
validAssets = images.slice(0, 4)
if (images.length > 4) trimmed = true
if (videosAndGifs.length > 0) {
toast.show({
type: 'info',
content:
'You can select either images or videos. Taking the images.',
a11yLabel:
'You can select either images or videos. Taking the images.',
})
}
} else if (mediaType === 'video') {
// For videos/GIFs: take only the first one, show toast if mixed media
validAssets = videosAndGifs.slice(0, 1)
if (images.length > 0 || videosAndGifs.length > 1) {
toast.show({
type: 'info',
content:
'You can select either images or videos. Taking the first video.',
a11yLabel:
'You can select either images or videos. Taking the first video.',
})
}
}
return {validAssets, mediaType, trimmed}
}
async function normalizeAssets(assetList: ImagePickerAsset[]) {
const normalizedAssets: ImagePickerAsset[] = []
const failedAssets: ImagePickerAsset[] = []
for (const asset of assetList) {
try {
let normalizedAsset = asset
// Check if we need to enrich metadata
const needsEnrichment =
!asset.mimeType ||
(asset.type === 'video' && !asset.duration) ||
(isGifAsset(asset, asset.uri?.split('.').pop()?.toLowerCase()) &&
!asset.duration)
if (needsEnrichment) {
const enrichedAsset = await getMissingMetadata(asset)
if (enrichedAsset) {
normalizedAsset = enrichedAsset
}
// Last ditch effort: infer mimeType from extension if still missing
if (!normalizedAsset.mimeType) {
normalizedAsset = {
...normalizedAsset,
mimeType:
normalizedAsset.type === 'image'
? 'image/jpeg'
: inferMimeTypeFromURI(normalizedAsset.uri || ''),
}
}
}
normalizedAssets.push(normalizedAsset)
} catch (error) {
console.error('Failed to normalize asset:', asset, error)
failedAssets.push(asset)
}
}
return {normalizedAssets, failedAssets}
}
function performFinalValidation(
assetList: ImagePickerAsset[],
mediaType: 'image' | 'video' | null,
) {
const validAssets: ImagePickerAsset[] = []
for (const asset of assetList) {
if (mediaType === 'image') {
// Images just need basic validation
if (asset.mimeType && asset.mimeType.startsWith('image/')) {
validAssets.push(asset)
} else {
console.error('Image failed final validation:', asset)
}
} else if (mediaType === 'video') {
// Videos and GIFs need duration and format checks
const isValid = validateVideoOrGIF(asset)
if (isValid) {
validAssets.push(asset)
} else {
console.error('Video/GIF failed final validation:', asset)
}
}
}
return validAssets
}
// 1. Early validation to detect asset format and count limits
const {validAssets, mediaType, trimmed} = performEarlyValidation(assets)
if (validAssets.length === 0) {
toast.show({
type: 'error',
content: 'No valid media files selected',
a11yLabel: 'No valid media files selected',
})
return
}
// Show toast if selected files were trimmed
if (trimmed) {
toast.show({
type: 'info',
content: 'Selection limited to first 4 files.',
a11yLabel: 'Selection limited to first 4 files.',
supportedAssets.push({
mimeType,
...asset,
})
}
// 2. Normalize assets by adding missing metadata
const {normalizedAssets, failedAssets} =
await normalizeAssets(validAssets)
if (failedAssets.length > 0) {
console.error('Some assets failed to normalize:', failedAssets)
if (normalizedAssets.length === 0) {
toast.show({
type: 'error',
content: 'Failed to process selected files',
a11yLabel: 'Failed to process selected files',
})
return
} else {
toast.show({
type: 'info',
content: 'Some files could not be processed. Using valid files.',
a11yLabel: 'Some files could not be processed. Using valid files.',
})
if (primaryMediaType === 'image') {
if (supportedAssets.length > selectionLimit) {
errors.add(SelectedAssetError.MaxImages)
supportedAssets = supportedAssets.slice(0, selectionLimit)
}
}
// 3. Final validation checks on normalized assets
const finalAssets = performFinalValidation(normalizedAssets, mediaType)
if (finalAssets.length === 0) {
toast.show({
type: 'error',
content: 'This media type is not supported',
a11yLabel: 'This media type is not supported',
})
return
}
// 4. Add finalized assets to the composer
if (mediaType === 'image') {
const composerImages = await generateComposerImages(finalAssets)
onAdd(composerImages)
} else {
// Both videos and GIFs get selected with onSelectVideo
onSelectVideo(finalAssets[0])
if (supportedAssets.length > 1) {
if (primaryMediaType === 'video') {
errors.add(SelectedAssetError.MaxVideos)
} else if (primaryMediaType === 'gif') {
errors.add(SelectedAssetError.MaxGIFs)
}
supportedAssets = supportedAssets.slice(0, 1)
}
}
onSelectAssets({
type: primaryMediaType!,
assets: supportedAssets,
errors: Array.from(errors).map(error => {
return {
[SelectedAssetError.Unsupported]: _(
msg`One or more of your selected files are not supported.`,
),
[SelectedAssetError.MixedTypes]: _(
msg`Selecting multiple media types is not supported.`,
),
[SelectedAssetError.MaxImages]: _(
msg`You can select up to ${MAX_IMAGES} total images.`,
),
[SelectedAssetError.MaxVideos]: _(
msg`You can only select one video at a time.`,
),
[SelectedAssetError.MaxGIFs]: _(
msg`You can only select one GIF at a time.`,
),
}[error]
}),
})
},
[onAdd, onSelectVideo],
[_, onSelectAssets, selectionLimit],
)
const onPressSelectMedia = useCallback(async () => {
@@ -280,20 +244,20 @@ export function SelectMediaBtn({disabled, onAdd, onSelectVideo}: Props) {
}
}
const {assets} = await sheetWrapper(
const {assets, canceled} = await sheetWrapper(
launchImageLibraryAsync({
exif: false,
mediaTypes: ['images', 'videos'],
quality: 1,
allowsMultipleSelection: true,
legacy: true,
selectionLimit: MAX_IMAGES,
selectionLimit,
preferredAssetRepresentationMode:
UIImagePickerPreferredAssetRepresentationMode.Current,
}),
)
if (!assets || assets.length === 0) return
if (canceled) return
await processSelectedAssets(assets)
}, [
@@ -301,6 +265,7 @@ export function SelectMediaBtn({disabled, onAdd, onSelectVideo}: Props) {
requestVideoAccessIfNeeded,
sheetWrapper,
processSelectedAssets,
selectionLimit,
])
return (
@@ -324,185 +289,3 @@ export function SelectMediaBtn({disabled, onAdd, onSelectVideo}: Props) {
</Button>
)
}
async function generateComposerImages(
images: ImagePickerAsset[],
): Promise<ComposerImage[]> {
const imageMetas = images.map(image => ({
mime: image.mimeType || 'image/jpeg',
height: image.height,
width: image.width,
path: image.uri,
size: getDataUriSize(image.uri),
}))
const results = await Promise.all(
imageMetas.map(img => createComposerImage(img)),
)
return results
}
function isGifAsset(asset: ImagePickerAsset, ext?: string): boolean {
return asset.mimeType === 'image/gif' || ext === 'gif'
}
function isVideoAsset(asset: ImagePickerAsset): boolean {
if (asset.type === 'video') {
return true
}
if (asset.mimeType && asset.mimeType.startsWith('video/')) {
return true
}
if (asset.uri && /\.(mp4|mov|avi|webm|m4v)$/i.test(asset.uri)) {
return true
}
// Check for data URI with video mime type
if (asset.uri && asset.uri.startsWith('data:video/')) {
return true
}
if (typeof asset.duration === 'number' && asset.duration > 0) {
return true
}
return false
}
function isImageAsset(asset: ImagePickerAsset): boolean {
if (asset.type === 'image') return true
if (asset.mimeType && asset.mimeType.startsWith('image/')) return true
return false
}
function validateVideoOrGIF(asset: ImagePickerAsset): boolean {
if (!asset) {
console.error('Asset is null or undefined')
return false
}
if (isWeb) {
// asset.duration is null for gifs (see the TODO in pickVideo.web.ts)
if (asset.duration && asset.duration > VIDEO_MAX_DURATION_MS) {
toast.show({
type: 'error',
content: 'Videos must be less than 3 minutes long',
a11yLabel: 'Videos must be less than 3 minutes long',
})
return false
}
// compression step on native converts to mp4, so no need to check there
if (
asset.mimeType &&
!SUPPORTED_MIME_TYPES.includes(asset.mimeType as SupportedMimeTypes)
) {
toast.show({
type: 'error',
content: 'This video format is not supported',
a11yLabel: 'This video format is not supported',
})
return false
}
} else {
// Check if asset exists and has duration property before accessing it
if (!asset.duration || typeof asset.duration !== 'number') {
toast.show({
type: 'error',
content: 'Please select a valid video file',
a11yLabel: 'Please select a valid video file',
})
return false
}
if (asset.duration > VIDEO_MAX_DURATION_MS) {
toast.show({
type: 'error',
content: 'Videos must be less than 3 minutes long',
a11yLabel: 'Videos must be less than 3 minutes long',
})
return false
}
}
return true
}
function inferMimeTypeFromURI(uri: string): string {
const ext = uri.split('.').pop()?.toLowerCase()
switch (ext) {
case 'mp4':
return 'video/mp4'
case 'mov':
return 'video/quicktime'
case 'webm':
return 'video/webm'
case 'avi':
return 'video/x-msvideo'
case 'gif':
return 'image/gif'
default:
return 'video/mp4'
}
}
async function getMissingMetadata(
asset: ImagePickerAsset,
): Promise<ImagePickerAsset | null> {
if (!isWeb || !asset.uri) return asset
if (asset.uri.startsWith('data:')) {
try {
const mimeTypeMatch = asset.uri.match(/^data:([^;]+)/)
const extractedMimeType = mimeTypeMatch ? mimeTypeMatch[1] : null
const response = await fetch(asset.uri)
const blob = await response.blob()
const file = new File([blob], 'file', {
type: blob.type || extractedMimeType || '',
})
let enrichedAsset = asset
// If it's a video, try to get video metadata
if (
extractedMimeType?.startsWith('video/') ||
blob.type.startsWith('video/')
) {
try {
const videoAsset = await getVideoMetadata(file)
enrichedAsset = {
...asset,
...videoAsset,
mimeType: videoAsset.mimeType || extractedMimeType || blob.type,
type: 'video',
}
} catch (error) {
console.error(
' if getting metadata for a video fails, using basic info',
error,
)
enrichedAsset = {
...asset,
mimeType: extractedMimeType || blob.type,
type: extractedMimeType?.startsWith('video/')
? 'video'
: asset.type,
}
}
} else {
// For non-video assets (images and GIFs), setting the mime type
enrichedAsset = {
...asset,
mimeType: extractedMimeType || blob.type,
}
}
return enrichedAsset
} catch (error) {
console.error('Failed to extract metadata:', error)
return null
}
}
return asset
}