From cc0e1f88eaf1d81eaada62481a7faa3db2b30f9b Mon Sep 17 00:00:00 2001 From: pfrazee <1270099+pfrazee@users.noreply.github.com> Date: Tue, 23 Jun 2026 03:17:56 +0000 Subject: [PATCH 01/43] Nightly source-language update --- src/locale/locales/en/messages.po | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/locale/locales/en/messages.po b/src/locale/locales/en/messages.po index 45d022d994..38b347aa63 100644 --- a/src/locale/locales/en/messages.po +++ b/src/locale/locales/en/messages.po @@ -345,7 +345,7 @@ msgstr "{0}/{1} {2, plural, one {member} other {members}}" #. Badge showing the current image position out of the total number of images in a gallery. #. placeholder {0}: index + 1 -#: src/components/images/Gallery/index.tsx:526 +#: src/components/images/Gallery/index.tsx:521 msgctxt "gallery-badge-image-position-numbers" msgid "{0}/{imageCount}" msgstr "{0}/{imageCount}" @@ -1435,9 +1435,9 @@ msgstr "" msgid "Already signed in as @{0}" msgstr "" -#: src/components/images/AutoSizedImage.tsx:204 -#: src/components/images/Gallery/index.tsx:588 -#: src/components/images/ImageLayoutGridItem.tsx:142 +#: src/components/images/AutoSizedImage.tsx:205 +#: src/components/images/Gallery/index.tsx:585 +#: src/components/images/ImageLayoutGridItem.tsx:135 #: src/components/Post/Embed/VideoEmbed/GifPresentationControls.tsx:94 #: src/view/com/composer/GifAltText.tsx:100 #: src/view/com/composer/photos/Gallery.tsx:211 @@ -2406,7 +2406,7 @@ msgstr "" msgid "Captions & alt text" msgstr "" -#: src/components/images/Gallery/index.tsx:275 +#: src/components/images/Gallery/index.tsx:273 msgid "carousel" msgstr "carousel" @@ -6097,12 +6097,12 @@ msgstr "" msgid "If you're trying to change your handle or email, do so before you deactivate." msgstr "" -#: src/components/images/ImageLayoutGridItem.tsx:96 +#: src/components/images/ImageLayoutGridItem.tsx:91 msgid "Image" msgstr "" #. placeholder {0}: index + 1 -#: src/components/images/Gallery/index.tsx:459 +#: src/components/images/Gallery/index.tsx:454 msgid "Image {0}" msgstr "Image {0}" @@ -6113,7 +6113,7 @@ msgid "Image {0} of {1}" msgstr "" #. placeholder {0}: index + 1 -#: src/components/images/Gallery/index.tsx:444 +#: src/components/images/Gallery/index.tsx:439 msgid "Image {0} of {imageCount}" msgstr "Image {0} of {imageCount}" @@ -6128,7 +6128,7 @@ msgid "Image cache cleared, freed {0}" msgstr "" #. placeholder {0}: images.length -#: src/components/images/Gallery/index.tsx:276 +#: src/components/images/Gallery/index.tsx:274 msgid "Image gallery, {0} images" msgstr "Image gallery, {0} images" @@ -8324,7 +8324,7 @@ msgstr "" msgid "Opens flow to sign in to your existing Bluesky account" msgstr "" -#: src/components/images/Gallery/index.tsx:460 +#: src/components/images/Gallery/index.tsx:455 msgid "Opens full image" msgstr "Opens full image" @@ -10975,7 +10975,7 @@ msgstr "" msgid "Skip to next step" msgstr "" -#: src/components/images/Gallery/index.tsx:443 +#: src/components/images/Gallery/index.tsx:438 msgid "slide" msgstr "slide" @@ -13259,8 +13259,8 @@ msgstr "" msgid "View your verifications" msgstr "" -#: src/components/images/AutoSizedImage.tsx:221 -#: src/components/images/AutoSizedImage.tsx:253 +#: src/components/images/AutoSizedImage.tsx:222 +#: src/components/images/AutoSizedImage.tsx:254 msgid "Views full image" msgstr "" From c753c949696f5bee8ec19d08ad4b732741a9f147 Mon Sep 17 00:00:00 2001 From: Spence Pope Date: Tue, 23 Jun 2026 11:33:23 -0400 Subject: [PATCH 02/43] Process videos on web with WebCodecs (#10955) Co-authored-by: Samuel Newman --- package.json | 1 + pnpm-lock.yaml | 23 ++ src/lib/media/video/compress.web.ts | 325 ++++++++++++++++-- src/lib/media/video/constants.ts | 8 + src/view/com/composer/Composer.tsx | 2 +- src/view/com/composer/SelectMediaButton.tsx | 14 +- .../videos/VideoTranscodeBackdrop.web.tsx | 25 +- .../videos/VideoTranscodeProgress.tsx | 3 - src/view/com/composer/videos/metadata.ts | 25 ++ src/view/com/composer/videos/metadata.web.ts | 126 +++++++ src/view/com/composer/videos/pickVideo.ts | 43 --- src/view/com/composer/videos/pickVideo.web.ts | 100 ------ 12 files changed, 512 insertions(+), 183 deletions(-) create mode 100644 src/lib/media/video/constants.ts create mode 100644 src/view/com/composer/videos/metadata.ts create mode 100644 src/view/com/composer/videos/metadata.web.ts delete mode 100644 src/view/com/composer/videos/pickVideo.ts delete mode 100644 src/view/com/composer/videos/pickVideo.web.ts diff --git a/package.json b/package.json index f261749e28..ccd213d743 100644 --- a/package.json +++ b/package.json @@ -204,6 +204,7 @@ "lodash.debounce": "^4.0.8", "lodash.shuffle": "^4.2.0", "lodash.throttle": "^4.1.1", + "mediabunny": "^1.25.3", "multiformats": "^13.4.2", "nanoid": "^5.0.5", "normalize-url": "^8.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6353790b2e..eee11bf2c3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -570,6 +570,9 @@ importers: lodash.throttle: specifier: ^4.1.1 version: 4.1.1 + mediabunny: + specifier: ^1.25.3 + version: 1.49.0 multiformats: specifier: ^13.4.2 version: 13.4.2 @@ -3494,6 +3497,12 @@ packages: '@types/connect@3.4.38': resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + '@types/dom-mediacapture-transform@0.1.11': + resolution: {integrity: sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ==} + + '@types/dom-webcodecs@0.1.13': + resolution: {integrity: sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==} + '@types/eslint-scope@3.7.7': resolution: {integrity: sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==} @@ -6849,6 +6858,9 @@ packages: resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} engines: {node: '>= 0.6'} + mediabunny@1.49.0: + resolution: {integrity: sha512-hDMtS/q22GFjyB3Yum+a6NHhDtVnye6hgaYTjx3DPUfnckRjzxFmXmCc3UQXdvo7d6PUCl7KM6Y9MRWzlvnAJQ==} + memfs@3.5.3: resolution: {integrity: sha512-UERzLsxzllchadvbPs5aolHh65ISpKpM+ccLbOJ8/vvpBKmAWf+la7dXFy7Mr0ySHbdHrFv5kGFCUHHe6GFEmw==} engines: {node: '>= 4.0.0'} @@ -12748,6 +12760,12 @@ snapshots: dependencies: '@types/node': 24.12.4 + '@types/dom-mediacapture-transform@0.1.11': + dependencies: + '@types/dom-webcodecs': 0.1.13 + + '@types/dom-webcodecs@0.1.13': {} + '@types/eslint-scope@3.7.7': dependencies: '@types/eslint': 9.6.1 @@ -16664,6 +16682,11 @@ snapshots: media-typer@0.3.0: {} + mediabunny@1.49.0: + dependencies: + '@types/dom-mediacapture-transform': 0.1.11 + '@types/dom-webcodecs': 0.1.13 + memfs@3.5.3: dependencies: fs-monkey: 1.1.0 diff --git a/src/lib/media/video/compress.web.ts b/src/lib/media/video/compress.web.ts index 83fdfde533..96e0fc43f0 100644 --- a/src/lib/media/video/compress.web.ts +++ b/src/lib/media/video/compress.web.ts @@ -1,56 +1,323 @@ import {type ImagePickerAsset} from 'expo-image-picker' +import { + ALL_FORMATS, + type AudioCodec, + BlobSource, + BufferTarget, + canEncodeAudio, + canEncodeVideo, + Conversion, + Input, + Mp4OutputFormat, + Output, + type VideoCodec, + WebMOutputFormat, +} from 'mediabunny' import {VIDEO_MAX_SIZE} from '#/lib/constants' import {VideoTooLargeError} from '#/lib/media/video/errors' +import {logger} from '#/logger' +import {hasWebCodecs} from '#/view/com/composer/videos/metadata' +import { + COMPRESSION_MAX_DIMENSION, + COMPRESSION_MIN_SIZE_BYTES, + COMPRESSION_TARGET_BITRATE, +} from './constants' import {type CompressedVideo} from './types' -// doesn't actually compress, converts to ArrayBuffer +// Codecs to try in order of preference +// avc (H.264) is most compatible, vp9/vp8 are fallbacks for WebM +const VIDEO_CODECS: VideoCodec[] = ['avc', 'hevc', 'vp9', 'vp8'] + export async function compressVideo( asset: ImagePickerAsset, - _opts?: { + opts?: { signal?: AbortSignal onProgress?: (progress: number) => void }, ): Promise { - const {mimeType, base64} = parseDataUrl(asset.uri) - const blob = base64ToBlob(base64, mimeType) - const uri = URL.createObjectURL(blob) + const {onProgress, signal} = opts || {} + logger.debug('compress: starting', { + uri: asset.uri.slice(0, 50), + hasWebCodecs: hasWebCodecs(), + }) + + const response = await fetch(asset.uri) + const blob = await response.blob() + + const isGif = blob.type === 'image/gif' + + logger.debug('compress: fetched blob', { + size: blob.size, + mimeType: blob.type, + isGif, + minSizeForCompression: COMPRESSION_MIN_SIZE_BYTES, + }) + + // Try MediaBunny compression if WebCodecs is available and file is large enough + // Skip GIFs - MediaBunny doesn't support them + if (hasWebCodecs() && blob.size >= COMPRESSION_MIN_SIZE_BYTES && !isGif) { + try { + return await doCompression(blob, asset.uri, {onProgress, signal}) + } catch (e) { + logger.warn('compress: MediaBunny compression failed, using original', { + safeMessage: e, + }) + } + } else { + logger.debug('compress: skipping compression', { + hasWebCodecs: hasWebCodecs(), + blobSize: blob.size, + minSize: COMPRESSION_MIN_SIZE_BYTES, + }) + } + + // No compression path - just return the blob as-is if (blob.size > VIDEO_MAX_SIZE) { throw new VideoTooLargeError() } return { + uri: asset.uri, size: blob.size, - uri, bytes: await blob.arrayBuffer(), + mimeType: blob.type || 'video/mp4', + } +} + +async function findEncodableVideoCodec( + width: number, + height: number, +): Promise<{codec: VideoCodec; useWebM: boolean} | null> { + for (const codec of VIDEO_CODECS) { + const canEncode = await canEncodeVideo(codec, { + width, + height, + bitrate: COMPRESSION_TARGET_BITRATE, + }) + logger.debug('compress: checking video codec', { + codec, + canEncode, + width, + height, + }) + if (canEncode) { + // vp8/vp9 need WebM container, others use MP4 + const useWebM = codec === 'vp8' || codec === 'vp9' + return {codec, useWebM} + } + } + return null +} + +// Audio codecs to try - aac for MP4, opus for WebM +const AUDIO_CODECS_MP4: AudioCodec[] = ['aac'] +const AUDIO_CODECS_WEBM: AudioCodec[] = ['opus', 'vorbis'] + +async function findEncodableAudioCodec( + audioTrack: Awaited>, + useWebM: boolean, +): Promise<{codec: AudioCodec} | null> { + if (!audioTrack) { + return null + } + + // First check if we can decode the source audio + const canDecodeSource = await audioTrack.canDecode() + logger.debug('compress: checking audio source', { + sourceCodec: audioTrack.codec, + canDecode: canDecodeSource, + channels: audioTrack.numberOfChannels, + sampleRate: audioTrack.sampleRate, + }) + + if (!canDecodeSource) { + return null + } + + const codecsToTry = useWebM ? AUDIO_CODECS_WEBM : AUDIO_CODECS_MP4 + + for (const codec of codecsToTry) { + const canEncode = await canEncodeAudio(codec, { + numberOfChannels: audioTrack.numberOfChannels, + sampleRate: audioTrack.sampleRate, + }) + logger.debug('compress: checking audio encode codec', { + codec, + canEncode, + }) + if (canEncode) { + return {codec} + } + } + + return null +} + +async function doCompression( + blob: Blob, + originalUri: string, + opts: { + onProgress?: (progress: number) => void + signal?: AbortSignal + }, +): Promise { + const {onProgress, signal} = opts + + const input = new Input({ + source: new BlobSource(blob), + formats: ALL_FORMATS, + }) + + // Get video track to determine dimensions for codec check + const videoTrack = await input.getPrimaryVideoTrack() + if (!videoTrack) { + input.dispose() + throw new Error('No video track found') + } + + // Get audio track to check if we can encode it + const audioTrack = await input.getPrimaryAudioTrack() + + const {width, height} = calculateDimensions( + videoTrack.displayWidth, + videoTrack.displayHeight, + COMPRESSION_MAX_DIMENSION, + ) + + logger.debug('compress: video dimensions', { + original: { + width: videoTrack.displayWidth, + height: videoTrack.displayHeight, + }, + target: {width, height}, + audioCodec: audioTrack?.codec, + }) + + // Find a video codec we can encode with + const codecInfo = await findEncodableVideoCodec(width, height) + if (!codecInfo) { + input.dispose() + throw new Error('No supported video codec available') + } + + // Check if we can encode the audio + const audioCodecInfo = await findEncodableAudioCodec( + audioTrack, + codecInfo.useWebM, + ) + + logger.debug('compress: using codecs', { + video: codecInfo.codec, + audio: audioCodecInfo?.codec ?? 'none', + useWebM: codecInfo.useWebM, + }) + + const target = new BufferTarget() + const output = new Output({ + format: codecInfo.useWebM ? new WebMOutputFormat() : new Mp4OutputFormat(), + target, + }) + + // If we have audio but can't encode it, bail out and use the original + if (audioTrack && !audioCodecInfo) { + input.dispose() + throw new Error( + `Cannot encode audio codec: ${audioTrack.codec ?? 'unknown'}`, + ) + } + + const conversion = await Conversion.init({ + input, + output, + video: { + codec: codecInfo.codec, + bitrate: COMPRESSION_TARGET_BITRATE, + width, + height, + fit: 'contain', + }, + audio: audioCodecInfo ? {codec: audioCodecInfo.codec} : undefined, + }) + + if (onProgress) { + conversion.onProgress = onProgress + } + + if (signal) { + signal.addEventListener( + 'abort', + () => { + logger.debug('compress: cancelled') + void conversion.cancel() + }, + {once: true}, + ) + } + + logger.debug('compress: starting conversion') + const startTime = performance.now() + + try { + await conversion.execute() + } finally { + input.dispose() + } + + const elapsed = performance.now() - startTime + const bytes = target.buffer + + if (!bytes) { + // mediabunny's BufferTarget reports a null buffer after a successful + // execute(). Should not happen in normal use; recoverable here because + // the outer compressVideo() catches and falls back to the original blob, + // but worth flagging in Sentry so we can chase the root cause. + const err = new Error('Compression produced empty output') + logger.error(err, {}) + throw err + } + + const mimeType = codecInfo.useWebM ? 'video/webm' : 'video/mp4' + + const savedBytes = blob.size - bytes.byteLength + const savedPercent = ((savedBytes / blob.size) * 100).toFixed(1) + + logger.debug('compress: completed', { + from: blob.type, + to: mimeType, + originalSize: blob.size, + compressedSize: bytes.byteLength, + savedBytes, + savedPercent: `${savedPercent}%`, + elapsedMs: Math.round(elapsed), + }) + + if (bytes.byteLength > VIDEO_MAX_SIZE) { + throw new VideoTooLargeError() + } + + return { + uri: originalUri, + size: bytes.byteLength, + bytes, mimeType, } } -function parseDataUrl(dataUrl: string) { - const [mimeType, base64] = dataUrl.slice('data:'.length).split(';base64,') - if (!mimeType || !base64) { - throw new Error('Invalid data URL') - } - return {mimeType, base64} -} - -function base64ToBlob(base64: string, mimeType: string) { - const byteCharacters = atob(base64) - const byteArrays = [] - - for (let offset = 0; offset < byteCharacters.length; offset += 512) { - const slice = byteCharacters.slice(offset, offset + 512) - const byteNumbers = new Array(slice.length) - - for (let i = 0; i < slice.length; i++) { - byteNumbers[i] = slice.charCodeAt(i) - } - - const byteArray = new Uint8Array(byteNumbers) - byteArrays.push(byteArray) +function calculateDimensions( + width: number, + height: number, + maxDimension: number, +): {width: number; height: number} { + const maxSide = Math.max(width, height) + if (maxSide <= maxDimension) { + return {width, height} } - return new Blob(byteArrays, {type: mimeType}) + const scale = maxDimension / maxSide + return { + width: Math.round(width * scale), + height: Math.round(height * scale), + } } diff --git a/src/lib/media/video/constants.ts b/src/lib/media/video/constants.ts new file mode 100644 index 0000000000..a328c64d68 --- /dev/null +++ b/src/lib/media/video/constants.ts @@ -0,0 +1,8 @@ +// Shared compression knobs. Mirrored between native (compress.ts) and web +// (compress.web.ts) so both platforms produce videos with the same target. + +export const COMPRESSION_TARGET_BITRATE = 3_000_000 // 3 Mbps +export const COMPRESSION_MAX_DIMENSION = 1920 +// Web skips compression entirely for files under this size; native applies its +// own threshold logic inside expo-bluesky-video-compress's probe step. +export const COMPRESSION_MIN_SIZE_BYTES = 25_000_000 diff --git a/src/view/com/composer/Composer.tsx b/src/view/com/composer/Composer.tsx index 54932286be..78a78a22a8 100644 --- a/src/view/com/composer/Composer.tsx +++ b/src/view/com/composer/Composer.tsx @@ -179,7 +179,7 @@ import { type VideoState, } from './state/video' import {type TextInputRef} from './text-input/TextInput.types' -import {getVideoMetadata} from './videos/pickVideo' +import {getVideoMetadata} from './videos/metadata' import {clearThumbnailCache} from './videos/VideoTranscodeBackdrop' type CancelRef = { diff --git a/src/view/com/composer/SelectMediaButton.tsx b/src/view/com/composer/SelectMediaButton.tsx index 1de39f5f54..d3cb92db7c 100644 --- a/src/view/com/composer/SelectMediaButton.tsx +++ b/src/view/com/composer/SelectMediaButton.tsx @@ -24,6 +24,7 @@ import {Image_Stroke2_Corner0_Rounded as ImageIcon} from '#/components/icons/Ima import * as toast from '#/components/Toast' import {IS_NATIVE, IS_WEB} from '#/env' import {isAnimatedGif} from './videos/isAnimatedGif' +import {hasWebCodecs} from './videos/metadata' export type SelectMediaButtonProps = { disabled?: boolean @@ -291,9 +292,15 @@ async function processImagePickerAssets( /* * Filesize appears to be stable across all platforms, so we can use it * to filter out large files on web. On native, we compress these anyway, - * so we only check on web. + * so we only check on web. On web, we can reject early if the browser + * doesn't support WebCodecs. */ - if (IS_WEB && asset.fileSize && asset.fileSize > VIDEO_MAX_SIZE) { + if ( + IS_WEB && + !hasWebCodecs() && + asset.fileSize && + asset.fileSize > VIDEO_MAX_SIZE + ) { errors.add(SelectedAssetError.FileTooBig) continue } @@ -309,8 +316,7 @@ async function processImagePickerAssets( if (type === 'gif') { /* * Filesize appears to be stable across all platforms, so we can use it - * to filter out large files on web. On native, we compress GIFs as - * videos anyway, so we only check on web. + * to filter out large files. We can't compress GIFs on either platform. */ if (IS_WEB && asset.fileSize && asset.fileSize > VIDEO_MAX_SIZE) { errors.add(SelectedAssetError.FileTooBig) diff --git a/src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx b/src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx index a04200f53a..cb8eb52bec 100644 --- a/src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx +++ b/src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx @@ -1,7 +1,26 @@ +import {atoms as a, flatten} from '#/alf' + export function clearThumbnailCache() { - // no-op + // no-op on web } -export function VideoTranscodeBackdrop() { - return null +export function VideoTranscodeBackdrop({uri}: {uri: string}) { + return ( +