From 1e8eccffd2d0082f52017f7181da63a16bfb0af1 Mon Sep 17 00:00:00 2001 From: Samuel Newman Date: Thu, 27 Nov 2025 16:31:19 +0200 Subject: [PATCH] process videos on web with webcodecs --- package.json | 1 + src/lib/media/video/compress.ts | 4 + src/lib/media/video/compress.web.ts | 310 ++++++++++++++++-- src/view/com/composer/Composer.tsx | 2 +- src/view/com/composer/SelectMediaButton.tsx | 14 +- .../videos/VideoTranscodeBackdrop.web.tsx | 25 +- .../videos/VideoTranscodeProgress.tsx | 3 - src/view/com/composer/videos/metadata.ts | 9 + src/view/com/composer/videos/metadata.web.ts | 120 +++++++ src/view/com/composer/videos/pickVideo.ts | 43 --- src/view/com/composer/videos/pickVideo.web.ts | 100 ------ 11 files changed, 448 insertions(+), 183 deletions(-) create mode 100644 src/view/com/composer/videos/metadata.ts create mode 100644 src/view/com/composer/videos/metadata.web.ts delete mode 100644 src/view/com/composer/videos/pickVideo.ts delete mode 100644 src/view/com/composer/videos/pickVideo.web.ts diff --git a/package.json b/package.json index f261749e28..ccd213d743 100644 --- a/package.json +++ b/package.json @@ -204,6 +204,7 @@ "lodash.debounce": "^4.0.8", "lodash.shuffle": "^4.2.0", "lodash.throttle": "^4.1.1", + "mediabunny": "^1.25.3", "multiformats": "^13.4.2", "nanoid": "^5.0.5", "normalize-url": "^8.0.0", diff --git a/src/lib/media/video/compress.ts b/src/lib/media/video/compress.ts index 1d00bfcea1..fe05d28328 100644 --- a/src/lib/media/video/compress.ts +++ b/src/lib/media/video/compress.ts @@ -7,6 +7,10 @@ import {extToMime} from './util' const MIN_SIZE_FOR_COMPRESSION = 25 // 25mb +export function hasWebCodecs(): boolean { + throw new Error('Native does not have WebCodecs.') +} + export async function compressVideo( file: ImagePickerAsset, opts?: { diff --git a/src/lib/media/video/compress.web.ts b/src/lib/media/video/compress.web.ts index 83fdfde533..06b900b626 100644 --- a/src/lib/media/video/compress.web.ts +++ b/src/lib/media/video/compress.web.ts @@ -1,56 +1,308 @@ import {type ImagePickerAsset} from 'expo-image-picker' +import { + ALL_FORMATS, + type AudioCodec, + BlobSource, + BufferTarget, + canEncodeAudio, + canEncodeVideo, + Conversion, + Input, + Mp4OutputFormat, + Output, + type VideoCodec, + WebMOutputFormat, +} from 'mediabunny' import {VIDEO_MAX_SIZE} from '#/lib/constants' import {VideoTooLargeError} from '#/lib/media/video/errors' +import {logger} from '#/logger' +import {hasWebCodecs} from '#/view/com/composer/videos/metadata' import {type CompressedVideo} from './types' -// doesn't actually compress, converts to ArrayBuffer +const TARGET_BITRATE = 3_000_000 // 3mbps, matches native +const MAX_DIMENSION = 1920 // matches native +const MIN_SIZE_FOR_COMPRESSION = 25 * 1000 * 1000 // 25mb, matches native + +// Codecs to try in order of preference +// avc (H.264) is most compatible, vp9/vp8 are fallbacks for WebM +const VIDEO_CODECS: VideoCodec[] = ['avc', 'hevc', 'vp9', 'vp8'] + export async function compressVideo( asset: ImagePickerAsset, - _opts?: { + opts?: { signal?: AbortSignal onProgress?: (progress: number) => void }, ): Promise { - const {mimeType, base64} = parseDataUrl(asset.uri) - const blob = base64ToBlob(base64, mimeType) - const uri = URL.createObjectURL(blob) + const {onProgress, signal} = opts || {} + logger.debug('compress: starting', { + uri: asset.uri.slice(0, 50), + hasWebCodecs: hasWebCodecs(), + }) + + const response = await fetch(asset.uri) + const blob = await response.blob() + + const isGif = blob.type === 'image/gif' + + logger.debug('compress: fetched blob', { + size: blob.size, + mimeType: blob.type, + isGif, + minSizeForCompression: MIN_SIZE_FOR_COMPRESSION, + }) + + // Try MediaBunny compression if WebCodecs is available and file is large enough + // Skip GIFs - MediaBunny doesn't support them + if (hasWebCodecs() && blob.size >= MIN_SIZE_FOR_COMPRESSION && !isGif) { + try { + return await doCompression(blob, asset.uri, {onProgress, signal}) + } catch (e) { + logger.warn('compress: MediaBunny compression failed, using original', { + safeMessage: e, + }) + } + } else { + logger.debug('compress: skipping compression', { + hasWebCodecs: hasWebCodecs(), + blobSize: blob.size, + minSize: MIN_SIZE_FOR_COMPRESSION, + }) + } + + // No compression path - just return the blob as-is if (blob.size > VIDEO_MAX_SIZE) { throw new VideoTooLargeError() } return { + uri: asset.uri, size: blob.size, - uri, bytes: await blob.arrayBuffer(), + mimeType: blob.type || 'video/mp4', + } +} + +async function findEncodableVideoCodec( + width: number, + height: number, +): Promise<{codec: VideoCodec; useWebM: boolean} | null> { + for (const codec of VIDEO_CODECS) { + const canEncode = await canEncodeVideo(codec, { + width, + height, + bitrate: TARGET_BITRATE, + }) + logger.debug('compress: checking video codec', { + codec, + canEncode, + width, + height, + }) + if (canEncode) { + // vp8/vp9 need WebM container, others use MP4 + const useWebM = codec === 'vp8' || codec === 'vp9' + return {codec, useWebM} + } + } + return null +} + +// Audio codecs to try - aac for MP4, opus for WebM +const AUDIO_CODECS_MP4: AudioCodec[] = ['aac'] +const AUDIO_CODECS_WEBM: AudioCodec[] = ['opus', 'vorbis'] + +async function findEncodableAudioCodec( + audioTrack: Awaited>, + useWebM: boolean, +): Promise<{codec: AudioCodec} | null> { + if (!audioTrack) { + return null + } + + // First check if we can decode the source audio + const canDecodeSource = await audioTrack.canDecode() + logger.debug('compress: checking audio source', { + sourceCodec: audioTrack.codec, + canDecode: canDecodeSource, + channels: audioTrack.numberOfChannels, + sampleRate: audioTrack.sampleRate, + }) + + if (!canDecodeSource) { + return null + } + + const codecsToTry = useWebM ? AUDIO_CODECS_WEBM : AUDIO_CODECS_MP4 + + for (const codec of codecsToTry) { + const canEncode = await canEncodeAudio(codec, { + numberOfChannels: audioTrack.numberOfChannels, + sampleRate: audioTrack.sampleRate, + }) + logger.debug('compress: checking audio encode codec', { + codec, + canEncode, + }) + if (canEncode) { + return {codec} + } + } + + return null +} + +async function doCompression( + blob: Blob, + originalUri: string, + opts: { + onProgress?: (progress: number) => void + signal?: AbortSignal + }, +): Promise { + const {onProgress, signal} = opts + + const input = new Input({ + source: new BlobSource(blob), + formats: ALL_FORMATS, + }) + + // Get video track to determine dimensions for codec check + const videoTrack = await input.getPrimaryVideoTrack() + if (!videoTrack) { + input.dispose() + throw new Error('No video track found') + } + + // Get audio track to check if we can encode it + const audioTrack = await input.getPrimaryAudioTrack() + + const {width, height} = calculateDimensions( + videoTrack.displayWidth, + videoTrack.displayHeight, + MAX_DIMENSION, + ) + + logger.debug('compress: video dimensions', { + original: { + width: videoTrack.displayWidth, + height: videoTrack.displayHeight, + }, + target: {width, height}, + audioCodec: audioTrack?.codec, + }) + + // Find a video codec we can encode with + const codecInfo = await findEncodableVideoCodec(width, height) + if (!codecInfo) { + input.dispose() + throw new Error('No supported video codec available') + } + + // Check if we can encode the audio + const audioCodecInfo = await findEncodableAudioCodec( + audioTrack, + codecInfo.useWebM, + ) + + logger.debug('compress: using codecs', { + video: codecInfo.codec, + audio: audioCodecInfo?.codec ?? 'none', + useWebM: codecInfo.useWebM, + }) + + const target = new BufferTarget() + const output = new Output({ + format: codecInfo.useWebM ? new WebMOutputFormat() : new Mp4OutputFormat(), + target, + }) + + // If we have audio but can't encode it, bail out and use the original + if (audioTrack && !audioCodecInfo) { + input.dispose() + throw new Error( + `Cannot encode audio codec: ${audioTrack.codec ?? 'unknown'}`, + ) + } + + const conversion = await Conversion.init({ + input, + output, + video: { + codec: codecInfo.codec, + bitrate: TARGET_BITRATE, + width, + height, + fit: 'contain', + }, + audio: audioCodecInfo ? {codec: audioCodecInfo.codec} : undefined, + }) + + if (onProgress) { + conversion.onProgress = onProgress + } + + if (signal) { + signal.addEventListener('abort', () => { + logger.debug('compress: cancelled') + conversion.cancel() + }) + } + + logger.debug('compress: starting conversion') + const startTime = performance.now() + + await conversion.execute() + + const elapsed = performance.now() - startTime + const bytes = target.buffer + + if (!bytes) { + throw new Error('MediaBunny compression produced no output') + } + + const mimeType = codecInfo.useWebM ? 'video/webm' : 'video/mp4' + + const savedBytes = blob.size - bytes.byteLength + const savedPercent = ((savedBytes / blob.size) * 100).toFixed(1) + + logger.debug('compress: completed', { + from: blob.type, + to: mimeType, + originalSize: blob.size, + compressedSize: bytes.byteLength, + savedBytes, + savedPercent: `${savedPercent}%`, + elapsedMs: Math.round(elapsed), + }) + + if (bytes.byteLength > VIDEO_MAX_SIZE) { + throw new VideoTooLargeError() + } + + return { + uri: originalUri, + size: bytes.byteLength, + bytes, mimeType, } } -function parseDataUrl(dataUrl: string) { - const [mimeType, base64] = dataUrl.slice('data:'.length).split(';base64,') - if (!mimeType || !base64) { - throw new Error('Invalid data URL') - } - return {mimeType, base64} -} - -function base64ToBlob(base64: string, mimeType: string) { - const byteCharacters = atob(base64) - const byteArrays = [] - - for (let offset = 0; offset < byteCharacters.length; offset += 512) { - const slice = byteCharacters.slice(offset, offset + 512) - const byteNumbers = new Array(slice.length) - - for (let i = 0; i < slice.length; i++) { - byteNumbers[i] = slice.charCodeAt(i) - } - - const byteArray = new Uint8Array(byteNumbers) - byteArrays.push(byteArray) +function calculateDimensions( + width: number, + height: number, + maxDimension: number, +): {width: number; height: number} { + const maxSide = Math.max(width, height) + if (maxSide <= maxDimension) { + return {width, height} } - return new Blob(byteArrays, {type: mimeType}) + const scale = maxDimension / maxSide + return { + width: Math.round(width * scale), + height: Math.round(height * scale), + } } diff --git a/src/view/com/composer/Composer.tsx b/src/view/com/composer/Composer.tsx index 54932286be..78a78a22a8 100644 --- a/src/view/com/composer/Composer.tsx +++ b/src/view/com/composer/Composer.tsx @@ -179,7 +179,7 @@ import { type VideoState, } from './state/video' import {type TextInputRef} from './text-input/TextInput.types' -import {getVideoMetadata} from './videos/pickVideo' +import {getVideoMetadata} from './videos/metadata' import {clearThumbnailCache} from './videos/VideoTranscodeBackdrop' type CancelRef = { diff --git a/src/view/com/composer/SelectMediaButton.tsx b/src/view/com/composer/SelectMediaButton.tsx index 1de39f5f54..d3cb92db7c 100644 --- a/src/view/com/composer/SelectMediaButton.tsx +++ b/src/view/com/composer/SelectMediaButton.tsx @@ -24,6 +24,7 @@ import {Image_Stroke2_Corner0_Rounded as ImageIcon} from '#/components/icons/Ima import * as toast from '#/components/Toast' import {IS_NATIVE, IS_WEB} from '#/env' import {isAnimatedGif} from './videos/isAnimatedGif' +import {hasWebCodecs} from './videos/metadata' export type SelectMediaButtonProps = { disabled?: boolean @@ -291,9 +292,15 @@ async function processImagePickerAssets( /* * Filesize appears to be stable across all platforms, so we can use it * to filter out large files on web. On native, we compress these anyway, - * so we only check on web. + * so we only check on web. On web, we can reject early if the browser + * doesn't support WebCodecs. */ - if (IS_WEB && asset.fileSize && asset.fileSize > VIDEO_MAX_SIZE) { + if ( + IS_WEB && + !hasWebCodecs() && + asset.fileSize && + asset.fileSize > VIDEO_MAX_SIZE + ) { errors.add(SelectedAssetError.FileTooBig) continue } @@ -309,8 +316,7 @@ async function processImagePickerAssets( if (type === 'gif') { /* * Filesize appears to be stable across all platforms, so we can use it - * to filter out large files on web. On native, we compress GIFs as - * videos anyway, so we only check on web. + * to filter out large files. We can't compress GIFs on either platform. */ if (IS_WEB && asset.fileSize && asset.fileSize > VIDEO_MAX_SIZE) { errors.add(SelectedAssetError.FileTooBig) diff --git a/src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx b/src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx index a04200f53a..cb8eb52bec 100644 --- a/src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx +++ b/src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx @@ -1,7 +1,26 @@ +import {atoms as a, flatten} from '#/alf' + export function clearThumbnailCache() { - // no-op + // no-op on web } -export function VideoTranscodeBackdrop() { - return null +export function VideoTranscodeBackdrop({uri}: {uri: string}) { + return ( +