Process videos on web with WebCodecs (#10955)

Co-authored-by: Samuel Newman <mozzius@protonmail.com>
2026-06-23 11:33:23 -04:00
parent cc0e1f88ea
commit c753c94969
12 changed files with 512 additions and 183 deletions
@@ -1,56 +1,323 @@
 import {type ImagePickerAsset} from 'expo-image-picker'
+import {
+  ALL_FORMATS,
+  type AudioCodec,
+  BlobSource,
+  BufferTarget,
+  canEncodeAudio,
+  canEncodeVideo,
+  Conversion,
+  Input,
+  Mp4OutputFormat,
+  Output,
+  type VideoCodec,
+  WebMOutputFormat,
+} from 'mediabunny'

 import {VIDEO_MAX_SIZE} from '#/lib/constants'
 import {VideoTooLargeError} from '#/lib/media/video/errors'
+import {logger} from '#/logger'
+import {hasWebCodecs} from '#/view/com/composer/videos/metadata'
+import {
+  COMPRESSION_MAX_DIMENSION,
+  COMPRESSION_MIN_SIZE_BYTES,
+  COMPRESSION_TARGET_BITRATE,
+} from './constants'
 import {type CompressedVideo} from './types'

-// doesn't actually compress, converts to ArrayBuffer
+// Codecs to try in order of preference
+// avc (H.264) is most compatible, vp9/vp8 are fallbacks for WebM
+const VIDEO_CODECS: VideoCodec[] = ['avc', 'hevc', 'vp9', 'vp8']
+
 export async function compressVideo(
  asset: ImagePickerAsset,
-  _opts?: {
+  opts?: {
    signal?: AbortSignal
    onProgress?: (progress: number) => void
  },
 ): Promise<CompressedVideo> {
-  const {mimeType, base64} = parseDataUrl(asset.uri)
-  const blob = base64ToBlob(base64, mimeType)
-  const uri = URL.createObjectURL(blob)
+  const {onProgress, signal} = opts || {}

+  logger.debug('compress: starting', {
+    uri: asset.uri.slice(0, 50),
+    hasWebCodecs: hasWebCodecs(),
+  })
+
+  const response = await fetch(asset.uri)
+  const blob = await response.blob()
+
+  const isGif = blob.type === 'image/gif'
+
+  logger.debug('compress: fetched blob', {
+    size: blob.size,
+    mimeType: blob.type,
+    isGif,
+    minSizeForCompression: COMPRESSION_MIN_SIZE_BYTES,
+  })
+
+  // Try MediaBunny compression if WebCodecs is available and file is large enough
+  // Skip GIFs - MediaBunny doesn't support them
+  if (hasWebCodecs() && blob.size >= COMPRESSION_MIN_SIZE_BYTES && !isGif) {
+    try {
+      return await doCompression(blob, asset.uri, {onProgress, signal})
+    } catch (e) {
+      logger.warn('compress: MediaBunny compression failed, using original', {
+        safeMessage: e,
+      })
+    }
+  } else {
+    logger.debug('compress: skipping compression', {
+      hasWebCodecs: hasWebCodecs(),
+      blobSize: blob.size,
+      minSize: COMPRESSION_MIN_SIZE_BYTES,
+    })
+  }
+
+  // No compression path - just return the blob as-is
  if (blob.size > VIDEO_MAX_SIZE) {
    throw new VideoTooLargeError()
  }

  return {
+    uri: asset.uri,
    size: blob.size,
-    uri,
    bytes: await blob.arrayBuffer(),
+    mimeType: blob.type || 'video/mp4',
+  }
+}
+
+async function findEncodableVideoCodec(
+  width: number,
+  height: number,
+): Promise<{codec: VideoCodec; useWebM: boolean} | null> {
+  for (const codec of VIDEO_CODECS) {
+    const canEncode = await canEncodeVideo(codec, {
+      width,
+      height,
+      bitrate: COMPRESSION_TARGET_BITRATE,
+    })
+    logger.debug('compress: checking video codec', {
+      codec,
+      canEncode,
+      width,
+      height,
+    })
+    if (canEncode) {
+      // vp8/vp9 need WebM container, others use MP4
+      const useWebM = codec === 'vp8' || codec === 'vp9'
+      return {codec, useWebM}
+    }
+  }
+  return null
+}
+
+// Audio codecs to try - aac for MP4, opus for WebM
+const AUDIO_CODECS_MP4: AudioCodec[] = ['aac']
+const AUDIO_CODECS_WEBM: AudioCodec[] = ['opus', 'vorbis']
+
+async function findEncodableAudioCodec(
+  audioTrack: Awaited<ReturnType<Input['getPrimaryAudioTrack']>>,
+  useWebM: boolean,
+): Promise<{codec: AudioCodec} | null> {
+  if (!audioTrack) {
+    return null
+  }
+
+  // First check if we can decode the source audio
+  const canDecodeSource = await audioTrack.canDecode()
+  logger.debug('compress: checking audio source', {
+    sourceCodec: audioTrack.codec,
+    canDecode: canDecodeSource,
+    channels: audioTrack.numberOfChannels,
+    sampleRate: audioTrack.sampleRate,
+  })
+
+  if (!canDecodeSource) {
+    return null
+  }
+
+  const codecsToTry = useWebM ? AUDIO_CODECS_WEBM : AUDIO_CODECS_MP4
+
+  for (const codec of codecsToTry) {
+    const canEncode = await canEncodeAudio(codec, {
+      numberOfChannels: audioTrack.numberOfChannels,
+      sampleRate: audioTrack.sampleRate,
+    })
+    logger.debug('compress: checking audio encode codec', {
+      codec,
+      canEncode,
+    })
+    if (canEncode) {
+      return {codec}
+    }
+  }
+
+  return null
+}
+
+async function doCompression(
+  blob: Blob,
+  originalUri: string,
+  opts: {
+    onProgress?: (progress: number) => void
+    signal?: AbortSignal
+  },
+): Promise<CompressedVideo> {
+  const {onProgress, signal} = opts
+
+  const input = new Input({
+    source: new BlobSource(blob),
+    formats: ALL_FORMATS,
+  })
+
+  // Get video track to determine dimensions for codec check
+  const videoTrack = await input.getPrimaryVideoTrack()
+  if (!videoTrack) {
+    input.dispose()
+    throw new Error('No video track found')
+  }
+
+  // Get audio track to check if we can encode it
+  const audioTrack = await input.getPrimaryAudioTrack()
+
+  const {width, height} = calculateDimensions(
+    videoTrack.displayWidth,
+    videoTrack.displayHeight,
+    COMPRESSION_MAX_DIMENSION,
+  )
+
+  logger.debug('compress: video dimensions', {
+    original: {
+      width: videoTrack.displayWidth,
+      height: videoTrack.displayHeight,
+    },
+    target: {width, height},
+    audioCodec: audioTrack?.codec,
+  })
+
+  // Find a video codec we can encode with
+  const codecInfo = await findEncodableVideoCodec(width, height)
+  if (!codecInfo) {
+    input.dispose()
+    throw new Error('No supported video codec available')
+  }
+
+  // Check if we can encode the audio
+  const audioCodecInfo = await findEncodableAudioCodec(
+    audioTrack,
+    codecInfo.useWebM,
+  )
+
+  logger.debug('compress: using codecs', {
+    video: codecInfo.codec,
+    audio: audioCodecInfo?.codec ?? 'none',
+    useWebM: codecInfo.useWebM,
+  })
+
+  const target = new BufferTarget()
+  const output = new Output({
+    format: codecInfo.useWebM ? new WebMOutputFormat() : new Mp4OutputFormat(),
+    target,
+  })
+
+  // If we have audio but can't encode it, bail out and use the original
+  if (audioTrack && !audioCodecInfo) {
+    input.dispose()
+    throw new Error(
+      `Cannot encode audio codec: ${audioTrack.codec ?? 'unknown'}`,
+    )
+  }
+
+  const conversion = await Conversion.init({
+    input,
+    output,
+    video: {
+      codec: codecInfo.codec,
+      bitrate: COMPRESSION_TARGET_BITRATE,
+      width,
+      height,
+      fit: 'contain',
+    },
+    audio: audioCodecInfo ? {codec: audioCodecInfo.codec} : undefined,
+  })
+
+  if (onProgress) {
+    conversion.onProgress = onProgress
+  }
+
+  if (signal) {
+    signal.addEventListener(
+      'abort',
+      () => {
+        logger.debug('compress: cancelled')
+        void conversion.cancel()
+      },
+      {once: true},
+    )
+  }
+
+  logger.debug('compress: starting conversion')
+  const startTime = performance.now()
+
+  try {
+    await conversion.execute()
+  } finally {
+    input.dispose()
+  }
+
+  const elapsed = performance.now() - startTime
+  const bytes = target.buffer
+
+  if (!bytes) {
+    // mediabunny's BufferTarget reports a null buffer after a successful
+    // execute(). Should not happen in normal use; recoverable here because
+    // the outer compressVideo() catches and falls back to the original blob,
+    // but worth flagging in Sentry so we can chase the root cause.
+    const err = new Error('Compression produced empty output')
+    logger.error(err, {})
+    throw err
+  }
+
+  const mimeType = codecInfo.useWebM ? 'video/webm' : 'video/mp4'
+
+  const savedBytes = blob.size - bytes.byteLength
+  const savedPercent = ((savedBytes / blob.size) * 100).toFixed(1)
+
+  logger.debug('compress: completed', {
+    from: blob.type,
+    to: mimeType,
+    originalSize: blob.size,
+    compressedSize: bytes.byteLength,
+    savedBytes,
+    savedPercent: `${savedPercent}%`,
+    elapsedMs: Math.round(elapsed),
+  })
+
+  if (bytes.byteLength > VIDEO_MAX_SIZE) {
+    throw new VideoTooLargeError()
+  }
+
+  return {
+    uri: originalUri,
+    size: bytes.byteLength,
+    bytes,
    mimeType,
  }
 }

-function parseDataUrl(dataUrl: string) {
-  const [mimeType, base64] = dataUrl.slice('data:'.length).split(';base64,')
-  if (!mimeType || !base64) {
-    throw new Error('Invalid data URL')
-  }
-  return {mimeType, base64}
-}
-
-function base64ToBlob(base64: string, mimeType: string) {
-  const byteCharacters = atob(base64)
-  const byteArrays = []
-
-  for (let offset = 0; offset < byteCharacters.length; offset += 512) {
-    const slice = byteCharacters.slice(offset, offset + 512)
-    const byteNumbers = new Array(slice.length)
-
-    for (let i = 0; i < slice.length; i++) {
-      byteNumbers[i] = slice.charCodeAt(i)
-    }
-
-    const byteArray = new Uint8Array(byteNumbers)
-    byteArrays.push(byteArray)
+function calculateDimensions(
+  width: number,
+  height: number,
+  maxDimension: number,
+): {width: number; height: number} {
+  const maxSide = Math.max(width, height)
+  if (maxSide <= maxDimension) {
+    return {width, height}
  }

-  return new Blob(byteArrays, {type: mimeType})
+  const scale = maxDimension / maxSide
+  return {
+    width: Math.round(width * scale),
+    height: Math.round(height * scale),
+  }
 }
@@ -0,0 +1,8 @@
+// Shared compression knobs. Mirrored between native (compress.ts) and web
+// (compress.web.ts) so both platforms produce videos with the same target.
+
+export const COMPRESSION_TARGET_BITRATE = 3_000_000 // 3 Mbps
+export const COMPRESSION_MAX_DIMENSION = 1920
+// Web skips compression entirely for files under this size; native applies its
+// own threshold logic inside expo-bluesky-video-compress's probe step.
+export const COMPRESSION_MIN_SIZE_BYTES = 25_000_000