process videos on web with webcodecs

package.json

@@ -204,6 +204,7 @@
     "lodash.debounce": "^4.0.8",
     "lodash.shuffle": "^4.2.0",
     "lodash.throttle": "^4.1.1",
+    "mediabunny": "^1.25.3",
     "multiformats": "^13.4.2",
     "nanoid": "^5.0.5",
     "normalize-url": "^8.0.0",

src/lib/media/video/compress.ts

@@ -7,6 +7,10 @@ import {extToMime} from './util'
 
 const MIN_SIZE_FOR_COMPRESSION = 25 // 25mb
 
+export function hasWebCodecs(): boolean {
+  throw new Error('Native does not have WebCodecs.')
+}
+
 export async function compressVideo(
   file: ImagePickerAsset,
   opts?: {

src/lib/media/video/compress.web.ts

@@ -1,56 +1,308 @@
 import {type ImagePickerAsset} from 'expo-image-picker'
+import {
+  ALL_FORMATS,
+  type AudioCodec,
+  BlobSource,
+  BufferTarget,
+  canEncodeAudio,
+  canEncodeVideo,
+  Conversion,
+  Input,
+  Mp4OutputFormat,
+  Output,
+  type VideoCodec,
+  WebMOutputFormat,
+} from 'mediabunny'
 
 import {VIDEO_MAX_SIZE} from '#/lib/constants'
 import {VideoTooLargeError} from '#/lib/media/video/errors'
+import {logger} from '#/logger'
+import {hasWebCodecs} from '#/view/com/composer/videos/metadata'
 import {type CompressedVideo} from './types'
 
-// doesn't actually compress, converts to ArrayBuffer
+const TARGET_BITRATE = 3_000_000 // 3mbps, matches native
+const MAX_DIMENSION = 1920 // matches native
+const MIN_SIZE_FOR_COMPRESSION = 25 * 1000 * 1000 // 25mb, matches native
+
+// Codecs to try in order of preference
+// avc (H.264) is most compatible, vp9/vp8 are fallbacks for WebM
+const VIDEO_CODECS: VideoCodec[] = ['avc', 'hevc', 'vp9', 'vp8']
+
 export async function compressVideo(
   asset: ImagePickerAsset,
-  _opts?: {
+  opts?: {
     signal?: AbortSignal
     onProgress?: (progress: number) => void
   },
 ): Promise<CompressedVideo> {
-  const {mimeType, base64} = parseDataUrl(asset.uri)
+  const {onProgress, signal} = opts || {}
-  const blob = base64ToBlob(base64, mimeType)
-  const uri = URL.createObjectURL(blob)
 
+  logger.debug('compress: starting', {
+    uri: asset.uri.slice(0, 50),
+    hasWebCodecs: hasWebCodecs(),
+  })
+
+  const response = await fetch(asset.uri)
+  const blob = await response.blob()
+
+  const isGif = blob.type === 'image/gif'
+
+  logger.debug('compress: fetched blob', {
+    size: blob.size,
+    mimeType: blob.type,
+    isGif,
+    minSizeForCompression: MIN_SIZE_FOR_COMPRESSION,
+  })
+
+  // Try MediaBunny compression if WebCodecs is available and file is large enough
+  // Skip GIFs - MediaBunny doesn't support them
+  if (hasWebCodecs() && blob.size >= MIN_SIZE_FOR_COMPRESSION && !isGif) {
+    try {
+      return await doCompression(blob, asset.uri, {onProgress, signal})
+    } catch (e) {
+      logger.warn('compress: MediaBunny compression failed, using original', {
+        safeMessage: e,
+      })
+    }
+  } else {
+    logger.debug('compress: skipping compression', {
+      hasWebCodecs: hasWebCodecs(),
+      blobSize: blob.size,
+      minSize: MIN_SIZE_FOR_COMPRESSION,
+    })
+  }
+
+  // No compression path - just return the blob as-is
   if (blob.size > VIDEO_MAX_SIZE) {
     throw new VideoTooLargeError()
   }
 
   return {
+    uri: asset.uri,
     size: blob.size,
-    uri,
     bytes: await blob.arrayBuffer(),
+    mimeType: blob.type || 'video/mp4',
+  }
+}
+
+async function findEncodableVideoCodec(
+  width: number,
+  height: number,
+): Promise<{codec: VideoCodec; useWebM: boolean} | null> {
+  for (const codec of VIDEO_CODECS) {
+    const canEncode = await canEncodeVideo(codec, {
+      width,
+      height,
+      bitrate: TARGET_BITRATE,
+    })
+    logger.debug('compress: checking video codec', {
+      codec,
+      canEncode,
+      width,
+      height,
+    })
+    if (canEncode) {
+      // vp8/vp9 need WebM container, others use MP4
+      const useWebM = codec === 'vp8' || codec === 'vp9'
+      return {codec, useWebM}
+    }
+  }
+  return null
+}
+
+// Audio codecs to try - aac for MP4, opus for WebM
+const AUDIO_CODECS_MP4: AudioCodec[] = ['aac']
+const AUDIO_CODECS_WEBM: AudioCodec[] = ['opus', 'vorbis']
+
+async function findEncodableAudioCodec(
+  audioTrack: Awaited<ReturnType<Input['getPrimaryAudioTrack']>>,
+  useWebM: boolean,
+): Promise<{codec: AudioCodec} | null> {
+  if (!audioTrack) {
+    return null
+  }
+
+  // First check if we can decode the source audio
+  const canDecodeSource = await audioTrack.canDecode()
+  logger.debug('compress: checking audio source', {
+    sourceCodec: audioTrack.codec,
+    canDecode: canDecodeSource,
+    channels: audioTrack.numberOfChannels,
+    sampleRate: audioTrack.sampleRate,
+  })
+
+  if (!canDecodeSource) {
+    return null
+  }
+
+  const codecsToTry = useWebM ? AUDIO_CODECS_WEBM : AUDIO_CODECS_MP4
+
+  for (const codec of codecsToTry) {
+    const canEncode = await canEncodeAudio(codec, {
+      numberOfChannels: audioTrack.numberOfChannels,
+      sampleRate: audioTrack.sampleRate,
+    })
+    logger.debug('compress: checking audio encode codec', {
+      codec,
+      canEncode,
+    })
+    if (canEncode) {
+      return {codec}
+    }
+  }
+
+  return null
+}
+
+async function doCompression(
+  blob: Blob,
+  originalUri: string,
+  opts: {
+    onProgress?: (progress: number) => void
+    signal?: AbortSignal
+  },
+): Promise<CompressedVideo> {
+  const {onProgress, signal} = opts
+
+  const input = new Input({
+    source: new BlobSource(blob),
+    formats: ALL_FORMATS,
+  })
+
+  // Get video track to determine dimensions for codec check
+  const videoTrack = await input.getPrimaryVideoTrack()
+  if (!videoTrack) {
+    input.dispose()
+    throw new Error('No video track found')
+  }
+
+  // Get audio track to check if we can encode it
+  const audioTrack = await input.getPrimaryAudioTrack()
+
+  const {width, height} = calculateDimensions(
+    videoTrack.displayWidth,
+    videoTrack.displayHeight,
+    MAX_DIMENSION,
+  )
+
+  logger.debug('compress: video dimensions', {
+    original: {
+      width: videoTrack.displayWidth,
+      height: videoTrack.displayHeight,
+    },
+    target: {width, height},
+    audioCodec: audioTrack?.codec,
+  })
+
+  // Find a video codec we can encode with
+  const codecInfo = await findEncodableVideoCodec(width, height)
+  if (!codecInfo) {
+    input.dispose()
+    throw new Error('No supported video codec available')
+  }
+
+  // Check if we can encode the audio
+  const audioCodecInfo = await findEncodableAudioCodec(
+    audioTrack,
+    codecInfo.useWebM,
+  )
+
+  logger.debug('compress: using codecs', {
+    video: codecInfo.codec,
+    audio: audioCodecInfo?.codec ?? 'none',
+    useWebM: codecInfo.useWebM,
+  })
+
+  const target = new BufferTarget()
+  const output = new Output({
+    format: codecInfo.useWebM ? new WebMOutputFormat() : new Mp4OutputFormat(),
+    target,
+  })
+
+  // If we have audio but can't encode it, bail out and use the original
+  if (audioTrack && !audioCodecInfo) {
+    input.dispose()
+    throw new Error(
+      `Cannot encode audio codec: ${audioTrack.codec ?? 'unknown'}`,
+    )
+  }
+
+  const conversion = await Conversion.init({
+    input,
+    output,
+    video: {
+      codec: codecInfo.codec,
+      bitrate: TARGET_BITRATE,
+      width,
+      height,
+      fit: 'contain',
+    },
+    audio: audioCodecInfo ? {codec: audioCodecInfo.codec} : undefined,
+  })
+
+  if (onProgress) {
+    conversion.onProgress = onProgress
+  }
+
+  if (signal) {
+    signal.addEventListener('abort', () => {
+      logger.debug('compress: cancelled')
+      conversion.cancel()
+    })
+  }
+
+  logger.debug('compress: starting conversion')
+  const startTime = performance.now()
+
+  await conversion.execute()
+
+  const elapsed = performance.now() - startTime
+  const bytes = target.buffer
+
+  if (!bytes) {
+    throw new Error('MediaBunny compression produced no output')
+  }
+
+  const mimeType = codecInfo.useWebM ? 'video/webm' : 'video/mp4'
+
+  const savedBytes = blob.size - bytes.byteLength
+  const savedPercent = ((savedBytes / blob.size) * 100).toFixed(1)
+
+  logger.debug('compress: completed', {
+    from: blob.type,
+    to: mimeType,
+    originalSize: blob.size,
+    compressedSize: bytes.byteLength,
+    savedBytes,
+    savedPercent: `${savedPercent}%`,
+    elapsedMs: Math.round(elapsed),
+  })
+
+  if (bytes.byteLength > VIDEO_MAX_SIZE) {
+    throw new VideoTooLargeError()
+  }
+
+  return {
+    uri: originalUri,
+    size: bytes.byteLength,
+    bytes,
     mimeType,
   }
 }
 
-function parseDataUrl(dataUrl: string) {
+function calculateDimensions(
-  const [mimeType, base64] = dataUrl.slice('data:'.length).split(';base64,')
+  width: number,
-  if (!mimeType || !base64) {
+  height: number,
-    throw new Error('Invalid data URL')
+  maxDimension: number,
-  }
+): {width: number; height: number} {
-  return {mimeType, base64}
+  const maxSide = Math.max(width, height)
-}
+  if (maxSide <= maxDimension) {
-
+    return {width, height}
-function base64ToBlob(base64: string, mimeType: string) {
-  const byteCharacters = atob(base64)
-  const byteArrays = []
-
-  for (let offset = 0; offset < byteCharacters.length; offset += 512) {
-    const slice = byteCharacters.slice(offset, offset + 512)
-    const byteNumbers = new Array(slice.length)
-
-    for (let i = 0; i < slice.length; i++) {
-      byteNumbers[i] = slice.charCodeAt(i)
-    }
-
-    const byteArray = new Uint8Array(byteNumbers)
-    byteArrays.push(byteArray)
   }
 
-  return new Blob(byteArrays, {type: mimeType})
+  const scale = maxDimension / maxSide
+  return {
+    width: Math.round(width * scale),
+    height: Math.round(height * scale),
+  }
 }

src/view/com/composer/Composer.tsx

@@ -179,7 +179,7 @@ import {
   type VideoState,
 } from './state/video'
 import {type TextInputRef} from './text-input/TextInput.types'
-import {getVideoMetadata} from './videos/pickVideo'
+import {getVideoMetadata} from './videos/metadata'
 import {clearThumbnailCache} from './videos/VideoTranscodeBackdrop'
 
 type CancelRef = {

src/view/com/composer/SelectMediaButton.tsx

@@ -24,6 +24,7 @@ import {Image_Stroke2_Corner0_Rounded as ImageIcon} from '#/components/icons/Ima
 import * as toast from '#/components/Toast'
 import {IS_NATIVE, IS_WEB} from '#/env'
 import {isAnimatedGif} from './videos/isAnimatedGif'
+import {hasWebCodecs} from './videos/metadata'
 
 export type SelectMediaButtonProps = {
   disabled?: boolean
@@ -291,9 +292,15 @@ async function processImagePickerAssets(
       /*
        * Filesize appears to be stable across all platforms, so we can use it
        * to filter out large files on web. On native, we compress these anyway,
-       * so we only check on web.
+       * so we only check on web. On web, we can reject early if the browser
+       * doesn't support WebCodecs.
        */
-      if (IS_WEB && asset.fileSize && asset.fileSize > VIDEO_MAX_SIZE) {
+      if (
+        IS_WEB &&
+        !hasWebCodecs() &&
+        asset.fileSize &&
+        asset.fileSize > VIDEO_MAX_SIZE
+      ) {
         errors.add(SelectedAssetError.FileTooBig)
         continue
       }
@@ -309,8 +316,7 @@ async function processImagePickerAssets(
     if (type === 'gif') {
       /*
        * Filesize appears to be stable across all platforms, so we can use it
-       * to filter out large files on web. On native, we compress GIFs as
+       * to filter out large files. We can't compress GIFs on either platform.
-       * videos anyway, so we only check on web.
        */
       if (IS_WEB && asset.fileSize && asset.fileSize > VIDEO_MAX_SIZE) {
         errors.add(SelectedAssetError.FileTooBig)

src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx

@@ -1,7 +1,26 @@
+import {atoms as a, flatten} from '#/alf'
+
 export function clearThumbnailCache() {
-  // no-op
+  // no-op on web
 }
 
-export function VideoTranscodeBackdrop() {
+export function VideoTranscodeBackdrop({uri}: {uri: string}) {
-  return null
+  return (
+    <video
+      src={uri}
+      style={flatten([
+        a.absolute,
+        a.inset_0,
+        a.h_full,
+        a.w_full,
+        {
+          objectFit: 'cover',
+          filter: 'blur(15px)',
+          transform: 'scale(1.1)', // hide blur edges
+        },
+      ])}
+      muted
+      playsInline
+    />
+  )
 }

src/view/com/composer/videos/VideoTranscodeProgress.tsx

@@ -5,7 +5,6 @@ import {type ImagePickerAsset} from 'expo-image-picker'
 
 import {atoms as a, useTheme} from '#/alf'
 import {ConstrainedImage} from '#/components/images/AutoSizedImage'
-import {IS_WEB} from '#/env'
 import {ExternalEmbedRemoveBtn} from '../ExternalEmbedRemoveBtn'
 import {VideoTranscodeBackdrop} from './VideoTranscodeBackdrop'
 
@@ -20,8 +19,6 @@ export function VideoTranscodeProgress({
 }) {
   const t = useTheme()
 
-  if (IS_WEB) return null
-
   let aspectRatio: number | undefined
   if (asset.width && asset.height) {
     const raw = asset.width / asset.height

src/view/com/composer/videos/metadata.ts

@@ -0,0 +1,9 @@
+import {type ImagePickerAsset} from 'expo-image-picker'
+
+export const getVideoMetadata = (_file: File): Promise<ImagePickerAsset> => {
+  throw new Error('getVideoMetadata is web only')
+}
+
+export function hasWebCodecs(): boolean {
+  throw new Error("hasWebCodecs is web only (also, no it doesn't)")
+}

src/view/com/composer/videos/metadata.web.ts

@@ -0,0 +1,120 @@
+import {type ImagePickerAsset} from 'expo-image-picker'
+import {ALL_FORMATS, BlobSource, Input} from 'mediabunny'
+
+import {logger} from '#/logger'
+
+export function hasWebCodecs(): boolean {
+  return (
+    typeof VideoEncoder !== 'undefined' && typeof VideoDecoder !== 'undefined'
+  )
+}
+
+export async function getVideoMetadata(file: File): Promise<ImagePickerAsset> {
+  const blobUrl = URL.createObjectURL(file)
+
+  logger.debug('metadata: starting', {
+    fileName: file.name,
+    fileSize: file.size,
+    mimeType: file.type,
+    hasWebCodecs: hasWebCodecs(),
+  })
+
+  if (hasWebCodecs()) {
+    try {
+      const result = await getMetadataWithWebCodecs(file, blobUrl)
+      logger.debug('metadata: MediaBunny succeeded', {
+        width: result.width,
+        height: result.height,
+        duration: result.duration,
+      })
+      return result
+    } catch (e) {
+      logger.warn('metadata: MediaBunny failed, using fallback', {
+        safeMessage: e,
+      })
+    }
+  }
+
+  // Fallback to old-fashioned browser APIs
+  const result = await getMetadataWithBrowserAPIs(file, blobUrl)
+  logger.debug('metadata: browser API succeeded', {
+    width: result.width,
+    height: result.height,
+    duration: result.duration,
+  })
+  return result
+}
+
+async function getMetadataWithWebCodecs(
+  file: File,
+  blobUrl: string,
+): Promise<ImagePickerAsset> {
+  const input = new Input({
+    source: new BlobSource(file),
+    formats: ALL_FORMATS,
+  })
+
+  try {
+    const [videoTrack, duration] = await Promise.all([
+      input.getPrimaryVideoTrack(),
+      input.computeDuration(),
+    ])
+
+    if (!videoTrack) {
+      throw new Error('No video track found')
+    }
+
+    return {
+      uri: blobUrl,
+      mimeType: file.type,
+      width: videoTrack.displayWidth,
+      height: videoTrack.displayHeight,
+      duration: duration * 1000, // convert seconds to ms
+    }
+  } finally {
+    input.dispose()
+  }
+}
+
+async function getMetadataWithBrowserAPIs(
+  file: File,
+  blobUrl: string,
+): Promise<ImagePickerAsset> {
+  return new Promise((resolve, reject) => {
+    if (file.type === 'image/gif') {
+      const img = new Image()
+      img.onload = () => {
+        resolve({
+          uri: blobUrl,
+          mimeType: 'image/gif',
+          width: img.width,
+          height: img.height,
+          duration: null,
+        })
+      }
+      img.onerror = () => {
+        URL.revokeObjectURL(blobUrl)
+        reject(new Error('Failed to load GIF'))
+      }
+      img.src = blobUrl
+    } else {
+      const video = document.createElement('video')
+      video.preload = 'metadata'
+      video.src = blobUrl
+
+      video.onloadedmetadata = () => {
+        resolve({
+          uri: blobUrl,
+          mimeType: file.type,
+          width: video.videoWidth,
+          height: video.videoHeight,
+          duration: video.duration * 1000,
+        })
+      }
+      video.onerror = () => {
+        URL.revokeObjectURL(blobUrl)
+        reject(new Error('Failed to load video metadata'))
+      }
+    }
+  })
+}

src/view/com/composer/videos/pickVideo.ts

@@ -1,43 +0,0 @@
-import {getVideoMetaData} from 'react-native-compressor'
-import {
-  type ImagePickerAsset,
-  launchImageLibraryAsync,
-  UIImagePickerPreferredAssetRepresentationMode,
-} from 'expo-image-picker'
-
-import {VIDEO_MAX_DURATION_MS} from '#/lib/constants'
-import {extToMime} from '#/lib/media/video/util'
-
-export async function pickVideo() {
-  return await launchImageLibraryAsync({
-    exif: false,
-    mediaTypes: ['videos'],
-    quality: 1,
-    legacy: true,
-    preferredAssetRepresentationMode:
-      UIImagePickerPreferredAssetRepresentationMode.Current,
-    videoMaxDuration: VIDEO_MAX_DURATION_MS / 1000,
-  })
-}
-
-/**
- * Gets video metadata from a file or uri, depending on the platform
- *
- * @param file File on web, uri on native
- */
-export async function getVideoMetadata(
-  file: File | string,
-): Promise<ImagePickerAsset> {
-  if (typeof file !== 'string')
-    throw new Error(
-      'getVideoMetadata was passed a File, when on native it should be a uri',
-    )
-  const metadata = await getVideoMetaData(file)
-  return {
-    uri: file,
-    mimeType: extToMime(metadata.extension),
-    width: metadata.width,
-    height: metadata.height,
-    duration: metadata.duration,
-  }
-}

src/view/com/composer/videos/pickVideo.web.ts

@@ -1,100 +0,0 @@
-import {type ImagePickerAsset, type ImagePickerResult} from 'expo-image-picker'
-
-import {SUPPORTED_MIME_TYPES} from '#/lib/constants'
-
-// mostly copied from expo-image-picker and adapted to support gifs
-// also adds support for reading video metadata
-
-export async function pickVideo(): Promise<ImagePickerResult> {
-  const input = document.createElement('input')
-  input.style.display = 'none'
-  input.setAttribute('type', 'file')
-  // TODO: do we need video/* here? -sfn
-  input.setAttribute('accept', SUPPORTED_MIME_TYPES.join(','))
-  input.setAttribute('id', String(Math.random()))
-
-  document.body.appendChild(input)
-
-  return new Promise(resolve => {
-    input.addEventListener('change', async () => {
-      if (input.files) {
-        const file = input.files[0]
-        resolve({
-          canceled: false,
-          assets: [await getVideoMetadata(file)],
-        })
-      } else {
-        resolve({canceled: true, assets: null})
-      }
-      document.body.removeChild(input)
-    })
-
-    const event = new MouseEvent('click')
-    input.dispatchEvent(event)
-  })
-}
-
-// TODO: we're converting to a dataUrl here, and then converting back to an
-// ArrayBuffer in the compressVideo function. This is a bit wasteful, but it
-// lets us use the ImagePickerAsset type, which the rest of the code expects.
-// We should unwind this and just pass the ArrayBuffer/objectUrl through the system
-// instead of a string -sfn
-export function getVideoMetadata(
-  file: File | string,
-): Promise<ImagePickerAsset> {
-  if (typeof file === 'string')
-    throw new Error(
-      'getVideoMetadata was passed a uri, when on web it should be a File',
-    )
-  return new Promise((resolve, reject) => {
-    const reader = new FileReader()
-    reader.onload = () => {
-      const uri = reader.result as string
-
-      if (file.type === 'image/gif') {
-        const img = new Image()
-        img.onload = () => {
-          resolve({
-            uri,
-            mimeType: 'image/gif',
-            width: img.width,
-            height: img.height,
-            // todo: calculate gif duration. seems possible if you read the bytes
-            // https://codepen.io/Ryman/pen/nZpYwY
-            // for now let's just let the server reject it, since that seems uncommon -sfn
-            duration: null,
-          })
-        }
-        img.onerror = (_ev, _source, _lineno, _colno, error) => {
-          console.log('Failed to grab GIF metadata', error)
-          reject(new Error('Failed to grab GIF metadata'))
-        }
-        img.src = uri
-      } else {
-        const video = document.createElement('video')
-        const blobUrl = URL.createObjectURL(file)
-
-        video.preload = 'metadata'
-        video.src = blobUrl
-
-        video.onloadedmetadata = () => {
-          URL.revokeObjectURL(blobUrl)
-          resolve({
-            uri,
-            mimeType: file.type,
-            width: video.videoWidth,
-            height: video.videoHeight,
-            // convert seconds to ms
-            duration: video.duration * 1000,
-          })
-        }
-        video.onerror = (_ev, _source, _lineno, _colno, error) => {
-          URL.revokeObjectURL(blobUrl)
-          console.log('Failed to grab video metadata', error)
-          reject(new Error('Failed to grab video metadata'))
-        }
-      }
-    }
-    reader.readAsDataURL(file)
-  })
-}