Process videos on web with WebCodecs (#10955)

Co-authored-by: Samuel Newman <mozzius@protonmail.com>

src/view/com/composer/Composer.tsx

@@ -179,7 +179,7 @@ import {
   type VideoState,
 } from './state/video'
 import {type TextInputRef} from './text-input/TextInput.types'
-import {getVideoMetadata} from './videos/pickVideo'
+import {getVideoMetadata} from './videos/metadata'
 import {clearThumbnailCache} from './videos/VideoTranscodeBackdrop'
 
 type CancelRef = {

src/view/com/composer/SelectMediaButton.tsx

@@ -24,6 +24,7 @@ import {Image_Stroke2_Corner0_Rounded as ImageIcon} from '#/components/icons/Ima
 import * as toast from '#/components/Toast'
 import {IS_NATIVE, IS_WEB} from '#/env'
 import {isAnimatedGif} from './videos/isAnimatedGif'
+import {hasWebCodecs} from './videos/metadata'
 
 export type SelectMediaButtonProps = {
   disabled?: boolean
@@ -291,9 +292,15 @@ async function processImagePickerAssets(
       /*
        * Filesize appears to be stable across all platforms, so we can use it
        * to filter out large files on web. On native, we compress these anyway,
-       * so we only check on web.
+       * so we only check on web. On web, we can reject early if the browser
+       * doesn't support WebCodecs.
        */
-      if (IS_WEB && asset.fileSize && asset.fileSize > VIDEO_MAX_SIZE) {
+      if (
+        IS_WEB &&
+        !hasWebCodecs() &&
+        asset.fileSize &&
+        asset.fileSize > VIDEO_MAX_SIZE
+      ) {
         errors.add(SelectedAssetError.FileTooBig)
         continue
       }
@@ -309,8 +316,7 @@ async function processImagePickerAssets(
     if (type === 'gif') {
       /*
        * Filesize appears to be stable across all platforms, so we can use it
-       * to filter out large files on web. On native, we compress GIFs as
-       * videos anyway, so we only check on web.
+       * to filter out large files. We can't compress GIFs on either platform.
        */
       if (IS_WEB && asset.fileSize && asset.fileSize > VIDEO_MAX_SIZE) {
         errors.add(SelectedAssetError.FileTooBig)

src/view/com/composer/videos/VideoTranscodeBackdrop.web.tsx

@@ -1,7 +1,26 @@
+import {atoms as a, flatten} from '#/alf'
+
 export function clearThumbnailCache() {
-  // no-op
+  // no-op on web
 }
 
-export function VideoTranscodeBackdrop() {
-  return null
+export function VideoTranscodeBackdrop({uri}: {uri: string}) {
+  return (
+    <video
+      src={uri}
+      style={flatten([
+        a.absolute,
+        a.inset_0,
+        a.h_full,
+        a.w_full,
+        {
+          objectFit: 'cover',
+          filter: 'blur(15px)',
+          transform: 'scale(1.1)', // hide blur edges
+        },
+      ])}
+      muted
+      playsInline
+    />
+  )
 }

src/view/com/composer/videos/VideoTranscodeProgress.tsx

@@ -5,7 +5,6 @@ import {type ImagePickerAsset} from 'expo-image-picker'
 
 import {atoms as a, useTheme} from '#/alf'
 import {ConstrainedImage} from '#/components/images/AutoSizedImage'
-import {IS_WEB} from '#/env'
 import {ExternalEmbedRemoveBtn} from '../ExternalEmbedRemoveBtn'
 import {VideoTranscodeBackdrop} from './VideoTranscodeBackdrop'
 
@@ -20,8 +19,6 @@ export function VideoTranscodeProgress({
 }) {
   const t = useTheme()
 
-  if (IS_WEB) return null
-
   let aspectRatio: number | undefined
   if (asset.width && asset.height) {
     const raw = asset.width / asset.height

src/view/com/composer/videos/metadata.ts

@@ -0,0 +1,25 @@
+import {getVideoMetaData} from 'react-native-compressor'
+import {type ImagePickerAsset} from 'expo-image-picker'
+
+import {extToMime} from '#/lib/media/video/util'
+
+export async function getVideoMetadata(
+  file: File | string,
+): Promise<ImagePickerAsset> {
+  if (typeof file !== 'string')
+    throw new Error(
+      'getVideoMetadata was passed a File, when on native it should be a uri',
+    )
+  const metadata = await getVideoMetaData(file)
+  return {
+    uri: file,
+    mimeType: extToMime(metadata.extension),
+    width: metadata.width,
+    height: metadata.height,
+    duration: metadata.duration,
+  }
+}
+
+export function hasWebCodecs(): boolean {
+  return false
+}

src/view/com/composer/videos/metadata.web.ts

@@ -0,0 +1,126 @@
+import {type ImagePickerAsset} from 'expo-image-picker'
+import {ALL_FORMATS, BlobSource, Input} from 'mediabunny'
+
+import {logger} from '#/logger'
+
+export function hasWebCodecs(): boolean {
+  return (
+    typeof VideoEncoder !== 'undefined' && typeof VideoDecoder !== 'undefined'
+  )
+}
+
+export async function getVideoMetadata(
+  file: File | string,
+): Promise<ImagePickerAsset> {
+  if (typeof file === 'string')
+    throw new Error(
+      'getVideoMetadata was passed a uri, when on web it should be a File',
+    )
+  const blobUrl = URL.createObjectURL(file)
+
+  logger.debug('metadata: starting', {
+    fileName: file.name,
+    fileSize: file.size,
+    mimeType: file.type,
+    hasWebCodecs: hasWebCodecs(),
+  })
+
+  if (hasWebCodecs()) {
+    try {
+      const result = await getMetadataWithWebCodecs(file, blobUrl)
+      logger.debug('metadata: WebCodecs succeeded', {
+        width: result.width,
+        height: result.height,
+        duration: result.duration,
+      })
+      return result
+    } catch (e) {
+      logger.warn('metadata: WebCodecs failed, using fallback', {
+        safeMessage: e,
+      })
+    }
+  }
+
+  // Fallback to old-fashioned browser APIs
+  const result = await getMetadataWithBrowserAPIs(file, blobUrl)
+  logger.debug('metadata: browser API succeeded', {
+    width: result.width,
+    height: result.height,
+    duration: result.duration,
+  })
+  return result
+}
+
+async function getMetadataWithWebCodecs(
+  file: File,
+  blobUrl: string,
+): Promise<ImagePickerAsset> {
+  const input = new Input({
+    source: new BlobSource(file),
+    formats: ALL_FORMATS,
+  })
+
+  try {
+    const [videoTrack, duration] = await Promise.all([
+      input.getPrimaryVideoTrack(),
+      input.computeDuration(),
+    ])
+
+    if (!videoTrack) {
+      throw new Error('No video track found')
+    }
+
+    return {
+      uri: blobUrl,
+      mimeType: file.type,
+      width: videoTrack.displayWidth,
+      height: videoTrack.displayHeight,
+      duration: duration * 1000, // convert seconds to ms
+    }
+  } finally {
+    input.dispose()
+  }
+}
+
+async function getMetadataWithBrowserAPIs(
+  file: File,
+  blobUrl: string,
+): Promise<ImagePickerAsset> {
+  return new Promise((resolve, reject) => {
+    if (file.type === 'image/gif') {
+      const img = new Image()
+      img.onload = () => {
+        resolve({
+          uri: blobUrl,
+          mimeType: 'image/gif',
+          width: img.width,
+          height: img.height,
+          duration: null,
+        })
+      }
+      img.onerror = () => {
+        URL.revokeObjectURL(blobUrl)
+        reject(new Error('Failed to load GIF'))
+      }
+      img.src = blobUrl
+    } else {
+      const video = document.createElement('video')
+      video.preload = 'metadata'
+      video.src = blobUrl
+
+      video.onloadedmetadata = () => {
+        resolve({
+          uri: blobUrl,
+          mimeType: file.type,
+          width: video.videoWidth,
+          height: video.videoHeight,
+          duration: video.duration * 1000,
+        })
+      }
+      video.onerror = () => {
+        URL.revokeObjectURL(blobUrl)
+        reject(new Error('Failed to load video metadata'))
+      }
+    }
+  })
+}

src/view/com/composer/videos/pickVideo.ts

@@ -1,43 +0,0 @@
-import {getVideoMetaData} from 'react-native-compressor'
-import {
-  type ImagePickerAsset,
-  launchImageLibraryAsync,
-  UIImagePickerPreferredAssetRepresentationMode,
-} from 'expo-image-picker'
-
-import {VIDEO_MAX_DURATION_MS} from '#/lib/constants'
-import {extToMime} from '#/lib/media/video/util'
-
-export async function pickVideo() {
-  return await launchImageLibraryAsync({
-    exif: false,
-    mediaTypes: ['videos'],
-    quality: 1,
-    legacy: true,
-    preferredAssetRepresentationMode:
-      UIImagePickerPreferredAssetRepresentationMode.Current,
-    videoMaxDuration: VIDEO_MAX_DURATION_MS / 1000,
-  })
-}
-
-/**
- * Gets video metadata from a file or uri, depending on the platform
- *
- * @param file File on web, uri on native
- */
-export async function getVideoMetadata(
-  file: File | string,
-): Promise<ImagePickerAsset> {
-  if (typeof file !== 'string')
-    throw new Error(
-      'getVideoMetadata was passed a File, when on native it should be a uri',
-    )
-  const metadata = await getVideoMetaData(file)
-  return {
-    uri: file,
-    mimeType: extToMime(metadata.extension),
-    width: metadata.width,
-    height: metadata.height,
-    duration: metadata.duration,
-  }
-}

src/view/com/composer/videos/pickVideo.web.ts

@@ -1,100 +0,0 @@
-import {type ImagePickerAsset, type ImagePickerResult} from 'expo-image-picker'
-
-import {SUPPORTED_MIME_TYPES} from '#/lib/constants'
-
-// mostly copied from expo-image-picker and adapted to support gifs
-// also adds support for reading video metadata
-
-export async function pickVideo(): Promise<ImagePickerResult> {
-  const input = document.createElement('input')
-  input.style.display = 'none'
-  input.setAttribute('type', 'file')
-  // TODO: do we need video/* here? -sfn
-  input.setAttribute('accept', SUPPORTED_MIME_TYPES.join(','))
-  input.setAttribute('id', String(Math.random()))
-
-  document.body.appendChild(input)
-
-  return new Promise(resolve => {
-    input.addEventListener('change', async () => {
-      if (input.files) {
-        const file = input.files[0]
-        resolve({
-          canceled: false,
-          assets: [await getVideoMetadata(file)],
-        })
-      } else {
-        resolve({canceled: true, assets: null})
-      }
-      document.body.removeChild(input)
-    })
-
-    const event = new MouseEvent('click')
-    input.dispatchEvent(event)
-  })
-}
-
-// TODO: we're converting to a dataUrl here, and then converting back to an
-// ArrayBuffer in the compressVideo function. This is a bit wasteful, but it
-// lets us use the ImagePickerAsset type, which the rest of the code expects.
-// We should unwind this and just pass the ArrayBuffer/objectUrl through the system
-// instead of a string -sfn
-export function getVideoMetadata(
-  file: File | string,
-): Promise<ImagePickerAsset> {
-  if (typeof file === 'string')
-    throw new Error(
-      'getVideoMetadata was passed a uri, when on web it should be a File',
-    )
-  return new Promise((resolve, reject) => {
-    const reader = new FileReader()
-    reader.onload = () => {
-      const uri = reader.result as string
-
-      if (file.type === 'image/gif') {
-        const img = new Image()
-        img.onload = () => {
-          resolve({
-            uri,
-            mimeType: 'image/gif',
-            width: img.width,
-            height: img.height,
-            // todo: calculate gif duration. seems possible if you read the bytes
-            // https://codepen.io/Ryman/pen/nZpYwY
-            // for now let's just let the server reject it, since that seems uncommon -sfn
-            duration: null,
-          })
-        }
-        img.onerror = (_ev, _source, _lineno, _colno, error) => {
-          console.log('Failed to grab GIF metadata', error)
-          reject(new Error('Failed to grab GIF metadata'))
-        }
-        img.src = uri
-      } else {
-        const video = document.createElement('video')
-        const blobUrl = URL.createObjectURL(file)
-
-        video.preload = 'metadata'
-        video.src = blobUrl
-
-        video.onloadedmetadata = () => {
-          URL.revokeObjectURL(blobUrl)
-          resolve({
-            uri,
-            mimeType: file.type,
-            width: video.videoWidth,
-            height: video.videoHeight,
-            // convert seconds to ms
-            duration: video.duration * 1000,
-          })
-        }
-        video.onerror = (_ev, _source, _lineno, _colno, error) => {
-          URL.revokeObjectURL(blobUrl)
-          console.log('Failed to grab video metadata', error)
-          reject(new Error('Failed to grab video metadata'))
-        }
-      }
-    }
-    reader.readAsDataURL(file)
-  })
-}