From 95726c6c6b16dbc78a387b42430e5516ee7dc2e8 Mon Sep 17 00:00:00 2001 From: Spence Pope Date: Tue, 30 Jun 2026 11:59:59 -0400 Subject: [PATCH] Replace react-native-compressor video path with native expo-bluesky-video-compress module [APP-2428] (#10954) Co-authored-by: Eric Bailey --- .../android/build.gradle | 34 + .../blueskyvideocompress/CodecSelector.kt | 55 ++ .../ExpoBlueskyVideoCompressModule.kt | 62 ++ .../blueskyvideocompress/InputSurface.kt | 96 +++ .../blueskyvideocompress/OutputSurface.kt | 61 ++ .../blueskyvideocompress/TextureRenderer.kt | 156 +++++ .../blueskyvideocompress/VideoCompressor.kt | 582 ++++++++++++++++++ .../blueskyvideocompress/VideoProber.kt | 125 ++++ .../expo-module.config.json | 9 + modules/expo-bluesky-video-compress/index.ts | 87 +++ .../ios/CodecCapability.swift | 21 + .../ios/ExpoBlueskyVideoCompress.podspec | 20 + .../ios/ExpoBlueskyVideoCompressModule.swift | 74 +++ .../ios/VideoCompressor.swift | 410 ++++++++++++ .../ios/VideoProber.swift | 105 ++++ .../src/ExpoBlueskyVideoCompressModule.ts | 23 + .../expo-bluesky-video-compress/src/types.ts | 45 ++ src/analytics/metrics/types.ts | 19 + src/lib/media/video/compress.ts | 17 +- src/lib/media/video/compress.web.ts | 55 +- src/lib/media/video/constants.ts | 6 +- src/lib/media/video/telemetry.ts | 24 +- src/lib/media/video/types.ts | 17 + src/view/com/composer/state/video.ts | 1 + 24 files changed, 2097 insertions(+), 7 deletions(-) create mode 100644 modules/expo-bluesky-video-compress/android/build.gradle create mode 100644 modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/CodecSelector.kt create mode 100644 modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/ExpoBlueskyVideoCompressModule.kt create mode 100644 modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt create mode 100644 modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt create mode 100644 modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt create mode 100644 modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt create mode 100644 modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt create mode 100644 modules/expo-bluesky-video-compress/expo-module.config.json create mode 100644 modules/expo-bluesky-video-compress/index.ts create mode 100644 modules/expo-bluesky-video-compress/ios/CodecCapability.swift create mode 100644 modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompress.podspec create mode 100644 modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompressModule.swift create mode 100644 modules/expo-bluesky-video-compress/ios/VideoCompressor.swift create mode 100644 modules/expo-bluesky-video-compress/ios/VideoProber.swift create mode 100644 modules/expo-bluesky-video-compress/src/ExpoBlueskyVideoCompressModule.ts create mode 100644 modules/expo-bluesky-video-compress/src/types.ts diff --git a/modules/expo-bluesky-video-compress/android/build.gradle b/modules/expo-bluesky-video-compress/android/build.gradle new file mode 100644 index 0000000000..be96c40154 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/build.gradle @@ -0,0 +1,34 @@ +apply plugin: 'com.android.library' + +group = 'expo.modules.blueskyvideocompress' +version = '1.0.0' + +def expoModulesCorePlugin = new File(project(":expo-modules-core").projectDir.absolutePath, "ExpoModulesCorePlugin.gradle") +apply from: expoModulesCorePlugin +applyKotlinExpoModulesCorePlugin() +useCoreDependencies() +useExpoPublishing() + +buildscript { + ext.safeExtGet = { prop, fallback -> + rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback + } +} + +android { + compileSdkVersion safeExtGet("compileSdkVersion", 34) + + namespace "expo.modules.blueskyvideocompress" + defaultConfig { + minSdkVersion safeExtGet("minSdkVersion", 21) + targetSdkVersion safeExtGet("targetSdkVersion", 34) + versionCode 1 + versionName "1.0.0" + } + lintOptions { + abortOnError false + } +} + +dependencies { +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/CodecSelector.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/CodecSelector.kt new file mode 100644 index 0000000000..f35698486a --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/CodecSelector.kt @@ -0,0 +1,55 @@ +package expo.modules.blueskyvideocompress + +import android.media.MediaCodecInfo +import android.media.MediaCodecList +import android.media.MediaFormat +import android.os.Build + +object CodecSelector { + // Source: https://github.com/numandev1/react-native-compressor/blob/f949b0868055178e7c8753e05202f784b1bcd589/android/src/main/java/com/reactnativecompressor/Video/VideoCompressor/compressor/Compressor.kt#L500 + private val AVC_DENYLIST = setOf( + "c2.qti.avc.encoder" + ) + + private val SOFTWARE_PREFIXES = listOf( + "OMX.google.", + "c2.android.", + "c2.google." + ) + + data class EncoderInfo( + val name: String, + val mime: String, + val isHardware: Boolean + ) + + fun findEncoder(mime: String, requireHardware: Boolean): EncoderInfo? { + val codecList = MediaCodecList(MediaCodecList.REGULAR_CODECS) + val candidates = codecList.codecInfos + .filter { it.isEncoder } + .filter { it.supportedTypes.any { t -> t.equals(mime, ignoreCase = true) } } + .filter { !it.name.contains("secure", ignoreCase = true) } + .filter { !(mime == MediaFormat.MIMETYPE_VIDEO_AVC && AVC_DENYLIST.contains(it.name)) } + + val hardware = candidates.filter { isHardware(it) } + val selected = if (requireHardware) { + hardware.firstOrNull() + } else { + hardware.firstOrNull() ?: candidates.firstOrNull() + } + selected ?: return null + return EncoderInfo( + name = selected.name, + mime = mime, + isHardware = isHardware(selected) + ) + } + + private fun isHardware(info: MediaCodecInfo): Boolean { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + info.isHardwareAccelerated + } else { + SOFTWARE_PREFIXES.none { info.name.startsWith(it, ignoreCase = true) } + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/ExpoBlueskyVideoCompressModule.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/ExpoBlueskyVideoCompressModule.kt new file mode 100644 index 0000000000..30b1def7c6 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/ExpoBlueskyVideoCompressModule.kt @@ -0,0 +1,62 @@ +package expo.modules.blueskyvideocompress + +import expo.modules.kotlin.modules.Module +import expo.modules.kotlin.modules.ModuleDefinition +import java.util.concurrent.ConcurrentHashMap + +class ExpoBlueskyVideoCompressModule : Module() { + private val activeCompressors = ConcurrentHashMap() + + override fun definition() = ModuleDefinition { + Name("ExpoBlueskyVideoCompress") + + Events("onProgress") + + AsyncFunction("probe") { uri: String -> + val context = appContext.reactContext + ?: throw Error("React context is null") + return@AsyncFunction VideoProber.probe(context, uri) + } + + AsyncFunction("compress") { uri: String, options: Map -> + val context = appContext.reactContext + ?: throw Error("React context is null") + val targetBitrate = (options["targetBitrate"] as? Number)?.toInt() ?: 0 + val maxSize = (options["maxSize"] as? Number)?.toInt() ?: 1920 + val codecPref = (options["codec"] as? String) ?: "auto" + val frameRateCap = ((options["frameRateCap"] as? Number)?.toInt() ?: 30).coerceAtLeast(1) + val jobId = (options["jobId"] as? Number)?.toInt() ?: 0 + + val compressor = VideoCompressor( + context = context, + uriString = uri, + targetBitrate = targetBitrate, + maxSize = maxSize, + codecPref = codecPref, + frameRateCap = frameRateCap, + jobId = jobId, + onProgress = { id, progress -> + sendEvent("onProgress", mapOf( + "id" to id, + "progress" to progress + )) + } + ) + + activeCompressors[jobId] = compressor + + try { + val result = compressor.compress() + activeCompressors.remove(jobId) + return@AsyncFunction result + } catch (e: Exception) { + activeCompressors.remove(jobId) + throw e + } + } + + Function("cancel") { jobId: Int -> + activeCompressors.remove(jobId)?.cancel() + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt new file mode 100644 index 0000000000..e2eee4768c --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt @@ -0,0 +1,96 @@ +package expo.modules.blueskyvideocompress + +import android.opengl.EGL14 +import android.opengl.EGLConfig +import android.opengl.EGLContext +import android.opengl.EGLDisplay +import android.opengl.EGLExt +import android.opengl.EGLSurface +import android.view.Surface + +class InputSurface(private val surface: Surface) { + private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY + private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT + private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE + + init { + eglSetup() + } + + private fun eglSetup() { + eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY) + if (eglDisplay === EGL14.EGL_NO_DISPLAY) { + throw RuntimeException("unable to get EGL14 display") + } + + val version = IntArray(2) + if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) { + throw RuntimeException("unable to initialize EGL14") + } + + val attribList = intArrayOf( + EGL14.EGL_RED_SIZE, 8, + EGL14.EGL_GREEN_SIZE, 8, + EGL14.EGL_BLUE_SIZE, 8, + EGL14.EGL_ALPHA_SIZE, 8, + EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, + EGL14.EGL_SURFACE_TYPE, EGL14.EGL_WINDOW_BIT, + EGL14.EGL_NONE + ) + val configs = arrayOfNulls(1) + val numConfigs = IntArray(1) + EGL14.eglChooseConfig(eglDisplay, attribList, 0, configs, 0, 1, numConfigs, 0) + checkEglError("eglChooseConfig") + + val contextAttribs = intArrayOf( + EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, + EGL14.EGL_NONE + ) + eglContext = EGL14.eglCreateContext( + eglDisplay, configs[0], EGL14.EGL_NO_CONTEXT, contextAttribs, 0 + ) + checkEglError("eglCreateContext") + + val surfaceAttribs = intArrayOf(EGL14.EGL_NONE) + eglSurface = EGL14.eglCreateWindowSurface( + eglDisplay, configs[0], surface, surfaceAttribs, 0 + ) + checkEglError("eglCreateWindowSurface") + } + + fun makeCurrent() { + EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext) + checkEglError("eglMakeCurrent") + } + + fun swapBuffers(): Boolean { + return EGL14.eglSwapBuffers(eglDisplay, eglSurface) + } + + fun setPresentationTime(nsecs: Long) { + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, nsecs) + } + + fun release() { + if (eglDisplay !== EGL14.EGL_NO_DISPLAY) { + EGL14.eglMakeCurrent( + eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT + ) + EGL14.eglDestroySurface(eglDisplay, eglSurface) + EGL14.eglDestroyContext(eglDisplay, eglContext) + EGL14.eglReleaseThread() + EGL14.eglTerminate(eglDisplay) + } + surface.release() + eglDisplay = EGL14.EGL_NO_DISPLAY + eglContext = EGL14.EGL_NO_CONTEXT + eglSurface = EGL14.EGL_NO_SURFACE + } + + private fun checkEglError(msg: String) { + val error = EGL14.eglGetError() + if (error != EGL14.EGL_SUCCESS) { + throw RuntimeException("$msg: EGL error: 0x${Integer.toHexString(error)}") + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt new file mode 100644 index 0000000000..fdf8c6eca2 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt @@ -0,0 +1,61 @@ +package expo.modules.blueskyvideocompress + +import android.graphics.SurfaceTexture +import android.os.Handler +import android.os.HandlerThread +import android.view.Surface + +class OutputSurface : SurfaceTexture.OnFrameAvailableListener { + private val renderer = TextureRenderer() + private var surfaceTexture: SurfaceTexture? = null + private val stMatrix = FloatArray(16) + private val callbackThread = HandlerThread("OutputSurfaceCallbacks") + val surface: Surface + + @Volatile + private var frameAvailable = false + private val frameSyncObject = Object() + + init { + renderer.surfaceCreated() + callbackThread.start() + val handler = Handler(callbackThread.looper) + surfaceTexture = SurfaceTexture(renderer.getTextureId()).also { + it.setOnFrameAvailableListener(this, handler) + } + surface = Surface(surfaceTexture) + } + + fun release() { + surface.release() + surfaceTexture?.release() + surfaceTexture = null + callbackThread.quitSafely() + } + + fun awaitNewImage() { + val timeoutMs = 2500L + synchronized(frameSyncObject) { + while (!frameAvailable) { + frameSyncObject.wait(timeoutMs) + if (!frameAvailable) { + throw RuntimeException("Surface frame wait timed out") + } + } + frameAvailable = false + } + surfaceTexture!!.updateTexImage() + } + + fun drawImage() { + surfaceTexture!!.getTransformMatrix(stMatrix) + renderer.drawFrame(stMatrix) + } + + override fun onFrameAvailable(st: SurfaceTexture) { + synchronized(frameSyncObject) { + frameAvailable = true + frameSyncObject.notifyAll() + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt new file mode 100644 index 0000000000..a0cb25bce1 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt @@ -0,0 +1,156 @@ +package expo.modules.blueskyvideocompress + +import android.opengl.GLES11Ext +import android.opengl.GLES20 +import android.opengl.Matrix +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.FloatBuffer + +class TextureRenderer { + companion object { + private const val FLOAT_SIZE_BYTES = 4 + private const val STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES + private const val POS_OFFSET = 0 + private const val UV_OFFSET = 3 + + private val VERTICES = floatArrayOf( + -1.0f, -1.0f, 0f, 0f, 0f, + 1.0f, -1.0f, 0f, 1f, 0f, + -1.0f, 1.0f, 0f, 0f, 1f, + 1.0f, 1.0f, 0f, 1f, 1f, + ) + + private const val VERTEX_SHADER = """ + uniform mat4 uMVPMatrix; + uniform mat4 uSTMatrix; + attribute vec4 aPosition; + attribute vec4 aTextureCoord; + varying vec2 vTextureCoord; + void main() { + gl_Position = uMVPMatrix * aPosition; + vTextureCoord = (uSTMatrix * aTextureCoord).xy; + } + """ + + private const val FRAGMENT_SHADER = """ + #extension GL_OES_EGL_image_external : require + precision mediump float; + varying vec2 vTextureCoord; + uniform samplerExternalOES sTexture; + void main() { + gl_FragColor = texture2D(sTexture, vTextureCoord); + } + """ + } + + private val vertices: FloatBuffer = + ByteBuffer.allocateDirect(VERTICES.size * FLOAT_SIZE_BYTES) + .order(ByteOrder.nativeOrder()) + .asFloatBuffer() + .apply { put(VERTICES); position(0) } + + private val mvpMatrix = FloatArray(16) + private var program = 0 + private var textureId = -1 + private var uMVPMatrixHandle = 0 + private var uSTMatrixHandle = 0 + private var aPositionHandle = 0 + private var aTextureCoordHandle = 0 + + init { + Matrix.setIdentityM(mvpMatrix, 0) + } + + fun getTextureId(): Int = textureId + + fun surfaceCreated() { + program = createProgram(VERTEX_SHADER, FRAGMENT_SHADER) + + aPositionHandle = GLES20.glGetAttribLocation(program, "aPosition") + aTextureCoordHandle = GLES20.glGetAttribLocation(program, "aTextureCoord") + uMVPMatrixHandle = GLES20.glGetUniformLocation(program, "uMVPMatrix") + uSTMatrixHandle = GLES20.glGetUniformLocation(program, "uSTMatrix") + + val textures = IntArray(1) + GLES20.glGenTextures(1, textures, 0) + textureId = textures[0] + + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId) + GLES20.glTexParameterf( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR.toFloat() + ) + GLES20.glTexParameterf( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat() + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE + ) + } + + fun drawFrame(stMatrix: FloatArray) { + GLES20.glClearColor(0f, 0f, 0f, 1f) + GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT or GLES20.GL_COLOR_BUFFER_BIT) + + GLES20.glUseProgram(program) + GLES20.glActiveTexture(GLES20.GL_TEXTURE0) + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId) + + vertices.position(POS_OFFSET) + GLES20.glVertexAttribPointer( + aPositionHandle, 3, GLES20.GL_FLOAT, false, STRIDE_BYTES, vertices + ) + GLES20.glEnableVertexAttribArray(aPositionHandle) + + vertices.position(UV_OFFSET) + GLES20.glVertexAttribPointer( + aTextureCoordHandle, 2, GLES20.GL_FLOAT, false, STRIDE_BYTES, vertices + ) + GLES20.glEnableVertexAttribArray(aTextureCoordHandle) + + GLES20.glUniformMatrix4fv(uMVPMatrixHandle, 1, false, mvpMatrix, 0) + GLES20.glUniformMatrix4fv(uSTMatrixHandle, 1, false, stMatrix, 0) + + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) + GLES20.glDisableVertexAttribArray(aPositionHandle) + GLES20.glDisableVertexAttribArray(aTextureCoordHandle) + } + + private fun createProgram(vertexSource: String, fragmentSource: String): Int { + val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource) + val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource) + val program = GLES20.glCreateProgram() + GLES20.glAttachShader(program, vertexShader) + GLES20.glAttachShader(program, fragmentShader) + GLES20.glLinkProgram(program) + val linkStatus = IntArray(1) + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0) + if (linkStatus[0] != GLES20.GL_TRUE) { + val log = GLES20.glGetProgramInfoLog(program) + GLES20.glDeleteProgram(program) + throw RuntimeException("Could not link program: $log") + } + return program + } + + private fun loadShader(type: Int, source: String): Int { + val shader = GLES20.glCreateShader(type) + GLES20.glShaderSource(shader, source) + GLES20.glCompileShader(shader) + val compiled = IntArray(1) + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0) + if (compiled[0] == 0) { + val log = GLES20.glGetShaderInfoLog(shader) + GLES20.glDeleteShader(shader) + throw RuntimeException("Could not compile shader $type: $log") + } + return shader + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt new file mode 100644 index 0000000000..e9c99bc9c4 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt @@ -0,0 +1,582 @@ +package expo.modules.blueskyvideocompress + +import android.content.Context +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaExtractor +import android.media.MediaFormat +import android.media.MediaMuxer +import android.net.Uri +import android.os.Build +import android.util.Log +import java.io.File +import java.nio.ByteBuffer +import java.util.UUID + +class VideoCompressor( + private val context: Context, + private val uriString: String, + private val targetBitrate: Int, + private val maxSize: Int, + private val codecPref: String, + private val frameRateCap: Int, + private val jobId: Int, + private val onProgress: (Int, Double) -> Unit +) { + companion object { + private const val TAG = "BskyVideoCompress" + private const val TIMEOUT_DEQUEUE = 100L + private const val I_FRAME_INTERVAL = 3 + } + + @Volatile + private var isCancelled = false + + fun cancel() { + isCancelled = true + } + + fun compress(): Map { + // 'auto' targets h264 — server pipeline is HLS, which favors h264 (HEVC needs + // fMP4 segments + commercial licensing). HEVC remains opt-in via codec: 'hevc'. + val tryHevc = codecPref == "hevc" + + if (tryHevc) { + try { + return doCompress(useHevc = true, allowSoftwareFallback = false) + } catch (e: Exception) { + if (codecPref == "hevc" || isCancelled) throw e + Log.w(TAG, "HEVC encode failed, falling back to h264", e) + } + } + return doCompress(useHevc = false, allowSoftwareFallback = true) + } + + private fun doCompress(useHevc: Boolean, allowSoftwareFallback: Boolean): Map { + val mime = if (useHevc) MediaFormat.MIMETYPE_VIDEO_HEVC else MediaFormat.MIMETYPE_VIDEO_AVC + val encoderInfo = CodecSelector.findEncoder(mime, requireHardware = !allowSoftwareFallback) + ?: throw RuntimeException("No encoder for $mime") + + try { + return runPipeline(encoderInfo, useHevc) + } catch (e: Exception) { + if (!allowSoftwareFallback || isCancelled || !encoderInfo.isHardware) throw e + Log.w(TAG, "Hardware encoder ${encoderInfo.name} failed, trying software", e) + val sw = CodecSelector.findEncoder(mime, requireHardware = false) + ?.takeIf { !it.isHardware } + ?: throw e + return runPipeline(sw, useHevc) + } + } + + private fun runPipeline( + encoderInfo: CodecSelector.EncoderInfo, + useHevc: Boolean + ): Map { + val outputFile = File(context.cacheDir, "${UUID.randomUUID()}.mp4") + + var extractor: MediaExtractor? = null + var muxer: MediaMuxer? = null + var encoder: MediaCodec? = null + var decoder: MediaCodec? = null + var inputSurface: InputSurface? = null + var outputSurface: OutputSurface? = null + var muxerStarted = false + var outputDims = Pair(0, 0) + var durationUs = 0L + + try { + val uri = Uri.parse(uriString) + extractor = MediaExtractor() + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + extractor.setDataSource(context, uri, null) + } else { + extractor.setDataSource(uriString) + } + + var videoTrackIndex = -1 + var audioTrackIndex = -1 + var videoFormat: MediaFormat? = null + var audioFormat: MediaFormat? = null + + for (i in 0 until extractor.trackCount) { + val format = extractor.getTrackFormat(i) + val trackMime = format.getString(MediaFormat.KEY_MIME) ?: continue + if (trackMime.startsWith("video/") && videoTrackIndex == -1) { + videoTrackIndex = i + videoFormat = format + } else if (trackMime.startsWith("audio/") && audioTrackIndex == -1) { + audioTrackIndex = i + audioFormat = format + } + } + + if (videoTrackIndex == -1 || videoFormat == null) { + throw RuntimeException("No video track found") + } + + val sourceWidth = videoFormat.getInteger(MediaFormat.KEY_WIDTH) + val sourceHeight = videoFormat.getInteger(MediaFormat.KEY_HEIGHT) + val rotation = if (videoFormat.containsKey(MediaFormat.KEY_ROTATION)) { + videoFormat.getInteger(MediaFormat.KEY_ROTATION) + } else 0 + durationUs = if (videoFormat.containsKey(MediaFormat.KEY_DURATION)) { + videoFormat.getLong(MediaFormat.KEY_DURATION) + } else 0L + val sourceFps = if (videoFormat.containsKey(MediaFormat.KEY_FRAME_RATE)) { + videoFormat.getInteger(MediaFormat.KEY_FRAME_RATE) + } else 30 + + outputDims = calculateOutputDims(sourceWidth, sourceHeight, rotation, maxSize) + val shouldPassthroughAudio = audioFormat != null && canPassthroughAudio(audioFormat) + val transcodedAudio: TranscodedAudio? = if ( + audioTrackIndex >= 0 && audioFormat != null && !shouldPassthroughAudio + ) { + transcodeAudioToAAC(audioTrackIndex, audioFormat) + } else null + muxer = MediaMuxer(outputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + + val effectiveBitrate = if (targetBitrate > 0) { + targetBitrate + } else if (useHevc) 2_500_000 else 3_000_000 + + val encoderFormat = MediaFormat.createVideoFormat( + encoderInfo.mime, outputDims.first, outputDims.second + ).apply { + setInteger( + MediaFormat.KEY_COLOR_FORMAT, + MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface + ) + setInteger(MediaFormat.KEY_BIT_RATE, effectiveBitrate) + setInteger( + MediaFormat.KEY_BITRATE_MODE, + MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR + ) + setInteger(MediaFormat.KEY_FRAME_RATE, frameRateCap) + setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + setInteger(MediaFormat.KEY_COLOR_STANDARD, MediaFormat.COLOR_STANDARD_BT709) + setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_SDR_VIDEO) + setInteger(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED) + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + setInteger(MediaFormat.KEY_PRIORITY, 0) + setInteger(MediaFormat.KEY_OPERATING_RATE, frameRateCap) + if (useHevc) { + setInteger( + MediaFormat.KEY_PROFILE, + MediaCodecInfo.CodecProfileLevel.HEVCProfileMain + ) + } else { + setInteger( + MediaFormat.KEY_PROFILE, + MediaCodecInfo.CodecProfileLevel.AVCProfileHigh + ) + setInteger( + MediaFormat.KEY_LEVEL, + MediaCodecInfo.CodecProfileLevel.AVCLevel41 + ) + } + } + } + + encoder = MediaCodec.createByCodecName(encoderInfo.name) + encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + val encoderInputSurface = encoder.createInputSurface() + inputSurface = InputSurface(encoderInputSurface) + inputSurface.makeCurrent() + outputSurface = OutputSurface() + encoder.start() + + decoder = MediaCodec.createDecoderByType( + videoFormat.getString(MediaFormat.KEY_MIME) ?: "video/avc" + ) + // Ask the decoder to tone-map HDR (HLG/PQ) sources to SDR. Vendors may + // ignore the hint, but where supported it produces correct BT.709 pixels + // for the encoder rather than HDR pixels mislabeled as SDR. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + videoFormat.setInteger( + MediaFormat.KEY_COLOR_TRANSFER_REQUEST, + MediaFormat.COLOR_TRANSFER_SDR_VIDEO + ) + } + decoder.configure(videoFormat, outputSurface.surface, null, 0) + decoder.start() + extractor.selectTrack(videoTrackIndex) + + val frameDropEnabled = sourceFps > frameRateCap + val targetFrameIntervalUs = if (frameDropEnabled) 1_000_000L / frameRateCap else 0L + var nextTargetPtsUs = 0L + + var muxerVideoTrack = -1 + var muxerAudioTrack = -1 + + val bufferInfo = MediaCodec.BufferInfo() + var inputDone = false + var decoderDone = false + var outputDone = false + var lastProgressMs = 0L + + while (!outputDone && !isCancelled) { + if (!inputDone) { + val idx = decoder.dequeueInputBuffer(TIMEOUT_DEQUEUE) + if (idx >= 0) { + val buf = decoder.getInputBuffer(idx) + if (buf != null) { + val sz = extractor.readSampleData(buf, 0) + if (sz < 0) { + decoder.queueInputBuffer( + idx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM + ) + inputDone = true + } else { + decoder.queueInputBuffer(idx, 0, sz, extractor.sampleTime, 0) + extractor.advance() + } + } + } + } + + if (!decoderDone) { + val status = decoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_DEQUEUE) + if (status >= 0) { + val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + val shouldRender = if (isEos) { + false + } else if (frameDropEnabled) { + if (bufferInfo.presentationTimeUs >= nextTargetPtsUs) { + nextTargetPtsUs = bufferInfo.presentationTimeUs + targetFrameIntervalUs + true + } else false + } else true + decoder.releaseOutputBuffer(status, shouldRender) + if (shouldRender) { + outputSurface.awaitNewImage() + outputSurface.drawImage() + inputSurface.setPresentationTime(bufferInfo.presentationTimeUs * 1000) + inputSurface.swapBuffers() + } + if (isEos) { + encoder.signalEndOfInputStream() + decoderDone = true + } + } + } + + var encoderDrained = false + while (!outputDone && !isCancelled && !encoderDrained) { + val encIdx = encoder.dequeueOutputBuffer(bufferInfo, 0) + when { + encIdx == MediaCodec.INFO_TRY_AGAIN_LATER -> encoderDrained = true + encIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + if (!muxerStarted) { + muxerVideoTrack = muxer.addTrack(encoder.outputFormat) + if (audioTrackIndex >= 0 && audioFormat != null) { + if (shouldPassthroughAudio) { + muxerAudioTrack = muxer.addTrack(audioFormat) + } else if (transcodedAudio != null) { + muxerAudioTrack = muxer.addTrack(transcodedAudio.outputFormat) + } + } + muxer.start() + muxerStarted = true + } + } + encIdx >= 0 -> { + val data = encoder.getOutputBuffer(encIdx) + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { + bufferInfo.size = 0 + } + if (data != null && bufferInfo.size > 0 && muxerStarted) { + muxer.writeSampleData(muxerVideoTrack, data, bufferInfo) + } + val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + encoder.releaseOutputBuffer(encIdx, false) + if (isEos) { + outputDone = true + } else if (durationUs > 0) { + val now = System.currentTimeMillis() + if (now - lastProgressMs >= 100) { + lastProgressMs = now + val p = (bufferInfo.presentationTimeUs.toDouble() / durationUs) + .coerceIn(0.0, 1.0) + onProgress(jobId, p) + } + } + } + } + } + } + + if (audioTrackIndex >= 0 && muxerAudioTrack >= 0 && muxerStarted && !isCancelled) { + if (shouldPassthroughAudio) { + passthroughAudio(audioTrackIndex, muxer, muxerAudioTrack) + } else if (transcodedAudio != null) { + writeTranscodedAudio(transcodedAudio.samples, muxer, muxerAudioTrack) + } + } + } finally { + try { decoder?.stop() } catch (_: Exception) {} + try { decoder?.release() } catch (_: Exception) {} + try { encoder?.stop() } catch (_: Exception) {} + try { encoder?.release() } catch (_: Exception) {} + try { outputSurface?.release() } catch (_: Exception) {} + try { inputSurface?.release() } catch (_: Exception) {} + try { extractor?.release() } catch (_: Exception) {} + try { + if (muxerStarted) muxer?.stop() + muxer?.release() + } catch (_: Exception) {} + } + + if (isCancelled) { + outputFile.delete() + throw RuntimeException("Compression cancelled") + } + + onProgress(jobId, 1.0) + val durationSeconds = durationUs / 1_000_000.0 + + return mapOf( + "uri" to "file://${outputFile.absolutePath}", + "size" to outputFile.length(), + "mimeType" to "video/mp4", + "width" to outputDims.first, + "height" to outputDims.second, + "duration" to durationSeconds, + "codec" to if (useHevc) "hevc" else "h264" + ) + } + + private fun passthroughAudio(audioTrackIndex: Int, muxer: MediaMuxer, muxerAudioTrack: Int) { + val audioExtractor = MediaExtractor() + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + audioExtractor.setDataSource(context, Uri.parse(uriString), null) + } else { + audioExtractor.setDataSource(uriString) + } + audioExtractor.selectTrack(audioTrackIndex) + audioExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC) + + val buffer = ByteBuffer.allocate(256 * 1024) + val info = MediaCodec.BufferInfo() + try { + while (!isCancelled) { + val sz = audioExtractor.readSampleData(buffer, 0) + if (sz < 0) break + info.offset = 0 + info.size = sz + info.presentationTimeUs = audioExtractor.sampleTime + info.flags = audioExtractor.sampleFlags + muxer.writeSampleData(muxerAudioTrack, buffer, info) + audioExtractor.advance() + } + } finally { + audioExtractor.release() + } + } + + private fun canPassthroughAudio(format: MediaFormat): Boolean { + val mime = format.getString(MediaFormat.KEY_MIME) ?: return false + return mime == MediaFormat.MIMETYPE_AUDIO_AAC + } + + private data class TranscodedAudio( + val outputFormat: MediaFormat, + val samples: List + ) { + data class Sample( + val bytes: ByteArray, + val presentationTimeUs: Long, + val flags: Int + ) + } + + // Re-encode non-AAC source audio (Opus, Vorbis, etc.) to AAC so the mp4 muxer + // can take it. iOS always re-encodes to AAC; without this Android would drop + // the audio track entirely. + private fun transcodeAudioToAAC( + audioTrackIndex: Int, + sourceFormat: MediaFormat + ): TranscodedAudio? { + val sourceMime = sourceFormat.getString(MediaFormat.KEY_MIME) ?: return null + val sampleRate = if (sourceFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE)) + sourceFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100 + val channelCount = if (sourceFormat.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) + sourceFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT).coerceIn(1, 2) else 2 + + val audioExtractor = MediaExtractor() + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + audioExtractor.setDataSource(context, Uri.parse(uriString), null) + } else { + audioExtractor.setDataSource(uriString) + } + audioExtractor.selectTrack(audioTrackIndex) + + var decoder: MediaCodec? = null + var encoder: MediaCodec? = null + try { + decoder = MediaCodec.createDecoderByType(sourceMime) + decoder.configure(sourceFormat, null, null, 0) + decoder.start() + + val encoderFormat = MediaFormat.createAudioFormat( + MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, channelCount + ).apply { + setInteger( + MediaFormat.KEY_AAC_PROFILE, + MediaCodecInfo.CodecProfileLevel.AACObjectLC + ) + setInteger(MediaFormat.KEY_BIT_RATE, 128_000) + setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 256 * 1024) + } + encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC) + encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + encoder.start() + + val samples = mutableListOf() + var outputFormat: MediaFormat? = null + var inputDone = false + var decoderDone = false + var encoderInputSignalled = false + var encoderDone = false + val info = MediaCodec.BufferInfo() + + while (!encoderDone && !isCancelled) { + if (!inputDone) { + val idx = decoder.dequeueInputBuffer(TIMEOUT_DEQUEUE) + if (idx >= 0) { + val buf = decoder.getInputBuffer(idx) + if (buf != null) { + val sz = audioExtractor.readSampleData(buf, 0) + if (sz < 0) { + decoder.queueInputBuffer( + idx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM + ) + inputDone = true + } else { + decoder.queueInputBuffer( + idx, 0, sz, audioExtractor.sampleTime, audioExtractor.sampleFlags + ) + audioExtractor.advance() + } + } + } + } + + if (!decoderDone) { + val status = decoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE) + when { + status == MediaCodec.INFO_TRY_AGAIN_LATER -> {} + status == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {} + status >= 0 -> { + val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + val data = decoder.getOutputBuffer(status) + if (data != null && info.size > 0) { + val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE) + if (encInIdx >= 0) { + val encInBuf = encoder.getInputBuffer(encInIdx) + if (encInBuf != null) { + encInBuf.clear() + data.position(info.offset) + data.limit(info.offset + info.size) + encInBuf.put(data) + encoder.queueInputBuffer( + encInIdx, 0, info.size, info.presentationTimeUs, 0 + ) + } + } + } + decoder.releaseOutputBuffer(status, false) + if (isEos) { + if (!encoderInputSignalled) { + val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE * 10) + if (encInIdx >= 0) { + encoder.queueInputBuffer( + encInIdx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM + ) + encoderInputSignalled = true + } + } + decoderDone = true + } + } + } + } + + val encOutIdx = encoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE) + when { + encOutIdx == MediaCodec.INFO_TRY_AGAIN_LATER -> {} + encOutIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + outputFormat = encoder.outputFormat + } + encOutIdx >= 0 -> { + val data = encoder.getOutputBuffer(encOutIdx) + val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + val isConfig = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0 + if (data != null && info.size > 0 && !isConfig) { + val bytes = ByteArray(info.size) + data.position(info.offset) + data.get(bytes, 0, info.size) + samples.add( + TranscodedAudio.Sample( + bytes = bytes, + presentationTimeUs = info.presentationTimeUs, + flags = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG.inv() + ) + ) + } + encoder.releaseOutputBuffer(encOutIdx, false) + if (isEos) encoderDone = true + } + } + } + + val fmt = outputFormat ?: return null + return TranscodedAudio(fmt, samples) + } catch (e: Exception) { + Log.w(TAG, "Audio transcode failed; dropping audio", e) + return null + } finally { + try { decoder?.stop() } catch (_: Exception) {} + try { decoder?.release() } catch (_: Exception) {} + try { encoder?.stop() } catch (_: Exception) {} + try { encoder?.release() } catch (_: Exception) {} + audioExtractor.release() + } + } + + private fun writeTranscodedAudio( + samples: List, + muxer: MediaMuxer, + muxerAudioTrack: Int + ) { + val info = MediaCodec.BufferInfo() + for (sample in samples) { + if (isCancelled) break + val buffer = ByteBuffer.wrap(sample.bytes) + info.offset = 0 + info.size = sample.bytes.size + info.presentationTimeUs = sample.presentationTimeUs + info.flags = sample.flags + muxer.writeSampleData(muxerAudioTrack, buffer, info) + } + } + + private fun calculateOutputDims(srcW: Int, srcH: Int, rotation: Int, maxSize: Int): Pair { + val isRotated = rotation == 90 || rotation == 270 + val displayW = if (isRotated) srcH else srcW + val displayH = if (isRotated) srcW else srcH + val scale: Float = when { + displayW <= maxSize && displayH <= maxSize -> 1.0f + displayW > displayH -> maxSize.toFloat() / displayW + else -> maxSize.toFloat() / displayH + } + return Pair( + roundToEven((displayW * scale).toInt()), + roundToEven((displayH * scale).toInt()) + ) + } + + private fun roundToEven(v: Int): Int = if (v % 2 == 0) v else v - 1 +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt new file mode 100644 index 0000000000..7c214bddd0 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt @@ -0,0 +1,125 @@ +package expo.modules.blueskyvideocompress + +import android.content.Context +import android.media.MediaExtractor +import android.media.MediaFormat +import android.media.MediaMetadataRetriever +import android.net.Uri +import android.os.Build + +object VideoProber { + fun probe(context: Context, uriString: String): Map { + val uri = Uri.parse(uriString) + val retriever = MediaMetadataRetriever() + + try { + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + retriever.setDataSource(context, uri) + } else { + retriever.setDataSource(uriString) + } + + val width = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH) + ?.toIntOrNull() ?: 0 + val height = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT) + ?.toIntOrNull() ?: 0 + val durationMs = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) + ?.toLongOrNull() ?: 0L + val rotation = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION) + ?.toIntOrNull() ?: 0 + val bitrate = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_BITRATE) + ?.toIntOrNull() ?: 0 + val hasAudio = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO) + ?.equals("yes") ?: false + val frameRate = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE) + ?.toFloatOrNull() ?: 0f + + val fileSize = getFileSize(context, uriString) + + val extractor = MediaExtractor() + var codec = "unknown" + var mimeType = "video/mp4" + var extractedFrameRate = frameRate + var isHDR = false + + try { + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + extractor.setDataSource(context, uri, null) + } else { + extractor.setDataSource(uriString) + } + + for (i in 0 until extractor.trackCount) { + val format = extractor.getTrackFormat(i) + val mime = format.getString(MediaFormat.KEY_MIME) + if (mime?.startsWith("video/") == true) { + mimeType = mime + codec = mime.removePrefix("video/") + if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) { + extractedFrameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE).toFloat() + } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && + format.containsKey(MediaFormat.KEY_COLOR_TRANSFER) + ) { + val transfer = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER) + isHDR = transfer == MediaFormat.COLOR_TRANSFER_HLG || + transfer == MediaFormat.COLOR_TRANSFER_ST2084 + } + // Dolby Vision tracks use codec-specific mimes that aren't covered by + // KEY_COLOR_TRANSFER on every device. + if (mime.contains("dolby-vision", ignoreCase = true)) { + isHDR = true + } + break + } + } + } finally { + extractor.release() + } + + val durationSeconds = durationMs / 1000.0 + val effectiveBitrate = if (bitrate > 0) { + bitrate + } else if (durationSeconds > 0 && fileSize > 0) { + (fileSize * 8 / durationSeconds).toInt() + } else { + 0 + } + + return mapOf( + "width" to width, + "height" to height, + "duration" to durationSeconds, + "bitrate" to effectiveBitrate, + "fileSize" to fileSize, + "mimeType" to mimeType, + "codec" to codec, + "hasAudio" to hasAudio, + "frameRate" to extractedFrameRate.toDouble(), + "rotation" to rotation, + "isHDR" to isHDR + ) + } finally { + retriever.release() + } + } + + private fun getFileSize(context: Context, uriString: String): Long { + return try { + if (uriString.startsWith("content://")) { + context.contentResolver.openFileDescriptor(Uri.parse(uriString), "r")?.use { + it.statSize + } ?: 0L + } else { + val path = if (uriString.startsWith("file://")) { + uriString.removePrefix("file://") + } else { + uriString + } + java.io.File(path).length() + } + } catch (_: Exception) { + 0L + } + } +} diff --git a/modules/expo-bluesky-video-compress/expo-module.config.json b/modules/expo-bluesky-video-compress/expo-module.config.json new file mode 100644 index 0000000000..c1c24a480d --- /dev/null +++ b/modules/expo-bluesky-video-compress/expo-module.config.json @@ -0,0 +1,9 @@ +{ + "platforms": ["ios", "android"], + "ios": { + "modules": ["ExpoBlueskyVideoCompressModule"] + }, + "android": { + "modules": ["expo.modules.blueskyvideocompress.ExpoBlueskyVideoCompressModule"] + } +} diff --git a/modules/expo-bluesky-video-compress/index.ts b/modules/expo-bluesky-video-compress/index.ts new file mode 100644 index 0000000000..423d7f3ac2 --- /dev/null +++ b/modules/expo-bluesky-video-compress/index.ts @@ -0,0 +1,87 @@ +import {type EventSubscription} from 'expo-modules-core' + +import NativeModule from './src/ExpoBlueskyVideoCompressModule' +import { + type CodecPreference, + type CompressCallbacks, + type CompressOptions, + type CompressResult, + type VideoMetadata, +} from './src/types' + +export type { + CodecPreference, + CompressCallbacks, + CompressOptions, + CompressResult, + VideoMetadata, +} + +class AbortError extends Error { + name = 'AbortError' + constructor() { + super('Aborted') + } +} + +let jobIdCounter = 0 + +export function probe(uri: string): Promise { + return NativeModule.probe(uri) +} + +export function compress( + uri: string, + options: CompressOptions = {}, + callbacks?: CompressCallbacks, +): Promise { + const jobId = ++jobIdCounter + let subscription: EventSubscription | undefined + + if (callbacks?.signal?.aborted) { + return Promise.reject(new AbortError()) + } + + const nativeOptions = { + targetBitrate: options.targetBitrate ?? 0, + maxSize: options.maxSize ?? 1920, + codec: options.codec ?? 'auto', + frameRateCap: options.frameRateCap ?? 30, + jobId, + } + + return new Promise((resolve, reject) => { + if (callbacks?.onProgress) { + subscription = NativeModule.addListener( + 'onProgress', + (event: {id: number; progress: number}) => { + if (event.id === jobId) { + callbacks.onProgress!(event.progress) + } + }, + ) + } + + const abortHandler = () => { + NativeModule.cancel(jobId) + subscription?.remove() + reject(new AbortError()) + } + + if (callbacks?.signal) { + callbacks.signal.addEventListener('abort', abortHandler, {once: true}) + } + + NativeModule.compress(uri, nativeOptions) + .then(result => { + callbacks?.signal?.removeEventListener('abort', abortHandler) + subscription?.remove() + resolve(result) + }) + .catch((error: unknown) => { + callbacks?.signal?.removeEventListener('abort', abortHandler) + subscription?.remove() + reject(error instanceof Error ? error : new Error(String(error))) + }) + }) +} diff --git a/modules/expo-bluesky-video-compress/ios/CodecCapability.swift b/modules/expo-bluesky-video-compress/ios/CodecCapability.swift new file mode 100644 index 0000000000..df95d9b7e8 --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/CodecCapability.swift @@ -0,0 +1,21 @@ +import VideoToolbox + +enum CodecCapability { + static let isHardwareHEVCEncodeAvailable: Bool = { + var encoderListCF: CFArray? + let status = VTCopyVideoEncoderList(nil, &encoderListCF) + guard status == noErr, let encoderList = encoderListCF as? [[String: Any]] else { + return false + } + return encoderList.contains { encoder in + guard let codecTypeValue = encoder[kVTVideoEncoderList_CodecType as String] as? Int, + codecTypeValue == Int(kCMVideoCodecType_HEVC) else { + return false + } + if let isHardware = encoder[kVTVideoEncoderList_IsHardwareAccelerated as String] as? Bool { + return isHardware + } + return true + } + }() +} diff --git a/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompress.podspec b/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompress.podspec new file mode 100644 index 0000000000..5bca8f0812 --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompress.podspec @@ -0,0 +1,20 @@ +Pod::Spec.new do |s| + s.name = 'ExpoBlueskyVideoCompress' + s.version = '1.0.0' + s.summary = 'Hardware-accelerated video compression for Bluesky' + s.description = 'Hardware-accelerated h264/HEVC video compression using AVAssetReader/Writer and VideoToolbox on iOS' + s.author = '' + s.homepage = 'https://github.com/bluesky-social/social-app' + s.platforms = { :ios => '15.1' } + s.source = { git: '' } + s.static_framework = true + + s.dependency 'ExpoModulesCore' + + s.pod_target_xcconfig = { + 'DEFINES_MODULE' => 'YES', + 'SWIFT_COMPILATION_MODE' => 'wholemodule' + } + + s.source_files = "**/*.{h,m,mm,swift,hpp,cpp}" +end diff --git a/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompressModule.swift b/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompressModule.swift new file mode 100644 index 0000000000..3bcea3c9f4 --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompressModule.swift @@ -0,0 +1,74 @@ +import AVFoundation +import ExpoModulesCore + +public class ExpoBlueskyVideoCompressModule: Module { + private var activeCompressors: [Int: VideoCompressor] = [:] + private let activeCompressorsLock = NSLock() + + public func definition() -> ModuleDefinition { + Name("ExpoBlueskyVideoCompress") + + Events("onProgress") + + AsyncFunction("probe") { (uri: String) -> [String: Any] in + let url = URL(string: uri) ?? URL(fileURLWithPath: uri) + return try await VideoProber.probe(url: url) + } + + AsyncFunction("compress") { (uri: String, options: [String: Any]) -> [String: Any] in + let url = URL(string: uri) ?? URL(fileURLWithPath: uri) + let targetBitrate = options["targetBitrate"] as? Int ?? 0 + let maxSize = options["maxSize"] as? Int ?? 1920 + let codecPref = options["codec"] as? String ?? "auto" + let frameRateCap = max(1, options["frameRateCap"] as? Int ?? 30) + let jobId = options["jobId"] as? Int ?? 0 + + let compressor = VideoCompressor( + url: url, + targetBitrate: targetBitrate, + maxSize: maxSize, + codecPref: codecPref, + frameRateCap: frameRateCap, + jobId: jobId, + onProgress: { [weak self] id, progress in + self?.sendEvent("onProgress", [ + "id": id, + "progress": progress, + ]) + } + ) + + self.setCompressor(jobId, compressor) + + do { + let result = try await compressor.compress() + self.setCompressor(jobId, nil) + return result + } catch { + self.setCompressor(jobId, nil) + throw error + } + } + + Function("cancel") { (jobId: Int) in + self.cancelCompressor(jobId) + } + } + + private func setCompressor(_ jobId: Int, _ compressor: VideoCompressor?) { + activeCompressorsLock.lock() + defer { activeCompressorsLock.unlock() } + if let compressor = compressor { + activeCompressors[jobId] = compressor + } else { + activeCompressors.removeValue(forKey: jobId) + } + } + + private func cancelCompressor(_ jobId: Int) { + activeCompressorsLock.lock() + let compressor = activeCompressors.removeValue(forKey: jobId) + activeCompressorsLock.unlock() + compressor?.cancel() + } +} diff --git a/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift b/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift new file mode 100644 index 0000000000..e4a75398ea --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift @@ -0,0 +1,410 @@ +import AVFoundation +import VideoToolbox + +class VideoCompressor { + private let url: URL + private let targetBitrate: Int + private let maxSize: Int + private let codecPref: String + private let frameRateCap: Int + private let jobId: Int + private let onProgress: (Int, Double) -> Void + private var isCancelled = false + + init( + url: URL, + targetBitrate: Int, + maxSize: Int, + codecPref: String, + frameRateCap: Int, + jobId: Int, + onProgress: @escaping (Int, Double) -> Void + ) { + self.url = url + self.targetBitrate = targetBitrate + self.maxSize = maxSize + self.codecPref = codecPref + self.frameRateCap = frameRateCap + self.jobId = jobId + self.onProgress = onProgress + } + + func cancel() { + isCancelled = true + } + + func compress() async throws -> [String: Any] { + let asset = AVURLAsset( + url: url, + options: [AVURLAssetPreferPreciseDurationAndTimingKey: true] + ) + let duration = try await asset.load(.duration) + let totalSeconds = CMTimeGetSeconds(duration) + guard totalSeconds > 0 else { throw err("Invalid video duration", code: 2) } + + let videoTracks = try await asset.loadTracks(withMediaType: .video) + guard let videoTrack = videoTracks.first else { + throw err("No video track found", code: 1) + } + + let naturalSize = try await videoTrack.load(.naturalSize) + let preferredTransform = try await videoTrack.load(.preferredTransform) + + let rotatedRect = CGRect(origin: .zero, size: naturalSize).applying(preferredTransform) + let displaySize = CGSize( + width: abs(rotatedRect.width), + height: abs(rotatedRect.height) + ) + let outputSize = scaleEvenly(displaySize: displaySize, maxSize: maxSize) + + let audioTracks = try await asset.loadTracks(withMediaType: .audio) + + // 'auto' targets h264 — server pipeline is HLS, which favors h264 (HEVC needs + // fMP4 segments + commercial licensing). HEVC remains opt-in via codec: 'hevc'. + let useHEVC: Bool + switch codecPref { + case "hevc": useHEVC = true + default: useHEVC = false + } + let codecType: AVVideoCodecType = useHEVC ? .hevc : .h264 + let profileLevel: String = useHEVC + ? kVTProfileLevel_HEVC_Main_AutoLevel as String + : kVTProfileLevel_H264_High_AutoLevel as String + + let effectiveBitrate = targetBitrate > 0 + ? targetBitrate + : (useHEVC ? 2_500_000 : 3_000_000) + + let outputURL = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathExtension("mp4") + + let videoComposition = makeRotatingComposition( + videoTrack: videoTrack, + preferredTransform: preferredTransform, + naturalSize: naturalSize, + outputSize: outputSize, + duration: duration + ) + + let reader = try AVAssetReader(asset: asset) + + let videoReaderSettings: [String: Any] = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA + ] + let videoReaderOutput = AVAssetReaderVideoCompositionOutput( + videoTracks: [videoTrack], + videoSettings: videoReaderSettings + ) + videoReaderOutput.videoComposition = videoComposition + videoReaderOutput.alwaysCopiesSampleData = false + guard reader.canAdd(videoReaderOutput) else { + throw err("Cannot read video track", code: 3) + } + reader.add(videoReaderOutput) + + var compressionProps: [String: Any] = [ + AVVideoAverageBitRateKey: effectiveBitrate, + AVVideoProfileLevelKey: profileLevel, + AVVideoMaxKeyFrameIntervalKey: max(frameRateCap * 3, 30), + AVVideoExpectedSourceFrameRateKey: frameRateCap, + AVVideoAllowFrameReorderingKey: false, + kVTCompressionPropertyKey_RealTime as String: true, + ] + let peakBytesPerSecond = Int(Double(effectiveBitrate) / 8.0 * 1.5) + compressionProps[kVTCompressionPropertyKey_DataRateLimits as String] = [ + peakBytesPerSecond, 1.0 + ] as CFArray + + let videoColorProps: [String: Any] = [ + AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2, + AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2, + AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_709_2, + ] + + let videoWriterSettings: [String: Any] = [ + AVVideoCodecKey: codecType, + AVVideoWidthKey: outputSize.width, + AVVideoHeightKey: outputSize.height, + AVVideoColorPropertiesKey: videoColorProps, + AVVideoCompressionPropertiesKey: compressionProps, + ] + let videoWriterInput = AVAssetWriterInput( + mediaType: .video, + outputSettings: videoWriterSettings + ) + videoWriterInput.expectsMediaDataInRealTime = false + + let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4) + writer.shouldOptimizeForNetworkUse = true + writer.metadata = [] + guard writer.canAdd(videoWriterInput) else { + throw err("Cannot write video track", code: 4) + } + writer.add(videoWriterInput) + + var audioReaderOutput: AVAssetReaderTrackOutput? + var audioWriterInput: AVAssetWriterInput? + if let audioTrack = audioTracks.first { + let audioDecoderSettings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsBigEndianKey: false, + AVLinearPCMIsNonInterleaved: false, + ] + let audioOutput = AVAssetReaderTrackOutput( + track: audioTrack, + outputSettings: audioDecoderSettings + ) + audioOutput.alwaysCopiesSampleData = false + if reader.canAdd(audioOutput) { + reader.add(audioOutput) + audioReaderOutput = audioOutput + + let audioEncoderSettings: [String: Any] = [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVSampleRateKey: 44100, + AVNumberOfChannelsKey: 2, + AVEncoderBitRateKey: 128_000, + ] + let audioInput = AVAssetWriterInput( + mediaType: .audio, + outputSettings: audioEncoderSettings + ) + audioInput.expectsMediaDataInRealTime = false + if writer.canAdd(audioInput) { + writer.add(audioInput) + audioWriterInput = audioInput + } + } + } + + guard reader.startReading() else { + throw reader.error ?? err("Reader failed to start", code: 8) + } + guard writer.startWriting() else { + throw writer.error ?? err("Writer failed to start", code: 9) + } + writer.startSession(atSourceTime: .zero) + + let minFrameIntervalSeconds = 1.0 / Double(frameRateCap) + let minFrameInterval = CMTime( + seconds: minFrameIntervalSeconds, + preferredTimescale: 600 + ) + + await withTaskGroup(of: Void.self) { group in + group.addTask { [self] in + await processVideoTrack( + readerOutput: videoReaderOutput, + writerInput: videoWriterInput, + totalDuration: totalSeconds, + minFrameInterval: minFrameInterval + ) + } + if let audioOutput = audioReaderOutput, let audioInput = audioWriterInput { + group.addTask { [self] in + await processAudioTrack( + readerOutput: audioOutput, + writerInput: audioInput + ) + } + } + await group.waitForAll() + } + + if isCancelled { + writer.cancelWriting() + try? FileManager.default.removeItem(at: outputURL) + throw err("Compression cancelled", code: 5) + } + + if reader.status == .failed { + let error = reader.error ?? err("Reader failed", code: 6) + writer.cancelWriting() + try? FileManager.default.removeItem(at: outputURL) + throw error + } + + await writer.finishWriting() + + if writer.status == .failed { + let error = writer.error ?? err("Writer failed", code: 7) + try? FileManager.default.removeItem(at: outputURL) + throw error + } + + onProgress(jobId, 1.0) + + let attributes = try FileManager.default.attributesOfItem(atPath: outputURL.path) + let fileSize = attributes[.size] as? Int ?? 0 + + return [ + "uri": outputURL.absoluteString, + "size": fileSize, + "mimeType": "video/mp4", + "width": outputSize.width, + "height": outputSize.height, + "duration": totalSeconds, + "codec": useHEVC ? "hevc" : "h264", + ] + } + + private func processVideoTrack( + readerOutput: AVAssetReaderOutput, + writerInput: AVAssetWriterInput, + totalDuration: Double, + minFrameInterval: CMTime + ) async { + var lastProgressTime: CFAbsoluteTime = 0 + var lastAppendedPTS: CMTime? + var finished = false + + await withCheckedContinuation { (continuation: CheckedContinuation) in + writerInput.requestMediaDataWhenReady( + on: DispatchQueue(label: "com.bsky.videocompress.video") + ) { + let finish = { + if !finished { + finished = true + writerInput.markAsFinished() + continuation.resume() + } + } + while writerInput.isReadyForMoreMediaData { + if finished { return } + if self.isCancelled { + finish() + return + } + guard let sampleBuffer = readerOutput.copyNextSampleBuffer() else { + finish() + return + } + + let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + if let last = lastAppendedPTS { + let delta = CMTimeSubtract(pts, last) + if CMTimeCompare(delta, minFrameInterval) < 0 { + continue + } + } + lastAppendedPTS = pts + + if !writerInput.append(sampleBuffer) { + finish() + return + } + + let now = CFAbsoluteTimeGetCurrent() + if now - lastProgressTime >= 0.1 { + lastProgressTime = now + let progress = min(CMTimeGetSeconds(pts) / totalDuration, 1.0) + self.onProgress(self.jobId, progress) + } + } + } + } + } + + private func processAudioTrack( + readerOutput: AVAssetReaderOutput, + writerInput: AVAssetWriterInput + ) async { + var finished = false + + await withCheckedContinuation { (continuation: CheckedContinuation) in + writerInput.requestMediaDataWhenReady( + on: DispatchQueue(label: "com.bsky.videocompress.audio") + ) { + let finish = { + if !finished { + finished = true + writerInput.markAsFinished() + continuation.resume() + } + } + while writerInput.isReadyForMoreMediaData { + if finished { return } + if self.isCancelled { + finish() + return + } + guard let sampleBuffer = readerOutput.copyNextSampleBuffer() else { + finish() + return + } + if !writerInput.append(sampleBuffer) { + finish() + return + } + } + } + } + } + + private func makeRotatingComposition( + videoTrack: AVAssetTrack, + preferredTransform: CGAffineTransform, + naturalSize: CGSize, + outputSize: (width: Int, height: Int), + duration: CMTime + ) -> AVMutableVideoComposition { + let composition = AVMutableVideoComposition() + composition.renderSize = CGSize(width: outputSize.width, height: outputSize.height) + composition.frameDuration = CMTime(value: 1, timescale: Int32(frameRateCap)) + + let rotatedRect = CGRect(origin: .zero, size: naturalSize).applying(preferredTransform) + let translate = CGAffineTransform( + translationX: -rotatedRect.minX, + y: -rotatedRect.minY + ) + let displaySize = CGSize( + width: abs(rotatedRect.width), + height: abs(rotatedRect.height) + ) + let scaleX = CGFloat(outputSize.width) / displaySize.width + let scaleY = CGFloat(outputSize.height) / displaySize.height + let scale = CGAffineTransform(scaleX: scaleX, y: scaleY) + let combined = preferredTransform.concatenating(translate).concatenating(scale) + + let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: videoTrack) + layerInstruction.setTransform(combined, at: .zero) + + let instruction = AVMutableVideoCompositionInstruction() + instruction.timeRange = CMTimeRange(start: .zero, duration: duration) + instruction.layerInstructions = [layerInstruction] + + composition.instructions = [instruction] + return composition + } + + private func scaleEvenly(displaySize: CGSize, maxSize: Int) -> (width: Int, height: Int) { + let cap = CGFloat(maxSize) + let scale: CGFloat + if displaySize.width <= cap && displaySize.height <= cap { + scale = 1.0 + } else if displaySize.width > displaySize.height { + scale = cap / displaySize.width + } else { + scale = cap / displaySize.height + } + return ( + roundToEven(Int(displaySize.width * scale)), + roundToEven(Int(displaySize.height * scale)) + ) + } + + private func roundToEven(_ value: Int) -> Int { + return value % 2 == 0 ? value : value - 1 + } + + private func err(_ message: String, code: Int) -> NSError { + return NSError( + domain: "ExpoBlueskyVideoCompress", + code: code, + userInfo: [NSLocalizedDescriptionKey: message] + ) + } +} diff --git a/modules/expo-bluesky-video-compress/ios/VideoProber.swift b/modules/expo-bluesky-video-compress/ios/VideoProber.swift new file mode 100644 index 0000000000..4636ed7d3c --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/VideoProber.swift @@ -0,0 +1,105 @@ +import AVFoundation +import UniformTypeIdentifiers + +struct VideoProber { + static func probe(url: URL) async throws -> [String: Any] { + let asset = AVURLAsset(url: url) + + let duration = try await asset.load(.duration) + let tracks = try await asset.loadTracks(withMediaType: .video) + + guard let videoTrack = tracks.first else { + throw NSError( + domain: "ExpoBlueskyVideoCompress", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "No video track found"] + ) + } + + let naturalSize = try await videoTrack.load(.naturalSize) + let preferredTransform = try await videoTrack.load(.preferredTransform) + let estimatedDataRate = try await videoTrack.load(.estimatedDataRate) + let nominalFrameRate = try await videoTrack.load(.nominalFrameRate) + let formatDescriptions = try await videoTrack.load(.formatDescriptions) + + var codec = "unknown" + var isHDR = false + if let formatDescription = formatDescriptions.first { + let subType = CMFormatDescriptionGetMediaSubType(formatDescription) + codec = fourCCToString(subType) + // Dolby Vision codecs are HDR by definition. + let dolbyVisionSubtypes: Set = ["dvhe", "dvh1", "dvav", "dva1"] + if dolbyVisionSubtypes.contains(codec) { + isHDR = true + } else if let extensions = CMFormatDescriptionGetExtensions(formatDescription) + as? [String: Any] + { + let transferKey = kCMFormatDescriptionExtension_TransferFunction as String + if let transfer = extensions[transferKey] as? String { + let hlg = kCMFormatDescriptionTransferFunction_ITU_R_2100_HLG as String + let pq = kCMFormatDescriptionTransferFunction_SMPTE_ST_2084_PQ as String + isHDR = transfer == hlg || transfer == pq + } + } + } + + let rotation = rotationFromTransform(preferredTransform) + let isRotated = rotation == 90 || rotation == 270 + let width = isRotated ? Int(naturalSize.height) : Int(naturalSize.width) + let height = isRotated ? Int(naturalSize.width) : Int(naturalSize.height) + + let audioTracks = try await asset.loadTracks(withMediaType: .audio) + let hasAudio = !audioTracks.isEmpty + + let fileSize: Int + if let attributes = try? FileManager.default.attributesOfItem(atPath: url.path), + let size = attributes[.size] as? Int { + fileSize = size + } else { + fileSize = 0 + } + + let mimeType: String + if let utType = UTType(filenameExtension: url.pathExtension) { + mimeType = utType.preferredMIMEType ?? "video/mp4" + } else { + mimeType = "video/mp4" + } + + let durationSeconds = CMTimeGetSeconds(duration) + var bitrate = Int(estimatedDataRate) + if bitrate == 0 && durationSeconds > 0 { + bitrate = Int(Double(fileSize * 8) / durationSeconds) + } + + return [ + "width": width, + "height": height, + "duration": durationSeconds, + "bitrate": bitrate, + "fileSize": fileSize, + "mimeType": mimeType, + "codec": codec, + "hasAudio": hasAudio, + "frameRate": nominalFrameRate, + "rotation": rotation, + "isHDR": isHDR + ] + } + + private static func rotationFromTransform(_ transform: CGAffineTransform) -> Int { + let angle = atan2(transform.b, transform.a) + let degrees = Int(round(angle * 180.0 / .pi)) + return ((degrees % 360) + 360) % 360 + } + + private static func fourCCToString(_ code: FourCharCode) -> String { + let chars: [Character] = [ + Character(UnicodeScalar((code >> 24) & 0xFF)!), + Character(UnicodeScalar((code >> 16) & 0xFF)!), + Character(UnicodeScalar((code >> 8) & 0xFF)!), + Character(UnicodeScalar(code & 0xFF)!) + ] + return String(chars).trimmingCharacters(in: .whitespaces) + } +} diff --git a/modules/expo-bluesky-video-compress/src/ExpoBlueskyVideoCompressModule.ts b/modules/expo-bluesky-video-compress/src/ExpoBlueskyVideoCompressModule.ts new file mode 100644 index 0000000000..dcf63c729f --- /dev/null +++ b/modules/expo-bluesky-video-compress/src/ExpoBlueskyVideoCompressModule.ts @@ -0,0 +1,23 @@ +import {type EventSubscription, requireNativeModule} from 'expo-modules-core' + +import { + type CompressResult, + type NativeCompressOptions, + type VideoMetadata, +} from './types' + +type ProgressEvent = {id: number; progress: number} + +interface ExpoBlueskyVideoCompressModule { + probe(uri: string): Promise + compress(uri: string, options: NativeCompressOptions): Promise + cancel(jobId: number): void + addListener( + eventName: 'onProgress', + listener: (event: ProgressEvent) => void, + ): EventSubscription +} + +export default requireNativeModule( + 'ExpoBlueskyVideoCompress', +) diff --git a/modules/expo-bluesky-video-compress/src/types.ts b/modules/expo-bluesky-video-compress/src/types.ts new file mode 100644 index 0000000000..df9b9ed1ac --- /dev/null +++ b/modules/expo-bluesky-video-compress/src/types.ts @@ -0,0 +1,45 @@ +export type CodecPreference = 'auto' | 'hevc' | 'h264' + +export type VideoMetadata = { + width: number + height: number + duration: number + bitrate: number + fileSize: number + mimeType: string + codec: string + hasAudio: boolean + frameRate: number + rotation: number + isHDR: boolean +} + +export type CompressOptions = { + targetBitrate?: number + maxSize?: number + codec?: CodecPreference + frameRateCap?: number +} + +export type CompressCallbacks = { + onProgress?: (progress: number) => void + signal?: AbortSignal +} + +export type CompressResult = { + uri: string + size: number + mimeType: string + width: number + height: number + duration: number + codec: 'h264' | 'hevc' +} + +export type NativeCompressOptions = { + targetBitrate: number + maxSize: number + codec: CodecPreference + frameRateCap: number + jobId: number +} diff --git a/src/analytics/metrics/types.ts b/src/analytics/metrics/types.ts index 8416a69b5e..3cb707733f 100644 --- a/src/analytics/metrics/types.ts +++ b/src/analytics/metrics/types.ts @@ -1365,6 +1365,25 @@ export type Events = { engine: string sourceBytes?: number } + // Native-only. Raw container metadata returned by the new module's probe() + // (bitrate, codec, HDR, frame rate, rotation, etc.). Fires once per upload + // between compressStarted and the compressSkipped/compressCompleted decision. + // The web (mediabunny) and legacy rn-compressor engines do not surface this. + 'video:upload:probed': { + uploadId: string + engine: string + mimeType: string + codec: string + width: number + height: number + duration: number + bitrate: number + fileSize: number + hasAudio: boolean + frameRate: number + rotation: number + isHDR: boolean + } 'video:upload:compressCompleted': { uploadId: string engine: string diff --git a/src/lib/media/video/compress.ts b/src/lib/media/video/compress.ts index dfb2e17b4b..72fb2a246c 100644 --- a/src/lib/media/video/compress.ts +++ b/src/lib/media/video/compress.ts @@ -2,7 +2,9 @@ import {getVideoMetaData, Video} from 'react-native-compressor' import {type ImagePickerAsset} from 'expo-image-picker' import {SUPPORTED_MIME_TYPES, type SupportedMimeTypes} from '#/lib/constants' -import {type CompressedVideo} from './types' +import {logger} from '#/logger' +import {probe} from '../../../../modules/expo-bluesky-video-compress' +import {type CompressedVideo, type ProbedMetadata} from './types' import {extToMime} from './util' const MIN_SIZE_FOR_COMPRESSION_BYTES = 25 * 1024 * 1024 // 25mb @@ -12,9 +14,20 @@ export async function compressVideo( opts?: { signal?: AbortSignal onProgress?: (progress: number) => void + onProbe?: (metadata: ProbedMetadata) => void }, ): Promise { - const {onProgress, signal} = opts || {} + const {onProgress, signal, onProbe} = opts || {} + + // Probe data is purely informational - fired into telemetry to validate + // future smart-skip thresholds. Failures must not block the upload. + if (onProbe && file.mimeType !== 'image/gif') { + try { + onProbe(await probe(file.uri)) + } catch (e) { + logger.debug('video probe failed', {safeMessage: e}) + } + } if (file.mimeType === 'image/gif') { // let's hope they're small enough that they don't need compression! diff --git a/src/lib/media/video/compress.web.ts b/src/lib/media/video/compress.web.ts index 62e8469351..7f96141b74 100644 --- a/src/lib/media/video/compress.web.ts +++ b/src/lib/media/video/compress.web.ts @@ -23,7 +23,7 @@ import { COMPRESSION_MIN_SIZE_BYTES, COMPRESSION_TARGET_BITRATE, } from './constants' -import {type CompressedVideo} from './types' +import {type CompressedVideo, type ProbedMetadata} from './types' // Codecs to try in order of preference // avc (H.264) is most compatible, vp9/vp8 are fallbacks for WebM @@ -34,9 +34,10 @@ export async function compressVideo( opts?: { signal?: AbortSignal onProgress?: (progress: number) => void + onProbe?: (metadata: ProbedMetadata) => void }, ): Promise { - const {onProgress, signal} = opts || {} + const {onProgress, signal, onProbe} = opts || {} logger.debug('compress: starting', { uri: asset.uri.slice(0, 50), @@ -49,6 +50,18 @@ export async function compressVideo( const isGif = blob.type === 'image/gif' const hasCodecs = hasWebCodecs() + // Probe pass: extract source metadata via mediabunny before any compression + // decision. Fires before doCompression so the probed event lands ahead of + // compressCompleted/compressSkipped in the funnel. GIFs and missing + // WebCodecs both skip - mediabunny needs a parseable container + decoder. + if (onProbe && !isGif && hasCodecs) { + try { + onProbe(await probeWithMediaBunny(blob)) + } catch (e) { + logger.debug('video probe failed', {safeMessage: e}) + } + } + logger.debug('compress: fetched blob', { size: blob.size, mimeType: blob.type, @@ -98,6 +111,44 @@ export async function compressVideo( } } +async function probeWithMediaBunny(blob: Blob): Promise { + const input = new Input({source: new BlobSource(blob), formats: ALL_FORMATS}) + try { + const videoTrack = await input.getPrimaryVideoTrack() + if (!videoTrack) { + throw new Error('No video track found') + } + const audioTrack = await input.getPrimaryAudioTrack() + const [codec, codedWidth, codedHeight, rotation, isHDR, stats, duration] = + await Promise.all([ + videoTrack.getCodec(), + videoTrack.getCodedWidth(), + videoTrack.getCodedHeight(), + videoTrack.getRotation(), + videoTrack.hasHighDynamicRange(), + // Sample a small fixed slice instead of scanning the whole file - we + // only want an approximate bitrate / frame rate for telemetry. + videoTrack.computePacketStats(100), + input.computeDuration(), + ]) + return { + mimeType: blob.type || 'video/mp4', + codec: codec ?? 'unknown', + width: codedWidth, + height: codedHeight, + duration, + bitrate: Math.round(stats.averageBitrate), + fileSize: blob.size, + hasAudio: audioTrack !== null, + frameRate: stats.averagePacketRate, + rotation, + isHDR, + } + } finally { + input.dispose() + } +} + async function findEncodableVideoCodec( width: number, height: number, diff --git a/src/lib/media/video/constants.ts b/src/lib/media/video/constants.ts index a328c64d68..60580efc8c 100644 --- a/src/lib/media/video/constants.ts +++ b/src/lib/media/video/constants.ts @@ -1,8 +1,10 @@ // Shared compression knobs. Mirrored between native (compress.ts) and web // (compress.web.ts) so both platforms produce videos with the same target. +// Target encode bitrate when we do compress. export const COMPRESSION_TARGET_BITRATE = 3_000_000 // 3 Mbps +// Output dimension cap when compressing, and skip threshold for source files. export const COMPRESSION_MAX_DIMENSION = 1920 -// Web skips compression entirely for files under this size; native applies its -// own threshold logic inside expo-bluesky-video-compress's probe step. +// Web only: files under this size skip compression entirely. Native applies +// its own threshold logic inside react-native-compressor. export const COMPRESSION_MIN_SIZE_BYTES = 25_000_000 diff --git a/src/lib/media/video/telemetry.ts b/src/lib/media/video/telemetry.ts index 7ef5a874ef..34eb431420 100644 --- a/src/lib/media/video/telemetry.ts +++ b/src/lib/media/video/telemetry.ts @@ -2,7 +2,10 @@ import {Platform} from 'react-native' import {type ImagePickerAsset} from 'expo-image-picker' import {nanoid} from 'nanoid/non-secure' -import {type VideoCompressSkipReason} from '#/lib/media/video/types' +import { + type ProbedMetadata, + type VideoCompressSkipReason, +} from '#/lib/media/video/types' import {Sentry} from '#/logger/sentry/lib' import {type Metrics} from '#/analytics/metrics' @@ -28,6 +31,7 @@ export type VideoTelemetry = { readonly engine: string picked: () => void compressStarted: () => void + probed: (metadata: ProbedMetadata) => void compressSkipped: (video: { size: number mimeType: string @@ -155,6 +159,24 @@ export function createVideoTelemetry({ }) }, + probed(metadata) { + metric('video:upload:probed', { + uploadId, + engine, + mimeType: metadata.mimeType, + codec: metadata.codec, + width: metadata.width, + height: metadata.height, + duration: metadata.duration, + bitrate: metadata.bitrate, + fileSize: metadata.fileSize, + hasAudio: metadata.hasAudio, + frameRate: metadata.frameRate, + rotation: metadata.rotation, + isHDR: metadata.isHDR, + }) + }, + compressSkipped({size, mimeType, skipReason}) { metric('video:upload:compressSkipped', { uploadId, diff --git a/src/lib/media/video/types.ts b/src/lib/media/video/types.ts index d22832794a..6825f03ba3 100644 --- a/src/lib/media/video/types.ts +++ b/src/lib/media/video/types.ts @@ -18,3 +18,20 @@ export type CompressedVideo = { // bytes were actually re-encoded. passthroughReason?: VideoCompressSkipReason } + +// Source container metadata read off the input before any encoding decision. +// Same shape across native (expo-bluesky-video-compress probe) and web +// (mediabunny Input + track inspection). Numbers are raw - no bucketing. +export type ProbedMetadata = { + mimeType: string + codec: string + width: number + height: number + duration: number + bitrate: number + fileSize: number + hasAudio: boolean + frameRate: number + rotation: number + isHDR: boolean +} diff --git a/src/view/com/composer/state/video.ts b/src/view/com/composer/state/video.ts index 6737d1f628..492eac1c5d 100644 --- a/src/view/com/composer/state/video.ts +++ b/src/view/com/composer/state/video.ts @@ -288,6 +288,7 @@ export async function processVideo( dispatch({type: 'update_progress', progress: trunc2dp(num), signal}) }, signal, + onProbe: metadata => telemetry.probed(metadata), }) } catch (e) { const message = getCompressErrorMessage(e, i18n)