diff --git a/modules/expo-bluesky-video-compress/android/build.gradle b/modules/expo-bluesky-video-compress/android/build.gradle new file mode 100644 index 0000000000..be96c40154 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/build.gradle @@ -0,0 +1,34 @@ +apply plugin: 'com.android.library' + +group = 'expo.modules.blueskyvideocompress' +version = '1.0.0' + +def expoModulesCorePlugin = new File(project(":expo-modules-core").projectDir.absolutePath, "ExpoModulesCorePlugin.gradle") +apply from: expoModulesCorePlugin +applyKotlinExpoModulesCorePlugin() +useCoreDependencies() +useExpoPublishing() + +buildscript { + ext.safeExtGet = { prop, fallback -> + rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback + } +} + +android { + compileSdkVersion safeExtGet("compileSdkVersion", 34) + + namespace "expo.modules.blueskyvideocompress" + defaultConfig { + minSdkVersion safeExtGet("minSdkVersion", 21) + targetSdkVersion safeExtGet("targetSdkVersion", 34) + versionCode 1 + versionName "1.0.0" + } + lintOptions { + abortOnError false + } +} + +dependencies { +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/CodecSelector.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/CodecSelector.kt new file mode 100644 index 0000000000..e06ccc99e2 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/CodecSelector.kt @@ -0,0 +1,64 @@ +package expo.modules.blueskyvideocompress + +import android.media.MediaCodecInfo +import android.media.MediaCodecList +import android.media.MediaFormat +import android.os.Build + +object CodecSelector { + // Source: https://github.com/numandev1/react-native-compressor/blob/f949b0868055178e7c8753e05202f784b1bcd589/android/src/main/java/com/reactnativecompressor/Video/VideoCompressor/compressor/Compressor.kt#L500 + private val AVC_DENYLIST = setOf( + "c2.qti.avc.encoder" + ) + + private val SOFTWARE_PREFIXES = listOf( + "OMX.google.", + "c2.android.", + "c2.google." + ) + + data class EncoderInfo( + val name: String, + val mime: String, + val isHardware: Boolean + ) + + fun selectEncoder(preferHevc: Boolean): EncoderInfo? { + if (preferHevc) { + val hevc = findEncoder(MediaFormat.MIMETYPE_VIDEO_HEVC, requireHardware = true) + if (hevc != null) return hevc + } + findEncoder(MediaFormat.MIMETYPE_VIDEO_AVC, requireHardware = true)?.let { return it } + return findEncoder(MediaFormat.MIMETYPE_VIDEO_AVC, requireHardware = false) + } + + fun findEncoder(mime: String, requireHardware: Boolean): EncoderInfo? { + val codecList = MediaCodecList(MediaCodecList.REGULAR_CODECS) + val candidates = codecList.codecInfos + .filter { it.isEncoder } + .filter { it.supportedTypes.any { t -> t.equals(mime, ignoreCase = true) } } + .filter { !it.name.contains("secure", ignoreCase = true) } + .filter { !(mime == MediaFormat.MIMETYPE_VIDEO_AVC && AVC_DENYLIST.contains(it.name)) } + + val hardware = candidates.filter { isHardware(it) } + val selected = if (requireHardware) { + hardware.firstOrNull() + } else { + hardware.firstOrNull() ?: candidates.firstOrNull() + } + selected ?: return null + return EncoderInfo( + name = selected.name, + mime = mime, + isHardware = isHardware(selected) + ) + } + + private fun isHardware(info: MediaCodecInfo): Boolean { + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + info.isHardwareAccelerated + } else { + SOFTWARE_PREFIXES.none { info.name.startsWith(it, ignoreCase = true) } + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/ExpoBlueskyVideoCompressModule.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/ExpoBlueskyVideoCompressModule.kt new file mode 100644 index 0000000000..345deedd66 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/ExpoBlueskyVideoCompressModule.kt @@ -0,0 +1,62 @@ +package expo.modules.blueskyvideocompress + +import expo.modules.kotlin.modules.Module +import expo.modules.kotlin.modules.ModuleDefinition + +class ExpoBlueskyVideoCompressModule : Module() { + private var currentCompressor: VideoCompressor? = null + + override fun definition() = ModuleDefinition { + Name("ExpoBlueskyVideoCompress") + + Events("onProgress") + + AsyncFunction("probe") { uri: String -> + val context = appContext.reactContext + ?: throw Error("React context is null") + return@AsyncFunction VideoProber.probe(context, uri) + } + + AsyncFunction("compress") { uri: String, options: Map -> + val context = appContext.reactContext + ?: throw Error("React context is null") + val targetBitrate = (options["targetBitrate"] as? Number)?.toInt() ?: 0 + val maxSize = (options["maxSize"] as? Number)?.toInt() ?: 1920 + val codecPref = (options["codec"] as? String) ?: "auto" + val frameRateCap = (options["frameRateCap"] as? Number)?.toInt() ?: 30 + val jobId = (options["jobId"] as? Number)?.toInt() ?: 0 + + val compressor = VideoCompressor( + context = context, + uriString = uri, + targetBitrate = targetBitrate, + maxSize = maxSize, + codecPref = codecPref, + frameRateCap = frameRateCap, + jobId = jobId, + onProgress = { id, progress -> + sendEvent("onProgress", mapOf( + "id" to id, + "progress" to progress + )) + } + ) + + currentCompressor = compressor + + try { + val result = compressor.compress() + currentCompressor = null + return@AsyncFunction result + } catch (e: Exception) { + currentCompressor = null + throw e + } + } + + Function("cancel") { + currentCompressor?.cancel() + currentCompressor = null + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt new file mode 100644 index 0000000000..e2eee4768c --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt @@ -0,0 +1,96 @@ +package expo.modules.blueskyvideocompress + +import android.opengl.EGL14 +import android.opengl.EGLConfig +import android.opengl.EGLContext +import android.opengl.EGLDisplay +import android.opengl.EGLExt +import android.opengl.EGLSurface +import android.view.Surface + +class InputSurface(private val surface: Surface) { + private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY + private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT + private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE + + init { + eglSetup() + } + + private fun eglSetup() { + eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY) + if (eglDisplay === EGL14.EGL_NO_DISPLAY) { + throw RuntimeException("unable to get EGL14 display") + } + + val version = IntArray(2) + if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) { + throw RuntimeException("unable to initialize EGL14") + } + + val attribList = intArrayOf( + EGL14.EGL_RED_SIZE, 8, + EGL14.EGL_GREEN_SIZE, 8, + EGL14.EGL_BLUE_SIZE, 8, + EGL14.EGL_ALPHA_SIZE, 8, + EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, + EGL14.EGL_SURFACE_TYPE, EGL14.EGL_WINDOW_BIT, + EGL14.EGL_NONE + ) + val configs = arrayOfNulls(1) + val numConfigs = IntArray(1) + EGL14.eglChooseConfig(eglDisplay, attribList, 0, configs, 0, 1, numConfigs, 0) + checkEglError("eglChooseConfig") + + val contextAttribs = intArrayOf( + EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, + EGL14.EGL_NONE + ) + eglContext = EGL14.eglCreateContext( + eglDisplay, configs[0], EGL14.EGL_NO_CONTEXT, contextAttribs, 0 + ) + checkEglError("eglCreateContext") + + val surfaceAttribs = intArrayOf(EGL14.EGL_NONE) + eglSurface = EGL14.eglCreateWindowSurface( + eglDisplay, configs[0], surface, surfaceAttribs, 0 + ) + checkEglError("eglCreateWindowSurface") + } + + fun makeCurrent() { + EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext) + checkEglError("eglMakeCurrent") + } + + fun swapBuffers(): Boolean { + return EGL14.eglSwapBuffers(eglDisplay, eglSurface) + } + + fun setPresentationTime(nsecs: Long) { + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, nsecs) + } + + fun release() { + if (eglDisplay !== EGL14.EGL_NO_DISPLAY) { + EGL14.eglMakeCurrent( + eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT + ) + EGL14.eglDestroySurface(eglDisplay, eglSurface) + EGL14.eglDestroyContext(eglDisplay, eglContext) + EGL14.eglReleaseThread() + EGL14.eglTerminate(eglDisplay) + } + surface.release() + eglDisplay = EGL14.EGL_NO_DISPLAY + eglContext = EGL14.EGL_NO_CONTEXT + eglSurface = EGL14.EGL_NO_SURFACE + } + + private fun checkEglError(msg: String) { + val error = EGL14.eglGetError() + if (error != EGL14.EGL_SUCCESS) { + throw RuntimeException("$msg: EGL error: 0x${Integer.toHexString(error)}") + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt new file mode 100644 index 0000000000..fdf8c6eca2 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt @@ -0,0 +1,61 @@ +package expo.modules.blueskyvideocompress + +import android.graphics.SurfaceTexture +import android.os.Handler +import android.os.HandlerThread +import android.view.Surface + +class OutputSurface : SurfaceTexture.OnFrameAvailableListener { + private val renderer = TextureRenderer() + private var surfaceTexture: SurfaceTexture? = null + private val stMatrix = FloatArray(16) + private val callbackThread = HandlerThread("OutputSurfaceCallbacks") + val surface: Surface + + @Volatile + private var frameAvailable = false + private val frameSyncObject = Object() + + init { + renderer.surfaceCreated() + callbackThread.start() + val handler = Handler(callbackThread.looper) + surfaceTexture = SurfaceTexture(renderer.getTextureId()).also { + it.setOnFrameAvailableListener(this, handler) + } + surface = Surface(surfaceTexture) + } + + fun release() { + surface.release() + surfaceTexture?.release() + surfaceTexture = null + callbackThread.quitSafely() + } + + fun awaitNewImage() { + val timeoutMs = 2500L + synchronized(frameSyncObject) { + while (!frameAvailable) { + frameSyncObject.wait(timeoutMs) + if (!frameAvailable) { + throw RuntimeException("Surface frame wait timed out") + } + } + frameAvailable = false + } + surfaceTexture!!.updateTexImage() + } + + fun drawImage() { + surfaceTexture!!.getTransformMatrix(stMatrix) + renderer.drawFrame(stMatrix) + } + + override fun onFrameAvailable(st: SurfaceTexture) { + synchronized(frameSyncObject) { + frameAvailable = true + frameSyncObject.notifyAll() + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt new file mode 100644 index 0000000000..a0cb25bce1 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt @@ -0,0 +1,156 @@ +package expo.modules.blueskyvideocompress + +import android.opengl.GLES11Ext +import android.opengl.GLES20 +import android.opengl.Matrix +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.FloatBuffer + +class TextureRenderer { + companion object { + private const val FLOAT_SIZE_BYTES = 4 + private const val STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES + private const val POS_OFFSET = 0 + private const val UV_OFFSET = 3 + + private val VERTICES = floatArrayOf( + -1.0f, -1.0f, 0f, 0f, 0f, + 1.0f, -1.0f, 0f, 1f, 0f, + -1.0f, 1.0f, 0f, 0f, 1f, + 1.0f, 1.0f, 0f, 1f, 1f, + ) + + private const val VERTEX_SHADER = """ + uniform mat4 uMVPMatrix; + uniform mat4 uSTMatrix; + attribute vec4 aPosition; + attribute vec4 aTextureCoord; + varying vec2 vTextureCoord; + void main() { + gl_Position = uMVPMatrix * aPosition; + vTextureCoord = (uSTMatrix * aTextureCoord).xy; + } + """ + + private const val FRAGMENT_SHADER = """ + #extension GL_OES_EGL_image_external : require + precision mediump float; + varying vec2 vTextureCoord; + uniform samplerExternalOES sTexture; + void main() { + gl_FragColor = texture2D(sTexture, vTextureCoord); + } + """ + } + + private val vertices: FloatBuffer = + ByteBuffer.allocateDirect(VERTICES.size * FLOAT_SIZE_BYTES) + .order(ByteOrder.nativeOrder()) + .asFloatBuffer() + .apply { put(VERTICES); position(0) } + + private val mvpMatrix = FloatArray(16) + private var program = 0 + private var textureId = -1 + private var uMVPMatrixHandle = 0 + private var uSTMatrixHandle = 0 + private var aPositionHandle = 0 + private var aTextureCoordHandle = 0 + + init { + Matrix.setIdentityM(mvpMatrix, 0) + } + + fun getTextureId(): Int = textureId + + fun surfaceCreated() { + program = createProgram(VERTEX_SHADER, FRAGMENT_SHADER) + + aPositionHandle = GLES20.glGetAttribLocation(program, "aPosition") + aTextureCoordHandle = GLES20.glGetAttribLocation(program, "aTextureCoord") + uMVPMatrixHandle = GLES20.glGetUniformLocation(program, "uMVPMatrix") + uSTMatrixHandle = GLES20.glGetUniformLocation(program, "uSTMatrix") + + val textures = IntArray(1) + GLES20.glGenTextures(1, textures, 0) + textureId = textures[0] + + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId) + GLES20.glTexParameterf( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR.toFloat() + ) + GLES20.glTexParameterf( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat() + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE + ) + } + + fun drawFrame(stMatrix: FloatArray) { + GLES20.glClearColor(0f, 0f, 0f, 1f) + GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT or GLES20.GL_COLOR_BUFFER_BIT) + + GLES20.glUseProgram(program) + GLES20.glActiveTexture(GLES20.GL_TEXTURE0) + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId) + + vertices.position(POS_OFFSET) + GLES20.glVertexAttribPointer( + aPositionHandle, 3, GLES20.GL_FLOAT, false, STRIDE_BYTES, vertices + ) + GLES20.glEnableVertexAttribArray(aPositionHandle) + + vertices.position(UV_OFFSET) + GLES20.glVertexAttribPointer( + aTextureCoordHandle, 2, GLES20.GL_FLOAT, false, STRIDE_BYTES, vertices + ) + GLES20.glEnableVertexAttribArray(aTextureCoordHandle) + + GLES20.glUniformMatrix4fv(uMVPMatrixHandle, 1, false, mvpMatrix, 0) + GLES20.glUniformMatrix4fv(uSTMatrixHandle, 1, false, stMatrix, 0) + + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) + GLES20.glDisableVertexAttribArray(aPositionHandle) + GLES20.glDisableVertexAttribArray(aTextureCoordHandle) + } + + private fun createProgram(vertexSource: String, fragmentSource: String): Int { + val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource) + val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource) + val program = GLES20.glCreateProgram() + GLES20.glAttachShader(program, vertexShader) + GLES20.glAttachShader(program, fragmentShader) + GLES20.glLinkProgram(program) + val linkStatus = IntArray(1) + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0) + if (linkStatus[0] != GLES20.GL_TRUE) { + val log = GLES20.glGetProgramInfoLog(program) + GLES20.glDeleteProgram(program) + throw RuntimeException("Could not link program: $log") + } + return program + } + + private fun loadShader(type: Int, source: String): Int { + val shader = GLES20.glCreateShader(type) + GLES20.glShaderSource(shader, source) + GLES20.glCompileShader(shader) + val compiled = IntArray(1) + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0) + if (compiled[0] == 0) { + val log = GLES20.glGetShaderInfoLog(shader) + GLES20.glDeleteShader(shader) + throw RuntimeException("Could not compile shader $type: $log") + } + return shader + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt new file mode 100644 index 0000000000..dc5ee5a41c --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt @@ -0,0 +1,372 @@ +package expo.modules.blueskyvideocompress + +import android.content.Context +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaExtractor +import android.media.MediaFormat +import android.media.MediaMuxer +import android.net.Uri +import android.os.Build +import android.util.Log +import java.io.File +import java.nio.ByteBuffer + +class VideoCompressor( + private val context: Context, + private val uriString: String, + private val targetBitrate: Int, + private val maxSize: Int, + private val codecPref: String, + private val frameRateCap: Int, + private val jobId: Int, + private val onProgress: (Int, Double) -> Unit +) { + companion object { + private const val TAG = "BskyVideoCompress" + private const val TIMEOUT_DEQUEUE = 100L + private const val I_FRAME_INTERVAL = 3 + } + + @Volatile + private var isCancelled = false + + fun cancel() { + isCancelled = true + } + + fun compress(): Map { + // 'auto' targets h264 — server pipeline is HLS, which favors h264 (HEVC needs + // fMP4 segments + commercial licensing). HEVC remains opt-in via codec: 'hevc'. + val tryHevc = codecPref == "hevc" + + if (tryHevc) { + try { + return doCompress(useHevc = true, allowSoftwareFallback = false) + } catch (e: Exception) { + if (codecPref == "hevc" || isCancelled) throw e + Log.w(TAG, "HEVC encode failed, falling back to h264", e) + } + } + return doCompress(useHevc = false, allowSoftwareFallback = true) + } + + private fun doCompress(useHevc: Boolean, allowSoftwareFallback: Boolean): Map { + val mime = if (useHevc) MediaFormat.MIMETYPE_VIDEO_HEVC else MediaFormat.MIMETYPE_VIDEO_AVC + val encoderInfo = CodecSelector.findEncoder(mime, requireHardware = !allowSoftwareFallback) + ?: throw RuntimeException("No encoder for $mime") + + try { + return runPipeline(encoderInfo, useHevc) + } catch (e: Exception) { + if (!allowSoftwareFallback || isCancelled || !encoderInfo.isHardware) throw e + Log.w(TAG, "Hardware encoder ${encoderInfo.name} failed, trying software", e) + val sw = CodecSelector.findEncoder(mime, requireHardware = false) + ?.takeIf { !it.isHardware } + ?: throw e + return runPipeline(sw, useHevc) + } + } + + private fun runPipeline( + encoderInfo: CodecSelector.EncoderInfo, + useHevc: Boolean + ): Map { + val outputFile = File(context.cacheDir, "${System.currentTimeMillis()}.mp4") + + var extractor: MediaExtractor? = null + var muxer: MediaMuxer? = null + var encoder: MediaCodec? = null + var decoder: MediaCodec? = null + var inputSurface: InputSurface? = null + var outputSurface: OutputSurface? = null + var muxerStarted = false + var outputDims = Pair(0, 0) + var durationUs = 0L + + try { + val uri = Uri.parse(uriString) + extractor = MediaExtractor() + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + extractor.setDataSource(context, uri, null) + } else { + extractor.setDataSource(uriString) + } + + var videoTrackIndex = -1 + var audioTrackIndex = -1 + var videoFormat: MediaFormat? = null + var audioFormat: MediaFormat? = null + + for (i in 0 until extractor.trackCount) { + val format = extractor.getTrackFormat(i) + val trackMime = format.getString(MediaFormat.KEY_MIME) ?: continue + if (trackMime.startsWith("video/") && videoTrackIndex == -1) { + videoTrackIndex = i + videoFormat = format + } else if (trackMime.startsWith("audio/") && audioTrackIndex == -1) { + audioTrackIndex = i + audioFormat = format + } + } + + if (videoTrackIndex == -1 || videoFormat == null) { + throw RuntimeException("No video track found") + } + + val sourceWidth = videoFormat.getInteger(MediaFormat.KEY_WIDTH) + val sourceHeight = videoFormat.getInteger(MediaFormat.KEY_HEIGHT) + val rotation = if (videoFormat.containsKey(MediaFormat.KEY_ROTATION)) { + videoFormat.getInteger(MediaFormat.KEY_ROTATION) + } else 0 + durationUs = if (videoFormat.containsKey(MediaFormat.KEY_DURATION)) { + videoFormat.getLong(MediaFormat.KEY_DURATION) + } else 0L + val sourceFps = if (videoFormat.containsKey(MediaFormat.KEY_FRAME_RATE)) { + videoFormat.getInteger(MediaFormat.KEY_FRAME_RATE) + } else 30 + + outputDims = calculateOutputDims(sourceWidth, sourceHeight, rotation, maxSize) + val shouldPassthroughAudio = audioFormat != null && canPassthroughAudio(audioFormat) + muxer = MediaMuxer(outputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) + + val effectiveBitrate = if (targetBitrate > 0) { + targetBitrate + } else if (useHevc) 2_500_000 else 3_000_000 + + val encoderFormat = MediaFormat.createVideoFormat( + encoderInfo.mime, outputDims.first, outputDims.second + ).apply { + setInteger( + MediaFormat.KEY_COLOR_FORMAT, + MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface + ) + setInteger(MediaFormat.KEY_BIT_RATE, effectiveBitrate) + setInteger( + MediaFormat.KEY_BITRATE_MODE, + MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR + ) + setInteger(MediaFormat.KEY_FRAME_RATE, frameRateCap) + setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + setInteger(MediaFormat.KEY_PRIORITY, 0) + setInteger(MediaFormat.KEY_OPERATING_RATE, frameRateCap) + if (useHevc) { + setInteger( + MediaFormat.KEY_PROFILE, + MediaCodecInfo.CodecProfileLevel.HEVCProfileMain + ) + } else { + setInteger( + MediaFormat.KEY_PROFILE, + MediaCodecInfo.CodecProfileLevel.AVCProfileHigh + ) + setInteger( + MediaFormat.KEY_LEVEL, + MediaCodecInfo.CodecProfileLevel.AVCLevel41 + ) + } + } + } + + encoder = MediaCodec.createByCodecName(encoderInfo.name) + encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + val encoderInputSurface = encoder.createInputSurface() + inputSurface = InputSurface(encoderInputSurface) + inputSurface.makeCurrent() + outputSurface = OutputSurface() + encoder.start() + + decoder = MediaCodec.createDecoderByType( + videoFormat.getString(MediaFormat.KEY_MIME) ?: "video/avc" + ) + decoder.configure(videoFormat, outputSurface.surface, null, 0) + decoder.start() + extractor.selectTrack(videoTrackIndex) + + val frameDropEnabled = sourceFps > frameRateCap + val targetFrameIntervalUs = if (frameDropEnabled) 1_000_000L / frameRateCap else 0L + var nextTargetPtsUs = 0L + + var muxerVideoTrack = -1 + var muxerAudioTrack = -1 + + val bufferInfo = MediaCodec.BufferInfo() + var inputDone = false + var decoderDone = false + var outputDone = false + var lastProgressMs = 0L + + while (!outputDone && !isCancelled) { + if (!inputDone) { + val idx = decoder.dequeueInputBuffer(TIMEOUT_DEQUEUE) + if (idx >= 0) { + val buf = decoder.getInputBuffer(idx) + if (buf != null) { + val sz = extractor.readSampleData(buf, 0) + if (sz < 0) { + decoder.queueInputBuffer( + idx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM + ) + inputDone = true + } else { + decoder.queueInputBuffer(idx, 0, sz, extractor.sampleTime, 0) + extractor.advance() + } + } + } + } + + if (!decoderDone) { + val status = decoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_DEQUEUE) + if (status >= 0) { + val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + val shouldRender = if (isEos) { + false + } else if (frameDropEnabled) { + if (bufferInfo.presentationTimeUs >= nextTargetPtsUs) { + nextTargetPtsUs = bufferInfo.presentationTimeUs + targetFrameIntervalUs + true + } else false + } else true + decoder.releaseOutputBuffer(status, shouldRender) + if (shouldRender) { + outputSurface.awaitNewImage() + outputSurface.drawImage() + inputSurface.setPresentationTime(bufferInfo.presentationTimeUs * 1000) + inputSurface.swapBuffers() + } + if (isEos) { + encoder.signalEndOfInputStream() + decoderDone = true + } + } + } + + var encoderDrained = false + while (!outputDone && !isCancelled && !encoderDrained) { + val encIdx = encoder.dequeueOutputBuffer(bufferInfo, 0) + when { + encIdx == MediaCodec.INFO_TRY_AGAIN_LATER -> encoderDrained = true + encIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + if (!muxerStarted) { + muxerVideoTrack = muxer.addTrack(encoder.outputFormat) + if (audioTrackIndex >= 0 && shouldPassthroughAudio && audioFormat != null) { + muxerAudioTrack = muxer.addTrack(audioFormat) + } + muxer.start() + muxerStarted = true + } + } + encIdx >= 0 -> { + val data = encoder.getOutputBuffer(encIdx) + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { + bufferInfo.size = 0 + } + if (data != null && bufferInfo.size > 0 && muxerStarted) { + muxer.writeSampleData(muxerVideoTrack, data, bufferInfo) + } + val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + encoder.releaseOutputBuffer(encIdx, false) + if (isEos) { + outputDone = true + } else if (durationUs > 0) { + val now = System.currentTimeMillis() + if (now - lastProgressMs >= 100) { + lastProgressMs = now + val p = (bufferInfo.presentationTimeUs.toDouble() / durationUs) + .coerceIn(0.0, 1.0) + onProgress(jobId, p) + } + } + } + } + } + } + + if (audioTrackIndex >= 0 && muxerAudioTrack >= 0 && muxerStarted && !isCancelled) { + passthroughAudio(audioTrackIndex, muxer, muxerAudioTrack) + } + } finally { + try { decoder?.stop() } catch (_: Exception) {} + try { decoder?.release() } catch (_: Exception) {} + try { encoder?.stop() } catch (_: Exception) {} + try { encoder?.release() } catch (_: Exception) {} + try { outputSurface?.release() } catch (_: Exception) {} + try { inputSurface?.release() } catch (_: Exception) {} + try { extractor?.release() } catch (_: Exception) {} + try { + if (muxerStarted) muxer?.stop() + muxer?.release() + } catch (_: Exception) {} + } + + if (isCancelled) { + outputFile.delete() + throw RuntimeException("Compression cancelled") + } + + onProgress(jobId, 1.0) + val durationSeconds = durationUs / 1_000_000.0 + + return mapOf( + "uri" to "file://${outputFile.absolutePath}", + "size" to outputFile.length(), + "mimeType" to "video/mp4", + "width" to outputDims.first, + "height" to outputDims.second, + "duration" to durationSeconds, + "codec" to if (useHevc) "hevc" else "h264" + ) + } + + private fun passthroughAudio(audioTrackIndex: Int, muxer: MediaMuxer, muxerAudioTrack: Int) { + val audioExtractor = MediaExtractor() + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + audioExtractor.setDataSource(context, Uri.parse(uriString), null) + } else { + audioExtractor.setDataSource(uriString) + } + audioExtractor.selectTrack(audioTrackIndex) + audioExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC) + + val buffer = ByteBuffer.allocate(256 * 1024) + val info = MediaCodec.BufferInfo() + try { + while (!isCancelled) { + val sz = audioExtractor.readSampleData(buffer, 0) + if (sz < 0) break + info.offset = 0 + info.size = sz + info.presentationTimeUs = audioExtractor.sampleTime + info.flags = audioExtractor.sampleFlags + muxer.writeSampleData(muxerAudioTrack, buffer, info) + audioExtractor.advance() + } + } finally { + audioExtractor.release() + } + } + + private fun canPassthroughAudio(format: MediaFormat): Boolean { + val mime = format.getString(MediaFormat.KEY_MIME) ?: return false + return mime == MediaFormat.MIMETYPE_AUDIO_AAC + } + + private fun calculateOutputDims(srcW: Int, srcH: Int, rotation: Int, maxSize: Int): Pair { + val isRotated = rotation == 90 || rotation == 270 + val displayW = if (isRotated) srcH else srcW + val displayH = if (isRotated) srcW else srcH + val scale: Float = when { + displayW <= maxSize && displayH <= maxSize -> 1.0f + displayW > displayH -> maxSize.toFloat() / displayW + else -> maxSize.toFloat() / displayH + } + return Pair( + roundToEven((displayW * scale).toInt()), + roundToEven((displayH * scale).toInt()) + ) + } + + private fun roundToEven(v: Int): Int = if (v % 2 == 0) v else v - 1 +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt new file mode 100644 index 0000000000..9d3d658e05 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt @@ -0,0 +1,110 @@ +package expo.modules.blueskyvideocompress + +import android.content.Context +import android.media.MediaExtractor +import android.media.MediaFormat +import android.media.MediaMetadataRetriever +import android.net.Uri + +object VideoProber { + fun probe(context: Context, uriString: String): Map { + val uri = Uri.parse(uriString) + val retriever = MediaMetadataRetriever() + + try { + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + retriever.setDataSource(context, uri) + } else { + retriever.setDataSource(uriString) + } + + val width = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH) + ?.toIntOrNull() ?: 0 + val height = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT) + ?.toIntOrNull() ?: 0 + val durationMs = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION) + ?.toLongOrNull() ?: 0L + val rotation = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION) + ?.toIntOrNull() ?: 0 + val bitrate = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_BITRATE) + ?.toIntOrNull() ?: 0 + val hasAudio = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO) + ?.equals("yes") ?: false + val frameRate = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE) + ?.toFloatOrNull() ?: 0f + + val fileSize = getFileSize(context, uriString) + + val extractor = MediaExtractor() + var codec = "unknown" + var mimeType = "video/mp4" + var extractedFrameRate = frameRate + + try { + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + extractor.setDataSource(context, uri, null) + } else { + extractor.setDataSource(uriString) + } + + for (i in 0 until extractor.trackCount) { + val format = extractor.getTrackFormat(i) + val mime = format.getString(MediaFormat.KEY_MIME) + if (mime?.startsWith("video/") == true) { + mimeType = mime + codec = mime.removePrefix("video/") + if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) { + extractedFrameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE).toFloat() + } + break + } + } + } finally { + extractor.release() + } + + val durationSeconds = durationMs / 1000.0 + val effectiveBitrate = if (bitrate > 0) { + bitrate + } else if (durationSeconds > 0 && fileSize > 0) { + (fileSize * 8 / durationSeconds).toInt() + } else { + 0 + } + + return mapOf( + "width" to width, + "height" to height, + "duration" to durationSeconds, + "bitrate" to effectiveBitrate, + "fileSize" to fileSize, + "mimeType" to mimeType, + "codec" to codec, + "hasAudio" to hasAudio, + "frameRate" to extractedFrameRate.toDouble(), + "rotation" to rotation + ) + } finally { + retriever.release() + } + } + + private fun getFileSize(context: Context, uriString: String): Long { + return try { + if (uriString.startsWith("content://")) { + context.contentResolver.openFileDescriptor(Uri.parse(uriString), "r")?.use { + it.statSize + } ?: 0L + } else { + val path = if (uriString.startsWith("file://")) { + uriString.removePrefix("file://") + } else { + uriString + } + java.io.File(path).length() + } + } catch (_: Exception) { + 0L + } + } +} diff --git a/modules/expo-bluesky-video-compress/expo-module.config.json b/modules/expo-bluesky-video-compress/expo-module.config.json new file mode 100644 index 0000000000..c1c24a480d --- /dev/null +++ b/modules/expo-bluesky-video-compress/expo-module.config.json @@ -0,0 +1,9 @@ +{ + "platforms": ["ios", "android"], + "ios": { + "modules": ["ExpoBlueskyVideoCompressModule"] + }, + "android": { + "modules": ["expo.modules.blueskyvideocompress.ExpoBlueskyVideoCompressModule"] + } +} diff --git a/modules/expo-bluesky-video-compress/index.ts b/modules/expo-bluesky-video-compress/index.ts new file mode 100644 index 0000000000..2dae0b4cfc --- /dev/null +++ b/modules/expo-bluesky-video-compress/index.ts @@ -0,0 +1,87 @@ +import {type EventSubscription} from 'expo-modules-core' + +import NativeModule from './src/ExpoBlueskyVideoCompressModule' +import { + type CodecPreference, + type CompressCallbacks, + type CompressOptions, + type CompressResult, + type VideoMetadata, +} from './src/types' + +export type { + CodecPreference, + CompressCallbacks, + CompressOptions, + CompressResult, + VideoMetadata, +} + +class AbortError extends Error { + name = 'AbortError' + constructor() { + super('Aborted') + } +} + +let jobIdCounter = 0 + +export function probe(uri: string): Promise { + return NativeModule.probe(uri) +} + +export function compress( + uri: string, + options: CompressOptions = {}, + callbacks?: CompressCallbacks, +): Promise { + const jobId = ++jobIdCounter + let subscription: EventSubscription | undefined + + if (callbacks?.signal?.aborted) { + return Promise.reject(new AbortError()) + } + + const nativeOptions = { + targetBitrate: options.targetBitrate ?? 0, + maxSize: options.maxSize ?? 1920, + codec: options.codec ?? 'auto', + frameRateCap: options.frameRateCap ?? 30, + jobId, + } + + return new Promise((resolve, reject) => { + if (callbacks?.onProgress) { + subscription = NativeModule.addListener( + 'onProgress', + (event: {id: number; progress: number}) => { + if (event.id === jobId) { + callbacks.onProgress!(event.progress) + } + }, + ) + } + + const abortHandler = () => { + NativeModule.cancel() + subscription?.remove() + reject(new AbortError()) + } + + if (callbacks?.signal) { + callbacks.signal.addEventListener('abort', abortHandler, {once: true}) + } + + NativeModule.compress(uri, nativeOptions) + .then(result => { + callbacks?.signal?.removeEventListener('abort', abortHandler) + subscription?.remove() + resolve(result) + }) + .catch((error: unknown) => { + callbacks?.signal?.removeEventListener('abort', abortHandler) + subscription?.remove() + reject(error instanceof Error ? error : new Error(String(error))) + }) + }) +} diff --git a/modules/expo-bluesky-video-compress/ios/CodecCapability.swift b/modules/expo-bluesky-video-compress/ios/CodecCapability.swift new file mode 100644 index 0000000000..df95d9b7e8 --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/CodecCapability.swift @@ -0,0 +1,21 @@ +import VideoToolbox + +enum CodecCapability { + static let isHardwareHEVCEncodeAvailable: Bool = { + var encoderListCF: CFArray? + let status = VTCopyVideoEncoderList(nil, &encoderListCF) + guard status == noErr, let encoderList = encoderListCF as? [[String: Any]] else { + return false + } + return encoderList.contains { encoder in + guard let codecTypeValue = encoder[kVTVideoEncoderList_CodecType as String] as? Int, + codecTypeValue == Int(kCMVideoCodecType_HEVC) else { + return false + } + if let isHardware = encoder[kVTVideoEncoderList_IsHardwareAccelerated as String] as? Bool { + return isHardware + } + return true + } + }() +} diff --git a/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompress.podspec b/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompress.podspec new file mode 100644 index 0000000000..5bca8f0812 --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompress.podspec @@ -0,0 +1,20 @@ +Pod::Spec.new do |s| + s.name = 'ExpoBlueskyVideoCompress' + s.version = '1.0.0' + s.summary = 'Hardware-accelerated video compression for Bluesky' + s.description = 'Hardware-accelerated h264/HEVC video compression using AVAssetReader/Writer and VideoToolbox on iOS' + s.author = '' + s.homepage = 'https://github.com/bluesky-social/social-app' + s.platforms = { :ios => '15.1' } + s.source = { git: '' } + s.static_framework = true + + s.dependency 'ExpoModulesCore' + + s.pod_target_xcconfig = { + 'DEFINES_MODULE' => 'YES', + 'SWIFT_COMPILATION_MODE' => 'wholemodule' + } + + s.source_files = "**/*.{h,m,mm,swift,hpp,cpp}" +end diff --git a/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompressModule.swift b/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompressModule.swift new file mode 100644 index 0000000000..d23dabb001 --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/ExpoBlueskyVideoCompressModule.swift @@ -0,0 +1,57 @@ +import AVFoundation +import ExpoModulesCore + +public class ExpoBlueskyVideoCompressModule: Module { + private var currentCompressor: VideoCompressor? + + public func definition() -> ModuleDefinition { + Name("ExpoBlueskyVideoCompress") + + Events("onProgress") + + AsyncFunction("probe") { (uri: String) -> [String: Any] in + let url = URL(string: uri) ?? URL(fileURLWithPath: uri) + return try await VideoProber.probe(url: url) + } + + AsyncFunction("compress") { (uri: String, options: [String: Any]) -> [String: Any] in + let url = URL(string: uri) ?? URL(fileURLWithPath: uri) + let targetBitrate = options["targetBitrate"] as? Int ?? 0 + let maxSize = options["maxSize"] as? Int ?? 1920 + let codecPref = options["codec"] as? String ?? "auto" + let frameRateCap = options["frameRateCap"] as? Int ?? 30 + let jobId = options["jobId"] as? Int ?? 0 + + let compressor = VideoCompressor( + url: url, + targetBitrate: targetBitrate, + maxSize: maxSize, + codecPref: codecPref, + frameRateCap: frameRateCap, + jobId: jobId, + onProgress: { [weak self] id, progress in + self?.sendEvent("onProgress", [ + "id": id, + "progress": progress, + ]) + } + ) + + self.currentCompressor = compressor + + do { + let result = try await compressor.compress() + self.currentCompressor = nil + return result + } catch { + self.currentCompressor = nil + throw error + } + } + + Function("cancel") { + self.currentCompressor?.cancel() + self.currentCompressor = nil + } + } +} diff --git a/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift b/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift new file mode 100644 index 0000000000..baaf0f7a51 --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift @@ -0,0 +1,397 @@ +import AVFoundation +import VideoToolbox + +class VideoCompressor { + private let url: URL + private let targetBitrate: Int + private let maxSize: Int + private let codecPref: String + private let frameRateCap: Int + private let jobId: Int + private let onProgress: (Int, Double) -> Void + private var isCancelled = false + + init( + url: URL, + targetBitrate: Int, + maxSize: Int, + codecPref: String, + frameRateCap: Int, + jobId: Int, + onProgress: @escaping (Int, Double) -> Void + ) { + self.url = url + self.targetBitrate = targetBitrate + self.maxSize = maxSize + self.codecPref = codecPref + self.frameRateCap = frameRateCap + self.jobId = jobId + self.onProgress = onProgress + } + + func cancel() { + isCancelled = true + } + + func compress() async throws -> [String: Any] { + let asset = AVURLAsset( + url: url, + options: [AVURLAssetPreferPreciseDurationAndTimingKey: true] + ) + let duration = try await asset.load(.duration) + let totalSeconds = CMTimeGetSeconds(duration) + guard totalSeconds > 0 else { throw err("Invalid video duration", code: 2) } + + let videoTracks = try await asset.loadTracks(withMediaType: .video) + guard let videoTrack = videoTracks.first else { + throw err("No video track found", code: 1) + } + + let naturalSize = try await videoTrack.load(.naturalSize) + let preferredTransform = try await videoTrack.load(.preferredTransform) + + let rotatedRect = CGRect(origin: .zero, size: naturalSize).applying(preferredTransform) + let displaySize = CGSize( + width: abs(rotatedRect.width), + height: abs(rotatedRect.height) + ) + let outputSize = scaleEvenly(displaySize: displaySize, maxSize: maxSize) + + let audioTracks = try await asset.loadTracks(withMediaType: .audio) + + // 'auto' targets h264 — server pipeline is HLS, which favors h264 (HEVC needs + // fMP4 segments + commercial licensing). HEVC remains opt-in via codec: 'hevc'. + let useHEVC: Bool + switch codecPref { + case "hevc": useHEVC = true + default: useHEVC = false + } + let codecType: AVVideoCodecType = useHEVC ? .hevc : .h264 + let profileLevel: String = useHEVC + ? kVTProfileLevel_HEVC_Main_AutoLevel as String + : kVTProfileLevel_H264_High_AutoLevel as String + + let effectiveBitrate = targetBitrate > 0 + ? targetBitrate + : (useHEVC ? 2_500_000 : 3_000_000) + + let outputURL = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathExtension("mp4") + + let videoComposition = makeRotatingComposition( + videoTrack: videoTrack, + preferredTransform: preferredTransform, + naturalSize: naturalSize, + outputSize: outputSize, + duration: duration + ) + + let reader = try AVAssetReader(asset: asset) + + let videoReaderSettings: [String: Any] = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA + ] + let videoReaderOutput = AVAssetReaderVideoCompositionOutput( + videoTracks: [videoTrack], + videoSettings: videoReaderSettings + ) + videoReaderOutput.videoComposition = videoComposition + videoReaderOutput.alwaysCopiesSampleData = false + guard reader.canAdd(videoReaderOutput) else { + throw err("Cannot read video track", code: 3) + } + reader.add(videoReaderOutput) + + var compressionProps: [String: Any] = [ + AVVideoAverageBitRateKey: effectiveBitrate, + AVVideoProfileLevelKey: profileLevel, + AVVideoMaxKeyFrameIntervalKey: max(frameRateCap * 3, 30), + AVVideoExpectedSourceFrameRateKey: frameRateCap, + AVVideoAllowFrameReorderingKey: false, + kVTCompressionPropertyKey_RealTime as String: true, + ] + let peakBytesPerSecond = Int(Double(effectiveBitrate) / 8.0 * 1.5) + compressionProps[kVTCompressionPropertyKey_DataRateLimits as String] = [ + peakBytesPerSecond, 1 + ] as CFArray + + let videoColorProps: [String: Any] = [ + AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2, + AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2, + AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_709_2, + ] + + let videoWriterSettings: [String: Any] = [ + AVVideoCodecKey: codecType, + AVVideoWidthKey: outputSize.width, + AVVideoHeightKey: outputSize.height, + AVVideoColorPropertiesKey: videoColorProps, + AVVideoCompressionPropertiesKey: compressionProps, + ] + let videoWriterInput = AVAssetWriterInput( + mediaType: .video, + outputSettings: videoWriterSettings + ) + videoWriterInput.expectsMediaDataInRealTime = false + + let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4) + writer.shouldOptimizeForNetworkUse = true + writer.metadata = [] + guard writer.canAdd(videoWriterInput) else { + throw err("Cannot write video track", code: 4) + } + writer.add(videoWriterInput) + + var audioReaderOutput: AVAssetReaderTrackOutput? + var audioWriterInput: AVAssetWriterInput? + if let audioTrack = audioTracks.first { + let audioDecoderSettings: [String: Any] = [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsBigEndianKey: false, + AVLinearPCMIsNonInterleaved: false, + ] + let audioOutput = AVAssetReaderTrackOutput( + track: audioTrack, + outputSettings: audioDecoderSettings + ) + audioOutput.alwaysCopiesSampleData = false + if reader.canAdd(audioOutput) { + reader.add(audioOutput) + audioReaderOutput = audioOutput + + let audioEncoderSettings: [String: Any] = [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVSampleRateKey: 44100, + AVNumberOfChannelsKey: 2, + AVEncoderBitRateKey: 128_000, + ] + let audioInput = AVAssetWriterInput( + mediaType: .audio, + outputSettings: audioEncoderSettings + ) + audioInput.expectsMediaDataInRealTime = false + if writer.canAdd(audioInput) { + writer.add(audioInput) + audioWriterInput = audioInput + } + } + } + + guard reader.startReading() else { + throw reader.error ?? err("Reader failed to start", code: 8) + } + guard writer.startWriting() else { + throw writer.error ?? err("Writer failed to start", code: 9) + } + writer.startSession(atSourceTime: .zero) + + let minFrameIntervalSeconds = 1.0 / Double(frameRateCap) + let minFrameInterval = CMTime( + seconds: minFrameIntervalSeconds, + preferredTimescale: 600 + ) + + await withTaskGroup(of: Void.self) { group in + group.addTask { [self] in + await processVideoTrack( + readerOutput: videoReaderOutput, + writerInput: videoWriterInput, + totalDuration: totalSeconds, + minFrameInterval: minFrameInterval + ) + } + if let audioOutput = audioReaderOutput, let audioInput = audioWriterInput { + group.addTask { [self] in + await processAudioTrack( + readerOutput: audioOutput, + writerInput: audioInput + ) + } + } + await group.waitForAll() + } + + if isCancelled { + writer.cancelWriting() + try? FileManager.default.removeItem(at: outputURL) + throw err("Compression cancelled", code: 5) + } + + if reader.status == .failed { + let error = reader.error ?? err("Reader failed", code: 6) + writer.cancelWriting() + try? FileManager.default.removeItem(at: outputURL) + throw error + } + + await writer.finishWriting() + + if writer.status == .failed { + let error = writer.error ?? err("Writer failed", code: 7) + try? FileManager.default.removeItem(at: outputURL) + throw error + } + + onProgress(jobId, 1.0) + + let attributes = try FileManager.default.attributesOfItem(atPath: outputURL.path) + let fileSize = attributes[.size] as? Int ?? 0 + + return [ + "uri": outputURL.absoluteString, + "size": fileSize, + "mimeType": "video/mp4", + "width": outputSize.width, + "height": outputSize.height, + "duration": totalSeconds, + "codec": useHEVC ? "hevc" : "h264", + ] + } + + private func processVideoTrack( + readerOutput: AVAssetReaderOutput, + writerInput: AVAssetWriterInput, + totalDuration: Double, + minFrameInterval: CMTime + ) async { + var lastProgressTime: CFAbsoluteTime = 0 + var lastAppendedPTS: CMTime? + + await withCheckedContinuation { (continuation: CheckedContinuation) in + writerInput.requestMediaDataWhenReady( + on: DispatchQueue(label: "com.bsky.videocompress.video") + ) { + while writerInput.isReadyForMoreMediaData { + if self.isCancelled { + writerInput.markAsFinished() + continuation.resume() + return + } + guard let sampleBuffer = readerOutput.copyNextSampleBuffer() else { + writerInput.markAsFinished() + continuation.resume() + return + } + + let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + if let last = lastAppendedPTS { + let delta = CMTimeSubtract(pts, last) + if CMTimeCompare(delta, minFrameInterval) < 0 { + continue + } + } + lastAppendedPTS = pts + + if !writerInput.append(sampleBuffer) { + writerInput.markAsFinished() + continuation.resume() + return + } + + let now = CFAbsoluteTimeGetCurrent() + if now - lastProgressTime >= 0.1 { + lastProgressTime = now + let progress = min(CMTimeGetSeconds(pts) / totalDuration, 1.0) + self.onProgress(self.jobId, progress) + } + } + } + } + } + + private func processAudioTrack( + readerOutput: AVAssetReaderOutput, + writerInput: AVAssetWriterInput + ) async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + writerInput.requestMediaDataWhenReady( + on: DispatchQueue(label: "com.bsky.videocompress.audio") + ) { + while writerInput.isReadyForMoreMediaData { + if self.isCancelled { + writerInput.markAsFinished() + continuation.resume() + return + } + guard let sampleBuffer = readerOutput.copyNextSampleBuffer() else { + writerInput.markAsFinished() + continuation.resume() + return + } + if !writerInput.append(sampleBuffer) { + writerInput.markAsFinished() + continuation.resume() + return + } + } + } + } + } + + private func makeRotatingComposition( + videoTrack: AVAssetTrack, + preferredTransform: CGAffineTransform, + naturalSize: CGSize, + outputSize: (width: Int, height: Int), + duration: CMTime + ) -> AVMutableVideoComposition { + let composition = AVMutableVideoComposition() + composition.renderSize = CGSize(width: outputSize.width, height: outputSize.height) + composition.frameDuration = CMTime(value: 1, timescale: Int32(frameRateCap)) + + let rotatedRect = CGRect(origin: .zero, size: naturalSize).applying(preferredTransform) + let translate = CGAffineTransform( + translationX: -rotatedRect.minX, + y: -rotatedRect.minY + ) + let displaySize = CGSize( + width: abs(rotatedRect.width), + height: abs(rotatedRect.height) + ) + let scaleX = CGFloat(outputSize.width) / displaySize.width + let scaleY = CGFloat(outputSize.height) / displaySize.height + let scale = CGAffineTransform(scaleX: scaleX, y: scaleY) + let combined = preferredTransform.concatenating(translate).concatenating(scale) + + let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: videoTrack) + layerInstruction.setTransform(combined, at: .zero) + + let instruction = AVMutableVideoCompositionInstruction() + instruction.timeRange = CMTimeRange(start: .zero, duration: duration) + instruction.layerInstructions = [layerInstruction] + + composition.instructions = [instruction] + return composition + } + + private func scaleEvenly(displaySize: CGSize, maxSize: Int) -> (width: Int, height: Int) { + let cap = CGFloat(maxSize) + let scale: CGFloat + if displaySize.width <= cap && displaySize.height <= cap { + scale = 1.0 + } else if displaySize.width > displaySize.height { + scale = cap / displaySize.width + } else { + scale = cap / displaySize.height + } + return ( + roundToEven(Int(displaySize.width * scale)), + roundToEven(Int(displaySize.height * scale)) + ) + } + + private func roundToEven(_ value: Int) -> Int { + return value % 2 == 0 ? value : value - 1 + } + + private func err(_ message: String, code: Int) -> NSError { + return NSError( + domain: "ExpoBlueskyVideoCompress", + code: code, + userInfo: [NSLocalizedDescriptionKey: message] + ) + } +} diff --git a/modules/expo-bluesky-video-compress/ios/VideoProber.swift b/modules/expo-bluesky-video-compress/ios/VideoProber.swift new file mode 100644 index 0000000000..e68371f369 --- /dev/null +++ b/modules/expo-bluesky-video-compress/ios/VideoProber.swift @@ -0,0 +1,88 @@ +import AVFoundation +import UniformTypeIdentifiers + +struct VideoProber { + static func probe(url: URL) async throws -> [String: Any] { + let asset = AVURLAsset(url: url) + + let duration = try await asset.load(.duration) + let tracks = try await asset.loadTracks(withMediaType: .video) + + guard let videoTrack = tracks.first else { + throw NSError( + domain: "ExpoBlueskyVideoCompress", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "No video track found"] + ) + } + + let naturalSize = try await videoTrack.load(.naturalSize) + let preferredTransform = try await videoTrack.load(.preferredTransform) + let estimatedDataRate = try await videoTrack.load(.estimatedDataRate) + let nominalFrameRate = try await videoTrack.load(.nominalFrameRate) + let formatDescriptions = try await videoTrack.load(.formatDescriptions) + + var codec = "unknown" + if let formatDescription = formatDescriptions.first { + codec = fourCCToString(CMFormatDescriptionGetMediaSubType(formatDescription)) + } + + let rotation = rotationFromTransform(preferredTransform) + let isRotated = rotation == 90 || rotation == 270 + let width = isRotated ? Int(naturalSize.height) : Int(naturalSize.width) + let height = isRotated ? Int(naturalSize.width) : Int(naturalSize.height) + + let audioTracks = try await asset.loadTracks(withMediaType: .audio) + let hasAudio = !audioTracks.isEmpty + + let fileSize: Int + if let attributes = try? FileManager.default.attributesOfItem(atPath: url.path), + let size = attributes[.size] as? Int { + fileSize = size + } else { + fileSize = 0 + } + + let mimeType: String + if let utType = UTType(filenameExtension: url.pathExtension) { + mimeType = utType.preferredMIMEType ?? "video/mp4" + } else { + mimeType = "video/mp4" + } + + let durationSeconds = CMTimeGetSeconds(duration) + var bitrate = Int(estimatedDataRate) + if bitrate == 0 && durationSeconds > 0 { + bitrate = Int(Double(fileSize * 8) / durationSeconds) + } + + return [ + "width": width, + "height": height, + "duration": durationSeconds, + "bitrate": bitrate, + "fileSize": fileSize, + "mimeType": mimeType, + "codec": codec, + "hasAudio": hasAudio, + "frameRate": nominalFrameRate, + "rotation": rotation + ] + } + + private static func rotationFromTransform(_ transform: CGAffineTransform) -> Int { + let angle = atan2(transform.b, transform.a) + let degrees = Int(round(angle * 180.0 / .pi)) + return ((degrees % 360) + 360) % 360 + } + + private static func fourCCToString(_ code: FourCharCode) -> String { + let chars: [Character] = [ + Character(UnicodeScalar((code >> 24) & 0xFF)!), + Character(UnicodeScalar((code >> 16) & 0xFF)!), + Character(UnicodeScalar((code >> 8) & 0xFF)!), + Character(UnicodeScalar(code & 0xFF)!) + ] + return String(chars).trimmingCharacters(in: .whitespaces) + } +} diff --git a/modules/expo-bluesky-video-compress/src/ExpoBlueskyVideoCompressModule.ts b/modules/expo-bluesky-video-compress/src/ExpoBlueskyVideoCompressModule.ts new file mode 100644 index 0000000000..cd3648735a --- /dev/null +++ b/modules/expo-bluesky-video-compress/src/ExpoBlueskyVideoCompressModule.ts @@ -0,0 +1,23 @@ +import {type EventSubscription, requireNativeModule} from 'expo-modules-core' + +import { + type CompressResult, + type NativeCompressOptions, + type VideoMetadata, +} from './types' + +type ProgressEvent = {id: number; progress: number} + +interface ExpoBlueskyVideoCompressModule { + probe(uri: string): Promise + compress(uri: string, options: NativeCompressOptions): Promise + cancel(): void + addListener( + eventName: 'onProgress', + listener: (event: ProgressEvent) => void, + ): EventSubscription +} + +export default requireNativeModule( + 'ExpoBlueskyVideoCompress', +) diff --git a/modules/expo-bluesky-video-compress/src/types.ts b/modules/expo-bluesky-video-compress/src/types.ts new file mode 100644 index 0000000000..d278ff752c --- /dev/null +++ b/modules/expo-bluesky-video-compress/src/types.ts @@ -0,0 +1,44 @@ +export type CodecPreference = 'auto' | 'hevc' | 'h264' + +export type VideoMetadata = { + width: number + height: number + duration: number + bitrate: number + fileSize: number + mimeType: string + codec: string + hasAudio: boolean + frameRate: number + rotation: number +} + +export type CompressOptions = { + targetBitrate?: number + maxSize?: number + codec?: CodecPreference + frameRateCap?: number +} + +export type CompressCallbacks = { + onProgress?: (progress: number) => void + signal?: AbortSignal +} + +export type CompressResult = { + uri: string + size: number + mimeType: string + width: number + height: number + duration: number + codec: 'h264' | 'hevc' +} + +export type NativeCompressOptions = { + targetBitrate: number + maxSize: number + codec: CodecPreference + frameRateCap: number + jobId: number +} diff --git a/src/lib/media/video/compress.ts b/src/lib/media/video/compress.ts index 1d00bfcea1..0a69e001f1 100644 --- a/src/lib/media/video/compress.ts +++ b/src/lib/media/video/compress.ts @@ -1,11 +1,13 @@ -import {getVideoMetaData, Video} from 'react-native-compressor' import {type ImagePickerAsset} from 'expo-image-picker' import {SUPPORTED_MIME_TYPES, type SupportedMimeTypes} from '#/lib/constants' +import {logger} from '#/logger' +import {compress, probe} from '../../../../modules/expo-bluesky-video-compress' import {type CompressedVideo} from './types' -import {extToMime} from './util' -const MIN_SIZE_FOR_COMPRESSION = 25 // 25mb +const PASSTHROUGH_BITRATE = 5_000_000 +const PASSTHROUGH_MAX_DIMENSION = 1920 +const MAX_UPLOAD_SIZE = 100 * 1000 * 1000 export async function compressVideo( file: ImagePickerAsset, @@ -14,43 +16,76 @@ export async function compressVideo( onProgress?: (progress: number) => void }, ): Promise { - const {onProgress, signal} = opts || {} + if (file.mimeType === 'image/gif') { + return {uri: file.uri, size: file.fileSize ?? -1, mimeType: 'image/gif'} + } const isAcceptableFormat = SUPPORTED_MIME_TYPES.includes( file.mimeType as SupportedMimeTypes, ) - if (file.mimeType === 'image/gif') { - // let's hope they're small enough that they don't need compression! - // this compression library doesn't support gifs - // worst case - server rejects them. I think that's fine -sfn - return {uri: file.uri, size: file.fileSize ?? -1, mimeType: 'image/gif'} + const metadata = await probe(file.uri) + + if (!shouldCompress(metadata, isAcceptableFormat)) { + return { + uri: file.uri, + size: metadata.fileSize, + mimeType: file.mimeType ?? 'video/mp4', + } } - const minimumFileSizeForCompress = isAcceptableFormat - ? MIN_SIZE_FOR_COMPRESSION - : 0 - - const compressed = await Video.compress( + const result = await compress( file.uri, { - compressionMethod: 'manual', - bitrate: 3_000_000, // 3mbps - maxSize: 1920, - // WARNING: this ONE SPECIFIC ARG is in MB -sfn - minimumFileSizeForCompress, - getCancellationId: id => { - if (signal) { - signal.addEventListener('abort', () => { - Video.cancelCompression(id) - }) - } - }, + targetBitrate: 3_000_000, + maxSize: PASSTHROUGH_MAX_DIMENSION, + codec: 'h264', + }, + { + onProgress: opts?.onProgress, + signal: opts?.signal, }, - onProgress, ) - const info = await getVideoMetaData(compressed) - - return {uri: compressed, size: info.size, mimeType: extToMime(info.extension)} + return { + uri: result.uri, + size: result.size, + mimeType: result.mimeType, + } +} + +function shouldCompress( + metadata: {bitrate: number; width: number; height: number; fileSize: number}, + isAcceptableFormat: boolean, +): boolean { + const maxDimension = Math.max(metadata.width, metadata.height) + const bitrateKbps = Math.round(metadata.bitrate / 1000) + const sizeMB = (metadata.fileSize / 1_000_000).toFixed(1) + + if (!isAcceptableFormat) { + logger.debug('shouldCompress: yes (unsupported format)') + return true + } + + if (metadata.fileSize > MAX_UPLOAD_SIZE) { + logger.debug(`shouldCompress: yes (file too large: ${sizeMB}MB)`) + return true + } + + if ( + metadata.bitrate <= PASSTHROUGH_BITRATE && + maxDimension <= PASSTHROUGH_MAX_DIMENSION + ) { + logger.debug( + `shouldCompress: no (${bitrateKbps}kbps, ${maxDimension}px, ${sizeMB}MB)`, + ) + return false + } + + if (metadata.bitrate > PASSTHROUGH_BITRATE) { + logger.debug(`shouldCompress: yes (bitrate ${bitrateKbps}kbps)`) + } else { + logger.debug(`shouldCompress: yes (dimension ${maxDimension}px)`) + } + return true }