diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt new file mode 100644 index 0000000000..c3e8e1b8b9 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/InputSurface.kt @@ -0,0 +1,103 @@ +package expo.modules.blueskyvideocompress + +import android.opengl.EGL14 +import android.opengl.EGLConfig +import android.opengl.EGLContext +import android.opengl.EGLDisplay +import android.opengl.EGLExt +import android.opengl.EGLSurface +import android.view.Surface + +/** + * EGL14 wrapper around the encoder's input Surface. Provides an EGL context and + * window surface so that the GL pipeline can render decoded frames onto the + * encoder's input at the target resolution. + */ +class InputSurface(private val surface: Surface) { + private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY + private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT + private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE + + init { + eglSetup() + } + + private fun eglSetup() { + eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY) + if (eglDisplay === EGL14.EGL_NO_DISPLAY) { + throw RuntimeException("unable to get EGL14 display") + } + + val version = IntArray(2) + if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) { + throw RuntimeException("unable to initialize EGL14") + } + + val attribList = intArrayOf( + EGL14.EGL_RED_SIZE, 8, + EGL14.EGL_GREEN_SIZE, 8, + EGL14.EGL_BLUE_SIZE, 8, + EGL14.EGL_ALPHA_SIZE, 8, + EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, + EGL14.EGL_NONE + ) + val configs = arrayOfNulls(1) + val numConfigs = IntArray(1) + EGL14.eglChooseConfig(eglDisplay, attribList, 0, configs, 0, 1, numConfigs, 0) + checkEglError("eglChooseConfig") + + val contextAttribs = intArrayOf( + EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, + EGL14.EGL_NONE + ) + eglContext = EGL14.eglCreateContext( + eglDisplay, configs[0], EGL14.EGL_NO_CONTEXT, contextAttribs, 0 + ) + checkEglError("eglCreateContext") + + val surfaceAttribs = intArrayOf(EGL14.EGL_NONE) + eglSurface = EGL14.eglCreateWindowSurface( + eglDisplay, configs[0], surface, surfaceAttribs, 0 + ) + checkEglError("eglCreateWindowSurface") + } + + fun makeCurrent() { + EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext) + checkEglError("eglMakeCurrent") + } + + fun swapBuffers(): Boolean { + val result = EGL14.eglSwapBuffers(eglDisplay, eglSurface) + checkEglError("eglSwapBuffers") + return result + } + + fun setPresentationTime(nsecs: Long) { + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, nsecs) + checkEglError("eglPresentationTimeANDROID") + } + + fun release() { + if (eglDisplay !== EGL14.EGL_NO_DISPLAY) { + EGL14.eglMakeCurrent( + eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT + ) + EGL14.eglDestroySurface(eglDisplay, eglSurface) + EGL14.eglDestroyContext(eglDisplay, eglContext) + EGL14.eglReleaseThread() + EGL14.eglTerminate(eglDisplay) + } + surface.release() + eglDisplay = EGL14.EGL_NO_DISPLAY + eglContext = EGL14.EGL_NO_CONTEXT + eglSurface = EGL14.EGL_NO_SURFACE + } + + private fun checkEglError(msg: String) { + val error = EGL14.eglGetError() + if (error != EGL14.EGL_SUCCESS) { + throw RuntimeException("$msg: EGL error: 0x${Integer.toHexString(error)}") + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt new file mode 100644 index 0000000000..6ec670a506 --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/OutputSurface.kt @@ -0,0 +1,80 @@ +package expo.modules.blueskyvideocompress + +import android.graphics.SurfaceTexture +import android.os.Handler +import android.os.HandlerThread +import android.view.Surface + +/** + * Wraps a SurfaceTexture to receive decoded frames and render them via GL. + * The decoder outputs to this surface, and drawImage() renders the latest + * frame through the TextureRenderer onto the current EGL surface (the encoder's + * input via InputSurface). + * + * Uses a dedicated HandlerThread for frame-available callbacks so they fire + * reliably regardless of the calling thread's Looper state. + */ +class OutputSurface : SurfaceTexture.OnFrameAvailableListener { + private val renderer = TextureRenderer() + private var surfaceTexture: SurfaceTexture? = null + private val stMatrix = FloatArray(16) + private val callbackThread = HandlerThread("OutputSurfaceCallbacks") + + val surface: Surface + + @Volatile + private var frameAvailable = false + private val frameSyncObject = Object() + + init { + renderer.surfaceCreated() + + callbackThread.start() + val handler = Handler(callbackThread.looper) + + surfaceTexture = SurfaceTexture(renderer.getTextureId()).also { + it.setOnFrameAvailableListener(this, handler) + } + surface = Surface(surfaceTexture) + } + + fun release() { + surface.release() + surfaceTexture?.release() + surfaceTexture = null + callbackThread.quitSafely() + } + + /** + * Blocks until a new frame is available from the decoder (up to 2500ms). + */ + fun awaitNewImage() { + val timeoutMs = 2500L + synchronized(frameSyncObject) { + while (!frameAvailable) { + frameSyncObject.wait(timeoutMs) + if (!frameAvailable) { + throw RuntimeException("Surface frame wait timed out") + } + } + frameAvailable = false + } + // Must be called outside synchronized to avoid deadlock with onFrameAvailable + surfaceTexture!!.updateTexImage() + } + + /** + * Renders the most recent frame through the GL pipeline. + */ + fun drawImage() { + surfaceTexture!!.getTransformMatrix(stMatrix) + renderer.drawFrame(stMatrix) + } + + override fun onFrameAvailable(st: SurfaceTexture) { + synchronized(frameSyncObject) { + frameAvailable = true + frameSyncObject.notifyAll() + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt new file mode 100644 index 0000000000..b0cb7063ef --- /dev/null +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/TextureRenderer.kt @@ -0,0 +1,209 @@ +package expo.modules.blueskyvideocompress + +import android.opengl.GLES11Ext +import android.opengl.GLES20 +import android.opengl.Matrix +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.FloatBuffer + +/** + * GLES2 renderer that draws an external OES texture (from SurfaceTexture) onto + * the current EGL surface. The GL viewport handles scaling from source to target + * dimensions automatically with GL_LINEAR filtering. + */ +class TextureRenderer { + companion object { + private const val FLOAT_SIZE_BYTES = 4 + private const val VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES + private const val VERTICES_DATA_POS_OFFSET = 0 + private const val VERTICES_DATA_UV_OFFSET = 3 + + private val VERTICES_DATA = floatArrayOf( + // X, Y, Z, U, V + -1.0f, -1.0f, 0f, 0f, 0f, + 1.0f, -1.0f, 0f, 1f, 0f, + -1.0f, 1.0f, 0f, 0f, 1f, + 1.0f, 1.0f, 0f, 1f, 1f, + ) + + private const val VERTEX_SHADER = """ + uniform mat4 uMVPMatrix; + uniform mat4 uSTMatrix; + attribute vec4 aPosition; + attribute vec4 aTextureCoord; + varying vec2 vTextureCoord; + void main() { + gl_Position = uMVPMatrix * aPosition; + vTextureCoord = (uSTMatrix * aTextureCoord).xy; + } + """ + + private const val FRAGMENT_SHADER = """ + #extension GL_OES_EGL_image_external : require + precision mediump float; + varying vec2 vTextureCoord; + uniform samplerExternalOES sTexture; + void main() { + gl_FragColor = texture2D(sTexture, vTextureCoord); + } + """ + } + + private val vertices: FloatBuffer = + ByteBuffer.allocateDirect(VERTICES_DATA.size * FLOAT_SIZE_BYTES) + .order(ByteOrder.nativeOrder()) + .asFloatBuffer() + .apply { + put(VERTICES_DATA) + position(0) + } + + private val mvpMatrix = FloatArray(16) + private val stMatrix = FloatArray(16) + + private var program = 0 + private var textureId = -1 + private var uMVPMatrixHandle = 0 + private var uSTMatrixHandle = 0 + private var aPositionHandle = 0 + private var aTextureCoordHandle = 0 + + init { + Matrix.setIdentityM(stMatrix, 0) + Matrix.setIdentityM(mvpMatrix, 0) + } + + fun getTextureId(): Int = textureId + + fun surfaceCreated() { + program = createProgram(VERTEX_SHADER, FRAGMENT_SHADER) + + aPositionHandle = GLES20.glGetAttribLocation(program, "aPosition") + checkGlError("glGetAttribLocation aPosition") + + aTextureCoordHandle = GLES20.glGetAttribLocation(program, "aTextureCoord") + checkGlError("glGetAttribLocation aTextureCoord") + + uMVPMatrixHandle = GLES20.glGetUniformLocation(program, "uMVPMatrix") + checkGlError("glGetUniformLocation uMVPMatrix") + + uSTMatrixHandle = GLES20.glGetUniformLocation(program, "uSTMatrix") + checkGlError("glGetUniformLocation uSTMatrix") + + // Create OES texture + val textures = IntArray(1) + GLES20.glGenTextures(1, textures, 0) + textureId = textures[0] + + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId) + checkGlError("glBindTexture") + + GLES20.glTexParameterf( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR.toFloat() + ) + GLES20.glTexParameterf( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat() + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE + ) + GLES20.glTexParameteri( + GLES11Ext.GL_TEXTURE_EXTERNAL_OES, + GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE + ) + checkGlError("glTexParameter") + } + + fun drawFrame(stMatrix: FloatArray) { + checkGlError("onDrawFrame start") + + GLES20.glClearColor(0f, 0f, 0f, 1f) + GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT or GLES20.GL_COLOR_BUFFER_BIT) + + GLES20.glUseProgram(program) + checkGlError("glUseProgram") + + GLES20.glActiveTexture(GLES20.GL_TEXTURE0) + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId) + + // Position + vertices.position(VERTICES_DATA_POS_OFFSET) + GLES20.glVertexAttribPointer( + aPositionHandle, 3, GLES20.GL_FLOAT, false, + VERTICES_DATA_STRIDE_BYTES, vertices + ) + checkGlError("glVertexAttribPointer aPosition") + GLES20.glEnableVertexAttribArray(aPositionHandle) + checkGlError("glEnableVertexAttribArray aPosition") + + // Texture coordinates + vertices.position(VERTICES_DATA_UV_OFFSET) + GLES20.glVertexAttribPointer( + aTextureCoordHandle, 2, GLES20.GL_FLOAT, false, + VERTICES_DATA_STRIDE_BYTES, vertices + ) + checkGlError("glVertexAttribPointer aTextureCoord") + GLES20.glEnableVertexAttribArray(aTextureCoordHandle) + checkGlError("glEnableVertexAttribArray aTextureCoord") + + // Matrices + GLES20.glUniformMatrix4fv(uMVPMatrixHandle, 1, false, mvpMatrix, 0) + GLES20.glUniformMatrix4fv(uSTMatrixHandle, 1, false, stMatrix, 0) + + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4) + checkGlError("glDrawArrays") + + GLES20.glFinish() + } + + private fun createProgram(vertexSource: String, fragmentSource: String): Int { + val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource) + val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource) + + val program = GLES20.glCreateProgram() + checkGlError("glCreateProgram") + + GLES20.glAttachShader(program, vertexShader) + checkGlError("glAttachShader vertex") + GLES20.glAttachShader(program, fragmentShader) + checkGlError("glAttachShader fragment") + GLES20.glLinkProgram(program) + + val linkStatus = IntArray(1) + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0) + if (linkStatus[0] != GLES20.GL_TRUE) { + val log = GLES20.glGetProgramInfoLog(program) + GLES20.glDeleteProgram(program) + throw RuntimeException("Could not link program: $log") + } + return program + } + + private fun loadShader(shaderType: Int, source: String): Int { + val shader = GLES20.glCreateShader(shaderType) + checkGlError("glCreateShader type=$shaderType") + + GLES20.glShaderSource(shader, source) + GLES20.glCompileShader(shader) + + val compiled = IntArray(1) + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0) + if (compiled[0] == 0) { + val log = GLES20.glGetShaderInfoLog(shader) + GLES20.glDeleteShader(shader) + throw RuntimeException("Could not compile shader $shaderType: $log") + } + return shader + } + + private fun checkGlError(op: String) { + val error = GLES20.glGetError() + if (error != GLES20.GL_NO_ERROR) { + throw RuntimeException("$op: glError $error") + } + } +} diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt index 15fdf2c708..68b1ff1dcd 100644 --- a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt @@ -9,7 +9,6 @@ import android.media.MediaMuxer import android.net.Uri import android.os.Build import android.util.Log -import android.view.Surface import java.io.File import java.nio.ByteBuffer @@ -23,11 +22,8 @@ class VideoCompressor( ) { companion object { private const val TAG = "VideoCompressor" - private const val TIMEOUT_US = 10_000L + private const val TIMEOUT_DEQUEUE = 100L // 100us private const val I_FRAME_INTERVAL = 3 - private const val AUDIO_AAC_BITRATE = 128_000 - private const val AUDIO_SAMPLE_RATE = 44100 - private const val AUDIO_CHANNELS = 2 } @Volatile @@ -156,12 +152,19 @@ class VideoCompressor( MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel41 ) + setInteger(MediaFormat.KEY_PRIORITY, 0) // realtime + setInteger(MediaFormat.KEY_OPERATING_RATE, Short.MAX_VALUE.toInt()) // max speed } } val encoder = MediaCodec.createByCodecName(encoderName) encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) - val inputSurface = encoder.createInputSurface() + + // GL pipeline: Decoder -> SurfaceTexture -> OpenGL ES 2.0 -> EGL Surface -> Encoder + val inputSurface = InputSurface(encoder.createInputSurface()) + inputSurface.makeCurrent() + val outputSurface = OutputSurface() + encoder.start() // Set up video decoder @@ -169,23 +172,28 @@ class VideoCompressor( val decoder = MediaCodec.createDecoderByType( videoFormat.getString(MediaFormat.KEY_MIME) ?: "video/avc" ) - // Output surface is the encoder's input surface for zero-copy pipeline - decoder.configure(decoderFormat, inputSurface, null, 0) + decoder.configure(decoderFormat, outputSurface.surface, null, 0) decoder.start() extractor.selectTrack(videoTrackIndex) + // Frame rate capping: drop frames above 30fps + val targetFps = frameRate.coerceAtMost(30) + val frameIntervalUs = 1_000_000L / targetFps + // Process video frames val bufferInfo = MediaCodec.BufferInfo() var inputDone = false + var decoderDone = false var outputDone = false var lastProgressTime = 0L + var lastRenderedPtsUs = -1L try { while (!outputDone && !isCancelled) { // Feed decoder if (!inputDone) { - val inputIndex = decoder.dequeueInputBuffer(TIMEOUT_US) + val inputIndex = decoder.dequeueInputBuffer(TIMEOUT_DEQUEUE) if (inputIndex >= 0) { val inputBuffer = decoder.getInputBuffer(inputIndex) ?: continue val sampleSize = extractor.readSampleData(inputBuffer, 0) @@ -205,71 +213,66 @@ class VideoCompressor( } } - // Drain decoder -> surface -> encoder - drainDecoder(decoder, bufferInfo) + // Drain decoder -> GL pipeline -> encoder + if (!decoderDone) { + decoderDone = drainDecoder( + decoder, encoder, inputSurface, outputSurface, + bufferInfo, frameIntervalUs, lastRenderedPtsUs + ) { pts -> lastRenderedPtsUs = pts } + } - // Drain encoder - val encoderOutputIndex = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_US) - when { - encoderOutputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { - if (!muxerStarted) { - muxerVideoTrack = muxer.addTrack(encoder.outputFormat) - // If we have audio, add it now too before starting muxer - if (audioTrackIndex != -1 && audioFormat != null) { - muxerAudioTrack = if (shouldPassthroughAudio) { - muxer.addTrack(audioFormat) - } else { - // Audio re-encode track will be added when audio encoder outputs format - -1 + // Drain all available encoder output + while (!outputDone && !isCancelled) { + val encoderOutputIndex = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_DEQUEUE) + when { + encoderOutputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + if (!muxerStarted) { + muxerVideoTrack = muxer.addTrack(encoder.outputFormat) + // Add audio track for passthrough (format known upfront) + if (audioTrackIndex != -1 && audioFormat != null && shouldPassthroughAudio) { + muxerAudioTrack = muxer.addTrack(audioFormat) } - } - if (audioTrackIndex == -1 || muxerAudioTrack >= 0) { muxer.start() muxerStarted = true } } - } - encoderOutputIndex >= 0 -> { - val outputBuffer = encoder.getOutputBuffer(encoderOutputIndex) - if (outputBuffer != null && - bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG == 0 && - bufferInfo.size > 0 && - muxerStarted) { - muxer.writeSampleData(muxerVideoTrack, outputBuffer, bufferInfo) - } + encoderOutputIndex >= 0 -> { + val outputBuffer = encoder.getOutputBuffer(encoderOutputIndex) + if (outputBuffer != null && + bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG == 0 && + bufferInfo.size > 0 && + muxerStarted) { + muxer.writeSampleData(muxerVideoTrack, outputBuffer, bufferInfo) + } - val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 - encoder.releaseOutputBuffer(encoderOutputIndex, false) + val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + encoder.releaseOutputBuffer(encoderOutputIndex, false) - if (isEos) { - outputDone = true - } + if (isEos) { + outputDone = true + } - // Progress reporting - if (durationUs > 0) { - val now = System.currentTimeMillis() - if (now - lastProgressTime >= 100) { - lastProgressTime = now - val progress = (bufferInfo.presentationTimeUs.toDouble() / durationUs) - .coerceIn(0.0, 1.0) - onProgress(jobId, progress) + // Progress reporting + if (durationUs > 0) { + val now = System.currentTimeMillis() + if (now - lastProgressTime >= 100) { + lastProgressTime = now + val progress = (bufferInfo.presentationTimeUs.toDouble() / durationUs) + .coerceIn(0.0, 1.0) + onProgress(jobId, progress) + } } } + else -> break // No more encoder output available right now } } } - // Process audio track - if (audioTrackIndex != -1 && audioFormat != null && !isCancelled) { - if (shouldPassthroughAudio) { - processAudioPassthrough( - extractor, audioTrackIndex, muxer, muxerAudioTrack, muxerStarted - ) - } else { - processAudioReencode( - extractor, audioTrackIndex, audioFormat, muxer, muxerStarted - ) - } + // Process audio track (passthrough only — non-AAC audio is skipped) + if (muxerAudioTrack >= 0 && muxerStarted && !isCancelled) { + processAudioPassthrough( + audioTrackIndex, muxer, muxerAudioTrack + ) } } finally { // Clean up resources @@ -277,6 +280,7 @@ class VideoCompressor( try { decoder.release() } catch (_: Exception) {} try { encoder.stop() } catch (_: Exception) {} try { encoder.release() } catch (_: Exception) {} + try { outputSurface.release() } catch (_: Exception) {} try { inputSurface.release() } catch (_: Exception) {} try { extractor.release() } catch (_: Exception) {} try { @@ -303,41 +307,66 @@ class VideoCompressor( ) } - private fun drainDecoder(decoder: MediaCodec, bufferInfo: MediaCodec.BufferInfo) { + /** + * Drains decoded frames through the GL pipeline to the encoder. + * Drops frames to cap at the target frame rate. + * + * @return true if the decoder signaled EOS (all frames decoded) + */ + private fun drainDecoder( + decoder: MediaCodec, + encoder: MediaCodec, + inputSurface: InputSurface, + outputSurface: OutputSurface, + bufferInfo: MediaCodec.BufferInfo, + frameIntervalUs: Long, + lastRenderedPtsUs: Long, + onRendered: (Long) -> Unit + ): Boolean { + var currentLastPts = lastRenderedPtsUs + while (true) { - val outputIndex = decoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_US) + val outputIndex = decoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_DEQUEUE) if (outputIndex < 0) break val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 - // Render to surface (encoder's input) - true means render - decoder.releaseOutputBuffer(outputIndex, bufferInfo.size > 0) - if (isEos) { - // Signal encoder that input is done - // Note: with Surface input, we signal EOS by calling signalEndOfInputStream - break + if (!isEos) { + // Frame rate capping: decide whether to render or drop this frame + val shouldRender = if (currentLastPts < 0) { + true + } else { + bufferInfo.presentationTimeUs - currentLastPts >= frameIntervalUs + } + + if (shouldRender && bufferInfo.size > 0) { + // Render through GL pipeline: decoder -> SurfaceTexture -> GL -> encoder + decoder.releaseOutputBuffer(outputIndex, true) + outputSurface.awaitNewImage() + outputSurface.drawImage() + inputSurface.setPresentationTime(bufferInfo.presentationTimeUs * 1000) + inputSurface.swapBuffers() + currentLastPts = bufferInfo.presentationTimeUs + onRendered(currentLastPts) + } else { + // Drop frame (don't render to surface) + decoder.releaseOutputBuffer(outputIndex, false) + } + } else { + decoder.releaseOutputBuffer(outputIndex, false) + encoder.signalEndOfInputStream() + return true } } - - // If decoder flagged EOS, signal encoder - if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { - // This may throw if already signaled - that's fine - try { - // We need access to encoder here - this is handled in the main loop - } catch (_: Exception) {} - } + return false } private fun processAudioPassthrough( - extractor: MediaExtractor, audioTrackIndex: Int, muxer: MediaMuxer, - muxerAudioTrack: Int, - muxerStarted: Boolean + muxerAudioTrack: Int ) { - if (!muxerStarted || muxerAudioTrack < 0) return - - // Need a separate extractor for audio since the first one is used for video + // Need a separate extractor for audio since the first one was used for video val audioExtractor = MediaExtractor() if (uri.startsWith("content://") || uri.startsWith("file://")) { audioExtractor.setDataSource(context, Uri.parse(uri), null) @@ -367,162 +396,13 @@ class VideoCompressor( } } - private fun processAudioReencode( - extractor: MediaExtractor, - audioTrackIndex: Int, - audioFormat: MediaFormat, - muxer: MediaMuxer, - muxerStarted: Boolean - ) { - // Set up separate extractor for audio - val audioExtractor = MediaExtractor() - if (uri.startsWith("content://") || uri.startsWith("file://")) { - audioExtractor.setDataSource(context, Uri.parse(uri), null) - } else { - audioExtractor.setDataSource(uri) - } - audioExtractor.selectTrack(audioTrackIndex) - - val audioMime = audioFormat.getString(MediaFormat.KEY_MIME) ?: "audio/mp4a-latm" - val sampleRate = if (audioFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE)) { - audioFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) - } else { - AUDIO_SAMPLE_RATE - } - val channelCount = if (audioFormat.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) { - audioFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT) - } else { - AUDIO_CHANNELS - } - - // Audio decoder - val audioDecoder = MediaCodec.createDecoderByType(audioMime) - audioDecoder.configure(audioFormat, null, null, 0) - audioDecoder.start() - - // Audio encoder - val audioEncoderFormat = MediaFormat.createAudioFormat( - MediaFormat.MIMETYPE_AUDIO_AAC, - sampleRate, - channelCount - ).apply { - setInteger(MediaFormat.KEY_BIT_RATE, AUDIO_AAC_BITRATE) - setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC) - } - val audioEncoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC) - audioEncoder.configure(audioEncoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) - audioEncoder.start() - - var muxerTrack = -1 - var localMuxerStarted = muxerStarted - val bufferInfo = MediaCodec.BufferInfo() - var inputDone = false - var decoderDone = false - var encoderDone = false - - try { - while (!encoderDone && !isCancelled) { - // Feed decoder - if (!inputDone) { - val inputIndex = audioDecoder.dequeueInputBuffer(TIMEOUT_US) - if (inputIndex >= 0) { - val inputBuffer = audioDecoder.getInputBuffer(inputIndex) ?: continue - val sampleSize = audioExtractor.readSampleData(inputBuffer, 0) - if (sampleSize < 0) { - audioDecoder.queueInputBuffer( - inputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM - ) - inputDone = true - } else { - audioDecoder.queueInputBuffer( - inputIndex, 0, sampleSize, audioExtractor.sampleTime, 0 - ) - audioExtractor.advance() - } - } - } - - // Drain decoder -> feed encoder - if (!decoderDone) { - val decoderOutputIndex = audioDecoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_US) - if (decoderOutputIndex >= 0) { - val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 - val decodedBuffer = audioDecoder.getOutputBuffer(decoderOutputIndex) - - if (decodedBuffer != null && bufferInfo.size > 0) { - val encoderInputIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_US) - if (encoderInputIndex >= 0) { - val encoderInputBuffer = audioEncoder.getInputBuffer(encoderInputIndex) - if (encoderInputBuffer != null) { - encoderInputBuffer.clear() - encoderInputBuffer.put(decodedBuffer) - audioEncoder.queueInputBuffer( - encoderInputIndex, 0, bufferInfo.size, - bufferInfo.presentationTimeUs, - if (isEos) MediaCodec.BUFFER_FLAG_END_OF_STREAM else 0 - ) - } - } - } else if (isEos) { - val encoderInputIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_US) - if (encoderInputIndex >= 0) { - audioEncoder.queueInputBuffer( - encoderInputIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM - ) - } - } - - audioDecoder.releaseOutputBuffer(decoderOutputIndex, false) - if (isEos) decoderDone = true - } - } - - // Drain encoder -> muxer - val encoderOutputIndex = audioEncoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_US) - when { - encoderOutputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { - if (muxerTrack < 0) { - muxerTrack = muxer.addTrack(audioEncoder.outputFormat) - if (!localMuxerStarted) { - muxer.start() - localMuxerStarted = true - } - } - } - encoderOutputIndex >= 0 -> { - val outputBuffer = audioEncoder.getOutputBuffer(encoderOutputIndex) - if (outputBuffer != null && - bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG == 0 && - bufferInfo.size > 0 && - muxerTrack >= 0) { - muxer.writeSampleData(muxerTrack, outputBuffer, bufferInfo) - } - encoderDone = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 - audioEncoder.releaseOutputBuffer(encoderOutputIndex, false) - } - } - } - } finally { - try { audioDecoder.stop() } catch (_: Exception) {} - try { audioDecoder.release() } catch (_: Exception) {} - try { audioEncoder.stop() } catch (_: Exception) {} - try { audioEncoder.release() } catch (_: Exception) {} - audioExtractor.release() - } - } - private fun canPassthroughAudio(format: MediaFormat): Boolean { val mime = format.getString(MediaFormat.KEY_MIME) ?: return false - if (mime != MediaFormat.MIMETYPE_AUDIO_AAC) return false - - // Check bitrate if available - if (format.containsKey(MediaFormat.KEY_BIT_RATE)) { - val bitrate = format.getInteger(MediaFormat.KEY_BIT_RATE) - return bitrate <= AUDIO_AAC_BITRATE - } - - // If no bitrate info, assume we should passthrough AAC - return true + // Always passthrough AAC regardless of bitrate — the server re-encodes + // everything anyway, and re-encoding audio here complicates muxer startup + // (MediaMuxer needs all tracks added before start(), but audio encoder + // output format isn't known until it produces FORMAT_CHANGED). + return mime == MediaFormat.MIMETYPE_AUDIO_AAC } private fun calculateOutputSize( diff --git a/modules/expo-bluesky-video-compress/index.ts b/modules/expo-bluesky-video-compress/index.ts index 11f771f366..1cadd0b806 100644 --- a/modules/expo-bluesky-video-compress/index.ts +++ b/modules/expo-bluesky-video-compress/index.ts @@ -9,6 +9,13 @@ import { export type {CompressOptions, CompressResult, VideoMetadata} +class AbortError extends Error { + name = 'AbortError' + constructor() { + super('Aborted') + } +} + let jobIdCounter = 0 export function probe(uri: string): Promise { @@ -27,7 +34,7 @@ export function compress( let subscription: EventSubscription | undefined if (callbacks?.signal?.aborted) { - return Promise.reject(new DOMException('Aborted', 'AbortError')) + return Promise.reject(new AbortError()) } return new Promise((resolve, reject) => { @@ -45,7 +52,7 @@ export function compress( const abortHandler = () => { NativeModule.cancel() subscription?.remove() - reject(new DOMException('Aborted', 'AbortError')) + reject(new AbortError()) } if (callbacks?.signal) { diff --git a/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift b/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift index 985debd594..f71e93e3c7 100644 --- a/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift +++ b/modules/expo-bluesky-video-compress/ios/VideoCompressor.swift @@ -357,13 +357,8 @@ class VideoCompressor { let mediaSubType = CMFormatDescriptionGetMediaSubType(formatDesc) - // Only passthrough AAC audio - guard mediaSubType == kAudioFormatMPEG4AAC else { - return false - } - - // Check bitrate - passthrough if <= 128kbps - let estimatedDataRate = try await audioTrack.load(.estimatedDataRate) - return estimatedDataRate <= 128_000 + // Always passthrough AAC regardless of bitrate — the server re-encodes + // everything anyway, and the bitrate difference is negligible. + return mediaSubType == kAudioFormatMPEG4AAC } } diff --git a/src/lib/media/video/compress.new.ts b/src/lib/media/video/compress.new.ts index 78d610418a..e78c3779c2 100644 --- a/src/lib/media/video/compress.new.ts +++ b/src/lib/media/video/compress.new.ts @@ -1,11 +1,12 @@ import {type ImagePickerAsset} from 'expo-image-picker' import {SUPPORTED_MIME_TYPES, type SupportedMimeTypes} from '#/lib/constants' +import {logger} from '#/logger' import {compress, probe} from '../../../../modules/expo-bluesky-video-compress' import {type CompressedVideo} from './types' // Skip compression if bitrate is at/below this threshold (bps) -const PASSTHROUGH_BITRATE = 3_300_000 +const PASSTHROUGH_BITRATE = 5_000_000 // Max dimension that doesn't need downscaling const PASSTHROUGH_MAX_DIMENSION = 1920 // Max file size the server accepts (bytes) @@ -62,25 +63,41 @@ function shouldCompress( metadata: {bitrate: number; width: number; height: number; fileSize: number}, isAcceptableFormat: boolean, ): boolean { + const maxDimension = Math.max(metadata.width, metadata.height) + const bitrateKbps = Math.round(metadata.bitrate / 1000) + const sizeMB = (metadata.fileSize / 1_000_000).toFixed(1) + // Always compress unacceptable formats (e.g. MOV → MP4) if (!isAcceptableFormat) { + logger.debug('shouldCompress: yes (unsupported format)') return true } // Must compress if over upload limit if (metadata.fileSize > MAX_UPLOAD_SIZE) { + logger.debug(`shouldCompress: yes (file too large: ${sizeMB}MB)`) return true } // Skip if already low bitrate, small resolution, and under upload limit - const maxDimension = Math.max(metadata.width, metadata.height) if ( metadata.bitrate <= PASSTHROUGH_BITRATE && - maxDimension <= PASSTHROUGH_MAX_DIMENSION && - metadata.fileSize <= MAX_UPLOAD_SIZE + maxDimension <= PASSTHROUGH_MAX_DIMENSION ) { + logger.debug( + `shouldCompress: no (${bitrateKbps}kbps, ${maxDimension}px, ${sizeMB}MB)`, + ) return false } + if (metadata.bitrate > PASSTHROUGH_BITRATE) { + logger.debug( + `shouldCompress: yes (bitrate ${bitrateKbps}kbps > ${PASSTHROUGH_BITRATE / 1000}kbps)`, + ) + } else { + logger.debug( + `shouldCompress: yes (dimension ${maxDimension}px > ${PASSTHROUGH_MAX_DIMENSION}px)`, + ) + } return true } diff --git a/src/lib/media/video/compress.ts b/src/lib/media/video/compress.ts index 32c9867ed4..fa39da0958 100644 --- a/src/lib/media/video/compress.ts +++ b/src/lib/media/video/compress.ts @@ -1,10 +1,15 @@ import {type ImagePickerAsset} from 'expo-image-picker' +import * as Toast from '#/components/Toast' +import {IS_DEV} from '#/env' import {type CompressedVideo} from './types' // Toggle for A/B comparison. Set to true to use the new Expo module. const USE_NEW_COMPRESSOR = false +// In dev, run both compressors and show a comparison toast. +const DEV_COMPARE_BOTH = true + export async function compressVideo( file: ImagePickerAsset, opts?: { @@ -12,6 +17,10 @@ export async function compressVideo( onProgress?: (progress: number) => void }, ): Promise { + if (IS_DEV && DEV_COMPARE_BOTH) { + return compressAndCompare(file, opts) + } + if (USE_NEW_COMPRESSOR) { const {compressVideo: compressVideoNew} = await import('./compress.new') return compressVideoNew(file, opts) @@ -22,3 +31,35 @@ export async function compressVideo( return compressVideoLegacy(file, opts) } } + +async function compressAndCompare( + file: ImagePickerAsset, + opts?: { + signal?: AbortSignal + onProgress?: (progress: number) => void + }, +): Promise { + const {compressVideo: compressVideoLegacy} = await import('./compress.legacy') + const {compressVideo: compressVideoNew} = await import('./compress.new') + + // Run legacy first (progress goes to UI) + const legacyStart = performance.now() + const legacyResult = await compressVideoLegacy(file, opts) + const legacyMs = performance.now() - legacyStart + + // Run new second + const newStart = performance.now() + const newResult = await compressVideoNew(file, opts) + const newMs = performance.now() - newStart + + const fmt = (ms: number, size: number) => + `${(ms / 1000).toFixed(1)}s → ${(size / 1_000_000).toFixed(1)}MB` + + Toast.show( + `legacy: ${fmt(legacyMs, legacyResult.size)}\nnew: ${fmt(newMs, newResult.size)}`, + {duration: 8000}, + ) + + // Return whichever is selected + return USE_NEW_COMPRESSOR ? newResult : legacyResult +}