Replace react-native-compressor video path with native expo-bluesky-video-compress module [APP-2428] (#10954)

Co-authored-by: Eric Bailey <git@esb.lol>
This commit is contained in:
Spence Pope
2026-06-30 11:59:59 -04:00
committed by GitHub
parent 43d2f939c2
commit 95726c6c6b
24 changed files with 2097 additions and 7 deletions
@@ -0,0 +1,34 @@
apply plugin: 'com.android.library'
group = 'expo.modules.blueskyvideocompress'
version = '1.0.0'
def expoModulesCorePlugin = new File(project(":expo-modules-core").projectDir.absolutePath, "ExpoModulesCorePlugin.gradle")
apply from: expoModulesCorePlugin
applyKotlinExpoModulesCorePlugin()
useCoreDependencies()
useExpoPublishing()
buildscript {
ext.safeExtGet = { prop, fallback ->
rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
}
}
android {
compileSdkVersion safeExtGet("compileSdkVersion", 34)
namespace "expo.modules.blueskyvideocompress"
defaultConfig {
minSdkVersion safeExtGet("minSdkVersion", 21)
targetSdkVersion safeExtGet("targetSdkVersion", 34)
versionCode 1
versionName "1.0.0"
}
lintOptions {
abortOnError false
}
}
dependencies {
}
@@ -0,0 +1,55 @@
package expo.modules.blueskyvideocompress
import android.media.MediaCodecInfo
import android.media.MediaCodecList
import android.media.MediaFormat
import android.os.Build
object CodecSelector {
// Source: https://github.com/numandev1/react-native-compressor/blob/f949b0868055178e7c8753e05202f784b1bcd589/android/src/main/java/com/reactnativecompressor/Video/VideoCompressor/compressor/Compressor.kt#L500
private val AVC_DENYLIST = setOf(
"c2.qti.avc.encoder"
)
private val SOFTWARE_PREFIXES = listOf(
"OMX.google.",
"c2.android.",
"c2.google."
)
data class EncoderInfo(
val name: String,
val mime: String,
val isHardware: Boolean
)
fun findEncoder(mime: String, requireHardware: Boolean): EncoderInfo? {
val codecList = MediaCodecList(MediaCodecList.REGULAR_CODECS)
val candidates = codecList.codecInfos
.filter { it.isEncoder }
.filter { it.supportedTypes.any { t -> t.equals(mime, ignoreCase = true) } }
.filter { !it.name.contains("secure", ignoreCase = true) }
.filter { !(mime == MediaFormat.MIMETYPE_VIDEO_AVC && AVC_DENYLIST.contains(it.name)) }
val hardware = candidates.filter { isHardware(it) }
val selected = if (requireHardware) {
hardware.firstOrNull()
} else {
hardware.firstOrNull() ?: candidates.firstOrNull()
}
selected ?: return null
return EncoderInfo(
name = selected.name,
mime = mime,
isHardware = isHardware(selected)
)
}
private fun isHardware(info: MediaCodecInfo): Boolean {
return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
info.isHardwareAccelerated
} else {
SOFTWARE_PREFIXES.none { info.name.startsWith(it, ignoreCase = true) }
}
}
}
@@ -0,0 +1,62 @@
package expo.modules.blueskyvideocompress
import expo.modules.kotlin.modules.Module
import expo.modules.kotlin.modules.ModuleDefinition
import java.util.concurrent.ConcurrentHashMap
class ExpoBlueskyVideoCompressModule : Module() {
private val activeCompressors = ConcurrentHashMap<Int, VideoCompressor>()
override fun definition() = ModuleDefinition {
Name("ExpoBlueskyVideoCompress")
Events("onProgress")
AsyncFunction("probe") { uri: String ->
val context = appContext.reactContext
?: throw Error("React context is null")
return@AsyncFunction VideoProber.probe(context, uri)
}
AsyncFunction("compress") { uri: String, options: Map<String, Any?> ->
val context = appContext.reactContext
?: throw Error("React context is null")
val targetBitrate = (options["targetBitrate"] as? Number)?.toInt() ?: 0
val maxSize = (options["maxSize"] as? Number)?.toInt() ?: 1920
val codecPref = (options["codec"] as? String) ?: "auto"
val frameRateCap = ((options["frameRateCap"] as? Number)?.toInt() ?: 30).coerceAtLeast(1)
val jobId = (options["jobId"] as? Number)?.toInt() ?: 0
val compressor = VideoCompressor(
context = context,
uriString = uri,
targetBitrate = targetBitrate,
maxSize = maxSize,
codecPref = codecPref,
frameRateCap = frameRateCap,
jobId = jobId,
onProgress = { id, progress ->
sendEvent("onProgress", mapOf(
"id" to id,
"progress" to progress
))
}
)
activeCompressors[jobId] = compressor
try {
val result = compressor.compress()
activeCompressors.remove(jobId)
return@AsyncFunction result
} catch (e: Exception) {
activeCompressors.remove(jobId)
throw e
}
}
Function("cancel") { jobId: Int ->
activeCompressors.remove(jobId)?.cancel()
}
}
}
@@ -0,0 +1,96 @@
package expo.modules.blueskyvideocompress
import android.opengl.EGL14
import android.opengl.EGLConfig
import android.opengl.EGLContext
import android.opengl.EGLDisplay
import android.opengl.EGLExt
import android.opengl.EGLSurface
import android.view.Surface
class InputSurface(private val surface: Surface) {
private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY
private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT
private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE
init {
eglSetup()
}
private fun eglSetup() {
eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY)
if (eglDisplay === EGL14.EGL_NO_DISPLAY) {
throw RuntimeException("unable to get EGL14 display")
}
val version = IntArray(2)
if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) {
throw RuntimeException("unable to initialize EGL14")
}
val attribList = intArrayOf(
EGL14.EGL_RED_SIZE, 8,
EGL14.EGL_GREEN_SIZE, 8,
EGL14.EGL_BLUE_SIZE, 8,
EGL14.EGL_ALPHA_SIZE, 8,
EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT,
EGL14.EGL_SURFACE_TYPE, EGL14.EGL_WINDOW_BIT,
EGL14.EGL_NONE
)
val configs = arrayOfNulls<EGLConfig>(1)
val numConfigs = IntArray(1)
EGL14.eglChooseConfig(eglDisplay, attribList, 0, configs, 0, 1, numConfigs, 0)
checkEglError("eglChooseConfig")
val contextAttribs = intArrayOf(
EGL14.EGL_CONTEXT_CLIENT_VERSION, 2,
EGL14.EGL_NONE
)
eglContext = EGL14.eglCreateContext(
eglDisplay, configs[0], EGL14.EGL_NO_CONTEXT, contextAttribs, 0
)
checkEglError("eglCreateContext")
val surfaceAttribs = intArrayOf(EGL14.EGL_NONE)
eglSurface = EGL14.eglCreateWindowSurface(
eglDisplay, configs[0], surface, surfaceAttribs, 0
)
checkEglError("eglCreateWindowSurface")
}
fun makeCurrent() {
EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext)
checkEglError("eglMakeCurrent")
}
fun swapBuffers(): Boolean {
return EGL14.eglSwapBuffers(eglDisplay, eglSurface)
}
fun setPresentationTime(nsecs: Long) {
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, nsecs)
}
fun release() {
if (eglDisplay !== EGL14.EGL_NO_DISPLAY) {
EGL14.eglMakeCurrent(
eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT
)
EGL14.eglDestroySurface(eglDisplay, eglSurface)
EGL14.eglDestroyContext(eglDisplay, eglContext)
EGL14.eglReleaseThread()
EGL14.eglTerminate(eglDisplay)
}
surface.release()
eglDisplay = EGL14.EGL_NO_DISPLAY
eglContext = EGL14.EGL_NO_CONTEXT
eglSurface = EGL14.EGL_NO_SURFACE
}
private fun checkEglError(msg: String) {
val error = EGL14.eglGetError()
if (error != EGL14.EGL_SUCCESS) {
throw RuntimeException("$msg: EGL error: 0x${Integer.toHexString(error)}")
}
}
}
@@ -0,0 +1,61 @@
package expo.modules.blueskyvideocompress
import android.graphics.SurfaceTexture
import android.os.Handler
import android.os.HandlerThread
import android.view.Surface
class OutputSurface : SurfaceTexture.OnFrameAvailableListener {
private val renderer = TextureRenderer()
private var surfaceTexture: SurfaceTexture? = null
private val stMatrix = FloatArray(16)
private val callbackThread = HandlerThread("OutputSurfaceCallbacks")
val surface: Surface
@Volatile
private var frameAvailable = false
private val frameSyncObject = Object()
init {
renderer.surfaceCreated()
callbackThread.start()
val handler = Handler(callbackThread.looper)
surfaceTexture = SurfaceTexture(renderer.getTextureId()).also {
it.setOnFrameAvailableListener(this, handler)
}
surface = Surface(surfaceTexture)
}
fun release() {
surface.release()
surfaceTexture?.release()
surfaceTexture = null
callbackThread.quitSafely()
}
fun awaitNewImage() {
val timeoutMs = 2500L
synchronized(frameSyncObject) {
while (!frameAvailable) {
frameSyncObject.wait(timeoutMs)
if (!frameAvailable) {
throw RuntimeException("Surface frame wait timed out")
}
}
frameAvailable = false
}
surfaceTexture!!.updateTexImage()
}
fun drawImage() {
surfaceTexture!!.getTransformMatrix(stMatrix)
renderer.drawFrame(stMatrix)
}
override fun onFrameAvailable(st: SurfaceTexture) {
synchronized(frameSyncObject) {
frameAvailable = true
frameSyncObject.notifyAll()
}
}
}
@@ -0,0 +1,156 @@
package expo.modules.blueskyvideocompress
import android.opengl.GLES11Ext
import android.opengl.GLES20
import android.opengl.Matrix
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.nio.FloatBuffer
class TextureRenderer {
companion object {
private const val FLOAT_SIZE_BYTES = 4
private const val STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES
private const val POS_OFFSET = 0
private const val UV_OFFSET = 3
private val VERTICES = floatArrayOf(
-1.0f, -1.0f, 0f, 0f, 0f,
1.0f, -1.0f, 0f, 1f, 0f,
-1.0f, 1.0f, 0f, 0f, 1f,
1.0f, 1.0f, 0f, 1f, 1f,
)
private const val VERTEX_SHADER = """
uniform mat4 uMVPMatrix;
uniform mat4 uSTMatrix;
attribute vec4 aPosition;
attribute vec4 aTextureCoord;
varying vec2 vTextureCoord;
void main() {
gl_Position = uMVPMatrix * aPosition;
vTextureCoord = (uSTMatrix * aTextureCoord).xy;
}
"""
private const val FRAGMENT_SHADER = """
#extension GL_OES_EGL_image_external : require
precision mediump float;
varying vec2 vTextureCoord;
uniform samplerExternalOES sTexture;
void main() {
gl_FragColor = texture2D(sTexture, vTextureCoord);
}
"""
}
private val vertices: FloatBuffer =
ByteBuffer.allocateDirect(VERTICES.size * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
.apply { put(VERTICES); position(0) }
private val mvpMatrix = FloatArray(16)
private var program = 0
private var textureId = -1
private var uMVPMatrixHandle = 0
private var uSTMatrixHandle = 0
private var aPositionHandle = 0
private var aTextureCoordHandle = 0
init {
Matrix.setIdentityM(mvpMatrix, 0)
}
fun getTextureId(): Int = textureId
fun surfaceCreated() {
program = createProgram(VERTEX_SHADER, FRAGMENT_SHADER)
aPositionHandle = GLES20.glGetAttribLocation(program, "aPosition")
aTextureCoordHandle = GLES20.glGetAttribLocation(program, "aTextureCoord")
uMVPMatrixHandle = GLES20.glGetUniformLocation(program, "uMVPMatrix")
uSTMatrixHandle = GLES20.glGetUniformLocation(program, "uSTMatrix")
val textures = IntArray(1)
GLES20.glGenTextures(1, textures, 0)
textureId = textures[0]
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId)
GLES20.glTexParameterf(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR.toFloat()
)
GLES20.glTexParameterf(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat()
)
GLES20.glTexParameteri(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE
)
GLES20.glTexParameteri(
GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE
)
}
fun drawFrame(stMatrix: FloatArray) {
GLES20.glClearColor(0f, 0f, 0f, 1f)
GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT or GLES20.GL_COLOR_BUFFER_BIT)
GLES20.glUseProgram(program)
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId)
vertices.position(POS_OFFSET)
GLES20.glVertexAttribPointer(
aPositionHandle, 3, GLES20.GL_FLOAT, false, STRIDE_BYTES, vertices
)
GLES20.glEnableVertexAttribArray(aPositionHandle)
vertices.position(UV_OFFSET)
GLES20.glVertexAttribPointer(
aTextureCoordHandle, 2, GLES20.GL_FLOAT, false, STRIDE_BYTES, vertices
)
GLES20.glEnableVertexAttribArray(aTextureCoordHandle)
GLES20.glUniformMatrix4fv(uMVPMatrixHandle, 1, false, mvpMatrix, 0)
GLES20.glUniformMatrix4fv(uSTMatrixHandle, 1, false, stMatrix, 0)
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
GLES20.glDisableVertexAttribArray(aPositionHandle)
GLES20.glDisableVertexAttribArray(aTextureCoordHandle)
}
private fun createProgram(vertexSource: String, fragmentSource: String): Int {
val vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource)
val fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource)
val program = GLES20.glCreateProgram()
GLES20.glAttachShader(program, vertexShader)
GLES20.glAttachShader(program, fragmentShader)
GLES20.glLinkProgram(program)
val linkStatus = IntArray(1)
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0)
if (linkStatus[0] != GLES20.GL_TRUE) {
val log = GLES20.glGetProgramInfoLog(program)
GLES20.glDeleteProgram(program)
throw RuntimeException("Could not link program: $log")
}
return program
}
private fun loadShader(type: Int, source: String): Int {
val shader = GLES20.glCreateShader(type)
GLES20.glShaderSource(shader, source)
GLES20.glCompileShader(shader)
val compiled = IntArray(1)
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0)
if (compiled[0] == 0) {
val log = GLES20.glGetShaderInfoLog(shader)
GLES20.glDeleteShader(shader)
throw RuntimeException("Could not compile shader $type: $log")
}
return shader
}
}
@@ -0,0 +1,582 @@
package expo.modules.blueskyvideocompress
import android.content.Context
import android.media.MediaCodec
import android.media.MediaCodecInfo
import android.media.MediaExtractor
import android.media.MediaFormat
import android.media.MediaMuxer
import android.net.Uri
import android.os.Build
import android.util.Log
import java.io.File
import java.nio.ByteBuffer
import java.util.UUID
class VideoCompressor(
private val context: Context,
private val uriString: String,
private val targetBitrate: Int,
private val maxSize: Int,
private val codecPref: String,
private val frameRateCap: Int,
private val jobId: Int,
private val onProgress: (Int, Double) -> Unit
) {
companion object {
private const val TAG = "BskyVideoCompress"
private const val TIMEOUT_DEQUEUE = 100L
private const val I_FRAME_INTERVAL = 3
}
@Volatile
private var isCancelled = false
fun cancel() {
isCancelled = true
}
fun compress(): Map<String, Any> {
// 'auto' targets h264 — server pipeline is HLS, which favors h264 (HEVC needs
// fMP4 segments + commercial licensing). HEVC remains opt-in via codec: 'hevc'.
val tryHevc = codecPref == "hevc"
if (tryHevc) {
try {
return doCompress(useHevc = true, allowSoftwareFallback = false)
} catch (e: Exception) {
if (codecPref == "hevc" || isCancelled) throw e
Log.w(TAG, "HEVC encode failed, falling back to h264", e)
}
}
return doCompress(useHevc = false, allowSoftwareFallback = true)
}
private fun doCompress(useHevc: Boolean, allowSoftwareFallback: Boolean): Map<String, Any> {
val mime = if (useHevc) MediaFormat.MIMETYPE_VIDEO_HEVC else MediaFormat.MIMETYPE_VIDEO_AVC
val encoderInfo = CodecSelector.findEncoder(mime, requireHardware = !allowSoftwareFallback)
?: throw RuntimeException("No encoder for $mime")
try {
return runPipeline(encoderInfo, useHevc)
} catch (e: Exception) {
if (!allowSoftwareFallback || isCancelled || !encoderInfo.isHardware) throw e
Log.w(TAG, "Hardware encoder ${encoderInfo.name} failed, trying software", e)
val sw = CodecSelector.findEncoder(mime, requireHardware = false)
?.takeIf { !it.isHardware }
?: throw e
return runPipeline(sw, useHevc)
}
}
private fun runPipeline(
encoderInfo: CodecSelector.EncoderInfo,
useHevc: Boolean
): Map<String, Any> {
val outputFile = File(context.cacheDir, "${UUID.randomUUID()}.mp4")
var extractor: MediaExtractor? = null
var muxer: MediaMuxer? = null
var encoder: MediaCodec? = null
var decoder: MediaCodec? = null
var inputSurface: InputSurface? = null
var outputSurface: OutputSurface? = null
var muxerStarted = false
var outputDims = Pair(0, 0)
var durationUs = 0L
try {
val uri = Uri.parse(uriString)
extractor = MediaExtractor()
if (uriString.startsWith("content://") || uriString.startsWith("file://")) {
extractor.setDataSource(context, uri, null)
} else {
extractor.setDataSource(uriString)
}
var videoTrackIndex = -1
var audioTrackIndex = -1
var videoFormat: MediaFormat? = null
var audioFormat: MediaFormat? = null
for (i in 0 until extractor.trackCount) {
val format = extractor.getTrackFormat(i)
val trackMime = format.getString(MediaFormat.KEY_MIME) ?: continue
if (trackMime.startsWith("video/") && videoTrackIndex == -1) {
videoTrackIndex = i
videoFormat = format
} else if (trackMime.startsWith("audio/") && audioTrackIndex == -1) {
audioTrackIndex = i
audioFormat = format
}
}
if (videoTrackIndex == -1 || videoFormat == null) {
throw RuntimeException("No video track found")
}
val sourceWidth = videoFormat.getInteger(MediaFormat.KEY_WIDTH)
val sourceHeight = videoFormat.getInteger(MediaFormat.KEY_HEIGHT)
val rotation = if (videoFormat.containsKey(MediaFormat.KEY_ROTATION)) {
videoFormat.getInteger(MediaFormat.KEY_ROTATION)
} else 0
durationUs = if (videoFormat.containsKey(MediaFormat.KEY_DURATION)) {
videoFormat.getLong(MediaFormat.KEY_DURATION)
} else 0L
val sourceFps = if (videoFormat.containsKey(MediaFormat.KEY_FRAME_RATE)) {
videoFormat.getInteger(MediaFormat.KEY_FRAME_RATE)
} else 30
outputDims = calculateOutputDims(sourceWidth, sourceHeight, rotation, maxSize)
val shouldPassthroughAudio = audioFormat != null && canPassthroughAudio(audioFormat)
val transcodedAudio: TranscodedAudio? = if (
audioTrackIndex >= 0 && audioFormat != null && !shouldPassthroughAudio
) {
transcodeAudioToAAC(audioTrackIndex, audioFormat)
} else null
muxer = MediaMuxer(outputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
val effectiveBitrate = if (targetBitrate > 0) {
targetBitrate
} else if (useHevc) 2_500_000 else 3_000_000
val encoderFormat = MediaFormat.createVideoFormat(
encoderInfo.mime, outputDims.first, outputDims.second
).apply {
setInteger(
MediaFormat.KEY_COLOR_FORMAT,
MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface
)
setInteger(MediaFormat.KEY_BIT_RATE, effectiveBitrate)
setInteger(
MediaFormat.KEY_BITRATE_MODE,
MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR
)
setInteger(MediaFormat.KEY_FRAME_RATE, frameRateCap)
setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
setInteger(MediaFormat.KEY_COLOR_STANDARD, MediaFormat.COLOR_STANDARD_BT709)
setInteger(MediaFormat.KEY_COLOR_TRANSFER, MediaFormat.COLOR_TRANSFER_SDR_VIDEO)
setInteger(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED)
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
setInteger(MediaFormat.KEY_PRIORITY, 0)
setInteger(MediaFormat.KEY_OPERATING_RATE, frameRateCap)
if (useHevc) {
setInteger(
MediaFormat.KEY_PROFILE,
MediaCodecInfo.CodecProfileLevel.HEVCProfileMain
)
} else {
setInteger(
MediaFormat.KEY_PROFILE,
MediaCodecInfo.CodecProfileLevel.AVCProfileHigh
)
setInteger(
MediaFormat.KEY_LEVEL,
MediaCodecInfo.CodecProfileLevel.AVCLevel41
)
}
}
}
encoder = MediaCodec.createByCodecName(encoderInfo.name)
encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
val encoderInputSurface = encoder.createInputSurface()
inputSurface = InputSurface(encoderInputSurface)
inputSurface.makeCurrent()
outputSurface = OutputSurface()
encoder.start()
decoder = MediaCodec.createDecoderByType(
videoFormat.getString(MediaFormat.KEY_MIME) ?: "video/avc"
)
// Ask the decoder to tone-map HDR (HLG/PQ) sources to SDR. Vendors may
// ignore the hint, but where supported it produces correct BT.709 pixels
// for the encoder rather than HDR pixels mislabeled as SDR.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
videoFormat.setInteger(
MediaFormat.KEY_COLOR_TRANSFER_REQUEST,
MediaFormat.COLOR_TRANSFER_SDR_VIDEO
)
}
decoder.configure(videoFormat, outputSurface.surface, null, 0)
decoder.start()
extractor.selectTrack(videoTrackIndex)
val frameDropEnabled = sourceFps > frameRateCap
val targetFrameIntervalUs = if (frameDropEnabled) 1_000_000L / frameRateCap else 0L
var nextTargetPtsUs = 0L
var muxerVideoTrack = -1
var muxerAudioTrack = -1
val bufferInfo = MediaCodec.BufferInfo()
var inputDone = false
var decoderDone = false
var outputDone = false
var lastProgressMs = 0L
while (!outputDone && !isCancelled) {
if (!inputDone) {
val idx = decoder.dequeueInputBuffer(TIMEOUT_DEQUEUE)
if (idx >= 0) {
val buf = decoder.getInputBuffer(idx)
if (buf != null) {
val sz = extractor.readSampleData(buf, 0)
if (sz < 0) {
decoder.queueInputBuffer(
idx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM
)
inputDone = true
} else {
decoder.queueInputBuffer(idx, 0, sz, extractor.sampleTime, 0)
extractor.advance()
}
}
}
}
if (!decoderDone) {
val status = decoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_DEQUEUE)
if (status >= 0) {
val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
val shouldRender = if (isEos) {
false
} else if (frameDropEnabled) {
if (bufferInfo.presentationTimeUs >= nextTargetPtsUs) {
nextTargetPtsUs = bufferInfo.presentationTimeUs + targetFrameIntervalUs
true
} else false
} else true
decoder.releaseOutputBuffer(status, shouldRender)
if (shouldRender) {
outputSurface.awaitNewImage()
outputSurface.drawImage()
inputSurface.setPresentationTime(bufferInfo.presentationTimeUs * 1000)
inputSurface.swapBuffers()
}
if (isEos) {
encoder.signalEndOfInputStream()
decoderDone = true
}
}
}
var encoderDrained = false
while (!outputDone && !isCancelled && !encoderDrained) {
val encIdx = encoder.dequeueOutputBuffer(bufferInfo, 0)
when {
encIdx == MediaCodec.INFO_TRY_AGAIN_LATER -> encoderDrained = true
encIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
if (!muxerStarted) {
muxerVideoTrack = muxer.addTrack(encoder.outputFormat)
if (audioTrackIndex >= 0 && audioFormat != null) {
if (shouldPassthroughAudio) {
muxerAudioTrack = muxer.addTrack(audioFormat)
} else if (transcodedAudio != null) {
muxerAudioTrack = muxer.addTrack(transcodedAudio.outputFormat)
}
}
muxer.start()
muxerStarted = true
}
}
encIdx >= 0 -> {
val data = encoder.getOutputBuffer(encIdx)
if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) {
bufferInfo.size = 0
}
if (data != null && bufferInfo.size > 0 && muxerStarted) {
muxer.writeSampleData(muxerVideoTrack, data, bufferInfo)
}
val isEos = bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
encoder.releaseOutputBuffer(encIdx, false)
if (isEos) {
outputDone = true
} else if (durationUs > 0) {
val now = System.currentTimeMillis()
if (now - lastProgressMs >= 100) {
lastProgressMs = now
val p = (bufferInfo.presentationTimeUs.toDouble() / durationUs)
.coerceIn(0.0, 1.0)
onProgress(jobId, p)
}
}
}
}
}
}
if (audioTrackIndex >= 0 && muxerAudioTrack >= 0 && muxerStarted && !isCancelled) {
if (shouldPassthroughAudio) {
passthroughAudio(audioTrackIndex, muxer, muxerAudioTrack)
} else if (transcodedAudio != null) {
writeTranscodedAudio(transcodedAudio.samples, muxer, muxerAudioTrack)
}
}
} finally {
try { decoder?.stop() } catch (_: Exception) {}
try { decoder?.release() } catch (_: Exception) {}
try { encoder?.stop() } catch (_: Exception) {}
try { encoder?.release() } catch (_: Exception) {}
try { outputSurface?.release() } catch (_: Exception) {}
try { inputSurface?.release() } catch (_: Exception) {}
try { extractor?.release() } catch (_: Exception) {}
try {
if (muxerStarted) muxer?.stop()
muxer?.release()
} catch (_: Exception) {}
}
if (isCancelled) {
outputFile.delete()
throw RuntimeException("Compression cancelled")
}
onProgress(jobId, 1.0)
val durationSeconds = durationUs / 1_000_000.0
return mapOf(
"uri" to "file://${outputFile.absolutePath}",
"size" to outputFile.length(),
"mimeType" to "video/mp4",
"width" to outputDims.first,
"height" to outputDims.second,
"duration" to durationSeconds,
"codec" to if (useHevc) "hevc" else "h264"
)
}
private fun passthroughAudio(audioTrackIndex: Int, muxer: MediaMuxer, muxerAudioTrack: Int) {
val audioExtractor = MediaExtractor()
if (uriString.startsWith("content://") || uriString.startsWith("file://")) {
audioExtractor.setDataSource(context, Uri.parse(uriString), null)
} else {
audioExtractor.setDataSource(uriString)
}
audioExtractor.selectTrack(audioTrackIndex)
audioExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC)
val buffer = ByteBuffer.allocate(256 * 1024)
val info = MediaCodec.BufferInfo()
try {
while (!isCancelled) {
val sz = audioExtractor.readSampleData(buffer, 0)
if (sz < 0) break
info.offset = 0
info.size = sz
info.presentationTimeUs = audioExtractor.sampleTime
info.flags = audioExtractor.sampleFlags
muxer.writeSampleData(muxerAudioTrack, buffer, info)
audioExtractor.advance()
}
} finally {
audioExtractor.release()
}
}
private fun canPassthroughAudio(format: MediaFormat): Boolean {
val mime = format.getString(MediaFormat.KEY_MIME) ?: return false
return mime == MediaFormat.MIMETYPE_AUDIO_AAC
}
private data class TranscodedAudio(
val outputFormat: MediaFormat,
val samples: List<Sample>
) {
data class Sample(
val bytes: ByteArray,
val presentationTimeUs: Long,
val flags: Int
)
}
// Re-encode non-AAC source audio (Opus, Vorbis, etc.) to AAC so the mp4 muxer
// can take it. iOS always re-encodes to AAC; without this Android would drop
// the audio track entirely.
private fun transcodeAudioToAAC(
audioTrackIndex: Int,
sourceFormat: MediaFormat
): TranscodedAudio? {
val sourceMime = sourceFormat.getString(MediaFormat.KEY_MIME) ?: return null
val sampleRate = if (sourceFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE))
sourceFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100
val channelCount = if (sourceFormat.containsKey(MediaFormat.KEY_CHANNEL_COUNT))
sourceFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT).coerceIn(1, 2) else 2
val audioExtractor = MediaExtractor()
if (uriString.startsWith("content://") || uriString.startsWith("file://")) {
audioExtractor.setDataSource(context, Uri.parse(uriString), null)
} else {
audioExtractor.setDataSource(uriString)
}
audioExtractor.selectTrack(audioTrackIndex)
var decoder: MediaCodec? = null
var encoder: MediaCodec? = null
try {
decoder = MediaCodec.createDecoderByType(sourceMime)
decoder.configure(sourceFormat, null, null, 0)
decoder.start()
val encoderFormat = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, channelCount
).apply {
setInteger(
MediaFormat.KEY_AAC_PROFILE,
MediaCodecInfo.CodecProfileLevel.AACObjectLC
)
setInteger(MediaFormat.KEY_BIT_RATE, 128_000)
setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 256 * 1024)
}
encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
encoder.start()
val samples = mutableListOf<TranscodedAudio.Sample>()
var outputFormat: MediaFormat? = null
var inputDone = false
var decoderDone = false
var encoderInputSignalled = false
var encoderDone = false
val info = MediaCodec.BufferInfo()
while (!encoderDone && !isCancelled) {
if (!inputDone) {
val idx = decoder.dequeueInputBuffer(TIMEOUT_DEQUEUE)
if (idx >= 0) {
val buf = decoder.getInputBuffer(idx)
if (buf != null) {
val sz = audioExtractor.readSampleData(buf, 0)
if (sz < 0) {
decoder.queueInputBuffer(
idx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM
)
inputDone = true
} else {
decoder.queueInputBuffer(
idx, 0, sz, audioExtractor.sampleTime, audioExtractor.sampleFlags
)
audioExtractor.advance()
}
}
}
}
if (!decoderDone) {
val status = decoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE)
when {
status == MediaCodec.INFO_TRY_AGAIN_LATER -> {}
status == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {}
status >= 0 -> {
val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
val data = decoder.getOutputBuffer(status)
if (data != null && info.size > 0) {
val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE)
if (encInIdx >= 0) {
val encInBuf = encoder.getInputBuffer(encInIdx)
if (encInBuf != null) {
encInBuf.clear()
data.position(info.offset)
data.limit(info.offset + info.size)
encInBuf.put(data)
encoder.queueInputBuffer(
encInIdx, 0, info.size, info.presentationTimeUs, 0
)
}
}
}
decoder.releaseOutputBuffer(status, false)
if (isEos) {
if (!encoderInputSignalled) {
val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE * 10)
if (encInIdx >= 0) {
encoder.queueInputBuffer(
encInIdx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM
)
encoderInputSignalled = true
}
}
decoderDone = true
}
}
}
}
val encOutIdx = encoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE)
when {
encOutIdx == MediaCodec.INFO_TRY_AGAIN_LATER -> {}
encOutIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
outputFormat = encoder.outputFormat
}
encOutIdx >= 0 -> {
val data = encoder.getOutputBuffer(encOutIdx)
val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
val isConfig = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0
if (data != null && info.size > 0 && !isConfig) {
val bytes = ByteArray(info.size)
data.position(info.offset)
data.get(bytes, 0, info.size)
samples.add(
TranscodedAudio.Sample(
bytes = bytes,
presentationTimeUs = info.presentationTimeUs,
flags = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG.inv()
)
)
}
encoder.releaseOutputBuffer(encOutIdx, false)
if (isEos) encoderDone = true
}
}
}
val fmt = outputFormat ?: return null
return TranscodedAudio(fmt, samples)
} catch (e: Exception) {
Log.w(TAG, "Audio transcode failed; dropping audio", e)
return null
} finally {
try { decoder?.stop() } catch (_: Exception) {}
try { decoder?.release() } catch (_: Exception) {}
try { encoder?.stop() } catch (_: Exception) {}
try { encoder?.release() } catch (_: Exception) {}
audioExtractor.release()
}
}
private fun writeTranscodedAudio(
samples: List<TranscodedAudio.Sample>,
muxer: MediaMuxer,
muxerAudioTrack: Int
) {
val info = MediaCodec.BufferInfo()
for (sample in samples) {
if (isCancelled) break
val buffer = ByteBuffer.wrap(sample.bytes)
info.offset = 0
info.size = sample.bytes.size
info.presentationTimeUs = sample.presentationTimeUs
info.flags = sample.flags
muxer.writeSampleData(muxerAudioTrack, buffer, info)
}
}
private fun calculateOutputDims(srcW: Int, srcH: Int, rotation: Int, maxSize: Int): Pair<Int, Int> {
val isRotated = rotation == 90 || rotation == 270
val displayW = if (isRotated) srcH else srcW
val displayH = if (isRotated) srcW else srcH
val scale: Float = when {
displayW <= maxSize && displayH <= maxSize -> 1.0f
displayW > displayH -> maxSize.toFloat() / displayW
else -> maxSize.toFloat() / displayH
}
return Pair(
roundToEven((displayW * scale).toInt()),
roundToEven((displayH * scale).toInt())
)
}
private fun roundToEven(v: Int): Int = if (v % 2 == 0) v else v - 1
}
@@ -0,0 +1,125 @@
package expo.modules.blueskyvideocompress
import android.content.Context
import android.media.MediaExtractor
import android.media.MediaFormat
import android.media.MediaMetadataRetriever
import android.net.Uri
import android.os.Build
object VideoProber {
fun probe(context: Context, uriString: String): Map<String, Any> {
val uri = Uri.parse(uriString)
val retriever = MediaMetadataRetriever()
try {
if (uriString.startsWith("content://") || uriString.startsWith("file://")) {
retriever.setDataSource(context, uri)
} else {
retriever.setDataSource(uriString)
}
val width = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_WIDTH)
?.toIntOrNull() ?: 0
val height = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_HEIGHT)
?.toIntOrNull() ?: 0
val durationMs = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)
?.toLongOrNull() ?: 0L
val rotation = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_VIDEO_ROTATION)
?.toIntOrNull() ?: 0
val bitrate = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_BITRATE)
?.toIntOrNull() ?: 0
val hasAudio = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_HAS_AUDIO)
?.equals("yes") ?: false
val frameRate = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_CAPTURE_FRAMERATE)
?.toFloatOrNull() ?: 0f
val fileSize = getFileSize(context, uriString)
val extractor = MediaExtractor()
var codec = "unknown"
var mimeType = "video/mp4"
var extractedFrameRate = frameRate
var isHDR = false
try {
if (uriString.startsWith("content://") || uriString.startsWith("file://")) {
extractor.setDataSource(context, uri, null)
} else {
extractor.setDataSource(uriString)
}
for (i in 0 until extractor.trackCount) {
val format = extractor.getTrackFormat(i)
val mime = format.getString(MediaFormat.KEY_MIME)
if (mime?.startsWith("video/") == true) {
mimeType = mime
codec = mime.removePrefix("video/")
if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
extractedFrameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE).toFloat()
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N &&
format.containsKey(MediaFormat.KEY_COLOR_TRANSFER)
) {
val transfer = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER)
isHDR = transfer == MediaFormat.COLOR_TRANSFER_HLG ||
transfer == MediaFormat.COLOR_TRANSFER_ST2084
}
// Dolby Vision tracks use codec-specific mimes that aren't covered by
// KEY_COLOR_TRANSFER on every device.
if (mime.contains("dolby-vision", ignoreCase = true)) {
isHDR = true
}
break
}
}
} finally {
extractor.release()
}
val durationSeconds = durationMs / 1000.0
val effectiveBitrate = if (bitrate > 0) {
bitrate
} else if (durationSeconds > 0 && fileSize > 0) {
(fileSize * 8 / durationSeconds).toInt()
} else {
0
}
return mapOf(
"width" to width,
"height" to height,
"duration" to durationSeconds,
"bitrate" to effectiveBitrate,
"fileSize" to fileSize,
"mimeType" to mimeType,
"codec" to codec,
"hasAudio" to hasAudio,
"frameRate" to extractedFrameRate.toDouble(),
"rotation" to rotation,
"isHDR" to isHDR
)
} finally {
retriever.release()
}
}
private fun getFileSize(context: Context, uriString: String): Long {
return try {
if (uriString.startsWith("content://")) {
context.contentResolver.openFileDescriptor(Uri.parse(uriString), "r")?.use {
it.statSize
} ?: 0L
} else {
val path = if (uriString.startsWith("file://")) {
uriString.removePrefix("file://")
} else {
uriString
}
java.io.File(path).length()
}
} catch (_: Exception) {
0L
}
}
}
@@ -0,0 +1,9 @@
{
"platforms": ["ios", "android"],
"ios": {
"modules": ["ExpoBlueskyVideoCompressModule"]
},
"android": {
"modules": ["expo.modules.blueskyvideocompress.ExpoBlueskyVideoCompressModule"]
}
}
@@ -0,0 +1,87 @@
import {type EventSubscription} from 'expo-modules-core'
import NativeModule from './src/ExpoBlueskyVideoCompressModule'
import {
type CodecPreference,
type CompressCallbacks,
type CompressOptions,
type CompressResult,
type VideoMetadata,
} from './src/types'
export type {
CodecPreference,
CompressCallbacks,
CompressOptions,
CompressResult,
VideoMetadata,
}
class AbortError extends Error {
name = 'AbortError'
constructor() {
super('Aborted')
}
}
let jobIdCounter = 0
export function probe(uri: string): Promise<VideoMetadata> {
return NativeModule.probe(uri)
}
export function compress(
uri: string,
options: CompressOptions = {},
callbacks?: CompressCallbacks,
): Promise<CompressResult> {
const jobId = ++jobIdCounter
let subscription: EventSubscription | undefined
if (callbacks?.signal?.aborted) {
return Promise.reject(new AbortError())
}
const nativeOptions = {
targetBitrate: options.targetBitrate ?? 0,
maxSize: options.maxSize ?? 1920,
codec: options.codec ?? 'auto',
frameRateCap: options.frameRateCap ?? 30,
jobId,
}
return new Promise<CompressResult>((resolve, reject) => {
if (callbacks?.onProgress) {
subscription = NativeModule.addListener(
'onProgress',
(event: {id: number; progress: number}) => {
if (event.id === jobId) {
callbacks.onProgress!(event.progress)
}
},
)
}
const abortHandler = () => {
NativeModule.cancel(jobId)
subscription?.remove()
reject(new AbortError())
}
if (callbacks?.signal) {
callbacks.signal.addEventListener('abort', abortHandler, {once: true})
}
NativeModule.compress(uri, nativeOptions)
.then(result => {
callbacks?.signal?.removeEventListener('abort', abortHandler)
subscription?.remove()
resolve(result)
})
.catch((error: unknown) => {
callbacks?.signal?.removeEventListener('abort', abortHandler)
subscription?.remove()
reject(error instanceof Error ? error : new Error(String(error)))
})
})
}
@@ -0,0 +1,21 @@
import VideoToolbox
enum CodecCapability {
static let isHardwareHEVCEncodeAvailable: Bool = {
var encoderListCF: CFArray?
let status = VTCopyVideoEncoderList(nil, &encoderListCF)
guard status == noErr, let encoderList = encoderListCF as? [[String: Any]] else {
return false
}
return encoderList.contains { encoder in
guard let codecTypeValue = encoder[kVTVideoEncoderList_CodecType as String] as? Int,
codecTypeValue == Int(kCMVideoCodecType_HEVC) else {
return false
}
if let isHardware = encoder[kVTVideoEncoderList_IsHardwareAccelerated as String] as? Bool {
return isHardware
}
return true
}
}()
}
@@ -0,0 +1,20 @@
Pod::Spec.new do |s|
s.name = 'ExpoBlueskyVideoCompress'
s.version = '1.0.0'
s.summary = 'Hardware-accelerated video compression for Bluesky'
s.description = 'Hardware-accelerated h264/HEVC video compression using AVAssetReader/Writer and VideoToolbox on iOS'
s.author = ''
s.homepage = 'https://github.com/bluesky-social/social-app'
s.platforms = { :ios => '15.1' }
s.source = { git: '' }
s.static_framework = true
s.dependency 'ExpoModulesCore'
s.pod_target_xcconfig = {
'DEFINES_MODULE' => 'YES',
'SWIFT_COMPILATION_MODE' => 'wholemodule'
}
s.source_files = "**/*.{h,m,mm,swift,hpp,cpp}"
end
@@ -0,0 +1,74 @@
import AVFoundation
import ExpoModulesCore
public class ExpoBlueskyVideoCompressModule: Module {
private var activeCompressors: [Int: VideoCompressor] = [:]
private let activeCompressorsLock = NSLock()
public func definition() -> ModuleDefinition {
Name("ExpoBlueskyVideoCompress")
Events("onProgress")
AsyncFunction("probe") { (uri: String) -> [String: Any] in
let url = URL(string: uri) ?? URL(fileURLWithPath: uri)
return try await VideoProber.probe(url: url)
}
AsyncFunction("compress") { (uri: String, options: [String: Any]) -> [String: Any] in
let url = URL(string: uri) ?? URL(fileURLWithPath: uri)
let targetBitrate = options["targetBitrate"] as? Int ?? 0
let maxSize = options["maxSize"] as? Int ?? 1920
let codecPref = options["codec"] as? String ?? "auto"
let frameRateCap = max(1, options["frameRateCap"] as? Int ?? 30)
let jobId = options["jobId"] as? Int ?? 0
let compressor = VideoCompressor(
url: url,
targetBitrate: targetBitrate,
maxSize: maxSize,
codecPref: codecPref,
frameRateCap: frameRateCap,
jobId: jobId,
onProgress: { [weak self] id, progress in
self?.sendEvent("onProgress", [
"id": id,
"progress": progress,
])
}
)
self.setCompressor(jobId, compressor)
do {
let result = try await compressor.compress()
self.setCompressor(jobId, nil)
return result
} catch {
self.setCompressor(jobId, nil)
throw error
}
}
Function("cancel") { (jobId: Int) in
self.cancelCompressor(jobId)
}
}
private func setCompressor(_ jobId: Int, _ compressor: VideoCompressor?) {
activeCompressorsLock.lock()
defer { activeCompressorsLock.unlock() }
if let compressor = compressor {
activeCompressors[jobId] = compressor
} else {
activeCompressors.removeValue(forKey: jobId)
}
}
private func cancelCompressor(_ jobId: Int) {
activeCompressorsLock.lock()
let compressor = activeCompressors.removeValue(forKey: jobId)
activeCompressorsLock.unlock()
compressor?.cancel()
}
}
@@ -0,0 +1,410 @@
import AVFoundation
import VideoToolbox
class VideoCompressor {
private let url: URL
private let targetBitrate: Int
private let maxSize: Int
private let codecPref: String
private let frameRateCap: Int
private let jobId: Int
private let onProgress: (Int, Double) -> Void
private var isCancelled = false
init(
url: URL,
targetBitrate: Int,
maxSize: Int,
codecPref: String,
frameRateCap: Int,
jobId: Int,
onProgress: @escaping (Int, Double) -> Void
) {
self.url = url
self.targetBitrate = targetBitrate
self.maxSize = maxSize
self.codecPref = codecPref
self.frameRateCap = frameRateCap
self.jobId = jobId
self.onProgress = onProgress
}
func cancel() {
isCancelled = true
}
func compress() async throws -> [String: Any] {
let asset = AVURLAsset(
url: url,
options: [AVURLAssetPreferPreciseDurationAndTimingKey: true]
)
let duration = try await asset.load(.duration)
let totalSeconds = CMTimeGetSeconds(duration)
guard totalSeconds > 0 else { throw err("Invalid video duration", code: 2) }
let videoTracks = try await asset.loadTracks(withMediaType: .video)
guard let videoTrack = videoTracks.first else {
throw err("No video track found", code: 1)
}
let naturalSize = try await videoTrack.load(.naturalSize)
let preferredTransform = try await videoTrack.load(.preferredTransform)
let rotatedRect = CGRect(origin: .zero, size: naturalSize).applying(preferredTransform)
let displaySize = CGSize(
width: abs(rotatedRect.width),
height: abs(rotatedRect.height)
)
let outputSize = scaleEvenly(displaySize: displaySize, maxSize: maxSize)
let audioTracks = try await asset.loadTracks(withMediaType: .audio)
// 'auto' targets h264 server pipeline is HLS, which favors h264 (HEVC needs
// fMP4 segments + commercial licensing). HEVC remains opt-in via codec: 'hevc'.
let useHEVC: Bool
switch codecPref {
case "hevc": useHEVC = true
default: useHEVC = false
}
let codecType: AVVideoCodecType = useHEVC ? .hevc : .h264
let profileLevel: String = useHEVC
? kVTProfileLevel_HEVC_Main_AutoLevel as String
: kVTProfileLevel_H264_High_AutoLevel as String
let effectiveBitrate = targetBitrate > 0
? targetBitrate
: (useHEVC ? 2_500_000 : 3_000_000)
let outputURL = FileManager.default.temporaryDirectory
.appendingPathComponent(UUID().uuidString)
.appendingPathExtension("mp4")
let videoComposition = makeRotatingComposition(
videoTrack: videoTrack,
preferredTransform: preferredTransform,
naturalSize: naturalSize,
outputSize: outputSize,
duration: duration
)
let reader = try AVAssetReader(asset: asset)
let videoReaderSettings: [String: Any] = [
kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA
]
let videoReaderOutput = AVAssetReaderVideoCompositionOutput(
videoTracks: [videoTrack],
videoSettings: videoReaderSettings
)
videoReaderOutput.videoComposition = videoComposition
videoReaderOutput.alwaysCopiesSampleData = false
guard reader.canAdd(videoReaderOutput) else {
throw err("Cannot read video track", code: 3)
}
reader.add(videoReaderOutput)
var compressionProps: [String: Any] = [
AVVideoAverageBitRateKey: effectiveBitrate,
AVVideoProfileLevelKey: profileLevel,
AVVideoMaxKeyFrameIntervalKey: max(frameRateCap * 3, 30),
AVVideoExpectedSourceFrameRateKey: frameRateCap,
AVVideoAllowFrameReorderingKey: false,
kVTCompressionPropertyKey_RealTime as String: true,
]
let peakBytesPerSecond = Int(Double(effectiveBitrate) / 8.0 * 1.5)
compressionProps[kVTCompressionPropertyKey_DataRateLimits as String] = [
peakBytesPerSecond, 1.0
] as CFArray
let videoColorProps: [String: Any] = [
AVVideoColorPrimariesKey: AVVideoColorPrimaries_ITU_R_709_2,
AVVideoTransferFunctionKey: AVVideoTransferFunction_ITU_R_709_2,
AVVideoYCbCrMatrixKey: AVVideoYCbCrMatrix_ITU_R_709_2,
]
let videoWriterSettings: [String: Any] = [
AVVideoCodecKey: codecType,
AVVideoWidthKey: outputSize.width,
AVVideoHeightKey: outputSize.height,
AVVideoColorPropertiesKey: videoColorProps,
AVVideoCompressionPropertiesKey: compressionProps,
]
let videoWriterInput = AVAssetWriterInput(
mediaType: .video,
outputSettings: videoWriterSettings
)
videoWriterInput.expectsMediaDataInRealTime = false
let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4)
writer.shouldOptimizeForNetworkUse = true
writer.metadata = []
guard writer.canAdd(videoWriterInput) else {
throw err("Cannot write video track", code: 4)
}
writer.add(videoWriterInput)
var audioReaderOutput: AVAssetReaderTrackOutput?
var audioWriterInput: AVAssetWriterInput?
if let audioTrack = audioTracks.first {
let audioDecoderSettings: [String: Any] = [
AVFormatIDKey: kAudioFormatLinearPCM,
AVLinearPCMBitDepthKey: 16,
AVLinearPCMIsFloatKey: false,
AVLinearPCMIsBigEndianKey: false,
AVLinearPCMIsNonInterleaved: false,
]
let audioOutput = AVAssetReaderTrackOutput(
track: audioTrack,
outputSettings: audioDecoderSettings
)
audioOutput.alwaysCopiesSampleData = false
if reader.canAdd(audioOutput) {
reader.add(audioOutput)
audioReaderOutput = audioOutput
let audioEncoderSettings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 44100,
AVNumberOfChannelsKey: 2,
AVEncoderBitRateKey: 128_000,
]
let audioInput = AVAssetWriterInput(
mediaType: .audio,
outputSettings: audioEncoderSettings
)
audioInput.expectsMediaDataInRealTime = false
if writer.canAdd(audioInput) {
writer.add(audioInput)
audioWriterInput = audioInput
}
}
}
guard reader.startReading() else {
throw reader.error ?? err("Reader failed to start", code: 8)
}
guard writer.startWriting() else {
throw writer.error ?? err("Writer failed to start", code: 9)
}
writer.startSession(atSourceTime: .zero)
let minFrameIntervalSeconds = 1.0 / Double(frameRateCap)
let minFrameInterval = CMTime(
seconds: minFrameIntervalSeconds,
preferredTimescale: 600
)
await withTaskGroup(of: Void.self) { group in
group.addTask { [self] in
await processVideoTrack(
readerOutput: videoReaderOutput,
writerInput: videoWriterInput,
totalDuration: totalSeconds,
minFrameInterval: minFrameInterval
)
}
if let audioOutput = audioReaderOutput, let audioInput = audioWriterInput {
group.addTask { [self] in
await processAudioTrack(
readerOutput: audioOutput,
writerInput: audioInput
)
}
}
await group.waitForAll()
}
if isCancelled {
writer.cancelWriting()
try? FileManager.default.removeItem(at: outputURL)
throw err("Compression cancelled", code: 5)
}
if reader.status == .failed {
let error = reader.error ?? err("Reader failed", code: 6)
writer.cancelWriting()
try? FileManager.default.removeItem(at: outputURL)
throw error
}
await writer.finishWriting()
if writer.status == .failed {
let error = writer.error ?? err("Writer failed", code: 7)
try? FileManager.default.removeItem(at: outputURL)
throw error
}
onProgress(jobId, 1.0)
let attributes = try FileManager.default.attributesOfItem(atPath: outputURL.path)
let fileSize = attributes[.size] as? Int ?? 0
return [
"uri": outputURL.absoluteString,
"size": fileSize,
"mimeType": "video/mp4",
"width": outputSize.width,
"height": outputSize.height,
"duration": totalSeconds,
"codec": useHEVC ? "hevc" : "h264",
]
}
private func processVideoTrack(
readerOutput: AVAssetReaderOutput,
writerInput: AVAssetWriterInput,
totalDuration: Double,
minFrameInterval: CMTime
) async {
var lastProgressTime: CFAbsoluteTime = 0
var lastAppendedPTS: CMTime?
var finished = false
await withCheckedContinuation { (continuation: CheckedContinuation<Void, Never>) in
writerInput.requestMediaDataWhenReady(
on: DispatchQueue(label: "com.bsky.videocompress.video")
) {
let finish = {
if !finished {
finished = true
writerInput.markAsFinished()
continuation.resume()
}
}
while writerInput.isReadyForMoreMediaData {
if finished { return }
if self.isCancelled {
finish()
return
}
guard let sampleBuffer = readerOutput.copyNextSampleBuffer() else {
finish()
return
}
let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
if let last = lastAppendedPTS {
let delta = CMTimeSubtract(pts, last)
if CMTimeCompare(delta, minFrameInterval) < 0 {
continue
}
}
lastAppendedPTS = pts
if !writerInput.append(sampleBuffer) {
finish()
return
}
let now = CFAbsoluteTimeGetCurrent()
if now - lastProgressTime >= 0.1 {
lastProgressTime = now
let progress = min(CMTimeGetSeconds(pts) / totalDuration, 1.0)
self.onProgress(self.jobId, progress)
}
}
}
}
}
private func processAudioTrack(
readerOutput: AVAssetReaderOutput,
writerInput: AVAssetWriterInput
) async {
var finished = false
await withCheckedContinuation { (continuation: CheckedContinuation<Void, Never>) in
writerInput.requestMediaDataWhenReady(
on: DispatchQueue(label: "com.bsky.videocompress.audio")
) {
let finish = {
if !finished {
finished = true
writerInput.markAsFinished()
continuation.resume()
}
}
while writerInput.isReadyForMoreMediaData {
if finished { return }
if self.isCancelled {
finish()
return
}
guard let sampleBuffer = readerOutput.copyNextSampleBuffer() else {
finish()
return
}
if !writerInput.append(sampleBuffer) {
finish()
return
}
}
}
}
}
private func makeRotatingComposition(
videoTrack: AVAssetTrack,
preferredTransform: CGAffineTransform,
naturalSize: CGSize,
outputSize: (width: Int, height: Int),
duration: CMTime
) -> AVMutableVideoComposition {
let composition = AVMutableVideoComposition()
composition.renderSize = CGSize(width: outputSize.width, height: outputSize.height)
composition.frameDuration = CMTime(value: 1, timescale: Int32(frameRateCap))
let rotatedRect = CGRect(origin: .zero, size: naturalSize).applying(preferredTransform)
let translate = CGAffineTransform(
translationX: -rotatedRect.minX,
y: -rotatedRect.minY
)
let displaySize = CGSize(
width: abs(rotatedRect.width),
height: abs(rotatedRect.height)
)
let scaleX = CGFloat(outputSize.width) / displaySize.width
let scaleY = CGFloat(outputSize.height) / displaySize.height
let scale = CGAffineTransform(scaleX: scaleX, y: scaleY)
let combined = preferredTransform.concatenating(translate).concatenating(scale)
let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: videoTrack)
layerInstruction.setTransform(combined, at: .zero)
let instruction = AVMutableVideoCompositionInstruction()
instruction.timeRange = CMTimeRange(start: .zero, duration: duration)
instruction.layerInstructions = [layerInstruction]
composition.instructions = [instruction]
return composition
}
private func scaleEvenly(displaySize: CGSize, maxSize: Int) -> (width: Int, height: Int) {
let cap = CGFloat(maxSize)
let scale: CGFloat
if displaySize.width <= cap && displaySize.height <= cap {
scale = 1.0
} else if displaySize.width > displaySize.height {
scale = cap / displaySize.width
} else {
scale = cap / displaySize.height
}
return (
roundToEven(Int(displaySize.width * scale)),
roundToEven(Int(displaySize.height * scale))
)
}
private func roundToEven(_ value: Int) -> Int {
return value % 2 == 0 ? value : value - 1
}
private func err(_ message: String, code: Int) -> NSError {
return NSError(
domain: "ExpoBlueskyVideoCompress",
code: code,
userInfo: [NSLocalizedDescriptionKey: message]
)
}
}
@@ -0,0 +1,105 @@
import AVFoundation
import UniformTypeIdentifiers
struct VideoProber {
static func probe(url: URL) async throws -> [String: Any] {
let asset = AVURLAsset(url: url)
let duration = try await asset.load(.duration)
let tracks = try await asset.loadTracks(withMediaType: .video)
guard let videoTrack = tracks.first else {
throw NSError(
domain: "ExpoBlueskyVideoCompress",
code: 1,
userInfo: [NSLocalizedDescriptionKey: "No video track found"]
)
}
let naturalSize = try await videoTrack.load(.naturalSize)
let preferredTransform = try await videoTrack.load(.preferredTransform)
let estimatedDataRate = try await videoTrack.load(.estimatedDataRate)
let nominalFrameRate = try await videoTrack.load(.nominalFrameRate)
let formatDescriptions = try await videoTrack.load(.formatDescriptions)
var codec = "unknown"
var isHDR = false
if let formatDescription = formatDescriptions.first {
let subType = CMFormatDescriptionGetMediaSubType(formatDescription)
codec = fourCCToString(subType)
// Dolby Vision codecs are HDR by definition.
let dolbyVisionSubtypes: Set<String> = ["dvhe", "dvh1", "dvav", "dva1"]
if dolbyVisionSubtypes.contains(codec) {
isHDR = true
} else if let extensions = CMFormatDescriptionGetExtensions(formatDescription)
as? [String: Any]
{
let transferKey = kCMFormatDescriptionExtension_TransferFunction as String
if let transfer = extensions[transferKey] as? String {
let hlg = kCMFormatDescriptionTransferFunction_ITU_R_2100_HLG as String
let pq = kCMFormatDescriptionTransferFunction_SMPTE_ST_2084_PQ as String
isHDR = transfer == hlg || transfer == pq
}
}
}
let rotation = rotationFromTransform(preferredTransform)
let isRotated = rotation == 90 || rotation == 270
let width = isRotated ? Int(naturalSize.height) : Int(naturalSize.width)
let height = isRotated ? Int(naturalSize.width) : Int(naturalSize.height)
let audioTracks = try await asset.loadTracks(withMediaType: .audio)
let hasAudio = !audioTracks.isEmpty
let fileSize: Int
if let attributes = try? FileManager.default.attributesOfItem(atPath: url.path),
let size = attributes[.size] as? Int {
fileSize = size
} else {
fileSize = 0
}
let mimeType: String
if let utType = UTType(filenameExtension: url.pathExtension) {
mimeType = utType.preferredMIMEType ?? "video/mp4"
} else {
mimeType = "video/mp4"
}
let durationSeconds = CMTimeGetSeconds(duration)
var bitrate = Int(estimatedDataRate)
if bitrate == 0 && durationSeconds > 0 {
bitrate = Int(Double(fileSize * 8) / durationSeconds)
}
return [
"width": width,
"height": height,
"duration": durationSeconds,
"bitrate": bitrate,
"fileSize": fileSize,
"mimeType": mimeType,
"codec": codec,
"hasAudio": hasAudio,
"frameRate": nominalFrameRate,
"rotation": rotation,
"isHDR": isHDR
]
}
private static func rotationFromTransform(_ transform: CGAffineTransform) -> Int {
let angle = atan2(transform.b, transform.a)
let degrees = Int(round(angle * 180.0 / .pi))
return ((degrees % 360) + 360) % 360
}
private static func fourCCToString(_ code: FourCharCode) -> String {
let chars: [Character] = [
Character(UnicodeScalar((code >> 24) & 0xFF)!),
Character(UnicodeScalar((code >> 16) & 0xFF)!),
Character(UnicodeScalar((code >> 8) & 0xFF)!),
Character(UnicodeScalar(code & 0xFF)!)
]
return String(chars).trimmingCharacters(in: .whitespaces)
}
}
@@ -0,0 +1,23 @@
import {type EventSubscription, requireNativeModule} from 'expo-modules-core'
import {
type CompressResult,
type NativeCompressOptions,
type VideoMetadata,
} from './types'
type ProgressEvent = {id: number; progress: number}
interface ExpoBlueskyVideoCompressModule {
probe(uri: string): Promise<VideoMetadata>
compress(uri: string, options: NativeCompressOptions): Promise<CompressResult>
cancel(jobId: number): void
addListener(
eventName: 'onProgress',
listener: (event: ProgressEvent) => void,
): EventSubscription
}
export default requireNativeModule<ExpoBlueskyVideoCompressModule>(
'ExpoBlueskyVideoCompress',
)
@@ -0,0 +1,45 @@
export type CodecPreference = 'auto' | 'hevc' | 'h264'
export type VideoMetadata = {
width: number
height: number
duration: number
bitrate: number
fileSize: number
mimeType: string
codec: string
hasAudio: boolean
frameRate: number
rotation: number
isHDR: boolean
}
export type CompressOptions = {
targetBitrate?: number
maxSize?: number
codec?: CodecPreference
frameRateCap?: number
}
export type CompressCallbacks = {
onProgress?: (progress: number) => void
signal?: AbortSignal
}
export type CompressResult = {
uri: string
size: number
mimeType: string
width: number
height: number
duration: number
codec: 'h264' | 'hevc'
}
export type NativeCompressOptions = {
targetBitrate: number
maxSize: number
codec: CodecPreference
frameRateCap: number
jobId: number
}
+19
View File
@@ -1365,6 +1365,25 @@ export type Events = {
engine: string
sourceBytes?: number
}
// Native-only. Raw container metadata returned by the new module's probe()
// (bitrate, codec, HDR, frame rate, rotation, etc.). Fires once per upload
// between compressStarted and the compressSkipped/compressCompleted decision.
// The web (mediabunny) and legacy rn-compressor engines do not surface this.
'video:upload:probed': {
uploadId: string
engine: string
mimeType: string
codec: string
width: number
height: number
duration: number
bitrate: number
fileSize: number
hasAudio: boolean
frameRate: number
rotation: number
isHDR: boolean
}
'video:upload:compressCompleted': {
uploadId: string
engine: string
+15 -2
View File
@@ -2,7 +2,9 @@ import {getVideoMetaData, Video} from 'react-native-compressor'
import {type ImagePickerAsset} from 'expo-image-picker'
import {SUPPORTED_MIME_TYPES, type SupportedMimeTypes} from '#/lib/constants'
import {type CompressedVideo} from './types'
import {logger} from '#/logger'
import {probe} from '../../../../modules/expo-bluesky-video-compress'
import {type CompressedVideo, type ProbedMetadata} from './types'
import {extToMime} from './util'
const MIN_SIZE_FOR_COMPRESSION_BYTES = 25 * 1024 * 1024 // 25mb
@@ -12,9 +14,20 @@ export async function compressVideo(
opts?: {
signal?: AbortSignal
onProgress?: (progress: number) => void
onProbe?: (metadata: ProbedMetadata) => void
},
): Promise<CompressedVideo> {
const {onProgress, signal} = opts || {}
const {onProgress, signal, onProbe} = opts || {}
// Probe data is purely informational - fired into telemetry to validate
// future smart-skip thresholds. Failures must not block the upload.
if (onProbe && file.mimeType !== 'image/gif') {
try {
onProbe(await probe(file.uri))
} catch (e) {
logger.debug('video probe failed', {safeMessage: e})
}
}
if (file.mimeType === 'image/gif') {
// let's hope they're small enough that they don't need compression!
+53 -2
View File
@@ -23,7 +23,7 @@ import {
COMPRESSION_MIN_SIZE_BYTES,
COMPRESSION_TARGET_BITRATE,
} from './constants'
import {type CompressedVideo} from './types'
import {type CompressedVideo, type ProbedMetadata} from './types'
// Codecs to try in order of preference
// avc (H.264) is most compatible, vp9/vp8 are fallbacks for WebM
@@ -34,9 +34,10 @@ export async function compressVideo(
opts?: {
signal?: AbortSignal
onProgress?: (progress: number) => void
onProbe?: (metadata: ProbedMetadata) => void
},
): Promise<CompressedVideo> {
const {onProgress, signal} = opts || {}
const {onProgress, signal, onProbe} = opts || {}
logger.debug('compress: starting', {
uri: asset.uri.slice(0, 50),
@@ -49,6 +50,18 @@ export async function compressVideo(
const isGif = blob.type === 'image/gif'
const hasCodecs = hasWebCodecs()
// Probe pass: extract source metadata via mediabunny before any compression
// decision. Fires before doCompression so the probed event lands ahead of
// compressCompleted/compressSkipped in the funnel. GIFs and missing
// WebCodecs both skip - mediabunny needs a parseable container + decoder.
if (onProbe && !isGif && hasCodecs) {
try {
onProbe(await probeWithMediaBunny(blob))
} catch (e) {
logger.debug('video probe failed', {safeMessage: e})
}
}
logger.debug('compress: fetched blob', {
size: blob.size,
mimeType: blob.type,
@@ -98,6 +111,44 @@ export async function compressVideo(
}
}
async function probeWithMediaBunny(blob: Blob): Promise<ProbedMetadata> {
const input = new Input({source: new BlobSource(blob), formats: ALL_FORMATS})
try {
const videoTrack = await input.getPrimaryVideoTrack()
if (!videoTrack) {
throw new Error('No video track found')
}
const audioTrack = await input.getPrimaryAudioTrack()
const [codec, codedWidth, codedHeight, rotation, isHDR, stats, duration] =
await Promise.all([
videoTrack.getCodec(),
videoTrack.getCodedWidth(),
videoTrack.getCodedHeight(),
videoTrack.getRotation(),
videoTrack.hasHighDynamicRange(),
// Sample a small fixed slice instead of scanning the whole file - we
// only want an approximate bitrate / frame rate for telemetry.
videoTrack.computePacketStats(100),
input.computeDuration(),
])
return {
mimeType: blob.type || 'video/mp4',
codec: codec ?? 'unknown',
width: codedWidth,
height: codedHeight,
duration,
bitrate: Math.round(stats.averageBitrate),
fileSize: blob.size,
hasAudio: audioTrack !== null,
frameRate: stats.averagePacketRate,
rotation,
isHDR,
}
} finally {
input.dispose()
}
}
async function findEncodableVideoCodec(
width: number,
height: number,
+4 -2
View File
@@ -1,8 +1,10 @@
// Shared compression knobs. Mirrored between native (compress.ts) and web
// (compress.web.ts) so both platforms produce videos with the same target.
// Target encode bitrate when we do compress.
export const COMPRESSION_TARGET_BITRATE = 3_000_000 // 3 Mbps
// Output dimension cap when compressing, and skip threshold for source files.
export const COMPRESSION_MAX_DIMENSION = 1920
// Web skips compression entirely for files under this size; native applies its
// own threshold logic inside expo-bluesky-video-compress's probe step.
// Web only: files under this size skip compression entirely. Native applies
// its own threshold logic inside react-native-compressor.
export const COMPRESSION_MIN_SIZE_BYTES = 25_000_000
+23 -1
View File
@@ -2,7 +2,10 @@ import {Platform} from 'react-native'
import {type ImagePickerAsset} from 'expo-image-picker'
import {nanoid} from 'nanoid/non-secure'
import {type VideoCompressSkipReason} from '#/lib/media/video/types'
import {
type ProbedMetadata,
type VideoCompressSkipReason,
} from '#/lib/media/video/types'
import {Sentry} from '#/logger/sentry/lib'
import {type Metrics} from '#/analytics/metrics'
@@ -28,6 +31,7 @@ export type VideoTelemetry = {
readonly engine: string
picked: () => void
compressStarted: () => void
probed: (metadata: ProbedMetadata) => void
compressSkipped: (video: {
size: number
mimeType: string
@@ -155,6 +159,24 @@ export function createVideoTelemetry({
})
},
probed(metadata) {
metric('video:upload:probed', {
uploadId,
engine,
mimeType: metadata.mimeType,
codec: metadata.codec,
width: metadata.width,
height: metadata.height,
duration: metadata.duration,
bitrate: metadata.bitrate,
fileSize: metadata.fileSize,
hasAudio: metadata.hasAudio,
frameRate: metadata.frameRate,
rotation: metadata.rotation,
isHDR: metadata.isHDR,
})
},
compressSkipped({size, mimeType, skipReason}) {
metric('video:upload:compressSkipped', {
uploadId,
+17
View File
@@ -18,3 +18,20 @@ export type CompressedVideo = {
// bytes were actually re-encoded.
passthroughReason?: VideoCompressSkipReason
}
// Source container metadata read off the input before any encoding decision.
// Same shape across native (expo-bluesky-video-compress probe) and web
// (mediabunny Input + track inspection). Numbers are raw - no bucketing.
export type ProbedMetadata = {
mimeType: string
codec: string
width: number
height: number
duration: number
bitrate: number
fileSize: number
hasAudio: boolean
frameRate: number
rotation: number
isHDR: boolean
}
+1
View File
@@ -288,6 +288,7 @@ export async function processVideo(
dispatch({type: 'update_progress', progress: trunc2dp(num), signal})
},
signal,
onProbe: metadata => telemetry.probed(metadata),
})
} catch (e) {
const message = getCompressErrorMessage(e, i18n)