From fd387904d48c0ee54a75e6e286921978c09bad33 Mon Sep 17 00:00:00 2001 From: vineyardbovines Date: Sat, 20 Jun 2026 14:43:57 -0400 Subject: [PATCH] Address re-review: AAC re-encode, probe fallthrough, HDR detection - Android non-AAC audio is now transcoded to AAC (matching iOS) via a source-decoder -> AAC-encoder pre-pass that captures the encoder's output format before the muxer starts, then writes buffered samples after the video pipeline finishes. Falls back to dropping audio if the transcode fails. - compress.ts wraps probe() in try/catch and falls through to passthrough on failure instead of throwing. - Probers expose isHDR (HLG/PQ via color transfer, plus Dolby Vision codecs/mimes); shouldCompress forces compression for HDR sources so the SDR BT.709 path always runs. Also sets KEY_COLOR_TRANSFER_REQUEST=SDR on the Android decoder (API 31+) so HDR sources tone-map to SDR pixels instead of being mislabeled. --- .../blueskyvideocompress/VideoCompressor.kt | 210 +++++++++++++++++- .../blueskyvideocompress/VideoProber.kt | 17 +- .../ios/VideoProber.swift | 21 +- .../expo-bluesky-video-compress/src/types.ts | 1 + src/lib/media/video/compress.ts | 29 ++- 5 files changed, 270 insertions(+), 8 deletions(-) diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt index f1728945ff..e9c99bc9c4 100644 --- a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt @@ -129,6 +129,11 @@ class VideoCompressor( outputDims = calculateOutputDims(sourceWidth, sourceHeight, rotation, maxSize) val shouldPassthroughAudio = audioFormat != null && canPassthroughAudio(audioFormat) + val transcodedAudio: TranscodedAudio? = if ( + audioTrackIndex >= 0 && audioFormat != null && !shouldPassthroughAudio + ) { + transcodeAudioToAAC(audioTrackIndex, audioFormat) + } else null muxer = MediaMuxer(outputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) val effectiveBitrate = if (targetBitrate > 0) { @@ -186,6 +191,15 @@ class VideoCompressor( decoder = MediaCodec.createDecoderByType( videoFormat.getString(MediaFormat.KEY_MIME) ?: "video/avc" ) + // Ask the decoder to tone-map HDR (HLG/PQ) sources to SDR. Vendors may + // ignore the hint, but where supported it produces correct BT.709 pixels + // for the encoder rather than HDR pixels mislabeled as SDR. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + videoFormat.setInteger( + MediaFormat.KEY_COLOR_TRANSFER_REQUEST, + MediaFormat.COLOR_TRANSFER_SDR_VIDEO + ) + } decoder.configure(videoFormat, outputSurface.surface, null, 0) decoder.start() extractor.selectTrack(videoTrackIndex) @@ -257,8 +271,12 @@ class VideoCompressor( encIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { if (!muxerStarted) { muxerVideoTrack = muxer.addTrack(encoder.outputFormat) - if (audioTrackIndex >= 0 && shouldPassthroughAudio && audioFormat != null) { - muxerAudioTrack = muxer.addTrack(audioFormat) + if (audioTrackIndex >= 0 && audioFormat != null) { + if (shouldPassthroughAudio) { + muxerAudioTrack = muxer.addTrack(audioFormat) + } else if (transcodedAudio != null) { + muxerAudioTrack = muxer.addTrack(transcodedAudio.outputFormat) + } } muxer.start() muxerStarted = true @@ -291,7 +309,11 @@ class VideoCompressor( } if (audioTrackIndex >= 0 && muxerAudioTrack >= 0 && muxerStarted && !isCancelled) { - passthroughAudio(audioTrackIndex, muxer, muxerAudioTrack) + if (shouldPassthroughAudio) { + passthroughAudio(audioTrackIndex, muxer, muxerAudioTrack) + } else if (transcodedAudio != null) { + writeTranscodedAudio(transcodedAudio.samples, muxer, muxerAudioTrack) + } } } finally { try { decoder?.stop() } catch (_: Exception) {} @@ -359,6 +381,188 @@ class VideoCompressor( return mime == MediaFormat.MIMETYPE_AUDIO_AAC } + private data class TranscodedAudio( + val outputFormat: MediaFormat, + val samples: List + ) { + data class Sample( + val bytes: ByteArray, + val presentationTimeUs: Long, + val flags: Int + ) + } + + // Re-encode non-AAC source audio (Opus, Vorbis, etc.) to AAC so the mp4 muxer + // can take it. iOS always re-encodes to AAC; without this Android would drop + // the audio track entirely. + private fun transcodeAudioToAAC( + audioTrackIndex: Int, + sourceFormat: MediaFormat + ): TranscodedAudio? { + val sourceMime = sourceFormat.getString(MediaFormat.KEY_MIME) ?: return null + val sampleRate = if (sourceFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE)) + sourceFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100 + val channelCount = if (sourceFormat.containsKey(MediaFormat.KEY_CHANNEL_COUNT)) + sourceFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT).coerceIn(1, 2) else 2 + + val audioExtractor = MediaExtractor() + if (uriString.startsWith("content://") || uriString.startsWith("file://")) { + audioExtractor.setDataSource(context, Uri.parse(uriString), null) + } else { + audioExtractor.setDataSource(uriString) + } + audioExtractor.selectTrack(audioTrackIndex) + + var decoder: MediaCodec? = null + var encoder: MediaCodec? = null + try { + decoder = MediaCodec.createDecoderByType(sourceMime) + decoder.configure(sourceFormat, null, null, 0) + decoder.start() + + val encoderFormat = MediaFormat.createAudioFormat( + MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, channelCount + ).apply { + setInteger( + MediaFormat.KEY_AAC_PROFILE, + MediaCodecInfo.CodecProfileLevel.AACObjectLC + ) + setInteger(MediaFormat.KEY_BIT_RATE, 128_000) + setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 256 * 1024) + } + encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC) + encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + encoder.start() + + val samples = mutableListOf() + var outputFormat: MediaFormat? = null + var inputDone = false + var decoderDone = false + var encoderInputSignalled = false + var encoderDone = false + val info = MediaCodec.BufferInfo() + + while (!encoderDone && !isCancelled) { + if (!inputDone) { + val idx = decoder.dequeueInputBuffer(TIMEOUT_DEQUEUE) + if (idx >= 0) { + val buf = decoder.getInputBuffer(idx) + if (buf != null) { + val sz = audioExtractor.readSampleData(buf, 0) + if (sz < 0) { + decoder.queueInputBuffer( + idx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM + ) + inputDone = true + } else { + decoder.queueInputBuffer( + idx, 0, sz, audioExtractor.sampleTime, audioExtractor.sampleFlags + ) + audioExtractor.advance() + } + } + } + } + + if (!decoderDone) { + val status = decoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE) + when { + status == MediaCodec.INFO_TRY_AGAIN_LATER -> {} + status == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {} + status >= 0 -> { + val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + val data = decoder.getOutputBuffer(status) + if (data != null && info.size > 0) { + val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE) + if (encInIdx >= 0) { + val encInBuf = encoder.getInputBuffer(encInIdx) + if (encInBuf != null) { + encInBuf.clear() + data.position(info.offset) + data.limit(info.offset + info.size) + encInBuf.put(data) + encoder.queueInputBuffer( + encInIdx, 0, info.size, info.presentationTimeUs, 0 + ) + } + } + } + decoder.releaseOutputBuffer(status, false) + if (isEos) { + if (!encoderInputSignalled) { + val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE * 10) + if (encInIdx >= 0) { + encoder.queueInputBuffer( + encInIdx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM + ) + encoderInputSignalled = true + } + } + decoderDone = true + } + } + } + } + + val encOutIdx = encoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE) + when { + encOutIdx == MediaCodec.INFO_TRY_AGAIN_LATER -> {} + encOutIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> { + outputFormat = encoder.outputFormat + } + encOutIdx >= 0 -> { + val data = encoder.getOutputBuffer(encOutIdx) + val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0 + val isConfig = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0 + if (data != null && info.size > 0 && !isConfig) { + val bytes = ByteArray(info.size) + data.position(info.offset) + data.get(bytes, 0, info.size) + samples.add( + TranscodedAudio.Sample( + bytes = bytes, + presentationTimeUs = info.presentationTimeUs, + flags = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG.inv() + ) + ) + } + encoder.releaseOutputBuffer(encOutIdx, false) + if (isEos) encoderDone = true + } + } + } + + val fmt = outputFormat ?: return null + return TranscodedAudio(fmt, samples) + } catch (e: Exception) { + Log.w(TAG, "Audio transcode failed; dropping audio", e) + return null + } finally { + try { decoder?.stop() } catch (_: Exception) {} + try { decoder?.release() } catch (_: Exception) {} + try { encoder?.stop() } catch (_: Exception) {} + try { encoder?.release() } catch (_: Exception) {} + audioExtractor.release() + } + } + + private fun writeTranscodedAudio( + samples: List, + muxer: MediaMuxer, + muxerAudioTrack: Int + ) { + val info = MediaCodec.BufferInfo() + for (sample in samples) { + if (isCancelled) break + val buffer = ByteBuffer.wrap(sample.bytes) + info.offset = 0 + info.size = sample.bytes.size + info.presentationTimeUs = sample.presentationTimeUs + info.flags = sample.flags + muxer.writeSampleData(muxerAudioTrack, buffer, info) + } + } + private fun calculateOutputDims(srcW: Int, srcH: Int, rotation: Int, maxSize: Int): Pair { val isRotated = rotation == 90 || rotation == 270 val displayW = if (isRotated) srcH else srcW diff --git a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt index 9d3d658e05..7c214bddd0 100644 --- a/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt +++ b/modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt @@ -5,6 +5,7 @@ import android.media.MediaExtractor import android.media.MediaFormat import android.media.MediaMetadataRetriever import android.net.Uri +import android.os.Build object VideoProber { fun probe(context: Context, uriString: String): Map { @@ -39,6 +40,7 @@ object VideoProber { var codec = "unknown" var mimeType = "video/mp4" var extractedFrameRate = frameRate + var isHDR = false try { if (uriString.startsWith("content://") || uriString.startsWith("file://")) { @@ -56,6 +58,18 @@ object VideoProber { if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) { extractedFrameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE).toFloat() } + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N && + format.containsKey(MediaFormat.KEY_COLOR_TRANSFER) + ) { + val transfer = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER) + isHDR = transfer == MediaFormat.COLOR_TRANSFER_HLG || + transfer == MediaFormat.COLOR_TRANSFER_ST2084 + } + // Dolby Vision tracks use codec-specific mimes that aren't covered by + // KEY_COLOR_TRANSFER on every device. + if (mime.contains("dolby-vision", ignoreCase = true)) { + isHDR = true + } break } } @@ -82,7 +96,8 @@ object VideoProber { "codec" to codec, "hasAudio" to hasAudio, "frameRate" to extractedFrameRate.toDouble(), - "rotation" to rotation + "rotation" to rotation, + "isHDR" to isHDR ) } finally { retriever.release() diff --git a/modules/expo-bluesky-video-compress/ios/VideoProber.swift b/modules/expo-bluesky-video-compress/ios/VideoProber.swift index e68371f369..4636ed7d3c 100644 --- a/modules/expo-bluesky-video-compress/ios/VideoProber.swift +++ b/modules/expo-bluesky-video-compress/ios/VideoProber.swift @@ -23,8 +23,24 @@ struct VideoProber { let formatDescriptions = try await videoTrack.load(.formatDescriptions) var codec = "unknown" + var isHDR = false if let formatDescription = formatDescriptions.first { - codec = fourCCToString(CMFormatDescriptionGetMediaSubType(formatDescription)) + let subType = CMFormatDescriptionGetMediaSubType(formatDescription) + codec = fourCCToString(subType) + // Dolby Vision codecs are HDR by definition. + let dolbyVisionSubtypes: Set = ["dvhe", "dvh1", "dvav", "dva1"] + if dolbyVisionSubtypes.contains(codec) { + isHDR = true + } else if let extensions = CMFormatDescriptionGetExtensions(formatDescription) + as? [String: Any] + { + let transferKey = kCMFormatDescriptionExtension_TransferFunction as String + if let transfer = extensions[transferKey] as? String { + let hlg = kCMFormatDescriptionTransferFunction_ITU_R_2100_HLG as String + let pq = kCMFormatDescriptionTransferFunction_SMPTE_ST_2084_PQ as String + isHDR = transfer == hlg || transfer == pq + } + } } let rotation = rotationFromTransform(preferredTransform) @@ -66,7 +82,8 @@ struct VideoProber { "codec": codec, "hasAudio": hasAudio, "frameRate": nominalFrameRate, - "rotation": rotation + "rotation": rotation, + "isHDR": isHDR ] } diff --git a/modules/expo-bluesky-video-compress/src/types.ts b/modules/expo-bluesky-video-compress/src/types.ts index d278ff752c..df9b9ed1ac 100644 --- a/modules/expo-bluesky-video-compress/src/types.ts +++ b/modules/expo-bluesky-video-compress/src/types.ts @@ -11,6 +11,7 @@ export type VideoMetadata = { hasAudio: boolean frameRate: number rotation: number + isHDR: boolean } export type CompressOptions = { diff --git a/src/lib/media/video/compress.ts b/src/lib/media/video/compress.ts index 0a69e001f1..ad593938d9 100644 --- a/src/lib/media/video/compress.ts +++ b/src/lib/media/video/compress.ts @@ -24,7 +24,19 @@ export async function compressVideo( file.mimeType as SupportedMimeTypes, ) - const metadata = await probe(file.uri) + let metadata + try { + metadata = await probe(file.uri) + } catch (e) { + logger.debug('probe failed, falling through to passthrough', { + safeMessage: e, + }) + return { + uri: file.uri, + size: file.fileSize ?? -1, + mimeType: file.mimeType ?? 'video/mp4', + } + } if (!shouldCompress(metadata, isAcceptableFormat)) { return { @@ -55,7 +67,13 @@ export async function compressVideo( } function shouldCompress( - metadata: {bitrate: number; width: number; height: number; fileSize: number}, + metadata: { + bitrate: number + width: number + height: number + fileSize: number + isHDR: boolean + }, isAcceptableFormat: boolean, ): boolean { const maxDimension = Math.max(metadata.width, metadata.height) @@ -67,6 +85,13 @@ function shouldCompress( return true } + // HDR sources need the SDR BT.709 tone-map in the compress path; otherwise we + // would upload HLG/PQ/Dolby Vision untouched. + if (metadata.isHDR) { + logger.debug('shouldCompress: yes (HDR source)') + return true + } + if (metadata.fileSize > MAX_UPLOAD_SIZE) { logger.debug(`shouldCompress: yes (file too large: ${sizeMB}MB)`) return true