acetheking987/bsky-social-app
1
0
Fork 0
Code Issues Actions Packages Projects Releases 1 Wiki Activity

Address re-review: AAC re-encode, probe fallthrough, HDR detection

Browse Source 
- Android non-AAC audio is now transcoded to AAC (matching iOS) via a
  source-decoder -> AAC-encoder pre-pass that captures the encoder's
  output format before the muxer starts, then writes buffered samples
  after the video pipeline finishes. Falls back to dropping audio if
  the transcode fails.
- compress.ts wraps probe() in try/catch and falls through to
  passthrough on failure instead of throwing.
- Probers expose isHDR (HLG/PQ via color transfer, plus Dolby Vision
  codecs/mimes); shouldCompress forces compression for HDR sources so
  the SDR BT.709 path always runs. Also sets
  KEY_COLOR_TRANSFER_REQUEST=SDR on the Android decoder (API 31+) so
  HDR sources tone-map to SDR pixels instead of being mislabeled.
This commit is contained in:
vineyardbovines
2026-06-20 14:43:57 -04:00
parent 34ee7bafda
commit fd387904d4
5 changed files with 270 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt
+207 -3
View File
@@ -129,6 +129,11 @@ class VideoCompressor(
outputDims = calculateOutputDims(sourceWidth, sourceHeight, rotation, maxSize)
val shouldPassthroughAudio = audioFormat != null && canPassthroughAudio(audioFormat)
val transcodedAudio: TranscodedAudio? = if (
audioTrackIndex >= 0 && audioFormat != null && !shouldPassthroughAudio
) {
transcodeAudioToAAC(audioTrackIndex, audioFormat)
} else null
muxer = MediaMuxer(outputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
val effectiveBitrate = if (targetBitrate > 0) {
@@ -186,6 +191,15 @@ class VideoCompressor(
decoder = MediaCodec.createDecoderByType(
videoFormat.getString(MediaFormat.KEY_MIME) ?: "video/avc"
)
// Ask the decoder to tone-map HDR (HLG/PQ) sources to SDR. Vendors may
// ignore the hint, but where supported it produces correct BT.709 pixels
// for the encoder rather than HDR pixels mislabeled as SDR.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
videoFormat.setInteger(
MediaFormat.KEY_COLOR_TRANSFER_REQUEST,
MediaFormat.COLOR_TRANSFER_SDR_VIDEO
)
}
decoder.configure(videoFormat, outputSurface.surface, null, 0)
decoder.start()
extractor.selectTrack(videoTrackIndex)
@@ -257,8 +271,12 @@ class VideoCompressor(
encIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
if (!muxerStarted) {
muxerVideoTrack = muxer.addTrack(encoder.outputFormat)
if (audioTrackIndex >= 0 && shouldPassthroughAudio && audioFormat != null) {
muxerAudioTrack = muxer.addTrack(audioFormat)
if (audioTrackIndex >= 0 && audioFormat != null) {
if (shouldPassthroughAudio) {
muxerAudioTrack = muxer.addTrack(audioFormat)
} else if (transcodedAudio != null) {
muxerAudioTrack = muxer.addTrack(transcodedAudio.outputFormat)
}
}
muxer.start()
muxerStarted = true
@@ -291,7 +309,11 @@ class VideoCompressor(
}
if (audioTrackIndex >= 0 && muxerAudioTrack >= 0 && muxerStarted && !isCancelled) {
passthroughAudio(audioTrackIndex, muxer, muxerAudioTrack)
if (shouldPassthroughAudio) {
passthroughAudio(audioTrackIndex, muxer, muxerAudioTrack)
} else if (transcodedAudio != null) {
writeTranscodedAudio(transcodedAudio.samples, muxer, muxerAudioTrack)
}
}
} finally {
try { decoder?.stop() } catch (_: Exception) {}
@@ -359,6 +381,188 @@ class VideoCompressor(
return mime == MediaFormat.MIMETYPE_AUDIO_AAC
}
private data class TranscodedAudio(
val outputFormat: MediaFormat,
val samples: List<Sample>
) {
data class Sample(
val bytes: ByteArray,
val presentationTimeUs: Long,
val flags: Int
)
}
// Re-encode non-AAC source audio (Opus, Vorbis, etc.) to AAC so the mp4 muxer
// can take it. iOS always re-encodes to AAC; without this Android would drop
// the audio track entirely.
private fun transcodeAudioToAAC(
audioTrackIndex: Int,
sourceFormat: MediaFormat
): TranscodedAudio? {
val sourceMime = sourceFormat.getString(MediaFormat.KEY_MIME) ?: return null
val sampleRate = if (sourceFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE))
sourceFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100
val channelCount = if (sourceFormat.containsKey(MediaFormat.KEY_CHANNEL_COUNT))
sourceFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT).coerceIn(1, 2) else 2
val audioExtractor = MediaExtractor()
if (uriString.startsWith("content://") || uriString.startsWith("file://")) {
audioExtractor.setDataSource(context, Uri.parse(uriString), null)
} else {
audioExtractor.setDataSource(uriString)
}
audioExtractor.selectTrack(audioTrackIndex)
var decoder: MediaCodec? = null
var encoder: MediaCodec? = null
try {
decoder = MediaCodec.createDecoderByType(sourceMime)
decoder.configure(sourceFormat, null, null, 0)
decoder.start()
val encoderFormat = MediaFormat.createAudioFormat(
MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, channelCount
).apply {
setInteger(
MediaFormat.KEY_AAC_PROFILE,
MediaCodecInfo.CodecProfileLevel.AACObjectLC
)
setInteger(MediaFormat.KEY_BIT_RATE, 128_000)
setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 256 * 1024)
}
encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
encoder.start()
val samples = mutableListOf<TranscodedAudio.Sample>()
var outputFormat: MediaFormat? = null
var inputDone = false
var decoderDone = false
var encoderInputSignalled = false
var encoderDone = false
val info = MediaCodec.BufferInfo()
while (!encoderDone && !isCancelled) {
if (!inputDone) {
val idx = decoder.dequeueInputBuffer(TIMEOUT_DEQUEUE)
if (idx >= 0) {
val buf = decoder.getInputBuffer(idx)
if (buf != null) {
val sz = audioExtractor.readSampleData(buf, 0)
if (sz < 0) {
decoder.queueInputBuffer(
idx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM
)
inputDone = true
} else {
decoder.queueInputBuffer(
idx, 0, sz, audioExtractor.sampleTime, audioExtractor.sampleFlags
)
audioExtractor.advance()
}
}
}
}
if (!decoderDone) {
val status = decoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE)
when {
status == MediaCodec.INFO_TRY_AGAIN_LATER -> {}
status == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {}
status >= 0 -> {
val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
val data = decoder.getOutputBuffer(status)
if (data != null && info.size > 0) {
val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE)
if (encInIdx >= 0) {
val encInBuf = encoder.getInputBuffer(encInIdx)
if (encInBuf != null) {
encInBuf.clear()
data.position(info.offset)
data.limit(info.offset + info.size)
encInBuf.put(data)
encoder.queueInputBuffer(
encInIdx, 0, info.size, info.presentationTimeUs, 0
)
}
}
}
decoder.releaseOutputBuffer(status, false)
if (isEos) {
if (!encoderInputSignalled) {
val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE * 10)
if (encInIdx >= 0) {
encoder.queueInputBuffer(
encInIdx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM
)
encoderInputSignalled = true
}
}
decoderDone = true
}
}
}
}
val encOutIdx = encoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE)
when {
encOutIdx == MediaCodec.INFO_TRY_AGAIN_LATER -> {}
encOutIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
outputFormat = encoder.outputFormat
}
encOutIdx >= 0 -> {
val data = encoder.getOutputBuffer(encOutIdx)
val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
val isConfig = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0
if (data != null && info.size > 0 && !isConfig) {
val bytes = ByteArray(info.size)
data.position(info.offset)
data.get(bytes, 0, info.size)
samples.add(
TranscodedAudio.Sample(
bytes = bytes,
presentationTimeUs = info.presentationTimeUs,
flags = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG.inv()
)
)
}
encoder.releaseOutputBuffer(encOutIdx, false)
if (isEos) encoderDone = true
}
}
}
val fmt = outputFormat ?: return null
return TranscodedAudio(fmt, samples)
} catch (e: Exception) {
Log.w(TAG, "Audio transcode failed; dropping audio", e)
return null
} finally {
try { decoder?.stop() } catch (_: Exception) {}
try { decoder?.release() } catch (_: Exception) {}
try { encoder?.stop() } catch (_: Exception) {}
try { encoder?.release() } catch (_: Exception) {}
audioExtractor.release()
}
}
private fun writeTranscodedAudio(
samples: List<TranscodedAudio.Sample>,
muxer: MediaMuxer,
muxerAudioTrack: Int
) {
val info = MediaCodec.BufferInfo()
for (sample in samples) {
if (isCancelled) break
val buffer = ByteBuffer.wrap(sample.bytes)
info.offset = 0
info.size = sample.bytes.size
info.presentationTimeUs = sample.presentationTimeUs
info.flags = sample.flags
muxer.writeSampleData(muxerAudioTrack, buffer, info)
}
}
private fun calculateOutputDims(srcW: Int, srcH: Int, rotation: Int, maxSize: Int): Pair<Int, Int> {
val isRotated = rotation == 90 || rotation == 270
val displayW = if (isRotated) srcH else srcW
modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt
+16 -1
View File
@@ -5,6 +5,7 @@ import android.media.MediaExtractor
import android.media.MediaFormat
import android.media.MediaMetadataRetriever
import android.net.Uri
import android.os.Build
object VideoProber {
fun probe(context: Context, uriString: String): Map<String, Any> {
@@ -39,6 +40,7 @@ object VideoProber {
var codec = "unknown"
var mimeType = "video/mp4"
var extractedFrameRate = frameRate
var isHDR = false
try {
if (uriString.startsWith("content://") || uriString.startsWith("file://")) {
@@ -56,6 +58,18 @@ object VideoProber {
if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
extractedFrameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE).toFloat()
}
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N &&
format.containsKey(MediaFormat.KEY_COLOR_TRANSFER)
) {
val transfer = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER)
isHDR = transfer == MediaFormat.COLOR_TRANSFER_HLG ||
transfer == MediaFormat.COLOR_TRANSFER_ST2084
}
// Dolby Vision tracks use codec-specific mimes that aren't covered by
// KEY_COLOR_TRANSFER on every device.
if (mime.contains("dolby-vision", ignoreCase = true)) {
isHDR = true
}
break
}
}
@@ -82,7 +96,8 @@ object VideoProber {
"codec" to codec,
"hasAudio" to hasAudio,
"frameRate" to extractedFrameRate.toDouble(),
"rotation" to rotation
"rotation" to rotation,
"isHDR" to isHDR
)
} finally {
retriever.release()
modules/expo-bluesky-video-compress/ios/VideoProber.swift
+19 -2
View File
@@ -23,8 +23,24 @@ struct VideoProber {
let formatDescriptions = try await videoTrack.load(.formatDescriptions)
var codec = "unknown"
var isHDR = false
if let formatDescription = formatDescriptions.first {
codec = fourCCToString(CMFormatDescriptionGetMediaSubType(formatDescription))
let subType = CMFormatDescriptionGetMediaSubType(formatDescription)
codec = fourCCToString(subType)
// Dolby Vision codecs are HDR by definition.
let dolbyVisionSubtypes: Set<String> = ["dvhe", "dvh1", "dvav", "dva1"]
if dolbyVisionSubtypes.contains(codec) {
isHDR = true
} else if let extensions = CMFormatDescriptionGetExtensions(formatDescription)
as? [String: Any]
{
let transferKey = kCMFormatDescriptionExtension_TransferFunction as String
if let transfer = extensions[transferKey] as? String {
let hlg = kCMFormatDescriptionTransferFunction_ITU_R_2100_HLG as String
let pq = kCMFormatDescriptionTransferFunction_SMPTE_ST_2084_PQ as String
isHDR = transfer == hlg || transfer == pq
}
}
}
let rotation = rotationFromTransform(preferredTransform)
@@ -66,7 +82,8 @@ struct VideoProber {
"codec": codec,
"hasAudio": hasAudio,
"frameRate": nominalFrameRate,
"rotation": rotation
"rotation": rotation,
"isHDR": isHDR
]
}
modules/expo-bluesky-video-compress/src/types.ts
+1
View File
@@ -11,6 +11,7 @@ export type VideoMetadata = {
hasAudio: boolean
frameRate: number
rotation: number
isHDR: boolean
}
export type CompressOptions = {
src/lib/media/video/compress.ts
+27 -2
View File
@@ -24,7 +24,19 @@ export async function compressVideo(
file.mimeType as SupportedMimeTypes,
)
const metadata = await probe(file.uri)
let metadata
try {
metadata = await probe(file.uri)
} catch (e) {
logger.debug('probe failed, falling through to passthrough', {
safeMessage: e,
})
return {
uri: file.uri,
size: file.fileSize ?? -1,
mimeType: file.mimeType ?? 'video/mp4',
}
}
if (!shouldCompress(metadata, isAcceptableFormat)) {
return {
@@ -55,7 +67,13 @@ export async function compressVideo(
}
function shouldCompress(
metadata: {bitrate: number; width: number; height: number; fileSize: number},
metadata: {
bitrate: number
width: number
height: number
fileSize: number
isHDR: boolean
},
isAcceptableFormat: boolean,
): boolean {
const maxDimension = Math.max(metadata.width, metadata.height)
@@ -67,6 +85,13 @@ function shouldCompress(
return true
}
// HDR sources need the SDR BT.709 tone-map in the compress path; otherwise we
// would upload HLG/PQ/Dolby Vision untouched.
if (metadata.isHDR) {
logger.debug('shouldCompress: yes (HDR source)')
return true
}
if (metadata.fileSize > MAX_UPLOAD_SIZE) {
logger.debug(`shouldCompress: yes (file too large: ${sizeMB}MB)`)
return true