Address re-review: AAC re-encode, probe fallthrough, HDR detection

- Android non-AAC audio is now transcoded to AAC (matching iOS) via a
  source-decoder -> AAC-encoder pre-pass that captures the encoder's
  output format before the muxer starts, then writes buffered samples
  after the video pipeline finishes. Falls back to dropping audio if
  the transcode fails.
- compress.ts wraps probe() in try/catch and falls through to
  passthrough on failure instead of throwing.
- Probers expose isHDR (HLG/PQ via color transfer, plus Dolby Vision
  codecs/mimes); shouldCompress forces compression for HDR sources so
  the SDR BT.709 path always runs. Also sets
  KEY_COLOR_TRANSFER_REQUEST=SDR on the Android decoder (API 31+) so
  HDR sources tone-map to SDR pixels instead of being mislabeled.

modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoCompressor.kt

@@ -129,6 +129,11 @@ class VideoCompressor(
 
       outputDims = calculateOutputDims(sourceWidth, sourceHeight, rotation, maxSize)
       val shouldPassthroughAudio = audioFormat != null && canPassthroughAudio(audioFormat)
+      val transcodedAudio: TranscodedAudio? = if (
+        audioTrackIndex >= 0 && audioFormat != null && !shouldPassthroughAudio
+      ) {
+        transcodeAudioToAAC(audioTrackIndex, audioFormat)
+      } else null
       muxer = MediaMuxer(outputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
 
       val effectiveBitrate = if (targetBitrate > 0) {
@@ -186,6 +191,15 @@ class VideoCompressor(
       decoder = MediaCodec.createDecoderByType(
         videoFormat.getString(MediaFormat.KEY_MIME) ?: "video/avc"
       )
+      // Ask the decoder to tone-map HDR (HLG/PQ) sources to SDR. Vendors may
+      // ignore the hint, but where supported it produces correct BT.709 pixels
+      // for the encoder rather than HDR pixels mislabeled as SDR.
+      if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
+        videoFormat.setInteger(
+          MediaFormat.KEY_COLOR_TRANSFER_REQUEST,
+          MediaFormat.COLOR_TRANSFER_SDR_VIDEO
+        )
+      }
       decoder.configure(videoFormat, outputSurface.surface, null, 0)
       decoder.start()
       extractor.selectTrack(videoTrackIndex)
@@ -257,8 +271,12 @@ class VideoCompressor(
             encIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
               if (!muxerStarted) {
                 muxerVideoTrack = muxer.addTrack(encoder.outputFormat)
-                if (audioTrackIndex >= 0 && shouldPassthroughAudio && audioFormat != null) {
+                if (audioTrackIndex >= 0 && audioFormat != null) {
+                  if (shouldPassthroughAudio) {
                     muxerAudioTrack = muxer.addTrack(audioFormat)
+                  } else if (transcodedAudio != null) {
+                    muxerAudioTrack = muxer.addTrack(transcodedAudio.outputFormat)
+                  }
                 }
                 muxer.start()
                 muxerStarted = true
@@ -291,7 +309,11 @@ class VideoCompressor(
       }
 
       if (audioTrackIndex >= 0 && muxerAudioTrack >= 0 && muxerStarted && !isCancelled) {
+        if (shouldPassthroughAudio) {
           passthroughAudio(audioTrackIndex, muxer, muxerAudioTrack)
+        } else if (transcodedAudio != null) {
+          writeTranscodedAudio(transcodedAudio.samples, muxer, muxerAudioTrack)
+        }
       }
     } finally {
       try { decoder?.stop() } catch (_: Exception) {}
@@ -359,6 +381,188 @@ class VideoCompressor(
     return mime == MediaFormat.MIMETYPE_AUDIO_AAC
   }
 
+  private data class TranscodedAudio(
+    val outputFormat: MediaFormat,
+    val samples: List<Sample>
+  ) {
+    data class Sample(
+      val bytes: ByteArray,
+      val presentationTimeUs: Long,
+      val flags: Int
+    )
+  }
+
+  // Re-encode non-AAC source audio (Opus, Vorbis, etc.) to AAC so the mp4 muxer
+  // can take it. iOS always re-encodes to AAC; without this Android would drop
+  // the audio track entirely.
+  private fun transcodeAudioToAAC(
+    audioTrackIndex: Int,
+    sourceFormat: MediaFormat
+  ): TranscodedAudio? {
+    val sourceMime = sourceFormat.getString(MediaFormat.KEY_MIME) ?: return null
+    val sampleRate = if (sourceFormat.containsKey(MediaFormat.KEY_SAMPLE_RATE))
+      sourceFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE) else 44100
+    val channelCount = if (sourceFormat.containsKey(MediaFormat.KEY_CHANNEL_COUNT))
+      sourceFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT).coerceIn(1, 2) else 2
+
+    val audioExtractor = MediaExtractor()
+    if (uriString.startsWith("content://") || uriString.startsWith("file://")) {
+      audioExtractor.setDataSource(context, Uri.parse(uriString), null)
+    } else {
+      audioExtractor.setDataSource(uriString)
+    }
+    audioExtractor.selectTrack(audioTrackIndex)
+
+    var decoder: MediaCodec? = null
+    var encoder: MediaCodec? = null
+    try {
+      decoder = MediaCodec.createDecoderByType(sourceMime)
+      decoder.configure(sourceFormat, null, null, 0)
+      decoder.start()
+
+      val encoderFormat = MediaFormat.createAudioFormat(
+        MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, channelCount
+      ).apply {
+        setInteger(
+          MediaFormat.KEY_AAC_PROFILE,
+          MediaCodecInfo.CodecProfileLevel.AACObjectLC
+        )
+        setInteger(MediaFormat.KEY_BIT_RATE, 128_000)
+        setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 256 * 1024)
+      }
+      encoder = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
+      encoder.configure(encoderFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
+      encoder.start()
+
+      val samples = mutableListOf<TranscodedAudio.Sample>()
+      var outputFormat: MediaFormat? = null
+      var inputDone = false
+      var decoderDone = false
+      var encoderInputSignalled = false
+      var encoderDone = false
+      val info = MediaCodec.BufferInfo()
+
+      while (!encoderDone && !isCancelled) {
+        if (!inputDone) {
+          val idx = decoder.dequeueInputBuffer(TIMEOUT_DEQUEUE)
+          if (idx >= 0) {
+            val buf = decoder.getInputBuffer(idx)
+            if (buf != null) {
+              val sz = audioExtractor.readSampleData(buf, 0)
+              if (sz < 0) {
+                decoder.queueInputBuffer(
+                  idx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM
+                )
+                inputDone = true
+              } else {
+                decoder.queueInputBuffer(
+                  idx, 0, sz, audioExtractor.sampleTime, audioExtractor.sampleFlags
+                )
+                audioExtractor.advance()
+              }
+            }
+          }
+        }
+
+        if (!decoderDone) {
+          val status = decoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE)
+          when {
+            status == MediaCodec.INFO_TRY_AGAIN_LATER -> {}
+            status == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {}
+            status >= 0 -> {
+              val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
+              val data = decoder.getOutputBuffer(status)
+              if (data != null && info.size > 0) {
+                val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE)
+                if (encInIdx >= 0) {
+                  val encInBuf = encoder.getInputBuffer(encInIdx)
+                  if (encInBuf != null) {
+                    encInBuf.clear()
+                    data.position(info.offset)
+                    data.limit(info.offset + info.size)
+                    encInBuf.put(data)
+                    encoder.queueInputBuffer(
+                      encInIdx, 0, info.size, info.presentationTimeUs, 0
+                    )
+                  }
+                }
+              }
+              decoder.releaseOutputBuffer(status, false)
+              if (isEos) {
+                if (!encoderInputSignalled) {
+                  val encInIdx = encoder.dequeueInputBuffer(TIMEOUT_DEQUEUE * 10)
+                  if (encInIdx >= 0) {
+                    encoder.queueInputBuffer(
+                      encInIdx, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM
+                    )
+                    encoderInputSignalled = true
+                  }
+                }
+                decoderDone = true
+              }
+            }
+          }
+        }
+
+        val encOutIdx = encoder.dequeueOutputBuffer(info, TIMEOUT_DEQUEUE)
+        when {
+          encOutIdx == MediaCodec.INFO_TRY_AGAIN_LATER -> {}
+          encOutIdx == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
+            outputFormat = encoder.outputFormat
+          }
+          encOutIdx >= 0 -> {
+            val data = encoder.getOutputBuffer(encOutIdx)
+            val isEos = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
+            val isConfig = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0
+            if (data != null && info.size > 0 && !isConfig) {
+              val bytes = ByteArray(info.size)
+              data.position(info.offset)
+              data.get(bytes, 0, info.size)
+              samples.add(
+                TranscodedAudio.Sample(
+                  bytes = bytes,
+                  presentationTimeUs = info.presentationTimeUs,
+                  flags = info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG.inv()
+                )
+              )
+            }
+            encoder.releaseOutputBuffer(encOutIdx, false)
+            if (isEos) encoderDone = true
+          }
+        }
+      }
+
+      val fmt = outputFormat ?: return null
+      return TranscodedAudio(fmt, samples)
+    } catch (e: Exception) {
+      Log.w(TAG, "Audio transcode failed; dropping audio", e)
+      return null
+    } finally {
+      try { decoder?.stop() } catch (_: Exception) {}
+      try { decoder?.release() } catch (_: Exception) {}
+      try { encoder?.stop() } catch (_: Exception) {}
+      try { encoder?.release() } catch (_: Exception) {}
+      audioExtractor.release()
+    }
+  }
+
+  private fun writeTranscodedAudio(
+    samples: List<TranscodedAudio.Sample>,
+    muxer: MediaMuxer,
+    muxerAudioTrack: Int
+  ) {
+    val info = MediaCodec.BufferInfo()
+    for (sample in samples) {
+      if (isCancelled) break
+      val buffer = ByteBuffer.wrap(sample.bytes)
+      info.offset = 0
+      info.size = sample.bytes.size
+      info.presentationTimeUs = sample.presentationTimeUs
+      info.flags = sample.flags
+      muxer.writeSampleData(muxerAudioTrack, buffer, info)
+    }
+  }
+
   private fun calculateOutputDims(srcW: Int, srcH: Int, rotation: Int, maxSize: Int): Pair<Int, Int> {
     val isRotated = rotation == 90 || rotation == 270
     val displayW = if (isRotated) srcH else srcW

modules/expo-bluesky-video-compress/android/src/main/java/expo/modules/blueskyvideocompress/VideoProber.kt

@@ -5,6 +5,7 @@ import android.media.MediaExtractor
 import android.media.MediaFormat
 import android.media.MediaMetadataRetriever
 import android.net.Uri
+import android.os.Build
 
 object VideoProber {
   fun probe(context: Context, uriString: String): Map<String, Any> {
@@ -39,6 +40,7 @@ object VideoProber {
       var codec = "unknown"
       var mimeType = "video/mp4"
       var extractedFrameRate = frameRate
+      var isHDR = false
 
       try {
         if (uriString.startsWith("content://") || uriString.startsWith("file://")) {
@@ -56,6 +58,18 @@ object VideoProber {
             if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
               extractedFrameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE).toFloat()
             }
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N &&
+              format.containsKey(MediaFormat.KEY_COLOR_TRANSFER)
+            ) {
+              val transfer = format.getInteger(MediaFormat.KEY_COLOR_TRANSFER)
+              isHDR = transfer == MediaFormat.COLOR_TRANSFER_HLG ||
+                transfer == MediaFormat.COLOR_TRANSFER_ST2084
+            }
+            // Dolby Vision tracks use codec-specific mimes that aren't covered by
+            // KEY_COLOR_TRANSFER on every device.
+            if (mime.contains("dolby-vision", ignoreCase = true)) {
+              isHDR = true
+            }
             break
           }
         }
@@ -82,7 +96,8 @@ object VideoProber {
         "codec" to codec,
         "hasAudio" to hasAudio,
         "frameRate" to extractedFrameRate.toDouble(),
-        "rotation" to rotation
+        "rotation" to rotation,
+        "isHDR" to isHDR
       )
     } finally {
       retriever.release()

modules/expo-bluesky-video-compress/ios/VideoProber.swift

@@ -23,8 +23,24 @@ struct VideoProber {
     let formatDescriptions = try await videoTrack.load(.formatDescriptions)
 
     var codec = "unknown"
+    var isHDR = false
     if let formatDescription = formatDescriptions.first {
-      codec = fourCCToString(CMFormatDescriptionGetMediaSubType(formatDescription))
+      let subType = CMFormatDescriptionGetMediaSubType(formatDescription)
+      codec = fourCCToString(subType)
+      // Dolby Vision codecs are HDR by definition.
+      let dolbyVisionSubtypes: Set<String> = ["dvhe", "dvh1", "dvav", "dva1"]
+      if dolbyVisionSubtypes.contains(codec) {
+        isHDR = true
+      } else if let extensions = CMFormatDescriptionGetExtensions(formatDescription)
+        as? [String: Any]
+      {
+        let transferKey = kCMFormatDescriptionExtension_TransferFunction as String
+        if let transfer = extensions[transferKey] as? String {
+          let hlg = kCMFormatDescriptionTransferFunction_ITU_R_2100_HLG as String
+          let pq = kCMFormatDescriptionTransferFunction_SMPTE_ST_2084_PQ as String
+          isHDR = transfer == hlg || transfer == pq
+        }
+      }
     }
 
     let rotation = rotationFromTransform(preferredTransform)
@@ -66,7 +82,8 @@ struct VideoProber {
       "codec": codec,
       "hasAudio": hasAudio,
       "frameRate": nominalFrameRate,
-      "rotation": rotation
+      "rotation": rotation,
+      "isHDR": isHDR
     ]
   }
 

modules/expo-bluesky-video-compress/src/types.ts

@@ -11,6 +11,7 @@ export type VideoMetadata = {
   hasAudio: boolean
   frameRate: number
   rotation: number
+  isHDR: boolean
 }
 
 export type CompressOptions = {

src/lib/media/video/compress.ts

@@ -24,7 +24,19 @@ export async function compressVideo(
     file.mimeType as SupportedMimeTypes,
   )
 
-  const metadata = await probe(file.uri)
+  let metadata
+  try {
+    metadata = await probe(file.uri)
+  } catch (e) {
+    logger.debug('probe failed, falling through to passthrough', {
+      safeMessage: e,
+    })
+    return {
+      uri: file.uri,
+      size: file.fileSize ?? -1,
+      mimeType: file.mimeType ?? 'video/mp4',
+    }
+  }
 
   if (!shouldCompress(metadata, isAcceptableFormat)) {
     return {
@@ -55,7 +67,13 @@ export async function compressVideo(
 }
 
 function shouldCompress(
-  metadata: {bitrate: number; width: number; height: number; fileSize: number},
+  metadata: {
+    bitrate: number
+    width: number
+    height: number
+    fileSize: number
+    isHDR: boolean
+  },
   isAcceptableFormat: boolean,
 ): boolean {
   const maxDimension = Math.max(metadata.width, metadata.height)
@@ -67,6 +85,13 @@ function shouldCompress(
     return true
   }
 
+  // HDR sources need the SDR BT.709 tone-map in the compress path; otherwise we
+  // would upload HLG/PQ/Dolby Vision untouched.
+  if (metadata.isHDR) {
+    logger.debug('shouldCompress: yes (HDR source)')
+    return true
+  }
+
   if (metadata.fileSize > MAX_UPLOAD_SIZE) {
     logger.debug(`shouldCompress: yes (file too large: ${sizeMB}MB)`)
     return true