Support double pcm encoding in extractor

RELEASENOTES.md

@@ -20,13 +20,18 @@
 *   Transformer:
 *   Track selection:
 *   Extractors:
+    *   WAV, Matroska, and MP4: Add support for 64-bit floating point PCM audio
+        ([#3090](https://github.com/androidx/media/pull/3090)).
 *   Inspector:
 *   Audio:
     *   Convert parameters of `AudioSink.configure` to data class. Custom
         overrides of `ForwardingAudioSink.configure` need to be migrated to the
         new method signature.
     *   Add support for maintaining pitch when time-stretching an audio stream
         with `EditedMediaItem.Builder#setSpeed(SpeedParameters)`.
+    *   Enhance `ToFloatPcmAudioProcessor` to support converting 8-bit PCM,
+        16-bit big-endian PCM, and 64-bit floating point PCM to 32-bit floating
+        point PCM ([#3090](https://github.com/androidx/media/pull/3090)).
 *   Video:
     *   Add support for skipping frames that are late during join rather than
         dropping in DecoderVideoRenderer.

libraries/common/src/main/java/androidx/media3/common/util/Util.java

@@ -2421,6 +2421,25 @@ public static Format getPcmFormat(AudioProcessor.AudioFormat audioFormat) {
     }
   }
 
+  /**
+   * Converts a sample bit depth to a corresponding little-endian float PCM encoding constant.
+   *
+   * @param bitDepth The bit depth. Supported values are 32 and 64.
+   * @return The corresponding float PCM encoding. If the bit depth is unsupported then {@link
+   *     C#ENCODING_INVALID} is returned.
+   */
+  @UnstableApi
+  public static @C.PcmEncoding int getFloatPcmEncoding(int bitDepth) {
+    switch (bitDepth) {
+      case 32:
+        return C.ENCODING_PCM_FLOAT;
+      case 64:
+        return C.ENCODING_PCM_DOUBLE;
+      default:
+        return C.ENCODING_INVALID;
+    }
+  }
+
   /**
    * Returns whether {@code encoding} is one of the linear PCM encodings.
    *

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ToFloatPcmAudioProcessor.java

@@ -24,11 +24,13 @@
 import java.nio.ByteBuffer;
 
 /**
- * An {@link AudioProcessor} that converts high resolution PCM audio to 32-bit float. The following
- * encodings are supported as input:
+ * An {@link AudioProcessor} that converts different PCM audio encodings to 32-bit float. The
+ * following encodings are supported as input:
  *
  * <ul>
+ *   <li>{@link C#ENCODING_PCM_8BIT}
  *   <li>{@link C#ENCODING_PCM_16BIT}
+ *   <li>{@link C#ENCODING_PCM_16BIT_BIG_ENDIAN}
  *   <li>{@link C#ENCODING_PCM_24BIT}
  *   <li>{@link C#ENCODING_PCM_24BIT_BIG_ENDIAN}
  *   <li>{@link C#ENCODING_PCM_32BIT}
@@ -47,7 +49,7 @@ public final class ToFloatPcmAudioProcessor extends BaseAudioProcessor {
   public AudioFormat onConfigure(AudioFormat inputAudioFormat)
       throws UnhandledAudioFormatException {
     @C.PcmEncoding int encoding = inputAudioFormat.encoding;
-    if (!Util.isEncodingHighResolutionPcm(encoding) && encoding != C.ENCODING_PCM_16BIT) {
+    if (!Util.isEncodingLinearPcm(encoding)) {
       throw new UnhandledAudioFormatException(inputAudioFormat);
     }
     return encoding != C.ENCODING_PCM_FLOAT
@@ -64,6 +66,13 @@ public void queueInput(ByteBuffer inputBuffer) {
 
     ByteBuffer buffer;
     switch (inputAudioFormat.encoding) {
+      case C.ENCODING_PCM_8BIT:
+        buffer = replaceOutputBuffer(size * 4);
+        for (int i = position; i < limit; i++) {
+          int pcm32BitInteger = (((inputBuffer.get(i) & 0xFF) - 128) << 24);
+          writePcm32BitFloat(pcm32BitInteger, buffer);
+        }
+        break;
       case C.ENCODING_PCM_16BIT:
         buffer = replaceOutputBuffer(size * 2);
         for (int i = position; i < limit; i += 2) {
@@ -72,6 +81,14 @@ public void queueInput(ByteBuffer inputBuffer) {
           writePcm32BitFloat(pcm32BitInteger, buffer);
         }
         break;
+      case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
+        buffer = replaceOutputBuffer(size * 2);
+        for (int i = position; i < limit; i += 2) {
+          int pcm32BitInteger =
+              ((inputBuffer.get(i + 1) & 0xFF) << 16) | ((inputBuffer.get(i) & 0xFF) << 24);
+          writePcm32BitFloat(pcm32BitInteger, buffer);
+        }
+        break;
       case C.ENCODING_PCM_24BIT:
         buffer = replaceOutputBuffer((size / 3) * 4);
         for (int i = position; i < limit; i += 3) {
@@ -120,8 +137,6 @@ public void queueInput(ByteBuffer inputBuffer) {
           buffer.putFloat((float) inputBuffer.getDouble(i));
         }
         break;
-      case C.ENCODING_PCM_8BIT:
-      case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
       case C.ENCODING_PCM_FLOAT:
       case C.ENCODING_INVALID:
       case Format.NO_VALUE:

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/ToFloatPcmAudioProcessorTest.java

@@ -16,6 +16,7 @@
 package androidx.media3.exoplayer.audio;
 
 import static androidx.media3.common.util.Util.getByteDepth;
+import static androidx.media3.test.utils.TestUtil.createByteArray;
 import static androidx.media3.test.utils.TestUtil.createByteBuffer;
 import static androidx.media3.test.utils.TestUtil.createFloatArray;
 import static com.google.common.truth.Truth.assertThat;
@@ -39,15 +40,17 @@ public class ToFloatPcmAudioProcessorTest {
    * <p>Can be one of:
    *
    * <ul>
+   *   <li>{@link C#ENCODING_PCM_8BIT}
    *   <li>{@link C#ENCODING_PCM_16BIT}
-   *   <li>{@link C#ENCODING_PCM_32BIT}
+   *   <li>{@link C#ENCODING_PCM_16BIT_BIG_ENDIAN}
    *   <li>{@link C#ENCODING_PCM_24BIT}
-   *   <li>{@link C#ENCODING_PCM_32BIT_BIG_ENDIAN}
    *   <li>{@link C#ENCODING_PCM_24BIT_BIG_ENDIAN}
+   *   <li>{@link C#ENCODING_PCM_32BIT}
+   *   <li>{@link C#ENCODING_PCM_32BIT_BIG_ENDIAN}
    *   <li>{@link C#ENCODING_PCM_DOUBLE}
    * </ul>
    */
-  @TestParameter({"2", "22", "21", "1610612736", "1342177280", "1879048192"})
+  @TestParameter({"3", "2", "268435456", "21", "1342177280", "22", "1610612736", "1879048192"})
   private int pcmEncoding;
 
   @Test
@@ -64,13 +67,18 @@ public void queueInput_withNonZeroValues_returnsCorrectConvertedValues() throws
   }
 
   @Test
-  public void queueInput_withZero_returnsZero() throws Exception {
+  public void queueInput_withSilence_returnsZero() throws Exception {
     ToFloatPcmAudioProcessor processor = new ToFloatPcmAudioProcessor();
     processor.configure(
         new AudioFormat(/* sampleRate= */ 44100, /* channelCount= */ 1, pcmEncoding));
     processor.flush(StreamMetadata.DEFAULT);
 
     ByteBuffer buffer = ByteBuffer.allocateDirect(getByteDepth(pcmEncoding));
+    if (pcmEncoding == C.ENCODING_PCM_8BIT) {
+      // 8-bit PCM is unsigned with a neutral midpoint of 128 (representing silence).
+      buffer.put((byte) 128);
+      buffer.flip();
+    }
 
     processor.queueInput(buffer);
     assertThat(createFloatArray(processor.getOutput())).isEqualTo(new float[] {0f});
@@ -87,7 +95,10 @@ public void configure_returnsFloatPcmEncoding() throws Exception {
 
   private static float getToleranceForEncoding(int pcmEncoding) {
     switch (pcmEncoding) {
+      case C.ENCODING_PCM_8BIT:
+        return 1f / 0x80;
       case C.ENCODING_PCM_16BIT:
+      case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
         return 1f / 0x8000;
       case C.ENCODING_PCM_32BIT:
       case C.ENCODING_PCM_32BIT_BIG_ENDIAN:
@@ -107,9 +118,19 @@ private static float getToleranceForEncoding(int pcmEncoding) {
    */
   private static ByteBuffer getTestSamplesForEncoding(int pcmEncoding) {
     switch (pcmEncoding) {
+      case C.ENCODING_PCM_8BIT:
+        return createByteBuffer(createByteArray(0xFF, 0, 0xC0, 0x40));
       case C.ENCODING_PCM_16BIT:
         return createByteBuffer(
             new short[] {Short.MAX_VALUE, Short.MIN_VALUE, 0x4000, (short) 0xC000});
+      case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
+        return createByteBuffer(
+            new short[] {
+              Short.reverseBytes(Short.MAX_VALUE),
+              Short.reverseBytes(Short.MIN_VALUE),
+              Short.reverseBytes((short) 0x4000),
+              Short.reverseBytes((short) 0xC000)
+            });
       case C.ENCODING_PCM_32BIT:
         return createByteBuffer(
             new int[] {

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/Mp4PlaybackTest.java

@@ -70,6 +70,7 @@ public static ImmutableList<Sample> mediaSamples() {
         Sample.forFile("sample_opus_fragmented.mp4"),
         Sample.forFile("sample_opus.mp4"),
         Sample.forFile("sample_alac.mp4"),
+        Sample.forFile("sample_fpcm_64le.mp4"),
         Sample.forFile("sample_partially_fragmented.mp4"),
         Sample.withSubtitles("sample_with_vobsub.mp4", "eng"),
         Sample.forFile("testvid_1022ms.mp4"),

libraries/extractor/src/main/java/androidx/media3/extractor/WavUtil.java

@@ -76,6 +76,7 @@ public static int getTypeForPcmEncoding(@C.PcmEncoding int pcmEncoding) {
       case C.ENCODING_PCM_32BIT:
         return TYPE_PCM;
       case C.ENCODING_PCM_FLOAT:
+      case C.ENCODING_PCM_DOUBLE:
         return TYPE_FLOAT;
       // TYPE_PCM is little endian so big endian formats don't match.
       case C.ENCODING_PCM_16BIT_BIG_ENDIAN:
@@ -98,7 +99,7 @@ public static int getTypeForPcmEncoding(@C.PcmEncoding int pcmEncoding) {
       case TYPE_WAVE_FORMAT_EXTENSIBLE:
         return Util.getPcmEncoding(bitsPerSample);
       case TYPE_FLOAT:
-        return bitsPerSample == 32 ? C.ENCODING_PCM_FLOAT : C.ENCODING_INVALID;
+        return Util.getFloatPcmEncoding(bitsPerSample);
       default:
         return C.ENCODING_INVALID;
     }

libraries/extractor/src/main/java/androidx/media3/extractor/mkv/MatroskaExtractor.java

@@ -2451,9 +2451,8 @@ public void initializeFormat(int trackId) throws ParserException {
           break;
         case CODEC_ID_PCM_FLOAT:
           mimeType = MimeTypes.AUDIO_RAW;
-          if (audioBitDepth == 32) {
-            pcmEncoding = C.ENCODING_PCM_FLOAT;
-          } else {
+          pcmEncoding = Util.getFloatPcmEncoding(audioBitDepth);
+          if (pcmEncoding == C.ENCODING_INVALID) {
             pcmEncoding = Format.NO_VALUE;
             mimeType = MimeTypes.AUDIO_UNKNOWN;
             Log.w(

libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java

@@ -2131,8 +2131,8 @@ private static void parseAudioSampleEntry(
       boolean isBigEndian = (formatSpecificFlags & (1 << 1)) != 0;
       if (!isFloat) {
         pcmEncoding = Util.getPcmEncoding(bitsPerSample, isBigEndian ? BIG_ENDIAN : LITTLE_ENDIAN);
-      } else if (!isBigEndian && bitsPerSample == 32) {
-        pcmEncoding = C.ENCODING_PCM_FLOAT;
+      } else if (!isBigEndian) {
+        pcmEncoding = Util.getFloatPcmEncoding(bitsPerSample);
       }
       if (pcmEncoding == C.ENCODING_INVALID) {
         pcmEncoding = Format.NO_VALUE;
@@ -2389,11 +2389,12 @@ private static void parseAudioSampleEntry(
         int sampleSize = parent.readUnsignedByte();
         if (atomType == Mp4Box.TYPE_ipcm) {
           pcmEncoding = Util.getPcmEncoding(sampleSize, byteOrder);
-        } else if (atomType == Mp4Box.TYPE_fpcm
-            && sampleSize == 32
-            && byteOrder.equals(LITTLE_ENDIAN)) {
-          // Only single-width little-endian floating point PCM is supported.
-          pcmEncoding = C.ENCODING_PCM_FLOAT;
+        } else if (atomType == Mp4Box.TYPE_fpcm && byteOrder.equals(LITTLE_ENDIAN)) {
+          // Only little-endian floating point PCM is supported.
+          pcmEncoding = Util.getFloatPcmEncoding(sampleSize);
+        }
+        if (pcmEncoding == C.ENCODING_INVALID) {
+          pcmEncoding = Format.NO_VALUE;
         }
         if (pcmEncoding != Format.NO_VALUE) {
           mimeType = MimeTypes.AUDIO_RAW;

libraries/extractor/src/test/java/androidx/media3/extractor/mkv/MatroskaExtractorTest.java

@@ -243,6 +243,14 @@ public void mkvSample_withDtsX() throws Exception {
         simulationConfig);
   }
 
+  @Test
+  public void mkaSample_withFpcm64le() throws Exception {
+    ExtractorAsserts.assertBehavior(
+        getExtractorFactory(subtitlesParsedDuringExtraction),
+        "media/mka/bear-pcm-f64le.mka",
+        simulationConfig);
+  }
+
   private static ExtractorAsserts.ExtractorFactory getExtractorFactory(
       boolean subtitlesParsedDuringExtraction) {
     SubtitleParser.Factory subtitleParserFactory;

libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorParameterizedTest.java

@@ -296,6 +296,11 @@ public void mp4SampleWith32leFpcm() throws Exception {
     assertExtractorBehavior("media/mp4/sample_fpcm_32le.mp4", /* peekLimit= */ 50);
   }
 
+  @Test
+  public void mp4SampleWith64leFpcm() throws Exception {
+    assertExtractorBehavior("media/mp4/sample_fpcm_64le.mp4", /* peekLimit= */ 50);
+  }
+
   // Only the rotation part of the transformation matrix is resolved (b/390422593 tracks supporting
   // reflection too).
   @Test

libraries/extractor/src/test/java/androidx/media3/extractor/wav/WavExtractorTest.java

@@ -74,4 +74,10 @@ public void sample_wav_format_extensible() throws Exception {
     ExtractorAsserts.assertBehavior(
         WavExtractor::new, "media/wav/sample_wav_format_extensible.wav", simulationConfig);
   }
+
+  @Test
+  public void sample_float64() throws Exception {
+    ExtractorAsserts.assertBehavior(
+        WavExtractor::new, "media/wav/sample_float64.wav", simulationConfig);
+  }
 }