Skip to content

Display and control your Android device

Encode recorded audio on the device

Romain Vimont5eed2c52c2f3
diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
index 45e217fc..7b571126 100644
--- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
+++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
@@ -4,21 +4,63 @@ import android.annotation.SuppressLint;
import android.annotation.TargetApi;
import android.media.AudioFormat;
import android.media.AudioRecord;
+import android.media.AudioTimestamp;
+import android.media.MediaCodec;
+import android.media.MediaFormat;
import android.media.MediaRecorder;
import android.os.Build;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.os.Looper;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
public final class AudioEncoder {
+ private static class InputTask {
+ private final int index;
+
+ InputTask(int index) {
+ this.index = index;
+ }
+ }
+
+ private static class OutputTask {
+ private final int index;
+ private final MediaCodec.BufferInfo bufferInfo;
+
+ OutputTask(int index, MediaCodec.BufferInfo bufferInfo) {
+ this.index = index;
+ this.bufferInfo = bufferInfo;
+ }
+ }
+
+ private static final String MIMETYPE = MediaFormat.MIMETYPE_AUDIO_OPUS;
private static final int SAMPLE_RATE = 48000;
private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
private static final int CHANNELS = 2;
private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
private static final int BYTES_PER_SAMPLE = 2;
+ private static final int BIT_RATE = 128000;
private static final int READ_MS = 5; // milliseconds
private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000;
+ // Capacity of 64 is in practice "infinite" (it is limited by the number of available MediaCodec buffers, typically 4).
+ // So many pending tasks would lead to an unacceptable delay anyway.
+ private final BlockingQueue<InputTask> inputTasks = new ArrayBlockingQueue<>(64);
+ private final BlockingQueue<OutputTask> outputTasks = new ArrayBlockingQueue<>(64);
+
private Thread thread;
+ private HandlerThread mediaCodecThread;
+
+ private Thread inputThread;
+ private Thread outputThread;
+
+ private boolean ended;
private static AudioFormat createAudioFormat() {
AudioFormat.Builder builder = new AudioFormat.Builder();
@@ -44,23 +86,80 @@ public final class AudioEncoder {
return builder.build();
}
- public void start() {
- AudioRecord recorder = createAudioRecord();
+ private static MediaFormat createFormat() {
+ MediaFormat format = new MediaFormat();
+ format.setString(MediaFormat.KEY_MIME, MIMETYPE);
+ format.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
+ format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS);
+ format.setInteger(MediaFormat.KEY_SAMPLE_RATE, SAMPLE_RATE);
+ return format;
+ }
+
+ @TargetApi(Build.VERSION_CODES.N)
+ private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException {
+ final AudioTimestamp timestamp = new AudioTimestamp();
+ long previousPts = 0;
+ long nextPts = 0;
+
+ while (!Thread.currentThread().isInterrupted()) {
+ InputTask task = inputTasks.take();
+ ByteBuffer buffer = mediaCodec.getInputBuffer(task.index);
+ int r = recorder.read(buffer, READ_SIZE);
+ if (r < 0) {
+ throw new IOException("Could not read audio: " + r);
+ }
+
+ long pts;
+
+ int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
+ if (ret == AudioRecord.SUCCESS) {
+ pts = timestamp.nanoTime / 1000;
+ } else {
+ if (nextPts == 0) {
+ Ln.w("Could not get any audio timestamp");
+ }
+ // compute from previous timestamp and packet size
+ pts = nextPts;
+ }
+
+ long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
+ nextPts = pts + durationUs;
+ if (previousPts != 0 && pts < previousPts) {
+ // Audio PTS may come from two sources:
+ // - recorder.getTimestamp() if the call works;
+ // - an estimation from the previous PTS and the packet size as a fallback.
+ //
+ // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
+ pts = previousPts + 1;
+ }
+
+ previousPts = pts;
+
+ mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0);
+ }
+ }
+
+ private void outputThread(MediaCodec mediaCodec) throws IOException, InterruptedException {
+ while (!Thread.currentThread().isInterrupted()) {
+ OutputTask task = outputTasks.take();
+ ByteBuffer buffer = mediaCodec.getOutputBuffer(task.index);
+ try {
+ Ln.i("Audio packet [pts=" + task.bufferInfo.presentationTimeUs + "] " + buffer.remaining() + " bytes");
+ } finally {
+ mediaCodec.releaseOutputBuffer(task.index, false);
+ }
+ }
+ }
+
+ public void start() {
thread = new Thread(() -> {
- recorder.startRecording();
try {
- byte[] buf = new byte[READ_SIZE];
- while (!Thread.currentThread().isInterrupted()) {
- int r = recorder.read(buf, 0, READ_SIZE);
- if (r > 0) {
- Ln.i("Audio captured: " + r + " bytes");
- } else {
- Ln.e("Audio capture error: " + r);
- }
- }
+ encode();
+ } catch (IOException e) {
+ Ln.e("Audio encoding error", e);
} finally {
- recorder.stop();
+ Ln.d("Audio encoder stopped");
}
});
thread.start();
@@ -68,7 +167,8 @@ public final class AudioEncoder {
public void stop() {
if (thread != null) {
- thread.interrupt();
+ // Just wake up the blocking wait from the thread, so that it properly releases all its resources and terminates
+ end();
}
}
@@ -77,4 +177,150 @@ public final class AudioEncoder {
thread.join();
}
}
+
+ private synchronized void end() {
+ ended = true;
+ notify();
+ }
+
+ private synchronized void waitEnded() {
+ try {
+ while (!ended) {
+ wait();
+ }
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+
+ @TargetApi(Build.VERSION_CODES.M)
+ public void encode() throws IOException {
+ MediaCodec mediaCodec = null;
+ AudioRecord recorder = null;
+
+ boolean mediaCodecStarted = false;
+ boolean recorderStarted = false;
+ try {
+ mediaCodec = MediaCodec.createEncoderByType(MIMETYPE); // may throw IOException
+
+ mediaCodecThread = new HandlerThread("AudioEncoder");
+ mediaCodecThread.start();
+
+ MediaFormat format = createFormat();
+ mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper()));
+ mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+
+ recorder = createAudioRecord();
+ recorder.startRecording();
+ recorderStarted = true;
+
+ final MediaCodec mediaCodecRef = mediaCodec;
+ final AudioRecord recorderRef = recorder;
+ inputThread = new Thread(() -> {
+ try {
+ inputThread(mediaCodecRef, recorderRef);
+ } catch (IOException | InterruptedException e) {
+ Ln.e("Audio capture error", e);
+ } finally {
+ end();
+ }
+ });
+
+ outputThread = new Thread(() -> {
+ try {
+ outputThread(mediaCodecRef);
+ } catch (InterruptedException e) {
+ // this is expected on close
+ } catch (IOException e) {
+ // Broken pipe is expected on close, because the socket is closed by the client
+ if (!IO.isBrokenPipe(e)) {
+ Ln.e("Audio encoding error", e);
+ }
+ } finally {
+ end();
+ }
+ });
+
+ mediaCodec.start();
+ mediaCodecStarted = true;
+ inputThread.start();
+ outputThread.start();
+
+ waitEnded();
+ } finally {
+ // Cleanup everything (either at the end or on error at any step of the initialization)
+ if (mediaCodecThread != null) {
+ Looper looper = mediaCodecThread.getLooper();
+ if (looper != null) {
+ looper.quitSafely();
+ }
+ }
+ if (inputThread != null) {
+ inputThread.interrupt();
+ }
+ if (outputThread != null) {
+ outputThread.interrupt();
+ }
+
+ try {
+ if (mediaCodecThread != null) {
+ mediaCodecThread.join();
+ }
+ if (inputThread != null) {
+ inputThread.join();
+ }
+ if (outputThread != null) {
+ outputThread.join();
+ }
+ } catch (InterruptedException e) {
+ // Should never happen
+ throw new AssertionError(e);
+ }
+
+ if (mediaCodec != null) {
+ if (mediaCodecStarted) {
+ mediaCodec.stop();
+ }
+ mediaCodec.release();
+ }
+ if (recorder != null) {
+ if (recorderStarted) {
+ recorder.stop();
+ }
+ recorder.release();
+ }
+ }
+ }
+
+ private class EncoderCallback extends MediaCodec.Callback {
+ @TargetApi(Build.VERSION_CODES.N)
+ @Override
+ public void onInputBufferAvailable(MediaCodec codec, int index) {
+ try {
+ inputTasks.put(new InputTask(index));
+ } catch (InterruptedException e) {
+ end();
+ }
+ }
+
+ @Override
+ public void onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo bufferInfo) {
+ try {
+ outputTasks.put(new OutputTask(index, bufferInfo));
+ } catch (InterruptedException e) {
+ end();
+ }
+ }
+
+ @Override
+ public void onError(MediaCodec codec, MediaCodec.CodecException e) {
+ Ln.e("MediaCodec error", e);
+ end();
+ }
+
+ @Override
+ public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) {
+ // ignore
+ }
+ }
}