Skip to content

Display and control your Android device

Extract audio capture

Romain Vimont65cc9d765d8d
diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java
new file mode 100644
index 00000000..3cef7801
--- /dev/null
+++ b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java
@@ -0,0 +1,148 @@
+package com.genymobile.scrcpy;
+
+import com.genymobile.scrcpy.wrappers.ServiceManager;
+
+import android.annotation.SuppressLint;
+import android.annotation.TargetApi;
+import android.content.ComponentName;
+import android.content.Intent;
+import android.media.AudioFormat;
+import android.media.AudioRecord;
+import android.media.AudioTimestamp;
+import android.media.MediaCodec;
+import android.media.MediaRecorder;
+import android.os.Build;
+import android.os.SystemClock;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public final class AudioCapture {
+
+ public static final int SAMPLE_RATE = 48000;
+ public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
+ public static final int CHANNELS = 2;
+ public static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
+ public static final int BYTES_PER_SAMPLE = 2;
+
+ private AudioRecord recorder;
+
+ private final AudioTimestamp timestamp = new AudioTimestamp();
+ private long previousPts = 0;
+ private long nextPts = 0;
+
+ public static int millisToBytes(int millis) {
+ return SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * millis / 1000;
+ }
+
+ private static AudioFormat createAudioFormat() {
+ AudioFormat.Builder builder = new AudioFormat.Builder();
+ builder.setEncoding(FORMAT);
+ builder.setSampleRate(SAMPLE_RATE);
+ builder.setChannelMask(CHANNEL_CONFIG);
+ return builder.build();
+ }
+
+ @TargetApi(Build.VERSION_CODES.M)
+ @SuppressLint({"WrongConstant", "MissingPermission"})
+ private static AudioRecord createAudioRecord() {
+ AudioRecord.Builder builder = new AudioRecord.Builder();
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
+ // On older APIs, Workarounds.fillAppInfo() must be called beforehand
+ builder.setContext(FakeContext.get());
+ }
+ builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
+ builder.setAudioFormat(createAudioFormat());
+ int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT);
+ // This buffer size does not impact latency
+ builder.setBufferSizeInBytes(8 * minBufferSize);
+ return builder.build();
+ }
+
+ private static void startWorkaroundAndroid11() {
+ if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
+ // Android 11 requires Apps to be at foreground to record audio.
+ // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground.
+ // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android
+ // shell ("com.android.shell").
+ // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the
+ // foreground.
+ if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
+ Intent intent = new Intent(Intent.ACTION_MAIN);
+ intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
+ intent.addCategory(Intent.CATEGORY_LAUNCHER);
+ intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity"));
+ ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent);
+ // Wait for activity to start
+ SystemClock.sleep(150);
+ }
+ }
+ }
+
+ private static void stopWorkaroundAndroid11() {
+ if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
+ ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME);
+ }
+ }
+
+ public void start() throws AudioCaptureForegroundException {
+ startWorkaroundAndroid11();
+ try {
+ recorder = createAudioRecord();
+ recorder.startRecording();
+ } catch (UnsupportedOperationException e) {
+ if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
+ Ln.e("Failed to start audio capture");
+ Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy.");
+ throw new AudioCaptureForegroundException();
+ }
+ throw e;
+ } finally {
+ stopWorkaroundAndroid11();
+ }
+ }
+
+ public void stop() {
+ if (recorder != null) {
+ // Will call .stop() if necessary, without throwing an IllegalStateException
+ recorder.release();
+ }
+ }
+
+ @TargetApi(Build.VERSION_CODES.N)
+ public int read(ByteBuffer directBuffer, int size, MediaCodec.BufferInfo outBufferInfo) throws IOException {
+ int r = recorder.read(directBuffer, size);
+ if (r < 0) {
+ return r;
+ }
+
+ long pts;
+
+ int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
+ if (ret == AudioRecord.SUCCESS) {
+ pts = timestamp.nanoTime / 1000;
+ } else {
+ if (nextPts == 0) {
+ Ln.w("Could not get any audio timestamp");
+ }
+ // compute from previous timestamp and packet size
+ pts = nextPts;
+ }
+
+ long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
+ nextPts = pts + durationUs;
+
+ if (previousPts != 0 && pts < previousPts) {
+ // Audio PTS may come from two sources:
+ // - recorder.getTimestamp() if the call works;
+ // - an estimation from the previous PTS and the packet size as a fallback.
+ //
+ // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
+ pts = previousPts + 1;
+ }
+ previousPts = pts;
+
+ outBufferInfo.set(0, r, pts, 0);
+ return r;
+ }
+}
diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
index cc786bdb..8b60d37e 100644
--- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
+++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java
@@ -1,22 +1,12 @@
package com.genymobile.scrcpy;
-import com.genymobile.scrcpy.wrappers.ServiceManager;
-
-import android.annotation.SuppressLint;
import android.annotation.TargetApi;
-import android.content.ComponentName;
-import android.content.Intent;
-import android.media.AudioFormat;
-import android.media.AudioRecord;
-import android.media.AudioTimestamp;
import android.media.MediaCodec;
import android.media.MediaFormat;
-import android.media.MediaRecorder;
import android.os.Build;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Looper;
-import android.os.SystemClock;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -44,14 +34,11 @@ public final class AudioEncoder {
}
}
- private static final int SAMPLE_RATE = 48000;
- private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
- private static final int CHANNELS = 2;
- private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;
- private static final int BYTES_PER_SAMPLE = 2;
+ private static final int SAMPLE_RATE = AudioCapture.SAMPLE_RATE;
+ private static final int CHANNELS = AudioCapture.CHANNELS;
private static final int READ_MS = 5; // milliseconds
- private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000;
+ private static final int READ_SIZE = AudioCapture.millisToBytes(READ_MS);
private final Streamer streamer;
private final int bitRate;
@@ -78,30 +65,6 @@ public final class AudioEncoder {
this.encoderName = encoderName;
}
- private static AudioFormat createAudioFormat() {
- AudioFormat.Builder builder = new AudioFormat.Builder();
- builder.setEncoding(FORMAT);
- builder.setSampleRate(SAMPLE_RATE);
- builder.setChannelMask(CHANNEL_CONFIG);
- return builder.build();
- }
-
- @TargetApi(Build.VERSION_CODES.M)
- @SuppressLint({"WrongConstant", "MissingPermission"})
- private static AudioRecord createAudioRecord() {
- AudioRecord.Builder builder = new AudioRecord.Builder();
- if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
- // On older APIs, Workarounds.fillAppInfo() must be called beforehand
- builder.setContext(FakeContext.get());
- }
- builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);
- builder.setAudioFormat(createAudioFormat());
- int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT);
- // This buffer size does not impact latency
- builder.setBufferSizeInBytes(8 * minBufferSize);
- return builder.build();
- }
-
private static MediaFormat createFormat(String mimeType, int bitRate, List<CodecOption> codecOptions) {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, mimeType);
@@ -122,47 +85,18 @@ public final class AudioEncoder {
}
@TargetApi(Build.VERSION_CODES.N)
- private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException {
- final AudioTimestamp timestamp = new AudioTimestamp();
- long previousPts = 0;
- long nextPts = 0;
+ private void inputThread(MediaCodec mediaCodec, AudioCapture capture) throws IOException, InterruptedException {
+ final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
while (!Thread.currentThread().isInterrupted()) {
InputTask task = inputTasks.take();
ByteBuffer buffer = mediaCodec.getInputBuffer(task.index);
- int r = recorder.read(buffer, READ_SIZE);
+ int r = capture.read(buffer, READ_SIZE, bufferInfo);
if (r < 0) {
throw new IOException("Could not read audio: " + r);
}
- long pts;
-
- int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
- if (ret == AudioRecord.SUCCESS) {
- pts = timestamp.nanoTime / 1000;
- } else {
- if (nextPts == 0) {
- Ln.w("Could not get any audio timestamp");
- }
- // compute from previous timestamp and packet size
- pts = nextPts;
- }
-
- long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
- nextPts = pts + durationUs;
-
- if (previousPts != 0 && pts < previousPts) {
- // Audio PTS may come from two sources:
- // - recorder.getTimestamp() if the call works;
- // - an estimation from the previous PTS and the packet size as a fallback.
- //
- // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
- pts = previousPts + 1;
- }
-
- previousPts = pts;
-
- mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0);
+ mediaCodec.queueInputBuffer(task.index, bufferInfo.offset, bufferInfo.size, bufferInfo.presentationTimeUs, bufferInfo.flags);
}
}
@@ -223,32 +157,6 @@ public final class AudioEncoder {
}
}
- private static void startWorkaroundAndroid11() {
- if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
- // Android 11 requires Apps to be at foreground to record audio.
- // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground.
- // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android
- // shell ("com.android.shell").
- // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the
- // foreground.
- if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
- Intent intent = new Intent(Intent.ACTION_MAIN);
- intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
- intent.addCategory(Intent.CATEGORY_LAUNCHER);
- intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity"));
- ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent);
- // Wait for activity to start
- SystemClock.sleep(150);
- }
- }
- }
-
- private static void stopWorkaroundAndroid11() {
- if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
- ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME);
- }
- }
-
@TargetApi(Build.VERSION_CODES.M)
public void encode() throws IOException, ConfigurationException, AudioCaptureForegroundException {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
@@ -258,10 +166,9 @@ public final class AudioEncoder {
}
MediaCodec mediaCodec = null;
- AudioRecord recorder = null;
+ AudioCapture capture = new AudioCapture();
boolean mediaCodecStarted = false;
- boolean recorderStarted = false;
try {
Codec codec = streamer.getCodec();
mediaCodec = createMediaCodec(codec, encoderName);
@@ -273,27 +180,13 @@ public final class AudioEncoder {
mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper()));
mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
- startWorkaroundAndroid11();
- try {
- recorder = createAudioRecord();
- recorder.startRecording();
- } catch (UnsupportedOperationException e) {
- if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
- Ln.e("Failed to start audio capture");
- Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy.");
- throw new AudioCaptureForegroundException();
- }
- throw e;
- } finally {
- stopWorkaroundAndroid11();
- }
- recorderStarted = true;
+ capture.start();
final MediaCodec mediaCodecRef = mediaCodec;
- final AudioRecord recorderRef = recorder;
+ final AudioCapture captureRef = capture;
inputThread = new Thread(() -> {
try {
- inputThread(mediaCodecRef, recorderRef);
+ inputThread(mediaCodecRef, captureRef);
} catch (IOException | InterruptedException e) {
Ln.e("Audio capture error", e);
} finally {
@@ -366,11 +259,8 @@ public final class AudioEncoder {
}
mediaCodec.release();
}
- if (recorder != null) {
- if (recorderStarted) {
- recorder.stop();
- }
- recorder.release();
+ if (capture != null) {
+ capture.stop();
}
}
}