Display and control your Android device
Extract audio capture
Romain Vimont65cc9d765d8d
diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.javanew file mode 100644index 00000000..3cef7801--- /dev/null+++ b/server/src/main/java/com/genymobile/scrcpy/AudioCapture.java@@ -0,0 +1,148 @@+package com.genymobile.scrcpy;++import com.genymobile.scrcpy.wrappers.ServiceManager;++import android.annotation.SuppressLint;+import android.annotation.TargetApi;+import android.content.ComponentName;+import android.content.Intent;+import android.media.AudioFormat;+import android.media.AudioRecord;+import android.media.AudioTimestamp;+import android.media.MediaCodec;+import android.media.MediaRecorder;+import android.os.Build;+import android.os.SystemClock;++import java.io.IOException;+import java.nio.ByteBuffer;++public final class AudioCapture {++ public static final int SAMPLE_RATE = 48000;+ public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;+ public static final int CHANNELS = 2;+ public static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;+ public static final int BYTES_PER_SAMPLE = 2;++ private AudioRecord recorder;++ private final AudioTimestamp timestamp = new AudioTimestamp();+ private long previousPts = 0;+ private long nextPts = 0;++ public static int millisToBytes(int millis) {+ return SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * millis / 1000;+ }++ private static AudioFormat createAudioFormat() {+ AudioFormat.Builder builder = new AudioFormat.Builder();+ builder.setEncoding(FORMAT);+ builder.setSampleRate(SAMPLE_RATE);+ builder.setChannelMask(CHANNEL_CONFIG);+ return builder.build();+ }++ @TargetApi(Build.VERSION_CODES.M)+ @SuppressLint({"WrongConstant", "MissingPermission"})+ private static AudioRecord createAudioRecord() {+ AudioRecord.Builder builder = new AudioRecord.Builder();+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {+ // On older APIs, Workarounds.fillAppInfo() must be called beforehand+ builder.setContext(FakeContext.get());+ }+ builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);+ builder.setAudioFormat(createAudioFormat());+ int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT);+ // This buffer size does not impact latency+ builder.setBufferSizeInBytes(8 * minBufferSize);+ return builder.build();+ }++ private static void startWorkaroundAndroid11() {+ if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {+ // Android 11 requires Apps to be at foreground to record audio.+ // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground.+ // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android+ // shell ("com.android.shell").+ // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the+ // foreground.+ if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {+ Intent intent = new Intent(Intent.ACTION_MAIN);+ intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);+ intent.addCategory(Intent.CATEGORY_LAUNCHER);+ intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity"));+ ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent);+ // Wait for activity to start+ SystemClock.sleep(150);+ }+ }+ }++ private static void stopWorkaroundAndroid11() {+ if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {+ ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME);+ }+ }++ public void start() throws AudioCaptureForegroundException {+ startWorkaroundAndroid11();+ try {+ recorder = createAudioRecord();+ recorder.startRecording();+ } catch (UnsupportedOperationException e) {+ if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {+ Ln.e("Failed to start audio capture");+ Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy.");+ throw new AudioCaptureForegroundException();+ }+ throw e;+ } finally {+ stopWorkaroundAndroid11();+ }+ }++ public void stop() {+ if (recorder != null) {+ // Will call .stop() if necessary, without throwing an IllegalStateException+ recorder.release();+ }+ }++ @TargetApi(Build.VERSION_CODES.N)+ public int read(ByteBuffer directBuffer, int size, MediaCodec.BufferInfo outBufferInfo) throws IOException {+ int r = recorder.read(directBuffer, size);+ if (r < 0) {+ return r;+ }++ long pts;++ int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);+ if (ret == AudioRecord.SUCCESS) {+ pts = timestamp.nanoTime / 1000;+ } else {+ if (nextPts == 0) {+ Ln.w("Could not get any audio timestamp");+ }+ // compute from previous timestamp and packet size+ pts = nextPts;+ }++ long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);+ nextPts = pts + durationUs;++ if (previousPts != 0 && pts < previousPts) {+ // Audio PTS may come from two sources:+ // - recorder.getTimestamp() if the call works;+ // - an estimation from the previous PTS and the packet size as a fallback.+ //+ // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.+ pts = previousPts + 1;+ }+ previousPts = pts;++ outBufferInfo.set(0, r, pts, 0);+ return r;+ }+}diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.javaindex cc786bdb..8b60d37e 100644--- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java+++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java@@ -1,22 +1,12 @@package com.genymobile.scrcpy;-import com.genymobile.scrcpy.wrappers.ServiceManager;--import android.annotation.SuppressLint;import android.annotation.TargetApi;-import android.content.ComponentName;-import android.content.Intent;-import android.media.AudioFormat;-import android.media.AudioRecord;-import android.media.AudioTimestamp;import android.media.MediaCodec;import android.media.MediaFormat;-import android.media.MediaRecorder;import android.os.Build;import android.os.Handler;import android.os.HandlerThread;import android.os.Looper;-import android.os.SystemClock;import java.io.IOException;import java.nio.ByteBuffer;@@ -44,14 +34,11 @@ public final class AudioEncoder {}}- private static final int SAMPLE_RATE = 48000;- private static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;- private static final int CHANNELS = 2;- private static final int FORMAT = AudioFormat.ENCODING_PCM_16BIT;- private static final int BYTES_PER_SAMPLE = 2;+ private static final int SAMPLE_RATE = AudioCapture.SAMPLE_RATE;+ private static final int CHANNELS = AudioCapture.CHANNELS;private static final int READ_MS = 5; // milliseconds- private static final int READ_SIZE = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE * READ_MS / 1000;+ private static final int READ_SIZE = AudioCapture.millisToBytes(READ_MS);private final Streamer streamer;private final int bitRate;@@ -78,30 +65,6 @@ public final class AudioEncoder {this.encoderName = encoderName;}- private static AudioFormat createAudioFormat() {- AudioFormat.Builder builder = new AudioFormat.Builder();- builder.setEncoding(FORMAT);- builder.setSampleRate(SAMPLE_RATE);- builder.setChannelMask(CHANNEL_CONFIG);- return builder.build();- }-- @TargetApi(Build.VERSION_CODES.M)- @SuppressLint({"WrongConstant", "MissingPermission"})- private static AudioRecord createAudioRecord() {- AudioRecord.Builder builder = new AudioRecord.Builder();- if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {- // On older APIs, Workarounds.fillAppInfo() must be called beforehand- builder.setContext(FakeContext.get());- }- builder.setAudioSource(MediaRecorder.AudioSource.REMOTE_SUBMIX);- builder.setAudioFormat(createAudioFormat());- int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, FORMAT);- // This buffer size does not impact latency- builder.setBufferSizeInBytes(8 * minBufferSize);- return builder.build();- }-private static MediaFormat createFormat(String mimeType, int bitRate, List<CodecOption> codecOptions) {MediaFormat format = new MediaFormat();format.setString(MediaFormat.KEY_MIME, mimeType);@@ -122,47 +85,18 @@ public final class AudioEncoder {}@TargetApi(Build.VERSION_CODES.N)- private void inputThread(MediaCodec mediaCodec, AudioRecord recorder) throws IOException, InterruptedException {- final AudioTimestamp timestamp = new AudioTimestamp();- long previousPts = 0;- long nextPts = 0;+ private void inputThread(MediaCodec mediaCodec, AudioCapture capture) throws IOException, InterruptedException {+ final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();while (!Thread.currentThread().isInterrupted()) {InputTask task = inputTasks.take();ByteBuffer buffer = mediaCodec.getInputBuffer(task.index);- int r = recorder.read(buffer, READ_SIZE);+ int r = capture.read(buffer, READ_SIZE, bufferInfo);if (r < 0) {throw new IOException("Could not read audio: " + r);}- long pts;-- int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);- if (ret == AudioRecord.SUCCESS) {- pts = timestamp.nanoTime / 1000;- } else {- if (nextPts == 0) {- Ln.w("Could not get any audio timestamp");- }- // compute from previous timestamp and packet size- pts = nextPts;- }-- long durationUs = r * 1000000 / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);- nextPts = pts + durationUs;-- if (previousPts != 0 && pts < previousPts) {- // Audio PTS may come from two sources:- // - recorder.getTimestamp() if the call works;- // - an estimation from the previous PTS and the packet size as a fallback.- //- // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.- pts = previousPts + 1;- }-- previousPts = pts;-- mediaCodec.queueInputBuffer(task.index, 0, r, pts, 0);+ mediaCodec.queueInputBuffer(task.index, bufferInfo.offset, bufferInfo.size, bufferInfo.presentationTimeUs, bufferInfo.flags);}}@@ -223,32 +157,6 @@ public final class AudioEncoder {}}- private static void startWorkaroundAndroid11() {- if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {- // Android 11 requires Apps to be at foreground to record audio.- // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground.- // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android- // shell ("com.android.shell").- // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the- // foreground.- if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {- Intent intent = new Intent(Intent.ACTION_MAIN);- intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);- intent.addCategory(Intent.CATEGORY_LAUNCHER);- intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity"));- ServiceManager.getActivityManager().startActivityAsUserWithFeature(intent);- // Wait for activity to start- SystemClock.sleep(150);- }- }- }-- private static void stopWorkaroundAndroid11() {- if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {- ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME);- }- }-@TargetApi(Build.VERSION_CODES.M)public void encode() throws IOException, ConfigurationException, AudioCaptureForegroundException {if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {@@ -258,10 +166,9 @@ public final class AudioEncoder {}MediaCodec mediaCodec = null;- AudioRecord recorder = null;+ AudioCapture capture = new AudioCapture();boolean mediaCodecStarted = false;- boolean recorderStarted = false;try {Codec codec = streamer.getCodec();mediaCodec = createMediaCodec(codec, encoderName);@@ -273,27 +180,13 @@ public final class AudioEncoder {mediaCodec.setCallback(new EncoderCallback(), new Handler(mediaCodecThread.getLooper()));mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);- startWorkaroundAndroid11();- try {- recorder = createAudioRecord();- recorder.startRecording();- } catch (UnsupportedOperationException e) {- if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {- Ln.e("Failed to start audio capture");- Ln.e("On Android 11, it is only possible to capture in foreground, make sure that the device is unlocked when starting scrcpy.");- throw new AudioCaptureForegroundException();- }- throw e;- } finally {- stopWorkaroundAndroid11();- }- recorderStarted = true;+ capture.start();final MediaCodec mediaCodecRef = mediaCodec;- final AudioRecord recorderRef = recorder;+ final AudioCapture captureRef = capture;inputThread = new Thread(() -> {try {- inputThread(mediaCodecRef, recorderRef);+ inputThread(mediaCodecRef, captureRef);} catch (IOException | InterruptedException e) {Ln.e("Audio capture error", e);} finally {@@ -366,11 +259,8 @@ public final class AudioEncoder {}mediaCodec.release();}- if (recorder != null) {- if (recorderStarted) {- recorder.stop();- }- recorder.release();+ if (capture != null) {+ capture.stop();}}}