Display and control your Android device
Add audio playback capture method
Romain Vimonta10f8cd79802
diff --git a/app/data/bash-completion/scrcpy b/app/data/bash-completion/scrcpyindex b35ea5e4..d5f129d0 100644--- a/app/data/bash-completion/scrcpy+++ b/app/data/bash-completion/scrcpy@@ -111,7 +111,7 @@ _scrcpy() {return;;--audio-source)- COMPREPLY=($(compgen -W 'output mic' -- "$cur"))+ COMPREPLY=($(compgen -W 'output mic playback' -- "$cur"))return;;--camera-facing)diff --git a/app/data/zsh-completion/_scrcpy b/app/data/zsh-completion/_scrcpyindex 5afca977..c49c24eb 100644--- a/app/data/zsh-completion/_scrcpy+++ b/app/data/zsh-completion/_scrcpy@@ -14,7 +14,7 @@ arguments=('--audio-codec=[Select the audio codec]:codec:(opus aac flac raw)''--audio-codec-options=[Set a list of comma-separated key\:type=value options for the device audio encoder]''--audio-encoder=[Use a specific MediaCodec audio encoder]'- '--audio-source=[Select the audio source]:source:(output mic)'+ '--audio-source=[Select the audio source]:source:(output mic playback)''--audio-output-buffer=[Configure the size of the SDL audio output buffer (in milliseconds)]'{-b,--video-bit-rate=}'[Encode the video at the given bit-rate]''--camera-ar=[Select the camera size by its aspect ratio]'diff --git a/app/scrcpy.1 b/app/scrcpy.1index 1c0c0f7a..19b4ab6b 100644--- a/app/scrcpy.1+++ b/app/scrcpy.1@@ -57,7 +57,13 @@ The available encoders can be listed by \fB\-\-list\-encoders\fR..TP.BI "\-\-audio\-source " source-Select the audio source (output or mic).+Select the audio source (output, mic or playback).++The "output" source forwards the whole audio output, and disables playback on the device.++The "playback" source captures the audio playback (Android apps can opt-out, so the whole output is not necessarily captured).++The "mic" source captures the microphone.Default is output.diff --git a/app/src/cli.c b/app/src/cli.cindex 9dd49538..1699e46d 100644--- a/app/src/cli.c+++ b/app/src/cli.c@@ -189,7 +189,13 @@ static const struct sc_option options[] = {.longopt_id = OPT_AUDIO_SOURCE,.longopt = "audio-source",.argdesc = "source",- .text = "Select the audio source (output or mic).\n"+ .text = "Select the audio source (output, mic or playback).\n"+ "The \"output\" source forwards the whole audio output, and "+ "disables playback on the device.\n"+ "The \"playback\" source captures the audio playback (Android "+ "apps can opt-out, so the whole output is not necessarily "+ "captured).\n"+ "The \"mic\" source captures the microphone.\n""Default is output.",},{@@ -1931,7 +1937,13 @@ parse_audio_source(const char *optarg, enum sc_audio_source *source) {return true;}- LOGE("Unsupported audio source: %s (expected output or mic)", optarg);+ if (!strcmp(optarg, "playback")) {+ *source = SC_AUDIO_SOURCE_PLAYBACK;+ return true;+ }++ LOGE("Unsupported audio source: %s (expected output, mic or playback)",+ optarg);return false;}diff --git a/app/src/options.h b/app/src/options.hindex 5ec809f0..403685e3 100644--- a/app/src/options.h+++ b/app/src/options.h@@ -59,6 +59,7 @@ enum sc_audio_source {SC_AUDIO_SOURCE_AUTO, // OUTPUT for video DISPLAY, MIC for video CAMERASC_AUDIO_SOURCE_OUTPUT,SC_AUDIO_SOURCE_MIC,+ SC_AUDIO_SOURCE_PLAYBACK,};enum sc_camera_facing {diff --git a/app/src/server.c b/app/src/server.cindex 721c91df..e32aa556 100644--- a/app/src/server.c+++ b/app/src/server.c@@ -203,6 +203,21 @@ sc_server_get_camera_facing_name(enum sc_camera_facing camera_facing) {}}+static const char *+sc_server_get_audio_source_name(enum sc_audio_source audio_source) {+ switch (audio_source) {+ case SC_AUDIO_SOURCE_OUTPUT:+ return "output";+ case SC_AUDIO_SOURCE_MIC:+ return "mic";+ case SC_AUDIO_SOURCE_PLAYBACK:+ return "playback";+ default:+ assert(!"unexpected audio source");+ return NULL;+ }+}+static sc_pidexecute_server(struct sc_server *server,const struct sc_server_params *params) {@@ -273,8 +288,9 @@ execute_server(struct sc_server *server,assert(params->video_source == SC_VIDEO_SOURCE_CAMERA);ADD_PARAM("video_source=camera");}- if (params->audio_source == SC_AUDIO_SOURCE_MIC) {- ADD_PARAM("audio_source=mic");+ if (params->audio_source != SC_AUDIO_SOURCE_OUTPUT) {+ ADD_PARAM("audio_source=%s",+ sc_server_get_audio_source_name(params->audio_source));}if (params->max_size) {ADD_PARAM("max_size=%" PRIu16, params->max_size);diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.javaindex 8a88e276..75d9ea15 100644--- a/server/src/main/java/com/genymobile/scrcpy/Server.java+++ b/server/src/main/java/com/genymobile/scrcpy/Server.java@@ -4,7 +4,9 @@ import com.genymobile.scrcpy.audio.AudioCapture;import com.genymobile.scrcpy.audio.AudioCodec;import com.genymobile.scrcpy.audio.AudioDirectCapture;import com.genymobile.scrcpy.audio.AudioEncoder;+import com.genymobile.scrcpy.audio.AudioPlaybackCapture;import com.genymobile.scrcpy.audio.AudioRawRecorder;+import com.genymobile.scrcpy.audio.AudioSource;import com.genymobile.scrcpy.control.ControlChannel;import com.genymobile.scrcpy.control.Controller;import com.genymobile.scrcpy.control.DeviceMessage;@@ -164,7 +166,8 @@ public final class Server {if (audio) {AudioCodec audioCodec = options.getAudioCodec();- AudioCapture audioCapture = new AudioDirectCapture(options.getAudioSource());+ AudioSource audioSource = options.getAudioSource();+ AudioCapture audioCapture = audioSource.isDirect() ? new AudioDirectCapture(audioSource) : new AudioPlaybackCapture();Streamer audioStreamer = new Streamer(connection.getAudioFd(), audioCodec, options.getSendCodecMeta(), options.getSendFrameMeta());AsyncProcessor audioRecorder;if (audioCodec == AudioCodec.RAW) {diff --git a/server/src/main/java/com/genymobile/scrcpy/audio/AudioPlaybackCapture.java b/server/src/main/java/com/genymobile/scrcpy/audio/AudioPlaybackCapture.javanew file mode 100644index 00000000..2a0d23fb--- /dev/null+++ b/server/src/main/java/com/genymobile/scrcpy/audio/AudioPlaybackCapture.java@@ -0,0 +1,130 @@+package com.genymobile.scrcpy.audio;++import com.genymobile.scrcpy.FakeContext;+import com.genymobile.scrcpy.util.Ln;++import android.annotation.SuppressLint;+import android.annotation.TargetApi;+import android.content.Context;+import android.media.AudioAttributes;+import android.media.AudioFormat;+import android.media.AudioManager;+import android.media.AudioRecord;+import android.media.MediaCodec;+import android.os.Build;++import java.lang.reflect.Method;+import java.nio.ByteBuffer;++public final class AudioPlaybackCapture implements AudioCapture {++ private AudioRecord recorder;+ private AudioRecordReader reader;++ @SuppressLint("PrivateApi")+ private AudioRecord createAudioRecord() throws AudioCaptureException {+ // See <https://github.com/Genymobile/scrcpy/issues/4380>+ try {+ Class<?> audioMixingRuleClass = Class.forName("android.media.audiopolicy.AudioMixingRule");+ Class<?> audioMixingRuleBuilderClass = Class.forName("android.media.audiopolicy.AudioMixingRule$Builder");++ // AudioMixingRule.Builder audioMixingRuleBuilder = new AudioMixingRule.Builder();+ Object audioMixingRuleBuilder = audioMixingRuleBuilderClass.getConstructor().newInstance();++ // audioMixingRuleBuilder.setTargetMixRole(AudioMixingRule.MIX_ROLE_PLAYERS);+ int mixRolePlayersConstant = audioMixingRuleClass.getField("MIX_ROLE_PLAYERS").getInt(null);+ Method setTargetMixRoleMethod = audioMixingRuleBuilderClass.getMethod("setTargetMixRole", int.class);+ setTargetMixRoleMethod.invoke(audioMixingRuleBuilder, mixRolePlayersConstant);++ AudioAttributes attributes = new AudioAttributes.Builder().setUsage(AudioAttributes.USAGE_MEDIA).build();++ // audioMixingRuleBuilder.addMixRule(AudioMixingRule.RULE_MATCH_ATTRIBUTE_USAGE, attributes);+ int ruleMatchAttributeUsageConstant = audioMixingRuleClass.getField("RULE_MATCH_ATTRIBUTE_USAGE").getInt(null);+ Method addMixRuleMethod = audioMixingRuleBuilderClass.getMethod("addMixRule", int.class, Object.class);+ addMixRuleMethod.invoke(audioMixingRuleBuilder, ruleMatchAttributeUsageConstant, attributes);++ // AudioMixingRule audioMixingRule = builder.build();+ Object audioMixingRule = audioMixingRuleBuilderClass.getMethod("build").invoke(audioMixingRuleBuilder);++ // audioMixingRuleBuilder.voiceCommunicationCaptureAllowed(true);+ Method voiceCommunicationCaptureAllowedMethod = audioMixingRuleBuilderClass.getMethod("voiceCommunicationCaptureAllowed", boolean.class);+ voiceCommunicationCaptureAllowedMethod.invoke(audioMixingRuleBuilder, true);++ Class<?> audioMixClass = Class.forName("android.media.audiopolicy.AudioMix");+ Class<?> audioMixBuilderClass = Class.forName("android.media.audiopolicy.AudioMix$Builder");++ // AudioMix.Builder audioMixBuilder = new AudioMix.Builder(audioMixingRule);+ Object audioMixBuilder = audioMixBuilderClass.getConstructor(audioMixingRuleClass).newInstance(audioMixingRule);++ // audioMixBuilder.setFormat(createAudioFormat());+ Method setFormat = audioMixBuilder.getClass().getMethod("setFormat", AudioFormat.class);+ setFormat.invoke(audioMixBuilder, AudioConfig.createAudioFormat());++ int routeFlags = audioMixClass.getField("ROUTE_FLAG_LOOP_BACK").getInt(null);++ // audioMixBuilder.setRouteFlags(routeFlag);+ Method setRouteFlags = audioMixBuilder.getClass().getMethod("setRouteFlags", int.class);+ setRouteFlags.invoke(audioMixBuilder, routeFlags);++ // AudioMix audioMix = audioMixBuilder.build();+ Object audioMix = audioMixBuilderClass.getMethod("build").invoke(audioMixBuilder);++ Class<?> audioPolicyClass = Class.forName("android.media.audiopolicy.AudioPolicy");+ Class<?> audioPolicyBuilderClass = Class.forName("android.media.audiopolicy.AudioPolicy$Builder");++ // AudioPolicy.Builder audioPolicyBuilder = new AudioPolicy.Builder();+ Object audioPolicyBuilder = audioPolicyBuilderClass.getConstructor(Context.class).newInstance(FakeContext.get());++ // audioPolicyBuilder.addMix(audioMix);+ Method addMixMethod = audioPolicyBuilderClass.getMethod("addMix", audioMixClass);+ addMixMethod.invoke(audioPolicyBuilder, audioMix);++ // AudioPolicy audioPolicy = audioPolicyBuilder.build();+ Object audioPolicy = audioPolicyBuilderClass.getMethod("build").invoke(audioPolicyBuilder);++ // AudioManager.registerAudioPolicyStatic(audioPolicy);+ Method registerAudioPolicyStaticMethod = AudioManager.class.getDeclaredMethod("registerAudioPolicyStatic", audioPolicyClass);+ registerAudioPolicyStaticMethod.setAccessible(true);+ int result = (int) registerAudioPolicyStaticMethod.invoke(null, audioPolicy);+ if (result != 0) {+ throw new RuntimeException("registerAudioPolicy() returned " + result);+ }++ // audioPolicy.createAudioRecordSink(audioPolicy);+ Method createAudioRecordSinkClass = audioPolicyClass.getMethod("createAudioRecordSink", audioMixClass);+ return (AudioRecord) createAudioRecordSinkClass.invoke(audioPolicy, audioMix);+ } catch (Exception e) {+ Ln.e("Could not capture audio playback", e);+ throw new AudioCaptureException();+ }+ }++ @Override+ public void checkCompatibility() throws AudioCaptureException {+ if (Build.VERSION.SDK_INT < Build.VERSION_CODES.TIRAMISU) {+ Ln.w("Audio disabled: audio playback capture source not supported before Android 13");+ throw new AudioCaptureException();+ }+ }++ @Override+ public void start() throws AudioCaptureException {+ recorder = createAudioRecord();+ recorder.startRecording();+ reader = new AudioRecordReader(recorder);+ }++ @Override+ public void stop() {+ if (recorder != null) {+ // Will call .stop() if necessary, without throwing an IllegalStateException+ recorder.release();+ }+ }++ @Override+ @TargetApi(Build.VERSION_CODES.N)+ public int read(ByteBuffer outDirectBuffer, MediaCodec.BufferInfo outBufferInfo) {+ return reader.read(outDirectBuffer, outBufferInfo);+ }+}diff --git a/server/src/main/java/com/genymobile/scrcpy/audio/AudioSource.java b/server/src/main/java/com/genymobile/scrcpy/audio/AudioSource.javaindex 7201dd39..6082f20e 100644--- a/server/src/main/java/com/genymobile/scrcpy/audio/AudioSource.java+++ b/server/src/main/java/com/genymobile/scrcpy/audio/AudioSource.java@@ -2,7 +2,8 @@ package com.genymobile.scrcpy.audio;public enum AudioSource {OUTPUT("output"),- MIC("mic");+ MIC("mic"),+ PLAYBACK("playback");private final String name;@@ -10,6 +11,10 @@ public enum AudioSource {this.name = name;}+ public boolean isDirect() {+ return this != PLAYBACK;+ }+public static AudioSource findByName(String name) {for (AudioSource audioSource : AudioSource.values()) {if (name.equals(audioSource.name)) {