Skip to content

Display and control your Android device

Extract SurfaceCapture from ScreenEncoder

Simon Chana2fb1b40f690
diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java
new file mode 100644
index 00000000..f9ac66b8
--- /dev/null
+++ b/server/src/main/java/com/genymobile/scrcpy/ScreenCapture.java
@@ -0,0 +1,83 @@
+package com.genymobile.scrcpy;
+
+import com.genymobile.scrcpy.wrappers.SurfaceControl;
+
+import android.graphics.Rect;
+import android.os.Build;
+import android.os.IBinder;
+import android.view.Surface;
+
+public class ScreenCapture extends SurfaceCapture implements Device.RotationListener, Device.FoldListener {
+
+ private final Device device;
+ private IBinder display;
+
+ public ScreenCapture(Device device) {
+ this.device = device;
+ }
+
+ @Override
+ public void init() {
+ display = createDisplay();
+ device.setRotationListener(this);
+ device.setFoldListener(this);
+ }
+
+ @Override
+ public void start(Surface surface) {
+ ScreenInfo screenInfo = device.getScreenInfo();
+ Rect contentRect = screenInfo.getContentRect();
+
+ // does not include the locked video orientation
+ Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
+ int videoRotation = screenInfo.getVideoRotation();
+ int layerStack = device.getLayerStack();
+ setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
+ }
+
+ @Override
+ public void release() {
+ device.setRotationListener(null);
+ device.setFoldListener(null);
+ SurfaceControl.destroyDisplay(display);
+ }
+
+ @Override
+ public Size getSize() {
+ return device.getScreenInfo().getVideoSize();
+ }
+
+ @Override
+ public void setMaxSize(int maxSize) {
+ device.setMaxSize(maxSize);
+ }
+
+ @Override
+ public void onFoldChanged(int displayId, boolean folded) {
+ requestReset();
+ }
+
+ @Override
+ public void onRotationChanged(int rotation) {
+ requestReset();
+ }
+
+ private static IBinder createDisplay() {
+ // Since Android 12 (preview), secure displays could not be created with shell permissions anymore.
+ // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S".
+ boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S".equals(
+ Build.VERSION.CODENAME));
+ return SurfaceControl.createDisplay("scrcpy", secure);
+ }
+
+ private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) {
+ SurfaceControl.openTransaction();
+ try {
+ SurfaceControl.setDisplaySurface(display, surface);
+ SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
+ SurfaceControl.setDisplayLayerStack(display, layerStack);
+ } finally {
+ SurfaceControl.closeTransaction();
+ }
+ }
+}
diff --git a/server/src/main/java/com/genymobile/scrcpy/Server.java b/server/src/main/java/com/genymobile/scrcpy/Server.java
index dc85c965..0f73a19b 100644
--- a/server/src/main/java/com/genymobile/scrcpy/Server.java
+++ b/server/src/main/java/com/genymobile/scrcpy/Server.java
@@ -132,7 +132,8 @@ public final class Server {
if (video) {
Streamer videoStreamer = new Streamer(connection.getVideoFd(), options.getVideoCodec(), options.getSendCodecMeta(),
options.getSendFrameMeta());
- ScreenEncoder screenEncoder = new ScreenEncoder(device, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
+ ScreenCapture screenCapture = new ScreenCapture(device);
+ SurfaceEncoder screenEncoder = new SurfaceEncoder(screenCapture, videoStreamer, options.getVideoBitRate(), options.getMaxFps(),
options.getVideoCodecOptions(), options.getVideoEncoder(), options.getDownsizeOnError());
asyncProcessors.add(screenEncoder);
}
diff --git a/server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java b/server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java
new file mode 100644
index 00000000..45a0fd2f
--- /dev/null
+++ b/server/src/main/java/com/genymobile/scrcpy/SurfaceCapture.java
@@ -0,0 +1,61 @@
+package com.genymobile.scrcpy;
+
+import android.view.Surface;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A video source which can be rendered on a Surface for encoding.
+ */
+public abstract class SurfaceCapture {
+
+ private final AtomicBoolean resetCapture = new AtomicBoolean();
+
+ /**
+ * Request the encoding session to be restarted, for example if the capture implementation detects that the video source size has changed (on
+ * device rotation for example).
+ */
+ protected void requestReset() {
+ resetCapture.set(true);
+ }
+
+ /**
+ * Consume the reset request (intended to be called by the encoder).
+ *
+ * @return {@code true} if a reset request was pending, {@code false} otherwise.
+ */
+ public boolean consumeReset() {
+ return resetCapture.getAndSet(false);
+ }
+
+ /**
+ * Called once before the capture starts.
+ */
+ public abstract void init();
+
+ /**
+ * Called after the capture ends (if and only if {@link #init()} has been called).
+ */
+ public abstract void release();
+
+ /**
+ * Start the capture to the target surface.
+ *
+ * @param surface the surface which will be encoded
+ */
+ public abstract void start(Surface surface);
+
+ /**
+ * Return the video size
+ *
+ * @return the video size
+ */
+ public abstract Size getSize();
+
+ /**
+ * Set the maximum capture size (set by the encoder if it does not support the current size).
+ *
+ * @param maxSize Maximum size
+ */
+ public abstract void setMaxSize(int maxSize);
+}
diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java
similarity index 74%
rename from server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java
rename to server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java
index 5a9db10d..4af31e89 100644
--- a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java
+++ b/server/src/main/java/com/genymobile/scrcpy/SurfaceEncoder.java
@@ -1,13 +1,8 @@
package com.genymobile.scrcpy;
-import com.genymobile.scrcpy.wrappers.SurfaceControl;
-
-import android.graphics.Rect;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
-import android.os.Build;
-import android.os.IBinder;
import android.os.Looper;
import android.os.SystemClock;
import android.view.Surface;
@@ -17,7 +12,7 @@ import java.nio.ByteBuffer;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
-public class ScreenEncoder implements Device.RotationListener, Device.FoldListener, AsyncProcessor {
+public class SurfaceEncoder implements AsyncProcessor {
private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds
private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms
@@ -27,9 +22,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
private static final int[] MAX_SIZE_FALLBACK = {2560, 1920, 1600, 1280, 1024, 800};
private static final int MAX_CONSECUTIVE_ERRORS = 3;
- private final AtomicBoolean resetCapture = new AtomicBoolean();
-
- private final Device device;
+ private final SurfaceCapture capture;
private final Streamer streamer;
private final String encoderName;
private final List<CodecOption> codecOptions;
@@ -43,9 +36,9 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
private Thread thread;
private final AtomicBoolean stopped = new AtomicBoolean();
- public ScreenEncoder(Device device, Streamer streamer, int videoBitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
+ public SurfaceEncoder(SurfaceCapture capture, Streamer streamer, int videoBitRate, int maxFps, List<CodecOption> codecOptions, String encoderName,
boolean downsizeOnError) {
- this.device = device;
+ this.capture = capture;
this.streamer = streamer;
this.videoBitRate = videoBitRate;
this.maxFps = maxFps;
@@ -54,51 +47,29 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
this.downsizeOnError = downsizeOnError;
}
- @Override
- public void onFoldChanged(int displayId, boolean folded) {
- resetCapture.set(true);
- }
-
- @Override
- public void onRotationChanged(int rotation) {
- resetCapture.set(true);
- }
-
- private boolean consumeResetCapture() {
- return resetCapture.getAndSet(false);
- }
-
private void streamScreen() throws IOException, ConfigurationException {
Codec codec = streamer.getCodec();
MediaCodec mediaCodec = createMediaCodec(codec, encoderName);
MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions);
- IBinder display = createDisplay();
- device.setRotationListener(this);
- device.setFoldListener(this);
- streamer.writeVideoHeader(device.getScreenInfo().getVideoSize());
+ capture.init();
- boolean alive;
try {
- do {
- ScreenInfo screenInfo = device.getScreenInfo();
- Rect contentRect = screenInfo.getContentRect();
+ streamer.writeVideoHeader(capture.getSize());
+
+ boolean alive;
- // include the locked video orientation
- Rect videoRect = screenInfo.getVideoSize().toRect();
- format.setInteger(MediaFormat.KEY_WIDTH, videoRect.width());
- format.setInteger(MediaFormat.KEY_HEIGHT, videoRect.height());
+ do {
+ Size size = capture.getSize();
+ format.setInteger(MediaFormat.KEY_WIDTH, size.getWidth());
+ format.setInteger(MediaFormat.KEY_HEIGHT, size.getHeight());
Surface surface = null;
try {
mediaCodec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
surface = mediaCodec.createInputSurface();
- // does not include the locked video orientation
- Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
- int videoRotation = screenInfo.getVideoRotation();
- int layerStack = device.getLayerStack();
- setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
+ capture.start(surface);
mediaCodec.start();
@@ -107,7 +78,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
mediaCodec.stop();
} catch (IllegalStateException | IllegalArgumentException e) {
Ln.e("Encoding error: " + e.getClass().getName() + ": " + e.getMessage());
- if (!prepareRetry(device, screenInfo)) {
+ if (!prepareRetry(size)) {
throw e;
}
Ln.i("Retrying...");
@@ -121,13 +92,11 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
} while (alive);
} finally {
mediaCodec.release();
- device.setRotationListener(null);
- device.setFoldListener(null);
- SurfaceControl.destroyDisplay(display);
+ capture.release();
}
}
- private boolean prepareRetry(Device device, ScreenInfo screenInfo) {
+ private boolean prepareRetry(Size currentSize) {
if (firstFrameSent) {
++consecutiveErrors;
if (consecutiveErrors >= MAX_CONSECUTIVE_ERRORS) {
@@ -147,7 +116,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
// Downsizing on error is only enabled if an encoding failure occurs before the first frame (downsizing later could be surprising)
- int newMaxSize = chooseMaxSizeFallback(screenInfo.getVideoSize());
+ int newMaxSize = chooseMaxSizeFallback(currentSize);
if (newMaxSize == 0) {
// Must definitively fail
return false;
@@ -155,7 +124,7 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
// Retry with a smaller device size
Ln.i("Retrying with -m" + newMaxSize + "...");
- device.setMaxSize(newMaxSize);
+ capture.setMaxSize(newMaxSize);
return true;
}
@@ -176,14 +145,14 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
boolean alive = true;
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
- while (!consumeResetCapture() && !eof) {
+ while (!capture.consumeReset() && !eof) {
if (stopped.get()) {
alive = false;
break;
}
int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
try {
- if (consumeResetCapture()) {
+ if (capture.consumeReset()) {
// must restart encoding with new size
break;
}
@@ -264,25 +233,6 @@ public class ScreenEncoder implements Device.RotationListener, Device.FoldListen
return format;
}
- private static IBinder createDisplay() {
- // Since Android 12 (preview), secure displays could not be created with shell permissions anymore.
- // On Android 12 preview, SDK_INT is still R (not S), but CODENAME is "S".
- boolean secure = Build.VERSION.SDK_INT < Build.VERSION_CODES.R || (Build.VERSION.SDK_INT == Build.VERSION_CODES.R && !"S"
- .equals(Build.VERSION.CODENAME));
- return SurfaceControl.createDisplay("scrcpy", secure);
- }
-
- private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) {
- SurfaceControl.openTransaction();
- try {
- SurfaceControl.setDisplaySurface(display, surface);
- SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
- SurfaceControl.setDisplayLayerStack(display, layerStack);
- } finally {
- SurfaceControl.closeTransaction();
- }
- }
-
@Override
public void start(TerminationListener listener) {
thread = new Thread(() -> {