Skip to content

Display and control your Android device

Encode video using MediaCodec API

Romain Vimont865ebb386239
diff --git a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java
index 6e71c63a..e739abe6 100644
--- a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java
+++ b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java
@@ -65,8 +65,8 @@ public class DesktopConnection implements Closeable {
outputStream.write(buffer, 0, buffer.length);
}
- public void sendVideoStream(byte[] videoStreamBuffer, int len) throws IOException {
- outputStream.write(videoStreamBuffer, 0, len);
+ public OutputStream getOutputStream() {
+ return outputStream;
}
public ControlEvent receiveControlEvent() throws IOException {
diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.java
index 4348127e..542f2035 100644
--- a/server/src/main/java/com/genymobile/scrcpy/Device.java
+++ b/server/src/main/java/com/genymobile/scrcpy/Device.java
@@ -45,14 +45,12 @@ public final class Device {
// Principle:
// - scale down the great side of the screen to maximumSize (if necessary);
// - scale down the other side so that the aspect ratio is preserved;
- // - ceil this value to the next multiple of 8 (H.264 only accepts multiples of 8)
- // - this may introduce black bands, so store the padding (typically a few pixels)
+ // - round this value to the nearest multiple of 8 (H.264 only accepts multiples of 8)
DisplayInfo displayInfo = serviceManager.getDisplayManager().getDisplayInfo();
boolean rotated = (displayInfo.getRotation() & 1) != 0;
Size deviceSize = displayInfo.getSize();
int w = deviceSize.getWidth();
int h = deviceSize.getHeight();
- int padding = 0;
if (maximumSize > 0) {
assert maximumSize % 8 == 0;
boolean portrait = h > w;
@@ -60,16 +58,15 @@ public final class Device {
int minor = portrait ? w : h;
if (major > maximumSize) {
int minorExact = minor * maximumSize / major;
- // +7 to ceil the value on rounding to the next multiple of 8,
- // so that any necessary black bands to keep the aspect ratio are added to the smallest dimension
- minor = (minorExact + 7) & ~7;
+ // +4 to round the value to the nearest multiple of 8
+ minor = (minorExact + 4) & ~7;
major = maximumSize;
- padding = minor - minorExact;
}
w = portrait ? minor : major;
h = portrait ? major : minor;
}
- return new ScreenInfo(deviceSize, new Size(w, h), padding, rotated);
+ Size videoSize = new Size(w, h);
+ return new ScreenInfo(deviceSize, videoSize, rotated);
}
public Point getPhysicalPoint(Position position) {
@@ -82,19 +79,9 @@ public final class Device {
return null;
}
Size deviceSize = screenInfo.getDeviceSize();
- int xPadding = screenInfo.getXPadding();
- int yPadding = screenInfo.getYPadding();
- int contentWidth = videoSize.getWidth() - xPadding;
- int contentHeight = videoSize.getHeight() - yPadding;
Point point = position.getPoint();
- int x = point.x - xPadding / 2;
- int y = point.y - yPadding / 2;
- if (x < 0 || x >= contentWidth || y < 0 || y >= contentHeight) {
- // out of screen
- return null;
- }
- int scaledX = x * deviceSize.getWidth() / videoSize.getWidth();
- int scaledY = y * deviceSize.getHeight() / videoSize.getHeight();
+ int scaledX = point.x * deviceSize.getWidth() / videoSize.getWidth();
+ int scaledY = point.y * deviceSize.getHeight() / videoSize.getHeight();
return new Point(scaledX, scaledY);
}
diff --git a/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java b/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java
index 441fbf8c..bf84f31b 100644
--- a/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java
+++ b/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java
@@ -4,25 +4,17 @@ import java.io.IOException;
public class ScrCpyServer {
- private static final String TAG = "scrcpy";
-
private static void scrcpy(Options options) throws IOException {
final Device device = new Device(options);
try (DesktopConnection connection = DesktopConnection.open(device)) {
- final ScreenStreamer streamer = new ScreenStreamer(device, connection);
- device.setRotationListener(new Device.RotationListener() {
- @Override
- public void onRotationChanged(int rotation) {
- streamer.reset();
- }
- });
+ ScreenEncoder screenEncoder = new ScreenEncoder();
// asynchronous
startEventController(device, connection);
try {
// synchronous
- streamer.streamScreen();
+ screenEncoder.streamScreen(device, connection.getOutputStream());
} catch (IOException e) {
Ln.e("Screen streaming interrupted", e);
}
diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java
new file mode 100644
index 00000000..2f33e4e3
--- /dev/null
+++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java
@@ -0,0 +1,149 @@
+package com.genymobile.scrcpy;
+
+import android.graphics.Rect;
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import android.media.MediaFormat;
+import android.os.IBinder;
+import android.view.Surface;
+
+import com.genymobile.scrcpy.wrappers.SurfaceControl;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ScreenEncoder implements Device.RotationListener {
+
+ private static final int DEFAULT_BIT_RATE = 4_000_000; // bits per second
+ private static final int DEFAULT_FRAME_RATE = 60; // fps
+ private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds
+
+ private static final int REPEAT_FRAME_DELAY = 6; // repeat after 6 frames
+
+ private final AtomicBoolean rotationChanged = new AtomicBoolean();
+
+ private int bitRate;
+ private int frameRate;
+ private int iFrameInterval;
+
+ public ScreenEncoder(int bitRate, int frameRate, int iFrameInterval) {
+ this.bitRate = bitRate;
+ this.frameRate = frameRate;
+ this.iFrameInterval = iFrameInterval;
+ }
+
+ public ScreenEncoder() {
+ this(DEFAULT_BIT_RATE, DEFAULT_FRAME_RATE, DEFAULT_I_FRAME_INTERVAL);
+ }
+
+ @Override
+ public void onRotationChanged(int rotation) {
+ rotationChanged.set(true);
+ }
+
+ public boolean checkRotationChanged() {
+ return rotationChanged.getAndSet(false);
+ }
+
+ public void streamScreen(Device device, OutputStream outputStream) throws IOException {
+ MediaFormat format = createFormat(bitRate, frameRate, iFrameInterval);
+ MediaCodec codec = createCodec();
+ IBinder display = createDisplay();
+ device.setRotationListener(this);
+ boolean alive;
+ try {
+ do {
+ Rect deviceRect = device.getScreenInfo().getDeviceSize().toRect();
+ Rect videoRect = device.getScreenInfo().getVideoSize().toRect();
+ setSize(format, videoRect.width(), videoRect.height());
+ configure(codec, format);
+ Surface surface = codec.createInputSurface();
+ setDisplaySurface(display, surface, deviceRect, videoRect);
+ codec.start();
+ try {
+ alive = encode(codec, outputStream);
+ } finally {
+ codec.stop();
+ surface.release();
+ }
+ } while (alive);
+ } finally {
+ device.setRotationListener(null);
+ destroyDisplay(display);
+ codec.release();
+ }
+ }
+
+ private boolean encode(MediaCodec codec, OutputStream outputStream) throws IOException {
+ byte[] buf = new byte[bitRate / 8]; // may contain up to 1 second of video
+ boolean eof = false;
+ MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
+ while (!checkRotationChanged() && !eof) {
+ int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
+ if (checkRotationChanged()) {
+ // must restart encoding with new size
+ break;
+ }
+ if (outputBufferId >= 0) {
+ ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
+ while (outputBuffer.hasRemaining()) {
+ int remaining = outputBuffer.remaining();
+ int len = Math.min(buf.length, remaining);
+ // the outputBuffer is probably direct (it has no underlying array), and LocalSocket does not expose channels,
+ // so we must copy the data locally to write them manually to the output stream
+ outputBuffer.get(buf, 0, len);
+ outputStream.write(buf, 0, len);
+ }
+ codec.releaseOutputBuffer(outputBufferId, false);
+ }
+ }
+
+ return !eof;
+ }
+
+ private static MediaCodec createCodec() throws IOException {
+ return MediaCodec.createEncoderByType("video/avc");
+ }
+
+ private static MediaFormat createFormat(int bitRate, int frameRate, int iFrameInterval) throws IOException {
+ MediaFormat format = new MediaFormat();
+ format.setString(MediaFormat.KEY_MIME, "video/avc");
+ format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
+ format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);
+ format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
+ format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval);
+ // display the very first frame, and recover from bad quality when no new frames
+ format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 1_000_000 * REPEAT_FRAME_DELAY / frameRate); // µs
+ return format;
+ }
+
+ private static IBinder createDisplay() {
+ return SurfaceControl.createDisplay("scrcpy", false);
+ }
+
+ private static void configure(MediaCodec codec, MediaFormat format) {
+ codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+ }
+
+ private static void setSize(MediaFormat format, int width, int height) {
+ format.setInteger(MediaFormat.KEY_WIDTH, width);
+ format.setInteger(MediaFormat.KEY_HEIGHT, height);
+ }
+
+ private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect) {
+ SurfaceControl.openTransaction();
+ try {
+ SurfaceControl.setDisplaySurface(display, surface);
+ SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect);
+ SurfaceControl.setDisplayLayerStack(display, 0);
+ } finally {
+ SurfaceControl.closeTransaction();
+ }
+ }
+
+ private static void destroyDisplay(IBinder display) {
+ SurfaceControl.destroyDisplay(display);
+ }
+}
diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java b/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java
index 827e9ee8..1f752758 100644
--- a/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java
+++ b/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java
@@ -3,13 +3,11 @@ package com.genymobile.scrcpy;
public final class ScreenInfo {
private final Size deviceSize;
private final Size videoSize;
- private final int padding; // padding inside the video stream, along the smallest dimension
private final boolean rotated;
- public ScreenInfo(Size deviceSize, Size videoSize, int padding, boolean rotated) {
+ public ScreenInfo(Size deviceSize, Size videoSize, boolean rotated) {
this.deviceSize = deviceSize;
this.videoSize = videoSize;
- this.padding = padding;
this.rotated = rotated;
}
@@ -21,19 +19,11 @@ public final class ScreenInfo {
return videoSize;
}
- public int getXPadding() {
- return videoSize.getWidth() < videoSize.getHeight() ? padding : 0;
- }
-
- public int getYPadding() {
- return videoSize.getHeight() < videoSize.getWidth() ? padding : 0;
- }
-
public ScreenInfo withRotation(int rotation) {
boolean newRotated = (rotation & 1) != 0;
if (rotated == newRotated) {
return this;
}
- return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), padding, newRotated);
+ return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), newRotated);
}
}
diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java b/server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java
deleted file mode 100644
index 38509105..00000000
--- a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package com.genymobile.scrcpy;
-
-import java.io.IOException;
-import java.io.InterruptedIOException;
-
-public class ScreenStreamer {
-
- private final Device device;
- private final DesktopConnection connection;
- private ScreenStreamerSession currentStreamer; // protected by 'this'
-
- public ScreenStreamer(Device device, DesktopConnection connection) {
- this.device = device;
- this.connection = connection;
- }
-
- private synchronized ScreenStreamerSession newScreenStreamerSession() {
- currentStreamer = new ScreenStreamerSession(device, connection);
- return currentStreamer;
- }
-
- public void streamScreen() throws IOException {
- while (true) {
- try {
- ScreenStreamerSession screenStreamer = newScreenStreamerSession();
- screenStreamer.streamScreen();
- } catch (InterruptedIOException e) {
- // the current screenrecord process has probably been killed due to reset(), start a new one without failing
- }
- }
- }
-
- public synchronized void reset() {
- if (currentStreamer != null) {
- // it will stop the blocking call to streamScreen(), so a new streamer will be started
- currentStreamer.stop();
- }
- }
-}
diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java b/server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java
deleted file mode 100644
index cb3c7bb1..00000000
--- a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java
+++ /dev/null
@@ -1,73 +0,0 @@
-package com.genymobile.scrcpy;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-public class ScreenStreamerSession {
-
- private final Device device;
- private final DesktopConnection connection;
- private Process screenRecordProcess; // protected by 'this'
- private final AtomicBoolean stopped = new AtomicBoolean();
- private final byte[] buffer = new byte[0x10000];
-
- public ScreenStreamerSession(Device device, DesktopConnection connection) {
- this.device = device;
- this.connection = connection;
- }
-
- public void streamScreen() throws IOException {
- // screenrecord may not record more than 3 minutes, so restart it on EOF
- while (!stopped.get() && streamScreenOnce()) ;
- }
-
- /**
- * Starts screenrecord once and relay its output to the desktop connection.
- *
- * @return {@code true} if EOF is reached, {@code false} otherwise (i.e. requested to stop).
- * @throws IOException if an I/O error occurred
- */
- private boolean streamScreenOnce() throws IOException {
- Ln.d("Recording...");
- Size videoSize = device.getScreenInfo().getVideoSize();
- Process process = startScreenRecord(videoSize);
- setCurrentProcess(process);
- InputStream inputStream = process.getInputStream();
- int r;
- while ((r = inputStream.read(buffer)) != -1 && !stopped.get()) {
- connection.sendVideoStream(buffer, r);
- }
- return r != -1;
- }
-
- public void stop() {
- // let the thread stop itself without breaking the video stream
- stopped.set(true);
- killCurrentProcess();
- }
-
- private static Process startScreenRecord(Size videoSize) throws IOException {
- List<String> command = new ArrayList<>();
- command.add("screenrecord");
- command.add("--output-format=h264");
- command.add("--size=" + videoSize.getWidth() + "x" + videoSize.getHeight());
- command.add("-");
- Process process = new ProcessBuilder(command).start();
- process.getOutputStream().close();
- return process;
- }
-
- private synchronized void setCurrentProcess(Process screenRecordProcess) {
- this.screenRecordProcess = screenRecordProcess;
- }
-
- private synchronized void killCurrentProcess() {
- if (screenRecordProcess != null) {
- screenRecordProcess.destroy();
- screenRecordProcess = null;
- }
- }
-}
diff --git a/server/src/main/java/com/genymobile/scrcpy/Size.java b/server/src/main/java/com/genymobile/scrcpy/Size.java
index 66d67b37..96026d87 100644
--- a/server/src/main/java/com/genymobile/scrcpy/Size.java
+++ b/server/src/main/java/com/genymobile/scrcpy/Size.java
@@ -1,5 +1,7 @@
package com.genymobile.scrcpy;
+import android.graphics.Rect;
+
import java.util.Objects;
public final class Size {
@@ -23,6 +25,10 @@ public final class Size {
return new Size(height, width);
}
+ public Rect toRect() {
+ return new Rect(0, 0, width, height);
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java
new file mode 100644
index 00000000..35efbc27
--- /dev/null
+++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java
@@ -0,0 +1,79 @@
+package com.genymobile.scrcpy.wrappers;
+
+import android.graphics.Rect;
+import android.os.IBinder;
+import android.view.Surface;
+
+public class SurfaceControl {
+
+ private static final Class<?> cls;
+
+ static {
+ try {
+ cls = Class.forName("android.view.SurfaceControl");
+ } catch (ClassNotFoundException e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ private SurfaceControl() {
+ // only static methods
+ }
+
+ public static void openTransaction() {
+ try {
+ cls.getMethod("openTransaction").invoke(null);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ public static void closeTransaction() {
+ try {
+ cls.getMethod("closeTransaction").invoke(null);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ public static void setDisplayProjection(IBinder displayToken, int orientation, Rect layerStackRect, Rect displayRect) {
+ try {
+ cls.getMethod("setDisplayProjection", IBinder.class, int.class, Rect.class, Rect.class)
+ .invoke(null, displayToken, orientation, layerStackRect, displayRect);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ public static void setDisplayLayerStack(IBinder displayToken, int layerStack) {
+ try {
+ cls.getMethod("setDisplayLayerStack", IBinder.class, int.class).invoke(null, displayToken, layerStack);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ public static void setDisplaySurface(IBinder displayToken, Surface surface) {
+ try {
+ cls.getMethod("setDisplaySurface", IBinder.class, Surface.class).invoke(null, displayToken, surface);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ public static IBinder createDisplay(String name, boolean secure) {
+ try {
+ return (IBinder) cls.getMethod("createDisplay", String.class, boolean.class).invoke(null, name, secure);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+
+ public static void destroyDisplay(IBinder displayToken) {
+ try {
+ cls.getMethod("destroyDisplay", IBinder.class).invoke(null, displayToken);
+ } catch (Exception e) {
+ throw new AssertionError(e);
+ }
+ }
+}