Display and control your Android device
Encode video using MediaCodec API
Romain Vimont865ebb386239
diff --git a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.javaindex 6e71c63a..e739abe6 100644--- a/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java+++ b/server/src/main/java/com/genymobile/scrcpy/DesktopConnection.java@@ -65,8 +65,8 @@ public class DesktopConnection implements Closeable {outputStream.write(buffer, 0, buffer.length);}- public void sendVideoStream(byte[] videoStreamBuffer, int len) throws IOException {- outputStream.write(videoStreamBuffer, 0, len);+ public OutputStream getOutputStream() {+ return outputStream;}public ControlEvent receiveControlEvent() throws IOException {diff --git a/server/src/main/java/com/genymobile/scrcpy/Device.java b/server/src/main/java/com/genymobile/scrcpy/Device.javaindex 4348127e..542f2035 100644--- a/server/src/main/java/com/genymobile/scrcpy/Device.java+++ b/server/src/main/java/com/genymobile/scrcpy/Device.java@@ -45,14 +45,12 @@ public final class Device {// Principle:// - scale down the great side of the screen to maximumSize (if necessary);// - scale down the other side so that the aspect ratio is preserved;- // - ceil this value to the next multiple of 8 (H.264 only accepts multiples of 8)- // - this may introduce black bands, so store the padding (typically a few pixels)+ // - round this value to the nearest multiple of 8 (H.264 only accepts multiples of 8)DisplayInfo displayInfo = serviceManager.getDisplayManager().getDisplayInfo();boolean rotated = (displayInfo.getRotation() & 1) != 0;Size deviceSize = displayInfo.getSize();int w = deviceSize.getWidth();int h = deviceSize.getHeight();- int padding = 0;if (maximumSize > 0) {assert maximumSize % 8 == 0;boolean portrait = h > w;@@ -60,16 +58,15 @@ public final class Device {int minor = portrait ? w : h;if (major > maximumSize) {int minorExact = minor * maximumSize / major;- // +7 to ceil the value on rounding to the next multiple of 8,- // so that any necessary black bands to keep the aspect ratio are added to the smallest dimension- minor = (minorExact + 7) & ~7;+ // +4 to round the value to the nearest multiple of 8+ minor = (minorExact + 4) & ~7;major = maximumSize;- padding = minor - minorExact;}w = portrait ? minor : major;h = portrait ? major : minor;}- return new ScreenInfo(deviceSize, new Size(w, h), padding, rotated);+ Size videoSize = new Size(w, h);+ return new ScreenInfo(deviceSize, videoSize, rotated);}public Point getPhysicalPoint(Position position) {@@ -82,19 +79,9 @@ public final class Device {return null;}Size deviceSize = screenInfo.getDeviceSize();- int xPadding = screenInfo.getXPadding();- int yPadding = screenInfo.getYPadding();- int contentWidth = videoSize.getWidth() - xPadding;- int contentHeight = videoSize.getHeight() - yPadding;Point point = position.getPoint();- int x = point.x - xPadding / 2;- int y = point.y - yPadding / 2;- if (x < 0 || x >= contentWidth || y < 0 || y >= contentHeight) {- // out of screen- return null;- }- int scaledX = x * deviceSize.getWidth() / videoSize.getWidth();- int scaledY = y * deviceSize.getHeight() / videoSize.getHeight();+ int scaledX = point.x * deviceSize.getWidth() / videoSize.getWidth();+ int scaledY = point.y * deviceSize.getHeight() / videoSize.getHeight();return new Point(scaledX, scaledY);}diff --git a/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java b/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.javaindex 441fbf8c..bf84f31b 100644--- a/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java+++ b/server/src/main/java/com/genymobile/scrcpy/ScrCpyServer.java@@ -4,25 +4,17 @@ import java.io.IOException;public class ScrCpyServer {- private static final String TAG = "scrcpy";-private static void scrcpy(Options options) throws IOException {final Device device = new Device(options);try (DesktopConnection connection = DesktopConnection.open(device)) {- final ScreenStreamer streamer = new ScreenStreamer(device, connection);- device.setRotationListener(new Device.RotationListener() {- @Override- public void onRotationChanged(int rotation) {- streamer.reset();- }- });+ ScreenEncoder screenEncoder = new ScreenEncoder();// asynchronousstartEventController(device, connection);try {// synchronous- streamer.streamScreen();+ screenEncoder.streamScreen(device, connection.getOutputStream());} catch (IOException e) {Ln.e("Screen streaming interrupted", e);}diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.javanew file mode 100644index 00000000..2f33e4e3--- /dev/null+++ b/server/src/main/java/com/genymobile/scrcpy/ScreenEncoder.java@@ -0,0 +1,149 @@+package com.genymobile.scrcpy;++import android.graphics.Rect;+import android.media.MediaCodec;+import android.media.MediaCodecInfo;+import android.media.MediaFormat;+import android.os.IBinder;+import android.view.Surface;++import com.genymobile.scrcpy.wrappers.SurfaceControl;++import java.io.IOException;+import java.io.OutputStream;+import java.nio.ByteBuffer;+import java.util.concurrent.atomic.AtomicBoolean;++public class ScreenEncoder implements Device.RotationListener {++ private static final int DEFAULT_BIT_RATE = 4_000_000; // bits per second+ private static final int DEFAULT_FRAME_RATE = 60; // fps+ private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds++ private static final int REPEAT_FRAME_DELAY = 6; // repeat after 6 frames++ private final AtomicBoolean rotationChanged = new AtomicBoolean();++ private int bitRate;+ private int frameRate;+ private int iFrameInterval;++ public ScreenEncoder(int bitRate, int frameRate, int iFrameInterval) {+ this.bitRate = bitRate;+ this.frameRate = frameRate;+ this.iFrameInterval = iFrameInterval;+ }++ public ScreenEncoder() {+ this(DEFAULT_BIT_RATE, DEFAULT_FRAME_RATE, DEFAULT_I_FRAME_INTERVAL);+ }++ @Override+ public void onRotationChanged(int rotation) {+ rotationChanged.set(true);+ }++ public boolean checkRotationChanged() {+ return rotationChanged.getAndSet(false);+ }++ public void streamScreen(Device device, OutputStream outputStream) throws IOException {+ MediaFormat format = createFormat(bitRate, frameRate, iFrameInterval);+ MediaCodec codec = createCodec();+ IBinder display = createDisplay();+ device.setRotationListener(this);+ boolean alive;+ try {+ do {+ Rect deviceRect = device.getScreenInfo().getDeviceSize().toRect();+ Rect videoRect = device.getScreenInfo().getVideoSize().toRect();+ setSize(format, videoRect.width(), videoRect.height());+ configure(codec, format);+ Surface surface = codec.createInputSurface();+ setDisplaySurface(display, surface, deviceRect, videoRect);+ codec.start();+ try {+ alive = encode(codec, outputStream);+ } finally {+ codec.stop();+ surface.release();+ }+ } while (alive);+ } finally {+ device.setRotationListener(null);+ destroyDisplay(display);+ codec.release();+ }+ }++ private boolean encode(MediaCodec codec, OutputStream outputStream) throws IOException {+ byte[] buf = new byte[bitRate / 8]; // may contain up to 1 second of video+ boolean eof = false;+ MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();+ while (!checkRotationChanged() && !eof) {+ int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);+ if (checkRotationChanged()) {+ // must restart encoding with new size+ break;+ }+ if (outputBufferId >= 0) {+ ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);+ while (outputBuffer.hasRemaining()) {+ int remaining = outputBuffer.remaining();+ int len = Math.min(buf.length, remaining);+ // the outputBuffer is probably direct (it has no underlying array), and LocalSocket does not expose channels,+ // so we must copy the data locally to write them manually to the output stream+ outputBuffer.get(buf, 0, len);+ outputStream.write(buf, 0, len);+ }+ codec.releaseOutputBuffer(outputBufferId, false);+ }+ }++ return !eof;+ }++ private static MediaCodec createCodec() throws IOException {+ return MediaCodec.createEncoderByType("video/avc");+ }++ private static MediaFormat createFormat(int bitRate, int frameRate, int iFrameInterval) throws IOException {+ MediaFormat format = new MediaFormat();+ format.setString(MediaFormat.KEY_MIME, "video/avc");+ format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);+ format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);+ format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);+ format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval);+ // display the very first frame, and recover from bad quality when no new frames+ format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, 1_000_000 * REPEAT_FRAME_DELAY / frameRate); // µs+ return format;+ }++ private static IBinder createDisplay() {+ return SurfaceControl.createDisplay("scrcpy", false);+ }++ private static void configure(MediaCodec codec, MediaFormat format) {+ codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);+ }++ private static void setSize(MediaFormat format, int width, int height) {+ format.setInteger(MediaFormat.KEY_WIDTH, width);+ format.setInteger(MediaFormat.KEY_HEIGHT, height);+ }++ private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect) {+ SurfaceControl.openTransaction();+ try {+ SurfaceControl.setDisplaySurface(display, surface);+ SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect);+ SurfaceControl.setDisplayLayerStack(display, 0);+ } finally {+ SurfaceControl.closeTransaction();+ }+ }++ private static void destroyDisplay(IBinder display) {+ SurfaceControl.destroyDisplay(display);+ }+}diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java b/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.javaindex 827e9ee8..1f752758 100644--- a/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java+++ b/server/src/main/java/com/genymobile/scrcpy/ScreenInfo.java@@ -3,13 +3,11 @@ package com.genymobile.scrcpy;public final class ScreenInfo {private final Size deviceSize;private final Size videoSize;- private final int padding; // padding inside the video stream, along the smallest dimensionprivate final boolean rotated;- public ScreenInfo(Size deviceSize, Size videoSize, int padding, boolean rotated) {+ public ScreenInfo(Size deviceSize, Size videoSize, boolean rotated) {this.deviceSize = deviceSize;this.videoSize = videoSize;- this.padding = padding;this.rotated = rotated;}@@ -21,19 +19,11 @@ public final class ScreenInfo {return videoSize;}- public int getXPadding() {- return videoSize.getWidth() < videoSize.getHeight() ? padding : 0;- }-- public int getYPadding() {- return videoSize.getHeight() < videoSize.getWidth() ? padding : 0;- }-public ScreenInfo withRotation(int rotation) {boolean newRotated = (rotation & 1) != 0;if (rotated == newRotated) {return this;}- return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), padding, newRotated);+ return new ScreenInfo(deviceSize.rotate(), videoSize.rotate(), newRotated);}}diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java b/server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.javadeleted file mode 100644index 38509105..00000000--- a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamer.java+++ /dev/null@@ -1,39 +0,0 @@-package com.genymobile.scrcpy;--import java.io.IOException;-import java.io.InterruptedIOException;--public class ScreenStreamer {-- private final Device device;- private final DesktopConnection connection;- private ScreenStreamerSession currentStreamer; // protected by 'this'-- public ScreenStreamer(Device device, DesktopConnection connection) {- this.device = device;- this.connection = connection;- }-- private synchronized ScreenStreamerSession newScreenStreamerSession() {- currentStreamer = new ScreenStreamerSession(device, connection);- return currentStreamer;- }-- public void streamScreen() throws IOException {- while (true) {- try {- ScreenStreamerSession screenStreamer = newScreenStreamerSession();- screenStreamer.streamScreen();- } catch (InterruptedIOException e) {- // the current screenrecord process has probably been killed due to reset(), start a new one without failing- }- }- }-- public synchronized void reset() {- if (currentStreamer != null) {- // it will stop the blocking call to streamScreen(), so a new streamer will be started- currentStreamer.stop();- }- }-}diff --git a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java b/server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.javadeleted file mode 100644index cb3c7bb1..00000000--- a/server/src/main/java/com/genymobile/scrcpy/ScreenStreamerSession.java+++ /dev/null@@ -1,73 +0,0 @@-package com.genymobile.scrcpy;--import java.io.IOException;-import java.io.InputStream;-import java.util.ArrayList;-import java.util.List;-import java.util.concurrent.atomic.AtomicBoolean;--public class ScreenStreamerSession {-- private final Device device;- private final DesktopConnection connection;- private Process screenRecordProcess; // protected by 'this'- private final AtomicBoolean stopped = new AtomicBoolean();- private final byte[] buffer = new byte[0x10000];-- public ScreenStreamerSession(Device device, DesktopConnection connection) {- this.device = device;- this.connection = connection;- }-- public void streamScreen() throws IOException {- // screenrecord may not record more than 3 minutes, so restart it on EOF- while (!stopped.get() && streamScreenOnce()) ;- }-- /**- * Starts screenrecord once and relay its output to the desktop connection.- *- * @return {@code true} if EOF is reached, {@code false} otherwise (i.e. requested to stop).- * @throws IOException if an I/O error occurred- */- private boolean streamScreenOnce() throws IOException {- Ln.d("Recording...");- Size videoSize = device.getScreenInfo().getVideoSize();- Process process = startScreenRecord(videoSize);- setCurrentProcess(process);- InputStream inputStream = process.getInputStream();- int r;- while ((r = inputStream.read(buffer)) != -1 && !stopped.get()) {- connection.sendVideoStream(buffer, r);- }- return r != -1;- }-- public void stop() {- // let the thread stop itself without breaking the video stream- stopped.set(true);- killCurrentProcess();- }-- private static Process startScreenRecord(Size videoSize) throws IOException {- List<String> command = new ArrayList<>();- command.add("screenrecord");- command.add("--output-format=h264");- command.add("--size=" + videoSize.getWidth() + "x" + videoSize.getHeight());- command.add("-");- Process process = new ProcessBuilder(command).start();- process.getOutputStream().close();- return process;- }-- private synchronized void setCurrentProcess(Process screenRecordProcess) {- this.screenRecordProcess = screenRecordProcess;- }-- private synchronized void killCurrentProcess() {- if (screenRecordProcess != null) {- screenRecordProcess.destroy();- screenRecordProcess = null;- }- }-}diff --git a/server/src/main/java/com/genymobile/scrcpy/Size.java b/server/src/main/java/com/genymobile/scrcpy/Size.javaindex 66d67b37..96026d87 100644--- a/server/src/main/java/com/genymobile/scrcpy/Size.java+++ b/server/src/main/java/com/genymobile/scrcpy/Size.java@@ -1,5 +1,7 @@package com.genymobile.scrcpy;+import android.graphics.Rect;+import java.util.Objects;public final class Size {@@ -23,6 +25,10 @@ public final class Size {return new Size(height, width);}+ public Rect toRect() {+ return new Rect(0, 0, width, height);+ }+@Overridepublic boolean equals(Object o) {if (this == o) return true;diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.javanew file mode 100644index 00000000..35efbc27--- /dev/null+++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/SurfaceControl.java@@ -0,0 +1,79 @@+package com.genymobile.scrcpy.wrappers;++import android.graphics.Rect;+import android.os.IBinder;+import android.view.Surface;++public class SurfaceControl {++ private static final Class<?> cls;++ static {+ try {+ cls = Class.forName("android.view.SurfaceControl");+ } catch (ClassNotFoundException e) {+ throw new AssertionError(e);+ }+ }++ private SurfaceControl() {+ // only static methods+ }++ public static void openTransaction() {+ try {+ cls.getMethod("openTransaction").invoke(null);+ } catch (Exception e) {+ throw new AssertionError(e);+ }+ }++ public static void closeTransaction() {+ try {+ cls.getMethod("closeTransaction").invoke(null);+ } catch (Exception e) {+ throw new AssertionError(e);+ }+ }++ public static void setDisplayProjection(IBinder displayToken, int orientation, Rect layerStackRect, Rect displayRect) {+ try {+ cls.getMethod("setDisplayProjection", IBinder.class, int.class, Rect.class, Rect.class)+ .invoke(null, displayToken, orientation, layerStackRect, displayRect);+ } catch (Exception e) {+ throw new AssertionError(e);+ }+ }++ public static void setDisplayLayerStack(IBinder displayToken, int layerStack) {+ try {+ cls.getMethod("setDisplayLayerStack", IBinder.class, int.class).invoke(null, displayToken, layerStack);+ } catch (Exception e) {+ throw new AssertionError(e);+ }+ }++ public static void setDisplaySurface(IBinder displayToken, Surface surface) {+ try {+ cls.getMethod("setDisplaySurface", IBinder.class, Surface.class).invoke(null, displayToken, surface);+ } catch (Exception e) {+ throw new AssertionError(e);+ }+ }++ public static IBinder createDisplay(String name, boolean secure) {+ try {+ return (IBinder) cls.getMethod("createDisplay", String.class, boolean.class).invoke(null, name, secure);+ } catch (Exception e) {+ throw new AssertionError(e);+ }+ }++ public static void destroyDisplay(IBinder displayToken) {+ try {+ cls.getMethod("destroyDisplay", IBinder.class).invoke(null, displayToken);+ } catch (Exception e) {+ throw new AssertionError(e);+ }+ }+}