diff --git a/app/src/demuxer.c b/app/src/demuxer.c index 690e1620..b98e3e4b 100644 --- a/app/src/demuxer.c +++ b/app/src/demuxer.c @@ -13,6 +13,7 @@ #define SC_PACKET_FLAG_CONFIG (UINT64_C(1) << 63) #define SC_PACKET_FLAG_KEY_FRAME (UINT64_C(1) << 62) +#define PACKET_FLAG_VIDEO_SESSION (UINT64_C(1) << 61) #define SC_PACKET_PTS_MASK (SC_PACKET_FLAG_KEY_FRAME - 1) @@ -76,6 +77,18 @@ sc_demuxer_recv_video_size(struct sc_demuxer *demuxer, uint32_t *width, *height = sc_read32be(data + 4); return true; } +static void +handle_video_session_packet(AVPacket *packet) { + if (!packet || !packet->data || packet->size < 12) { + LOGE("Invalid packet data"); + return; + } + const uint8_t *data = packet->data; + int width = sc_read32be(data); + int height = sc_read32be(data + 4); + int orientation = sc_read32be(data + 8); + LOGD("Orientation=%d, Width=%d, Height=%d, Size=%d", orientation, width, height, packet->size); +} uint32_t last_screen_orientation = -1; @@ -127,6 +140,11 @@ sc_demuxer_recv_packet(struct sc_demuxer *demuxer, AVPacket *packet) { return false; } + if (pts_flags & PACKET_FLAG_VIDEO_SESSION) { + handle_video_session_packet(packet); + return true; + } + if (pts_flags & SC_PACKET_FLAG_CONFIG) { packet->pts = AV_NOPTS_VALUE; } else { diff --git a/server/src/main/java/com/genymobile/scrcpy/device/Device.java b/server/src/main/java/com/genymobile/scrcpy/device/Device.java index 687ec4a4..732c7ba8 100644 --- a/server/src/main/java/com/genymobile/scrcpy/device/Device.java +++ b/server/src/main/java/com/genymobile/scrcpy/device/Device.java @@ -24,6 +24,7 @@ import android.view.InputDevice; import android.view.InputEvent; import android.view.KeyCharacterMap; import android.view.KeyEvent; +import android.graphics.Point; import java.util.ArrayList; import java.util.List; @@ -220,6 +221,14 @@ public final class Device { DisplayInfo displayInfo = ServiceManager.getDisplayManager().getDisplayInfo(displayId); return displayInfo.getRotation(); } + public static Size getSize(int displayId){ + assert displayId != DISPLAY_ID_NONE; + if (displayId == 0) { + return ServiceManager.getWindowManager().getSize(displayId); + } + DisplayInfo displayInfo = ServiceManager.getDisplayManager().getDisplayInfo(displayId); + return displayInfo.getSize(); + } public static List listApps() { List apps = new ArrayList<>(); diff --git a/server/src/main/java/com/genymobile/scrcpy/device/Streamer.java b/server/src/main/java/com/genymobile/scrcpy/device/Streamer.java index a3627d08..589a23fc 100644 --- a/server/src/main/java/com/genymobile/scrcpy/device/Streamer.java +++ b/server/src/main/java/com/genymobile/scrcpy/device/Streamer.java @@ -13,9 +13,9 @@ import java.nio.ByteOrder; import java.util.Arrays; public final class Streamer { - private static final long PACKET_FLAG_CONFIG = 1L << 63; private static final long PACKET_FLAG_KEY_FRAME = 1L << 62; + public static final long PACKET_FLAG_VIDEO_SESSION = 1L << 61; private final FileDescriptor fd; private final Codec codec; @@ -23,7 +23,8 @@ public final class Streamer { private final boolean sendFrameMeta; private final int displayId; - private final ByteBuffer headerBuffer = ByteBuffer.allocate(16); + private final int HEADER_PACKET_SIZE = 12; + private final ByteBuffer headerBuffer = ByteBuffer.allocate(HEADER_PACKET_SIZE); public Streamer(FileDescriptor fd, Codec codec, boolean sendCodecMeta, boolean sendFrameMeta, int displayId) { this.fd = fd; @@ -48,7 +49,7 @@ public final class Streamer { public void writeVideoHeader(Size videoSize) throws IOException { if (sendCodecMeta) { - ByteBuffer buffer = ByteBuffer.allocate(12); + ByteBuffer buffer = ByteBuffer.allocate(HEADER_PACKET_SIZE); buffer.putInt(codec.getId()); buffer.putInt(videoSize.getWidth()); buffer.putInt(videoSize.getHeight()); diff --git a/server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.java b/server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.java index 236a5f48..7b722927 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/SurfaceEncoder.java @@ -3,9 +3,11 @@ package com.genymobile.scrcpy.video; import com.genymobile.scrcpy.AndroidVersions; import com.genymobile.scrcpy.AsyncProcessor; import com.genymobile.scrcpy.Options; +import com.genymobile.scrcpy.device.Device; import com.genymobile.scrcpy.device.ConfigurationException; import com.genymobile.scrcpy.device.Size; import com.genymobile.scrcpy.device.Streamer; +import com.genymobile.scrcpy.device.Orientation; import com.genymobile.scrcpy.util.Codec; import com.genymobile.scrcpy.util.CodecOption; import com.genymobile.scrcpy.util.CodecUtils; @@ -25,6 +27,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; public class SurfaceEncoder implements AsyncProcessor { @@ -44,6 +47,16 @@ public class SurfaceEncoder implements AsyncProcessor { private final float maxFps; private final boolean downsizeOnError; + private int orientation = -100; + private final int displayId; + private boolean sendCodeMeta; + private boolean sendFrameMeta; + private int maxSize; + private Size originalSize; + private Size displaySize; + private Orientation captureOrientation; + private Orientation.Lock captureOrientationLock; + private boolean firstFrameSent; private int consecutiveErrors; @@ -60,13 +73,22 @@ public class SurfaceEncoder implements AsyncProcessor { this.codecOptions = options.getVideoCodecOptions(); this.encoderName = options.getVideoEncoder(); this.downsizeOnError = options.getDownsizeOnError(); + + this.displayId = options.getDisplayId(); + this.sendCodeMeta = options.getSendCodecMeta(); + this.sendFrameMeta = options.getSendFrameMeta(); + this.originalSize = Device.getSize(this.displayId); + int maxSize = options.getMaxSize(); + int max = maxSize == 0 ? chooseMaxSizeFallback(this.originalSize) : maxSize; + this.displaySize = this.originalSize.limit(max).round8(); + this.captureOrientation = options.getCaptureOrientation(); + this.captureOrientationLock = options.getCaptureOrientationLock(); } private void streamCapture() throws IOException, ConfigurationException { Codec codec = streamer.getCodec(); MediaCodec mediaCodec = createMediaCodec(codec, encoderName); MediaFormat format = createFormat(codec.getMimeType(), videoBitRate, maxFps, codecOptions); - capture.init(reset); try { @@ -182,7 +204,7 @@ public class SurfaceEncoder implements AsyncProcessor { return true; } - private static int chooseMaxSizeFallback(Size failedSize) { + public static int chooseMaxSizeFallback(Size failedSize) { int currentMaxSize = Math.max(failedSize.getWidth(), failedSize.getHeight()); for (int value : MAX_SIZE_FALLBACK) { if (value < currentMaxSize) { @@ -194,6 +216,28 @@ public class SurfaceEncoder implements AsyncProcessor { return 0; } + private Size getSize(int orientation){ + switch (orientation) { + case 0: + case 4: + return this.displaySize; + case 1: + case 3: + return new Size(this.displaySize.getHeight(),this.displaySize.getWidth()); + } + return new Size(0,0); + } + private ByteBuffer getCurrentVideoSession(int orientation){ + ByteBuffer buffer = ByteBuffer.allocate(20); + Size size = getSize(orientation); + int width = size.getWidth(); + int height = size.getHeight(); + buffer.putInt(width); + buffer.putInt(height); + buffer.putInt(orientation); + buffer.flip(); + return buffer; + } private void encode(MediaCodec codec, Streamer streamer) throws IOException { MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); @@ -201,6 +245,16 @@ public class SurfaceEncoder implements AsyncProcessor { do { int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1); try { + // If the frame width and height do not change, it will not be re-encoded + // Therefore, it is necessary to monitor each frame. + // For example: horizontal (1->3), vertical (0->4) + int orientation = Device.getCurrentRotation(displayId); + if(this.orientation != orientation){ + this.orientation = orientation; + ByteBuffer videoSession = getCurrentVideoSession(orientation); + streamer.writePacket(videoSession,Streamer.PACKET_FLAG_VIDEO_SESSION,false,false); + } + eos = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0; // On EOS, there might be data or not, depending on bufferInfo.size if (outputBufferId >= 0 && bufferInfo.size > 0) { @@ -212,7 +266,6 @@ public class SurfaceEncoder implements AsyncProcessor { firstFrameSent = true; consecutiveErrors = 0; } - streamer.writePacket(codecBuffer, bufferInfo); } } finally { diff --git a/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java b/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java index 7ba5cc06..498ba6cf 100644 --- a/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java +++ b/server/src/main/java/com/genymobile/scrcpy/wrappers/WindowManager.java @@ -2,12 +2,15 @@ package com.genymobile.scrcpy.wrappers; import com.genymobile.scrcpy.AndroidVersions; import com.genymobile.scrcpy.util.Ln; +import com.genymobile.scrcpy.device.Size; import android.annotation.TargetApi; +import android.graphics.Point; import android.os.Build; import android.os.IInterface; import android.view.IDisplayWindowListener; +// import java.awt.Point; import java.lang.reflect.Method; public final class WindowManager { @@ -42,6 +45,26 @@ public final class WindowManager { this.manager = manager; } + private Method getGetSizeMethod; + private Method getGetSizeMethod() throws NoSuchMethodException { + if (getGetSizeMethod == null) { + Class cls = manager.getClass(); + getGetSizeMethod = cls.getMethod("getInitialDisplaySize",int.class, Point.class); + } + return getGetSizeMethod; + } + public Size getSize(int displayId) { + try { + Method method = getGetSizeMethod(); + Point size = new Point(); + method.invoke(manager, displayId, size); + return new Size(size.x, size.y); + } catch (ReflectiveOperationException e) { + Ln.e("Could not invoke method", e); + return new Size(0,0); + } + } + private Method getGetRotationMethod() throws NoSuchMethodException { if (getRotationMethod == null) { Class cls = manager.getClass();