From 999191bea98871e995945afd1e487073935e43e4 Mon Sep 17 00:00:00 2001 From: Teofilis Martisius Date: Wed, 25 Aug 2021 22:00:40 +0100 Subject: [PATCH 1/5] Changes to turn scrcpy into a library and hook into video frames. --- app/meson.build | 4 + app/src/decoder.h | 2 +- app/src/scrcpy.c | 199 ++++++++++++++++++++++++++++++++-------------- app/src/scrcpy.h | 24 ++++++ 4 files changed, 167 insertions(+), 62 deletions(-) diff --git a/app/meson.build b/app/meson.build index f5345803..9642b49f 100644 --- a/app/meson.build +++ b/app/meson.build @@ -149,6 +149,10 @@ executable('scrcpy', src, install: true, c_args: []) +library('scrcpy', src, + dependencies: dependencies, + include_directories: src_dir) + install_man('scrcpy.1') diff --git a/app/src/decoder.h b/app/src/decoder.h index 257f751a..810a21f8 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -8,7 +8,7 @@ #include #include -#define DECODER_MAX_SINKS 2 +#define DECODER_MAX_SINKS 5 struct decoder { struct sc_packet_sink packet_sink; // packet sink trait diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 6a285788..5738a636 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -41,6 +41,24 @@ struct scrcpy { struct controller controller; struct file_handler file_handler; struct input_manager input_manager; + // do not allocate this on stack, keep it in the struct + struct stream_callbacks stream_cbs; + + // status of scrcpy process + bool server_started; + bool file_handler_initialized; + bool recorder_initialized; +#ifdef HAVE_V4L2 + bool v4l2_sink_initialized; +#endif + bool stream_started; + bool controller_initialized; + bool controller_started; + bool screen_initialized; + + // External sinks- allocated on HEAP. Remember them so that they can be freed later. + struct sc_frame_sink *external_sinks[DECODER_MAX_SINKS]; + unsigned external_sink_count; }; #ifdef _WIN32 @@ -239,28 +257,16 @@ stream_on_eos(struct stream *stream, void *userdata) { SDL_PushEvent(&stop_event); } -bool -scrcpy(const struct scrcpy_options *options) { - static struct scrcpy scrcpy; - struct scrcpy *s = &scrcpy; +struct scrcpy_process * +scrcpy_start(const struct scrcpy_options *options) { + struct scrcpy *s = malloc(sizeof(struct scrcpy)); + struct scrcpy_process *p = malloc(sizeof(struct scrcpy)); + p->scrcpy_struct = s; if (!server_init(&s->server)) { - return false; + return NULL; } - bool ret = false; - - bool server_started = false; - bool file_handler_initialized = false; - bool recorder_initialized = false; -#ifdef HAVE_V4L2 - bool v4l2_sink_initialized = false; -#endif - bool stream_started = false; - bool controller_initialized = false; - bool controller_started = false; - bool screen_initialized = false; - bool record = !!options->record_filename; struct server_params params = { .serial = options->serial, @@ -281,29 +287,32 @@ scrcpy(const struct scrcpy_options *options) { .power_off_on_close = options->power_off_on_close, }; if (!server_start(&s->server, ¶ms)) { - goto end; + scrcpy_stop(p); + return NULL; } - server_started = true; + s->server_started = true; if (!sdl_init_and_configure(options->display, options->render_driver, options->disable_screensaver)) { - goto end; + scrcpy_stop(p); + return NULL; } char device_name[DEVICE_NAME_FIELD_LENGTH]; - struct size frame_size; - if (!server_connect_to(&s->server, device_name, &frame_size)) { - goto end; + if (!server_connect_to(&s->server, device_name, &p->frame_size)) { + scrcpy_stop(p); + return NULL; } if (options->display && options->control) { if (!file_handler_init(&s->file_handler, s->server.serial, options->push_target)) { - goto end; + scrcpy_stop(p); + return NULL; } - file_handler_initialized = true; + s->file_handler_initialized = true; } struct decoder *dec = NULL; @@ -311,6 +320,7 @@ scrcpy(const struct scrcpy_options *options) { #ifdef HAVE_V4L2 needs_decoder |= !!options->v4l2_device; #endif + needs_decoder |= options->force_decoder; if (needs_decoder) { decoder_init(&s->decoder); dec = &s->decoder; @@ -321,19 +331,19 @@ scrcpy(const struct scrcpy_options *options) { if (!recorder_init(&s->recorder, options->record_filename, options->record_format, - frame_size)) { - goto end; + p->frame_size)) { + scrcpy_stop(p); + return NULL; } rec = &s->recorder; - recorder_initialized = true; + s->recorder_initialized = true; } av_log_set_callback(av_log_callback); - static const struct stream_callbacks stream_cbs = { - .on_eos = stream_on_eos, - }; - stream_init(&s->stream, s->server.video_socket, &stream_cbs, NULL); + // don't allocate callbacks on stack + s->stream_cbs.on_eos = stream_on_eos; + stream_init(&s->stream, s->server.video_socket, &s->stream_cbs, NULL); if (dec) { stream_add_sink(&s->stream, &dec->packet_sink); @@ -345,14 +355,16 @@ scrcpy(const struct scrcpy_options *options) { if (options->control) { if (!controller_init(&s->controller, s->server.control_socket)) { - goto end; + scrcpy_stop(p); + return NULL; } - controller_initialized = true; + s->controller_initialized = true; if (!controller_start(&s->controller)) { - goto end; + scrcpy_stop(p); + return NULL; } - controller_started = true; + s->controller_started = true; if (options->turn_screen_off) { struct control_msg msg; @@ -371,7 +383,7 @@ scrcpy(const struct scrcpy_options *options) { struct screen_params screen_params = { .window_title = window_title, - .frame_size = frame_size, + .frame_size = p->frame_size, .always_on_top = options->always_on_top, .window_x = options->window_x, .window_y = options->window_y, @@ -385,96 +397,161 @@ scrcpy(const struct scrcpy_options *options) { }; if (!screen_init(&s->screen, &screen_params)) { - goto end; + scrcpy_stop(p); + return NULL; } - screen_initialized = true; + s->screen_initialized = true; decoder_add_sink(&s->decoder, &s->screen.frame_sink); } #ifdef HAVE_V4L2 if (options->v4l2_device) { - if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device, frame_size, + if (!sc_v4l2_sink_init(&s->v4l2_sink, options->v4l2_device, p->frame_size, options->v4l2_buffer)) { - goto end; + scrcpy_stop(p); + return NULL; } decoder_add_sink(&s->decoder, &s->v4l2_sink.frame_sink); - v4l2_sink_initialized = true; + s->v4l2_sink_initialized = true; } #endif // now we consumed the header values, the socket receives the video stream // start the stream if (!stream_start(&s->stream)) { - goto end; + scrcpy_stop(p); + return NULL; } - stream_started = true; + s->stream_started = true; + return p; +} + +bool +scrcpy_loop(struct scrcpy_process *p, const struct scrcpy_options *options) { + struct scrcpy *s = p->scrcpy_struct; input_manager_init(&s->input_manager, &s->controller, &s->screen, options); - ret = event_loop(s, options); + int ret = event_loop(s, options); LOGD("quit..."); + return ret; +} - // Close the window immediately on closing, because screen_destroy() may - // only be called once the stream thread is joined (it may take time) - screen_hide_window(&s->screen); -end: +void +scrcpy_stop(struct scrcpy_process *p) { + struct scrcpy *s = p->scrcpy_struct; + + if (s->screen_initialized) { + // Close the window immediately on closing, because screen_destroy() may + // only be called once the stream thread is joined (it may take time) + screen_hide_window(&s->screen); + } + // The stream is not stopped explicitly, because it will stop by itself on // end-of-stream - if (controller_started) { + if (s->controller_started) { controller_stop(&s->controller); } - if (file_handler_initialized) { + if (s->file_handler_initialized) { file_handler_stop(&s->file_handler); } - if (screen_initialized) { + if (s->screen_initialized) { screen_interrupt(&s->screen); } - if (server_started) { + if (s->server_started) { // shutdown the sockets and kill the server server_stop(&s->server); } // now that the sockets are shutdown, the stream and controller are // interrupted, we can join them - if (stream_started) { + if (s->stream_started) { stream_join(&s->stream); } #ifdef HAVE_V4L2 - if (v4l2_sink_initialized) { + if (s->v4l2_sink_initialized) { sc_v4l2_sink_destroy(&s->v4l2_sink); } #endif // Destroy the screen only after the stream is guaranteed to be finished, // because otherwise the screen could receive new frames after destruction - if (screen_initialized) { + if (s->screen_initialized) { screen_join(&s->screen); screen_destroy(&s->screen); } - if (controller_started) { + if (s->controller_started) { controller_join(&s->controller); } - if (controller_initialized) { + if (s->controller_initialized) { controller_destroy(&s->controller); } - if (recorder_initialized) { + if (s->recorder_initialized) { recorder_destroy(&s->recorder); } - if (file_handler_initialized) { + if (s->file_handler_initialized) { file_handler_join(&s->file_handler); file_handler_destroy(&s->file_handler); } server_destroy(&s->server); + // free up sinks + while (s->external_sink_count > 0) { + struct sc_frame_sink *sink = s->external_sinks[--s->external_sink_count]; + const struct sc_frame_sink_ops *ops = sink->ops; + free((void*)ops); + free(sink); + } + + // given that these structures were allocated in heap, free them + free(s); + free(p); +} + +bool +scrcpy(const struct scrcpy_options *options) { + struct scrcpy_process *p = scrcpy_start(options); + if (p == NULL) { + return false; + } + bool ret = scrcpy_loop(p, options); + scrcpy_stop(p); return ret; } + +// use void* so that external clients don't have to deal with scrcpy internals. +// TODO: how do I force JavaCPP AVFrame to use AVFrame type it already has from ffmpeg library mapping? +bool +scrcpy_add_sink(struct scrcpy_process *p, + bool (*open)(void *sink), + void (*close)(void *sink), + bool (*push)(void *sink, const void *avframe) + ) { + struct scrcpy *s = p->scrcpy_struct; + + if (s->external_sink_count >= DECODER_MAX_SINKS) { + return false; + } + + struct sc_frame_sink *sink = malloc(sizeof(struct sc_frame_sink)); + struct sc_frame_sink_ops *ops = malloc(sizeof(struct sc_frame_sink_ops)); + + ops->open = (bool (*)(struct sc_frame_sink *sink)) open; + ops->close = (void (*)(struct sc_frame_sink *sink)) close; + ops->push = (bool (*)(struct sc_frame_sink *sink, const AVFrame *frame))push; + sink->ops = ops; + s->external_sinks[s->external_sink_count++] = sink; + + decoder_add_sink(&s->decoder, sink); + return true; +} diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index 8b76fb25..e1ab5bfb 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -2,6 +2,7 @@ #define SCRCPY_H #include "common.h" +#include "coords.h" #include #include @@ -87,6 +88,7 @@ struct scrcpy_options { bool always_on_top; bool control; bool display; + bool force_decoder; // force scrcpy to always initialize a decoder bool turn_screen_off; bool prefer_text; bool window_borderless; @@ -150,6 +152,28 @@ struct scrcpy_options { .power_off_on_close = false, \ } +struct scrcpy_process { + // To be cast to (struct scrcpy *) + // don't use the real struct type, as it's internal and should not be interfered by outside process; + // also, JavaCPP is unable to parse struct scrcpy properly. + void *scrcpy_struct; + // other properties to be available to external systems + struct size frame_size; +}; + +struct scrcpy_process * +scrcpy_start(const struct scrcpy_options *options); + +// add an external frame sink. +bool +scrcpy_add_sink(struct scrcpy_process *p, + bool (*open)(void *sink), + void (*close)(void *sink), + bool (*push)(void *sink, const void *avframe)); + +void +scrcpy_stop(struct scrcpy_process *p); + bool scrcpy(const struct scrcpy_options *options); From e3e162ffdc343de7dc2b3957a5c61b684c75de0f Mon Sep 17 00:00:00 2001 From: nkh0472 Date: Wed, 25 Aug 2021 11:09:58 +0800 Subject: [PATCH 2/5] Upgrade platform-tools (31.0.3) for Windows Include the latest version of adb in Windows releases. PR #2588 Signed-off-by: Romain Vimont --- prebuilt-deps/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prebuilt-deps/Makefile b/prebuilt-deps/Makefile index d75d0a5c..75736f8c 100644 --- a/prebuilt-deps/Makefile +++ b/prebuilt-deps/Makefile @@ -35,6 +35,6 @@ prepare-sdl2: SDL2-2.0.14 prepare-adb: - @./prepare-dep https://dl.google.com/android/repository/platform-tools_r31.0.2-windows.zip \ - d560cb8ded83ae04763b94632673481f14843a5969256569623cfeac82db4ba5 \ + @./prepare-dep https://dl.google.com/android/repository/platform-tools_r31.0.3-windows.zip \ + 0f4b8fdd26af2c3733539d6eebb3c2ed499ea1d4bb1f4e0ecc2d6016961a6e24 \ platform-tools From e1c519b7150ef94b7e9517f5abafb88dbc296550 Mon Sep 17 00:00:00 2001 From: nkh0472 Date: Wed, 25 Aug 2021 11:11:41 +0800 Subject: [PATCH 3/5] Upgrade SDL (2.0.16) for Windows Include the latest version of SDL in Windows releases. PR #2589 Signed-off-by: Romain Vimont --- cross_win32.txt | 2 +- cross_win64.txt | 2 +- prebuilt-deps/Makefile | 6 +++--- release.mk | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cross_win32.txt b/cross_win32.txt index 0d8a843a..4db17be7 100644 --- a/cross_win32.txt +++ b/cross_win32.txt @@ -17,4 +17,4 @@ endian = 'little' [properties] prebuilt_ffmpeg_shared = 'ffmpeg-4.3.1-win32-shared' prebuilt_ffmpeg_dev = 'ffmpeg-4.3.1-win32-dev' -prebuilt_sdl2 = 'SDL2-2.0.14/i686-w64-mingw32' +prebuilt_sdl2 = 'SDL2-2.0.16/i686-w64-mingw32' diff --git a/cross_win64.txt b/cross_win64.txt index 6a39c391..d03f0272 100644 --- a/cross_win64.txt +++ b/cross_win64.txt @@ -17,4 +17,4 @@ endian = 'little' [properties] prebuilt_ffmpeg_shared = 'ffmpeg-4.3.1-win64-shared' prebuilt_ffmpeg_dev = 'ffmpeg-4.3.1-win64-dev' -prebuilt_sdl2 = 'SDL2-2.0.14/x86_64-w64-mingw32' +prebuilt_sdl2 = 'SDL2-2.0.16/x86_64-w64-mingw32' diff --git a/prebuilt-deps/Makefile b/prebuilt-deps/Makefile index 75736f8c..dced047c 100644 --- a/prebuilt-deps/Makefile +++ b/prebuilt-deps/Makefile @@ -30,9 +30,9 @@ prepare-ffmpeg-dev-win64: ffmpeg-4.3.1-win64-dev prepare-sdl2: - @./prepare-dep https://libsdl.org/release/SDL2-devel-2.0.14-mingw.tar.gz \ - 405eaff3eb18f2e08fe669ef9e63bc9a8710b7d343756f238619761e9b60407d \ - SDL2-2.0.14 + @./prepare-dep https://libsdl.org/release/SDL2-devel-2.0.16-mingw.tar.gz \ + 2bfe48628aa9635c12eac7d421907e291525de1d0b04b3bca4a5bd6e6c881a6f \ + SDL2-2.0.16 prepare-adb: @./prepare-dep https://dl.google.com/android/repository/platform-tools_r31.0.3-windows.zip \ diff --git a/release.mk b/release.mk index 2a026135..e327654c 100644 --- a/release.mk +++ b/release.mk @@ -102,7 +102,7 @@ dist-win32: build-server build-win32 cp prebuilt-deps/platform-tools/adb.exe "$(DIST)/$(WIN32_TARGET_DIR)/" cp prebuilt-deps/platform-tools/AdbWinApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" cp prebuilt-deps/platform-tools/AdbWinUsbApi.dll "$(DIST)/$(WIN32_TARGET_DIR)/" - cp prebuilt-deps/SDL2-2.0.14/i686-w64-mingw32/bin/SDL2.dll "$(DIST)/$(WIN32_TARGET_DIR)/" + cp prebuilt-deps/SDL2-2.0.16/i686-w64-mingw32/bin/SDL2.dll "$(DIST)/$(WIN32_TARGET_DIR)/" dist-win64: build-server build-win64 mkdir -p "$(DIST)/$(WIN64_TARGET_DIR)" @@ -118,7 +118,7 @@ dist-win64: build-server build-win64 cp prebuilt-deps/platform-tools/adb.exe "$(DIST)/$(WIN64_TARGET_DIR)/" cp prebuilt-deps/platform-tools/AdbWinApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" cp prebuilt-deps/platform-tools/AdbWinUsbApi.dll "$(DIST)/$(WIN64_TARGET_DIR)/" - cp prebuilt-deps/SDL2-2.0.14/x86_64-w64-mingw32/bin/SDL2.dll "$(DIST)/$(WIN64_TARGET_DIR)/" + cp prebuilt-deps/SDL2-2.0.16/x86_64-w64-mingw32/bin/SDL2.dll "$(DIST)/$(WIN64_TARGET_DIR)/" zip-win32: dist-win32 cd "$(DIST)/$(WIN32_TARGET_DIR)"; \ From de3853d91041ce84ad7cb76fcda613700063ad5b Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Sat, 28 Aug 2021 13:56:39 +0200 Subject: [PATCH 4/5] Use SOURCE_MOUSE for scroll events This has no practical impact (AFAIK), but a scroll events should come from a mouse. Refs #2602 --- server/src/main/java/com/genymobile/scrcpy/Controller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/Controller.java b/server/src/main/java/com/genymobile/scrcpy/Controller.java index 92986241..45882bb9 100644 --- a/server/src/main/java/com/genymobile/scrcpy/Controller.java +++ b/server/src/main/java/com/genymobile/scrcpy/Controller.java @@ -241,7 +241,7 @@ public class Controller { MotionEvent event = MotionEvent .obtain(lastTouchDown, now, MotionEvent.ACTION_SCROLL, 1, pointerProperties, pointerCoords, 0, 0, 1f, 1f, DEFAULT_DEVICE_ID, 0, - InputDevice.SOURCE_TOUCHSCREEN, 0); + InputDevice.SOURCE_MOUSE, 0); return device.injectEvent(event); } From ad7212beb1b0ef1a88261ab59c40125a7e2c195c Mon Sep 17 00:00:00 2001 From: TeofilisMartisius Date: Mon, 30 Aug 2021 23:48:26 +0100 Subject: [PATCH 5/5] Add API to scrcpy library to send control messages to Android device. --- app/src/scrcpy.c | 7 +++++++ app/src/scrcpy.h | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 5738a636..cf61e453 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -555,3 +555,10 @@ scrcpy_add_sink(struct scrcpy_process *p, decoder_add_sink(&s->decoder, sink); return true; } + +void +scrcpy_push_event(struct scrcpy_process *p, + const struct control_msg *msg) { + struct scrcpy *s = p->scrcpy_struct; + controller_push_msg(&s->controller, msg); +} diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index e1ab5bfb..bdc0623d 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -3,6 +3,7 @@ #include "common.h" #include "coords.h" +#include "control_msg.h" #include #include @@ -171,6 +172,10 @@ scrcpy_add_sink(struct scrcpy_process *p, void (*close)(void *sink), bool (*push)(void *sink, const void *avframe)); +void +scrcpy_push_event(struct scrcpy_process *p, + const struct control_msg *msg); + void scrcpy_stop(struct scrcpy_process *p);