From 158a3e5d2b1714c45bb4e3c94bb494f49c2c7966 Mon Sep 17 00:00:00 2001 From: Frank Leon Rose Date: Fri, 24 Jan 2020 13:04:24 -0500 Subject: [PATCH 1/9] Adds h264 as format for recording Useful for piping into `ffmpeg`. --- app/src/cli.c | 4 ++++ app/src/recorder.c | 1 + app/src/recorder.h | 1 + 3 files changed, 6 insertions(+) diff --git a/app/src/cli.c b/app/src/cli.c index d9e1013a..ef31a723 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -307,6 +307,10 @@ parse_record_format(const char *optarg, enum recorder_format *format) { *format = RECORDER_FORMAT_MKV; return true; } + if (!strcmp(optarg, "h264")) { + *format = RECORDER_FORMAT_H264; + return true; + } LOGE("Unsupported format: %s (expected mp4 or mkv)", optarg); return false; } diff --git a/app/src/recorder.c b/app/src/recorder.c index 465b24e8..9492f0fa 100644 --- a/app/src/recorder.c +++ b/app/src/recorder.c @@ -107,6 +107,7 @@ recorder_destroy(struct recorder *recorder) { static const char * recorder_get_format_name(enum recorder_format format) { switch (format) { + case RECORDER_FORMAT_H264: return "h264"; case RECORDER_FORMAT_MP4: return "mp4"; case RECORDER_FORMAT_MKV: return "matroska"; default: return NULL; diff --git a/app/src/recorder.h b/app/src/recorder.h index 4f5d526c..0b2420f6 100644 --- a/app/src/recorder.h +++ b/app/src/recorder.h @@ -14,6 +14,7 @@ enum recorder_format { RECORDER_FORMAT_AUTO, RECORDER_FORMAT_MP4, RECORDER_FORMAT_MKV, + RECORDER_FORMAT_H264, }; struct record_packet { From d16a0d260c564706b27cff4ef952c772f6245e99 Mon Sep 17 00:00:00 2001 From: Frank Leon Rose Date: Fri, 24 Jan 2020 16:48:24 -0500 Subject: [PATCH 2/9] Adds ability to capture a screenshot to PNG --- app/meson.build | 6 +- app/src/capture.c | 254 +++++++++++++++++++++++++++++++++++++++++++ app/src/capture.h | 21 ++++ app/src/cli.c | 10 +- app/src/events.h | 1 + app/src/main.c | 4 + app/src/scrcpy.c | 22 +++- app/src/scrcpy.h | 2 + app/src/stream.c | 18 ++- app/src/stream.h | 4 +- app/tests/test_cli.c | 2 + 11 files changed, 337 insertions(+), 7 deletions(-) create mode 100644 app/src/capture.c create mode 100644 app/src/capture.h diff --git a/app/meson.build b/app/meson.build index 3bcb9bc1..6a9ff316 100644 --- a/app/meson.build +++ b/app/meson.build @@ -1,5 +1,6 @@ src = [ 'src/main.c', + 'src/capture.c', 'src/cli.c', 'src/command.c', 'src/control_msg.c', @@ -30,6 +31,8 @@ if not get_option('crossbuild_windows') dependency('libavformat'), dependency('libavcodec'), dependency('libavutil'), + dependency('libswscale'), + dependency('libpng'), dependency('sdl2'), ] @@ -168,7 +171,8 @@ if get_option('buildtype') == 'debug' 'tests/test_queue.c', ]], ['test_strutil', [ - 'tests/test_strutil.c', + 'tests/test_s + til.c', 'src/util/str_util.c', ]], ] diff --git a/app/src/capture.c b/app/src/capture.c new file mode 100644 index 00000000..4e5c60bc --- /dev/null +++ b/app/src/capture.c @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2001 Fabrice Bellard + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +/** + * @file + * video decoding with libavcodec API example + * + * @example decode_video.c + */ + +#include "capture.h" + +#include +#include +#include + +#include +#include +#include + +#include + +#include "events.h" +#include "util/log.h" + +static const char *string_error(int err) { + switch (err) { + case AVERROR_EOF: return "End of File"; + case AVERROR_BUG: return "Bug"; + case AVERROR_BUG2: return "Bug2"; + case AVERROR_BUFFER_TOO_SMALL: return "Buffer Too Small"; + case AVERROR_EXIT: return "Exit Requested"; + case AVERROR_DECODER_NOT_FOUND: return "Decoder Not Found"; + case AVERROR_DEMUXER_NOT_FOUND: return "Demuxer Not Found"; + case AVERROR_ENCODER_NOT_FOUND: return "Encoder Not Found"; + case AVERROR_EXTERNAL: return "External Error"; + case AVERROR_FILTER_NOT_FOUND: return "Filter Not Found"; + case AVERROR_MUXER_NOT_FOUND: return "Muxer Not Found"; + case AVERROR_OPTION_NOT_FOUND: return "Option Not Found"; + case AVERROR_PATCHWELCOME: return "Patch Welcome"; + case AVERROR_PROTOCOL_NOT_FOUND: return "Protocol Not Found"; + case AVERROR_STREAM_NOT_FOUND: return "Stream Not Found"; + case AVERROR_INVALIDDATA: return "Invalid Data (waiting for index frame?)"; + case AVERROR(EINVAL): return "Invalid"; + case AVERROR(ENOMEM): return "No Memory"; + case AVERROR(EAGAIN): return "Try Again"; + default: return "Unknown"; + } +} + +static void notify_complete() { + static SDL_Event new_frame_event = { + .type = EVENT_SCREEN_CAPTURE_COMPLETE, + }; + SDL_PushEvent(&new_frame_event); +} + +// Thanks, https://stackoverflow.com/a/18313791 +// Also, https://raw.githubusercontent.com/FFmpeg/FFmpeg/master/doc/examples/decode_video.c + +static bool in_frame_to_png( + AVIOContext *output_context, + AVCodecContext *codecCtx, AVFrame *inframe) { + struct SwsContext * swCtx = sws_getContext(codecCtx->width, + codecCtx->height, + codecCtx->pix_fmt, + codecCtx->width, + codecCtx->height, + AV_PIX_FMT_RGB24, + SWS_FAST_BILINEAR, 0, 0, 0); + + AVFrame * rgbFrame = av_frame_alloc(); + LOGV("Image frame width: %d height: %d", codecCtx->width, codecCtx->height); + rgbFrame->width = codecCtx->width; + rgbFrame->height = codecCtx->height; + rgbFrame->format = AV_PIX_FMT_RGB24; + av_image_alloc( + rgbFrame->data, rgbFrame->linesize, codecCtx->width, + codecCtx->height, AV_PIX_FMT_RGB24, 1); + sws_scale( + swCtx, inframe->data, inframe->linesize, 0, + inframe->height, rgbFrame->data, rgbFrame->linesize); + + AVCodec *outCodec = avcodec_find_encoder(AV_CODEC_ID_PNG); + if (!outCodec) { + LOGE("Failed to find PNG codec"); + return false; + } + AVCodecContext *outCodecCtx = avcodec_alloc_context3(outCodec); + if (!outCodecCtx) { + LOGE("Unable to allocate PNG codec context"); + return false; + } + + outCodecCtx->width = codecCtx->width; + outCodecCtx->height = codecCtx->height; + outCodecCtx->pix_fmt = AV_PIX_FMT_RGB24; + outCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO; + if (codecCtx->time_base.num && codecCtx->time_base.num) { + outCodecCtx->time_base.num = codecCtx->time_base.num; + outCodecCtx->time_base.den = codecCtx->time_base.den; + } else { + outCodecCtx->time_base.num = 1; + outCodecCtx->time_base.den = 30; // FPS + } + + int ret = avcodec_open2(outCodecCtx, outCodec, NULL); + if (ret < 0) { + LOGE("Failed to open PNG codec: %s", string_error(ret)); + return false; + } + + ret = avcodec_send_frame(outCodecCtx, rgbFrame); + av_frame_free(&rgbFrame); + if (ret < 0) { + LOGE("Failed to send frame to output codex"); + return false; + } + + AVPacket outPacket; + av_init_packet(&outPacket); + outPacket.size = 0; + outPacket.data = NULL; + ret = avcodec_receive_packet(outCodecCtx, &outPacket); + avcodec_close(outCodecCtx); + av_free(outCodecCtx); + if (ret >= 0) { + // Dump packet + avio_write(output_context, outPacket.data, outPacket.size); + notify_complete(); + av_packet_unref(&outPacket); + return true; + } else { + LOGE("Failed to receive packet"); + return false; + } +} + +// Decodes a video packet into PNG image, if possible. +// Sends a new video packet `packet` into the active decoding +// context `dec_ctx`. +// Returns `true` when the decoding yielded a PNG. +static bool decode_packet_to_png( + AVIOContext *output_context, + AVCodecContext *dec_ctx, + const AVPacket *packet, + AVFrame *working_frame) { + LOGV("Sending video packet: %p Size: %d", packet, packet ? packet->size : 0); + + int ret = avcodec_send_packet(dec_ctx, packet); + if (ret < 0) { + LOGW( "Error sending a packet for decoding: %s", string_error(ret)); + return false; + } + + bool found = false; + while (ret >= 0) { + ret = avcodec_receive_frame(dec_ctx, working_frame); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + LOGV("No frame received: %s", string_error(ret)); + break; // We're done + } else if (ret < 0) { + LOGW("Error during decoding: %s", string_error(ret)); + } + + LOGI("Found frame: %d", dec_ctx->frame_number); + if (in_frame_to_png(output_context, dec_ctx, working_frame)) { + found = true; + break; + } else { + LOGE("Failed to generate PNG"); + } + } + return found; +} + +bool capture_init(struct capture *capture, const char *filename) { + capture->filename = SDL_strdup(filename); + if (!capture->filename) { + LOGE("Could not strdup filename"); + return false; + } + + int ret = avio_open(&capture->file_context, capture->filename, + AVIO_FLAG_WRITE); + if (ret < 0) { + LOGE("Failed to open output file: %s", capture->filename); + return false; + } + + /* find the MPEG-1 video decoder */ + const AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); + if (!codec) { + LOGE("Codec not found: %d", AV_CODEC_ID_H264); + return false; + } + + capture->context = avcodec_alloc_context3(codec); + if (!capture->context) { + LOGE("Could not allocate video codec context"); + return false; + } + + /* For some codecs, such as msmpeg4 and mpeg4, width and height + MUST be initialized there because this information is not + available in the bitstream. */ + + /* open it */ + if (avcodec_open2(capture->context, codec, NULL) < 0) { + LOGE("Could not open codec"); + return false; + } + + // Create a reusable frame buffer. + capture->working_frame = av_frame_alloc(); + if (!capture->working_frame) { + LOGE("Could not allocate video frame"); + return false; + } + return true; +} + +void capture_destroy(struct capture *capture) { + av_frame_free(&capture->working_frame); + avcodec_free_context(&capture->context); + avio_close(capture->file_context); +} + +bool capture_push(struct capture *capture, const AVPacket *packet) { + if (decode_packet_to_png(capture->file_context, capture->context, packet, capture->working_frame)) { + LOGI("Parsed PNG from incoming packets."); + return true; + } + return false; +} diff --git a/app/src/capture.h b/app/src/capture.h new file mode 100644 index 00000000..71549b02 --- /dev/null +++ b/app/src/capture.h @@ -0,0 +1,21 @@ +#ifndef CAPTURE_H +#define CAPTURE_H + +#include +#include +#include + +struct capture { + char *filename; + AVIOContext *file_context; + AVCodecContext *context; + AVFrame *working_frame; +}; + +bool capture_init(struct capture *capture, const char *filename); + +void capture_destroy(struct capture *capture); + +bool capture_push(struct capture *capture, const AVPacket *packet); + +#endif // CAPTURE_H \ No newline at end of file diff --git a/app/src/cli.c b/app/src/cli.c index ef31a723..4b74efab 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -363,6 +363,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { {"record-format", required_argument, NULL, OPT_RECORD_FORMAT}, {"render-expired-frames", no_argument, NULL, OPT_RENDER_EXPIRED_FRAMES}, + {"screen-capture", required_argument, NULL, 'C'}, {"serial", required_argument, NULL, 's'}, {"show-touches", no_argument, NULL, 't'}, {"turn-screen-off", no_argument, NULL, 'S'}, @@ -383,7 +384,7 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { optind = 0; // reset to start from the first argument in tests int c; - while ((c = getopt_long(argc, argv, "b:c:fF:hm:nNp:r:s:StTv", long_options, + while ((c = getopt_long(argc, argv, "b:cC:fF:hm:nNp:r:s:StTv", long_options, NULL)) != -1) { switch (c) { case 'b': @@ -391,6 +392,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { return false; } break; + case 'C': + opts->capture_filename = optarg; + break; case 'c': LOGW("Deprecated option -c. Use --crop instead."); // fall through @@ -494,8 +498,8 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { } } - if (!opts->display && !opts->record_filename) { - LOGE("-N/--no-display requires screen recording (-r/--record)"); + if (!opts->display && !opts->record_filename && !opts->capture_filename) { + LOGE("-N/--no-display requires screen recording (-r/--record) or screen capture (-C/--screen-capture)"); return false; } diff --git a/app/src/events.h b/app/src/events.h index e9512048..07c3a04a 100644 --- a/app/src/events.h +++ b/app/src/events.h @@ -1,3 +1,4 @@ #define EVENT_NEW_SESSION SDL_USEREVENT #define EVENT_NEW_FRAME (SDL_USEREVENT + 1) #define EVENT_STREAM_STOPPED (SDL_USEREVENT + 2) +#define EVENT_SCREEN_CAPTURE_COMPLETE (SDL_USEREVENT + 3) diff --git a/app/src/main.c b/app/src/main.c index d683c508..5aea485d 100644 --- a/app/src/main.c +++ b/app/src/main.c @@ -3,6 +3,7 @@ #include #include #include +#include #define SDL_MAIN_HANDLED // avoid link error on Linux Windows Subsystem #include @@ -27,6 +28,9 @@ print_version(void) { fprintf(stderr, " - libavutil %d.%d.%d\n", LIBAVUTIL_VERSION_MAJOR, LIBAVUTIL_VERSION_MINOR, LIBAVUTIL_VERSION_MICRO); + fprintf(stderr, " - libswscale %d.%d.%d\n", LIBSWSCALE_VERSION_MAJOR, + LIBSWSCALE_VERSION_MINOR, + LIBSWSCALE_VERSION_MICRO); } int diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 17be1ed4..6fe2965a 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -18,6 +18,7 @@ #include "file_handler.h" #include "fps_counter.h" #include "input_manager.h" +#include "capture.h" #include "recorder.h" #include "screen.h" #include "server.h" @@ -35,6 +36,7 @@ static struct video_buffer video_buffer; static struct stream stream; static struct decoder decoder; static struct recorder recorder; +static struct capture capture; static struct controller controller; static struct file_handler file_handler; @@ -131,6 +133,9 @@ handle_event(SDL_Event *event, bool control) { case EVENT_STREAM_STOPPED: LOGD("Video stream stopped"); return EVENT_RESULT_STOPPED_BY_EOS; + case EVENT_SCREEN_CAPTURE_COMPLETE: + LOGD("Screen capture completed"); + return EVENT_RESULT_STOPPED_BY_USER; case SDL_QUIT: LOGD("User requested to quit"); return EVENT_RESULT_STOPPED_BY_USER; @@ -277,6 +282,7 @@ av_log_callback(void *avcl, int level, const char *fmt, va_list vl) { bool scrcpy(const struct scrcpy_options *options) { + bool captures = !!options->capture_filename; bool record = !!options->record_filename; struct server_params params = { .crop = options->crop, @@ -304,6 +310,7 @@ scrcpy(const struct scrcpy_options *options) { bool video_buffer_initialized = false; bool file_handler_initialized = false; bool recorder_initialized = false; + bool capture_initialized = false; bool stream_started = false; bool controller_initialized = false; bool controller_started = false; @@ -363,9 +370,19 @@ scrcpy(const struct scrcpy_options *options) { recorder_initialized = true; } + struct capture *cap = NULL; + if (captures) { + if (!capture_init(&capture, + options->capture_filename)) { + goto end; + } + cap = &capture; + capture_initialized = true; + } + av_log_set_callback(av_log_callback); - stream_init(&stream, server.video_socket, dec, rec); + stream_init(&stream, server.video_socket, dec, rec, cap); // now we consumed the header values, the socket receives the video stream // start the stream @@ -460,6 +477,9 @@ end: recorder_destroy(&recorder); } + if (capture_initialized) { + capture_destroy(&capture); + } if (file_handler_initialized) { file_handler_join(&file_handler); file_handler_destroy(&file_handler); diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index 75de8717..de937d92 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -11,6 +11,7 @@ struct scrcpy_options { const char *serial; const char *crop; + const char *capture_filename; const char *record_filename; const char *window_title; const char *push_target; @@ -37,6 +38,7 @@ struct scrcpy_options { #define SCRCPY_OPTIONS_DEFAULT { \ .serial = NULL, \ .crop = NULL, \ + .capture_filename = NULL, \ .record_filename = NULL, \ .window_title = NULL, \ .push_target = NULL, \ diff --git a/app/src/stream.c b/app/src/stream.c index dd2dbd76..8aeff0e5 100644 --- a/app/src/stream.c +++ b/app/src/stream.c @@ -12,6 +12,7 @@ #include "compat.h" #include "decoder.h" #include "events.h" +#include "capture.h" #include "recorder.h" #include "util/buffer_util.h" #include "util/log.h" @@ -71,6 +72,10 @@ notify_stopped(void) { static bool process_config_packet(struct stream *stream, AVPacket *packet) { + if (stream->capture && !capture_push(stream->capture, packet)) { + LOGW("Could not send config packet to capture"); + return true; // We're Ok. Keep sending packets. + } if (stream->recorder && !recorder_push(stream->recorder, packet)) { LOGE("Could not send config packet to recorder"); return false; @@ -93,6 +98,15 @@ process_frame(struct stream *stream, AVPacket *packet) { } } + if (stream->capture) { + packet->dts = packet->pts; + + if (!capture_push(stream->capture, packet)) { + LOGE("Could not send packet to capture"); + return false; + } + } + return true; } @@ -270,9 +284,11 @@ end: void stream_init(struct stream *stream, socket_t socket, - struct decoder *decoder, struct recorder *recorder) { + struct decoder *decoder, struct recorder *recorder, + struct capture *capture) { stream->socket = socket; stream->decoder = decoder, + stream->capture = capture; stream->recorder = recorder; stream->has_pending = false; } diff --git a/app/src/stream.h b/app/src/stream.h index f7c5e475..ed571b31 100644 --- a/app/src/stream.h +++ b/app/src/stream.h @@ -18,6 +18,7 @@ struct stream { SDL_Thread *thread; struct decoder *decoder; struct recorder *recorder; + struct capture *capture; AVCodecContext *codec_ctx; AVCodecParserContext *parser; // successive packets may need to be concatenated, until a non-config @@ -28,7 +29,8 @@ struct stream { void stream_init(struct stream *stream, socket_t socket, - struct decoder *decoder, struct recorder *recorder); + struct decoder *decoder, struct recorder *recorder, + struct capture *capture); bool stream_start(struct stream *stream); diff --git a/app/tests/test_cli.c b/app/tests/test_cli.c index 539c3c94..49ebe3b8 100644 --- a/app/tests/test_cli.c +++ b/app/tests/test_cli.c @@ -55,6 +55,7 @@ static void test_options(void) { "--record", "file", "--record-format", "mkv", "--render-expired-frames", + "--screen-capture", "filename.png", "--serial", "0123456789abcdef", "--show-touches", "--turn-screen-off", @@ -83,6 +84,7 @@ static void test_options(void) { assert(!strcmp(opts->record_filename, "file")); assert(opts->record_format == RECORDER_FORMAT_MKV); assert(opts->render_expired_frames); + assert(!strcmp(opts->capture_filename, "filename.png")); assert(!strcmp(opts->serial, "0123456789abcdef")); assert(opts->show_touches); assert(opts->turn_screen_off); From e0444fdcaefb4c259bc9f36e8d9b560b2d615a25 Mon Sep 17 00:00:00 2001 From: Frank Leon Rose Date: Mon, 27 Jan 2020 15:08:24 -0500 Subject: [PATCH 3/9] Runs capture processing on a separate thread --- app/src/capture.c | 143 ++++++++++++++++++++++++++++++++++++++++++++-- app/src/capture.h | 29 ++++++++++ app/src/command.c | 1 + app/src/stream.c | 15 ++++- 4 files changed, 182 insertions(+), 6 deletions(-) diff --git a/app/src/capture.c b/app/src/capture.c index 4e5c60bc..eb64dd80 100644 --- a/app/src/capture.c +++ b/app/src/capture.c @@ -40,6 +40,7 @@ #include #include "events.h" +#include "util/lock.h" #include "util/log.h" static const char *string_error(int err) { @@ -200,6 +201,25 @@ bool capture_init(struct capture *capture, const char *filename) { return false; } + capture->mutex = SDL_CreateMutex(); + if (!capture->mutex) { + LOGC("Could not create mutex"); + SDL_free(capture->filename); + return false; + } + + capture->queue_cond = SDL_CreateCond(); + if (!capture->queue_cond) { + LOGC("Could not create capture cond"); + SDL_DestroyMutex(capture->mutex); + SDL_free(capture->filename); + return false; + } + + queue_init(&capture->queue); + capture->stopped = false; + capture->finished = false; + int ret = avio_open(&capture->file_context, capture->filename, AVIO_FLAG_WRITE); if (ret < 0) { @@ -245,10 +265,123 @@ void capture_destroy(struct capture *capture) { avio_close(capture->file_context); } -bool capture_push(struct capture *capture, const AVPacket *packet) { - if (decode_packet_to_png(capture->file_context, capture->context, packet, capture->working_frame)) { - LOGI("Parsed PNG from incoming packets."); - return true; +static bool capture_process(struct capture *capture, const AVPacket *packet) { + if (capture->finished) { + LOGV("Skipping redundant call to capture_push"); + } else { + bool found_png = decode_packet_to_png(capture->file_context, capture->context, packet, capture->working_frame); + if (found_png) { + LOGI("Parsed PNG from incoming packets."); + capture->finished = found_png; + } } - return false; + return capture->finished; +} + +static struct capture_packet * +capture_packet_new(const AVPacket *packet) { + struct capture_packet *rec = SDL_malloc(sizeof(*rec)); + if (!rec) { + return NULL; + } + + // av_packet_ref() does not initialize all fields in old FFmpeg versions + // See + av_init_packet(&rec->packet); + + if (av_packet_ref(&rec->packet, packet)) { + SDL_free(rec); + return NULL; + } + return rec; +} + +static void +capture_packet_delete(struct capture_packet *rec) { + av_packet_unref(&rec->packet); + SDL_free(rec); +} + +static int +run_capture(void *data) { + struct capture *capture = data; + + for (;;) { + mutex_lock(capture->mutex); + + while (!capture->stopped && queue_is_empty(&capture->queue)) { + cond_wait(capture->queue_cond, capture->mutex); + } + + // if stopped is set, continue to process the remaining events (to + // finish the capture) before actually stopping + + if (capture->stopped && queue_is_empty(&capture->queue)) { + mutex_unlock(capture->mutex); + break; + } + + struct capture_packet *rec; + queue_take(&capture->queue, next, &rec); + + mutex_unlock(capture->mutex); + + bool ok = capture_process(capture, &rec->packet); + capture_packet_delete(rec); + if (ok) { + break; + } + } + + LOGD("capture thread ended"); + + return 0; +} + +bool +capture_start(struct capture *capture) { + + capture->thread = SDL_CreateThread(run_capture, "capture", capture); + if (!capture->thread) { + LOGC("Could not start capture thread"); + return false; + } + + return true; +} + +void +capture_stop(struct capture *capture) { + mutex_lock(capture->mutex); + capture->stopped = true; + cond_signal(capture->queue_cond); + mutex_unlock(capture->mutex); +} + +void +capture_join(struct capture *capture) { + SDL_WaitThread(capture->thread, NULL); +} + +bool +capture_push(struct capture *capture, const AVPacket *packet) { + mutex_lock(capture->mutex); + assert(!capture->stopped); + + if (capture->finished) { + // reject any new packet (this will stop the stream) + return false; + } + + struct capture_packet *rec = capture_packet_new(packet); + if (!rec) { + LOGC("Could not allocate capture packet"); + return false; + } + + queue_push(&capture->queue, next, rec); + cond_signal(capture->queue_cond); + + mutex_unlock(capture->mutex); + return true; } diff --git a/app/src/capture.h b/app/src/capture.h index 71549b02..0f721db1 100644 --- a/app/src/capture.h +++ b/app/src/capture.h @@ -4,18 +4,47 @@ #include #include #include +#include +#include + +#include "config.h" +#include "common.h" +#include "util/queue.h" + +struct capture_packet { + AVPacket packet; + struct capture_packet *next; +}; + +struct capture_queue QUEUE(struct capture_packet); struct capture { char *filename; AVIOContext *file_context; AVCodecContext *context; AVFrame *working_frame; + + SDL_Thread *thread; + SDL_mutex *mutex; + SDL_cond *queue_cond; + bool finished; // PNG has been captured + bool stopped; // set on recorder_stop() by the stream reader + struct capture_queue queue; }; bool capture_init(struct capture *capture, const char *filename); void capture_destroy(struct capture *capture); +bool +capture_start(struct capture *capture); + +void +capture_stop(struct capture *capture); + +void +capture_join(struct capture *capture); + bool capture_push(struct capture *capture, const AVPacket *packet); #endif // CAPTURE_H \ No newline at end of file diff --git a/app/src/command.c b/app/src/command.c index 63afccb4..66a27128 100644 --- a/app/src/command.c +++ b/app/src/command.c @@ -96,6 +96,7 @@ adb_execute(const char *serial, const char *const adb_cmd[], size_t len) { cmd[len + i] = NULL; enum process_result r = cmd_execute(cmd, &process); if (r != PROCESS_SUCCESS) { + LOGE("adb process execution returned an error."); show_adb_err_msg(r, cmd); return PROCESS_NONE; } diff --git a/app/src/stream.c b/app/src/stream.c index 8aeff0e5..974bcf52 100644 --- a/app/src/stream.c +++ b/app/src/stream.c @@ -228,10 +228,17 @@ run_stream(void *data) { } } + if (stream->capture) { + if (!capture_start(stream->capture)) { + LOGE("Could not start capture"); + goto finally_stop_and_join_recorder; + } + } + stream->parser = av_parser_init(AV_CODEC_ID_H264); if (!stream->parser) { LOGE("Could not initialize parser"); - goto finally_stop_and_join_recorder; + goto finally_stop_and_join_capture; } // We must only pass complete frames to av_parser_parse2()! @@ -261,6 +268,12 @@ run_stream(void *data) { } av_parser_close(stream->parser); +finally_stop_and_join_capture: + if (stream->capture) { + capture_stop(stream->capture); + LOGI("Finishing capture..."); + capture_join(stream->capture); + } finally_stop_and_join_recorder: if (stream->recorder) { recorder_stop(stream->recorder); From 575e6b0a4274b78f150f6851a74a270f92d1265e Mon Sep 17 00:00:00 2001 From: Frank Leon Rose Date: Mon, 27 Jan 2020 15:09:17 -0500 Subject: [PATCH 4/9] Parameterizes PNG target dimensions --- app/src/capture.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/src/capture.c b/app/src/capture.c index eb64dd80..8e2ea40d 100644 --- a/app/src/capture.c +++ b/app/src/capture.c @@ -81,22 +81,23 @@ static void notify_complete() { static bool in_frame_to_png( AVIOContext *output_context, AVCodecContext *codecCtx, AVFrame *inframe) { + int targetHeight = codecCtx->height; + int targetWidth = codecCtx->width; struct SwsContext * swCtx = sws_getContext(codecCtx->width, codecCtx->height, codecCtx->pix_fmt, - codecCtx->width, - codecCtx->height, + targetWidth, + targetHeight, AV_PIX_FMT_RGB24, SWS_FAST_BILINEAR, 0, 0, 0); AVFrame * rgbFrame = av_frame_alloc(); - LOGV("Image frame width: %d height: %d", codecCtx->width, codecCtx->height); - rgbFrame->width = codecCtx->width; - rgbFrame->height = codecCtx->height; + LOGV("Image frame width: %d height: %d scaling to %d x %d", codecCtx->width, codecCtx->height, targetWidth, targetHeight); + rgbFrame->width = targetWidth; + rgbFrame->height = targetHeight; rgbFrame->format = AV_PIX_FMT_RGB24; av_image_alloc( - rgbFrame->data, rgbFrame->linesize, codecCtx->width, - codecCtx->height, AV_PIX_FMT_RGB24, 1); + rgbFrame->data, rgbFrame->linesize, targetWidth, targetHeight, AV_PIX_FMT_RGB24, 1); sws_scale( swCtx, inframe->data, inframe->linesize, 0, inframe->height, rgbFrame->data, rgbFrame->linesize); @@ -112,8 +113,8 @@ static bool in_frame_to_png( return false; } - outCodecCtx->width = codecCtx->width; - outCodecCtx->height = codecCtx->height; + outCodecCtx->width = targetWidth; + outCodecCtx->height = targetHeight; outCodecCtx->pix_fmt = AV_PIX_FMT_RGB24; outCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO; if (codecCtx->time_base.num && codecCtx->time_base.num) { From 173dc8a3fbd7bfc58aada02326f9469ad80e129c Mon Sep 17 00:00:00 2001 From: Frank Leon Rose Date: Mon, 27 Jan 2020 15:11:04 -0500 Subject: [PATCH 5/9] Instrumented to observe timing of execution `gprof` did not yield insights of sufficient detail. --- app/src/capture.c | 17 ++++++++++++++++- app/src/main.c | 1 + app/src/scrcpy.c | 10 ++++++++++ app/src/server.c | 11 +++++++++++ app/src/stream.c | 4 +++- app/src/sys/unix/command.c | 21 +++++++++++++++++++++ app/src/util/log.h | 3 +++ 7 files changed, 65 insertions(+), 2 deletions(-) diff --git a/app/src/capture.c b/app/src/capture.c index 8e2ea40d..4d02bc6b 100644 --- a/app/src/capture.c +++ b/app/src/capture.c @@ -81,6 +81,7 @@ static void notify_complete() { static bool in_frame_to_png( AVIOContext *output_context, AVCodecContext *codecCtx, AVFrame *inframe) { + uint64_t start = get_timestamp(); int targetHeight = codecCtx->height; int targetWidth = codecCtx->width; struct SwsContext * swCtx = sws_getContext(codecCtx->width, @@ -102,6 +103,8 @@ static bool in_frame_to_png( swCtx, inframe->data, inframe->linesize, 0, inframe->height, rgbFrame->data, rgbFrame->linesize); + LOGV("Scaling image: %llu", get_timestamp() - start); + AVCodec *outCodec = avcodec_find_encoder(AV_CODEC_ID_PNG); if (!outCodec) { LOGE("Failed to find PNG codec"); @@ -122,7 +125,7 @@ static bool in_frame_to_png( outCodecCtx->time_base.den = codecCtx->time_base.den; } else { outCodecCtx->time_base.num = 1; - outCodecCtx->time_base.den = 30; // FPS + outCodecCtx->time_base.den = 10; // FPS } int ret = avcodec_open2(outCodecCtx, outCodec, NULL); @@ -145,11 +148,14 @@ static bool in_frame_to_png( ret = avcodec_receive_packet(outCodecCtx, &outPacket); avcodec_close(outCodecCtx); av_free(outCodecCtx); + LOGV("Converted to PNG: %llu", get_timestamp() - start); if (ret >= 0) { // Dump packet avio_write(output_context, outPacket.data, outPacket.size); notify_complete(); av_packet_unref(&outPacket); + LOGV("Wrote file: %llu", get_timestamp() - start); + log_timestamp("Capture written"); return true; } else { LOGE("Failed to receive packet"); @@ -267,6 +273,9 @@ void capture_destroy(struct capture *capture) { } static bool capture_process(struct capture *capture, const AVPacket *packet) { + log_timestamp("Processing packet"); + static uint64_t total = 0; + uint64_t start = get_timestamp(); if (capture->finished) { LOGV("Skipping redundant call to capture_push"); } else { @@ -276,6 +285,9 @@ static bool capture_process(struct capture *capture, const AVPacket *packet) { capture->finished = found_png; } } + uint64_t duration = get_timestamp() - start; + total += duration; + LOGV("Capture step microseconds: %llu total: %llu", duration, total); return capture->finished; } @@ -305,6 +317,7 @@ capture_packet_delete(struct capture_packet *rec) { static int run_capture(void *data) { + log_timestamp("Running capture thread"); struct capture *capture = data; for (;;) { @@ -341,6 +354,7 @@ run_capture(void *data) { bool capture_start(struct capture *capture) { + log_timestamp("Starting capture thread"); capture->thread = SDL_CreateThread(run_capture, "capture", capture); if (!capture->thread) { @@ -366,6 +380,7 @@ capture_join(struct capture *capture) { bool capture_push(struct capture *capture, const AVPacket *packet) { + log_timestamp("Received packet"); mutex_lock(capture->mutex); assert(!capture->stopped); diff --git a/app/src/main.c b/app/src/main.c index 5aea485d..5daa68f9 100644 --- a/app/src/main.c +++ b/app/src/main.c @@ -45,6 +45,7 @@ main(int argc, char *argv[]) { #ifndef NDEBUG SDL_LogSetAllPriority(SDL_LOG_PRIORITY_DEBUG); #endif + SDL_LogSetAllPriority(SDL_LOG_PRIORITY_VERBOSE); struct scrcpy_cli_args args = { .opts = SCRCPY_OPTIONS_DEFAULT, diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 6fe2965a..4563f295 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -282,6 +282,8 @@ av_log_callback(void *avcl, int level, const char *fmt, va_list vl) { bool scrcpy(const struct scrcpy_options *options) { + log_timestamp("scrcpy Started"); + bool captures = !!options->capture_filename; bool record = !!options->record_filename; struct server_params params = { @@ -315,10 +317,12 @@ scrcpy(const struct scrcpy_options *options) { bool controller_initialized = false; bool controller_started = false; + log_timestamp("sdl_init_and_configure"); if (!sdl_init_and_configure(options->display)) { goto end; } + log_timestamp("server_connect_to"); if (!server_connect_to(&server)) { goto end; } @@ -329,12 +333,14 @@ scrcpy(const struct scrcpy_options *options) { // screenrecord does not send frames when the screen content does not // change therefore, we transmit the screen size before the video stream, // to be able to init the window immediately + log_timestamp("device_read_info"); if (!device_read_info(server.video_socket, device_name, &frame_size)) { goto end; } struct decoder *dec = NULL; if (options->display) { + log_timestamp("Display and fps_counter_init"); if (!fps_counter_init(&fps_counter)) { goto end; } @@ -380,6 +386,7 @@ scrcpy(const struct scrcpy_options *options) { capture_initialized = true; } + log_timestamp("stream_init"); av_log_set_callback(av_log_callback); stream_init(&stream, server.video_socket, dec, rec, cap); @@ -437,9 +444,12 @@ scrcpy(const struct scrcpy_options *options) { input_manager.prefer_text = options->prefer_text; + log_timestamp("Starting event_loop"); ret = event_loop(options->display, options->control); + log_timestamp("Terminated event_loop"); LOGD("quit..."); + log_timestamp("stream_init"); screen_destroy(&screen); end: diff --git a/app/src/server.c b/app/src/server.c index ff167aeb..2c9bbbd9 100644 --- a/app/src/server.c +++ b/app/src/server.c @@ -121,6 +121,7 @@ disable_tunnel(struct server *server) { static process_t execute_server(struct server *server, const struct server_params *params) { + log_timestamp("Starting device server"); char max_size_string[6]; char bit_rate_string[11]; char max_fps_string[6]; @@ -158,6 +159,7 @@ execute_server(struct server *server, const struct server_params *params) { // Port: 5005 // Then click on "Debug" #endif + LOGV("Executing ADB command"); return adb_execute(server->serial, cmd, sizeof(cmd) / sizeof(cmd[0])); } @@ -231,11 +233,13 @@ server_start(struct server *server, const char *serial, } if (!push_server(serial)) { + LOGE("Failed to push server to device."); SDL_free(server->serial); return false; } if (!enable_tunnel(server)) { + LOGE("Failed to enable tunnel."); SDL_free(server->serial); return false; } @@ -259,6 +263,7 @@ server_start(struct server *server, const char *serial, } } + LOGV("Executing server"); // server will connect to our server socket server->process = execute_server(server, params); @@ -278,12 +283,15 @@ server_start(struct server *server, const char *serial, bool server_connect_to(struct server *server) { + log_timestamp("Starting server_connect_to"); if (!server->tunnel_forward) { + log_timestamp("Accepting video socket"); server->video_socket = net_accept(server->server_socket); if (server->video_socket == INVALID_SOCKET) { return false; } + log_timestamp("Accepting control socket"); server->control_socket = net_accept(server->server_socket); if (server->control_socket == INVALID_SOCKET) { // the video_socket will be cleaned up on destroy @@ -295,6 +303,7 @@ server_connect_to(struct server *server) { } else { uint32_t attempts = 100; uint32_t delay = 100; // ms + log_timestamp("Connecting video socket"); server->video_socket = connect_to_server(server->local_port, attempts, delay); if (server->video_socket == INVALID_SOCKET) { @@ -302,6 +311,7 @@ server_connect_to(struct server *server) { } // we know that the device is listening, we don't need several attempts + log_timestamp("Connecting control socket"); server->control_socket = net_connect(IPV4_LOCALHOST, server->local_port); if (server->control_socket == INVALID_SOCKET) { @@ -313,6 +323,7 @@ server_connect_to(struct server *server) { disable_tunnel(server); // ignore failure server->tunnel_enabled = false; + log_timestamp("Finished server_connect_to"); return true; } diff --git a/app/src/stream.c b/app/src/stream.c index 974bcf52..6770cbc5 100644 --- a/app/src/stream.c +++ b/app/src/stream.c @@ -60,6 +60,8 @@ stream_recv_packet(struct stream *stream, AVPacket *packet) { packet->pts = pts != NO_PTS ? (int64_t) pts : AV_NOPTS_VALUE; + log_timestamp("Received packet via socket stream"); + return true; } @@ -308,7 +310,7 @@ stream_init(struct stream *stream, socket_t socket, bool stream_start(struct stream *stream) { - LOGD("Starting stream thread"); + log_timestamp("Starting stream thread"); stream->thread = SDL_CreateThread(run_stream, "stream", stream); if (!stream->thread) { diff --git a/app/src/sys/unix/command.c b/app/src/sys/unix/command.c index fbcf2355..a3a62d21 100644 --- a/app/src/sys/unix/command.c +++ b/app/src/sys/unix/command.c @@ -14,16 +14,35 @@ #include #include #include +#include +#include #include #include #include #include "util/log.h" +uint64_t get_timestamp() { + struct timeval tv; + gettimeofday(&tv,NULL); + return tv.tv_sec*(uint64_t)1000000+tv.tv_usec; +} + +static uint64_t StartTime = 0; +void log_timestamp(const char *tag) { + uint64_t now = get_timestamp(); + if (StartTime==0) { + StartTime = now; + } + LOGV("Timestamp: %llu ms %s", (now-StartTime) / 1000, tag); +} + enum process_result cmd_execute(const char *const argv[], pid_t *pid) { int fd[2]; + uint64_t start = get_timestamp(); + if (pipe(fd) == -1) { perror("pipe"); return PROCESS_ERROR_GENERIC; @@ -31,6 +50,7 @@ cmd_execute(const char *const argv[], pid_t *pid) { enum process_result ret = PROCESS_SUCCESS; + log_timestamp("Running command"); *pid = fork(); if (*pid == -1) { perror("fork"); @@ -48,6 +68,7 @@ cmd_execute(const char *const argv[], pid_t *pid) { ret = PROCESS_ERROR_GENERIC; goto end; } + LOGV("Command microseconds: %s %llu ms", argv[0], (get_timestamp()-start)/1000); } else if (*pid == 0) { // child close read side close(fd[0]); diff --git a/app/src/util/log.h b/app/src/util/log.h index 5955c7fb..eaacf949 100644 --- a/app/src/util/log.h +++ b/app/src/util/log.h @@ -10,4 +10,7 @@ #define LOGE(...) SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, __VA_ARGS__) #define LOGC(...) SDL_LogCritical(SDL_LOG_CATEGORY_APPLICATION, __VA_ARGS__) +#include +uint64_t get_timestamp(); +void log_timestamp(const char *tag); #endif From 48f1c6c2d69a9c2db9f7fa01890d3642da2bf38f Mon Sep 17 00:00:00 2001 From: Frank Leon Rose Date: Mon, 5 Apr 2021 15:55:14 +0100 Subject: [PATCH 6/9] Fix compiler warning --- app/src/capture.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/src/capture.c b/app/src/capture.c index 4d02bc6b..960cdc3e 100644 --- a/app/src/capture.c +++ b/app/src/capture.c @@ -100,7 +100,10 @@ static bool in_frame_to_png( av_image_alloc( rgbFrame->data, rgbFrame->linesize, targetWidth, targetHeight, AV_PIX_FMT_RGB24, 1); sws_scale( - swCtx, inframe->data, inframe->linesize, 0, + swCtx, + // const_cast(inframe->data) + (const uint8_t * const*)(inframe->data), + inframe->linesize, 0, inframe->height, rgbFrame->data, rgbFrame->linesize); LOGV("Scaling image: %llu", get_timestamp() - start); From 0ddd0b366cffaf86f5208c1ceee510f34d307b97 Mon Sep 17 00:00:00 2001 From: Frank Leon Rose Date: Mon, 5 Apr 2021 15:55:47 +0100 Subject: [PATCH 7/9] Fix log message to show source and dest sizes --- app/src/capture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/capture.c b/app/src/capture.c index 960cdc3e..60c3e800 100644 --- a/app/src/capture.c +++ b/app/src/capture.c @@ -93,7 +93,7 @@ static bool in_frame_to_png( SWS_FAST_BILINEAR, 0, 0, 0); AVFrame * rgbFrame = av_frame_alloc(); - LOGV("Image frame width: %d height: %d scaling to %d x %d", codecCtx->width, codecCtx->height, targetWidth, targetHeight); + LOGV("Image frame width: %d height: %d scaling to %d x %d", *inframe->linesize, inframe->height, targetWidth, targetHeight); rgbFrame->width = targetWidth; rgbFrame->height = targetHeight; rgbFrame->format = AV_PIX_FMT_RGB24; From 6fd343d7f99e53cabb0944f47269b2303215693a Mon Sep 17 00:00:00 2001 From: Frank Leon Rose Date: Mon, 5 Apr 2021 15:56:27 +0100 Subject: [PATCH 8/9] Add note about better quality image scaling --- app/src/capture.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/src/capture.c b/app/src/capture.c index 60c3e800..a4805927 100644 --- a/app/src/capture.c +++ b/app/src/capture.c @@ -90,7 +90,11 @@ static bool in_frame_to_png( targetWidth, targetHeight, AV_PIX_FMT_RGB24, - SWS_FAST_BILINEAR, 0, 0, 0); + // NOTE(frankleonrose): These options offer better picture + // quality in case of scaling, according to https://stackoverflow.com/a/46169884 + // SWS_BILINEAR | SWS_FULL_CHR_H_INT | SWS_ACCURATE_RND, + SWS_FAST_BILINEAR, + 0, 0, 0); AVFrame * rgbFrame = av_frame_alloc(); LOGV("Image frame width: %d height: %d scaling to %d x %d", *inframe->linesize, inframe->height, targetWidth, targetHeight); From 5bda0527a9b8feb0314d19633756d58958e0ddbb Mon Sep 17 00:00:00 2001 From: Frank Leon Rose Date: Mon, 5 Apr 2021 16:39:16 +0100 Subject: [PATCH 9/9] Add "(with Mobot extensions)" to reported version --- app/src/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main.c b/app/src/main.c index 49910ffa..50505dd8 100644 --- a/app/src/main.c +++ b/app/src/main.c @@ -15,7 +15,7 @@ static void print_version(void) { - fprintf(stderr, "scrcpy %s\n\n", SCRCPY_VERSION); + fprintf(stderr, "scrcpy %s (with Mobot extensions)\n\n", SCRCPY_VERSION); fprintf(stderr, "dependencies:\n"); fprintf(stderr, " - SDL %d.%d.%d\n", SDL_MAJOR_VERSION, SDL_MINOR_VERSION,