From 03cc0cf5431538c9b83e656f0f15c4f21d84803f Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 12 Feb 2021 22:34:34 +0100 Subject: [PATCH] swscalewip --- app/meson.build | 1 + app/src/cli.c | 43 ++++++++++++- app/src/frame_texture.c | 131 +++++++++++++++++++++++++++++++++++++--- app/src/frame_texture.h | 8 ++- app/src/scrcpy.c | 20 ++++++ app/src/scrcpy.h | 14 +++++ app/src/screen.c | 3 +- 7 files changed, 208 insertions(+), 12 deletions(-) diff --git a/app/meson.build b/app/meson.build index 6a36c2c1..3ed15855 100644 --- a/app/meson.build +++ b/app/meson.build @@ -47,6 +47,7 @@ if not get_option('crossbuild_windows') dependency('libavformat'), dependency('libavcodec'), dependency('libavutil'), + dependency('libswscale'), dependency('sdl2'), ] diff --git a/app/src/cli.c b/app/src/cli.c index d9ba6e30..1d08e03a 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -643,6 +643,7 @@ guess_record_format(const char *filename) { static bool parse_scale_filter(const char *optarg, enum sc_scale_filter *filter) { + // TODO store in a map and loop over the entries instead if (!strcmp(optarg, "none")) { *filter = SC_SCALE_FILTER_NONE; return true; @@ -651,8 +652,48 @@ parse_scale_filter(const char *optarg, enum sc_scale_filter *filter) { *filter = SC_SCALE_FILTER_TRILINEAR; return true; } + if (!strcmp(optarg, "bilinear")) { + *filter = SC_SCALE_FILTER_BILINEAR; + return true; + } + if (!strcmp(optarg, "bicubic")) { + *filter = SC_SCALE_FILTER_BICUBIC; + return true; + } + if (!strcmp(optarg, "x")) { + *filter = SC_SCALE_FILTER_X; + return true; + } + if (!strcmp(optarg, "point")) { + *filter = SC_SCALE_FILTER_POINT; + return true; + } + if (!strcmp(optarg, "area")) { + *filter = SC_SCALE_FILTER_AREA; + return true; + } + if (!strcmp(optarg, "bicublin")) { + *filter = SC_SCALE_FILTER_BICUBLIN; + return true; + } + if (!strcmp(optarg, "gauss")) { + *filter = SC_SCALE_FILTER_GAUSS; + return true; + } + if (!strcmp(optarg, "sinc")) { + *filter = SC_SCALE_FILTER_SINC; + return true; + } + if (!strcmp(optarg, "lanczos")) { + *filter = SC_SCALE_FILTER_LANCZOS; + return true; + } + if (!strcmp(optarg, "spline")) { + *filter = SC_SCALE_FILTER_SPLINE; + return true; + } LOGE("Unsupported scale filter: %s " - "(expected \"none\" or \"trilinear\")", optarg); + "(expected one of [TODO])", optarg); return false; } diff --git a/app/src/frame_texture.c b/app/src/frame_texture.c index 7baf89e2..b7faabce 100644 --- a/app/src/frame_texture.c +++ b/app/src/frame_texture.c @@ -1,11 +1,24 @@ #include "frame_texture.h" +#include +#include +#include + #include "util/log.h" +static inline bool +is_swscale_enabled(enum sc_scale_filter scale_filter) { + return scale_filter != SC_SCALE_FILTER_NONE + && scale_filter != SC_SCALE_FILTER_TRILINEAR; +} + static SDL_Texture * create_texture(struct sc_frame_texture *ftex, struct size size) { SDL_Renderer *renderer = ftex->renderer; - SDL_Texture *texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_YV12, + SDL_PixelFormatEnum fmt = is_swscale_enabled(ftex->scale_filter) + ? SDL_PIXELFORMAT_RGB24 + : SDL_PIXELFORMAT_YV12; + SDL_Texture *texture = SDL_CreateTexture(renderer, fmt, SDL_TEXTUREACCESS_STREAMING, size.width, size.height); if (!texture) { @@ -51,20 +64,18 @@ sc_frame_texture_init(struct sc_frame_texture *ftex, SDL_Renderer *renderer, bool supports_mipmaps = sc_opengl_version_at_least(gl, 3, 0, /* OpenGL 3.0+ */ 2, 0 /* OpenGL ES 2.0+ */); - if (supports_mipmaps) { - LOGI("Trilinear filtering enabled"); - } else { + if (!supports_mipmaps) { LOGW("Trilinear filtering disabled " "(OpenGL 3.0+ or ES 2.0+ required)"); - scale_filter = SC_SCALE_FILTER_NONE; + ftex->scale_filter = SC_SCALE_FILTER_NONE; } - } else { - LOGI("Trilinear filtering disabled"); } } else if (scale_filter == SC_SCALE_FILTER_TRILINEAR) { LOGD("Trilinear filtering disabled (not an OpenGL renderer)"); } + LOGD("Scale filter: %s\n", sc_scale_filter_name(ftex->scale_filter)); + LOGI("Initial texture: %" PRIu16 "x%" PRIu16, initial_size.width, initial_size.height); ftex->renderer = renderer; @@ -76,6 +87,9 @@ sc_frame_texture_init(struct sc_frame_texture *ftex, SDL_Renderer *renderer, } ftex->texture_size = initial_size; + ftex->decoded_frame = NULL; + memset(ftex->data, 0, sizeof(ftex->data)); + memset(ftex->linesize, 0, sizeof(ftex->linesize)); return true; } @@ -87,8 +101,60 @@ sc_frame_texture_destroy(struct sc_frame_texture *ftex) { } } -bool -sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame) { +static int +to_sws_filter(enum sc_scale_filter scale_filter) { + switch (scale_filter) { + case SC_SCALE_FILTER_BILINEAR: return SWS_BILINEAR; + case SC_SCALE_FILTER_BICUBIC: return SWS_BICUBIC; + case SC_SCALE_FILTER_X: return SWS_X; + case SC_SCALE_FILTER_POINT: return SWS_POINT; + case SC_SCALE_FILTER_AREA: return SWS_AREA; + case SC_SCALE_FILTER_BICUBLIN: return SWS_BICUBLIN; + case SC_SCALE_FILTER_GAUSS: return SWS_GAUSS; + case SC_SCALE_FILTER_SINC: return SWS_SINC; + case SC_SCALE_FILTER_LANCZOS: return SWS_LANCZOS; + case SC_SCALE_FILTER_SPLINE: return SWS_SPLINE; + default: assert(!"unsupported filter"); + } +} + +static bool +screen_generate_resized_frame(struct sc_frame_texture *ftex, + struct size target_size) { + assert(is_swscale_enabled(ftex->scale_filter)); + // TODO + + if (ftex->data[0]) { + av_freep(&ftex->data[0]); + } + + int ret = av_image_alloc(ftex->data, ftex->linesize, target_size.width, + target_size.height, AV_PIX_FMT_RGB24, 16); + if (ret < 0) { + return false; + } + + const AVFrame *input = ftex->decoded_frame; + + int flags = to_sws_filter(ftex->scale_filter); + struct SwsContext *ctx = + sws_getContext(input->width, input->height, AV_PIX_FMT_YUV420P, + target_size.width, target_size.height, AV_PIX_FMT_RGB24, + flags, NULL, NULL, NULL); + if (!ctx) { + return false; + } + + sws_scale(ctx, (const uint8_t *const *) input->data, input->linesize, 0, + input->height, ftex->data, ftex->linesize); + sws_freeContext(ctx); + + return true; +} + +static bool +sc_frame_texture_update_direct(struct sc_frame_texture *ftex, + const AVFrame *frame) { struct size frame_size = {frame->width, frame->height}; if (ftex->texture_size.width != frame_size.width || ftex->texture_size.height != frame_size.height) { @@ -119,3 +185,50 @@ sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame) { return true; } + +static bool +sc_frame_texture_update_swscale(struct sc_frame_texture *ftex, + const AVFrame *frame, struct size target_size) { + if (ftex->texture_size.width != target_size.width + || ftex->texture_size.height != target_size.height) { + // Frame dimensions changed, destroy texture + SDL_DestroyTexture(ftex->texture); + + ftex->texture = create_texture(ftex, target_size); + if (!ftex->texture) { + LOGC("Could not create texture: %s", SDL_GetError()); + return false; + } + + ftex->texture_size = target_size; + } + + // The frame is valid until the next call to + // video_buffer_take_rendering_frame() + ftex->decoded_frame = frame; + + bool ok = screen_generate_resized_frame(ftex, target_size); + if (!ok) { + LOGE("Failed to resize frame\n"); + return false; + } + + SDL_UpdateTexture(ftex->texture, NULL, ftex->data[0], ftex->linesize[0]); + + if (ftex->scale_filter == SC_SCALE_FILTER_TRILINEAR) { + SDL_GL_BindTexture(ftex->texture, NULL, NULL); + ftex->gl.GenerateMipmap(GL_TEXTURE_2D); + SDL_GL_UnbindTexture(ftex->texture); + } + + return true; +} + +bool +sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame, + struct size target_size) { + if (is_swscale_enabled(ftex->scale_filter)) { + return sc_frame_texture_update_swscale(ftex, frame, target_size); + } + return sc_frame_texture_update_direct(ftex, frame); +} diff --git a/app/src/frame_texture.h b/app/src/frame_texture.h index 773621be..1c54dcdb 100644 --- a/app/src/frame_texture.h +++ b/app/src/frame_texture.h @@ -19,6 +19,11 @@ struct sc_frame_texture { SDL_Texture *texture; struct size texture_size; + + // For swscaling + const AVFrame *decoded_frame; // owned by the video_buffer + uint8_t *data[4]; + int linesize[4]; }; bool @@ -30,6 +35,7 @@ void sc_frame_texture_destroy(struct sc_frame_texture *ftex); bool -sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame); +sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame, + struct size target_size); #endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 82b65018..090883d3 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -29,6 +29,26 @@ #include "util/log.h" #include "util/net.h" +static const char *sc_scale_filter_names[SC_SCALE_FILTER__COUNT] = { + [SC_SCALE_FILTER_NONE] = "none", + [SC_SCALE_FILTER_TRILINEAR] = "trilinear", + [SC_SCALE_FILTER_BILINEAR] = "bilinear", + [SC_SCALE_FILTER_BICUBIC] = "bicubic", + [SC_SCALE_FILTER_X] = "x", + [SC_SCALE_FILTER_POINT] = "point", + [SC_SCALE_FILTER_AREA] = "area", + [SC_SCALE_FILTER_BICUBLIN] = "bicublin", + [SC_SCALE_FILTER_GAUSS] = "gauss", + [SC_SCALE_FILTER_SINC] = "sinc", + [SC_SCALE_FILTER_LANCZOS] = "lanczos", + [SC_SCALE_FILTER_SPLINE] = "spline", +}; + +const char * +sc_scale_filter_name(enum sc_scale_filter scale_filter) { + return sc_scale_filter_names[scale_filter]; +} + static struct server server; static struct screen screen = SCREEN_INITIALIZER; static struct fps_counter fps_counter; diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index a89fa59c..ce9af7a2 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -44,8 +44,22 @@ struct sc_port_range { enum sc_scale_filter { SC_SCALE_FILTER_NONE, SC_SCALE_FILTER_TRILINEAR, // mipmaps + SC_SCALE_FILTER_BILINEAR, + SC_SCALE_FILTER_BICUBIC, + SC_SCALE_FILTER_X, + SC_SCALE_FILTER_POINT, + SC_SCALE_FILTER_AREA, + SC_SCALE_FILTER_BICUBLIN, + SC_SCALE_FILTER_GAUSS, + SC_SCALE_FILTER_SINC, + SC_SCALE_FILTER_LANCZOS, + SC_SCALE_FILTER_SPLINE, + SC_SCALE_FILTER__COUNT, }; +const char * +sc_scale_filter_name(enum sc_scale_filter scale_filter); + #define SC_WINDOW_POSITION_UNDEFINED (-0x8000) struct scrcpy_options { diff --git a/app/src/screen.c b/app/src/screen.c index 90b0c196..ebbfe7df 100644 --- a/app/src/screen.c +++ b/app/src/screen.c @@ -363,7 +363,8 @@ screen_update_frame(struct screen *screen, struct video_buffer *vb) { struct size new_frame_size = {frame->width, frame->height}; prepare_for_frame(screen, new_frame_size); - bool ok = sc_frame_texture_update(&screen->ftex, frame); + struct size rect_size = {screen->rect.w, screen->rect.h}; + bool ok = sc_frame_texture_update(&screen->ftex, frame, rect_size); if (!ok) { return false; }