swscalewip

app/meson.build

@@ -47,6 +47,7 @@ if not get_option('crossbuild_windows')
         dependency('libavformat'),
         dependency('libavcodec'),
         dependency('libavutil'),
+        dependency('libswscale'),
         dependency('sdl2'),
     ]
 

app/src/cli.c

@@ -643,6 +643,7 @@ guess_record_format(const char *filename) {
 
 static bool
 parse_scale_filter(const char *optarg, enum sc_scale_filter *filter) {
+    // TODO store in a map and loop over the entries instead
     if (!strcmp(optarg, "none")) {
         *filter = SC_SCALE_FILTER_NONE;
         return true;
@@ -651,8 +652,48 @@ parse_scale_filter(const char *optarg, enum sc_scale_filter *filter) {
         *filter = SC_SCALE_FILTER_TRILINEAR;
         return true;
     }
+    if (!strcmp(optarg, "bilinear")) {
+        *filter = SC_SCALE_FILTER_BILINEAR;
+        return true;
+    }
+    if (!strcmp(optarg, "bicubic")) {
+        *filter = SC_SCALE_FILTER_BICUBIC;
+        return true;
+    }
+    if (!strcmp(optarg, "x")) {
+        *filter = SC_SCALE_FILTER_X;
+        return true;
+    }
+    if (!strcmp(optarg, "point")) {
+        *filter = SC_SCALE_FILTER_POINT;
+        return true;
+    }
+    if (!strcmp(optarg, "area")) {
+        *filter = SC_SCALE_FILTER_AREA;
+        return true;
+    }
+    if (!strcmp(optarg, "bicublin")) {
+        *filter = SC_SCALE_FILTER_BICUBLIN;
+        return true;
+    }
+    if (!strcmp(optarg, "gauss")) {
+        *filter = SC_SCALE_FILTER_GAUSS;
+        return true;
+    }
+    if (!strcmp(optarg, "sinc")) {
+        *filter = SC_SCALE_FILTER_SINC;
+        return true;
+    }
+    if (!strcmp(optarg, "lanczos")) {
+        *filter = SC_SCALE_FILTER_LANCZOS;
+        return true;
+    }
+    if (!strcmp(optarg, "spline")) {
+        *filter = SC_SCALE_FILTER_SPLINE;
+        return true;
+    }
     LOGE("Unsupported scale filter: %s "
-         "(expected \"none\" or \"trilinear\")", optarg);
+         "(expected one of [TODO])", optarg);
     return false;
 }
 

app/src/frame_texture.c

@@ -1,11 +1,24 @@
 #include "frame_texture.h"
 
+#include <assert.h>
+#include <libavutil/imgutils.h>
+#include <libswscale/swscale.h>
+
 #include "util/log.h"
 
+static inline bool
+is_swscale_enabled(enum sc_scale_filter scale_filter) {
+    return scale_filter != SC_SCALE_FILTER_NONE
+        && scale_filter != SC_SCALE_FILTER_TRILINEAR;
+}
+
 static SDL_Texture *
 create_texture(struct sc_frame_texture *ftex, struct size size) {
     SDL_Renderer *renderer = ftex->renderer;
-    SDL_Texture *texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_YV12,
+    SDL_PixelFormatEnum fmt = is_swscale_enabled(ftex->scale_filter)
+                            ? SDL_PIXELFORMAT_RGB24
+                            : SDL_PIXELFORMAT_YV12;
+    SDL_Texture *texture = SDL_CreateTexture(renderer, fmt,
                                              SDL_TEXTUREACCESS_STREAMING,
                                              size.width, size.height);
     if (!texture) {
@@ -51,20 +64,18 @@ sc_frame_texture_init(struct sc_frame_texture *ftex, SDL_Renderer *renderer,
             bool supports_mipmaps =
                 sc_opengl_version_at_least(gl, 3, 0, /* OpenGL 3.0+ */
                                                2, 0  /* OpenGL ES 2.0+ */);
-            if (supports_mipmaps) {
+            if (!supports_mipmaps) {
-                LOGI("Trilinear filtering enabled");
-            } else {
                 LOGW("Trilinear filtering disabled "
                      "(OpenGL 3.0+ or ES 2.0+ required)");
-                scale_filter = SC_SCALE_FILTER_NONE;
+                ftex->scale_filter = SC_SCALE_FILTER_NONE;
             }
-        } else {
-            LOGI("Trilinear filtering disabled");
         }
     } else if (scale_filter == SC_SCALE_FILTER_TRILINEAR) {
         LOGD("Trilinear filtering disabled (not an OpenGL renderer)");
     }
 
+    LOGD("Scale filter: %s\n", sc_scale_filter_name(ftex->scale_filter));
+
     LOGI("Initial texture: %" PRIu16 "x%" PRIu16, initial_size.width,
                                                   initial_size.height);
     ftex->renderer = renderer;
@@ -76,6 +87,9 @@ sc_frame_texture_init(struct sc_frame_texture *ftex, SDL_Renderer *renderer,
     }
 
     ftex->texture_size = initial_size;
+    ftex->decoded_frame = NULL;
+    memset(ftex->data, 0, sizeof(ftex->data));
+    memset(ftex->linesize, 0, sizeof(ftex->linesize));
 
     return true;
 }
@@ -87,8 +101,60 @@ sc_frame_texture_destroy(struct sc_frame_texture *ftex) {
     }
 }
 
-bool
+static int
-sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame) {
+to_sws_filter(enum sc_scale_filter scale_filter) {
+    switch (scale_filter) {
+        case SC_SCALE_FILTER_BILINEAR:  return SWS_BILINEAR;
+        case SC_SCALE_FILTER_BICUBIC:   return SWS_BICUBIC;
+        case SC_SCALE_FILTER_X:         return SWS_X;
+        case SC_SCALE_FILTER_POINT:     return SWS_POINT;
+        case SC_SCALE_FILTER_AREA:      return SWS_AREA;
+        case SC_SCALE_FILTER_BICUBLIN:  return SWS_BICUBLIN;
+        case SC_SCALE_FILTER_GAUSS:     return SWS_GAUSS;
+        case SC_SCALE_FILTER_SINC:      return SWS_SINC;
+        case SC_SCALE_FILTER_LANCZOS:   return SWS_LANCZOS;
+        case SC_SCALE_FILTER_SPLINE:    return SWS_SPLINE;
+        default: assert(!"unsupported filter");
+    }
+}
+
+static bool
+screen_generate_resized_frame(struct sc_frame_texture *ftex,
+                              struct size target_size) {
+    assert(is_swscale_enabled(ftex->scale_filter));
+    // TODO
+
+    if (ftex->data[0]) {
+        av_freep(&ftex->data[0]);
+    }
+
+    int ret = av_image_alloc(ftex->data, ftex->linesize, target_size.width,
+                             target_size.height, AV_PIX_FMT_RGB24, 16);
+    if (ret < 0) {
+        return false;
+    }
+
+    const AVFrame *input = ftex->decoded_frame;
+
+    int flags = to_sws_filter(ftex->scale_filter);
+    struct SwsContext *ctx =
+        sws_getContext(input->width, input->height, AV_PIX_FMT_YUV420P,
+                       target_size.width, target_size.height, AV_PIX_FMT_RGB24,
+                       flags, NULL, NULL, NULL);
+    if (!ctx) {
+        return false;
+    }
+
+    sws_scale(ctx, (const uint8_t *const *) input->data, input->linesize, 0,
+              input->height, ftex->data, ftex->linesize);
+    sws_freeContext(ctx);
+
+    return true;
+}
+
+static bool
+sc_frame_texture_update_direct(struct sc_frame_texture *ftex,
+                               const AVFrame *frame) {
     struct size frame_size = {frame->width, frame->height};
     if (ftex->texture_size.width != frame_size.width
             || ftex->texture_size.height != frame_size.height) {
@@ -119,3 +185,50 @@ sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame) {
 
     return true;
 }
+
+static bool
+sc_frame_texture_update_swscale(struct sc_frame_texture *ftex,
+                                const AVFrame *frame, struct size target_size) {
+    if (ftex->texture_size.width != target_size.width
+            || ftex->texture_size.height != target_size.height) {
+        // Frame dimensions changed, destroy texture
+        SDL_DestroyTexture(ftex->texture);
+
+        ftex->texture = create_texture(ftex, target_size);
+        if (!ftex->texture) {
+            LOGC("Could not create texture: %s", SDL_GetError());
+            return false;
+        }
+
+        ftex->texture_size = target_size;
+    }
+
+    // The frame is valid until the next call to
+    // video_buffer_take_rendering_frame()
+    ftex->decoded_frame = frame;
+
+    bool ok = screen_generate_resized_frame(ftex, target_size);
+    if (!ok) {
+        LOGE("Failed to resize frame\n");
+        return false;
+    }
+
+    SDL_UpdateTexture(ftex->texture, NULL, ftex->data[0], ftex->linesize[0]);
+
+    if (ftex->scale_filter == SC_SCALE_FILTER_TRILINEAR) {
+        SDL_GL_BindTexture(ftex->texture, NULL, NULL);
+        ftex->gl.GenerateMipmap(GL_TEXTURE_2D);
+        SDL_GL_UnbindTexture(ftex->texture);
+    }
+
+    return true;
+}
+
+bool
+sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame,
+                        struct size target_size) {
+    if (is_swscale_enabled(ftex->scale_filter)) {
+        return sc_frame_texture_update_swscale(ftex, frame, target_size);
+    }
+    return sc_frame_texture_update_direct(ftex, frame);
+}

app/src/frame_texture.h

@@ -19,6 +19,11 @@ struct sc_frame_texture {
 
     SDL_Texture *texture;
     struct size texture_size;
+
+    // For swscaling
+    const AVFrame *decoded_frame; // owned by the video_buffer
+    uint8_t *data[4];
+    int linesize[4];
 };
 
 bool
@@ -30,6 +35,7 @@ void
 sc_frame_texture_destroy(struct sc_frame_texture *ftex);
 
 bool
-sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame);
+sc_frame_texture_update(struct sc_frame_texture *ftex, const AVFrame *frame,
+                        struct size target_size);
 
 #endif

app/src/scrcpy.c

@@ -29,6 +29,26 @@
 #include "util/log.h"
 #include "util/net.h"
 
+static const char *sc_scale_filter_names[SC_SCALE_FILTER__COUNT] = {
+    [SC_SCALE_FILTER_NONE] = "none",
+    [SC_SCALE_FILTER_TRILINEAR] = "trilinear",
+    [SC_SCALE_FILTER_BILINEAR] = "bilinear",
+    [SC_SCALE_FILTER_BICUBIC] = "bicubic",
+    [SC_SCALE_FILTER_X] = "x",
+    [SC_SCALE_FILTER_POINT] = "point",
+    [SC_SCALE_FILTER_AREA] = "area",
+    [SC_SCALE_FILTER_BICUBLIN] = "bicublin",
+    [SC_SCALE_FILTER_GAUSS] = "gauss",
+    [SC_SCALE_FILTER_SINC] = "sinc",
+    [SC_SCALE_FILTER_LANCZOS] = "lanczos",
+    [SC_SCALE_FILTER_SPLINE] = "spline",
+};
+
+const char *
+sc_scale_filter_name(enum sc_scale_filter scale_filter) {
+    return sc_scale_filter_names[scale_filter];
+}
+
 static struct server server;
 static struct screen screen = SCREEN_INITIALIZER;
 static struct fps_counter fps_counter;

app/src/scrcpy.h

@@ -44,8 +44,22 @@ struct sc_port_range {
 enum sc_scale_filter {
     SC_SCALE_FILTER_NONE,
     SC_SCALE_FILTER_TRILINEAR, // mipmaps
+    SC_SCALE_FILTER_BILINEAR,
+    SC_SCALE_FILTER_BICUBIC,
+    SC_SCALE_FILTER_X,
+    SC_SCALE_FILTER_POINT,
+    SC_SCALE_FILTER_AREA,
+    SC_SCALE_FILTER_BICUBLIN,
+    SC_SCALE_FILTER_GAUSS,
+    SC_SCALE_FILTER_SINC,
+    SC_SCALE_FILTER_LANCZOS,
+    SC_SCALE_FILTER_SPLINE,
+    SC_SCALE_FILTER__COUNT,
 };
 
+const char *
+sc_scale_filter_name(enum sc_scale_filter scale_filter);
+
 #define SC_WINDOW_POSITION_UNDEFINED (-0x8000)
 
 struct scrcpy_options {

app/src/screen.c

@@ -363,7 +363,8 @@ screen_update_frame(struct screen *screen, struct video_buffer *vb) {
     struct size new_frame_size = {frame->width, frame->height};
     prepare_for_frame(screen, new_frame_size);
 
-    bool ok = sc_frame_texture_update(&screen->ftex, frame);
+    struct size rect_size = {screen->rect.w, screen->rect.h};
+    bool ok = sc_frame_texture_update(&screen->ftex, frame, rect_size);
     if (!ok) {
         return false;
     }