audio_player WIP

app/meson.build

@@ -4,6 +4,7 @@ src = [
     'src/adb/adb_device.c',
     'src/adb/adb_parser.c',
     'src/adb/adb_tunnel.c',
+    'src/audio_player.c',
     'src/cli.c',
     'src/clock.c',
     'src/compat.c',

app/src/audio_player.c

@@ -0,0 +1,150 @@
+#include "audio_player.h"
+
+#include "util/log.h"
+
+/** Downcast frame_sink to sc_v4l2_sink */
+#define DOWNCAST(SINK) container_of(SINK, struct sc_audio_player, frame_sink)
+
+void
+sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) {
+    struct sc_audio_player *ap = userdata;
+
+    // This callback is called with the lock used by SDL_AudioDeviceLock(), so
+    // the bytebuf is protected
+
+    assert(len_int > 0);
+    size_t len = len_int;
+
+    size_t read = sc_bytebuf_read_remaining(&ap->buf);
+    if (read) {
+        if (read > len) {
+            read = len;
+        }
+        sc_bytebuf_read(&ap->buf, stream, read);
+    }
+    if (read < len) {
+        // Insert silence
+        memset(stream + read, 0, len - read);
+    }
+}
+
+static SDL_AudioFormat
+sc_audio_player_ffmpeg_to_sdl_format(enum AVSampleFormat format) {
+    switch (format) {
+        case AV_SAMPLE_FMT_S16:
+            return AUDIO_S16;
+        case AV_SAMPLE_FMT_S32:
+            return AUDIO_S32;
+        case AV_SAMPLE_FMT_FLT:
+            return AUDIO_F32;
+        default:
+            LOGE("Unsupported FFmpeg sample format: %s",
+                 av_get_sample_fmt_name(format));
+            return 0;
+    }
+}
+
+static bool
+sc_audio_player_frame_sink_open(struct sc_frame_sink *sink,
+                                const AVCodecContext *ctx) {
+    struct sc_audio_player *ap = DOWNCAST(sink);
+
+    SDL_AudioFormat format =
+        sc_audio_player_ffmpeg_to_sdl_format(ctx->sample_fmt);
+    if (!format) {
+        // error already logged
+        //return false;
+        format = AUDIO_F32; // it's planar, but for now there is only 1 channel
+    }
+    LOGI("%d\n", ctx->sample_rate);
+
+    SDL_AudioSpec desired = {
+        .freq = ctx->sample_rate,
+        .format = format,
+        .channels = 1,
+        .samples = 2048,
+        .callback = sc_audio_player_sdl_callback,
+        .userdata = ap,
+    };
+    SDL_AudioSpec obtained;
+
+    ap->device = SDL_OpenAudioDevice(NULL, 0, &desired, &obtained, 0);
+    if (!ap->device) {
+        LOGE("Could not open audio device: %s", SDL_GetError());
+        return false;
+    }
+
+    SDL_PauseAudioDevice(ap->device, 0);
+
+    return true;
+}
+
+static void
+sc_audio_player_frame_sink_close(struct sc_frame_sink *sink) {
+    struct sc_audio_player *ap = DOWNCAST(sink);
+
+    assert(ap->device);
+    SDL_PauseAudioDevice(ap->device, 1);
+    SDL_CloseAudioDevice(ap->device);
+}
+
+static bool
+sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) {
+    struct sc_audio_player *ap = DOWNCAST(sink);
+
+    const uint8_t *data = frame->data[0];
+    size_t size = frame->linesize[0];
+
+    // TODO convert to non planar format
+    // TODO then re-enable stereo
+    // TODO clock drift compensation
+
+    // It should almost always be possible to write without lock
+    bool can_write_without_lock = size <= ap->safe_empty_buffer;
+    if (can_write_without_lock) {
+        sc_bytebuf_prepare_write(&ap->buf, data, size);
+    }
+
+    SDL_LockAudioDevice(ap->device);
+    if (can_write_without_lock) {
+        sc_bytebuf_commit_write(&ap->buf, size);
+    } else {
+        sc_bytebuf_write(&ap->buf, data, size);
+    }
+
+    // The next time, it will remain at least the current empty space
+    ap->safe_empty_buffer = sc_bytebuf_write_remaining(&ap->buf);
+    SDL_UnlockAudioDevice(ap->device);
+
+    return true;
+}
+
+bool
+sc_audio_player_init(struct sc_audio_player *ap,
+                     const struct sc_audio_player_callbacks *cbs,
+                     void *cbs_userdata) {
+    bool ok = sc_bytebuf_init(&ap->buf, 128 * 1024);
+    if (!ok) {
+        return false;
+    }
+
+    ap->safe_empty_buffer = sc_bytebuf_write_remaining(&ap->buf);
+
+    assert(cbs && cbs->on_ended);
+    ap->cbs = cbs;
+    ap->cbs_userdata = cbs_userdata;
+
+    static const struct sc_frame_sink_ops ops = {
+        .open = sc_audio_player_frame_sink_open,
+        .close = sc_audio_player_frame_sink_close,
+        .push = sc_audio_player_frame_sink_push,
+    };
+
+    ap->frame_sink.ops = &ops;
+    return true;
+}
+
+void
+sc_audio_player_destroy(struct sc_audio_player *ap) {
+    sc_bytebuf_destroy(&ap->buf);
+}

app/src/audio_player.h

@@ -0,0 +1,40 @@
+#ifndef SC_AUDIO_PLAYER_H
+#define SC_AUDIO_PLAYER_H
+
+#include "common.h"
+
+#include <stdbool.h>
+#include "trait/frame_sink.h"
+#include <util/bytebuf.h>
+#include <util/thread.h>
+
+#include <libavformat/avformat.h>
+#include <SDL2/SDL.h>
+
+struct sc_audio_player {
+    struct sc_frame_sink frame_sink;
+
+    SDL_AudioDeviceID device;
+
+    // protected by SDL_AudioDeviceLock()
+    struct sc_bytebuf buf;
+    // Number of bytes which could be written without locking
+    size_t safe_empty_buffer;
+
+    const struct sc_audio_player_callbacks *cbs;
+    void *cbs_userdata;
+};
+
+struct sc_audio_player_callbacks {
+    void (*on_ended)(struct sc_audio_player *ap, bool success, void *userdata);
+};
+
+bool
+sc_audio_player_init(struct sc_audio_player *ap,
+                     const struct sc_audio_player_callbacks *cbs,
+                     void *cbs_userdata);
+
+void
+sc_audio_player_destroy(struct sc_audio_player *ap);
+
+#endif

app/src/scrcpy.c

@@ -13,6 +13,7 @@
 # include <windows.h>
 #endif
 
+#include "audio_player.h"
 #include "controller.h"
 #include "decoder.h"
 #include "demuxer.h"
@@ -40,6 +41,7 @@
 struct scrcpy {
     struct sc_server server;
     struct sc_screen screen;
+    struct sc_audio_player audio_player;
     struct sc_demuxer video_demuxer;
     struct sc_demuxer audio_demuxer;
     struct sc_decoder video_decoder;
@@ -215,6 +217,17 @@ sc_recorder_on_ended(struct sc_recorder *recorder, bool success,
     }
 }
 
+static void
+sc_audio_player_on_ended(struct sc_audio_player *ap, bool success,
+                         void *userdata) {
+    (void) ap;
+    (void) userdata;
+
+    if (!success) {
+        // TODO
+    }
+}
+
 static void
 sc_video_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos,
                           void *userdata) {
@@ -301,6 +314,7 @@ scrcpy(struct scrcpy_options *options) {
     bool file_pusher_initialized = false;
     bool recorder_initialized = false;
     bool recorder_started = false;
+    bool audio_player_initialized = false;
 #ifdef HAVE_V4L2
     bool v4l2_sink_initialized = false;
 #endif
@@ -383,9 +397,16 @@ scrcpy(struct scrcpy_options *options) {
     }
 
     // Initialize SDL video in addition if display is enabled
-    if (options->display && SDL_Init(SDL_INIT_VIDEO)) {
+    if (options->display) {
-        LOGE("Could not initialize SDL: %s", SDL_GetError());
+        if (SDL_Init(SDL_INIT_VIDEO)) {
-        goto end;
+            LOGE("Could not initialize SDL video: %s", SDL_GetError());
+            goto end;
+        }
+
+        if (options->audio && SDL_Init(SDL_INIT_AUDIO)) {
+            LOGE("Could not initialize SDL audio: %s", SDL_GetError());
+            goto end;
+        }
     }
 
     sdl_configure(options->display, options->disable_screensaver);
@@ -663,6 +684,19 @@ aoa_hid_end:
         screen_initialized = true;
 
         sc_decoder_add_sink(&s->video_decoder, &s->screen.frame_sink);
+
+        if (options->audio) {
+            static const struct sc_audio_player_callbacks audio_player_cbs = {
+                .on_ended = sc_audio_player_on_ended,
+            };
+            if (!sc_audio_player_init(&s->audio_player,
+                                      &audio_player_cbs, NULL)) {
+                goto end;
+            }
+            audio_player_initialized = true;
+
+            sc_decoder_add_sink(&s->audio_decoder, &s->audio_player.frame_sink);
+        }
     }
 
 #ifdef HAVE_V4L2
@@ -783,6 +817,10 @@ end:
         sc_recorder_destroy(&s->recorder);
     }
 
+    if (audio_player_initialized) {
+        sc_audio_player_destroy(&s->audio_player);
+    }
+
     if (file_pusher_initialized) {
         sc_file_pusher_join(&s->file_pusher);
         sc_file_pusher_destroy(&s->file_pusher);

server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java

@@ -40,7 +40,7 @@ public final class AudioEncoder {
     }
 
     private static final int SAMPLE_RATE = 48000;
-    private static final int CHANNELS = 2;
+    private static final int CHANNELS = 1;
 
     private static final int BUFFER_MS = 10; // milliseconds
     private static final int BUFFER_SIZE = SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000;