From a68500aa1306dd2ec320049319c0f12c4914e6dc Mon Sep 17 00:00:00 2001 From: Romain Vimont Date: Fri, 24 Feb 2023 21:29:10 +0100 Subject: [PATCH] audio_player WIP --- app/meson.build | 1 + app/src/audio_player.c | 150 ++++++++++++++++++ app/src/audio_player.h | 40 +++++ app/src/scrcpy.c | 44 ++++- .../com/genymobile/scrcpy/AudioEncoder.java | 2 +- 5 files changed, 233 insertions(+), 4 deletions(-) create mode 100644 app/src/audio_player.c create mode 100644 app/src/audio_player.h diff --git a/app/meson.build b/app/meson.build index aa4b1989..562a5358 100644 --- a/app/meson.build +++ b/app/meson.build @@ -4,6 +4,7 @@ src = [ 'src/adb/adb_device.c', 'src/adb/adb_parser.c', 'src/adb/adb_tunnel.c', + 'src/audio_player.c', 'src/cli.c', 'src/clock.c', 'src/compat.c', diff --git a/app/src/audio_player.c b/app/src/audio_player.c new file mode 100644 index 00000000..69e163e1 --- /dev/null +++ b/app/src/audio_player.c @@ -0,0 +1,150 @@ +#include "audio_player.h" + +#include "util/log.h" + +/** Downcast frame_sink to sc_v4l2_sink */ +#define DOWNCAST(SINK) container_of(SINK, struct sc_audio_player, frame_sink) + +void +sc_audio_player_sdl_callback(void *userdata, uint8_t *stream, int len_int) { + struct sc_audio_player *ap = userdata; + + // This callback is called with the lock used by SDL_AudioDeviceLock(), so + // the bytebuf is protected + + assert(len_int > 0); + size_t len = len_int; + + size_t read = sc_bytebuf_read_remaining(&ap->buf); + if (read) { + if (read > len) { + read = len; + } + sc_bytebuf_read(&ap->buf, stream, read); + } + if (read < len) { + // Insert silence + memset(stream + read, 0, len - read); + } +} + +static SDL_AudioFormat +sc_audio_player_ffmpeg_to_sdl_format(enum AVSampleFormat format) { + switch (format) { + case AV_SAMPLE_FMT_S16: + return AUDIO_S16; + case AV_SAMPLE_FMT_S32: + return AUDIO_S32; + case AV_SAMPLE_FMT_FLT: + return AUDIO_F32; + default: + LOGE("Unsupported FFmpeg sample format: %s", + av_get_sample_fmt_name(format)); + return 0; + } +} + +static bool +sc_audio_player_frame_sink_open(struct sc_frame_sink *sink, + const AVCodecContext *ctx) { + struct sc_audio_player *ap = DOWNCAST(sink); + + SDL_AudioFormat format = + sc_audio_player_ffmpeg_to_sdl_format(ctx->sample_fmt); + if (!format) { + // error already logged + //return false; + format = AUDIO_F32; // it's planar, but for now there is only 1 channel + } + LOGI("%d\n", ctx->sample_rate); + + SDL_AudioSpec desired = { + .freq = ctx->sample_rate, + .format = format, + .channels = 1, + .samples = 2048, + .callback = sc_audio_player_sdl_callback, + .userdata = ap, + }; + SDL_AudioSpec obtained; + + ap->device = SDL_OpenAudioDevice(NULL, 0, &desired, &obtained, 0); + if (!ap->device) { + LOGE("Could not open audio device: %s", SDL_GetError()); + return false; + } + + SDL_PauseAudioDevice(ap->device, 0); + + return true; +} + +static void +sc_audio_player_frame_sink_close(struct sc_frame_sink *sink) { + struct sc_audio_player *ap = DOWNCAST(sink); + + assert(ap->device); + SDL_PauseAudioDevice(ap->device, 1); + SDL_CloseAudioDevice(ap->device); +} + +static bool +sc_audio_player_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { + struct sc_audio_player *ap = DOWNCAST(sink); + + const uint8_t *data = frame->data[0]; + size_t size = frame->linesize[0]; + + // TODO convert to non planar format + // TODO then re-enable stereo + // TODO clock drift compensation + + // It should almost always be possible to write without lock + bool can_write_without_lock = size <= ap->safe_empty_buffer; + if (can_write_without_lock) { + sc_bytebuf_prepare_write(&ap->buf, data, size); + } + + SDL_LockAudioDevice(ap->device); + if (can_write_without_lock) { + sc_bytebuf_commit_write(&ap->buf, size); + } else { + sc_bytebuf_write(&ap->buf, data, size); + } + + // The next time, it will remain at least the current empty space + ap->safe_empty_buffer = sc_bytebuf_write_remaining(&ap->buf); + SDL_UnlockAudioDevice(ap->device); + + return true; +} + +bool +sc_audio_player_init(struct sc_audio_player *ap, + const struct sc_audio_player_callbacks *cbs, + void *cbs_userdata) { + bool ok = sc_bytebuf_init(&ap->buf, 128 * 1024); + if (!ok) { + return false; + } + + ap->safe_empty_buffer = sc_bytebuf_write_remaining(&ap->buf); + + assert(cbs && cbs->on_ended); + ap->cbs = cbs; + ap->cbs_userdata = cbs_userdata; + + static const struct sc_frame_sink_ops ops = { + .open = sc_audio_player_frame_sink_open, + .close = sc_audio_player_frame_sink_close, + .push = sc_audio_player_frame_sink_push, + }; + + ap->frame_sink.ops = &ops; + return true; +} + +void +sc_audio_player_destroy(struct sc_audio_player *ap) { + sc_bytebuf_destroy(&ap->buf); +} diff --git a/app/src/audio_player.h b/app/src/audio_player.h new file mode 100644 index 00000000..fbe6aac0 --- /dev/null +++ b/app/src/audio_player.h @@ -0,0 +1,40 @@ +#ifndef SC_AUDIO_PLAYER_H +#define SC_AUDIO_PLAYER_H + +#include "common.h" + +#include +#include "trait/frame_sink.h" +#include +#include + +#include +#include + +struct sc_audio_player { + struct sc_frame_sink frame_sink; + + SDL_AudioDeviceID device; + + // protected by SDL_AudioDeviceLock() + struct sc_bytebuf buf; + // Number of bytes which could be written without locking + size_t safe_empty_buffer; + + const struct sc_audio_player_callbacks *cbs; + void *cbs_userdata; +}; + +struct sc_audio_player_callbacks { + void (*on_ended)(struct sc_audio_player *ap, bool success, void *userdata); +}; + +bool +sc_audio_player_init(struct sc_audio_player *ap, + const struct sc_audio_player_callbacks *cbs, + void *cbs_userdata); + +void +sc_audio_player_destroy(struct sc_audio_player *ap); + +#endif diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index eb70749a..a12d4a78 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -13,6 +13,7 @@ # include #endif +#include "audio_player.h" #include "controller.h" #include "decoder.h" #include "demuxer.h" @@ -40,6 +41,7 @@ struct scrcpy { struct sc_server server; struct sc_screen screen; + struct sc_audio_player audio_player; struct sc_demuxer video_demuxer; struct sc_demuxer audio_demuxer; struct sc_decoder video_decoder; @@ -215,6 +217,17 @@ sc_recorder_on_ended(struct sc_recorder *recorder, bool success, } } +static void +sc_audio_player_on_ended(struct sc_audio_player *ap, bool success, + void *userdata) { + (void) ap; + (void) userdata; + + if (!success) { + // TODO + } +} + static void sc_video_demuxer_on_ended(struct sc_demuxer *demuxer, bool eos, void *userdata) { @@ -301,6 +314,7 @@ scrcpy(struct scrcpy_options *options) { bool file_pusher_initialized = false; bool recorder_initialized = false; bool recorder_started = false; + bool audio_player_initialized = false; #ifdef HAVE_V4L2 bool v4l2_sink_initialized = false; #endif @@ -383,9 +397,16 @@ scrcpy(struct scrcpy_options *options) { } // Initialize SDL video in addition if display is enabled - if (options->display && SDL_Init(SDL_INIT_VIDEO)) { - LOGE("Could not initialize SDL: %s", SDL_GetError()); - goto end; + if (options->display) { + if (SDL_Init(SDL_INIT_VIDEO)) { + LOGE("Could not initialize SDL video: %s", SDL_GetError()); + goto end; + } + + if (options->audio && SDL_Init(SDL_INIT_AUDIO)) { + LOGE("Could not initialize SDL audio: %s", SDL_GetError()); + goto end; + } } sdl_configure(options->display, options->disable_screensaver); @@ -663,6 +684,19 @@ aoa_hid_end: screen_initialized = true; sc_decoder_add_sink(&s->video_decoder, &s->screen.frame_sink); + + if (options->audio) { + static const struct sc_audio_player_callbacks audio_player_cbs = { + .on_ended = sc_audio_player_on_ended, + }; + if (!sc_audio_player_init(&s->audio_player, + &audio_player_cbs, NULL)) { + goto end; + } + audio_player_initialized = true; + + sc_decoder_add_sink(&s->audio_decoder, &s->audio_player.frame_sink); + } } #ifdef HAVE_V4L2 @@ -783,6 +817,10 @@ end: sc_recorder_destroy(&s->recorder); } + if (audio_player_initialized) { + sc_audio_player_destroy(&s->audio_player); + } + if (file_pusher_initialized) { sc_file_pusher_join(&s->file_pusher); sc_file_pusher_destroy(&s->file_pusher); diff --git a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java index 3b012b11..8f950d47 100644 --- a/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java +++ b/server/src/main/java/com/genymobile/scrcpy/AudioEncoder.java @@ -40,7 +40,7 @@ public final class AudioEncoder { } private static final int SAMPLE_RATE = 48000; - private static final int CHANNELS = 2; + private static final int CHANNELS = 1; private static final int BUFFER_MS = 10; // milliseconds private static final int BUFFER_SIZE = SAMPLE_RATE * CHANNELS * BUFFER_MS / 1000;