mirrors/scrcpy
1
0
Fork 0
mirror of https://github.com/Genymobile/scrcpy.git synced 2025-04-20 03:25:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement audio forwarding

Browse source 
Use Android Open Accessory 2 to redirect the device audio output to the
computer, creating a new audio input source.

Record this new source and play it to the default output.

<https://source.android.com/devices/accessories/aoa2#audio-support>
This commit is contained in:
Romain Vimont 2018-10-21 19:08:43 +02:00
parent b6f0f2bedf
commit fae8f353ce
11 changed files with 557 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
BUILD.md
View file

@ -20,14 +20,15 @@ the following files to a directory accessible from your `PATH`:
- `AdbWinApi.dll`
- `AdbWinUsbApi.dll`
The client requires [FFmpeg] and [LibSDL2]. Just follow the instructions.
The client requires [FFmpeg], [LibSDL2] and [LibUSB]. Just follow the
instructions.
[adb]: https://developer.android.com/studio/command-line/adb.html
[platform-tools]: https://developer.android.com/studio/releases/platform-tools.html
[platform-tools-windows]: https://dl.google.com/android/repository/platform-tools-latest-windows.zip
[ffmpeg]: https://en.wikipedia.org/wiki/FFmpeg
[LibSDL2]: https://en.wikipedia.org/wiki/Simple_DirectMedia_Layer
[LibUSB]: https://en.wikipedia.org/wiki/Libusb
## System-specific steps
@ -40,12 +41,12 @@ Install the required packages from your package manager.
```bash
# runtime dependencies
sudo apt install ffmpeg libsdl2-2.0.0
sudo apt install ffmpeg libsdl2-2.0.0 libusb-1.0-0
# client build dependencies
sudo apt install make gcc pkg-config meson \
libavcodec-dev libavformat-dev libavutil-dev \
libsdl2-dev
libsdl2-dev libusb-1.0-0-dev
# server build dependencies
sudo apt install openjdk-8-jdk
@ -109,7 +110,8 @@ pacman -S mingw-w64-x86_64-SDL2 \
pacman -S mingw-w64-x86_64-make \
mingw-w64-x86_64-gcc \
mingw-w64-x86_64-pkg-config \
mingw-w64-x86_64-meson
mingw-w64-x86_64-meson \
mingw-w64-x86_64-libusb
```
For a 32 bits version, replace `x86_64` by `i686`:
@ -123,7 +125,8 @@ pacman -S mingw-w64-i686-SDL2 \
pacman -S mingw-w64-i686-make \
mingw-w64-i686-gcc \
mingw-w64-i686-pkg-config \
mingw-w64-i686-meson
mingw-w64-i686-meson \
mingw-w64-i686-libusb
```
Java (>= 7) is not available in MSYS2, so if you plan to build the server,
@ -144,7 +147,7 @@ Install the packages with [Homebrew]:
brew install sdl2 ffmpeg
# client build dependencies
brew install pkg-config meson
brew install pkg-config meson libusb
```
Additionally, if you want to build the server, install Java 8 from Caskroom, and

7
README.md
View file

@ -123,6 +123,13 @@ The app may be started directly in fullscreen:
scrcpy -f
```
To enable audio forwarding (experimental):
```
scrcpy -a
```
## Shortcuts
| Action | Shortcut |

14
app/meson.build
View file

@ -19,6 +19,13 @@ src = [
'src/tiny_xpm.c',
]
if get_option('audio_support')
src += [
'src/aoa.c',
'src/audio.c',
]
endif
if not get_option('crossbuild_windows')
# native build
@ -29,6 +36,10 @@ if not get_option('crossbuild_windows')
dependency('sdl2'),
]
if get_option('audio_support')
dependencies += dependency('libusb-1.0')
endif
else
# cross-compile mingw32 build (from Linux to Windows)
@ -126,6 +137,9 @@ conf.set('SKIP_FRAMES', get_option('skip_frames'))
# enable High DPI support
conf.set('HIDPI_SUPPORT', get_option('hidpi_support'))
# enable audio support (enable audio forwarding with --forward-audio)
conf.set('AUDIO_SUPPORT', get_option('audio_support'))
# disable console on Windows
conf.set('WINDOWS_NOCONSOLE', get_option('windows_noconsole'))

206
app/src/aoa.c Normal file
View file

@ -0,0 +1,206 @@
#include "aoa.h"
#include "command.h" // must be first to include "winsock2.h" before "windows.h"
#include <libusb-1.0/libusb.h>
#include "log.h"
// <https://source.android.com/devices/accessories/aoa2>
#define AOA_GET_PROTOCOL 51
#define AOA_START_ACCESSORY 53
#define AOA_SET_AUDIO_MODE 58
#define AUDIO_MODE_NO_AUDIO 0
#define AUDIO_MODE_S16LSB_STEREO_44100HZ 1
#define DEFAULT_TIMEOUT 1000
typedef struct control_params {
uint8_t request_type;
uint8_t request;
uint16_t value;
uint16_t index;
unsigned char *data;
uint16_t length;
unsigned int timeout;
} control_params;
static void log_libusb_error(enum libusb_error errcode) {
LOGE("%s", libusb_strerror(errcode));
}
static SDL_bool control_transfer(libusb_device_handle *handle, control_params *params) {
int r = libusb_control_transfer(handle,
params->request_type,
params->request,
params->value,
params->index,
params->data,
params->length,
params->timeout);
if (r < 0) {
log_libusb_error(r);
return SDL_FALSE;
}
return SDL_TRUE;
}
static SDL_bool get_serial(libusb_device *device, struct libusb_device_descriptor *desc, unsigned char *data, int length) {
libusb_device_handle *handle;
int r;
if ((r = libusb_open(device, &handle))) {
// silently ignore
LOGD("USB: cannot open device %04x:%04x (%s)", desc->idVendor, desc->idProduct, libusb_strerror(r));
return SDL_FALSE;
}
if (!desc->iSerialNumber) {
LOGD("USB: device %04x:%04x has no serial number available", desc->idVendor, desc->idProduct);
libusb_close(handle);
return SDL_FALSE;
}
if ((r = libusb_get_string_descriptor_ascii(handle, desc->iSerialNumber, data, length)) <= 0) {
// silently ignore
LOGD("USB: cannot read serial of device %04x:%04x (%s)", desc->idVendor, desc->idProduct, libusb_strerror(r));
libusb_close(handle);
return SDL_FALSE;
}
data[length - 1] = '\0'; // just in case
libusb_close(handle);
return SDL_TRUE;
}
static libusb_device *find_device(const char *serial) {
libusb_device **list;
libusb_device *found = NULL;
ssize_t cnt = libusb_get_device_list(NULL, &list);
ssize_t i = 0;
if (cnt < 0) {
log_libusb_error(cnt);
return NULL;
}
for (i = 0; i < cnt; ++i) {
libusb_device *device = list[i];
struct libusb_device_descriptor desc;
libusb_get_device_descriptor(device, &desc);
char usb_serial[128];
if (get_serial(device, &desc, (unsigned char *) usb_serial, sizeof(usb_serial))) {
if (!strncmp(serial, usb_serial, sizeof(usb_serial))) {
libusb_ref_device(device);
found = device;
LOGD("USB device with serial %s found: %04x:%04x", serial, desc.idVendor, desc.idProduct);
break;
}
}
}
libusb_free_device_list(list, 1);
return found;
}
static SDL_bool aoa_get_protocol(libusb_device_handle *handle, uint16_t *version) {
unsigned char data[2];
control_params params = {
.request_type = LIBUSB_ENDPOINT_IN | LIBUSB_REQUEST_TYPE_VENDOR,
.request = AOA_GET_PROTOCOL,
.value = 0,
.index = 0,
.data = data,
.length = sizeof(data),
.timeout = DEFAULT_TIMEOUT
};
if (control_transfer(handle, &params)) {
// little endian
*version = (data[1] << 8) | data[0];
return SDL_TRUE;
}
return SDL_FALSE;
}
static SDL_bool set_audio_mode(libusb_device_handle *handle, uint16_t mode) {
control_params params = {
.request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR,
.request = AOA_SET_AUDIO_MODE,
// <https://source.android.com/devices/accessories/aoa2.html#audio-support>
.value = mode,
.index = 0, // unused
.data = NULL,
.length = 0,
.timeout = DEFAULT_TIMEOUT
};
return control_transfer(handle, &params);
}
static SDL_bool start_accessory(libusb_device_handle *handle) {
control_params params = {
.request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR,
.request = AOA_START_ACCESSORY,
.value = 0, // unused
.index = 0, // unused
.data = NULL,
.length = 0,
.timeout = DEFAULT_TIMEOUT
};
return control_transfer(handle, &params);
}
SDL_bool aoa_init(void) {
return !libusb_init(NULL);
}
void aoa_exit(void) {
libusb_exit(NULL);
}
SDL_bool aoa_forward_audio(const char *serial, SDL_bool forward) {
LOGD("%s audio accessory...", forward ? "Enabling" : "Disabling");
libusb_device *device = find_device(serial);
if (!device) {
LOGE("Cannot find USB device having serial %s", serial);
return SDL_FALSE;
}
SDL_bool ret = SDL_FALSE;
libusb_device_handle *handle;
int r = libusb_open(device, &handle);
if (r) {
log_libusb_error(r);
goto finally_unref_device;
}
uint16_t version;
if (!aoa_get_protocol(handle, &version)) {
LOGE("Cannot get AOA protocol version");
goto finally_close_handle;
}
LOGD("Device AOA version: %" PRIu16 "\n", version);
if (version < 2) {
LOGE("Device does not support AOA 2: %" PRIu16, version);
goto finally_close_handle;
}
uint16_t mode = forward ? AUDIO_MODE_S16LSB_STEREO_44100HZ : AUDIO_MODE_NO_AUDIO;
if (!set_audio_mode(handle, mode)) {
LOGE("Cannot set audio mode: %" PRIu16, mode);
goto finally_close_handle;
}
if (!start_accessory(handle)) {
LOGE("Cannot start accessory");
return SDL_FALSE;
}
ret = SDL_TRUE;
finally_close_handle:
libusb_close(handle);
finally_unref_device:
libusb_unref_device(device);
return ret;
}

15
app/src/aoa.h Normal file
View file

@ -0,0 +1,15 @@
#ifndef AOA_H
#define AOA_H
#include <SDL2/SDL_stdinc.h>
#define AUDIO_MODE_NO_AUDIO 0
#define AUDIO_MODE_S16LSB_STEREO_44100HZ 1
SDL_bool aoa_init(void);
void aoa_exit(void);
// serial must not be NULL
SDL_bool aoa_forward_audio(const char *serial, SDL_bool forward);
#endif

205
app/src/audio.c Normal file
View file

@ -0,0 +1,205 @@
#include "audio.h"
#include <SDL2/SDL.h>
#include "aoa.h"
#include "command.h"
#include "log.h"
SDL_bool sdl_audio_init(void) {
if (SDL_InitSubSystem(SDL_INIT_AUDIO)) {
LOGC("Could not initialize SDL audio: %s", SDL_GetError());
return SDL_FALSE;
}
return SDL_TRUE;
}
static void init_audio_spec(SDL_AudioSpec *spec) {
SDL_zero(*spec);
spec->freq = 44100;
spec->format = AUDIO_S16LSB;
spec->channels = 2;
spec->samples = 1024;
}
SDL_bool audio_player_init(struct audio_player *player, const char *serial) {
player->serial = SDL_strdup(serial);
return !!player->serial;
}
void audio_player_destroy(struct audio_player *player) {
SDL_free((void *) player->serial);
}
static void audio_input_callback(void *userdata, Uint8 *stream, int len) {
struct audio_player *player = userdata;
if (SDL_QueueAudio(player->output_device, stream, len)) {
LOGE("Cannot queue audio: %s", SDL_GetError());
}
}
static int get_matching_audio_device(const char *serial, int count) {
for (int i = 0; i < count; ++i) {
LOGD("Audio input #%d: %s", i, SDL_GetAudioDeviceName(i, 1));
}
char model[128];
int r = adb_read_model(serial, model, sizeof(model));
if (r <= 0) {
LOGE("Cannot read Android device model");
return -1;
}
LOGD("Device model is: %s", model);
// iterate backwards since the matching device is probably the last one
for (int i = count - 1; i >= 0; i--) {
// model is a NUL-terminated string
const char *name = SDL_GetAudioDeviceName(i, 1);
if (strstr(name, model)) {
// the device name contains the device model, we found it!
return i;
}
}
return -1;
}
static SDL_AudioDeviceID open_accessory_audio_input(struct audio_player *player) {
int count = SDL_GetNumAudioDevices(1);
if (!count) {
LOGE("No audio input source found");
return 0;
}
int selected = get_matching_audio_device(player->serial, count);
if (selected == -1) {
LOGE("Cannot find the Android accessory audio input source");
return 0;
}
const char *selected_name = SDL_GetAudioDeviceName(selected, 1);
LOGI("Selecting audio input source: %s", selected_name);
SDL_AudioSpec spec;
init_audio_spec(&spec);
spec.callback = audio_input_callback;
spec.userdata = player;
int id = SDL_OpenAudioDevice(selected_name, 1, &spec, NULL, 0);
if (!id) {
LOGE("Cannot open audio input: %s", SDL_GetError());
}
return id;
}
static SDL_AudioDeviceID open_default_audio_output() {
SDL_AudioSpec spec;
init_audio_spec(&spec);
int id = SDL_OpenAudioDevice(NULL, 0, &spec, NULL, 0);
if (!id) {
LOGE("Cannot open audio output: %s", SDL_GetError());
}
return id;
}
SDL_bool audio_player_open(struct audio_player *player) {
player->output_device = open_default_audio_output();
if (!player->output_device) {
return SDL_FALSE;
}
player->input_device = open_accessory_audio_input(player);
if (!player->input_device) {
SDL_CloseAudioDevice(player->output_device);
return SDL_FALSE;
}
return SDL_TRUE;
}
static void audio_player_set_paused(struct audio_player *player, SDL_bool paused) {
SDL_PauseAudioDevice(player->input_device, paused);
SDL_PauseAudioDevice(player->output_device, paused);
}
void audio_player_play(struct audio_player *player) {
audio_player_set_paused(player, SDL_FALSE);
}
void audio_player_pause(struct audio_player *player) {
audio_player_set_paused(player, SDL_TRUE);
}
void audio_player_close(struct audio_player *player) {
SDL_CloseAudioDevice(player->input_device);
SDL_CloseAudioDevice(player->output_device);
}
SDL_bool audio_forwarding_start(struct audio_player *player, const char *serial) {
if (!aoa_init()) {
LOGE("Cannot initialize AOA");
return SDL_FALSE;
}
char serialno[128];
if (!serial) {
LOGD("No serial provided, request it to the device");
int r = adb_read_serialno(NULL, serialno, sizeof(serialno));
if (r <= 0) {
LOGE("Cannot read serial from the device");
goto error_aoa_exit;
}
LOGD("Device serial is %s", serialno);
serial = serialno;
}
if (!audio_player_init(player, serial)) {
LOGE("Cannot initialize audio player");
goto error_aoa_exit;
}
// adb connection will be reset!
if (!aoa_forward_audio(player->serial, SDL_TRUE)) {
LOGE("AOA audio forwarding failed");
goto error_destroy_player;
}
LOGI("Audio accessory enabled");
if (!sdl_audio_init()) {
goto error_disable_audio_forwarding;
}
LOGI("Waiting 2s for USB reconfiguration...");
SDL_Delay(2000);
if (!audio_player_open(player)) {
goto error_disable_audio_forwarding;
}
audio_player_play(player);
return SDL_TRUE;
error_disable_audio_forwarding:
if (!aoa_forward_audio(serial, SDL_FALSE)) {
LOGW("Cannot disable audio forwarding");
}
error_destroy_player:
audio_player_destroy(player);
error_aoa_exit:
aoa_exit();
return SDL_FALSE;
}
void audio_forwarding_stop(struct audio_player *player) {
audio_player_close(player);
if (aoa_forward_audio(player->serial, SDL_FALSE)) {
LOGI("Audio forwarding disabled");
} else {
LOGW("Cannot disable audio forwarding");
}
aoa_exit();
audio_player_destroy(player);
}

29
app/src/audio.h Normal file
View file

@ -0,0 +1,29 @@
#ifndef AUDIO_H
#define AUDIO_H
#include <SDL2/SDL_audio.h>
#include <SDL2/SDL_stdinc.h>
struct audio_player {
const char *serial;
SDL_AudioDeviceID input_device;
SDL_AudioDeviceID output_device;
};
SDL_bool sdl_audio_init(void);
// serial must not be NULL
SDL_bool audio_player_init(struct audio_player *player, const char *serial);
void audio_player_destroy(struct audio_player *player);
SDL_bool audio_player_open(struct audio_player *player);
void audio_player_close(struct audio_player *player);
void audio_player_play(struct audio_player *player);
void audio_player_pause(struct audio_player *player);
// for convenience, these functions handle everything
SDL_bool audio_forwarding_start(struct audio_player *player, const char *serial);
void audio_forwarding_stop(struct audio_player *player);
#endif

51
app/src/main.c
View file

@ -18,6 +18,9 @@ struct args {
Uint16 port;
Uint16 max_size;
Uint32 bit_rate;
#ifdef AUDIO_SUPPORT
SDL_bool forward_audio;
#endif
};
static void usage(const char *arg0) {
@ -25,6 +28,12 @@ static void usage(const char *arg0) {
"Usage: %s [options]\n"
"\n"
"Options:\n"
#ifdef AUDIO_SUPPORT
"\n"
" -a, --forward-audio\n"
" Forward audio from the device to the computer over USB\n"
" (experimental).\n"
#endif
"\n"
" -b, --bit-rate value\n"
" Encode the video at the given bit-rate, expressed in bits/s.\n"
@ -202,20 +211,34 @@ static SDL_bool parse_port(char *optarg, Uint16 *port) {
static SDL_bool parse_args(struct args *args, int argc, char *argv[]) {
static const struct option long_options[] = {
{"bit-rate", required_argument, NULL, 'b'},
{"crop", required_argument, NULL, 'c'},
{"fullscreen", no_argument, NULL, 'f'},
{"help", no_argument, NULL, 'h'},
{"max-size", required_argument, NULL, 'm'},
{"port", required_argument, NULL, 'p'},
{"serial", required_argument, NULL, 's'},
{"show-touches", no_argument, NULL, 't'},
{"version", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0 },
#ifdef AUDIO_SUPPORT
{"forward-audio", no_argument, NULL, 'a'},
#endif
{"bit-rate", required_argument, NULL, 'b'},
{"crop", required_argument, NULL, 'c'},
{"fullscreen", no_argument, NULL, 'f'},
{"help", no_argument, NULL, 'h'},
{"max-size", required_argument, NULL, 'm'},
{"port", required_argument, NULL, 'p'},
{"serial", required_argument, NULL, 's'},
{"show-touches", no_argument, NULL, 't'},
{"version", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0 },
};
int c;
while ((c = getopt_long(argc, argv, "b:c:fhm:p:s:tv", long_options, NULL)) != -1) {
#ifdef AUDIO_SUPPORT
# define AUDIO_SHORT_PARAM "a"
#else
# define AUDIO_SHORT_PARAM
#endif
while ((c = getopt_long(argc, argv, AUDIO_SHORT_PARAM "b:c:fhm:p:s:tv",
long_options, NULL)) != -1) {
switch (c) {
#ifdef AUDIO_SUPPORT
case 'a':
args->forward_audio = SDL_TRUE;
break;
#endif
case 'b':
if (!parse_bit_rate(optarg, &args->bit_rate)) {
return SDL_FALSE;
@ -279,6 +302,9 @@ int main(int argc, char *argv[]) {
.port = DEFAULT_LOCAL_PORT,
.max_size = DEFAULT_MAX_SIZE,
.bit_rate = DEFAULT_BIT_RATE,
#ifdef AUDIO_SUPPORT
.forward_audio = SDL_FALSE,
#endif
};
if (!parse_args(&args, argc, argv)) {
return 1;
@ -314,6 +340,9 @@ int main(int argc, char *argv[]) {
.bit_rate = args.bit_rate,
.show_touches = args.show_touches,
.fullscreen = args.fullscreen,
#ifdef AUDIO_SUPPORT
.forward_audio = args.forward_audio,
#endif
};
int res = scrcpy(&options) ? 0 : 1;

31
app/src/scrcpy.c
View file

@ -7,6 +7,9 @@
#include <sys/time.h>
#include <SDL2/SDL.h>
#include "config.h"
#include "aoa.h"
#include "audio.h"
#include "command.h"
#include "common.h"
#include "controller.h"
@ -31,6 +34,10 @@ static struct decoder decoder;
static struct controller controller;
static struct file_handler file_handler;
#ifdef AUDIO_SUPPORT
static struct audio_player audio_player;
#endif
static struct input_manager input_manager = {
.controller = &controller,
.frames = &frames,
@ -143,6 +150,17 @@ SDL_bool scrcpy(const struct scrcpy_options *options) {
return SDL_FALSE;
}
SDL_bool ret = SDL_TRUE;
#ifdef AUDIO_SUPPORT
if (options->forward_audio) {
if (!audio_forwarding_start(&audio_player, options->serial)) {
ret = SDL_FALSE;
goto finally_destroy_server;
}
}
#endif
process_t proc_show_touches = PROCESS_NONE;
SDL_bool show_touches_waited;
if (options->show_touches) {
@ -151,15 +169,13 @@ SDL_bool scrcpy(const struct scrcpy_options *options) {
show_touches_waited = SDL_FALSE;
}
SDL_bool ret = SDL_TRUE;
if (!SDL_SetHint(SDL_HINT_NO_SIGNAL_HANDLERS, "1")) {
LOGW("Cannot request to keep default signal handlers");
}
if (!sdl_video_init()) {
ret = SDL_FALSE;
goto finally_destroy_server;
goto finally_revert_show_touches;
}
socket_t device_socket = server_connect_to(&server);
@ -248,7 +264,7 @@ finally_destroy_file_handler:
file_handler_destroy(&file_handler);
finally_destroy_frames:
frames_destroy(&frames);
finally_destroy_server:
finally_revert_show_touches:
if (options->show_touches) {
if (!show_touches_waited) {
// wait the process which enabled "show touches"
@ -258,7 +274,12 @@ finally_destroy_server:
proc_show_touches = set_show_touches_enabled(options->serial, SDL_FALSE);
wait_show_touches(proc_show_touches);
}
#ifdef AUDIO_SUPPORT
if (options->forward_audio) {
audio_forwarding_stop(&audio_player);
}
#endif
finally_destroy_server:
server_destroy(&server);
return ret;

4
app/src/scrcpy.h
View file

@ -2,6 +2,7 @@
#define SCRCPY_H
#include <SDL2/SDL_stdinc.h>
#include "config.h"
struct scrcpy_options {
const char *serial;
@ -11,6 +12,9 @@ struct scrcpy_options {
Uint32 bit_rate;
SDL_bool show_touches;
SDL_bool fullscreen;
#ifdef AUDIO_SUPPORT
SDL_bool forward_audio;
#endif
};
SDL_bool scrcpy(const struct scrcpy_options *options);

1
meson_options.txt
View file

@ -6,3 +6,4 @@ option('prebuilt_server', type: 'string', description: 'Path of the prebuilt ser
option('override_server_path', type: 'string', description: 'Hardcoded path to find the server at runtime')
option('skip_frames', type: 'boolean', value: true, description: 'Always display the most recent frame')
option('hidpi_support', type: 'boolean', value: true, description: 'Enable High DPI support')
option('audio_support', type: 'boolean', value: true, description: 'Enable audio support')