diff --git a/README.md b/README.md index be71a75e..4c02f5bd 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,15 @@ and extract the following files to a directory accessible from your `PATH`: Make sure you [enabled adb debugging][enable-adb] on your device(s). -The client requires [FFmpeg] and [LibSDL2]. +The client requires [FFmpeg], [LibSDL2] and [LibUSB]. [adb]: https://developer.android.com/studio/command-line/adb.html [enable-adb]: https://developer.android.com/studio/command-line/adb.html#Enabling [platform-tools]: https://developer.android.com/studio/releases/platform-tools.html [platform-tools-windows]: https://dl.google.com/android/repository/platform-tools-latest-windows.zip [ffmpeg]: https://en.wikipedia.org/wiki/FFmpeg -[LibSDL2]: https://en.wikipedia.org/wiki/Simple_DirectMedia_Layer +[libsdl2]: https://en.wikipedia.org/wiki/Simple_DirectMedia_Layer +[libusb]: https://en.wikipedia.org/wiki/Libusb ## Build and install @@ -44,12 +45,12 @@ Install the required packages from your package manager. ```bash # runtime dependencies -sudo apt install ffmpeg libsdl2-2.0.0 +sudo apt install ffmpeg libsdl2-2.0.0 libusb-1.0-0 # client build dependencies sudo apt install make gcc pkg-config meson \ libavcodec-dev libavformat-dev libavutil-dev \ - libsdl2-dev + libsdl2-dev libusb-1.0-0-dev # server build dependencies sudo apt install openjdk-8-jdk @@ -104,7 +105,8 @@ pacman -S mingw-w64-x86_64-SDL2 \ pacman -S mingw-w64-x86_64-make \ mingw-w64-x86_64-gcc \ mingw-w64-x86_64-pkg-config \ - mingw-w64-x86_64-meson + mingw-w64-x86_64-meson \ + mingw-w64-x86_64-libusb ``` For a 32 bits version, replace `x86_64` by `i686`: @@ -146,7 +148,7 @@ Instead, you may want to build it manually. Install the packages: brew install sdl2 ffmpeg # client build dependencies -brew install pkg-config meson +brew install pkg-config meson libusb ``` Additionally, if you want to build the server, install Java 8 from Caskroom, and @@ -274,6 +276,12 @@ To show physical touches while scrcpy is running: scrcpy -t ``` +To enable audio forwarding: + +```bash +scrcpy -a +``` + To run without installing: ```bash diff --git a/app/meson.build b/app/meson.build index a206dd58..ef708eb7 100644 --- a/app/meson.build +++ b/app/meson.build @@ -18,6 +18,13 @@ src = [ 'src/tinyxpm.c', ] +if get_option('audio_support') + src += [ + 'src/aoa.c', + 'src/audio.c' + ] +endif + dependencies = [ dependency('libavformat'), dependency('libavcodec'), @@ -25,6 +32,10 @@ dependencies = [ dependency('sdl2'), ] +if get_option('audio_support') + dependencies += dependency('libusb-1.0') +endif + cc = meson.get_compiler('c') if host_machine.system() == 'windows' @@ -83,6 +94,9 @@ conf.set('SKIP_FRAMES', get_option('skip_frames')) # enable High DPI support conf.set('HIDPI_SUPPORT', get_option('hidpi_support')) +# enable audio support (enable audio forwarding with --forward-audio) +conf.set('AUDIO_SUPPORT', get_option('audio_support')) + configure_file(configuration: conf, output: 'config.h') src_dir = include_directories('src') diff --git a/app/src/aoa.c b/app/src/aoa.c new file mode 100644 index 00000000..c0c035a7 --- /dev/null +++ b/app/src/aoa.c @@ -0,0 +1,206 @@ +#include "aoa.h" + +#include "command.h" // must be first to include "winsock2.h" before "windows.h" +#include +#include "log.h" + +// +#define AOA_GET_PROTOCOL 51 +#define AOA_START_ACCESSORY 53 +#define AOA_SET_AUDIO_MODE 58 + +#define AUDIO_MODE_NO_AUDIO 0 +#define AUDIO_MODE_S16LSB_STEREO_44100HZ 1 + +#define DEFAULT_TIMEOUT 1000 + +typedef struct control_params { + uint8_t request_type; + uint8_t request; + uint16_t value; + uint16_t index; + unsigned char *data; + uint16_t length; + unsigned int timeout; +} control_params; + +static void log_libusb_error(enum libusb_error errcode) { + LOGE("%s", libusb_strerror(errcode)); +} + +static SDL_bool control_transfer(libusb_device_handle *handle, control_params *params) { + int r = libusb_control_transfer(handle, + params->request_type, + params->request, + params->value, + params->index, + params->data, + params->length, + params->timeout); + if (r < 0) { + log_libusb_error(r); + return SDL_FALSE; + } + return SDL_TRUE; +} + +static SDL_bool get_serial(libusb_device *device, struct libusb_device_descriptor *desc, unsigned char *data, int length) { + + libusb_device_handle *handle; + int r; + if ((r = libusb_open(device, &handle))) { + // silently ignore + LOGD("USB: cannot open device %04x:%04x (%s)", desc->idVendor, desc->idProduct, libusb_strerror(r)); + return SDL_FALSE; + } + + if (!desc->iSerialNumber) { + LOGD("USB: device %04x:%04x has no serial number available", desc->idVendor, desc->idProduct); + libusb_close(handle); + return SDL_FALSE; + } + + if ((r = libusb_get_string_descriptor_ascii(handle, desc->iSerialNumber, data, length)) <= 0) { + // silently ignore + LOGD("USB: cannot read serial of device %04x:%04x (%s)", desc->idVendor, desc->idProduct, libusb_strerror(r)); + libusb_close(handle); + return SDL_FALSE; + } + data[length - 1] = '\0'; // just in case + + libusb_close(handle); + return SDL_TRUE; +} + +static libusb_device *find_device(const char *serial) { + libusb_device **list; + libusb_device *found = NULL; + ssize_t cnt = libusb_get_device_list(NULL, &list); + ssize_t i = 0; + if (cnt < 0) { + log_libusb_error(cnt); + return NULL; + } + for (i = 0; i < cnt; ++i) { + libusb_device *device = list[i]; + + struct libusb_device_descriptor desc; + libusb_get_device_descriptor(device, &desc); + + char usb_serial[128]; + if (get_serial(device, &desc, (unsigned char *) usb_serial, sizeof(usb_serial))) { + if (!strncmp(serial, usb_serial, sizeof(usb_serial))) { + libusb_ref_device(device); + found = device; + LOGD("USB device with serial %s found: %04x:%04x", serial, desc.idVendor, desc.idProduct); + break; + } + } + } + libusb_free_device_list(list, 1); + return found; +} + +static SDL_bool aoa_get_protocol(libusb_device_handle *handle, uint16_t *version) { + unsigned char data[2]; + control_params params = { + .request_type = LIBUSB_ENDPOINT_IN | LIBUSB_REQUEST_TYPE_VENDOR, + .request = AOA_GET_PROTOCOL, + .value = 0, + .index = 0, + .data = data, + .length = sizeof(data), + .timeout = DEFAULT_TIMEOUT + }; + if (control_transfer(handle, ¶ms)) { + // little endian + *version = (data[1] << 8) | data[0]; + return SDL_TRUE; + } + return SDL_FALSE; +} + +static SDL_bool set_audio_mode(libusb_device_handle *handle, uint16_t mode) { + control_params params = { + .request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR, + .request = AOA_SET_AUDIO_MODE, + // + .value = mode, + .index = 0, // unused + .data = NULL, + .length = 0, + .timeout = DEFAULT_TIMEOUT + }; + return control_transfer(handle, ¶ms); +} + +static SDL_bool start_accessory(libusb_device_handle *handle) { + control_params params = { + .request_type = LIBUSB_ENDPOINT_OUT | LIBUSB_REQUEST_TYPE_VENDOR, + .request = AOA_START_ACCESSORY, + .value = 0, // unused + .index = 0, // unused + .data = NULL, + .length = 0, + .timeout = DEFAULT_TIMEOUT + }; + return control_transfer(handle, ¶ms); +} + +SDL_bool aoa_init(void) { + return !libusb_init(NULL); +} + +void aoa_exit(void) { + libusb_exit(NULL); +} + +SDL_bool aoa_forward_audio(const char *serial, SDL_bool forward) { + LOGD("%s audio accessory...", forward ? "Enabling" : "Disabling"); + libusb_device *device = find_device(serial); + if (!device) { + LOGE("Cannot find USB device having serial %s", serial); + return SDL_FALSE; + } + + SDL_bool ret = SDL_FALSE; + + libusb_device_handle *handle; + int r = libusb_open(device, &handle); + if (r) { + log_libusb_error(r); + goto finally_unref_device; + } + + uint16_t version; + if (!aoa_get_protocol(handle, &version)) { + LOGE("Cannot get AOA protocol version"); + goto finally_close_handle; + } + + LOGD("Device AOA version: %" PRIu16 "\n", version); + if (version < 2) { + LOGE("Device does not support AOA 2: %" PRIu16, version); + goto finally_close_handle; + } + + uint16_t mode = forward ? AUDIO_MODE_S16LSB_STEREO_44100HZ : AUDIO_MODE_NO_AUDIO; + if (!set_audio_mode(handle, mode)) { + LOGE("Cannot set audio mode: %" PRIu16, mode); + goto finally_close_handle; + } + + if (!start_accessory(handle)) { + LOGE("Cannot start accessory"); + return SDL_FALSE; + } + + ret = SDL_TRUE; + +finally_close_handle: + libusb_close(handle); +finally_unref_device: + libusb_unref_device(device); + + return ret; +} diff --git a/app/src/aoa.h b/app/src/aoa.h new file mode 100644 index 00000000..6ad07c07 --- /dev/null +++ b/app/src/aoa.h @@ -0,0 +1,15 @@ +#ifndef AOA_H +#define AOA_H + +#include + +#define AUDIO_MODE_NO_AUDIO 0 +#define AUDIO_MODE_S16LSB_STEREO_44100HZ 1 + +SDL_bool aoa_init(void); +void aoa_exit(void); + +// serial must not be NULL +SDL_bool aoa_forward_audio(const char *serial, SDL_bool forward); + +#endif diff --git a/app/src/audio.c b/app/src/audio.c new file mode 100644 index 00000000..d670c826 --- /dev/null +++ b/app/src/audio.c @@ -0,0 +1,205 @@ +#include "audio.h" + +#include +#include "aoa.h" +#include "command.h" +#include "log.h" + +SDL_bool sdl_audio_init(void) { + if (SDL_InitSubSystem(SDL_INIT_AUDIO)) { + LOGC("Could not initialize SDL audio: %s", SDL_GetError()); + return SDL_FALSE; + } + return SDL_TRUE; +} + +static void init_audio_spec(SDL_AudioSpec *spec) { + SDL_zero(*spec); + spec->freq = 44100; + spec->format = AUDIO_S16LSB; + spec->channels = 2; + spec->samples = 1024; +} + +SDL_bool audio_player_init(struct audio_player *player, const char *serial) { + player->serial = SDL_strdup(serial); + return !!player->serial; +} + +void audio_player_destroy(struct audio_player *player) { + SDL_free((void *) player->serial); +} + +static void audio_input_callback(void *userdata, Uint8 *stream, int len) { + struct audio_player *player = userdata; + if (SDL_QueueAudio(player->output_device, stream, len)) { + LOGE("Cannot queue audio: %s", SDL_GetError()); + } +} + +static int get_matching_audio_device(const char *serial, int count) { + for (int i = 0; i < count; ++i) { + LOGD("Audio input #%d: %s", i, SDL_GetAudioDeviceName(i, 1)); + } + + char model[128]; + int r = adb_read_model(serial, model, sizeof(model)); + if (r <= 0) { + LOGE("Cannot read Android device model"); + return -1; + } + + LOGD("Device model is: %s", model); + + // iterate backwards since the matching device is probably the last one + for (int i = count - 1; i >= 0; i--) { + // model is a NUL-terminated string + const char *name = SDL_GetAudioDeviceName(i, 1); + if (strstr(name, model)) { + // the device name contains the device model, we found it! + return i; + } + } + return -1; +} + +static SDL_AudioDeviceID open_accessory_audio_input(struct audio_player *player) { + int count = SDL_GetNumAudioDevices(1); + if (!count) { + LOGE("No audio input source found"); + return 0; + } + + int selected = get_matching_audio_device(player->serial, count); + if (selected == -1) { + LOGE("Cannot find the Android accessory audio input source"); + return 0; + } + + const char *selected_name = SDL_GetAudioDeviceName(selected, 1); + LOGI("Selecting audio input source: %s", selected_name); + + SDL_AudioSpec spec; + init_audio_spec(&spec); + spec.callback = audio_input_callback; + spec.userdata = player; + + int id = SDL_OpenAudioDevice(selected_name, 1, &spec, NULL, 0); + if (!id) { + LOGE("Cannot open audio input: %s", SDL_GetError()); + } + return id; +} + +static SDL_AudioDeviceID open_default_audio_output() { + SDL_AudioSpec spec; + init_audio_spec(&spec); + int id = SDL_OpenAudioDevice(NULL, 0, &spec, NULL, 0); + if (!id) { + LOGE("Cannot open audio output: %s", SDL_GetError()); + } + return id; +} + +SDL_bool audio_player_open(struct audio_player *player) { + player->output_device = open_default_audio_output(); + if (!player->output_device) { + return SDL_FALSE; + } + + player->input_device = open_accessory_audio_input(player); + if (!player->input_device) { + SDL_CloseAudioDevice(player->output_device); + return SDL_FALSE; + } + + return SDL_TRUE; +} + +static void audio_player_set_paused(struct audio_player *player, SDL_bool paused) { + SDL_PauseAudioDevice(player->input_device, paused); + SDL_PauseAudioDevice(player->output_device, paused); +} + +void audio_player_play(struct audio_player *player) { + audio_player_set_paused(player, SDL_FALSE); +} + +void audio_player_pause(struct audio_player *player) { + audio_player_set_paused(player, SDL_TRUE); +} + +void audio_player_close(struct audio_player *player) { + SDL_CloseAudioDevice(player->input_device); + SDL_CloseAudioDevice(player->output_device); +} + +SDL_bool audio_forwarding_start(struct audio_player *player, const char *serial) { + if (!aoa_init()) { + LOGE("Cannot initialize AOA"); + return SDL_FALSE; + } + + char serialno[128]; + if (!serial) { + LOGD("No serial provided, request it to the device"); + int r = adb_read_serialno(NULL, serialno, sizeof(serialno)); + if (r <= 0) { + LOGE("Cannot read serial from the device"); + goto error_aoa_exit; + } + LOGD("Device serial is %s", serialno); + serial = serialno; + } + + if (!audio_player_init(player, serial)) { + LOGE("Cannot initialize audio player"); + goto error_aoa_exit; + } + + // adb connection will be reset! + if (!aoa_forward_audio(player->serial, SDL_TRUE)) { + LOGE("AOA audio forwarding failed"); + goto error_destroy_player; + } + + LOGI("Audio accessory enabled"); + + if (!sdl_audio_init()) { + goto error_disable_audio_forwarding; + } + + LOGI("Waiting 2s for USB reconfiguration..."); + SDL_Delay(2000); + + if (!audio_player_open(player)) { + goto error_disable_audio_forwarding; + } + + audio_player_play(player); + return SDL_TRUE; + +error_disable_audio_forwarding: + if (!aoa_forward_audio(serial, SDL_FALSE)) { + LOGW("Cannot disable audio forwarding"); + } +error_destroy_player: + audio_player_destroy(player); +error_aoa_exit: + aoa_exit(); + + return SDL_FALSE; +} + +void audio_forwarding_stop(struct audio_player *player) { + audio_player_close(player); + + if (aoa_forward_audio(player->serial, SDL_FALSE)) { + LOGI("Audio forwarding disabled"); + } else { + LOGW("Cannot disable audio forwarding"); + } + aoa_exit(); + + audio_player_destroy(player); +} diff --git a/app/src/audio.h b/app/src/audio.h new file mode 100644 index 00000000..a961e38e --- /dev/null +++ b/app/src/audio.h @@ -0,0 +1,29 @@ +#ifndef AUDIO_H +#define AUDIO_H + +#include +#include + +struct audio_player { + const char *serial; + SDL_AudioDeviceID input_device; + SDL_AudioDeviceID output_device; +}; + +SDL_bool sdl_audio_init(void); + +// serial must not be NULL +SDL_bool audio_player_init(struct audio_player *player, const char *serial); +void audio_player_destroy(struct audio_player *player); + +SDL_bool audio_player_open(struct audio_player *player); +void audio_player_close(struct audio_player *player); + +void audio_player_play(struct audio_player *player); +void audio_player_pause(struct audio_player *player); + +// for convenience, these functions handle everything +SDL_bool audio_forwarding_start(struct audio_player *player, const char *serial); +void audio_forwarding_stop(struct audio_player *player); + +#endif diff --git a/app/src/main.c b/app/src/main.c index df8b7f91..722a919e 100644 --- a/app/src/main.c +++ b/app/src/main.c @@ -13,6 +13,9 @@ struct args { SDL_bool help; SDL_bool version; SDL_bool show_touches; +#ifdef AUDIO_SUPPORT + SDL_bool forward_audio; +#endif Uint16 port; Uint16 max_size; Uint32 bit_rate; @@ -23,6 +26,12 @@ static void usage(const char *arg0) { "Usage: %s [options]\n" "\n" "Options:\n" +#ifdef AUDIO_SUPPORT + "\n" + " -a, --forward-audio\n" + " Forward audio from the device to the computer over USB\n" + " (experimental).\n" +#endif "\n" " -b, --bit-rate value\n" " Encode the video at the given bit-rate, expressed in bits/s.\n" @@ -188,18 +197,31 @@ static SDL_bool parse_port(char *optarg, Uint16 *port) { static SDL_bool parse_args(struct args *args, int argc, char *argv[]) { static const struct option long_options[] = { - {"bit-rate", required_argument, NULL, 'b'}, - {"help", no_argument, NULL, 'h'}, - {"max-size", required_argument, NULL, 'm'}, - {"port", required_argument, NULL, 'p'}, - {"serial", required_argument, NULL, 's'}, - {"show-touches", no_argument, NULL, 't'}, - {"version", no_argument, NULL, 'v'}, - {NULL, 0, NULL, 0 }, +#ifdef AUDIO_SUPPORT + {"forward-audio", no_argument, NULL, 'a'}, +#endif + {"bit-rate", required_argument, NULL, 'b'}, + {"help", no_argument, NULL, 'h'}, + {"max-size", required_argument, NULL, 'm'}, + {"port", required_argument, NULL, 'p'}, + {"serial", required_argument, NULL, 's'}, + {"show-touches", no_argument, NULL, 't'}, + {"version", no_argument, NULL, 'v'}, + {NULL, 0, NULL, 0 }, }; int c; - while ((c = getopt_long(argc, argv, "b:hm:p:s:tv", long_options, NULL)) != -1) { +#ifdef AUDIO_SUPPORT +# define AUDIO_SHORT_PARAM "a" +#else +# define AUDIO_SHORT_PARAM +#endif + while ((c = getopt_long(argc, argv, AUDIO_SHORT_PARAM "b:hm:p:s:tv", long_options, NULL)) != -1) { switch (c) { +#ifdef AUDIO_SUPPORT + case 'a': + args->forward_audio = SDL_TRUE; + break; +#endif case 'b': if (!parse_bit_rate(optarg, &args->bit_rate)) { return SDL_FALSE; @@ -256,6 +278,9 @@ int main(int argc, char *argv[]) { .port = DEFAULT_LOCAL_PORT, .max_size = DEFAULT_MAX_SIZE, .bit_rate = DEFAULT_BIT_RATE, +#ifdef AUDIO_SUPPORT + .forward_audio = SDL_FALSE, +#endif }; if (!parse_args(&args, argc, argv)) { return 1; @@ -287,6 +312,9 @@ int main(int argc, char *argv[]) { .max_size = args.max_size, .bit_rate = args.bit_rate, .show_touches = args.show_touches, +#ifdef AUDIO_SUPPORT + .forward_audio = args.forward_audio, +#endif }; int res = scrcpy(&options) ? 0 : 1; diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 208d3057..c3ee732e 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -7,6 +7,8 @@ #include #include +#include "aoa.h" +#include "audio.h" #include "command.h" #include "common.h" #include "controller.h" @@ -29,6 +31,10 @@ static struct frames frames; static struct decoder decoder; static struct controller controller; +#ifdef AUDIO_SUPPORT +static struct audio_player audio_player; +#endif + static struct input_manager input_manager = { .controller = &controller, .frames = &frames, @@ -120,9 +126,8 @@ static void wait_show_touches(process_t process) { } SDL_bool scrcpy(const struct scrcpy_options *options) { - if (!server_start(&server, options->serial, options->port, - options->max_size, options->bit_rate)) { - return SDL_FALSE; + if (!SDL_SetHint(SDL_HINT_NO_SIGNAL_HANDLERS, "1")) { + LOGW("Cannot request to keep default signal handlers"); } process_t proc_show_touches; @@ -133,10 +138,20 @@ SDL_bool scrcpy(const struct scrcpy_options *options) { show_touches_waited = SDL_FALSE; } +#ifdef AUDIO_SUPPORT + if (options->forward_audio) { + if (!audio_forwarding_start(&audio_player, options->serial)) { + return SDL_FALSE; + } + } +#endif + SDL_bool ret = SDL_TRUE; - if (!SDL_SetHint(SDL_HINT_NO_SIGNAL_HANDLERS, "1")) { - LOGW("Cannot request to keep default signal handlers"); + if (!server_start(&server, options->serial, options->port, + options->max_size, options->bit_rate)) { + ret = SDL_FALSE; + goto finally_disable_audio_forwarding; } if (!sdl_video_init()) { @@ -228,6 +243,12 @@ finally_destroy_server: } server_destroy(&server); +finally_disable_audio_forwarding: +#ifdef AUDIO_SUPPORT + if (options->forward_audio) { + audio_forwarding_stop(&audio_player); + } +#endif return ret; } diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index 583cd608..fb564971 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -2,6 +2,7 @@ #define SCRCPY_H #include +#include "config.h" struct scrcpy_options { const char *serial; @@ -9,6 +10,9 @@ struct scrcpy_options { Uint16 max_size; Uint32 bit_rate; SDL_bool show_touches; +#ifdef AUDIO_SUPPORT + SDL_bool forward_audio; +#endif }; SDL_bool scrcpy(const struct scrcpy_options *options); diff --git a/meson_options.txt b/meson_options.txt index b154bd97..3719d69e 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -4,3 +4,4 @@ option('prebuilt_server', type: 'string', description: 'Path of the prebuilt ser option('override_server_path', type: 'string', description: 'Hardcoded path to find the server at runtime') option('skip_frames', type: 'boolean', value: true, description: 'Always display the most recent frame') option('hidpi_support', type: 'boolean', value: true, description: 'Enable High DPI support') +option('audio_support', type: 'boolean', value: true, description: 'Enable audio support')