From d39161f7539b9d335e38385ffff5c3b4c21c7122 Mon Sep 17 00:00:00 2001 From: Marco Martinelli <6640057+martinellimarco@users.noreply.github.com> Date: Sun, 4 Apr 2021 00:10:44 +0200 Subject: [PATCH] Add support for v4l2loopback It allows to send the video stream to /dev/videoN, so that it can be captured (like a webcam) by any v4l2-capable tool. PR #2232 PR #2233 PR #2268 Co-authored-by: Romain Vimont --- app/meson.build | 12 ++ app/scrcpy.1 | 6 + app/src/cli.c | 31 ++++ app/src/decoder.h | 2 +- app/src/main.c | 14 ++ app/src/scrcpy.c | 35 ++++- app/src/scrcpy.h | 2 + app/src/v4l2_sink.c | 341 ++++++++++++++++++++++++++++++++++++++++++++ app/src/v4l2_sink.h | 39 +++++ 9 files changed, 480 insertions(+), 2 deletions(-) create mode 100644 app/src/v4l2_sink.c create mode 100644 app/src/v4l2_sink.h diff --git a/app/meson.build b/app/meson.build index c2a59741..c1dbf489 100644 --- a/app/meson.build +++ b/app/meson.build @@ -33,6 +33,11 @@ else src += [ 'src/sys/unix/process.c' ] endif +v4l2_support = host_machine.system() == 'linux' +if v4l2_support + src += [ 'src/v4l2_sink.c' ] +endif + check_functions = [ 'strdup' ] @@ -49,6 +54,10 @@ if not get_option('crossbuild_windows') dependency('sdl2'), ] + if v4l2_support + dependencies += dependency('libavdevice') + endif + else # cross-compile mingw32 build (from Linux to Windows) @@ -124,6 +133,9 @@ conf.set('SERVER_DEBUGGER', get_option('server_debugger')) # select the debugger method ('old' for Android < 9, 'new' for Android >= 9) conf.set('SERVER_DEBUGGER_METHOD_NEW', get_option('server_debugger_method') == 'new') +# enable V4L2 support (linux only) +conf.set('HAVE_V4L2', v4l2_support) + configure_file(configuration: conf, output: 'config.h') src_dir = include_directories('src') diff --git a/app/scrcpy.1 b/app/scrcpy.1 index 51c618a8..644d9bcc 100644 --- a/app/scrcpy.1 +++ b/app/scrcpy.1 @@ -185,6 +185,12 @@ Enable "show touches" on start, restore the initial value on exit. It only shows physical touches (not clicks from scrcpy). +.TP +.BI "\-\-v4l2_sink " /dev/videoN +Output to v4l2loopback device. + +It requires to lock the video orientation (see --lock-video-orientation). + .TP .BI "\-V, \-\-verbosity " value Set the log level ("debug", "info", "warn" or "error"). diff --git a/app/src/cli.c b/app/src/cli.c index 6d5fd6b8..16c56c6a 100644 --- a/app/src/cli.c +++ b/app/src/cli.c @@ -176,6 +176,13 @@ scrcpy_print_usage(const char *arg0) { " on exit.\n" " It only shows physical touches (not clicks from scrcpy).\n" "\n" +#ifdef HAVE_V4L2 + " --v4l2_sink /dev/videoN\n" + " Output to v4l2loopback device.\n" + " It requires to lock the video orientation (see\n" + " --lock-video-orientation).\n" + "\n" +#endif " -V, --verbosity value\n" " Set the log level (debug, info, warn or error).\n" #ifndef NDEBUG @@ -676,6 +683,7 @@ guess_record_format(const char *filename) { #define OPT_LEGACY_PASTE 1024 #define OPT_ENCODER_NAME 1025 #define OPT_POWER_OFF_ON_CLOSE 1026 +#define OPT_V4L2_SINK 1027 bool scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { @@ -717,6 +725,9 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { {"show-touches", no_argument, NULL, 't'}, {"stay-awake", no_argument, NULL, 'w'}, {"turn-screen-off", no_argument, NULL, 'S'}, +#ifdef HAVE_V4L2 + {"v4l2_sink", required_argument, NULL, OPT_V4L2_SINK}, +#endif {"verbosity", required_argument, NULL, 'V'}, {"version", no_argument, NULL, 'v'}, {"window-title", required_argument, NULL, OPT_WINDOW_TITLE}, @@ -901,16 +912,36 @@ scrcpy_parse_args(struct scrcpy_cli_args *args, int argc, char *argv[]) { case OPT_POWER_OFF_ON_CLOSE: opts->power_off_on_close = true; break; +#ifdef HAVE_V4L2 + case OPT_V4L2_SINK: + opts->v4l2_device = optarg; + break; +#endif default: // getopt prints the error message on stderr return false; } } +#ifdef HAVE_V4L2 + if (!opts->display && !opts->record_filename && !opts->v4l2_device) { + LOGE("-N/--no-display requires either screen recording (-r/--record)" + " or sink to v4l2loopback device (--v4l2_sink)"); + return false; + } + + if (opts->v4l2_device && opts->lock_video_orientation + == SC_LOCK_VIDEO_ORIENTATION_UNLOCKED) { + LOGI("Video orientation is locked for v4l2 sink. " + "See --lock-video-orientation."); + opts->lock_video_orientation = SC_LOCK_VIDEO_ORIENTATION_INITIAL; + } +#else if (!opts->display && !opts->record_filename) { LOGE("-N/--no-display requires screen recording (-r/--record)"); return false; } +#endif int index = optind; if (index < argc) { diff --git a/app/src/decoder.h b/app/src/decoder.h index bae97869..257f751a 100644 --- a/app/src/decoder.h +++ b/app/src/decoder.h @@ -8,7 +8,7 @@ #include #include -#define DECODER_MAX_SINKS 1 +#define DECODER_MAX_SINKS 2 struct decoder { struct sc_packet_sink packet_sink; // packet sink trait diff --git a/app/src/main.c b/app/src/main.c index e1e44f68..a468aed7 100644 --- a/app/src/main.c +++ b/app/src/main.c @@ -6,6 +6,9 @@ #include #include #include +#ifdef HAVE_V4L2 +# include +#endif #define SDL_MAIN_HANDLED // avoid link error on Linux Windows Subsystem #include @@ -28,6 +31,11 @@ print_version(void) { fprintf(stderr, " - libavutil %d.%d.%d\n", LIBAVUTIL_VERSION_MAJOR, LIBAVUTIL_VERSION_MINOR, LIBAVUTIL_VERSION_MICRO); +#ifdef HAVE_V4L2 + fprintf(stderr, " - libavdevice %d.%d.%d\n", LIBAVDEVICE_VERSION_MAJOR, + LIBAVDEVICE_VERSION_MINOR, + LIBAVDEVICE_VERSION_MICRO); +#endif } static SDL_LogPriority @@ -90,6 +98,12 @@ main(int argc, char *argv[]) { av_register_all(); #endif +#ifdef HAVE_V4L2 + if (args.opts.v4l2_device) { + avdevice_register_all(); + } +#endif + if (avformat_network_init()) { return 1; } diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c index 4de62389..e847037e 100644 --- a/app/src/scrcpy.c +++ b/app/src/scrcpy.c @@ -27,6 +27,9 @@ #include "tiny_xpm.h" #include "util/log.h" #include "util/net.h" +#ifdef HAVE_V4L2 +# include "v4l2_sink.h" +#endif static struct server server; static struct screen screen; @@ -34,6 +37,9 @@ static struct fps_counter fps_counter; static struct stream stream; static struct decoder decoder; static struct recorder recorder; +#ifdef HAVE_V4L2 +static struct sc_v4l2_sink v4l2_sink; +#endif static struct controller controller; static struct file_handler file_handler; @@ -247,6 +253,9 @@ scrcpy(const struct scrcpy_options *options) { bool fps_counter_initialized = false; bool file_handler_initialized = false; bool recorder_initialized = false; +#ifdef HAVE_V4L2 + bool v4l2_sink_initialized = false; +#endif bool stream_started = false; bool controller_initialized = false; bool controller_started = false; @@ -295,7 +304,6 @@ scrcpy(const struct scrcpy_options *options) { goto end; } - struct decoder *dec = NULL; if (options->display) { if (!fps_counter_init(&fps_counter)) { goto end; @@ -309,7 +317,14 @@ scrcpy(const struct scrcpy_options *options) { } file_handler_initialized = true; } + } + struct decoder *dec = NULL; + bool needs_decoder = options->display; +#ifdef HAVE_V4L2 + needs_decoder |= !!options->v4l2_device; +#endif + if (needs_decoder) { decoder_init(&decoder); dec = &decoder; } @@ -386,6 +401,18 @@ scrcpy(const struct scrcpy_options *options) { } } +#ifdef HAVE_V4L2 + if (options->v4l2_device) { + if (!sc_v4l2_sink_init(&v4l2_sink, options->v4l2_device, frame_size)) { + goto end; + } + + decoder_add_sink(&decoder, &v4l2_sink.frame_sink); + + v4l2_sink_initialized = true; + } +#endif + // now we consumed the header values, the socket receives the video stream // start the stream if (!stream_start(&stream)) { @@ -426,6 +453,12 @@ end: stream_join(&stream); } +#ifdef HAVE_V4L2 + if (v4l2_sink_initialized) { + sc_v4l2_sink_destroy(&v4l2_sink); + } +#endif + // Destroy the screen only after the stream is guaranteed to be finished, // because otherwise the screen could receive new frames after destruction if (screen_initialized) { diff --git a/app/src/scrcpy.h b/app/src/scrcpy.h index 7c2689ab..405dc7f3 100644 --- a/app/src/scrcpy.h +++ b/app/src/scrcpy.h @@ -62,6 +62,7 @@ struct scrcpy_options { const char *render_driver; const char *codec_options; const char *encoder_name; + const char *v4l2_device; enum sc_log_level log_level; enum sc_record_format record_format; struct sc_port_range port_range; @@ -103,6 +104,7 @@ struct scrcpy_options { .render_driver = NULL, \ .codec_options = NULL, \ .encoder_name = NULL, \ + .v4l2_device = NULL, \ .log_level = SC_LOG_LEVEL_INFO, \ .record_format = SC_RECORD_FORMAT_AUTO, \ .port_range = { \ diff --git a/app/src/v4l2_sink.c b/app/src/v4l2_sink.c new file mode 100644 index 00000000..b7d06bb6 --- /dev/null +++ b/app/src/v4l2_sink.c @@ -0,0 +1,341 @@ +#include "v4l2_sink.h" + +#include "util/log.h" +#include "util/str_util.h" + +/** Downcast frame_sink to sc_v4l2_sink */ +#define DOWNCAST(SINK) container_of(SINK, struct sc_v4l2_sink, frame_sink) + +static const AVRational SCRCPY_TIME_BASE = {1, 1000000}; // timestamps in us + +static const AVOutputFormat * +find_muxer(const char *name) { +#ifdef SCRCPY_LAVF_HAS_NEW_MUXER_ITERATOR_API + void *opaque = NULL; +#endif + const AVOutputFormat *oformat = NULL; + do { +#ifdef SCRCPY_LAVF_HAS_NEW_MUXER_ITERATOR_API + oformat = av_muxer_iterate(&opaque); +#else + oformat = av_oformat_next(oformat); +#endif + // until null or containing the requested name + } while (oformat && !strlist_contains(oformat->name, ',', name)); + return oformat; +} + +static bool +write_header(struct sc_v4l2_sink *vs, const AVPacket *packet) { + AVStream *ostream = vs->format_ctx->streams[0]; + + uint8_t *extradata = av_malloc(packet->size * sizeof(uint8_t)); + if (!extradata) { + LOGC("Could not allocate extradata"); + return false; + } + + // copy the first packet to the extra data + memcpy(extradata, packet->data, packet->size); + + ostream->codecpar->extradata = extradata; + ostream->codecpar->extradata_size = packet->size; + + int ret = avformat_write_header(vs->format_ctx, NULL); + if (ret < 0) { + LOGE("Failed to write header to %s", vs->device_name); + return false; + } + + return true; +} + +static void +rescale_packet(struct sc_v4l2_sink *vs, AVPacket *packet) { + AVStream *ostream = vs->format_ctx->streams[0]; + av_packet_rescale_ts(packet, SCRCPY_TIME_BASE, ostream->time_base); +} + +static bool +write_packet(struct sc_v4l2_sink *vs, AVPacket *packet) { + if (!vs->header_written) { + bool ok = write_header(vs, packet); + if (!ok) { + return false; + } + vs->header_written = true; + return true; + } + + rescale_packet(vs, packet); + + bool ok = av_write_frame(vs->format_ctx, packet) >= 0; + + // Failing to write the last frame is not very serious, no future frame may + // depend on it, so the resulting file will still be valid + (void) ok; + + return true; +} + +static bool +encode_and_write_frame(struct sc_v4l2_sink *vs, const AVFrame *frame) { + int ret = avcodec_send_frame(vs->encoder_ctx, frame); + if (ret < 0 && ret != AVERROR(EAGAIN)) { + LOGE("Could not send v4l2 video frame: %d", ret); + return false; + } + + AVPacket *packet = &vs->packet; + ret = avcodec_receive_packet(vs->encoder_ctx, packet); + if (ret == 0) { + // A packet was received + + bool ok = write_packet(vs, packet); + if (!ok) { + LOGW("Could not send packet to v4l2 sink"); + return false; + } + av_packet_unref(packet); + } else if (ret != AVERROR(EAGAIN)) { + LOGE("Could not receive v4l2 video packet: %d", ret); + return false; + } + + return true; +} + +static int +run_v4l2_sink(void *data) { + struct sc_v4l2_sink *vs = data; + + for (;;) { + sc_mutex_lock(&vs->mutex); + + while (!vs->stopped && vs->vb.pending_frame_consumed) { + sc_cond_wait(&vs->cond, &vs->mutex); + } + + if (vs->stopped) { + sc_mutex_unlock(&vs->mutex); + break; + } + + sc_mutex_unlock(&vs->mutex); + + video_buffer_consume(&vs->vb, vs->frame); + bool ok = encode_and_write_frame(vs, vs->frame); + if (!ok) { + LOGE("Could not send frame to v4l2 sink"); + break; + } + } + + LOGD("V4l2 thread ended"); + + return 0; +} + +static bool +sc_v4l2_sink_open(struct sc_v4l2_sink *vs) { + bool ok = video_buffer_init(&vs->vb); + if (!ok) { + return false; + } + + ok = sc_mutex_init(&vs->mutex); + if (!ok) { + LOGC("Could not create mutex"); + goto error_video_buffer_destroy; + } + + ok = sc_cond_init(&vs->cond); + if (!ok) { + LOGC("Could not create cond"); + goto error_mutex_destroy; + } + + // FIXME + const AVOutputFormat *format = find_muxer("video4linux2,v4l2"); + if (!format) { + LOGE("Could not find v4l2 muxer"); + goto error_cond_destroy; + } + + const AVCodec *encoder = avcodec_find_encoder(AV_CODEC_ID_RAWVIDEO); + if (!encoder) { + LOGE("Raw video encoder not found"); + return false; + } + + vs->format_ctx = avformat_alloc_context(); + if (!vs->format_ctx) { + LOGE("Could not allocate v4l2 output context"); + return false; + } + + // contrary to the deprecated API (av_oformat_next()), av_muxer_iterate() + // returns (on purpose) a pointer-to-const, but AVFormatContext.oformat + // still expects a pointer-to-non-const (it has not be updated accordingly) + // + vs->format_ctx->oformat = (AVOutputFormat *) format; + vs->format_ctx->url = strdup(vs->device_name); + if (!vs->format_ctx->url) { + LOGE("Could not strdup v4l2 device name"); + goto error_avformat_free_context; + return false; + } + + AVStream *ostream = avformat_new_stream(vs->format_ctx, encoder); + if (!ostream) { + LOGE("Could not allocate new v4l2 stream"); + goto error_avformat_free_context; + return false; + } + + ostream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + ostream->codecpar->codec_id = encoder->id; + ostream->codecpar->format = AV_PIX_FMT_YUV420P; + ostream->codecpar->width = vs->frame_size.width; + ostream->codecpar->height = vs->frame_size.height; + + int ret = avio_open(&vs->format_ctx->pb, vs->device_name, AVIO_FLAG_WRITE); + if (ret < 0) { + LOGE("Failed to open output device: %s", vs->device_name); + // ostream will be cleaned up during context cleaning + goto error_avformat_free_context; + } + + vs->encoder_ctx = avcodec_alloc_context3(encoder); + if (!vs->encoder_ctx) { + LOGC("Could not allocate codec context for v4l2"); + goto error_avio_close; + } + + vs->encoder_ctx->width = vs->frame_size.width; + vs->encoder_ctx->height = vs->frame_size.height; + vs->encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P; + vs->encoder_ctx->time_base.num = 1; + vs->encoder_ctx->time_base.den = 1; + + if (avcodec_open2(vs->encoder_ctx, encoder, NULL) < 0) { + LOGE("Could not open codec for v4l2"); + goto error_avcodec_free_context; + } + + vs->frame = av_frame_alloc(); + if (!vs->frame) { + LOGE("Could not create v4l2 frame"); + goto error_avcodec_close; + } + + LOGD("Starting v4l2 thread"); + ok = sc_thread_create(&vs->thread, run_v4l2_sink, "v4l2", vs); + if (!ok) { + LOGC("Could not start v4l2 thread"); + goto error_av_frame_free; + } + + vs->header_written = false; + vs->stopped = false; + + LOGI("v4l2 sink started to device: %s", vs->device_name); + + return true; + +error_av_frame_free: + av_frame_free(&vs->frame); +error_avcodec_close: + avcodec_close(vs->encoder_ctx); +error_avcodec_free_context: + avcodec_free_context(&vs->encoder_ctx); +error_avio_close: + avio_close(vs->format_ctx->pb); +error_avformat_free_context: + avformat_free_context(vs->format_ctx); +error_cond_destroy: + sc_cond_destroy(&vs->cond); +error_mutex_destroy: + sc_mutex_destroy(&vs->mutex); +error_video_buffer_destroy: + video_buffer_destroy(&vs->vb); + + return false; +} + +static void +sc_v4l2_sink_close(struct sc_v4l2_sink *vs) { + sc_mutex_lock(&vs->mutex); + vs->stopped = true; + sc_cond_signal(&vs->cond); + sc_mutex_unlock(&vs->mutex); + + sc_thread_join(&vs->thread, NULL); + + av_frame_free(&vs->frame); + avcodec_close(vs->encoder_ctx); + avcodec_free_context(&vs->encoder_ctx); + avio_close(vs->format_ctx->pb); + avformat_free_context(vs->format_ctx); + sc_cond_destroy(&vs->cond); + sc_mutex_destroy(&vs->mutex); + video_buffer_destroy(&vs->vb); +} + +static bool +sc_v4l2_sink_push(struct sc_v4l2_sink *vs, const AVFrame *frame) { + bool ok = video_buffer_push(&vs->vb, frame, NULL); + if (!ok) { + return false; + } + + // signal possible change of vs->vb.pending_frame_consumed + sc_cond_signal(&vs->cond); + + return true; +} + +static bool +sc_v4l2_frame_sink_open(struct sc_frame_sink *sink) { + struct sc_v4l2_sink *vs = DOWNCAST(sink); + return sc_v4l2_sink_open(vs); +} + +static void +sc_v4l2_frame_sink_close(struct sc_frame_sink *sink) { + struct sc_v4l2_sink *vs = DOWNCAST(sink); + sc_v4l2_sink_close(vs); +} + +static bool +sc_v4l2_frame_sink_push(struct sc_frame_sink *sink, const AVFrame *frame) { + struct sc_v4l2_sink *vs = DOWNCAST(sink); + return sc_v4l2_sink_push(vs, frame); +} + +bool +sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name, + struct size frame_size) { + vs->device_name = strdup(device_name); + if (!vs->device_name) { + LOGE("Could not strdup v4l2 device name"); + return false; + } + + vs->frame_size = frame_size; + + static const struct sc_frame_sink_ops ops = { + .open = sc_v4l2_frame_sink_open, + .close = sc_v4l2_frame_sink_close, + .push = sc_v4l2_frame_sink_push, + }; + + vs->frame_sink.ops = &ops; + + return true; +} + +void +sc_v4l2_sink_destroy(struct sc_v4l2_sink *vs) { + free(vs->device_name); +} diff --git a/app/src/v4l2_sink.h b/app/src/v4l2_sink.h new file mode 100644 index 00000000..9d2ee149 --- /dev/null +++ b/app/src/v4l2_sink.h @@ -0,0 +1,39 @@ +#ifndef SC_V4L2_SINK_H +#define SC_V4L2_SINK_H + +#include "common.h" + +#include "coords.h" +#include "trait/frame_sink.h" +#include "video_buffer.h" + +#include + +struct sc_v4l2_sink { + struct sc_frame_sink frame_sink; // frame sink trait + + struct video_buffer vb; + AVFormatContext *format_ctx; + AVCodecContext *encoder_ctx; + + char *device_name; + struct size frame_size; + + sc_thread thread; + sc_mutex mutex; + sc_cond cond; + bool stopped; + bool header_written; + + AVFrame *frame; + AVPacket packet; +}; + +bool +sc_v4l2_sink_init(struct sc_v4l2_sink *vs, const char *device_name, + struct size frame_size); + +void +sc_v4l2_sink_destroy(struct sc_v4l2_sink *vs); + +#endif