Skip to content

Display and control your Android device

Select the video decoder (hwdec)

Romain Vimontc0dc7b2e768a
diff --git a/app/meson.build b/app/meson.build
index db35f514..413bba5a 100644
--- a/app/meson.build
+++ b/app/meson.build
@@ -19,6 +19,7 @@ src = [
'src/file_pusher.c',
'src/fps_counter.c',
'src/frame_buffer.c',
+ 'src/hwdec.c',
'src/icon.c',
'src/input_manager.c',
'src/keyboard_sdk.c',
diff --git a/app/src/decoder.c b/app/src/decoder.c
index 27bc0f5c..c6911153 100644
--- a/app/src/decoder.c
+++ b/app/src/decoder.c
@@ -37,6 +37,12 @@ sc_decoder_open(struct sc_decoder *decoder, const AVCodec *codec,
decoder->ctx->flags |= AV_CODEC_FLAG_COPY_OPAQUE;
}
+ if (decoder->hwdec && !sc_hwdec_configure(decoder->hwdec, decoder->ctx)) {
+ // Use software decoding (the codec context is left untouched)
+ LOGW("Decoder '%s': hardware decoding unavailable, using software "
+ "decoding", decoder->name);
+ }
+
r = avcodec_open2(decoder->ctx, codec, NULL);
if (r < 0) {
LOGE("Decoder '%s': could not open codec", decoder->name);
@@ -346,7 +352,8 @@ sc_decoder_packet_sink_push_session(struct sc_packet_sink *sink,
}
bool
-sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque,
+sc_decoder_init(struct sc_decoder *decoder, const char *name,
+ struct sc_hwdec *hwdec, bool copy_opaque,
const struct sc_decoder_callbacks *cbs, void *cbs_userdata) {
bool ok = sc_mutex_init(&decoder->mutex);
if (!ok) {
@@ -362,6 +369,7 @@ sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque,
decoder->name = name; // statically allocated
sc_frame_source_init(&decoder->frame_source);
+ decoder->hwdec = hwdec;
decoder->copy_opaque = copy_opaque;
sc_vecdeque_init(&decoder->queue);
diff --git a/app/src/decoder.h b/app/src/decoder.h
index da5724d3..d3530476 100644
--- a/app/src/decoder.h
+++ b/app/src/decoder.h
@@ -6,6 +6,7 @@
#include <libavcodec/avcodec.h>
#include "coords.h"
+#include "hwdec.h"
#include "trait/frame_source.h"
#include "trait/packet_sink.h"
#include "util/thread.h"
@@ -33,6 +34,8 @@ struct sc_decoder {
const char *name; // must be statically allocated (e.g. a string literal)
bool copy_opaque;
+ struct sc_hwdec *hwdec; // optional, only for video
+
sc_thread thread;
sc_mutex mutex;
sc_cond cond;
@@ -59,8 +62,11 @@ struct sc_decoder_callbacks {
// The name must be statically allocated (e.g. a string literal)
//
// copy_opaque: Set to true to forward the AVPacket.opaque_ref to the AVFrame
+// hwdec: The video decoder backend (NULL for audio), owned by the caller (must
+// outlive the decoder)
bool
-sc_decoder_init(struct sc_decoder *decoder, const char *name, bool copy_opaque,
+sc_decoder_init(struct sc_decoder *decoder, const char *name,
+ struct sc_hwdec *hwdec, bool copy_opaque,
const struct sc_decoder_callbacks *cbs, void *cbs_userdata);
bool
diff --git a/app/src/hwdec.c b/app/src/hwdec.c
new file mode 100644
index 00000000..3960b3a8
--- /dev/null
+++ b/app/src/hwdec.c
@@ -0,0 +1,81 @@
+#include "hwdec.h"
+
+#include <assert.h>
+
+#include "util/log.h"
+
+bool
+sc_hwdec_init(struct sc_hwdec *hwdec, enum AVHWDeviceType hw_type,
+ SDL_Renderer *renderer) {
+ (void) renderer; // only used by some hardware decoders
+
+ hwdec->hw_type = hw_type;
+ switch (hw_type) {
+ case AV_HWDEVICE_TYPE_NONE:
+ hwdec->name = "software";
+ hwdec->hw_device_ctx = NULL;
+ break;
+ default:
+ LOGE("No decoder for hardware device type: %s",
+ av_hwdevice_get_type_name(hw_type));
+ return false;
+ }
+
+ LOGI("Video decoding: %s", hwdec->name);
+ return true;
+}
+
+static bool
+sc_hwdec_is_supported_by_codec(const AVCodec *codec,
+ enum AVHWDeviceType hw_type) {
+ for (int i = 0;; ++i) {
+ const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i);
+ if (!config) {
+ return false;
+ }
+
+ bool device_ctx =
+ config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX;
+ if (device_ctx && config->device_type == hw_type) {
+ return true;
+ }
+ }
+}
+
+bool
+sc_hwdec_configure(struct sc_hwdec *hwdec, AVCodecContext *ctx) {
+ assert(ctx->codec_type == AVMEDIA_TYPE_VIDEO);
+ assert(ctx->codec);
+ assert(!ctx->hw_device_ctx);
+
+ if (hwdec->hw_type == AV_HWDEVICE_TYPE_NONE) {
+ // Software decoding, nothing to configure
+ return true;
+ }
+
+ if (!sc_hwdec_is_supported_by_codec(ctx->codec, hwdec->hw_type)) {
+ LOGW("%s is not supported by the %s decoder",
+ hwdec->name, ctx->codec->name);
+ return false;
+ }
+
+ assert(hwdec->hw_device_ctx);
+ ctx->hw_device_ctx = av_buffer_ref(hwdec->hw_device_ctx);
+ if (!ctx->hw_device_ctx) {
+ LOG_OOM();
+ return false;
+ }
+
+ // Some backends do not expose plain Baseline (only Constrained Baseline).
+ // Hardware supporting Main/High can still decode Baseline.
+ ctx->hwaccel_flags |= AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH;
+
+ return true;
+}
+
+void
+sc_hwdec_destroy(struct sc_hwdec *hwdec) {
+ if (hwdec->hw_device_ctx) {
+ av_buffer_unref(&hwdec->hw_device_ctx);
+ }
+}
diff --git a/app/src/hwdec.h b/app/src/hwdec.h
new file mode 100644
index 00000000..7960798c
--- /dev/null
+++ b/app/src/hwdec.h
@@ -0,0 +1,30 @@
+#ifndef SC_HWDEC_H
+#define SC_HWDEC_H
+
+#include "common.h"
+
+#include <stdbool.h>
+
+#include <libavcodec/avcodec.h>
+#include <libavutil/buffer.h>
+#include <libavutil/hwcontext.h>
+#include <SDL3/SDL.h>
+
+struct sc_hwdec {
+ const char *name; // must be statically allocated (e.g. a string literal)
+
+ enum AVHWDeviceType hw_type;
+ AVBufferRef *hw_device_ctx;
+};
+
+bool
+sc_hwdec_init(struct sc_hwdec *hwdec, enum AVHWDeviceType hw_type,
+ SDL_Renderer *renderer);
+
+bool
+sc_hwdec_configure(struct sc_hwdec *hwdec, AVCodecContext *ctx);
+
+void
+sc_hwdec_destroy(struct sc_hwdec *hwdec);
+
+#endif
diff --git a/app/src/interop/interop.c b/app/src/interop/interop.c
index f87d9db4..a599e1f0 100644
--- a/app/src/interop/interop.c
+++ b/app/src/interop/interop.c
@@ -6,20 +6,20 @@
#include "util/log.h"
struct sc_interop *
-sc_interop_new(enum AVHWDeviceType hw_type, SDL_Renderer *renderer,
+sc_interop_new(enum sc_hwdec_mode hwdec_mode, SDL_Renderer *renderer,
struct sc_opengl *gl, bool mipmaps) {
- switch (hw_type) {
- case AV_HWDEVICE_TYPE_NONE: {
- struct sc_interop_software *sw =
- sc_interop_software_new(renderer, gl, mipmaps);
- if (!sw) {
- return NULL;
- }
+ bool any = hwdec_mode == SC_HWDEC_MODE_AUTO;
+
+ if (any || hwdec_mode == SC_HWDEC_MODE_DISABLED) {
+ struct sc_interop_software *sw =
+ sc_interop_software_new(renderer, gl, mipmaps);
+ if (sw) {
return &sw->interop;
}
- default:
- LOGE("hw_type not supported: %s", av_hwdevice_get_type_name(hw_type));
+ LOGI("Software interop failed");
}
+
+ LOGE("No compatible interop");
return NULL;
}
diff --git a/app/src/interop/interop.h b/app/src/interop/interop.h
index 4bc7dbfe..fa5a4395 100644
--- a/app/src/interop/interop.h
+++ b/app/src/interop/interop.h
@@ -12,10 +12,12 @@
#include "coords.h"
#include "opengl.h"
+#include "options.h"
struct sc_interop {
const char *name;
+ enum AVHWDeviceType hw_type;
enum AVPixelFormat pix_fmt;
SDL_Texture *texture;
@@ -37,7 +39,7 @@ struct sc_interop_ops {
};
struct sc_interop *
-sc_interop_new(enum AVHWDeviceType hw_type, SDL_Renderer *renderer,
+sc_interop_new(enum sc_hwdec_mode hwdec_mode, SDL_Renderer *renderer,
struct sc_opengl *gl, bool mipmaps);
void
diff --git a/app/src/interop/interop_software.c b/app/src/interop/interop_software.c
index 043e1ccf..cccc6dbd 100644
--- a/app/src/interop/interop_software.c
+++ b/app/src/interop/interop_software.c
@@ -169,6 +169,7 @@ sc_interop_software_new(SDL_Renderer *renderer, struct sc_opengl *gl,
};
sw->interop.name = "software";
+ sw->interop.hw_type = AV_HWDEVICE_TYPE_NONE;
sw->interop.pix_fmt = AV_PIX_FMT_YUV420P;
sw->interop.texture = NULL;
sw->interop.ops = &ops;
diff --git a/app/src/options.h b/app/src/options.h
index ffe88fe5..fd6ce355 100644
--- a/app/src/options.h
+++ b/app/src/options.h
@@ -76,6 +76,18 @@ enum sc_hwdec_mode {
SC_HWDEC_MODE_DISABLED,
};
+static inline const char *
+sc_hwdec_mode_get_name(enum sc_hwdec_mode mode) {
+ switch (mode) {
+ case SC_HWDEC_MODE_AUTO:
+ return "auto";
+ case SC_HWDEC_MODE_DISABLED:
+ return "disabled";
+ default:
+ return "(unknown)";
+ }
+}
+
enum sc_camera_facing {
SC_CAMERA_FACING_ANY,
SC_CAMERA_FACING_FRONT,
diff --git a/app/src/scrcpy.c b/app/src/scrcpy.c
index ccc5d408..77a51c27 100644
--- a/app/src/scrcpy.c
+++ b/app/src/scrcpy.c
@@ -20,6 +20,7 @@
#include "demuxer.h"
#include "events.h"
#include "file_pusher.h"
+#include "hwdec.h"
#include "keyboard_sdk.h"
#include "mouse_sdk.h"
#include "recorder.h"
@@ -58,6 +59,7 @@ struct scrcpy {
struct sc_demuxer audio_demuxer;
struct sc_decoder video_decoder;
struct sc_decoder audio_decoder;
+ struct sc_hwdec hwdec;
struct sc_recorder recorder;
struct sc_video_regulator video_regulator;
#ifdef HAVE_V4L2
@@ -332,6 +334,41 @@ set_terminal_title_with_prefix(const char *value) {
sc_term_set_title(title);
}
+static bool
+sc_init_video_hwdec(struct sc_hwdec *hwdec, enum sc_hwdec_mode mode,
+ struct sc_screen *screen) {
+ enum AVHWDeviceType hw_type = AV_HWDEVICE_TYPE_NONE;
+ SDL_Renderer *renderer = NULL;
+ if (screen) {
+ hw_type = sc_screen_get_hw_type(screen);
+ renderer = screen->renderer;
+ }
+ if (sc_hwdec_init(hwdec, hw_type, renderer)) {
+ return true;
+ }
+
+ // The software decoder cannot fail
+ assert(hw_type != AV_HWDEVICE_TYPE_NONE);
+
+ if (mode != SC_HWDEC_MODE_AUTO) {
+ LOGE("Hardware decoder %s unavailable",
+ av_hwdevice_get_type_name(hw_type));
+ return false;
+ }
+
+ // The screen created its interop according to the requested hardware
+ // decoder mode, but in the end the hardware decoder is unavailable, fall
+ // back to software decoding and replace the screen interop with a software
+ // interop.
+ LOGI("Hardware decoding unavailable; using software decoding");
+ if (!sc_screen_disable_hwdec(screen)) {
+ return false;
+ }
+
+ // Initialize software decoder
+ return sc_hwdec_init(hwdec, AV_HWDEVICE_TYPE_NONE, NULL);
+}
+
enum scrcpy_exit_code
scrcpy(struct scrcpy_options *options) {
static struct scrcpy scrcpy;
@@ -362,6 +399,7 @@ scrcpy(struct scrcpy_options *options) {
bool video_regulator_initialized = false;
bool video_demuxer_started = false;
bool audio_demuxer_started = false;
+ bool hwdec_initialized = false;
bool video_decoder_initialized = false;
bool video_decoder_started = false;
bool audio_decoder_initialized = false;
@@ -796,6 +834,7 @@ aoa_complete:
.render_fit = options->render_fit,
.orientation = options->display_orientation,
.mipmaps = options->mipmaps,
+ .hwdec_mode = options->hwdec_mode,
.fullscreen = options->fullscreen,
.start_fps_counter = options->start_fps_counter,
};
@@ -812,10 +851,18 @@ aoa_complete:
needs_video_decoder |= !!options->v4l2_device;
#endif
if (needs_video_decoder) {
+ struct sc_screen *screen = options->video_playback ? &s->screen : NULL;
+ assert(!screen || screen_initialized);
+
+ if (!sc_init_video_hwdec(&s->hwdec, options->hwdec_mode, screen)) {
+ goto end;
+ }
+ hwdec_initialized = true;
+
// If a video buffer is present, then the recv date must be forwarded
// from the AVPacket to the AVFrame
bool copy_opaque = has_video_buffer;
- if (!sc_decoder_init(&s->video_decoder, "video", copy_opaque,
+ if (!sc_decoder_init(&s->video_decoder, "video", &s->hwdec, copy_opaque,
&decoder_cbs, NULL)) {
goto end;
}
@@ -831,7 +878,7 @@ aoa_complete:
}
if (needs_audio_decoder) {
- if (!sc_decoder_init(&s->audio_decoder, "audio", false,
+ if (!sc_decoder_init(&s->audio_decoder, "audio", NULL, false,
&decoder_cbs, NULL)) {
goto end;
}
@@ -1114,6 +1161,11 @@ end:
sc_decoder_destroy(&s->video_decoder);
}
+ // The hwdec must outlive the video decoder
+ if (hwdec_initialized) {
+ sc_hwdec_destroy(&s->hwdec);
+ }
+
if (audio_decoder_initialized) {
sc_decoder_destroy(&s->audio_decoder);
}
diff --git a/app/src/screen.c b/app/src/screen.c
index 3e6bad71..fec823e7 100644
--- a/app/src/screen.c
+++ b/app/src/screen.c
@@ -645,7 +645,7 @@ sc_screen_init(struct sc_screen *screen,
bool mipmaps = params->video && params->mipmaps;
ok = sc_texture_init(&screen->tex, screen->renderer, mipmaps,
- AV_HWDEVICE_TYPE_NONE);
+ params->hwdec_mode);
if (!ok) {
goto error_destroy_renderer;
}
diff --git a/app/src/screen.h b/app/src/screen.h
index 72503116..9b641542 100644
--- a/app/src/screen.h
+++ b/app/src/screen.h
@@ -146,6 +146,8 @@ struct sc_screen_params {
enum sc_orientation orientation;
bool mipmaps;
+ enum sc_hwdec_mode hwdec_mode;
+
bool fullscreen;
bool start_fps_counter;
};
@@ -209,4 +211,14 @@ struct sc_point
sc_screen_convert_window_to_frame_coords(struct sc_screen *screen,
int32_t x, int32_t y);
+static inline enum AVHWDeviceType
+sc_screen_get_hw_type(struct sc_screen *screen) {
+ return sc_texture_get_hw_type(&screen->tex);
+}
+
+static inline bool
+sc_screen_disable_hwdec(struct sc_screen *screen) {
+ return sc_texture_disable_hwdec(&screen->tex);
+}
+
#endif
diff --git a/app/src/texture.c b/app/src/texture.c
index a18fc686..5998c22e 100644
--- a/app/src/texture.c
+++ b/app/src/texture.c
@@ -3,11 +3,13 @@
#include <assert.h>
#include <string.h>
+#include <libavutil/pixdesc.h>
+
#include "util/log.h"
bool
sc_texture_init(struct sc_texture *tex, SDL_Renderer *renderer, bool mipmaps,
- enum AVHWDeviceType hw_type) {
+ enum sc_hwdec_mode hwdec_mode) {
const char *renderer_name = SDL_GetRendererName(renderer);
LOGI("Renderer: %s", renderer_name ? renderer_name : "(unknown)");
@@ -44,11 +46,13 @@ sc_texture_init(struct sc_texture *tex, SDL_Renderer *renderer, bool mipmaps,
}
tex->renderer = renderer;
- tex->interop = sc_interop_new(hw_type, tex->renderer, gl, tex->mipmaps);
+ tex->interop = sc_interop_new(hwdec_mode, tex->renderer, gl, tex->mipmaps);
if (!tex->interop) {
return false;
}
+ LOGI("Interop: %s", tex->interop->name);
+
return true;
}
@@ -63,16 +67,55 @@ sc_texture_get(struct sc_texture *tex) {
return tex->interop->texture;
}
+enum AVHWDeviceType
+sc_texture_get_hw_type(struct sc_texture *tex) {
+ return tex->interop->hw_type;
+}
+
struct sc_size
sc_texture_get_frame_size(struct sc_texture *tex) {
assert(tex->interop && tex->interop->texture);
return tex->interop->frame_size;
}
+bool
+sc_texture_disable_hwdec(struct sc_texture *tex) {
+ assert(tex->interop);
+ assert(tex->interop->hw_type != AV_HWDEVICE_TYPE_NONE);
+
+ struct sc_opengl *gl = tex->has_gl ? &tex->gl : NULL;
+ struct sc_interop *interop =
+ sc_interop_new(SC_HWDEC_MODE_DISABLED, tex->renderer, gl, tex->mipmaps);
+ if (!interop) {
+ return false;
+ }
+
+ LOGD("Interop: %s dropped", tex->interop->name);
+ sc_interop_delete(tex->interop);
+
+ tex->interop = interop;
+
+ LOGI("Interop: %s", tex->interop->name);
+ return true;
+}
+
bool
sc_texture_update(struct sc_texture *tex, const AVFrame *frame) {
+ assert(tex->interop);
+
+ if (tex->interop->hw_type != AV_HWDEVICE_TYPE_NONE
+ && tex->interop->pix_fmt != frame->format) {
+ LOGI("Incompatible frame, switching to software interop");
+ if (!sc_texture_disable_hwdec(tex)) {
+ return false;
+ }
+ }
+
struct sc_interop *interop = tex->interop;
- if (!interop) {
+
+ if (interop->pix_fmt != frame->format) {
+ const char *name = av_get_pix_fmt_name(frame->format);
+ LOGE("Unsupported frames format: %s", name ? name : "(unknown)");
return false;
}
diff --git a/app/src/texture.h b/app/src/texture.h
index 6b536d68..f450355e 100644
--- a/app/src/texture.h
+++ b/app/src/texture.h
@@ -12,6 +12,7 @@
#include "coords.h"
#include "interop/interop.h"
#include "opengl.h"
+#include "options.h"
struct sc_texture {
SDL_Renderer *renderer; // owned by the caller
@@ -25,7 +26,7 @@ struct sc_texture {
bool
sc_texture_init(struct sc_texture *tex, SDL_Renderer *renderer, bool mipmaps,
- enum AVHWDeviceType hw_type);
+ enum sc_hwdec_mode hwdec_mode);
void
sc_texture_destroy(struct sc_texture *tex);
@@ -42,4 +43,10 @@ sc_texture_update(struct sc_texture *tex, const AVFrame *frame);
void
sc_texture_reset(struct sc_texture *tex);
+enum AVHWDeviceType
+sc_texture_get_hw_type(struct sc_texture *tex);
+
+bool
+sc_texture_disable_hwdec(struct sc_texture *tex);
+
#endif