Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

feat(media-server): decode hevc/av1 on the gpu with cuvid resize before nvenc

Richie McIlroy68c5f45e9afc
diff --git a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts
index f598e3d80..b698a77bd 100644
--- a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts
+++ b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts
@@ -15,8 +15,8 @@ import {
} from "../../lib/media-video";
import {
buildNvencEncodeArgs,
- buildNvencScaleFilter,
canDecodeOnGpu,
+ fitWithinEven,
getVideoEncoderStatus,
type NvencOptions,
nvencCq,
@@ -33,6 +33,7 @@ const defaults: NvencOptions = {
lookahead: 20,
spatialAq: true,
temporalAq: false,
+ gpuDecodeCodecs: ["hevc", "av1"],
};
const opts: ProcessVideoArgsInput["opts"] = {
maxWidth: 1920,
@@ -63,8 +64,7 @@ const nvencVideoArgs = [
"-g",
"250",
];
-const gpuScale =
- "scale_cuda='min(1920,iw)':'min(1080,ih)':force_original_aspect_ratio=decrease:format=nv12,scale_cuda=trunc(iw/2)*2:trunc(ih/2)*2";
+const gpuScale = "scale_cuda=format=nv12";
const cpuScale =
"scale='min(1920,iw)':'min(1080,ih)':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2";
@@ -104,6 +104,8 @@ function processArgs(
extraInputArgs: [],
extraOutputArgs: [],
decodeThreads: "2",
+ sourceWidth: source.width,
+ sourceHeight: source.height,
...overrides,
};
}
@@ -159,7 +161,14 @@ describe("nvenc encoder selection", () => {
lookahead: 0,
spatialAq: false,
temporalAq: true,
+ gpuDecodeCodecs: ["hevc", "av1"],
});
+ expect(
+ nvencOptionsFromEnv({
+ ...enabled,
+ CAP_NVENC_GPU_DECODE_CODECS: "h264, HEVC,vp8,nope",
+ })?.gpuDecodeCodecs,
+ ).toEqual(["h264", "hevc"]);
});
test("ignores invalid tuning overrides", () => {
@@ -213,14 +222,31 @@ describe("nvenc encoder selection", () => {
]);
});
- test("scales on the GPU with the CPU path's box and even dimensions", () => {
- expect(buildNvencScaleFilter(1920, 1080)).toBe(gpuScale);
+ test("fits the GPU resize to the CPU path's box and even dimensions", () => {
+ expect(fitWithinEven(3840, 2160, 1920, 1080)).toEqual({
+ width: 1920,
+ height: 1080,
+ });
+ expect(fitWithinEven(3024, 1964, 1920, 1080)).toEqual({
+ width: 1662,
+ height: 1080,
+ });
+ expect(fitWithinEven(1079, 1919, 1920, 1080)).toEqual({
+ width: 606,
+ height: 1080,
+ });
+ expect(fitWithinEven(1281, 721, 1920, 1080)).toEqual({
+ width: 1280,
+ height: 720,
+ });
});
- test("decodes on the GPU only for unrotated NVDEC codecs", () => {
- for (const codec of ["h264", "hevc", "vp9", "av1"]) {
+ test("decodes on the GPU only for configured, unrotated codecs", () => {
+ for (const codec of ["hevc", "av1"]) {
expect(canDecodeOnGpu(codec, 0)).toBe(true);
}
+ expect(canDecodeOnGpu("h264", 0)).toBe(false);
+ expect(canDecodeOnGpu("h264", 0, ["h264"])).toBe(true);
expect(canDecodeOnGpu("vp8", 0)).toBe(false);
expect(canDecodeOnGpu("prores", 0)).toBe(false);
expect(canDecodeOnGpu("hevc", 90)).toBe(false);
@@ -307,6 +333,10 @@ describe("process video args", () => {
"cuda",
"-hwaccel_output_format",
"cuda",
+ "-c:v",
+ "h264_cuvid",
+ "-resize",
+ "1662x1080",
"-i",
"in.mp4",
...nvencVideoArgs,
@@ -336,6 +366,10 @@ describe("process video args", () => {
"cuda",
"-hwaccel_output_format",
"cuda",
+ "-c:v",
+ "hevc_cuvid",
+ "-resize",
+ "1920x1080",
"-i",
"in.mov",
...nvencVideoArgs,
@@ -374,6 +408,8 @@ describe("process video args", () => {
"cuda",
"-hwaccel_output_format",
"cuda",
+ "-c:v",
+ "vp9_cuvid",
"-i",
"in.webm",
...nvencVideoArgs,
diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts
index feecf5830..591996736 100644
--- a/apps/media-server/src/lib/media-video.ts
+++ b/apps/media-server/src/lib/media-video.ts
@@ -23,11 +23,12 @@ import {
import { probeVideoFile } from "./media-probe";
import { fetchMedia, materializeMedia } from "./media-transfer";
import {
+ buildGpuDecodeInputArgs,
buildNvencEncodeArgs,
- buildNvencScaleFilter,
canDecodeOnGpu,
+ fitWithinEven,
+ GPU_DECODE_FORMAT_FILTER,
logNvencFallback,
- NVENC_HWACCEL_INPUT_ARGS,
type NvencOptions,
nvencFallbackReason,
nvencOptionsFromEnv,
@@ -1622,6 +1623,8 @@ export async function processVideo(
audioTranscode,
extraInputArgs,
extraOutputArgs,
+ sourceWidth: metadata.width,
+ sourceHeight: metadata.height,
decodeThreads: decodeThreadsFromEnv(
"MEDIA_SERVER_PROCESS_DECODE_THREADS",
2,
@@ -1636,6 +1639,7 @@ export async function processVideo(
const gpuDecode = canDecodeOnGpu(
metadata.videoCodec,
await probeVideoRotation(inputPath, abortSignal),
+ nvenc.gpuDecodeCodecs,
);
try {
await runTranscodeCommand(
@@ -1698,18 +1702,37 @@ export interface ProcessVideoArgsInput {
extraInputArgs: string[];
extraOutputArgs: string[];
decodeThreads: string;
+ sourceWidth?: number;
+ sourceHeight?: number;
nvenc?: { options: NvencOptions; gpuDecode: boolean } | null;
}
export function buildProcessVideoArgs(input: ProcessVideoArgsInput): string[] {
const { opts, nvenc } = input;
- const gpuDecode = Boolean(nvenc?.gpuDecode);
+ const gpuDecode = Boolean(
+ nvenc?.gpuDecode && input.sourceWidth && input.sourceHeight,
+ );
+ const source = {
+ width: input.sourceWidth ?? 0,
+ height: input.sourceHeight ?? 0,
+ };
const cpuScaleFilter = `scale='min(${opts.maxWidth},iw)':'min(${opts.maxHeight},ih)':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2`;
const ffmpegArgs: string[] = [
"ffmpeg",
"-threads",
input.decodeThreads,
- ...(gpuDecode ? NVENC_HWACCEL_INPUT_ARGS : []),
+ ...(gpuDecode
+ ? buildGpuDecodeInputArgs(
+ input.videoCodec,
+ source,
+ fitWithinEven(
+ source.width,
+ source.height,
+ opts.maxWidth,
+ opts.maxHeight,
+ ),
+ )
+ : []),
...input.extraInputArgs,
"-i",
input.inputPath,
@@ -1727,7 +1750,7 @@ export function buildProcessVideoArgs(input: ProcessVideoArgsInput): string[] {
ffmpegArgs.push(
...buildNvencEncodeArgs(nvenc.options, opts.crf),
...(gpuDecode
- ? ["-vf", buildNvencScaleFilter(opts.maxWidth, opts.maxHeight)]
+ ? ["-vf", GPU_DECODE_FORMAT_FILTER]
: ["-vf", cpuScaleFilter, "-pix_fmt", "yuv420p"]),
);
} else {
diff --git a/apps/media-server/src/lib/nvenc-encoder.ts b/apps/media-server/src/lib/nvenc-encoder.ts
index 6a784780f..d79432aec 100644
--- a/apps/media-server/src/lib/nvenc-encoder.ts
+++ b/apps/media-server/src/lib/nvenc-encoder.ts
@@ -5,6 +5,8 @@ const MAX_LOOKAHEAD = 32;
const GOP_LENGTH = "250";
const NVENC_PRESETS = new Set(["p1", "p2", "p3", "p4", "p5", "p6", "p7"]);
const NVDEC_CODECS = new Set(["h264", "hevc", "vp9", "av1"]);
+// H.264 and VP9 decode faster on the host CPU than on L4 NVDEC; HEVC and AV1 do not.
+const DEFAULT_GPU_DECODE_CODECS = ["hevc", "av1"];
const MAX_FALLBACK_REASON_LENGTH = 500;
export const NVENC_HWACCEL_INPUT_ARGS = [
@@ -23,6 +25,7 @@ export interface NvencOptions {
lookahead: number;
spatialAq: boolean;
temporalAq: boolean;
+ gpuDecodeCodecs: string[];
}
interface NvencAttempt {
@@ -82,9 +85,18 @@ export function nvencOptionsFromEnv(
DEFAULT_LOOKAHEAD,
spatialAq: parseToggle(env.CAP_NVENC_SPATIAL_AQ) ?? true,
temporalAq: parseToggle(env.CAP_NVENC_TEMPORAL_AQ) ?? false,
+ gpuDecodeCodecs: parseGpuDecodeCodecs(env.CAP_NVENC_GPU_DECODE_CODECS),
};
}
+function parseGpuDecodeCodecs(raw: string | undefined): string[] {
+ if (raw === undefined || raw.trim() === "") return DEFAULT_GPU_DECODE_CODECS;
+ return raw
+ .split(",")
+ .map((codec) => codec.trim().toLowerCase())
+ .filter((codec) => NVDEC_CODECS.has(codec));
+}
+
export function nvencCq(crf: number, cqOffset: number): string {
const base = Number.isFinite(crf) ? crf : 23;
const cq = Math.min(51, Math.max(1, base + cqOffset));
@@ -121,19 +133,47 @@ export function buildNvencEncodeArgs(
];
}
-export function buildNvencScaleFilter(
+// Mirrors the CPU filter chain: fit within the box keeping aspect (rounded like libavfilter), then floor to even.
+export function fitWithinEven(
+ width: number,
+ height: number,
maxWidth: number,
maxHeight: number,
-): string {
- return `scale_cuda='min(${maxWidth},iw)':'min(${maxHeight},ih)':force_original_aspect_ratio=decrease:format=nv12,scale_cuda=trunc(iw/2)*2:trunc(ih/2)*2`;
+): { width: number; height: number } {
+ let w = Math.min(maxWidth, width);
+ let h = Math.min(maxHeight, height);
+ const tmpW = Math.round((h * width) / height);
+ const tmpH = Math.round((w * height) / width);
+ w = Math.min(tmpW, w);
+ h = Math.min(tmpH, h);
+ return { width: Math.trunc(w / 2) * 2, height: Math.trunc(h / 2) * 2 };
}
+// cuvid resizes inside NVDEC for free; scale_cuda then only converts 10-bit surfaces and passes 8-bit through.
+export function buildGpuDecodeInputArgs(
+ videoCodec: string,
+ source: { width: number; height: number },
+ target: { width: number; height: number },
+): string[] {
+ return [
+ ...NVENC_HWACCEL_INPUT_ARGS,
+ "-c:v",
+ `${videoCodec}_cuvid`,
+ ...(target.width !== source.width || target.height !== source.height
+ ? ["-resize", `${target.width}x${target.height}`]
+ : []),
+ ];
+}
+
+export const GPU_DECODE_FORMAT_FILTER = "scale_cuda=format=nv12";
+
// FFmpeg skips autorotation for hardware frames, so rotated sources decode on the CPU to keep the libx264 geometry.
export function canDecodeOnGpu(
videoCodec: string,
rotation: number | null,
+ gpuDecodeCodecs: string[] = DEFAULT_GPU_DECODE_CODECS,
): boolean {
- return NVDEC_CODECS.has(videoCodec) && rotation === 0;
+ return gpuDecodeCodecs.includes(videoCodec) && rotation === 0;
}
export function nvencFallbackReason(