Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
feat(media-server): decode hevc/av1 on the gpu with cuvid resize before nvenc
Richie McIlroy68c5f45e9afc
diff --git a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.tsindex f598e3d80..b698a77bd 100644--- a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts+++ b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts@@ -15,8 +15,8 @@ import {} from "../../lib/media-video";import {buildNvencEncodeArgs,- buildNvencScaleFilter,canDecodeOnGpu,+ fitWithinEven,getVideoEncoderStatus,type NvencOptions,nvencCq,@@ -33,6 +33,7 @@ const defaults: NvencOptions = {lookahead: 20,spatialAq: true,temporalAq: false,+ gpuDecodeCodecs: ["hevc", "av1"],};const opts: ProcessVideoArgsInput["opts"] = {maxWidth: 1920,@@ -63,8 +64,7 @@ const nvencVideoArgs = ["-g","250",];-const gpuScale =- "scale_cuda='min(1920,iw)':'min(1080,ih)':force_original_aspect_ratio=decrease:format=nv12,scale_cuda=trunc(iw/2)*2:trunc(ih/2)*2";+const gpuScale = "scale_cuda=format=nv12";const cpuScale ="scale='min(1920,iw)':'min(1080,ih)':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2";@@ -104,6 +104,8 @@ function processArgs(extraInputArgs: [],extraOutputArgs: [],decodeThreads: "2",+ sourceWidth: source.width,+ sourceHeight: source.height,...overrides,};}@@ -159,7 +161,14 @@ describe("nvenc encoder selection", () => {lookahead: 0,spatialAq: false,temporalAq: true,+ gpuDecodeCodecs: ["hevc", "av1"],});+ expect(+ nvencOptionsFromEnv({+ ...enabled,+ CAP_NVENC_GPU_DECODE_CODECS: "h264, HEVC,vp8,nope",+ })?.gpuDecodeCodecs,+ ).toEqual(["h264", "hevc"]);});test("ignores invalid tuning overrides", () => {@@ -213,14 +222,31 @@ describe("nvenc encoder selection", () => {]);});- test("scales on the GPU with the CPU path's box and even dimensions", () => {- expect(buildNvencScaleFilter(1920, 1080)).toBe(gpuScale);+ test("fits the GPU resize to the CPU path's box and even dimensions", () => {+ expect(fitWithinEven(3840, 2160, 1920, 1080)).toEqual({+ width: 1920,+ height: 1080,+ });+ expect(fitWithinEven(3024, 1964, 1920, 1080)).toEqual({+ width: 1662,+ height: 1080,+ });+ expect(fitWithinEven(1079, 1919, 1920, 1080)).toEqual({+ width: 606,+ height: 1080,+ });+ expect(fitWithinEven(1281, 721, 1920, 1080)).toEqual({+ width: 1280,+ height: 720,+ });});- test("decodes on the GPU only for unrotated NVDEC codecs", () => {- for (const codec of ["h264", "hevc", "vp9", "av1"]) {+ test("decodes on the GPU only for configured, unrotated codecs", () => {+ for (const codec of ["hevc", "av1"]) {expect(canDecodeOnGpu(codec, 0)).toBe(true);}+ expect(canDecodeOnGpu("h264", 0)).toBe(false);+ expect(canDecodeOnGpu("h264", 0, ["h264"])).toBe(true);expect(canDecodeOnGpu("vp8", 0)).toBe(false);expect(canDecodeOnGpu("prores", 0)).toBe(false);expect(canDecodeOnGpu("hevc", 90)).toBe(false);@@ -307,6 +333,10 @@ describe("process video args", () => {"cuda","-hwaccel_output_format","cuda",+ "-c:v",+ "h264_cuvid",+ "-resize",+ "1662x1080","-i","in.mp4",...nvencVideoArgs,@@ -336,6 +366,10 @@ describe("process video args", () => {"cuda","-hwaccel_output_format","cuda",+ "-c:v",+ "hevc_cuvid",+ "-resize",+ "1920x1080","-i","in.mov",...nvencVideoArgs,@@ -374,6 +408,8 @@ describe("process video args", () => {"cuda","-hwaccel_output_format","cuda",+ "-c:v",+ "vp9_cuvid","-i","in.webm",...nvencVideoArgs,diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.tsindex feecf5830..591996736 100644--- a/apps/media-server/src/lib/media-video.ts+++ b/apps/media-server/src/lib/media-video.ts@@ -23,11 +23,12 @@ import {import { probeVideoFile } from "./media-probe";import { fetchMedia, materializeMedia } from "./media-transfer";import {+ buildGpuDecodeInputArgs,buildNvencEncodeArgs,- buildNvencScaleFilter,canDecodeOnGpu,+ fitWithinEven,+ GPU_DECODE_FORMAT_FILTER,logNvencFallback,- NVENC_HWACCEL_INPUT_ARGS,type NvencOptions,nvencFallbackReason,nvencOptionsFromEnv,@@ -1622,6 +1623,8 @@ export async function processVideo(audioTranscode,extraInputArgs,extraOutputArgs,+ sourceWidth: metadata.width,+ sourceHeight: metadata.height,decodeThreads: decodeThreadsFromEnv("MEDIA_SERVER_PROCESS_DECODE_THREADS",2,@@ -1636,6 +1639,7 @@ export async function processVideo(const gpuDecode = canDecodeOnGpu(metadata.videoCodec,await probeVideoRotation(inputPath, abortSignal),+ nvenc.gpuDecodeCodecs,);try {await runTranscodeCommand(@@ -1698,18 +1702,37 @@ export interface ProcessVideoArgsInput {extraInputArgs: string[];extraOutputArgs: string[];decodeThreads: string;+ sourceWidth?: number;+ sourceHeight?: number;nvenc?: { options: NvencOptions; gpuDecode: boolean } | null;}export function buildProcessVideoArgs(input: ProcessVideoArgsInput): string[] {const { opts, nvenc } = input;- const gpuDecode = Boolean(nvenc?.gpuDecode);+ const gpuDecode = Boolean(+ nvenc?.gpuDecode && input.sourceWidth && input.sourceHeight,+ );+ const source = {+ width: input.sourceWidth ?? 0,+ height: input.sourceHeight ?? 0,+ };const cpuScaleFilter = `scale='min(${opts.maxWidth},iw)':'min(${opts.maxHeight},ih)':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2`;const ffmpegArgs: string[] = ["ffmpeg","-threads",input.decodeThreads,- ...(gpuDecode ? NVENC_HWACCEL_INPUT_ARGS : []),+ ...(gpuDecode+ ? buildGpuDecodeInputArgs(+ input.videoCodec,+ source,+ fitWithinEven(+ source.width,+ source.height,+ opts.maxWidth,+ opts.maxHeight,+ ),+ )+ : []),...input.extraInputArgs,"-i",input.inputPath,@@ -1727,7 +1750,7 @@ export function buildProcessVideoArgs(input: ProcessVideoArgsInput): string[] {ffmpegArgs.push(...buildNvencEncodeArgs(nvenc.options, opts.crf),...(gpuDecode- ? ["-vf", buildNvencScaleFilter(opts.maxWidth, opts.maxHeight)]+ ? ["-vf", GPU_DECODE_FORMAT_FILTER]: ["-vf", cpuScaleFilter, "-pix_fmt", "yuv420p"]),);} else {diff --git a/apps/media-server/src/lib/nvenc-encoder.ts b/apps/media-server/src/lib/nvenc-encoder.tsindex 6a784780f..d79432aec 100644--- a/apps/media-server/src/lib/nvenc-encoder.ts+++ b/apps/media-server/src/lib/nvenc-encoder.ts@@ -5,6 +5,8 @@ const MAX_LOOKAHEAD = 32;const GOP_LENGTH = "250";const NVENC_PRESETS = new Set(["p1", "p2", "p3", "p4", "p5", "p6", "p7"]);const NVDEC_CODECS = new Set(["h264", "hevc", "vp9", "av1"]);+// H.264 and VP9 decode faster on the host CPU than on L4 NVDEC; HEVC and AV1 do not.+const DEFAULT_GPU_DECODE_CODECS = ["hevc", "av1"];const MAX_FALLBACK_REASON_LENGTH = 500;export const NVENC_HWACCEL_INPUT_ARGS = [@@ -23,6 +25,7 @@ export interface NvencOptions {lookahead: number;spatialAq: boolean;temporalAq: boolean;+ gpuDecodeCodecs: string[];}interface NvencAttempt {@@ -82,9 +85,18 @@ export function nvencOptionsFromEnv(DEFAULT_LOOKAHEAD,spatialAq: parseToggle(env.CAP_NVENC_SPATIAL_AQ) ?? true,temporalAq: parseToggle(env.CAP_NVENC_TEMPORAL_AQ) ?? false,+ gpuDecodeCodecs: parseGpuDecodeCodecs(env.CAP_NVENC_GPU_DECODE_CODECS),};}+function parseGpuDecodeCodecs(raw: string | undefined): string[] {+ if (raw === undefined || raw.trim() === "") return DEFAULT_GPU_DECODE_CODECS;+ return raw+ .split(",")+ .map((codec) => codec.trim().toLowerCase())+ .filter((codec) => NVDEC_CODECS.has(codec));+}+export function nvencCq(crf: number, cqOffset: number): string {const base = Number.isFinite(crf) ? crf : 23;const cq = Math.min(51, Math.max(1, base + cqOffset));@@ -121,19 +133,47 @@ export function buildNvencEncodeArgs(];}-export function buildNvencScaleFilter(+// Mirrors the CPU filter chain: fit within the box keeping aspect (rounded like libavfilter), then floor to even.+export function fitWithinEven(+ width: number,+ height: number,maxWidth: number,maxHeight: number,-): string {- return `scale_cuda='min(${maxWidth},iw)':'min(${maxHeight},ih)':force_original_aspect_ratio=decrease:format=nv12,scale_cuda=trunc(iw/2)*2:trunc(ih/2)*2`;+): { width: number; height: number } {+ let w = Math.min(maxWidth, width);+ let h = Math.min(maxHeight, height);+ const tmpW = Math.round((h * width) / height);+ const tmpH = Math.round((w * height) / width);+ w = Math.min(tmpW, w);+ h = Math.min(tmpH, h);+ return { width: Math.trunc(w / 2) * 2, height: Math.trunc(h / 2) * 2 };}+// cuvid resizes inside NVDEC for free; scale_cuda then only converts 10-bit surfaces and passes 8-bit through.+export function buildGpuDecodeInputArgs(+ videoCodec: string,+ source: { width: number; height: number },+ target: { width: number; height: number },+): string[] {+ return [+ ...NVENC_HWACCEL_INPUT_ARGS,+ "-c:v",+ `${videoCodec}_cuvid`,+ ...(target.width !== source.width || target.height !== source.height+ ? ["-resize", `${target.width}x${target.height}`]+ : []),+ ];+}++export const GPU_DECODE_FORMAT_FILTER = "scale_cuda=format=nv12";+// FFmpeg skips autorotation for hardware frames, so rotated sources decode on the CPU to keep the libx264 geometry.export function canDecodeOnGpu(videoCodec: string,rotation: number | null,+ gpuDecodeCodecs: string[] = DEFAULT_GPU_DECODE_CODECS,): boolean {- return NVDEC_CODECS.has(videoCodec) && rotation === 0;+ return gpuDecodeCodecs.includes(videoCodec) && rotation === 0;}export function nvencFallbackReason(