Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(media-server): log the real reason when recording verification or nvenc fails
Richie McIlroyaec430c97bc6
diff --git a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.tsindex 597855b44..d5516bb73 100644--- a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts+++ b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts@@ -1,5 +1,6 @@import { describe, expect, test } from "bun:test";import { join } from "node:path";+import { FfmpegExitError, ffmpegErrorLines } from "../../lib/ffmpeg-errors";import type { VideoMetadata } from "../../lib/job-manager";import { decodeThreadsFromEnv } from "../../lib/media-common";import {@@ -286,6 +287,76 @@ describe("nvenc fallback", () => {expect(nvencFallbackReason(new Error("killed"), true)).toBeNull();});+ test("leads with the encoder error printed before the progress tail", () => {+ const progress = [+ "frame=0",+ "fps=0.00",+ "stream_0_0_q=0.0",+ "bitrate=N/A",+ "total_size=0",+ "out_time_us=N/A",+ "out_time_ms=N/A",+ "out_time=N/A",+ "dup_frames=0",+ "drop_frames=0",+ "speed=N/A",+ "progress=end",+ ];+ const stderr = [+ "Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'https://bucket.s3.us-east-1.amazonaws.com/user-123/video-456/raw.mp4?X-Amz-Credential=AKIAEXAMPLE%2F20261023&X-Amz-Signature=deadbeef':",+ "[h264_nvenc @ 0x5581c2a0] InitializeEncoder failed: invalid param (8): Invalid Level.",+ "[vost#0:0/h264_nvenc @ 0x5581c3b0] Error while opening encoder - maybe incorrect parameters such as bit_rate, rate, width or height.",+ "Error opening output file /tmp/cap-media-server/instance-1/user-123-out.mp4.",+ ...progress,+ "Conversion failed!",+ ];+ const reason = nvencFallbackReason(+ new FfmpegExitError(+ `FFmpeg exited with code 1. Last stderr: ${stderr.slice(-10).join(" | ")}`,+ ffmpegErrorLines(stderr.join("\n")),+ ),+ false,+ );+ expect(reason).toStartWith(+ "[h264_nvenc @ 0x5581c2a0] InitializeEncoder failed: invalid param (8): Invalid Level.",+ );+ expect(reason).toContain("Error while opening encoder");+ expect(reason).toContain("Error opening output file <path>");+ expect(reason).toContain("Conversion failed!");+ expect(reason).not.toContain("amazonaws");+ expect(reason).not.toContain("X-Amz");+ expect(reason).not.toContain("user-123");+ expect(reason?.length).toBeLessThanOrEqual(500);+ expect(+ ffmpegErrorLines(+ "Reconfiguring filter graph because video parameters changed to yuv420p(tv), 1280x720\r\nprogress=continue\nerror_rate=0\n",+ ),+ ).toEqual([+ "Reconfiguring filter graph because video parameters changed to yuv420p(tv), 1280x720",+ ]);+ expect(+ ffmpegErrorLines(+ [+ "[https @ 0x1] HTTP error 403 Forbidden",+ `[in#0 @ 0x2] Error opening input: Server returned 403 Forbidden (access denied)`,+ ].join("\n"),+ ),+ ).toHaveLength(2);+ expect(+ nvencFallbackReason(+ new FfmpegExitError(+ "FFmpeg exited with code 1. Last stderr: Conversion failed!",+ [+ "Error opening input file https://bucket.example/raw.mp4?X-Amz-Signature=secret",+ ],+ ),+ false,+ ),+ ).toBe(+ "Error opening input file <url> || FFmpeg exited with code 1. Last stderr: Conversion failed!",+ );+ });+test("reports the last attempt through the encoder status", () => {const before = getVideoEncoderStatus(enabled).nvenc;recordNvencAttempt("process", "No NVENC capable devices found");@@ -684,6 +755,12 @@ describe("nvenc jobs on hosts without an NVENC device", () => {expect(result.height).toBe(120);expect(result.audioCodec).toBe("aac");expectRecordedAttempt("process", attempts);+ const lastAttempt = getVideoEncoderStatus(enabled).nvenc?.lastAttempt;+ if (lastAttempt?.fellBack) {+ const [cause] = lastAttempt.reason?.split(" || ") ?? [];+ expect(cause).not.toStartWith("FFmpeg exited");+ expect(cause).not.toMatch(/out_time=|total_size=|progress=/);+ }} finally {await output.cleanup();}diff --git a/apps/media-server/src/__tests__/routes/recording-verification.test.ts b/apps/media-server/src/__tests__/routes/recording-verification.test.tsindex 7d3beb673..7355cc15c 100644--- a/apps/media-server/src/__tests__/routes/recording-verification.test.ts+++ b/apps/media-server/src/__tests__/routes/recording-verification.test.ts@@ -108,6 +108,17 @@ async function terminalWebhook(jobId: string): Promise<JobProgress> {return payload;}+function captureFailureLogs() {+ const info = spyOn(console, "info");+ const events = () =>+ info.mock.calls.flatMap(([line]) => {+ if (typeof line !== "string" || !line.startsWith("{")) return [];+ const event = JSON.parse(line) as Record<string, unknown>;+ return event.event === "recording_verification_failed" ? [event] : [];+ });+ return { events, restore: () => info.mockRestore() };+}+function waitForDecoderCancellation() {let joined = false;verify.mockImplementation(async (_input, options) => {@@ -508,4 +519,94 @@ describe("asynchronous recording verification", () => {expect(failed.metadata).toBeUndefined();},);++ test("logs why permanent verification failed without leaking the presigned URL", async () => {+ const logs = captureFailureLogs();+ try {+ verify.mockRejectedValue(+ new Error(+ "Recording full decode failed: https://bucket.s3.amazonaws.com/user-1/video-1/raw.mp4?X-Amz-Credential=AKIAEXAMPLE&X-Amz-Signature=deadbeef: Invalid data found when processing input",+ ),+ );+ const jobId = await startJob();+ const failed = await terminalWebhook(jobId);+ expect(failed.errorCode).toBe("source-invalid");+ const [event, ...rest] = logs.events();+ expect(rest).toEqual([]);+ expect(event).toMatchObject({+ jobId,+ videoId: body.videoId,+ phase: "error",+ errorCode: "source-invalid",+ retryable: false,+ stage: "decode",+ reason:+ "Recording full decode failed: <url> Invalid data found when processing input",+ });+ const line = JSON.stringify(event);+ expect(line).not.toContain("amazonaws");+ expect(line).not.toContain("X-Amz");+ expect(line).not.toContain("deadbeef");+ expect(line).not.toContain(body.videoUrl);+ } finally {+ logs.restore();+ }+ });++ test("logs which uploaded metadata check failed", async () => {+ const logs = captureFailureLogs();+ try {+ probe.mockResolvedValue({ ...metadata, fileSize: 64, duration: 2 });+ const jobId = await startJob();+ const failed = await terminalWebhook(jobId);+ expect(failed.errorCode).toBe("source-invalid");+ expect(logs.events()).toEqual([+ expect.objectContaining({+ jobId,+ errorCode: "source-invalid",+ retryable: false,+ stage: "metadata",+ reason: "Uploaded recording does not match the completed local file",+ mismatch: {+ expectedFileSize: 128,+ actualFileSize: 64,+ expectedDuration: 5,+ actualDuration: 2,+ requiredAudio: false,+ audioCodec: "aac",+ },+ }),+ ]);+ expect(verify).not.toHaveBeenCalled();+ } finally {+ logs.restore();+ }+ });++ test("logs the original probe error behind a retryable failure", async () => {+ const logs = captureFailureLogs();+ try {+ probe.mockRejectedValue(+ new Error(+ "Media input is not accessible (503) for /tmp/cap-media-server/instance-1/user-1.mp4?X-Amz-Signature=deadbeef",+ ),+ );+ const jobId = await startJob();+ const failed = await terminalWebhook(jobId);+ expect(failed.error).toBe(+ "Recording verification temporarily unavailable (503)",+ );+ expect(logs.events()).toEqual([+ expect.objectContaining({+ jobId,+ errorCode: "processing-unavailable",+ retryable: true,+ stage: "probe",+ reason: "Media input is not accessible (503) for <path>",+ }),+ ]);+ } finally {+ logs.restore();+ }+ });});diff --git a/apps/media-server/src/lib/ffmpeg-errors.ts b/apps/media-server/src/lib/ffmpeg-errors.tsnew file mode 100644index 000000000..b9b8998d4--- /dev/null+++ b/apps/media-server/src/lib/ffmpeg-errors.ts@@ -0,0 +1,30 @@+const MAX_FFMPEG_ERROR_LINES = 3;+const ERROR_LINE_PATTERN =+ /error|invalid|failed|unsupported|cannot|unknown|unrecognized|not found|no such|no capable|reconfiguring/i;+const PROGRESS_LINE_PATTERN = /^\s*[\w.]+=\S*\s*$/;++export class FfmpegExitError extends Error {+ constructor(+ message: string,+ readonly errorLines: string[],+ ) {+ super(message);+ }+}++function isFfmpegErrorLine(line: string): boolean {+ return ERROR_LINE_PATTERN.test(line) && !PROGRESS_LINE_PATTERN.test(line);+}++export function collectFfmpegErrorLine(errorLines: string[], line: string) {+ if (errorLines.length < MAX_FFMPEG_ERROR_LINES && isFfmpegErrorLine(line))+ errorLines.push(line.trim());+}++export function ffmpegErrorLines(stderr: string): string[] {+ const errorLines: string[] = [];+ for (const line of stderr.split(/\r\n|\r|\n/)) {+ collectFfmpegErrorLine(errorLines, line);+ }+ return errorLines;+}diff --git a/apps/media-server/src/lib/log-redaction.ts b/apps/media-server/src/lib/log-redaction.tsnew file mode 100644index 000000000..bf30f4bca--- /dev/null+++ b/apps/media-server/src/lib/log-redaction.ts@@ -0,0 +1,30 @@+const MAX_LOG_REASON_LENGTH = 300;++const REDACTIONS: [RegExp, string][] = [+ [/[a-z][a-z0-9+.-]*:\/\/\S+/gi, "<url>"],+ [/\b[\w.-]+\.amazonaws\.com\S*/gi, "<url>"],+ [/X-Amz-[A-Za-z-]+=[^\s&]*/gi, "<redacted>"],+ [/\?[^\s?=]+=\S*/g, "?<redacted>"],+ [/\bBearer\s+\S+/gi, "Bearer <redacted>"],+ [+ /\b(token|secret|signature|password|credential|api[_-]?key|access[_-]?key)(\s*[=:]\s*)\S+/gi,+ "$1$2<redacted>",+ ],+ [/(?<![\w.~-])\/(?:[^\s/:'"]+\/)+[^\s/:'",)]*/g, "<path>"],+];++export function redactLogText(text: string): string {+ let redacted = text;+ for (const [pattern, replacement] of REDACTIONS) {+ redacted = redacted.replace(pattern, replacement);+ }+ return redacted.replace(/\s+/g, " ").trim();+}++export function logReason(+ error: unknown,+ maxLength = MAX_LOG_REASON_LENGTH,+): string {+ const message = error instanceof Error ? error.message : String(error);+ return redactLogText(message).slice(0, maxLength) || "unknown error";+}diff --git a/apps/media-server/src/lib/media-edit.ts b/apps/media-server/src/lib/media-edit.tsindex 91a9f0483..88e65bf9c 100644--- a/apps/media-server/src/lib/media-edit.ts+++ b/apps/media-server/src/lib/media-edit.ts@@ -1,5 +1,6 @@import { writeFile } from "node:fs/promises";import { file, spawn } from "bun";+import { FfmpegExitError, ffmpegErrorLines } from "./ffmpeg-errors";import type { VideoMetadata } from "./job-manager";import {buildNvencEncodeArgs,@@ -326,8 +327,9 @@ async function runFfmpegCommand(]);if (exitCode !== 0) {- throw new Error(+ throw new FfmpegExitError(`FFmpeg exited with code ${exitCode}. Last stderr: ${stderrText.slice(-2000)}`,+ ffmpegErrorLines(stderrText),);}})(),diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.tsindex bdff52896..f46c56a14 100644--- a/apps/media-server/src/lib/media-video.ts+++ b/apps/media-server/src/lib/media-video.ts@@ -10,6 +10,7 @@ import {remainingTimeoutMs,} from "./capcodec-encoder";import { uploadDriveResumable } from "./drive-resumable-upload";+import { collectFfmpegErrorLine, FfmpegExitError } from "./ffmpeg-errors";import type { VideoMetadata } from "./job-manager";import {DOWNLOAD_TIMEOUT_MS,@@ -1831,6 +1832,7 @@ async function runTranscodeCommand(const stderrLines: string[] = [];const maxStderrLines = 50;+ const errorLines: string[] = [];try {await withTimeout(@@ -1857,6 +1859,7 @@ async function runTranscodeCommand(if (stderrLines.length > maxStderrLines) {stderrLines.shift();}+ collectFfmpegErrorLine(errorLines, line);const progress = parseProgressFromStderr(line, totalDurationUs);if (progress !== null && onProgress) {onProgress(progress, `Encoding: ${Math.round(progress)}%`);@@ -1869,8 +1872,9 @@ async function runTranscodeCommand(const exitCode = await proc.exited;if (exitCode !== 0) {- throw new Error(+ throw new FfmpegExitError(`FFmpeg exited with code ${exitCode}. Last stderr: ${stderrLines.slice(-10).join(" | ")}`,+ errorLines,);}diff --git a/apps/media-server/src/lib/nvenc-encoder.ts b/apps/media-server/src/lib/nvenc-encoder.tsindex eb2848ba9..162d56b65 100644--- a/apps/media-server/src/lib/nvenc-encoder.ts+++ b/apps/media-server/src/lib/nvenc-encoder.ts@@ -1,3 +1,6 @@+import { FfmpegExitError } from "./ffmpeg-errors";+import { redactLogText } from "./log-redaction";+const DEFAULT_PRESET = "p1";const DEFAULT_CQ_OFFSET = 8;const DEFAULT_LOOKAHEAD = 0;@@ -8,6 +11,7 @@ const NVDEC_CODECS = new Set(["h264", "hevc", "vp9", "av1"]);// H.264 and VP9 decode faster on the host CPU than on L4 NVDEC; HEVC and AV1 do not.const DEFAULT_GPU_DECODE_CODECS = ["hevc", "av1"];const MAX_FALLBACK_REASON_LENGTH = 500;+const MAX_FFMPEG_CAUSE_LENGTH = 300;export const NVENC_HWACCEL_INPUT_ARGS = ["-hwaccel",@@ -192,10 +196,19 @@ export function nvencFallbackReason(aborted: boolean,): string | null {if (aborted) return null;- const message = (- error instanceof Error ? error.message : String(error)- ).replace(/[a-z][a-z0-9+.-]*:\/\/\S+/gi, "<url>");- return message.slice(-MAX_FALLBACK_REASON_LENGTH) || "unknown error";+ const message = redactLogText(+ error instanceof Error ? error.message : String(error),+ );+ // The message tail is mostly -progress output; the encoder's own error is printed earlier.+ const cause =+ error instanceof FfmpegExitError && error.errorLines.length > 0+ ? `${redactLogText(error.errorLines.join(" | ")).slice(0, MAX_FFMPEG_CAUSE_LENGTH)} || `+ : "";+ return (+ (+ cause + message.slice(-(MAX_FALLBACK_REASON_LENGTH - cause.length))+ ).trim() || "unknown error"+ );}export function recordNvencAttempt(diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.tsindex 09495309f..66d49c4a3 100644--- a/apps/media-server/src/routes/video.ts+++ b/apps/media-server/src/routes/video.ts@@ -42,6 +42,7 @@ import {setRequestIdleTimeout,trackBackgroundTask,} from "../lib/lifecycle";+import { logReason } from "../lib/log-redaction";import { PROCESS_TIMEOUT_MS, UPLOAD_TIMEOUT_MS } from "../lib/media-common";import {getTimeoutMs as getEditRenderTimeoutMs,@@ -109,6 +110,7 @@ const PROBE_ERROR_CODE = ["FF", "PROBE_ERROR"].join("");const VIDEO_BUSY_RETRY_AFTER_SECONDS = 15;const VIDEO_MEMORY_PRESSURE_ERROR ="SERVER_BUSY: container memory pressure is too high";+const MUX_FAILURE_LOG_LENGTH = 1_000;const SEGMENT_DOWNLOAD_MAX_ATTEMPTS = 3;const SEGMENT_DOWNLOAD_RETRY_BASE_MS = 250;const JOB_TRANSFER_ALLOWANCE_MS = 2 * UPLOAD_TIMEOUT_MS;@@ -2029,10 +2031,20 @@ video.post("/verify-recording", async (c) => {["complete", "error", "cancelled"].includes(current.phase))return;+ const errorCode = classifySourceError(error);+ logVideoEvent("recording_verification_failed", {+ jobId,+ videoId: body.videoId,+ phase: "error",+ errorCode,+ retryable: errorCode === "processing-unavailable",+ stage: "transfer",+ reason: logReason(error),+ });updateJob(jobId, {phase: "error",error: "Recording transfer could not complete",- errorCode: classifySourceError(error),+ errorCode,});sendCurrentJobWebhook(jobId);}),@@ -2047,21 +2059,35 @@ async function verifyUploadedRecordingAsync(const abortController = new AbortController();if (!updateJob(jobId, { abortController, phase: "processing", progress: 0 }))return;+ let stage = "probe";+ let cause: unknown;+ let mismatch: Record<string, unknown> | undefined;try {const metadata = await probeVideo(body.videoUrl).catch((error: unknown) => {if (error instanceof MediaTransferBudgetError) throw error;+ cause = error;throw new Error(RECORDING_VERIFICATION_RETRY_ERROR);});+ stage = "metadata";if (metadata.fileSize !== body.fileSize ||(body.duration !== undefined &&!isDurationClose(metadata.duration, body.duration)) ||(body.requiredAudio && !metadata.audioCodec)) {+ mismatch = {+ expectedFileSize: body.fileSize,+ actualFileSize: metadata.fileSize,+ expectedDuration: body.duration,+ actualDuration: metadata.duration,+ requiredAudio: body.requiredAudio,+ audioCodec: metadata.audioCodec,+ };throw new Error("Uploaded recording does not match the completed local file",);}+ stage = "decode";if (!beginRecordingVerification(jobId,@@ -2085,6 +2111,7 @@ async function verifyUploadedRecordingAsync(}),),);+ stage = "identity";if (verified.fileSize !== body.fileSize)throw new Error("Verified recording size does not match the local file");if (body.generation && !verified.remoteSha256)@@ -2122,15 +2149,27 @@ async function verifyUploadedRecordingAsync((error instanceof Error &&(error.message === VIDEO_MEMORY_PRESSURE_ERROR ||error.message === RECORDING_VERIFICATION_RETRY_ERROR));+ const phase =+ !retryable && abortController.signal.aborted ? "cancelled" : "error";+ const errorCode = retryable+ ? "processing-unavailable"+ : classifySourceError(error);+ logVideoEvent("recording_verification_failed", {+ jobId,+ videoId: body.videoId,+ phase,+ errorCode,+ retryable,+ stage,+ reason: logReason(cause ?? error),+ ...(mismatch && { mismatch }),+ });updateJob(jobId, {- phase:- !retryable && abortController.signal.aborted ? "cancelled" : "error",+ phase,error: retryable? RECORDING_VERIFICATION_RETRY_ERROR: "Uploaded recording content could not be verified; retain the local recording",- errorCode: retryable- ? "processing-unavailable"- : classifySourceError(error),+ errorCode,});}const job = getJob(jobId);@@ -2707,6 +2746,22 @@ async function downloadSegmentsBatchTracked(return outputPaths;}+function muxFailureErrorCode(+ error: unknown,+ stageErrorCode: RecordingErrorCode,+): RecordingErrorCode {+ if (error instanceof MediaTransferBudgetError)+ return "processing-budget-exhausted";+ if (error instanceof MediaDownloadError) return error.errorCode;+ if (+ isRetryableRecordingVerificationError(error) ||+ isBusyError(error) ||+ isTimeoutError(error)+ )+ return "processing-unavailable";+ return stageErrorCode;+}+function sendCurrentJobWebhook(jobId: string): void {const job = getJob(jobId);if (!job) {@@ -3146,7 +3201,8 @@ async function muxSegmentsAsync(jobId,videoId,durationMs: Date.now() - startedAt,- error: error instanceof Error ? error.message : String(error),+ errorCode: muxFailureErrorCode(error, errorCode),+ error: logReason(error, MUX_FAILURE_LOG_LENGTH),resources: getSystemResources(),});if (!outputUploadStarted) {@@ -3163,16 +3219,7 @@ async function muxSegmentsAsync(abortController.signal.aborted && !isBusyError(error)? "cancelled": "error",- errorCode:- error instanceof MediaTransferBudgetError- ? "processing-budget-exhausted"- : error instanceof MediaDownloadError- ? error.errorCode- : isRetryableRecordingVerificationError(error) ||- isBusyError(error) ||- isTimeoutError(error)- ? "processing-unavailable"- : errorCode,+ errorCode: muxFailureErrorCode(error, errorCode),error: isRetryableRecordingVerificationError(error)? "Recording verification temporarily unavailable (503)": error instanceof Error