Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(media-server): log the real reason when recording verification or nvenc fails

Richie McIlroyaec430c97bc6
diff --git a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts
index 597855b44..d5516bb73 100644
--- a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts
+++ b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts
@@ -1,5 +1,6 @@
import { describe, expect, test } from "bun:test";
import { join } from "node:path";
+import { FfmpegExitError, ffmpegErrorLines } from "../../lib/ffmpeg-errors";
import type { VideoMetadata } from "../../lib/job-manager";
import { decodeThreadsFromEnv } from "../../lib/media-common";
import {
@@ -286,6 +287,76 @@ describe("nvenc fallback", () => {
expect(nvencFallbackReason(new Error("killed"), true)).toBeNull();
});
+ test("leads with the encoder error printed before the progress tail", () => {
+ const progress = [
+ "frame=0",
+ "fps=0.00",
+ "stream_0_0_q=0.0",
+ "bitrate=N/A",
+ "total_size=0",
+ "out_time_us=N/A",
+ "out_time_ms=N/A",
+ "out_time=N/A",
+ "dup_frames=0",
+ "drop_frames=0",
+ "speed=N/A",
+ "progress=end",
+ ];
+ const stderr = [
+ "Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'https://bucket.s3.us-east-1.amazonaws.com/user-123/video-456/raw.mp4?X-Amz-Credential=AKIAEXAMPLE%2F20261023&X-Amz-Signature=deadbeef':",
+ "[h264_nvenc @ 0x5581c2a0] InitializeEncoder failed: invalid param (8): Invalid Level.",
+ "[vost#0:0/h264_nvenc @ 0x5581c3b0] Error while opening encoder - maybe incorrect parameters such as bit_rate, rate, width or height.",
+ "Error opening output file /tmp/cap-media-server/instance-1/user-123-out.mp4.",
+ ...progress,
+ "Conversion failed!",
+ ];
+ const reason = nvencFallbackReason(
+ new FfmpegExitError(
+ `FFmpeg exited with code 1. Last stderr: ${stderr.slice(-10).join(" | ")}`,
+ ffmpegErrorLines(stderr.join("\n")),
+ ),
+ false,
+ );
+ expect(reason).toStartWith(
+ "[h264_nvenc @ 0x5581c2a0] InitializeEncoder failed: invalid param (8): Invalid Level.",
+ );
+ expect(reason).toContain("Error while opening encoder");
+ expect(reason).toContain("Error opening output file <path>");
+ expect(reason).toContain("Conversion failed!");
+ expect(reason).not.toContain("amazonaws");
+ expect(reason).not.toContain("X-Amz");
+ expect(reason).not.toContain("user-123");
+ expect(reason?.length).toBeLessThanOrEqual(500);
+ expect(
+ ffmpegErrorLines(
+ "Reconfiguring filter graph because video parameters changed to yuv420p(tv), 1280x720\r\nprogress=continue\nerror_rate=0\n",
+ ),
+ ).toEqual([
+ "Reconfiguring filter graph because video parameters changed to yuv420p(tv), 1280x720",
+ ]);
+ expect(
+ ffmpegErrorLines(
+ [
+ "[https @ 0x1] HTTP error 403 Forbidden",
+ `[in#0 @ 0x2] Error opening input: Server returned 403 Forbidden (access denied)`,
+ ].join("\n"),
+ ),
+ ).toHaveLength(2);
+ expect(
+ nvencFallbackReason(
+ new FfmpegExitError(
+ "FFmpeg exited with code 1. Last stderr: Conversion failed!",
+ [
+ "Error opening input file https://bucket.example/raw.mp4?X-Amz-Signature=secret",
+ ],
+ ),
+ false,
+ ),
+ ).toBe(
+ "Error opening input file <url> || FFmpeg exited with code 1. Last stderr: Conversion failed!",
+ );
+ });
+
test("reports the last attempt through the encoder status", () => {
const before = getVideoEncoderStatus(enabled).nvenc;
recordNvencAttempt("process", "No NVENC capable devices found");
@@ -684,6 +755,12 @@ describe("nvenc jobs on hosts without an NVENC device", () => {
expect(result.height).toBe(120);
expect(result.audioCodec).toBe("aac");
expectRecordedAttempt("process", attempts);
+ const lastAttempt = getVideoEncoderStatus(enabled).nvenc?.lastAttempt;
+ if (lastAttempt?.fellBack) {
+ const [cause] = lastAttempt.reason?.split(" || ") ?? [];
+ expect(cause).not.toStartWith("FFmpeg exited");
+ expect(cause).not.toMatch(/out_time=|total_size=|progress=/);
+ }
} finally {
await output.cleanup();
}
diff --git a/apps/media-server/src/__tests__/routes/recording-verification.test.ts b/apps/media-server/src/__tests__/routes/recording-verification.test.ts
index 7d3beb673..7355cc15c 100644
--- a/apps/media-server/src/__tests__/routes/recording-verification.test.ts
+++ b/apps/media-server/src/__tests__/routes/recording-verification.test.ts
@@ -108,6 +108,17 @@ async function terminalWebhook(jobId: string): Promise<JobProgress> {
return payload;
}
+function captureFailureLogs() {
+ const info = spyOn(console, "info");
+ const events = () =>
+ info.mock.calls.flatMap(([line]) => {
+ if (typeof line !== "string" || !line.startsWith("{")) return [];
+ const event = JSON.parse(line) as Record<string, unknown>;
+ return event.event === "recording_verification_failed" ? [event] : [];
+ });
+ return { events, restore: () => info.mockRestore() };
+}
+
function waitForDecoderCancellation() {
let joined = false;
verify.mockImplementation(async (_input, options) => {
@@ -508,4 +519,94 @@ describe("asynchronous recording verification", () => {
expect(failed.metadata).toBeUndefined();
},
);
+
+ test("logs why permanent verification failed without leaking the presigned URL", async () => {
+ const logs = captureFailureLogs();
+ try {
+ verify.mockRejectedValue(
+ new Error(
+ "Recording full decode failed: https://bucket.s3.amazonaws.com/user-1/video-1/raw.mp4?X-Amz-Credential=AKIAEXAMPLE&X-Amz-Signature=deadbeef: Invalid data found when processing input",
+ ),
+ );
+ const jobId = await startJob();
+ const failed = await terminalWebhook(jobId);
+ expect(failed.errorCode).toBe("source-invalid");
+ const [event, ...rest] = logs.events();
+ expect(rest).toEqual([]);
+ expect(event).toMatchObject({
+ jobId,
+ videoId: body.videoId,
+ phase: "error",
+ errorCode: "source-invalid",
+ retryable: false,
+ stage: "decode",
+ reason:
+ "Recording full decode failed: <url> Invalid data found when processing input",
+ });
+ const line = JSON.stringify(event);
+ expect(line).not.toContain("amazonaws");
+ expect(line).not.toContain("X-Amz");
+ expect(line).not.toContain("deadbeef");
+ expect(line).not.toContain(body.videoUrl);
+ } finally {
+ logs.restore();
+ }
+ });
+
+ test("logs which uploaded metadata check failed", async () => {
+ const logs = captureFailureLogs();
+ try {
+ probe.mockResolvedValue({ ...metadata, fileSize: 64, duration: 2 });
+ const jobId = await startJob();
+ const failed = await terminalWebhook(jobId);
+ expect(failed.errorCode).toBe("source-invalid");
+ expect(logs.events()).toEqual([
+ expect.objectContaining({
+ jobId,
+ errorCode: "source-invalid",
+ retryable: false,
+ stage: "metadata",
+ reason: "Uploaded recording does not match the completed local file",
+ mismatch: {
+ expectedFileSize: 128,
+ actualFileSize: 64,
+ expectedDuration: 5,
+ actualDuration: 2,
+ requiredAudio: false,
+ audioCodec: "aac",
+ },
+ }),
+ ]);
+ expect(verify).not.toHaveBeenCalled();
+ } finally {
+ logs.restore();
+ }
+ });
+
+ test("logs the original probe error behind a retryable failure", async () => {
+ const logs = captureFailureLogs();
+ try {
+ probe.mockRejectedValue(
+ new Error(
+ "Media input is not accessible (503) for /tmp/cap-media-server/instance-1/user-1.mp4?X-Amz-Signature=deadbeef",
+ ),
+ );
+ const jobId = await startJob();
+ const failed = await terminalWebhook(jobId);
+ expect(failed.error).toBe(
+ "Recording verification temporarily unavailable (503)",
+ );
+ expect(logs.events()).toEqual([
+ expect.objectContaining({
+ jobId,
+ errorCode: "processing-unavailable",
+ retryable: true,
+ stage: "probe",
+ reason: "Media input is not accessible (503) for <path>",
+ }),
+ ]);
+ } finally {
+ logs.restore();
+ }
+ });
});
diff --git a/apps/media-server/src/lib/ffmpeg-errors.ts b/apps/media-server/src/lib/ffmpeg-errors.ts
new file mode 100644
index 000000000..b9b8998d4
--- /dev/null
+++ b/apps/media-server/src/lib/ffmpeg-errors.ts
@@ -0,0 +1,30 @@
+const MAX_FFMPEG_ERROR_LINES = 3;
+const ERROR_LINE_PATTERN =
+ /error|invalid|failed|unsupported|cannot|unknown|unrecognized|not found|no such|no capable|reconfiguring/i;
+const PROGRESS_LINE_PATTERN = /^\s*[\w.]+=\S*\s*$/;
+
+export class FfmpegExitError extends Error {
+ constructor(
+ message: string,
+ readonly errorLines: string[],
+ ) {
+ super(message);
+ }
+}
+
+function isFfmpegErrorLine(line: string): boolean {
+ return ERROR_LINE_PATTERN.test(line) && !PROGRESS_LINE_PATTERN.test(line);
+}
+
+export function collectFfmpegErrorLine(errorLines: string[], line: string) {
+ if (errorLines.length < MAX_FFMPEG_ERROR_LINES && isFfmpegErrorLine(line))
+ errorLines.push(line.trim());
+}
+
+export function ffmpegErrorLines(stderr: string): string[] {
+ const errorLines: string[] = [];
+ for (const line of stderr.split(/\r\n|\r|\n/)) {
+ collectFfmpegErrorLine(errorLines, line);
+ }
+ return errorLines;
+}
diff --git a/apps/media-server/src/lib/log-redaction.ts b/apps/media-server/src/lib/log-redaction.ts
new file mode 100644
index 000000000..bf30f4bca
--- /dev/null
+++ b/apps/media-server/src/lib/log-redaction.ts
@@ -0,0 +1,30 @@
+const MAX_LOG_REASON_LENGTH = 300;
+
+const REDACTIONS: [RegExp, string][] = [
+ [/[a-z][a-z0-9+.-]*:\/\/\S+/gi, "<url>"],
+ [/\b[\w.-]+\.amazonaws\.com\S*/gi, "<url>"],
+ [/X-Amz-[A-Za-z-]+=[^\s&]*/gi, "<redacted>"],
+ [/\?[^\s?=]+=\S*/g, "?<redacted>"],
+ [/\bBearer\s+\S+/gi, "Bearer <redacted>"],
+ [
+ /\b(token|secret|signature|password|credential|api[_-]?key|access[_-]?key)(\s*[=:]\s*)\S+/gi,
+ "$1$2<redacted>",
+ ],
+ [/(?<![\w.~-])\/(?:[^\s/:'"]+\/)+[^\s/:'",)]*/g, "<path>"],
+];
+
+export function redactLogText(text: string): string {
+ let redacted = text;
+ for (const [pattern, replacement] of REDACTIONS) {
+ redacted = redacted.replace(pattern, replacement);
+ }
+ return redacted.replace(/\s+/g, " ").trim();
+}
+
+export function logReason(
+ error: unknown,
+ maxLength = MAX_LOG_REASON_LENGTH,
+): string {
+ const message = error instanceof Error ? error.message : String(error);
+ return redactLogText(message).slice(0, maxLength) || "unknown error";
+}
diff --git a/apps/media-server/src/lib/media-edit.ts b/apps/media-server/src/lib/media-edit.ts
index 91a9f0483..88e65bf9c 100644
--- a/apps/media-server/src/lib/media-edit.ts
+++ b/apps/media-server/src/lib/media-edit.ts
@@ -1,5 +1,6 @@
import { writeFile } from "node:fs/promises";
import { file, spawn } from "bun";
+import { FfmpegExitError, ffmpegErrorLines } from "./ffmpeg-errors";
import type { VideoMetadata } from "./job-manager";
import {
buildNvencEncodeArgs,
@@ -326,8 +327,9 @@ async function runFfmpegCommand(
]);
if (exitCode !== 0) {
- throw new Error(
+ throw new FfmpegExitError(
`FFmpeg exited with code ${exitCode}. Last stderr: ${stderrText.slice(-2000)}`,
+ ffmpegErrorLines(stderrText),
);
}
})(),
diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts
index bdff52896..f46c56a14 100644
--- a/apps/media-server/src/lib/media-video.ts
+++ b/apps/media-server/src/lib/media-video.ts
@@ -10,6 +10,7 @@ import {
remainingTimeoutMs,
} from "./capcodec-encoder";
import { uploadDriveResumable } from "./drive-resumable-upload";
+import { collectFfmpegErrorLine, FfmpegExitError } from "./ffmpeg-errors";
import type { VideoMetadata } from "./job-manager";
import {
DOWNLOAD_TIMEOUT_MS,
@@ -1831,6 +1832,7 @@ async function runTranscodeCommand(
const stderrLines: string[] = [];
const maxStderrLines = 50;
+ const errorLines: string[] = [];
try {
await withTimeout(
@@ -1857,6 +1859,7 @@ async function runTranscodeCommand(
if (stderrLines.length > maxStderrLines) {
stderrLines.shift();
}
+ collectFfmpegErrorLine(errorLines, line);
const progress = parseProgressFromStderr(line, totalDurationUs);
if (progress !== null && onProgress) {
onProgress(progress, `Encoding: ${Math.round(progress)}%`);
@@ -1869,8 +1872,9 @@ async function runTranscodeCommand(
const exitCode = await proc.exited;
if (exitCode !== 0) {
- throw new Error(
+ throw new FfmpegExitError(
`FFmpeg exited with code ${exitCode}. Last stderr: ${stderrLines.slice(-10).join(" | ")}`,
+ errorLines,
);
}
diff --git a/apps/media-server/src/lib/nvenc-encoder.ts b/apps/media-server/src/lib/nvenc-encoder.ts
index eb2848ba9..162d56b65 100644
--- a/apps/media-server/src/lib/nvenc-encoder.ts
+++ b/apps/media-server/src/lib/nvenc-encoder.ts
@@ -1,3 +1,6 @@
+import { FfmpegExitError } from "./ffmpeg-errors";
+import { redactLogText } from "./log-redaction";
+
const DEFAULT_PRESET = "p1";
const DEFAULT_CQ_OFFSET = 8;
const DEFAULT_LOOKAHEAD = 0;
@@ -8,6 +11,7 @@ const NVDEC_CODECS = new Set(["h264", "hevc", "vp9", "av1"]);
// H.264 and VP9 decode faster on the host CPU than on L4 NVDEC; HEVC and AV1 do not.
const DEFAULT_GPU_DECODE_CODECS = ["hevc", "av1"];
const MAX_FALLBACK_REASON_LENGTH = 500;
+const MAX_FFMPEG_CAUSE_LENGTH = 300;
export const NVENC_HWACCEL_INPUT_ARGS = [
"-hwaccel",
@@ -192,10 +196,19 @@ export function nvencFallbackReason(
aborted: boolean,
): string | null {
if (aborted) return null;
- const message = (
- error instanceof Error ? error.message : String(error)
- ).replace(/[a-z][a-z0-9+.-]*:\/\/\S+/gi, "<url>");
- return message.slice(-MAX_FALLBACK_REASON_LENGTH) || "unknown error";
+ const message = redactLogText(
+ error instanceof Error ? error.message : String(error),
+ );
+ // The message tail is mostly -progress output; the encoder's own error is printed earlier.
+ const cause =
+ error instanceof FfmpegExitError && error.errorLines.length > 0
+ ? `${redactLogText(error.errorLines.join(" | ")).slice(0, MAX_FFMPEG_CAUSE_LENGTH)} || `
+ : "";
+ return (
+ (
+ cause + message.slice(-(MAX_FALLBACK_REASON_LENGTH - cause.length))
+ ).trim() || "unknown error"
+ );
}
export function recordNvencAttempt(
diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts
index 09495309f..66d49c4a3 100644
--- a/apps/media-server/src/routes/video.ts
+++ b/apps/media-server/src/routes/video.ts
@@ -42,6 +42,7 @@ import {
setRequestIdleTimeout,
trackBackgroundTask,
} from "../lib/lifecycle";
+import { logReason } from "../lib/log-redaction";
import { PROCESS_TIMEOUT_MS, UPLOAD_TIMEOUT_MS } from "../lib/media-common";
import {
getTimeoutMs as getEditRenderTimeoutMs,
@@ -109,6 +110,7 @@ const PROBE_ERROR_CODE = ["FF", "PROBE_ERROR"].join("");
const VIDEO_BUSY_RETRY_AFTER_SECONDS = 15;
const VIDEO_MEMORY_PRESSURE_ERROR =
"SERVER_BUSY: container memory pressure is too high";
+const MUX_FAILURE_LOG_LENGTH = 1_000;
const SEGMENT_DOWNLOAD_MAX_ATTEMPTS = 3;
const SEGMENT_DOWNLOAD_RETRY_BASE_MS = 250;
const JOB_TRANSFER_ALLOWANCE_MS = 2 * UPLOAD_TIMEOUT_MS;
@@ -2029,10 +2031,20 @@ video.post("/verify-recording", async (c) => {
["complete", "error", "cancelled"].includes(current.phase)
)
return;
+ const errorCode = classifySourceError(error);
+ logVideoEvent("recording_verification_failed", {
+ jobId,
+ videoId: body.videoId,
+ phase: "error",
+ errorCode,
+ retryable: errorCode === "processing-unavailable",
+ stage: "transfer",
+ reason: logReason(error),
+ });
updateJob(jobId, {
phase: "error",
error: "Recording transfer could not complete",
- errorCode: classifySourceError(error),
+ errorCode,
});
sendCurrentJobWebhook(jobId);
}),
@@ -2047,21 +2059,35 @@ async function verifyUploadedRecordingAsync(
const abortController = new AbortController();
if (!updateJob(jobId, { abortController, phase: "processing", progress: 0 }))
return;
+ let stage = "probe";
+ let cause: unknown;
+ let mismatch: Record<string, unknown> | undefined;
try {
const metadata = await probeVideo(body.videoUrl).catch((error: unknown) => {
if (error instanceof MediaTransferBudgetError) throw error;
+ cause = error;
throw new Error(RECORDING_VERIFICATION_RETRY_ERROR);
});
+ stage = "metadata";
if (
metadata.fileSize !== body.fileSize ||
(body.duration !== undefined &&
!isDurationClose(metadata.duration, body.duration)) ||
(body.requiredAudio && !metadata.audioCodec)
) {
+ mismatch = {
+ expectedFileSize: body.fileSize,
+ actualFileSize: metadata.fileSize,
+ expectedDuration: body.duration,
+ actualDuration: metadata.duration,
+ requiredAudio: body.requiredAudio,
+ audioCodec: metadata.audioCodec,
+ };
throw new Error(
"Uploaded recording does not match the completed local file",
);
}
+ stage = "decode";
if (
!beginRecordingVerification(
jobId,
@@ -2085,6 +2111,7 @@ async function verifyUploadedRecordingAsync(
}),
),
);
+ stage = "identity";
if (verified.fileSize !== body.fileSize)
throw new Error("Verified recording size does not match the local file");
if (body.generation && !verified.remoteSha256)
@@ -2122,15 +2149,27 @@ async function verifyUploadedRecordingAsync(
(error instanceof Error &&
(error.message === VIDEO_MEMORY_PRESSURE_ERROR ||
error.message === RECORDING_VERIFICATION_RETRY_ERROR));
+ const phase =
+ !retryable && abortController.signal.aborted ? "cancelled" : "error";
+ const errorCode = retryable
+ ? "processing-unavailable"
+ : classifySourceError(error);
+ logVideoEvent("recording_verification_failed", {
+ jobId,
+ videoId: body.videoId,
+ phase,
+ errorCode,
+ retryable,
+ stage,
+ reason: logReason(cause ?? error),
+ ...(mismatch && { mismatch }),
+ });
updateJob(jobId, {
- phase:
- !retryable && abortController.signal.aborted ? "cancelled" : "error",
+ phase,
error: retryable
? RECORDING_VERIFICATION_RETRY_ERROR
: "Uploaded recording content could not be verified; retain the local recording",
- errorCode: retryable
- ? "processing-unavailable"
- : classifySourceError(error),
+ errorCode,
});
}
const job = getJob(jobId);
@@ -2707,6 +2746,22 @@ async function downloadSegmentsBatchTracked(
return outputPaths;
}
+function muxFailureErrorCode(
+ error: unknown,
+ stageErrorCode: RecordingErrorCode,
+): RecordingErrorCode {
+ if (error instanceof MediaTransferBudgetError)
+ return "processing-budget-exhausted";
+ if (error instanceof MediaDownloadError) return error.errorCode;
+ if (
+ isRetryableRecordingVerificationError(error) ||
+ isBusyError(error) ||
+ isTimeoutError(error)
+ )
+ return "processing-unavailable";
+ return stageErrorCode;
+}
+
function sendCurrentJobWebhook(jobId: string): void {
const job = getJob(jobId);
if (!job) {
@@ -3146,7 +3201,8 @@ async function muxSegmentsAsync(
jobId,
videoId,
durationMs: Date.now() - startedAt,
- error: error instanceof Error ? error.message : String(error),
+ errorCode: muxFailureErrorCode(error, errorCode),
+ error: logReason(error, MUX_FAILURE_LOG_LENGTH),
resources: getSystemResources(),
});
if (!outputUploadStarted) {
@@ -3163,16 +3219,7 @@ async function muxSegmentsAsync(
abortController.signal.aborted && !isBusyError(error)
? "cancelled"
: "error",
- errorCode:
- error instanceof MediaTransferBudgetError
- ? "processing-budget-exhausted"
- : error instanceof MediaDownloadError
- ? error.errorCode
- : isRetryableRecordingVerificationError(error) ||
- isBusyError(error) ||
- isTimeoutError(error)
- ? "processing-unavailable"
- : errorCode,
+ errorCode: muxFailureErrorCode(error, errorCode),
error: isRetryableRecordingVerificationError(error)
? "Recording verification temporarily unavailable (503)"
: error instanceof Error