Skip to content

Open source CapCut alternative.

TypeScript

fix(site): light cloud proxies and honest worker failures [rebuild]

David Leb43d67a51d87
diff --git a/site/src/cut/lib/mediaProxy.test.ts b/site/src/cut/lib/mediaProxy.test.ts
index 5a952616..504e6e42 100644
--- a/site/src/cut/lib/mediaProxy.test.ts
+++ b/site/src/cut/lib/mediaProxy.test.ts
@@ -11,11 +11,17 @@ await stubModule("./mediaSync", import.meta.url, {
storedMediaUrl: async (projectId: string, fileName: string) => `stored:${projectId}/${fileName}`,
});
let storedReads = true;
+/** The codec the stored proxy file carries: the worker's and the Mac's are
+ * HEVC; a page without HEVC builds AV1 or VP9. */
+let proxyCodec: "hevc" | "av1" = "hevc";
await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, {
probeMediaFile: (async () => {
if (!storedReads) throw new Error("gone");
return { hasVideo: true };
}) as never,
+ withMedia: (async (_src: unknown, fn: (input: unknown) => Promise<unknown>) =>
+ fn({ canRead: async () => storedReads })) as never,
+ hasUndecodableVideo: async () => proxyCodec === "hevc" && !hevc,
});
const { ensureProxy, HEAVY_PROXY_MAX_HEIGHT, needsProxy, proxyUnreadable, useProxyJobs } = await import("./mediaProxy");
const { useEditor } = await import("./store");
@@ -90,7 +96,18 @@ describe("needsProxy", () => {
expect(needsProxy(h264("hd", { width: 1920, height: 1080, sizeBytes: 7_000_000 }), true)).toBe(false);
expect(needsProxy(h264("wide", { width: 2048, height: 1080, sizeBytes: 6_000_000 }), true)).toBe(false);
expect(needsProxy(h264("uhd"))).toBe(false);
- expect(needsProxy(h264("uhd", { proxy: { fileName: "uhd.proxy.mp4", sizeBytes: 1 } }), true)).toBe(false);
+ expect(needsProxy(h264("uhd", { proxy: { fileName: "uhd.mp4.light.mp4", sizeBytes: 1 } }), true)).toBe(false);
+ });
+
+ // The first heavy proxies were built at the account's quality with no
+ // ceiling, and a dense 1080p master came back heavier than itself (20 Mbps
+ // over 14). A heavy video's proxy is the capped light copy; one carrying
+ // the old full-quality proxy asks again, and gets the light one.
+ test("a heavy streamed video on the old full-quality proxy needs the light one", () => {
+ expect(needsProxy(h264("uhd", { proxy: { fileName: "uhd.proxy.mp4", sizeBytes: 1 } }), true)).toBe(true);
+ expect(needsProxy(h264("uhd", { proxy: { fileName: "uhd.mp4.light-1.mp4", sizeBytes: 1 } }), true)).toBe(false);
+ // ProRes keeps its own proxy, wherever it plays.
+ expect(needsProxy(prores("a", { proxy: { fileName: "a.proxy.mp4", sizeBytes: 1 } }), true)).toBe(false);
});
});
@@ -243,6 +260,36 @@ describe("proxyUnreadable", () => {
hevc = true;
});
+ test("a browser project's own proxy still heals on a browser with no HEVC", async () => {
+ // The page built it (AV1 or VP9 where HEVC is missing), so it decodes
+ // here; a failure means the file, and the usual drop and heal apply.
+ hevc = false;
+ proxyCodec = "av1";
+ storedReads = true;
+ const asset = broken("pagebuilt");
+ useEditor.setState({ projectId: "p", assets: [asset] });
+ const b = backend("browser", () => Response.json({}));
+ await proxyUnreadable(asset, b.api);
+ expect(useEditor.getState().assets[0].proxyUrl).toBe("stored:p/pagebuilt.proxy.mp4");
+ hevc = true;
+ proxyCodec = "hevc";
+ });
+
+ test("a Mac project's HEVC proxy stays on the doc in a browser with no HEVC", async () => {
+ // The engine encodes HEVC, and a Mac project opens in any browser on
+ // that Mac: Firefox cannot play the proxy, Safari still can.
+ features = new Set(["media.proxy"]);
+ hevc = false;
+ const asset = broken("machevc");
+ useEditor.setState({ projectId: "p", assets: [asset] });
+ const b = backend("local", () => Response.json({ fileName: "machevc.proxy.mp4", sizeBytes: 9 }));
+ await proxyUnreadable(asset, b.api);
+ expect(useEditor.getState().assets[0].proxy).toEqual(asset.proxy!);
+ expect(useEditor.getState().assets[0].proxyUrl).toBeUndefined();
+ expect(b.calls).toHaveLength(0);
+ hevc = true;
+ });
+
test("a shared view drops the proxy for itself and builds nothing", async () => {
const asset = broken("viewer");
useEditor.setState({ projectId: "p", assets: [asset], readOnly: true });
diff --git a/site/src/cut/lib/mediaProxy.ts b/site/src/cut/lib/mediaProxy.ts
index a2eda365..df117fc5 100644
--- a/site/src/cut/lib/mediaProxy.ts
+++ b/site/src/cut/lib/mediaProxy.ts
@@ -33,9 +33,10 @@ import { resolveRegisteredBlob } from "./backend/browser/registry";
import { pollCloudJob } from "./cloudJob";
import { cutColor } from "./colorSettings";
import { uploadProjectMediaTo } from "./media";
-import { frameSinkIsCustom, probeMediaFile } from "./mediaRead";
+import { frameSinkIsCustom, hasUndecodableVideo, probeMediaFile, withMedia } from "./mediaRead";
import { forgetLocalMediaUrl, storedMediaUrl } from "./mediaSync";
import { encodeProxyInPage, proxyNameFor, proxyOnTheMac, type Landed } from "./proxyEncode";
+import { isLightProxyName } from "./proxyNames";
import { releaseProxyHold, useEditor } from "./store";
import type { MediaAsset, StoredAsset } from "./types";
@@ -63,8 +64,12 @@ export function isHeavyVideo(asset: ProxyFacts): boolean {
* heavy video when the project streams its media (`streamed`, a cloud
* project). */
export function needsProxy(asset: ProxyFacts, streamed = false): boolean {
- if (asset.type !== "video" || asset.proxy || asset.block) return false;
- return isProres(asset) || (streamed && isHeavyVideo(asset));
+ if (asset.type !== "video" || asset.block) return false;
+ if (isProres(asset)) return !asset.proxy;
+ if (!streamed || !isHeavyVideo(asset)) return false;
+ // A heavy video's proxy is the light one; an older full-quality proxy
+ // streams no lighter than its master, so it is asked for again.
+ return !asset.proxy || !isLightProxyName(asset.proxy.fileName);
}
/** The codec string of the worker's proxy: HEVC Main10. */
@@ -165,9 +170,9 @@ export async function proxyUnreadable(asset: MediaAsset, backend: CutBackend = g
// The master plays from this moment; whether the proxy leaves the doc
// waits on why it failed.
s.holdProxyOff(asset.id);
- // A page with no HEVC decoder cannot play the worker's proxy, however
+ // A page with no decoder for the proxy's codec cannot play it, however
// sound the file: the proxy stays on the doc for the viewers who can.
- if (!(await proxyDecodes())) return;
+ if (await proxyCodecMissing(live, backend)) return;
const broken = live.proxy;
releaseProxyHold(asset.id);
s.dropAssetProxy(asset.id);
@@ -191,6 +196,22 @@ export async function proxyUnreadable(asset: MediaAsset, backend: CutBackend = g
if (now) await ensureProxy(projectId, now, backend);
}
+/** True when the proxy's file reads but this page has no decoder for its
+ * video, e.g. the worker's or the Mac's HEVC in Firefox. A file that does not
+ * read at all is broken, and false. */
+async function proxyCodecMissing(asset: MediaAsset, backend: CutBackend): Promise<boolean> {
+ const projectId = useEditor.getState().projectId;
+ if (!projectId || !asset.proxy) {
+ return false;
+ }
+ try {
+ const url = await storedMediaUrl(projectId, asset.proxy.fileName, backend);
+ return await withMedia(url, async (input) => (await input.canRead()) && (await hasUndecodableVideo(input)));
+ } catch {
+ return false;
+ }
+}
+
/** Every asset in the open project that still needs a proxy. */
export function ensureProjectProxies(): void {
const s = useEditor.getState();
diff --git a/site/src/cut/lib/proxyNames.test.ts b/site/src/cut/lib/proxyNames.test.ts
new file mode 100644
index 00000000..e6395ead
--- /dev/null
+++ b/site/src/cut/lib/proxyNames.test.ts
@@ -0,0 +1,12 @@
+import { expect, test } from "bun:test";
+import { isLightProxyName, lightProxyNameFor } from "./proxyNames";
+
+/** Two masters that share a stem (clip.mov, clip.mp4) are different footage,
+ * so their light proxies keep the whole master name apart. */
+test("a light proxy is named after the whole master name", () => {
+ expect(lightProxyNameFor("clip.mov")).toBe("clip.mov.light.mp4");
+ expect(lightProxyNameFor("clip.mp4")).not.toBe(lightProxyNameFor("clip.mov"));
+ expect(isLightProxyName("clip.mov.light.mp4")).toBe(true);
+ expect(isLightProxyName("clip.mov.light-1.mp4")).toBe(true);
+ expect(isLightProxyName("clip.proxy.mp4")).toBe(false);
+});
diff --git a/site/src/cut/lib/proxyNames.ts b/site/src/cut/lib/proxyNames.ts
new file mode 100644
index 00000000..c6bce1e0
--- /dev/null
+++ b/site/src/cut/lib/proxyNames.ts
@@ -0,0 +1,22 @@
+/**
+ * The light proxy: a heavy video's preview copy, built to stream at a fast
+ * cut's pace rather than to grade from. It is capped in bitrate as well as
+ * height — a quality target alone gave a dense 1080p master a proxy heavier
+ * than itself — and it carries its own name, so a project still holding an
+ * older full-quality proxy for such a video asks again and gets this one.
+ * Its bitrate ceiling is the cutProxy setting. Shared by the page, the cloud
+ * route and the worker.
+ */
+
+/** The light proxy's file name beside its master: the whole master name, so
+ * clip.mov and clip.mp4 never share one. */
+export const lightProxyNameFor = (fileName: string) => `${fileName}.light.mp4`;
+
+/** Whether a proxy's file name is a light proxy's (a deduped `.light-1.mp4`
+ * included). */
+export const isLightProxyName = (fileName: string) => /\.light(-\d+)?\.mp4$/.test(fileName);
+
+/** What a failed proxy job's row records in `result.fault` when ffmpeg ran and
+ * refused the master: the file's own fault, which the cloud route holds off
+ * retrying. A row without it failed for a passing reason and retries freely. */
+export const PROXY_SOURCE_FAULT = "source";
diff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.ts
index 32c681d8..92b3bfe1 100644
--- a/site/src/cut/server/cloud/jobs.ts
+++ b/site/src/cut/server/cloud/jobs.ts
@@ -2,6 +2,7 @@
// manage CutRenderJob rows — the worker claims and executes them. Response
// shapes byte-match the engine's export routes (http/export.ts + server/jobs.ts).
import type { Prisma } from "@/generated/prisma/client";
+import { getGlobalSetting } from "@/lib/config/effective";
import { prisma } from "@/lib/prisma";
import { EXPORT_QUOTA_MARGIN, renderJobCheck } from "./limits";
import { wakeRenderWorker } from "./wake";
@@ -13,9 +14,12 @@ import { contentTypeFor } from "../serveFile";
import { containerOfName, deliveryContainer, exportBaseName, jobWantsStems, specMediaFiles, STEMS_MIME, stemsArchiveName, type ExportContainer, type ExportFiles } from "../../lib/exportDelivery";
import { LOUDNESS_IDS, type LoudnessId } from "../../lib/loudnessSettings";
import type { DocExportAudio } from "../../lib/exportPresets";
+import { PROXY_SOURCE_FAULT } from "../../lib/proxyNames";
import { addUsage, outputByteCeiling, quotaCheck } from "./usage";
import { caught, err, redirect } from "./util";
+const HOUR_MS = 60 * 60 * 1000;
+
/** A master's file name without its extension: what its proxy's name starts
* with (server/proxy.ts `proxyNameFor`). */
const proxyStem = (fileName: string) => fileName.replace(/\.[^./\\]+$/, "") || "media";
@@ -943,20 +947,43 @@ export const jobsCloud = {
select: { id: true },
});
if (!source) return err("Media file missing from project.", 404);
+ // A heavy video's request (it names a height) wants the light proxy
+ // (lib/proxyNames.ts); a full-quality one built earlier doesn't answer it.
+ const light = maxHeight !== undefined;
const built = await prisma.cutMediaObject.findFirst({
- where: { userId, projectId, kind: "proxy", uploadState: "complete", fileName: { startsWith: `${proxyStem(file)}.proxy` } },
+ where: { userId, projectId, kind: "proxy", uploadState: "complete", fileName: { startsWith: light ? `${file}.light` : `${proxyStem(file)}.proxy` } },
select: { fileName: true, bytes: true },
orderBy: { createdAt: "desc" },
});
if (built) return Response.json({ fileName: built.fileName, sizeBytes: Number(built.bytes) });
- const running = await prisma.cutRenderJob.findFirst({
+ const runningRows = await prisma.cutRenderJob.findMany({
where: { userId, projectId, kind: "proxy", state: { in: ["queued", "running"] }, spec: { path: ["file"], equals: file } },
- select: { id: true },
+ select: { id: true, spec: true },
});
+ const running = runningRows.find((r) => ((r.spec as { maxHeight?: number } | null)?.maxHeight !== undefined) === light);
if (running) {
wakeRenderWorker();
return Response.json({ jobId: running.id });
}
+ // A master ffmpeg refused lately would be refused the same way: the
+ // preview stays on the master instead of costing a worker run on every
+ // open. A build that failed for a passing reason retries at once.
+ const { retryAfterHours } = await getGlobalSetting("cutProxy");
+ const failedRows = await prisma.cutRenderJob.findMany({
+ where: {
+ userId,
+ projectId,
+ kind: "proxy",
+ state: "error",
+ createdAt: { gt: new Date(Date.now() - retryAfterHours * HOUR_MS) },
+ spec: { path: ["file"], equals: file },
+ result: { path: ["fault"], equals: PROXY_SOURCE_FAULT },
+ },
+ select: { spec: true },
+ });
+ if (failedRows.some((r) => ((r.spec as { maxHeight?: number } | null)?.maxHeight !== undefined) === light)) {
+ return err("The preview proxy could not be made from this file lately; the original plays instead.", 409);
+ }
const capped = await renderJobCheck(userId);
if (capped) return capped;
const row = await prisma.cutRenderJob.create({
diff --git a/site/src/cut/server/proxy.test.ts b/site/src/cut/server/proxy.test.ts
index 8c9bcb81..0d9ace17 100644
--- a/site/src/cut/server/proxy.test.ts
+++ b/site/src/cut/server/proxy.test.ts
@@ -63,7 +63,7 @@ describe("proxy", () => {
expect(args[args.indexOf("-movflags") + 1]).toBe("+faststart");
// A keyframe every second: the preview seeks into the proxy at every
// cut, and a seek decodes from the keyframe before it.
- expect(args[args.indexOf("-force_key_frames") + 1]).toBe("expr:gte(t,n_forced*1)");
+ expect(args[args.indexOf("-force_key_frames") + 1]).toBe("expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+1))");
const vt = proxyFfmpegArgs("in.mov", "out.mp4", {
encoder: "hevc_videotoolbox",
@@ -79,6 +79,25 @@ describe("proxy", () => {
expect(vt[vt.indexOf("-c:v") + 1]).toBe("hevc_videotoolbox");
});
+ test("keyframes fall a second apart from the first frame, whatever time the master starts at", () => {
+ const source = { matrix: "bt709" as const, fullRange: false, height: 1080, hasAudio: false };
+ const args = proxyFfmpegArgs("in.ts", "out.mp4", { encoder: "libx265", source, maxHeight: 1080, crf: 20 });
+ // With -copyts, t is the master's own timestamp: counting forced frames
+ // against it made every frame a keyframe on a file starting at 1000 s.
+ expect(args[args.indexOf("-force_key_frames") + 1]).toBe("expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+1))");
+ });
+
+ test("a light proxy holds a bitrate ceiling in either encoder", () => {
+ const source = { matrix: "bt709" as const, fullRange: false, height: 2160, hasAudio: false };
+ const x265 = proxyFfmpegArgs("in.mp4", "out.mp4", { encoder: "libx265", source, maxHeight: 1080, crf: 20, maxKbps: 6000 });
+ expect(x265.join(" ")).toContain("-maxrate 6000k -bufsize 6000k");
+ const vt = proxyFfmpegArgs("in.mp4", "out.mp4", { encoder: "hevc_videotoolbox", source, maxHeight: 1080, crf: 20, maxKbps: 6000 });
+ expect(vt.join(" ")).toContain("-b:v 6000k");
+ expect(vt).not.toContain("-q:v");
+ // A ProRes proxy keeps its quality target and no ceiling.
+ expect(proxyFfmpegArgs("in.mp4", "out.mp4", { encoder: "libx265", source, maxHeight: 1080, crf: 20 })).not.toContain("-maxrate");
+ });
+
test("reads the fixture as a 10-bit BT.2020 ProRes master with sound", async () => {
expect(await probeProxySource(FIXTURE)).toEqual({ matrix: "bt2020nc", fullRange: false, height: 64, hasAudio: true });
});
diff --git a/site/src/cut/server/proxy.ts b/site/src/cut/server/proxy.ts
index 5b443cac..0c0337d6 100644
--- a/site/src/cut/server/proxy.ts
+++ b/site/src/cut/server/proxy.ts
@@ -47,6 +47,9 @@ export interface ProxyArgs {
maxHeight: number;
/** x265 crf; VideoToolbox's constant quality maps from it. */
crf: number;
+ /** A light proxy's bitrate ceiling, kbit/s (lib/proxyNames.ts): x265 holds
+ * it under its crf, VideoToolbox encodes at it. Absent = quality alone. */
+ maxKbps?: number;
}
/** swscale's name for a matrix. */
@@ -76,10 +79,21 @@ export function proxyFfmpegArgs(src: string, out: string, opts: ProxyArgs): stri
// the stream options below cover the container.
"setparams=color_primaries=bt709:color_trc=iec61966-2-1:colorspace=bt709:range=tv",
].join(",");
+ const kbps = opts.maxKbps && opts.maxKbps > 0 ? Math.round(opts.maxKbps) : 0;
const video =
encoder === "libx265"
- ? ["-c:v", "libx265", "-preset", "medium", "-crf", String(opts.crf), "-profile:v", "main10", "-x265-params", "log-level=error"]
- : ["-c:v", "hevc_videotoolbox", "-profile:v", "main10", "-q:v", String(vtQuality(opts.crf)), "-allow_sw", "1"];
+ ? [
+ "-c:v", "libx265", "-preset", "medium", "-crf", String(opts.crf),
+ // A one-second buffer: the ceiling holds over every stretch the
+ // preview fetches between keyframes.
+ ...(kbps ? ["-maxrate", `${kbps}k`, "-bufsize", `${kbps}k`] : []),
+ "-profile:v", "main10", "-x265-params", "log-level=error",
+ ]
+ : [
+ "-c:v", "hevc_videotoolbox", "-profile:v", "main10",
+ ...(kbps ? ["-b:v", `${kbps}k`] : ["-q:v", String(vtQuality(opts.crf))]),
+ "-allow_sw", "1",
+ ];
return [
"-y",
// The master's timestamps, unshifted: the preview maps clip time to
@@ -96,7 +110,9 @@ export function proxyFfmpegArgs(src: string, out: string, opts: ProxyArgs): stri
...video,
// A keyframe every second of the master's time: the preview seeks into
// the proxy at every cut, and a seek decodes from the keyframe before it.
- "-force_key_frames", "expr:gte(t,n_forced*1)",
+ // Counted from the last forced one, since -copyts keeps a master's own
+ // start time (a transport stream can start at 1000 s).
+ "-force_key_frames", "expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+1))",
"-tag:v", "hvc1",
...(source.hasAudio ? ["-c:a", "aac", "-b:a", "192k"] : []),
"-movflags", "+faststart",
@@ -168,13 +184,13 @@ export async function makeProxy(
handle: RenderHandle,
src: string,
out: string,
- opts: { maxHeight: number; crf: number; onProgress?: (seconds: number) => void }
+ opts: { maxHeight: number; crf: number; maxKbps?: number; onProgress?: (seconds: number) => void }
): Promise<ProxyOutcome> {
const [source, dims, encoder] = await Promise.all([probeProxySource(src), videoDimensions(src), proxyEncoder()]);
// The display height decides the scale — a phone's sideways file is
// measured upright everywhere else in Cut.
if (dims) source.height = dims.height;
- await runFfmpeg(handle, proxyFfmpegArgs(src, out, { encoder, source, maxHeight: opts.maxHeight, crf: opts.crf }), opts.onProgress);
+ await runFfmpeg(handle, proxyFfmpegArgs(src, out, { encoder, source, maxHeight: opts.maxHeight, crf: opts.crf, maxKbps: opts.maxKbps }), opts.onProgress);
const [info, outDims] = await Promise.all([stat(out), videoDimensions(out)]);
return { sizeBytes: info.size, ...(outDims ? { width: outDims.width, height: outDims.height } : {}) };
}
diff --git a/site/src/cut/worker/cardJob.ts b/site/src/cut/worker/cardJob.ts
index 84201297..56292330 100644
--- a/site/src/cut/worker/cardJob.ts
+++ b/site/src/cut/worker/cardJob.ts
@@ -26,10 +26,12 @@ const GIF_RECIPES = [
const MAX_GIF_BYTES = 3 * 1024 * 1024;
/** ffmpeg for the derivations: short, bounded, and its own runner so the
- * engine's export pipeline keeps a single entry point. */
-function ffmpeg(args: string[]): Promise<void> {
+ * engine's export pipeline keeps a single entry point. The process rides the
+ * job's handle, so a cancel kills it and a shutdown can tell it stopped it. */
+function ffmpeg(handle: RenderHandle, args: string[]): Promise<void> {
return new Promise((resolve, reject) => {
const proc = spawn("ffmpeg", ["-hide_banner", "-y", ...args]);
+ handle.proc = proc;
let tail = "";
proc.stderr.on("data", (chunk: Buffer) => {
tail = (tail + chunk.toString()).slice(-2000);
@@ -54,18 +56,18 @@ async function bytesOf(file: string): Promise<number> {
/** Still frame at t=0 — the video's actual first frame, not a poster picked
* from the middle. */
-async function renderStill(source: string, dest: string): Promise<number> {
- await ffmpeg(["-i", source, "-frames:v", "1", "-q:v", "3", dest]);
+async function renderStill(handle: RenderHandle, source: string, dest: string): Promise<number> {
+ await ffmpeg(handle, ["-i", source, "-frames:v", "1", "-q:v", "3", dest]);
return bytesOf(dest);
}
/** Animated card, stepping down through the recipes until it fits. The last
* recipe's output ships whatever its size — a card slightly over budget beats
* no card. */
-async function renderAnimation(source: string, dest: string): Promise<number> {
+async function renderAnimation(handle: RenderHandle, source: string, dest: string): Promise<number> {
let bytes = 0;
for (const [i, recipe] of GIF_RECIPES.entries()) {
- await ffmpeg([
+ await ffmpeg(handle, [
"-i",
source,
"-vf",
@@ -98,8 +100,8 @@ export async function storeCardArtifacts(
const dir = path.dirname(handle.outPath);
const stillPath = path.join(dir, "card.jpg");
const animPath = path.join(dir, "card.gif");
- const stillBytes = await renderStill(handle.outPath, stillPath);
- const animBytes = await renderAnimation(handle.outPath, animPath);
+ const stillBytes = await renderStill(handle, handle.outPath, stillPath);
+ const animBytes = await renderAnimation(handle, handle.outPath, animPath);
// Same guard the export path uses: a project deleted mid-render must not
// leave storage charged for bytes nothing can free.
diff --git a/site/src/cut/worker/db.test.ts b/site/src/cut/worker/db.test.ts
index 537f9d84..78afde73 100644
--- a/site/src/cut/worker/db.test.ts
+++ b/site/src/cut/worker/db.test.ts
@@ -187,3 +187,11 @@ test("a running project preview prevents a second preview claim while other work
expect(await claimNextJob()).toBeNull();
expect(rows[1].state).toBe("queued");
});
+
+test("an export waits behind no proxy: proxies take the tier after the watched work", async () => {
+ // Opening a cloud project queues a proxy for every heavy clip; an export
+ // asked for a moment later used to wait behind all of them.
+ queueHarness([...Array<string>(12).fill("proxy"), "export", "preview"]);
+ expect((await claimNextJob())?.kind).toBe("export");
+ expect((await claimNextJob())?.kind).toBe("proxy");
+});
diff --git a/site/src/cut/worker/db.ts b/site/src/cut/worker/db.ts
index cdb4d6ca..0403ba65 100644
--- a/site/src/cut/worker/db.ts
+++ b/site/src/cut/worker/db.ts
@@ -39,18 +39,20 @@ export interface ClaimedJob {
/** The kinds someone is watching happen: an export, a URL import, or a chat
* turn has a progress surface on screen, while a hover proxy and a share card
* are background polish nobody is waiting on. */
-const WATCHED_KINDS = ["export", "import_url", "convert", "proxy", "agent_turn"];
+const WATCHED_KINDS = ["export", "import_url", "convert", "agent_turn"];
/** Atomically claim the next queued job: the updateMany's state guard makes
* exactly one worker win each row, so replicas never double-run a job.
*
- * Watched kinds go first, FIFO within a tier. Strict FIFO across all kinds let
+ * Watched kinds go first, then proxies, FIFO within a tier. Strict FIFO across all kinds let
* a proxy render queued moments earlier hold up an export the user is staring
* at — the wait was invisible in the UI, because the export's elapsed clock
* only starts once it is claimed. */
export async function claimNextJob(): Promise<ClaimedJob | null> {
// Editor command batches stay queued for the card that owns their document.
- for (const kind of [{ in: WATCHED_KINDS }, { in: ["preview", "card", "hls"] }]) {
+ // Proxies come next: opening a cloud project queues one for every heavy
+ // clip, and an export asked for a moment later must not wait behind them.
+ for (const kind of [{ in: WATCHED_KINDS }, { in: ["proxy"] }, { in: ["preview", "card", "hls"] }]) {
const candidates = await prisma.cutRenderJob.findMany({
where: { state: "queued", kind },
orderBy: { createdAt: "asc" },
diff --git a/site/src/cut/worker/main.ts b/site/src/cut/worker/main.ts
index 95af5663..dce10903 100644
--- a/site/src/cut/worker/main.ts
+++ b/site/src/cut/worker/main.ts
@@ -9,6 +9,7 @@ import { runImportUrlJob } from "./importUrlJob";
import { runTurnJob } from "./turnJob";
import { describeRuntime, installHeadlessRuntime } from "../lib/headless/runtime";
import { deleteObjects } from "./r2";
+import { failedJobFate, InputFault } from "./settle";
// The cloud render worker: a headless loop that claims CutRenderJob rows the
// hosted API queues (export, preview, import_url, convert, proxy, agent_turn),
@@ -41,6 +42,9 @@ const SWEEP_EVERY_MS = 15_000;
// inside that window, keeping enough of it back to requeue whatever is still
// going when it closes.
const DRAIN_MS = 13 * 60_000;
+/** Set when the platform sends SIGTERM: a run whose process stops from here
+ * on is the shutdown's doing (`failedJobFate`). */
+let signaled = false;
/** The longest a job of each kind may run. Everything else about a job's cost
* is bounded when it is queued; this bounds the one thing that isn't — a
@@ -200,15 +204,27 @@ async function runJob(job: ClaimedJob): Promise<void> {
: raw;
// A canceled job's row already says so; the running-state guard also
// keeps this from resurrecting a row canceled between watcher ticks.
- if (!entry.canceled) {
+ const fate = failedJobFate({ canceled: entry.canceled, signaled, proc: entry.handle.proc });
+ if (fate === "requeue") {
+ // Its overlays stay for the replica that claims it next.
+ entry.retry = true;
await prisma.cutRenderJob
.updateMany({
where: { id: job.id, state: "running" },
- data: { state: "error", error: message },
+ data: { state: "queued", claimedAt: null, progress: 0 },
+ })
+ .catch(() => {});
+ } else if (fate === "error") {
+ // A failure on the job's input records which, for the route to read.
+ const fault = err instanceof InputFault ? { result: { fault: err.fault } } : {};
+ await prisma.cutRenderJob
+ .updateMany({
+ where: { id: job.id, state: "running" },
+ data: { state: "error", error: message, ...fault },
})
.catch(() => {});
}
- console.error(`[cut-worker] ${job.kind} ${job.id} ${entry.canceled ? "canceled" : "failed"}: ${raw}`);
+ console.error(`[cut-worker] ${job.kind} ${job.id} ${entry.canceled ? "canceled" : fate === "requeue" ? "requeued on shutdown" : "failed"}: ${raw}`);
// A canceled turn retires this process once its cleanup settles: the
// abort tears the agent loop down, and a fresh process guarantees the
// next claim starts with clean per-user transport bindings even if some
@@ -258,7 +274,10 @@ async function stop(): Promise<void> {
}
async function main(): Promise<void> {
- process.on("SIGTERM", () => void stop());
+ process.on("SIGTERM", () => {
+ signaled = true;
+ void stop();
+ });
process.on("SIGINT", () => void stop());
// Canvas, decoders, Web Audio, and fonts, standing in for the page a turn
// would otherwise run in. A worker missing a native module loses that
diff --git a/site/src/cut/worker/proxyJob.test.ts b/site/src/cut/worker/proxyJob.test.ts
index 4b040866..5f5b89d7 100644
--- a/site/src/cut/worker/proxyJob.test.ts
+++ b/site/src/cut/worker/proxyJob.test.ts
@@ -1,7 +1,10 @@
import { describe, expect, mock, test } from "bun:test";
import { writeFile } from "node:fs/promises";
import type { prisma } from "@/lib/prisma";
+import type { RenderHandle } from "../server/exportPipeline";
import { runProxyJob, type ProxyJobIO } from "./proxyJob";
+import { PROXY_SOURCE_FAULT } from "../lib/proxyNames";
+import { InputFault } from "./settle";
function harness(taken: string[] = [], spec: Record<string, unknown> = { file: "clip.mov" }) {
const findMany = mock(async (_args: unknown) => { void _args; return taken.map((fileName) => ({ fileName })); });
@@ -18,15 +21,25 @@ function harness(taken: string[] = [], spec: Record<string, unknown> = { file: "
return { sizeBytes: 5, width: 64, height: 64 };
}),
duration: mock(async () => 2),
- settings: mock(async () => ({ proxyMaxHeight: 1080, proxyCrf: 20 })),
+ settings: mock(async () => ({ proxyMaxHeight: 1080, proxyCrf: 20, lightKbps: 6000 })),
} satisfies ProxyJobIO;
const job = { id: "proxy-1", userId: "owner", projectId: "project", kind: "proxy", outName: null, spec };
- const handle = { tmpDir: "", outPath: "", progress: 0, log: [] as string[] };
+ const handle: RenderHandle = { tmpDir: "", outPath: "", progress: 0, log: [] as string[] };
const run = (canceled = () => false) => runProxyJob(job, handle, canceled, io);
return { run, io, handle, findMany };
}
describe("proxy worker", () => {
+ test("a heavy video's job builds the capped light copy under its own name", async () => {
+ const h = harness([], { file: "clip.mov", maxHeight: 1080 });
+ expect((await h.run()).fileName).toBe("clip.mov.light.mp4");
+ expect(h.io.make.mock.calls[0][3]).toMatchObject({ maxHeight: 1080, maxKbps: 6000 });
+ // A ProRes job keeps the full-quality proxy, uncapped.
+ const pro = harness();
+ expect((await pro.run()).fileName).toBe("clip.proxy.mp4");
+ expect((pro.io.make.mock.calls[0][3] as { maxKbps?: number }).maxKbps).toBeUndefined();
+ });
+
test("a job that asks for a smaller copy gets it, never one past the account's size", async () => {
const small = harness([], { file: "clip.mov", maxHeight: 720 });
await small.run();
@@ -73,4 +86,24 @@ describe("proxy worker", () => {
await expect(h.run(() => true)).rejects.toThrow("canceled");
expect(h.io.make).not.toHaveBeenCalled();
});
+
+ // A master ffmpeg refuses fails the same way on every try; a build killed
+ // by a signal or lost to storage does not, and the row tells them apart.
+ test("an encode that refuses the master records the file's fault; a killed one does not", async () => {
+ const refused = harness();
+ refused.io.make.mockImplementation(async (handle: unknown) => {
+ (handle as RenderHandle).proc = { signalCode: null, exitCode: 1 } as never;
+ throw new Error("ffmpeg exited with code 1.");
+ });
+ const err = await refused.run().catch((e: unknown) => e);
+ expect(err instanceof InputFault).toBe(true);
+ expect((err as InputFault).fault).toBe(PROXY_SOURCE_FAULT);
+
+ const killed = harness();
+ killed.io.make.mockImplementation(async (handle: unknown) => {
+ (handle as RenderHandle).proc = { signalCode: "SIGKILL", exitCode: null } as never;
+ throw new Error("Export stalled — no ffmpeg output for 120s.");
+ });
+ expect((await killed.run().catch((e: unknown) => e)) instanceof InputFault).toBe(false);
+ });
});
diff --git a/site/src/cut/worker/proxyJob.ts b/site/src/cut/worker/proxyJob.ts
index 7362decd..1c9229be 100644
--- a/site/src/cut/worker/proxyJob.ts
+++ b/site/src/cut/worker/proxyJob.ts
@@ -4,11 +4,13 @@ import path from "node:path";
import { getGlobalSetting } from "@/lib/config/effective";
import type { RenderHandle } from "../server/exportPipeline";
import { makeProxy, proxyNameFor } from "../server/proxy";
+import { lightProxyNameFor, PROXY_SOURCE_FAULT } from "../lib/proxyNames";
import { probeDuration } from "../server/frames";
import { dedupeName } from "../server/cloud/util";
import { prisma, registerObject, unregisterObjects, type ClaimedJob } from "./db";
import { deleteObjects, downloadToFile, mediaKey, uploadFile } from "./r2";
import { runLibraryProxyJob } from "./libraryProxyJob";
+import { InputFault, refusedInput } from "./settle";
import type { LibraryAsset } from "../server/library";
/** What a proxy job records in CutRenderJob.result — the same shape the
@@ -30,7 +32,7 @@ export interface ProxyJobIO {
unregister: typeof unregisterObjects;
make: typeof makeProxy;
duration: typeof probeDuration;
- settings: () => Promise<{ proxyMaxHeight: number; proxyCrf: number }>;
+ settings: () => Promise<{ proxyMaxHeight: number; proxyCrf: number; lightKbps: number }>;
}
const defaultIO: ProxyJobIO = {
@@ -42,7 +44,10 @@ const defaultIO: ProxyJobIO = {
unregister: unregisterObjects,
make: makeProxy,
duration: probeDuration,
- settings: () => getGlobalSetting("cutColor"),
+ settings: async () => {
+ const [color, proxy] = await Promise.all([getGlobalSetting("cutColor"), getGlobalSetting("cutProxy")]);
+ return { ...color, lightKbps: proxy.lightKbps };
+ },
};
/**
@@ -78,7 +83,10 @@ export async function runProxyJob(
where: { projectId, kind: { in: ["media", "proxy"] } },
select: { fileName: true },
});
- const fileName = dedupeName(proxyNameFor(file), new Set(rows.map((r) => r.fileName)));
+ // A job that names a height is a heavy video's: its proxy is the light
+ // copy, capped in bitrate and named apart from a full-quality one.
+ const light = !!(maxHeight && maxHeight > 0);
+ const fileName = dedupeName(light ? lightProxyNameFor(file) : proxyNameFor(file), new Set(rows.map((r) => r.fileName)));
const out = path.join(tmp, fileName);
handle.outPath = out;
@@ -88,9 +96,14 @@ export async function runProxyJob(
// the account allows.
maxHeight: maxHeight && maxHeight > 0 ? Math.min(settings.proxyMaxHeight, maxHeight) : settings.proxyMaxHeight,
crf: settings.proxyCrf,
+ ...(light ? { maxKbps: settings.lightKbps } : {}),
onProgress: (seconds) => {
if (total > 0) handle.progress = Math.min(0.99, seconds / total);
},
+ }).catch((e: unknown) => {
+ // ffmpeg ran to its own end and refused the master: the row says the
+ // file is at fault, so the route waits before trying it again.
+ throw !isCanceled() && refusedInput(handle.proc) ? new InputFault(PROXY_SOURCE_FAULT, e) : e;
});
if (isCanceled()) throw new Error("Proxy canceled.");
diff --git a/site/src/cut/worker/settle.test.ts b/site/src/cut/worker/settle.test.ts
new file mode 100644
index 00000000..7792accc
--- /dev/null
+++ b/site/src/cut/worker/settle.test.ts
@@ -0,0 +1,31 @@
+import { expect, test } from "bun:test";
+import { failedJobFate } from "./settle";
+
+/** A deploy rolls the worker's replicas with SIGTERM, and the signal reaches
+ * the job's ffmpeg too: it stops mid-encode and the run fails a minute in. A
+ * proxy job died that way while the drain was meant to let it finish. A run
+ * whose process the shutdown stopped goes back on the queue for the next
+ * replica; any other failure during the drain is the job's own. */
+
+const killed = { signalCode: "SIGTERM", exitCode: null };
+/** ffmpeg traps SIGTERM and exits 255 ("received signal 15"). */
+const trapped = { signalCode: null, exitCode: 255 };
+const failed = { signalCode: null, exitCode: 1 };
+
+test("a run whose process the shutdown stopped goes back on the queue", () => {
+ expect(failedJobFate({ canceled: false, signaled: true, proc: killed })).toBe("requeue");
+ expect(failedJobFate({ canceled: false, signaled: true, proc: trapped })).toBe("requeue");
+});
+
+test("a run that fails on its own during the drain is an error", () => {
+ // A turn's model or tool error, a corrupt input: re-running repeats it,
+ // and a turn would edit and charge twice.
+ expect(failedJobFate({ canceled: false, signaled: true })).toBe("error");
+ expect(failedJobFate({ canceled: false, signaled: true, proc: failed })).toBe("error");
+ expect(failedJobFate({ canceled: false, signaled: false, proc: killed })).toBe("error");
+});
+
+test("a canceled run's row already says so", () => {
+ expect(failedJobFate({ canceled: true, signaled: true, proc: killed })).toBe("leave");
+ expect(failedJobFate({ canceled: true, signaled: false })).toBe("leave");
+});
diff --git a/site/src/cut/worker/settle.ts b/site/src/cut/worker/settle.ts
new file mode 100644
index 00000000..08e14991
--- /dev/null
+++ b/site/src/cut/worker/settle.ts
@@ -0,0 +1,44 @@
+/** The exit code ffmpeg gives when it traps a termination signal and stops
+ * ("Exiting normally, received signal 15"). */
+const FFMPEG_SIGNAL_EXIT = 255;
+
+/** How the job's last child process ended, as a ChildProcess reports it. */
+type ProcExit = { signalCode: string | null; exitCode: number | null };
+
+/**
+ * What becomes of a job whose run threw. A canceled job's row already says so
+ * and is left alone. Once the worker has been sent SIGTERM, the platform's
+ * signal reaches the job's ffmpeg as well, which stops mid-encode: a run whose
+ * process ended that way goes back on the queue for the next replica. Any
+ * other failure, during the drain or not, is the job's own error.
+ */
+export function failedJobFate(run: { canceled: boolean; signaled: boolean; proc?: ProcExit }): "leave" | "requeue" | "error" {
+ if (run.canceled) {
+ return "leave";
+ }
+ return run.signaled && run.proc && stoppedBySignal(run.proc) ? "requeue" : "error";
+}
+
+function stoppedBySignal(proc: ProcExit): boolean {
+ return proc.signalCode !== null || proc.exitCode === FFMPEG_SIGNAL_EXIT;
+}
+
+/** True when the job's process ran to its own end and refused its input: a
+ * non-zero exit that no signal caused. */
+export function refusedInput(proc: ProcExit | undefined): boolean {
+ if (!proc || proc.exitCode === null || proc.exitCode === 0) {
+ return false;
+ }
+ return !stoppedBySignal(proc);
+}
+
+/** A run that failed on its input, carrying the `fault` its row records in
+ * `result` beside the message. */
+export class InputFault extends Error {
+ constructor(
+ readonly fault: string,
+ cause: unknown
+ ) {
+ super(cause instanceof Error ? cause.message : String(cause));
+ }
+}
diff --git a/site/src/lib/config/registry.ts b/site/src/lib/config/registry.ts
index 5dc7c75d..9d1e9d95 100644
--- a/site/src/lib/config/registry.ts
+++ b/site/src/lib/config/registry.ts
@@ -435,6 +435,23 @@ export const SETTINGS = defineSettings({
description:
"The ramp on the sound a split edit carries past its picture, and how lining up a separately recorded sound listens, how sure it must be, and how long a recording it searches.",
},
+ cutProxy: {
+ schema: z
+ .object({
+ // A heavy video's light proxy: its bitrate ceiling at the 1080p cap,
+ // kbit/s, so a fast cut streams it from the cloud in time.
+ lightKbps: z.number().int().min(500).max(50_000),
+ // How long a master ffmpeg could not make a proxy from keeps the
+ // cloud from trying again, hours.
+ retryAfterHours: z.number().min(0).max(24 * 30),
+ })
+ .strict(),
+ default: { lightKbps: 6000, retryAfterHours: 24 },
+ public: false,
+ title: "Preview proxies",
+ description:
+ "The light proxy's bitrate ceiling for heavy cloud videos, and how long a master that could not be proxied waits before the cloud tries it again.",
+ },
cutMotion: {
schema: z.object({ motionBlur: z.number().min(0.05).max(1) }).strict(),
default: { motionBlur: 0.5 },