Open source CapCut alternative.
TypeScript
fix(site): light cloud proxies and honest worker failures [rebuild]
David Leb43d67a51d87
diff --git a/site/src/cut/lib/mediaProxy.test.ts b/site/src/cut/lib/mediaProxy.test.tsindex 5a952616..504e6e42 100644--- a/site/src/cut/lib/mediaProxy.test.ts+++ b/site/src/cut/lib/mediaProxy.test.ts@@ -11,11 +11,17 @@ await stubModule("./mediaSync", import.meta.url, {storedMediaUrl: async (projectId: string, fileName: string) => `stored:${projectId}/${fileName}`,});let storedReads = true;+/** The codec the stored proxy file carries: the worker's and the Mac's are+ * HEVC; a page without HEVC builds AV1 or VP9. */+let proxyCodec: "hevc" | "av1" = "hevc";await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, {probeMediaFile: (async () => {if (!storedReads) throw new Error("gone");return { hasVideo: true };}) as never,+ withMedia: (async (_src: unknown, fn: (input: unknown) => Promise<unknown>) =>+ fn({ canRead: async () => storedReads })) as never,+ hasUndecodableVideo: async () => proxyCodec === "hevc" && !hevc,});const { ensureProxy, HEAVY_PROXY_MAX_HEIGHT, needsProxy, proxyUnreadable, useProxyJobs } = await import("./mediaProxy");const { useEditor } = await import("./store");@@ -90,7 +96,18 @@ describe("needsProxy", () => {expect(needsProxy(h264("hd", { width: 1920, height: 1080, sizeBytes: 7_000_000 }), true)).toBe(false);expect(needsProxy(h264("wide", { width: 2048, height: 1080, sizeBytes: 6_000_000 }), true)).toBe(false);expect(needsProxy(h264("uhd"))).toBe(false);- expect(needsProxy(h264("uhd", { proxy: { fileName: "uhd.proxy.mp4", sizeBytes: 1 } }), true)).toBe(false);+ expect(needsProxy(h264("uhd", { proxy: { fileName: "uhd.mp4.light.mp4", sizeBytes: 1 } }), true)).toBe(false);+ });++ // The first heavy proxies were built at the account's quality with no+ // ceiling, and a dense 1080p master came back heavier than itself (20 Mbps+ // over 14). A heavy video's proxy is the capped light copy; one carrying+ // the old full-quality proxy asks again, and gets the light one.+ test("a heavy streamed video on the old full-quality proxy needs the light one", () => {+ expect(needsProxy(h264("uhd", { proxy: { fileName: "uhd.proxy.mp4", sizeBytes: 1 } }), true)).toBe(true);+ expect(needsProxy(h264("uhd", { proxy: { fileName: "uhd.mp4.light-1.mp4", sizeBytes: 1 } }), true)).toBe(false);+ // ProRes keeps its own proxy, wherever it plays.+ expect(needsProxy(prores("a", { proxy: { fileName: "a.proxy.mp4", sizeBytes: 1 } }), true)).toBe(false);});});@@ -243,6 +260,36 @@ describe("proxyUnreadable", () => {hevc = true;});+ test("a browser project's own proxy still heals on a browser with no HEVC", async () => {+ // The page built it (AV1 or VP9 where HEVC is missing), so it decodes+ // here; a failure means the file, and the usual drop and heal apply.+ hevc = false;+ proxyCodec = "av1";+ storedReads = true;+ const asset = broken("pagebuilt");+ useEditor.setState({ projectId: "p", assets: [asset] });+ const b = backend("browser", () => Response.json({}));+ await proxyUnreadable(asset, b.api);+ expect(useEditor.getState().assets[0].proxyUrl).toBe("stored:p/pagebuilt.proxy.mp4");+ hevc = true;+ proxyCodec = "hevc";+ });++ test("a Mac project's HEVC proxy stays on the doc in a browser with no HEVC", async () => {+ // The engine encodes HEVC, and a Mac project opens in any browser on+ // that Mac: Firefox cannot play the proxy, Safari still can.+ features = new Set(["media.proxy"]);+ hevc = false;+ const asset = broken("machevc");+ useEditor.setState({ projectId: "p", assets: [asset] });+ const b = backend("local", () => Response.json({ fileName: "machevc.proxy.mp4", sizeBytes: 9 }));+ await proxyUnreadable(asset, b.api);+ expect(useEditor.getState().assets[0].proxy).toEqual(asset.proxy!);+ expect(useEditor.getState().assets[0].proxyUrl).toBeUndefined();+ expect(b.calls).toHaveLength(0);+ hevc = true;+ });+test("a shared view drops the proxy for itself and builds nothing", async () => {const asset = broken("viewer");useEditor.setState({ projectId: "p", assets: [asset], readOnly: true });diff --git a/site/src/cut/lib/mediaProxy.ts b/site/src/cut/lib/mediaProxy.tsindex a2eda365..df117fc5 100644--- a/site/src/cut/lib/mediaProxy.ts+++ b/site/src/cut/lib/mediaProxy.ts@@ -33,9 +33,10 @@ import { resolveRegisteredBlob } from "./backend/browser/registry";import { pollCloudJob } from "./cloudJob";import { cutColor } from "./colorSettings";import { uploadProjectMediaTo } from "./media";-import { frameSinkIsCustom, probeMediaFile } from "./mediaRead";+import { frameSinkIsCustom, hasUndecodableVideo, probeMediaFile, withMedia } from "./mediaRead";import { forgetLocalMediaUrl, storedMediaUrl } from "./mediaSync";import { encodeProxyInPage, proxyNameFor, proxyOnTheMac, type Landed } from "./proxyEncode";+import { isLightProxyName } from "./proxyNames";import { releaseProxyHold, useEditor } from "./store";import type { MediaAsset, StoredAsset } from "./types";@@ -63,8 +64,12 @@ export function isHeavyVideo(asset: ProxyFacts): boolean {* heavy video when the project streams its media (`streamed`, a cloud* project). */export function needsProxy(asset: ProxyFacts, streamed = false): boolean {- if (asset.type !== "video" || asset.proxy || asset.block) return false;- return isProres(asset) || (streamed && isHeavyVideo(asset));+ if (asset.type !== "video" || asset.block) return false;+ if (isProres(asset)) return !asset.proxy;+ if (!streamed || !isHeavyVideo(asset)) return false;+ // A heavy video's proxy is the light one; an older full-quality proxy+ // streams no lighter than its master, so it is asked for again.+ return !asset.proxy || !isLightProxyName(asset.proxy.fileName);}/** The codec string of the worker's proxy: HEVC Main10. */@@ -165,9 +170,9 @@ export async function proxyUnreadable(asset: MediaAsset, backend: CutBackend = g// The master plays from this moment; whether the proxy leaves the doc// waits on why it failed.s.holdProxyOff(asset.id);- // A page with no HEVC decoder cannot play the worker's proxy, however+ // A page with no decoder for the proxy's codec cannot play it, however// sound the file: the proxy stays on the doc for the viewers who can.- if (!(await proxyDecodes())) return;+ if (await proxyCodecMissing(live, backend)) return;const broken = live.proxy;releaseProxyHold(asset.id);s.dropAssetProxy(asset.id);@@ -191,6 +196,22 @@ export async function proxyUnreadable(asset: MediaAsset, backend: CutBackend = gif (now) await ensureProxy(projectId, now, backend);}+/** True when the proxy's file reads but this page has no decoder for its+ * video, e.g. the worker's or the Mac's HEVC in Firefox. A file that does not+ * read at all is broken, and false. */+async function proxyCodecMissing(asset: MediaAsset, backend: CutBackend): Promise<boolean> {+ const projectId = useEditor.getState().projectId;+ if (!projectId || !asset.proxy) {+ return false;+ }+ try {+ const url = await storedMediaUrl(projectId, asset.proxy.fileName, backend);+ return await withMedia(url, async (input) => (await input.canRead()) && (await hasUndecodableVideo(input)));+ } catch {+ return false;+ }+}+/** Every asset in the open project that still needs a proxy. */export function ensureProjectProxies(): void {const s = useEditor.getState();diff --git a/site/src/cut/lib/proxyNames.test.ts b/site/src/cut/lib/proxyNames.test.tsnew file mode 100644index 00000000..e6395ead--- /dev/null+++ b/site/src/cut/lib/proxyNames.test.ts@@ -0,0 +1,12 @@+import { expect, test } from "bun:test";+import { isLightProxyName, lightProxyNameFor } from "./proxyNames";++/** Two masters that share a stem (clip.mov, clip.mp4) are different footage,+ * so their light proxies keep the whole master name apart. */+test("a light proxy is named after the whole master name", () => {+ expect(lightProxyNameFor("clip.mov")).toBe("clip.mov.light.mp4");+ expect(lightProxyNameFor("clip.mp4")).not.toBe(lightProxyNameFor("clip.mov"));+ expect(isLightProxyName("clip.mov.light.mp4")).toBe(true);+ expect(isLightProxyName("clip.mov.light-1.mp4")).toBe(true);+ expect(isLightProxyName("clip.proxy.mp4")).toBe(false);+});diff --git a/site/src/cut/lib/proxyNames.ts b/site/src/cut/lib/proxyNames.tsnew file mode 100644index 00000000..c6bce1e0--- /dev/null+++ b/site/src/cut/lib/proxyNames.ts@@ -0,0 +1,22 @@+/**+ * The light proxy: a heavy video's preview copy, built to stream at a fast+ * cut's pace rather than to grade from. It is capped in bitrate as well as+ * height — a quality target alone gave a dense 1080p master a proxy heavier+ * than itself — and it carries its own name, so a project still holding an+ * older full-quality proxy for such a video asks again and gets this one.+ * Its bitrate ceiling is the cutProxy setting. Shared by the page, the cloud+ * route and the worker.+ */++/** The light proxy's file name beside its master: the whole master name, so+ * clip.mov and clip.mp4 never share one. */+export const lightProxyNameFor = (fileName: string) => `${fileName}.light.mp4`;++/** Whether a proxy's file name is a light proxy's (a deduped `.light-1.mp4`+ * included). */+export const isLightProxyName = (fileName: string) => /\.light(-\d+)?\.mp4$/.test(fileName);++/** What a failed proxy job's row records in `result.fault` when ffmpeg ran and+ * refused the master: the file's own fault, which the cloud route holds off+ * retrying. A row without it failed for a passing reason and retries freely. */+export const PROXY_SOURCE_FAULT = "source";diff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.tsindex 32c681d8..92b3bfe1 100644--- a/site/src/cut/server/cloud/jobs.ts+++ b/site/src/cut/server/cloud/jobs.ts@@ -2,6 +2,7 @@// manage CutRenderJob rows — the worker claims and executes them. Response// shapes byte-match the engine's export routes (http/export.ts + server/jobs.ts).import type { Prisma } from "@/generated/prisma/client";+import { getGlobalSetting } from "@/lib/config/effective";import { prisma } from "@/lib/prisma";import { EXPORT_QUOTA_MARGIN, renderJobCheck } from "./limits";import { wakeRenderWorker } from "./wake";@@ -13,9 +14,12 @@ import { contentTypeFor } from "../serveFile";import { containerOfName, deliveryContainer, exportBaseName, jobWantsStems, specMediaFiles, STEMS_MIME, stemsArchiveName, type ExportContainer, type ExportFiles } from "../../lib/exportDelivery";import { LOUDNESS_IDS, type LoudnessId } from "../../lib/loudnessSettings";import type { DocExportAudio } from "../../lib/exportPresets";+import { PROXY_SOURCE_FAULT } from "../../lib/proxyNames";import { addUsage, outputByteCeiling, quotaCheck } from "./usage";import { caught, err, redirect } from "./util";+const HOUR_MS = 60 * 60 * 1000;+/** A master's file name without its extension: what its proxy's name starts* with (server/proxy.ts `proxyNameFor`). */const proxyStem = (fileName: string) => fileName.replace(/\.[^./\\]+$/, "") || "media";@@ -943,20 +947,43 @@ export const jobsCloud = {select: { id: true },});if (!source) return err("Media file missing from project.", 404);+ // A heavy video's request (it names a height) wants the light proxy+ // (lib/proxyNames.ts); a full-quality one built earlier doesn't answer it.+ const light = maxHeight !== undefined;const built = await prisma.cutMediaObject.findFirst({- where: { userId, projectId, kind: "proxy", uploadState: "complete", fileName: { startsWith: `${proxyStem(file)}.proxy` } },+ where: { userId, projectId, kind: "proxy", uploadState: "complete", fileName: { startsWith: light ? `${file}.light` : `${proxyStem(file)}.proxy` } },select: { fileName: true, bytes: true },orderBy: { createdAt: "desc" },});if (built) return Response.json({ fileName: built.fileName, sizeBytes: Number(built.bytes) });- const running = await prisma.cutRenderJob.findFirst({+ const runningRows = await prisma.cutRenderJob.findMany({where: { userId, projectId, kind: "proxy", state: { in: ["queued", "running"] }, spec: { path: ["file"], equals: file } },- select: { id: true },+ select: { id: true, spec: true },});+ const running = runningRows.find((r) => ((r.spec as { maxHeight?: number } | null)?.maxHeight !== undefined) === light);if (running) {wakeRenderWorker();return Response.json({ jobId: running.id });}+ // A master ffmpeg refused lately would be refused the same way: the+ // preview stays on the master instead of costing a worker run on every+ // open. A build that failed for a passing reason retries at once.+ const { retryAfterHours } = await getGlobalSetting("cutProxy");+ const failedRows = await prisma.cutRenderJob.findMany({+ where: {+ userId,+ projectId,+ kind: "proxy",+ state: "error",+ createdAt: { gt: new Date(Date.now() - retryAfterHours * HOUR_MS) },+ spec: { path: ["file"], equals: file },+ result: { path: ["fault"], equals: PROXY_SOURCE_FAULT },+ },+ select: { spec: true },+ });+ if (failedRows.some((r) => ((r.spec as { maxHeight?: number } | null)?.maxHeight !== undefined) === light)) {+ return err("The preview proxy could not be made from this file lately; the original plays instead.", 409);+ }const capped = await renderJobCheck(userId);if (capped) return capped;const row = await prisma.cutRenderJob.create({diff --git a/site/src/cut/server/proxy.test.ts b/site/src/cut/server/proxy.test.tsindex 8c9bcb81..0d9ace17 100644--- a/site/src/cut/server/proxy.test.ts+++ b/site/src/cut/server/proxy.test.ts@@ -63,7 +63,7 @@ describe("proxy", () => {expect(args[args.indexOf("-movflags") + 1]).toBe("+faststart");// A keyframe every second: the preview seeks into the proxy at every// cut, and a seek decodes from the keyframe before it.- expect(args[args.indexOf("-force_key_frames") + 1]).toBe("expr:gte(t,n_forced*1)");+ expect(args[args.indexOf("-force_key_frames") + 1]).toBe("expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+1))");const vt = proxyFfmpegArgs("in.mov", "out.mp4", {encoder: "hevc_videotoolbox",@@ -79,6 +79,25 @@ describe("proxy", () => {expect(vt[vt.indexOf("-c:v") + 1]).toBe("hevc_videotoolbox");});+ test("keyframes fall a second apart from the first frame, whatever time the master starts at", () => {+ const source = { matrix: "bt709" as const, fullRange: false, height: 1080, hasAudio: false };+ const args = proxyFfmpegArgs("in.ts", "out.mp4", { encoder: "libx265", source, maxHeight: 1080, crf: 20 });+ // With -copyts, t is the master's own timestamp: counting forced frames+ // against it made every frame a keyframe on a file starting at 1000 s.+ expect(args[args.indexOf("-force_key_frames") + 1]).toBe("expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+1))");+ });++ test("a light proxy holds a bitrate ceiling in either encoder", () => {+ const source = { matrix: "bt709" as const, fullRange: false, height: 2160, hasAudio: false };+ const x265 = proxyFfmpegArgs("in.mp4", "out.mp4", { encoder: "libx265", source, maxHeight: 1080, crf: 20, maxKbps: 6000 });+ expect(x265.join(" ")).toContain("-maxrate 6000k -bufsize 6000k");+ const vt = proxyFfmpegArgs("in.mp4", "out.mp4", { encoder: "hevc_videotoolbox", source, maxHeight: 1080, crf: 20, maxKbps: 6000 });+ expect(vt.join(" ")).toContain("-b:v 6000k");+ expect(vt).not.toContain("-q:v");+ // A ProRes proxy keeps its quality target and no ceiling.+ expect(proxyFfmpegArgs("in.mp4", "out.mp4", { encoder: "libx265", source, maxHeight: 1080, crf: 20 })).not.toContain("-maxrate");+ });+test("reads the fixture as a 10-bit BT.2020 ProRes master with sound", async () => {expect(await probeProxySource(FIXTURE)).toEqual({ matrix: "bt2020nc", fullRange: false, height: 64, hasAudio: true });});diff --git a/site/src/cut/server/proxy.ts b/site/src/cut/server/proxy.tsindex 5b443cac..0c0337d6 100644--- a/site/src/cut/server/proxy.ts+++ b/site/src/cut/server/proxy.ts@@ -47,6 +47,9 @@ export interface ProxyArgs {maxHeight: number;/** x265 crf; VideoToolbox's constant quality maps from it. */crf: number;+ /** A light proxy's bitrate ceiling, kbit/s (lib/proxyNames.ts): x265 holds+ * it under its crf, VideoToolbox encodes at it. Absent = quality alone. */+ maxKbps?: number;}/** swscale's name for a matrix. */@@ -76,10 +79,21 @@ export function proxyFfmpegArgs(src: string, out: string, opts: ProxyArgs): stri// the stream options below cover the container."setparams=color_primaries=bt709:color_trc=iec61966-2-1:colorspace=bt709:range=tv",].join(",");+ const kbps = opts.maxKbps && opts.maxKbps > 0 ? Math.round(opts.maxKbps) : 0;const video =encoder === "libx265"- ? ["-c:v", "libx265", "-preset", "medium", "-crf", String(opts.crf), "-profile:v", "main10", "-x265-params", "log-level=error"]- : ["-c:v", "hevc_videotoolbox", "-profile:v", "main10", "-q:v", String(vtQuality(opts.crf)), "-allow_sw", "1"];+ ? [+ "-c:v", "libx265", "-preset", "medium", "-crf", String(opts.crf),+ // A one-second buffer: the ceiling holds over every stretch the+ // preview fetches between keyframes.+ ...(kbps ? ["-maxrate", `${kbps}k`, "-bufsize", `${kbps}k`] : []),+ "-profile:v", "main10", "-x265-params", "log-level=error",+ ]+ : [+ "-c:v", "hevc_videotoolbox", "-profile:v", "main10",+ ...(kbps ? ["-b:v", `${kbps}k`] : ["-q:v", String(vtQuality(opts.crf))]),+ "-allow_sw", "1",+ ];return ["-y",// The master's timestamps, unshifted: the preview maps clip time to@@ -96,7 +110,9 @@ export function proxyFfmpegArgs(src: string, out: string, opts: ProxyArgs): stri...video,// A keyframe every second of the master's time: the preview seeks into// the proxy at every cut, and a seek decodes from the keyframe before it.- "-force_key_frames", "expr:gte(t,n_forced*1)",+ // Counted from the last forced one, since -copyts keeps a master's own+ // start time (a transport stream can start at 1000 s).+ "-force_key_frames", "expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+1))","-tag:v", "hvc1",...(source.hasAudio ? ["-c:a", "aac", "-b:a", "192k"] : []),"-movflags", "+faststart",@@ -168,13 +184,13 @@ export async function makeProxy(handle: RenderHandle,src: string,out: string,- opts: { maxHeight: number; crf: number; onProgress?: (seconds: number) => void }+ opts: { maxHeight: number; crf: number; maxKbps?: number; onProgress?: (seconds: number) => void }): Promise<ProxyOutcome> {const [source, dims, encoder] = await Promise.all([probeProxySource(src), videoDimensions(src), proxyEncoder()]);// The display height decides the scale — a phone's sideways file is// measured upright everywhere else in Cut.if (dims) source.height = dims.height;- await runFfmpeg(handle, proxyFfmpegArgs(src, out, { encoder, source, maxHeight: opts.maxHeight, crf: opts.crf }), opts.onProgress);+ await runFfmpeg(handle, proxyFfmpegArgs(src, out, { encoder, source, maxHeight: opts.maxHeight, crf: opts.crf, maxKbps: opts.maxKbps }), opts.onProgress);const [info, outDims] = await Promise.all([stat(out), videoDimensions(out)]);return { sizeBytes: info.size, ...(outDims ? { width: outDims.width, height: outDims.height } : {}) };}diff --git a/site/src/cut/worker/cardJob.ts b/site/src/cut/worker/cardJob.tsindex 84201297..56292330 100644--- a/site/src/cut/worker/cardJob.ts+++ b/site/src/cut/worker/cardJob.ts@@ -26,10 +26,12 @@ const GIF_RECIPES = [const MAX_GIF_BYTES = 3 * 1024 * 1024;/** ffmpeg for the derivations: short, bounded, and its own runner so the- * engine's export pipeline keeps a single entry point. */-function ffmpeg(args: string[]): Promise<void> {+ * engine's export pipeline keeps a single entry point. The process rides the+ * job's handle, so a cancel kills it and a shutdown can tell it stopped it. */+function ffmpeg(handle: RenderHandle, args: string[]): Promise<void> {return new Promise((resolve, reject) => {const proc = spawn("ffmpeg", ["-hide_banner", "-y", ...args]);+ handle.proc = proc;let tail = "";proc.stderr.on("data", (chunk: Buffer) => {tail = (tail + chunk.toString()).slice(-2000);@@ -54,18 +56,18 @@ async function bytesOf(file: string): Promise<number> {/** Still frame at t=0 — the video's actual first frame, not a poster picked* from the middle. */-async function renderStill(source: string, dest: string): Promise<number> {- await ffmpeg(["-i", source, "-frames:v", "1", "-q:v", "3", dest]);+async function renderStill(handle: RenderHandle, source: string, dest: string): Promise<number> {+ await ffmpeg(handle, ["-i", source, "-frames:v", "1", "-q:v", "3", dest]);return bytesOf(dest);}/** Animated card, stepping down through the recipes until it fits. The last* recipe's output ships whatever its size — a card slightly over budget beats* no card. */-async function renderAnimation(source: string, dest: string): Promise<number> {+async function renderAnimation(handle: RenderHandle, source: string, dest: string): Promise<number> {let bytes = 0;for (const [i, recipe] of GIF_RECIPES.entries()) {- await ffmpeg([+ await ffmpeg(handle, ["-i",source,"-vf",@@ -98,8 +100,8 @@ export async function storeCardArtifacts(const dir = path.dirname(handle.outPath);const stillPath = path.join(dir, "card.jpg");const animPath = path.join(dir, "card.gif");- const stillBytes = await renderStill(handle.outPath, stillPath);- const animBytes = await renderAnimation(handle.outPath, animPath);+ const stillBytes = await renderStill(handle, handle.outPath, stillPath);+ const animBytes = await renderAnimation(handle, handle.outPath, animPath);// Same guard the export path uses: a project deleted mid-render must not// leave storage charged for bytes nothing can free.diff --git a/site/src/cut/worker/db.test.ts b/site/src/cut/worker/db.test.tsindex 537f9d84..78afde73 100644--- a/site/src/cut/worker/db.test.ts+++ b/site/src/cut/worker/db.test.ts@@ -187,3 +187,11 @@ test("a running project preview prevents a second preview claim while other workexpect(await claimNextJob()).toBeNull();expect(rows[1].state).toBe("queued");});++test("an export waits behind no proxy: proxies take the tier after the watched work", async () => {+ // Opening a cloud project queues a proxy for every heavy clip; an export+ // asked for a moment later used to wait behind all of them.+ queueHarness([...Array<string>(12).fill("proxy"), "export", "preview"]);+ expect((await claimNextJob())?.kind).toBe("export");+ expect((await claimNextJob())?.kind).toBe("proxy");+});diff --git a/site/src/cut/worker/db.ts b/site/src/cut/worker/db.tsindex cdb4d6ca..0403ba65 100644--- a/site/src/cut/worker/db.ts+++ b/site/src/cut/worker/db.ts@@ -39,18 +39,20 @@ export interface ClaimedJob {/** The kinds someone is watching happen: an export, a URL import, or a chat* turn has a progress surface on screen, while a hover proxy and a share card* are background polish nobody is waiting on. */-const WATCHED_KINDS = ["export", "import_url", "convert", "proxy", "agent_turn"];+const WATCHED_KINDS = ["export", "import_url", "convert", "agent_turn"];/** Atomically claim the next queued job: the updateMany's state guard makes* exactly one worker win each row, so replicas never double-run a job.*- * Watched kinds go first, FIFO within a tier. Strict FIFO across all kinds let+ * Watched kinds go first, then proxies, FIFO within a tier. Strict FIFO across all kinds let* a proxy render queued moments earlier hold up an export the user is staring* at — the wait was invisible in the UI, because the export's elapsed clock* only starts once it is claimed. */export async function claimNextJob(): Promise<ClaimedJob | null> {// Editor command batches stay queued for the card that owns their document.- for (const kind of [{ in: WATCHED_KINDS }, { in: ["preview", "card", "hls"] }]) {+ // Proxies come next: opening a cloud project queues one for every heavy+ // clip, and an export asked for a moment later must not wait behind them.+ for (const kind of [{ in: WATCHED_KINDS }, { in: ["proxy"] }, { in: ["preview", "card", "hls"] }]) {const candidates = await prisma.cutRenderJob.findMany({where: { state: "queued", kind },orderBy: { createdAt: "asc" },diff --git a/site/src/cut/worker/main.ts b/site/src/cut/worker/main.tsindex 95af5663..dce10903 100644--- a/site/src/cut/worker/main.ts+++ b/site/src/cut/worker/main.ts@@ -9,6 +9,7 @@ import { runImportUrlJob } from "./importUrlJob";import { runTurnJob } from "./turnJob";import { describeRuntime, installHeadlessRuntime } from "../lib/headless/runtime";import { deleteObjects } from "./r2";+import { failedJobFate, InputFault } from "./settle";// The cloud render worker: a headless loop that claims CutRenderJob rows the// hosted API queues (export, preview, import_url, convert, proxy, agent_turn),@@ -41,6 +42,9 @@ const SWEEP_EVERY_MS = 15_000;// inside that window, keeping enough of it back to requeue whatever is still// going when it closes.const DRAIN_MS = 13 * 60_000;+/** Set when the platform sends SIGTERM: a run whose process stops from here+ * on is the shutdown's doing (`failedJobFate`). */+let signaled = false;/** The longest a job of each kind may run. Everything else about a job's cost* is bounded when it is queued; this bounds the one thing that isn't — a@@ -200,15 +204,27 @@ async function runJob(job: ClaimedJob): Promise<void> {: raw;// A canceled job's row already says so; the running-state guard also// keeps this from resurrecting a row canceled between watcher ticks.- if (!entry.canceled) {+ const fate = failedJobFate({ canceled: entry.canceled, signaled, proc: entry.handle.proc });+ if (fate === "requeue") {+ // Its overlays stay for the replica that claims it next.+ entry.retry = true;await prisma.cutRenderJob.updateMany({where: { id: job.id, state: "running" },- data: { state: "error", error: message },+ data: { state: "queued", claimedAt: null, progress: 0 },+ })+ .catch(() => {});+ } else if (fate === "error") {+ // A failure on the job's input records which, for the route to read.+ const fault = err instanceof InputFault ? { result: { fault: err.fault } } : {};+ await prisma.cutRenderJob+ .updateMany({+ where: { id: job.id, state: "running" },+ data: { state: "error", error: message, ...fault },}).catch(() => {});}- console.error(`[cut-worker] ${job.kind} ${job.id} ${entry.canceled ? "canceled" : "failed"}: ${raw}`);+ console.error(`[cut-worker] ${job.kind} ${job.id} ${entry.canceled ? "canceled" : fate === "requeue" ? "requeued on shutdown" : "failed"}: ${raw}`);// A canceled turn retires this process once its cleanup settles: the// abort tears the agent loop down, and a fresh process guarantees the// next claim starts with clean per-user transport bindings even if some@@ -258,7 +274,10 @@ async function stop(): Promise<void> {}async function main(): Promise<void> {- process.on("SIGTERM", () => void stop());+ process.on("SIGTERM", () => {+ signaled = true;+ void stop();+ });process.on("SIGINT", () => void stop());// Canvas, decoders, Web Audio, and fonts, standing in for the page a turn// would otherwise run in. A worker missing a native module loses thatdiff --git a/site/src/cut/worker/proxyJob.test.ts b/site/src/cut/worker/proxyJob.test.tsindex 4b040866..5f5b89d7 100644--- a/site/src/cut/worker/proxyJob.test.ts+++ b/site/src/cut/worker/proxyJob.test.ts@@ -1,7 +1,10 @@import { describe, expect, mock, test } from "bun:test";import { writeFile } from "node:fs/promises";import type { prisma } from "@/lib/prisma";+import type { RenderHandle } from "../server/exportPipeline";import { runProxyJob, type ProxyJobIO } from "./proxyJob";+import { PROXY_SOURCE_FAULT } from "../lib/proxyNames";+import { InputFault } from "./settle";function harness(taken: string[] = [], spec: Record<string, unknown> = { file: "clip.mov" }) {const findMany = mock(async (_args: unknown) => { void _args; return taken.map((fileName) => ({ fileName })); });@@ -18,15 +21,25 @@ function harness(taken: string[] = [], spec: Record<string, unknown> = { file: "return { sizeBytes: 5, width: 64, height: 64 };}),duration: mock(async () => 2),- settings: mock(async () => ({ proxyMaxHeight: 1080, proxyCrf: 20 })),+ settings: mock(async () => ({ proxyMaxHeight: 1080, proxyCrf: 20, lightKbps: 6000 })),} satisfies ProxyJobIO;const job = { id: "proxy-1", userId: "owner", projectId: "project", kind: "proxy", outName: null, spec };- const handle = { tmpDir: "", outPath: "", progress: 0, log: [] as string[] };+ const handle: RenderHandle = { tmpDir: "", outPath: "", progress: 0, log: [] as string[] };const run = (canceled = () => false) => runProxyJob(job, handle, canceled, io);return { run, io, handle, findMany };}describe("proxy worker", () => {+ test("a heavy video's job builds the capped light copy under its own name", async () => {+ const h = harness([], { file: "clip.mov", maxHeight: 1080 });+ expect((await h.run()).fileName).toBe("clip.mov.light.mp4");+ expect(h.io.make.mock.calls[0][3]).toMatchObject({ maxHeight: 1080, maxKbps: 6000 });+ // A ProRes job keeps the full-quality proxy, uncapped.+ const pro = harness();+ expect((await pro.run()).fileName).toBe("clip.proxy.mp4");+ expect((pro.io.make.mock.calls[0][3] as { maxKbps?: number }).maxKbps).toBeUndefined();+ });+test("a job that asks for a smaller copy gets it, never one past the account's size", async () => {const small = harness([], { file: "clip.mov", maxHeight: 720 });await small.run();@@ -73,4 +86,24 @@ describe("proxy worker", () => {await expect(h.run(() => true)).rejects.toThrow("canceled");expect(h.io.make).not.toHaveBeenCalled();});++ // A master ffmpeg refuses fails the same way on every try; a build killed+ // by a signal or lost to storage does not, and the row tells them apart.+ test("an encode that refuses the master records the file's fault; a killed one does not", async () => {+ const refused = harness();+ refused.io.make.mockImplementation(async (handle: unknown) => {+ (handle as RenderHandle).proc = { signalCode: null, exitCode: 1 } as never;+ throw new Error("ffmpeg exited with code 1.");+ });+ const err = await refused.run().catch((e: unknown) => e);+ expect(err instanceof InputFault).toBe(true);+ expect((err as InputFault).fault).toBe(PROXY_SOURCE_FAULT);++ const killed = harness();+ killed.io.make.mockImplementation(async (handle: unknown) => {+ (handle as RenderHandle).proc = { signalCode: "SIGKILL", exitCode: null } as never;+ throw new Error("Export stalled — no ffmpeg output for 120s.");+ });+ expect((await killed.run().catch((e: unknown) => e)) instanceof InputFault).toBe(false);+ });});diff --git a/site/src/cut/worker/proxyJob.ts b/site/src/cut/worker/proxyJob.tsindex 7362decd..1c9229be 100644--- a/site/src/cut/worker/proxyJob.ts+++ b/site/src/cut/worker/proxyJob.ts@@ -4,11 +4,13 @@ import path from "node:path";import { getGlobalSetting } from "@/lib/config/effective";import type { RenderHandle } from "../server/exportPipeline";import { makeProxy, proxyNameFor } from "../server/proxy";+import { lightProxyNameFor, PROXY_SOURCE_FAULT } from "../lib/proxyNames";import { probeDuration } from "../server/frames";import { dedupeName } from "../server/cloud/util";import { prisma, registerObject, unregisterObjects, type ClaimedJob } from "./db";import { deleteObjects, downloadToFile, mediaKey, uploadFile } from "./r2";import { runLibraryProxyJob } from "./libraryProxyJob";+import { InputFault, refusedInput } from "./settle";import type { LibraryAsset } from "../server/library";/** What a proxy job records in CutRenderJob.result — the same shape the@@ -30,7 +32,7 @@ export interface ProxyJobIO {unregister: typeof unregisterObjects;make: typeof makeProxy;duration: typeof probeDuration;- settings: () => Promise<{ proxyMaxHeight: number; proxyCrf: number }>;+ settings: () => Promise<{ proxyMaxHeight: number; proxyCrf: number; lightKbps: number }>;}const defaultIO: ProxyJobIO = {@@ -42,7 +44,10 @@ const defaultIO: ProxyJobIO = {unregister: unregisterObjects,make: makeProxy,duration: probeDuration,- settings: () => getGlobalSetting("cutColor"),+ settings: async () => {+ const [color, proxy] = await Promise.all([getGlobalSetting("cutColor"), getGlobalSetting("cutProxy")]);+ return { ...color, lightKbps: proxy.lightKbps };+ },};/**@@ -78,7 +83,10 @@ export async function runProxyJob(where: { projectId, kind: { in: ["media", "proxy"] } },select: { fileName: true },});- const fileName = dedupeName(proxyNameFor(file), new Set(rows.map((r) => r.fileName)));+ // A job that names a height is a heavy video's: its proxy is the light+ // copy, capped in bitrate and named apart from a full-quality one.+ const light = !!(maxHeight && maxHeight > 0);+ const fileName = dedupeName(light ? lightProxyNameFor(file) : proxyNameFor(file), new Set(rows.map((r) => r.fileName)));const out = path.join(tmp, fileName);handle.outPath = out;@@ -88,9 +96,14 @@ export async function runProxyJob(// the account allows.maxHeight: maxHeight && maxHeight > 0 ? Math.min(settings.proxyMaxHeight, maxHeight) : settings.proxyMaxHeight,crf: settings.proxyCrf,+ ...(light ? { maxKbps: settings.lightKbps } : {}),onProgress: (seconds) => {if (total > 0) handle.progress = Math.min(0.99, seconds / total);},+ }).catch((e: unknown) => {+ // ffmpeg ran to its own end and refused the master: the row says the+ // file is at fault, so the route waits before trying it again.+ throw !isCanceled() && refusedInput(handle.proc) ? new InputFault(PROXY_SOURCE_FAULT, e) : e;});if (isCanceled()) throw new Error("Proxy canceled.");diff --git a/site/src/cut/worker/settle.test.ts b/site/src/cut/worker/settle.test.tsnew file mode 100644index 00000000..7792accc--- /dev/null+++ b/site/src/cut/worker/settle.test.ts@@ -0,0 +1,31 @@+import { expect, test } from "bun:test";+import { failedJobFate } from "./settle";++/** A deploy rolls the worker's replicas with SIGTERM, and the signal reaches+ * the job's ffmpeg too: it stops mid-encode and the run fails a minute in. A+ * proxy job died that way while the drain was meant to let it finish. A run+ * whose process the shutdown stopped goes back on the queue for the next+ * replica; any other failure during the drain is the job's own. */++const killed = { signalCode: "SIGTERM", exitCode: null };+/** ffmpeg traps SIGTERM and exits 255 ("received signal 15"). */+const trapped = { signalCode: null, exitCode: 255 };+const failed = { signalCode: null, exitCode: 1 };++test("a run whose process the shutdown stopped goes back on the queue", () => {+ expect(failedJobFate({ canceled: false, signaled: true, proc: killed })).toBe("requeue");+ expect(failedJobFate({ canceled: false, signaled: true, proc: trapped })).toBe("requeue");+});++test("a run that fails on its own during the drain is an error", () => {+ // A turn's model or tool error, a corrupt input: re-running repeats it,+ // and a turn would edit and charge twice.+ expect(failedJobFate({ canceled: false, signaled: true })).toBe("error");+ expect(failedJobFate({ canceled: false, signaled: true, proc: failed })).toBe("error");+ expect(failedJobFate({ canceled: false, signaled: false, proc: killed })).toBe("error");+});++test("a canceled run's row already says so", () => {+ expect(failedJobFate({ canceled: true, signaled: true, proc: killed })).toBe("leave");+ expect(failedJobFate({ canceled: true, signaled: false })).toBe("leave");+});diff --git a/site/src/cut/worker/settle.ts b/site/src/cut/worker/settle.tsnew file mode 100644index 00000000..08e14991--- /dev/null+++ b/site/src/cut/worker/settle.ts@@ -0,0 +1,44 @@+/** The exit code ffmpeg gives when it traps a termination signal and stops+ * ("Exiting normally, received signal 15"). */+const FFMPEG_SIGNAL_EXIT = 255;++/** How the job's last child process ended, as a ChildProcess reports it. */+type ProcExit = { signalCode: string | null; exitCode: number | null };++/**+ * What becomes of a job whose run threw. A canceled job's row already says so+ * and is left alone. Once the worker has been sent SIGTERM, the platform's+ * signal reaches the job's ffmpeg as well, which stops mid-encode: a run whose+ * process ended that way goes back on the queue for the next replica. Any+ * other failure, during the drain or not, is the job's own error.+ */+export function failedJobFate(run: { canceled: boolean; signaled: boolean; proc?: ProcExit }): "leave" | "requeue" | "error" {+ if (run.canceled) {+ return "leave";+ }+ return run.signaled && run.proc && stoppedBySignal(run.proc) ? "requeue" : "error";+}++function stoppedBySignal(proc: ProcExit): boolean {+ return proc.signalCode !== null || proc.exitCode === FFMPEG_SIGNAL_EXIT;+}++/** True when the job's process ran to its own end and refused its input: a+ * non-zero exit that no signal caused. */+export function refusedInput(proc: ProcExit | undefined): boolean {+ if (!proc || proc.exitCode === null || proc.exitCode === 0) {+ return false;+ }+ return !stoppedBySignal(proc);+}++/** A run that failed on its input, carrying the `fault` its row records in+ * `result` beside the message. */+export class InputFault extends Error {+ constructor(+ readonly fault: string,+ cause: unknown+ ) {+ super(cause instanceof Error ? cause.message : String(cause));+ }+}diff --git a/site/src/lib/config/registry.ts b/site/src/lib/config/registry.tsindex 5dc7c75d..9d1e9d95 100644--- a/site/src/lib/config/registry.ts+++ b/site/src/lib/config/registry.ts@@ -435,6 +435,23 @@ export const SETTINGS = defineSettings({description:"The ramp on the sound a split edit carries past its picture, and how lining up a separately recorded sound listens, how sure it must be, and how long a recording it searches.",},+ cutProxy: {+ schema: z+ .object({+ // A heavy video's light proxy: its bitrate ceiling at the 1080p cap,+ // kbit/s, so a fast cut streams it from the cloud in time.+ lightKbps: z.number().int().min(500).max(50_000),+ // How long a master ffmpeg could not make a proxy from keeps the+ // cloud from trying again, hours.+ retryAfterHours: z.number().min(0).max(24 * 30),+ })+ .strict(),+ default: { lightKbps: 6000, retryAfterHours: 24 },+ public: false,+ title: "Preview proxies",+ description:+ "The light proxy's bitrate ceiling for heavy cloud videos, and how long a master that could not be proxied waits before the cloud tries it again.",+ },cutMotion: {schema: z.object({ motionBlur: z.number().min(0.05).max(1) }).strict(),default: { motionBlur: 0.5 },