Open source CapCut alternative.
TypeScript
fix(site): keep tab export claims off the worker queue [rebuild]
David Le066a07828eee
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.tsindex eef0677a..8598624d 100644--- a/site/src/cut/lib/exportClient.ts+++ b/site/src/cut/lib/exportClient.ts@@ -1826,6 +1826,20 @@ export async function runBrowserExport(// The render keeps reading the project's store-served blob URLs after the// user opens another project; the hold keeps them alive until it finishes.holdRegistered(`/api/cut/projects/${projectId}/`);+ // The claim's row hears from the tab on a timer, so the worker's stale sweep+ // tells a render still drawing from a tab that closed mid-render.+ let reported = 0;+ const heartbeat = setInterval(+ () =>+ void backend+ .fetch(`/api/cut/export/client/${jobId}/progress`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ ratio: reported }),+ })+ .catch(() => {}),+ CLIENT_HEARTBEAT_MS+ );try {const rendered = await renderProjectToMp4(doc, settings, {// Read the asset's URL at the moment it is needed rather than off the@@ -1835,7 +1849,10 @@ export async function runBrowserExport(renderAssetUrl(projectId, asset),signal: opts.signal,// The render is nearly all of the work; the upload is the tail of the bar.- onProgress: ({ ratio }) => opts.onProgress?.(ratio * 0.9),+ onProgress: ({ ratio }) => {+ reported = ratio * 0.9;+ opts.onProgress?.(reported);+ },});try {@@ -1876,6 +1893,7 @@ export async function runBrowserExport(.catch(() => {});throw err;} finally {+ clearInterval(heartbeat);releaseRegistered(`/api/cut/projects/${projectId}/`);}}diff --git a/site/src/cut/lib/exportDelivery.ts b/site/src/cut/lib/exportDelivery.tsindex 87c7b15e..f7e190cf 100644--- a/site/src/cut/lib/exportDelivery.ts+++ b/site/src/cut/lib/exportDelivery.ts@@ -147,6 +147,12 @@ export function isStemsName(name: string): boolean {* beside it, whose name it claims too. */export type ExportFiles = "file" | "fileAndStems";+/** Whether a stored export job is a tab render's claim: the browser draws+ * and encodes it, so no worker can run it. */+export function isTabExport(stored: unknown): boolean {+ return (stored as { client?: boolean } | null)?.client === true;+}+/** Whether a stored export job asked for stems, however it was queued: a* tab render's claim, a document export, or a built spec with a stem plan. */export function jobWantsStems(stored: unknown): boolean {diff --git a/site/src/cut/server/cloud/exportJobs.test.ts b/site/src/cut/server/cloud/exportJobs.test.tsindex 2b35f061..e7684039 100644--- a/site/src/cut/server/cloud/exportJobs.test.ts+++ b/site/src/cut/server/cloud/exportJobs.test.ts@@ -89,3 +89,27 @@ test("a released render with stems takes its own unregistered stems", async () =await jobsCloud.exportClientRelease("u", "j");expect(deleted).toEqual(["cut/u/p/exports/Trip.mp4", "cut/u/p/exports/Trip stems.zip"]);});++// A row the worker once failed while the tab drew it lands clean, and the+// tab's heartbeat touches only its own running claim.+test("a finished tab render clears an error left on its row", async () => {+ job = { id: "j", kind: "export", projectId: "p", outName: "Trip.mp4", state: "error", spec: { client: true } };+ sizes.set("cut/u/p/exports/Trip.mp4", 50 * MB);+ const res = await jobsCloud.exportClientComplete("u", post({ jobId: "j" }));+ expect(res.status).toBe(200);+ expect(tx.cutRenderJob.update.mock.calls.at(-1)).toEqual([+ { where: { id: "j" }, data: { state: "done", progress: 1, outputKey: "cut/u/p/exports/Trip.mp4", error: null } },+ ] as never);+});++test("the tab's heartbeat bumps its own running claim, clamping the ratio", async () => {+ const res = await jobsCloud.exportClientProgress("u", "j", post({ ratio: 1.4 }));+ expect(res.status).toBe(200);+ const [args] = prisma.cutRenderJob.updateMany.mock.calls.at(-1) as unknown as [{ where: unknown; data: { progress?: number; updatedAt: Date } }];+ expect(args.where).toEqual({ id: "j", userId: "u", kind: "export", state: "running", spec: { path: ["client"], equals: true } });+ expect(args.data.progress).toBe(1);+ expect(args.data.updatedAt instanceof Date).toBe(true);+ await jobsCloud.exportClientProgress("u", "j", post({}));+ const [bare] = prisma.cutRenderJob.updateMany.mock.calls.at(-1) as unknown as [{ data: Record<string, unknown> }];+ expect("progress" in bare.data).toBe(false);+});diff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.tsindex 92b3bfe1..87ee3eff 100644--- a/site/src/cut/server/cloud/jobs.ts+++ b/site/src/cut/server/cloud/jobs.ts@@ -702,7 +702,8 @@ export const jobsCloud = {await addUsage(tx, userId, bytes);await tx.cutRenderJob.update({where: { id: row.id },- data: { state: "done", progress: 1, outputKey: key },+ // A row the worker once mistook for its own may carry its error.+ data: { state: "done", progress: 1, outputKey: key, error: null },});});return Response.json({ id: row.id, outName });@@ -711,6 +712,27 @@ export const jobsCloud = {}},+ /**+ * A tab render's heartbeat: its progress so far, and a fresh updatedAt that+ * tells the worker's stale sweep the tab is still drawing. A claim whose tab+ * goes quiet for minutes is dismissed there.+ */+ async exportClientProgress(userId: string, jobId: string, req: Request) {+ try {+ const body = (await req.json().catch(() => ({}))) as { ratio?: unknown };+ const ratio = typeof body.ratio === "number" && Number.isFinite(body.ratio)+ ? Math.min(1, Math.max(0, body.ratio))+ : undefined;+ await prisma.cutRenderJob.updateMany({+ where: { id: jobId, userId, kind: "export", state: "running", spec: { path: ["client"], equals: true } },+ data: { updatedAt: new Date(), ...(ratio === undefined ? {} : { progress: ratio }) },+ });+ return Response.json({ ok: true });+ } catch (e) {+ return caught(e, "Could not record the export's progress.");+ }+ },+/*** Give back a name a browser render claimed but will not use.*diff --git a/site/src/cut/server/cloud/routes.ts b/site/src/cut/server/cloud/routes.tsindex 4f45a571..3ff4b177 100644--- a/site/src/cut/server/cloud/routes.ts+++ b/site/src/cut/server/cloud/routes.ts@@ -108,6 +108,7 @@ const CUT_CLOUD_ROUTES: CloudRoute[] = [// register the finished file. No queue in between — the tab is the worker.{ method: "POST", path: "/api/cut-cloud/export/client/presign", handler: (r, u) => jobsCloud.exportClientPresign(u, r) },{ method: "POST", path: "/api/cut-cloud/export/client/complete", handler: (r, u) => jobsCloud.exportClientComplete(u, r) },+ { method: "POST", path: "/api/cut-cloud/export/client/:jobId/progress", handler: (r, u, p) => jobsCloud.exportClientProgress(u, p.jobId, r) },{ method: "POST", path: "/api/cut-cloud/export/client/:jobId/release", handler: (_r, u, p) => jobsCloud.exportClientRelease(u, p.jobId) },{ method: "GET", path: "/api/cut-cloud/export/:jobId", handler: (_r, u, p) => jobsCloud.exportStatus(u, p.jobId) },{ method: "DELETE", path: "/api/cut-cloud/export/:jobId", handler: (_r, u, p) => jobsCloud.exportCancel(u, p.jobId) },diff --git a/site/src/cut/worker/exportJob.ts b/site/src/cut/worker/exportJob.tsindex ce1c1500..258ea70a 100644--- a/site/src/cut/worker/exportJob.ts+++ b/site/src/cut/worker/exportJob.ts@@ -9,7 +9,7 @@ import { prisma, registerObjectIn, storageTransaction, type ClaimedJob } from ".import { deleteObjects, downloadToFile, exportKey, mediaKey, mimeFor, uploadFile } from "./r2";import { overlayKey } from "../server/cloud/r2";import { STORAGE_FULL } from "../lib/operationFailure";-import { specMediaFiles, STEMS_MIME, stemsArchiveName } from "../lib/exportDelivery";+import { isTabExport, specMediaFiles, STEMS_MIME, stemsArchiveName } from "../lib/exportDelivery";import { runnerSession } from "./session";import { bindCutColor } from "../lib/colorSettings";import { bindCutExportPasses } from "../lib/exportPassSettings";@@ -133,6 +133,8 @@ export async function runExportJob(if (!projectId) throw new Error("Export job has no project.");if (stored.fromDoc && !stored.fromDoc.snapshot?.doc)throw new Error("This render has no captured document. Queue the export again.");+ // A tab render's claim carries no spec: the tab that made it draws it.+ if (isTabExport(stored)) throw new Error("This export renders in the browser tab that started it.");if (!stored.fromDoc && (!stored.spec || !Array.isArray(stored.spec.clips))) {throw new Error("Malformed export spec.");}diff --git a/site/src/cut/worker/main.ts b/site/src/cut/worker/main.tsindex dce10903..3aaaecba 100644--- a/site/src/cut/worker/main.ts+++ b/site/src/cut/worker/main.ts@@ -9,7 +9,7 @@ import { runImportUrlJob } from "./importUrlJob";import { runTurnJob } from "./turnJob";import { describeRuntime, installHeadlessRuntime } from "../lib/headless/runtime";import { deleteObjects } from "./r2";-import { failedJobFate, InputFault } from "./settle";+import { failedJobFate, InputFault, staleRunningFate, TAB_EXPORT_SILENCE_MS } from "./settle";// The cloud render worker: a headless loop that claims CutRenderJob rows the// hosted API queues (export, preview, import_url, convert, proxy, agent_turn),@@ -80,20 +80,42 @@ const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));/** Requeue "running" rows whose heartbeat (updatedAt) has gone quiet: their* worker died without SIGTERM (OOM, host kill), so nothing else will ever- * settle them. Own in-flight jobs are excluded by id. */+ * settle them. Own in-flight jobs are excluded by id. A tab render's claim is+ * the tab's, never queued: one whose tab went silent is dismissed instead+ * (`staleRunningFate`). */async function sweepStaleRunning(): Promise<void> {- const { count } = await prisma.cutRenderJob.updateMany({+ const now = Date.now();+ const stale = await prisma.cutRenderJob.findMany({where: {state: "running",- updatedAt: { lt: new Date(Date.now() - STALE_RUNNING_MS) },+ updatedAt: { lt: new Date(now - STALE_RUNNING_MS) },id: { notIn: [...active.keys()] },// A command batch belongs to the editor in the ChatGPT card; the hosted// API expires one whose editor went quiet.kind: { not: "commands" },},- data: { state: "queued", claimedAt: null, progress: 0 },+ select: { id: true, spec: true, updatedAt: true },});- if (count > 0) console.log(`[cut-worker] requeued ${count} stale running job(s)`);+ const ids = (fate: ReturnType<typeof staleRunningFate>) =>+ stale.filter((row) => staleRunningFate(row, now) === fate).map((row) => row.id);+ const requeue = ids("requeue");+ const dismiss = ids("dismiss");+ // The state and heartbeat guards again, so a row that moved or beat since+ // the read is left alone.+ if (requeue.length > 0) {+ const { count } = await prisma.cutRenderJob.updateMany({+ where: { id: { in: requeue }, state: "running", updatedAt: { lt: new Date(now - STALE_RUNNING_MS) } },+ data: { state: "queued", claimedAt: null, progress: 0 },+ });+ if (count > 0) console.log(`[cut-worker] requeued ${count} stale running job(s)`);+ }+ if (dismiss.length > 0) {+ const { count } = await prisma.cutRenderJob.updateMany({+ where: { id: { in: dismiss }, state: "running", updatedAt: { lt: new Date(now - TAB_EXPORT_SILENCE_MS) } },+ data: { state: "dismissed" },+ });+ if (count > 0) console.log(`[cut-worker] dismissed ${count} tab export(s) whose tab went quiet`);+ }}/** Mirror one running job to its row every WATCH_MS: push the pipeline'sdiff --git a/site/src/cut/worker/settle.test.ts b/site/src/cut/worker/settle.test.tsindex 7792accc..c96c7591 100644--- a/site/src/cut/worker/settle.test.ts+++ b/site/src/cut/worker/settle.test.ts@@ -1,5 +1,5 @@import { expect, test } from "bun:test";-import { failedJobFate } from "./settle";+import { failedJobFate, staleRunningFate, TAB_EXPORT_SILENCE_MS } from "./settle";/** A deploy rolls the worker's replicas with SIGTERM, and the signal reaches* the job's ffmpeg too: it stops mid-encode and the run fails a minute in. A@@ -29,3 +29,19 @@ test("a canceled run's row already says so", () => {expect(failedJobFate({ canceled: true, signaled: true, proc: killed })).toBe("leave");expect(failedJobFate({ canceled: true, signaled: false })).toBe("leave");});++// A cloud project's export renders in the tab, which holds a "running" row as+// its claim. The sweep used to requeue that row a minute in, and a worker then+// failed it with "Malformed export spec." while the tab was still drawing.+test("a quiet worker job goes back on the queue", () => {+ const now = Date.now();+ expect(staleRunningFate({ spec: { fromDoc: {} }, updatedAt: new Date(now - 90_000) }, now)).toBe("requeue");+ expect(staleRunningFate({ spec: null, updatedAt: new Date(now - 90_000) }, now)).toBe("requeue");+});++test("a tab render's claim is never queued, and is let go once its tab is silent", () => {+ const now = Date.now();+ const claim = { client: true };+ expect(staleRunningFate({ spec: claim, updatedAt: new Date(now - 90_000) }, now)).toBe("wait");+ expect(staleRunningFate({ spec: { ...claim, stems: true }, updatedAt: new Date(now - TAB_EXPORT_SILENCE_MS - 1) }, now)).toBe("dismiss");+});diff --git a/site/src/cut/worker/settle.ts b/site/src/cut/worker/settle.tsindex 08e14991..a6ff10bb 100644--- a/site/src/cut/worker/settle.ts+++ b/site/src/cut/worker/settle.ts@@ -1,3 +1,5 @@+import { isTabExport } from "../lib/exportDelivery";+/** The exit code ffmpeg gives when it traps a termination signal and stops* ("Exiting normally, received signal 15"). */const FFMPEG_SIGNAL_EXIT = 255;@@ -42,3 +44,22 @@ export class InputFault extends Error {super(cause instanceof Error ? cause.message : String(cause));}}++/** How long a tab render's claim may go without the tab's heartbeat before+ * the tab counts as closed. A hidden tab's timers can be held to one a+ * minute, so this is several of those. */+export const TAB_EXPORT_SILENCE_MS = 5 * 60_000;++/**+ * What the stale sweep does with a quiet "running" row. A worker's job lost+ * its worker and goes back on the queue. A tab render's claim never belonged+ * to a worker, which has nothing to draw it from, so it is never queued: once+ * its tab has been silent past TAB_EXPORT_SILENCE_MS the claim is dismissed,+ * which frees its slot.+ */+export function staleRunningFate(row: { spec: unknown; updatedAt: Date }, now: number): "requeue" | "dismiss" | "wait" {+ if (!isTabExport(row.spec)) {+ return "requeue";+ }+ return now - row.updatedAt.getTime() > TAB_EXPORT_SILENCE_MS ? "dismiss" : "wait";+}