Skip to content

Open source CapCut alternative.

TypeScript

fix(site): keep tab export claims off the worker queue [rebuild]

David Le066a07828eee
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index eef0677a..8598624d 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -1826,6 +1826,20 @@ export async function runBrowserExport(
// The render keeps reading the project's store-served blob URLs after the
// user opens another project; the hold keeps them alive until it finishes.
holdRegistered(`/api/cut/projects/${projectId}/`);
+ // The claim's row hears from the tab on a timer, so the worker's stale sweep
+ // tells a render still drawing from a tab that closed mid-render.
+ let reported = 0;
+ const heartbeat = setInterval(
+ () =>
+ void backend
+ .fetch(`/api/cut/export/client/${jobId}/progress`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ ratio: reported }),
+ })
+ .catch(() => {}),
+ CLIENT_HEARTBEAT_MS
+ );
try {
const rendered = await renderProjectToMp4(doc, settings, {
// Read the asset's URL at the moment it is needed rather than off the
@@ -1835,7 +1849,10 @@ export async function runBrowserExport(
renderAssetUrl(projectId, asset),
signal: opts.signal,
// The render is nearly all of the work; the upload is the tail of the bar.
- onProgress: ({ ratio }) => opts.onProgress?.(ratio * 0.9),
+ onProgress: ({ ratio }) => {
+ reported = ratio * 0.9;
+ opts.onProgress?.(reported);
+ },
});
try {
@@ -1876,6 +1893,7 @@ export async function runBrowserExport(
.catch(() => {});
throw err;
} finally {
+ clearInterval(heartbeat);
releaseRegistered(`/api/cut/projects/${projectId}/`);
}
}
diff --git a/site/src/cut/lib/exportDelivery.ts b/site/src/cut/lib/exportDelivery.ts
index 87c7b15e..f7e190cf 100644
--- a/site/src/cut/lib/exportDelivery.ts
+++ b/site/src/cut/lib/exportDelivery.ts
@@ -147,6 +147,12 @@ export function isStemsName(name: string): boolean {
* beside it, whose name it claims too. */
export type ExportFiles = "file" | "fileAndStems";
+/** Whether a stored export job is a tab render's claim: the browser draws
+ * and encodes it, so no worker can run it. */
+export function isTabExport(stored: unknown): boolean {
+ return (stored as { client?: boolean } | null)?.client === true;
+}
+
/** Whether a stored export job asked for stems, however it was queued: a
* tab render's claim, a document export, or a built spec with a stem plan. */
export function jobWantsStems(stored: unknown): boolean {
diff --git a/site/src/cut/server/cloud/exportJobs.test.ts b/site/src/cut/server/cloud/exportJobs.test.ts
index 2b35f061..e7684039 100644
--- a/site/src/cut/server/cloud/exportJobs.test.ts
+++ b/site/src/cut/server/cloud/exportJobs.test.ts
@@ -89,3 +89,27 @@ test("a released render with stems takes its own unregistered stems", async () =
await jobsCloud.exportClientRelease("u", "j");
expect(deleted).toEqual(["cut/u/p/exports/Trip.mp4", "cut/u/p/exports/Trip stems.zip"]);
});
+
+// A row the worker once failed while the tab drew it lands clean, and the
+// tab's heartbeat touches only its own running claim.
+test("a finished tab render clears an error left on its row", async () => {
+ job = { id: "j", kind: "export", projectId: "p", outName: "Trip.mp4", state: "error", spec: { client: true } };
+ sizes.set("cut/u/p/exports/Trip.mp4", 50 * MB);
+ const res = await jobsCloud.exportClientComplete("u", post({ jobId: "j" }));
+ expect(res.status).toBe(200);
+ expect(tx.cutRenderJob.update.mock.calls.at(-1)).toEqual([
+ { where: { id: "j" }, data: { state: "done", progress: 1, outputKey: "cut/u/p/exports/Trip.mp4", error: null } },
+ ] as never);
+});
+
+test("the tab's heartbeat bumps its own running claim, clamping the ratio", async () => {
+ const res = await jobsCloud.exportClientProgress("u", "j", post({ ratio: 1.4 }));
+ expect(res.status).toBe(200);
+ const [args] = prisma.cutRenderJob.updateMany.mock.calls.at(-1) as unknown as [{ where: unknown; data: { progress?: number; updatedAt: Date } }];
+ expect(args.where).toEqual({ id: "j", userId: "u", kind: "export", state: "running", spec: { path: ["client"], equals: true } });
+ expect(args.data.progress).toBe(1);
+ expect(args.data.updatedAt instanceof Date).toBe(true);
+ await jobsCloud.exportClientProgress("u", "j", post({}));
+ const [bare] = prisma.cutRenderJob.updateMany.mock.calls.at(-1) as unknown as [{ data: Record<string, unknown> }];
+ expect("progress" in bare.data).toBe(false);
+});
diff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.ts
index 92b3bfe1..87ee3eff 100644
--- a/site/src/cut/server/cloud/jobs.ts
+++ b/site/src/cut/server/cloud/jobs.ts
@@ -702,7 +702,8 @@ export const jobsCloud = {
await addUsage(tx, userId, bytes);
await tx.cutRenderJob.update({
where: { id: row.id },
- data: { state: "done", progress: 1, outputKey: key },
+ // A row the worker once mistook for its own may carry its error.
+ data: { state: "done", progress: 1, outputKey: key, error: null },
});
});
return Response.json({ id: row.id, outName });
@@ -711,6 +712,27 @@ export const jobsCloud = {
}
},
+ /**
+ * A tab render's heartbeat: its progress so far, and a fresh updatedAt that
+ * tells the worker's stale sweep the tab is still drawing. A claim whose tab
+ * goes quiet for minutes is dismissed there.
+ */
+ async exportClientProgress(userId: string, jobId: string, req: Request) {
+ try {
+ const body = (await req.json().catch(() => ({}))) as { ratio?: unknown };
+ const ratio = typeof body.ratio === "number" && Number.isFinite(body.ratio)
+ ? Math.min(1, Math.max(0, body.ratio))
+ : undefined;
+ await prisma.cutRenderJob.updateMany({
+ where: { id: jobId, userId, kind: "export", state: "running", spec: { path: ["client"], equals: true } },
+ data: { updatedAt: new Date(), ...(ratio === undefined ? {} : { progress: ratio }) },
+ });
+ return Response.json({ ok: true });
+ } catch (e) {
+ return caught(e, "Could not record the export's progress.");
+ }
+ },
+
/**
* Give back a name a browser render claimed but will not use.
*
diff --git a/site/src/cut/server/cloud/routes.ts b/site/src/cut/server/cloud/routes.ts
index 4f45a571..3ff4b177 100644
--- a/site/src/cut/server/cloud/routes.ts
+++ b/site/src/cut/server/cloud/routes.ts
@@ -108,6 +108,7 @@ const CUT_CLOUD_ROUTES: CloudRoute[] = [
// register the finished file. No queue in between — the tab is the worker.
{ method: "POST", path: "/api/cut-cloud/export/client/presign", handler: (r, u) => jobsCloud.exportClientPresign(u, r) },
{ method: "POST", path: "/api/cut-cloud/export/client/complete", handler: (r, u) => jobsCloud.exportClientComplete(u, r) },
+ { method: "POST", path: "/api/cut-cloud/export/client/:jobId/progress", handler: (r, u, p) => jobsCloud.exportClientProgress(u, p.jobId, r) },
{ method: "POST", path: "/api/cut-cloud/export/client/:jobId/release", handler: (_r, u, p) => jobsCloud.exportClientRelease(u, p.jobId) },
{ method: "GET", path: "/api/cut-cloud/export/:jobId", handler: (_r, u, p) => jobsCloud.exportStatus(u, p.jobId) },
{ method: "DELETE", path: "/api/cut-cloud/export/:jobId", handler: (_r, u, p) => jobsCloud.exportCancel(u, p.jobId) },
diff --git a/site/src/cut/worker/exportJob.ts b/site/src/cut/worker/exportJob.ts
index ce1c1500..258ea70a 100644
--- a/site/src/cut/worker/exportJob.ts
+++ b/site/src/cut/worker/exportJob.ts
@@ -9,7 +9,7 @@ import { prisma, registerObjectIn, storageTransaction, type ClaimedJob } from ".
import { deleteObjects, downloadToFile, exportKey, mediaKey, mimeFor, uploadFile } from "./r2";
import { overlayKey } from "../server/cloud/r2";
import { STORAGE_FULL } from "../lib/operationFailure";
-import { specMediaFiles, STEMS_MIME, stemsArchiveName } from "../lib/exportDelivery";
+import { isTabExport, specMediaFiles, STEMS_MIME, stemsArchiveName } from "../lib/exportDelivery";
import { runnerSession } from "./session";
import { bindCutColor } from "../lib/colorSettings";
import { bindCutExportPasses } from "../lib/exportPassSettings";
@@ -133,6 +133,8 @@ export async function runExportJob(
if (!projectId) throw new Error("Export job has no project.");
if (stored.fromDoc && !stored.fromDoc.snapshot?.doc)
throw new Error("This render has no captured document. Queue the export again.");
+ // A tab render's claim carries no spec: the tab that made it draws it.
+ if (isTabExport(stored)) throw new Error("This export renders in the browser tab that started it.");
if (!stored.fromDoc && (!stored.spec || !Array.isArray(stored.spec.clips))) {
throw new Error("Malformed export spec.");
}
diff --git a/site/src/cut/worker/main.ts b/site/src/cut/worker/main.ts
index dce10903..3aaaecba 100644
--- a/site/src/cut/worker/main.ts
+++ b/site/src/cut/worker/main.ts
@@ -9,7 +9,7 @@ import { runImportUrlJob } from "./importUrlJob";
import { runTurnJob } from "./turnJob";
import { describeRuntime, installHeadlessRuntime } from "../lib/headless/runtime";
import { deleteObjects } from "./r2";
-import { failedJobFate, InputFault } from "./settle";
+import { failedJobFate, InputFault, staleRunningFate, TAB_EXPORT_SILENCE_MS } from "./settle";
// The cloud render worker: a headless loop that claims CutRenderJob rows the
// hosted API queues (export, preview, import_url, convert, proxy, agent_turn),
@@ -80,20 +80,42 @@ const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
/** Requeue "running" rows whose heartbeat (updatedAt) has gone quiet: their
* worker died without SIGTERM (OOM, host kill), so nothing else will ever
- * settle them. Own in-flight jobs are excluded by id. */
+ * settle them. Own in-flight jobs are excluded by id. A tab render's claim is
+ * the tab's, never queued: one whose tab went silent is dismissed instead
+ * (`staleRunningFate`). */
async function sweepStaleRunning(): Promise<void> {
- const { count } = await prisma.cutRenderJob.updateMany({
+ const now = Date.now();
+ const stale = await prisma.cutRenderJob.findMany({
where: {
state: "running",
- updatedAt: { lt: new Date(Date.now() - STALE_RUNNING_MS) },
+ updatedAt: { lt: new Date(now - STALE_RUNNING_MS) },
id: { notIn: [...active.keys()] },
// A command batch belongs to the editor in the ChatGPT card; the hosted
// API expires one whose editor went quiet.
kind: { not: "commands" },
},
- data: { state: "queued", claimedAt: null, progress: 0 },
+ select: { id: true, spec: true, updatedAt: true },
});
- if (count > 0) console.log(`[cut-worker] requeued ${count} stale running job(s)`);
+ const ids = (fate: ReturnType<typeof staleRunningFate>) =>
+ stale.filter((row) => staleRunningFate(row, now) === fate).map((row) => row.id);
+ const requeue = ids("requeue");
+ const dismiss = ids("dismiss");
+ // The state and heartbeat guards again, so a row that moved or beat since
+ // the read is left alone.
+ if (requeue.length > 0) {
+ const { count } = await prisma.cutRenderJob.updateMany({
+ where: { id: { in: requeue }, state: "running", updatedAt: { lt: new Date(now - STALE_RUNNING_MS) } },
+ data: { state: "queued", claimedAt: null, progress: 0 },
+ });
+ if (count > 0) console.log(`[cut-worker] requeued ${count} stale running job(s)`);
+ }
+ if (dismiss.length > 0) {
+ const { count } = await prisma.cutRenderJob.updateMany({
+ where: { id: { in: dismiss }, state: "running", updatedAt: { lt: new Date(now - TAB_EXPORT_SILENCE_MS) } },
+ data: { state: "dismissed" },
+ });
+ if (count > 0) console.log(`[cut-worker] dismissed ${count} tab export(s) whose tab went quiet`);
+ }
}
/** Mirror one running job to its row every WATCH_MS: push the pipeline's
diff --git a/site/src/cut/worker/settle.test.ts b/site/src/cut/worker/settle.test.ts
index 7792accc..c96c7591 100644
--- a/site/src/cut/worker/settle.test.ts
+++ b/site/src/cut/worker/settle.test.ts
@@ -1,5 +1,5 @@
import { expect, test } from "bun:test";
-import { failedJobFate } from "./settle";
+import { failedJobFate, staleRunningFate, TAB_EXPORT_SILENCE_MS } from "./settle";
/** A deploy rolls the worker's replicas with SIGTERM, and the signal reaches
* the job's ffmpeg too: it stops mid-encode and the run fails a minute in. A
@@ -29,3 +29,19 @@ test("a canceled run's row already says so", () => {
expect(failedJobFate({ canceled: true, signaled: true, proc: killed })).toBe("leave");
expect(failedJobFate({ canceled: true, signaled: false })).toBe("leave");
});
+
+// A cloud project's export renders in the tab, which holds a "running" row as
+// its claim. The sweep used to requeue that row a minute in, and a worker then
+// failed it with "Malformed export spec." while the tab was still drawing.
+test("a quiet worker job goes back on the queue", () => {
+ const now = Date.now();
+ expect(staleRunningFate({ spec: { fromDoc: {} }, updatedAt: new Date(now - 90_000) }, now)).toBe("requeue");
+ expect(staleRunningFate({ spec: null, updatedAt: new Date(now - 90_000) }, now)).toBe("requeue");
+});
+
+test("a tab render's claim is never queued, and is let go once its tab is silent", () => {
+ const now = Date.now();
+ const claim = { client: true };
+ expect(staleRunningFate({ spec: claim, updatedAt: new Date(now - 90_000) }, now)).toBe("wait");
+ expect(staleRunningFate({ spec: { ...claim, stems: true }, updatedAt: new Date(now - TAB_EXPORT_SILENCE_MS - 1) }, now)).toBe("dismiss");
+});
diff --git a/site/src/cut/worker/settle.ts b/site/src/cut/worker/settle.ts
index 08e14991..a6ff10bb 100644
--- a/site/src/cut/worker/settle.ts
+++ b/site/src/cut/worker/settle.ts
@@ -1,3 +1,5 @@
+import { isTabExport } from "../lib/exportDelivery";
+
/** The exit code ffmpeg gives when it traps a termination signal and stops
* ("Exiting normally, received signal 15"). */
const FFMPEG_SIGNAL_EXIT = 255;
@@ -42,3 +44,22 @@ export class InputFault extends Error {
super(cause instanceof Error ? cause.message : String(cause));
}
}
+
+/** How long a tab render's claim may go without the tab's heartbeat before
+ * the tab counts as closed. A hidden tab's timers can be held to one a
+ * minute, so this is several of those. */
+export const TAB_EXPORT_SILENCE_MS = 5 * 60_000;
+
+/**
+ * What the stale sweep does with a quiet "running" row. A worker's job lost
+ * its worker and goes back on the queue. A tab render's claim never belonged
+ * to a worker, which has nothing to draw it from, so it is never queued: once
+ * its tab has been silent past TAB_EXPORT_SILENCE_MS the claim is dismissed,
+ * which frees its slot.
+ */
+export function staleRunningFate(row: { spec: unknown; updatedAt: Date }, now: number): "requeue" | "dismiss" | "wait" {
+ if (!isTabExport(row.spec)) {
+ return "requeue";
+ }
+ return now - row.updatedAt.getTime() > TAB_EXPORT_SILENCE_MS ? "dismiss" : "wait";
+}