Skip to content

Open source CapCut alternative.

TypeScript

feat(site): a whole cut renders for a client that cannot build a spec

David Le0530f89da7bd
diff --git a/docs/guides/cut/local-compute.md b/docs/guides/cut/local-compute.md
index 07212284..139f478f 100644
--- a/docs/guides/cut/local-compute.md
+++ b/docs/guides/cut/local-compute.md
@@ -47,6 +47,8 @@ Two facts decide whether a tab can carry a render, both probed per export: origi
The worker also takes the renders the editor fires on its own — hover proxies, share cards, streaming ladders. Moving export to the Mac would be a third path and there is no longer much to win from it.
+A client with no compositor at all asks by project id instead. The phone sends the project and a size; the worker opens the stored document into the editor store, runs the same payload builder the export dialog runs, paints the overlay stills straight into the render's own scratch dir, and encodes. This is the headless runtime doing what it was built for — canvas, decoders, fonts standing in for the page — so the phone's export is the editor's export, not an approximation of it.
+
## Converting footage
A camera writes what it likes. A phone's `.mov` is HEVC, a screen recorder's is ProRes, and a camera file often carries raw PCM sound — formats some browsers have no decoder for. Cut reads media through the browser's own decoders, so a file this browser cannot decode is a file it cannot preview, cut, or render, and the fix is to rewrite it as MP4 with H.264 picture and AAC sound.
diff --git a/site/src/cut/lib/exportClient.ts b/site/src/cut/lib/exportClient.ts
index fdc1fe01..771f8c3d 100644
--- a/site/src/cut/lib/exportClient.ts
+++ b/site/src/cut/lib/exportClient.ts
@@ -190,7 +190,7 @@ export interface ExportDoc {
/** The neutral built cut: the engine spec plus the browser-rendered overlay
* PNGs. The local path serializes it to the engine's multipart form; the
* cloud path presigns the PNGs to R2 and posts the spec as JSON. */
-interface ExportPayload {
+export interface ExportPayload {
spec: object;
pngs: { name: string; blob: Blob }[];
}
@@ -409,7 +409,7 @@ function renderClipBorderPng(
/** Build the export spec + overlay PNGs from the cut. Media already lives in
* the project folder — the spec references it by file name; only overlay PNGs
* travel with the request. Shared by full exports and the low-res hover proxy. */
-async function buildExportPayload(
+export async function buildExportPayload(
projectId: string,
doc: ExportDoc,
settings: ExportSettings,
diff --git a/site/src/cut/lib/headless/docExport.ts b/site/src/cut/lib/headless/docExport.ts
new file mode 100644
index 00000000..05b09efd
--- /dev/null
+++ b/site/src/cut/lib/headless/docExport.ts
@@ -0,0 +1,80 @@
+import { writeFile } from "node:fs/promises";
+import path from "node:path";
+import {
+ buildExportPayload,
+ EXPORT_PRESETS,
+ originalSettings,
+ presetSettings,
+ type ExportDoc,
+ type ExportSettings,
+} from "../exportClient";
+import { useEditor } from "../store";
+import { bindHeadlessSession, type HeadlessSession } from "./bind";
+import { openCloudProject } from "./docSession";
+
+// A whole-timeline export built where the tab used to be. A client that can
+// render the cut itself sends the finished spec up; a client that cannot —
+// the phone — asks for the project by id, and this builds the same spec from
+// the stored document: open the doc into the editor store, run the export
+// payload builder the dialog runs, and drop the overlay pictures straight
+// into the render's own tmp dir.
+
+/** The sizes a doc-built export offers, named as the export dialog names
+ * them. `original` is derived from the footage on the timeline; the rest are
+ * the fixed presets flipped to the project's ratio. */
+export const DOC_EXPORT_PRESETS = ["original", ...EXPORT_PRESETS.map((p) => p.id)] as const;
+
+export type DocExportPreset = (typeof DOC_EXPORT_PRESETS)[number];
+
+export function isDocExportPreset(value: unknown): value is DocExportPreset {
+ return typeof value === "string" && (DOC_EXPORT_PRESETS as readonly string[]).includes(value);
+}
+
+/** The editor store's current contents as the builder's neutral document. */
+function docFromStore(): ExportDoc {
+ const s = useEditor.getState();
+ return {
+ aspect: s.aspect,
+ assets: s.assets,
+ clips: s.clips,
+ audioClips: s.audioClips,
+ overlays: s.overlays,
+ subtitles: s.subtitles,
+ fadeIn: s.fadeIn,
+ fadeOut: s.fadeOut,
+ background: s.background,
+ };
+}
+
+function settingsFor(preset: DocExportPreset, doc: ExportDoc): ExportSettings {
+ const fixed = EXPORT_PRESETS.find((p) => p.id === preset);
+ return fixed
+ ? presetSettings(fixed, doc.aspect)
+ : originalSettings(doc.aspect, doc.clips, doc.assets);
+}
+
+/**
+ * Build the render spec for a cloud project's whole timeline and stage its
+ * overlay pictures in `tmpDir` — the directory the pipeline reads stills from
+ * by base name.
+ */
+export async function buildDocExportSpec(
+ session: HeadlessSession,
+ projectId: string,
+ preset: DocExportPreset,
+ tmpDir: string
+): Promise<object> {
+ bindHeadlessSession(session);
+ await openCloudProject(session, projectId);
+ const doc = docFromStore();
+ const payload = await buildExportPayload(projectId, doc, settingsFor(preset, doc), "export");
+ await Promise.all(
+ payload.pngs.map(async (p) =>
+ writeFile(
+ path.join(tmpDir, path.basename(p.name)),
+ Buffer.from(await p.blob.arrayBuffer())
+ )
+ )
+ );
+ return payload.spec;
+}
diff --git a/site/src/cut/server/cloud/jobs.ts b/site/src/cut/server/cloud/jobs.ts
index 2c7e9e3f..89ef4160 100644
--- a/site/src/cut/server/cloud/jobs.ts
+++ b/site/src/cut/server/cloud/jobs.ts
@@ -206,6 +206,42 @@ export const jobsCloud = {
}
},
+ /**
+ * Queue a whole-timeline export for a client that cannot build a render spec
+ * — the phone. The row carries the project and the size; the worker opens
+ * the project document and builds the spec itself, so what renders is the
+ * cut as it stands, overlays and captions and all, exactly as the web
+ * dialog would have rendered it.
+ */
+ async exportFromDoc(userId: string, projectId: string, req: Request) {
+ try {
+ const body = (await req.json().catch(() => ({}))) as { preset?: string };
+ const project = await getProject(userId, projectId);
+ if (!project) return err("Project not found.", 404);
+ const capped = await renderJobCheck(userId);
+ if (capped) return capped;
+ const over = await quotaCheck(userId, 0, EXPORT_QUOTA_MARGIN);
+ if (over) return over;
+ // The worker owns the size list (it builds the spec), and falls back to
+ // the source-matched original for anything it does not know.
+ const preset = typeof body.preset === "string" ? body.preset : "original";
+ const outName = await exportName(userId, projectId, project.name);
+ const row = await prisma.cutRenderJob.create({
+ data: {
+ userId,
+ projectId,
+ kind: "export",
+ spec: { fromDoc: { preset } } as unknown as Prisma.InputJsonValue,
+ outName,
+ },
+ });
+ wakeRenderWorker();
+ return Response.json({ id: row.id, outName });
+ } catch (e) {
+ return caught(e, "Export failed to start.");
+ }
+ },
+
async exportStatus(userId: string, jobId: string) {
const row = await findJob(userId, jobId);
if (!row) return err("Unknown export.", 404);
diff --git a/site/src/cut/server/cloud/routes.ts b/site/src/cut/server/cloud/routes.ts
index 97a3fa5a..6a3b3ed6 100644
--- a/site/src/cut/server/cloud/routes.ts
+++ b/site/src/cut/server/cloud/routes.ts
@@ -46,6 +46,9 @@ const CUT_CLOUD_ROUTES: CloudRoute[] = [
{ method: "POST", path: "/api/cut-cloud/projects/:id/duplicate", handler: (_r, u, p) => copyJobs.requestDuplicate(u, p.id) },
{ method: "GET", path: "/api/cut-cloud/copy-jobs/:jobId", handler: (_r, u, p) => copyJobs.ownerStatus(u, p.jobId) },
{ method: "GET", path: "/api/cut-cloud/projects/:id/exports", handler: (_r, u, p) => projectsCloud.listExports(u, p.id) },
+ // A render asked for by id alone: the worker builds the spec from the stored
+ // document, for clients with no compositor of their own (the phone).
+ { method: "POST", path: "/api/cut-cloud/projects/:id/export", handler: (r, u, p) => jobsCloud.exportFromDoc(u, p.id, r) },
{ method: "GET", path: "/api/cut-cloud/projects/:id/exports/:file", handler: (r, u, p) => projectsCloud.serveExport(u, p.id, p.file, wantsDownload(r)) },
{ method: "DELETE", path: "/api/cut-cloud/projects/:id/exports/:file", handler: (_r, u, p) => projectsCloud.removeExport(u, p.id, p.file) },
{ method: "GET", path: "/api/cut-cloud/projects/:id/media/:file", handler: (r, u, p) => projectsCloud.serveMedia(u, p.id, p.file, wantsDownload(r)) },
diff --git a/site/src/cut/worker/exportJob.ts b/site/src/cut/worker/exportJob.ts
index 9b2b9d9f..ed976ed5 100644
--- a/site/src/cut/worker/exportJob.ts
+++ b/site/src/cut/worker/exportJob.ts
@@ -1,10 +1,12 @@
import { mkdir, mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
+import { buildDocExportSpec, isDocExportPreset } from "../lib/headless/docExport";
import { runExport, type ExportSpec, type RenderHandle } from "../server/exportPipeline";
import { storeCardArtifacts } from "./cardJob";
import { prisma, registerObject, type ClaimedJob } from "./db";
import { downloadToFile, exportKey, mediaKey, previewKey, uploadFile } from "./r2";
+import { runnerSession } from "./session";
/** The stored spec of an export/preview CutRenderJob: the engine export spec
* plus the R2 keys of the browser-rendered overlay PNGs (title/caption stills
@@ -12,6 +14,10 @@ import { downloadToFile, exportKey, mediaKey, previewKey, uploadFile } from "./r
* and puts the already-deduped export name on the row's outName column. */
export interface ExportJobSpec {
spec: ExportSpec;
+ /** A render asked for by a client that cannot build a spec — the phone. The
+ * row carries the size and nothing else; the worker opens the project doc
+ * and builds the spec itself. */
+ fromDoc?: { preset?: string };
overlays?: { name: string; key: string }[];
/** HLS only: whether the client burned captions into this render, decided by
* what the share grants. Recorded on the ladder so a share that later stops
@@ -103,20 +109,37 @@ export async function runExportJob(
job: ClaimedJob,
handle: RenderHandle
): Promise<{ outputKey: string; outName: string }> {
- const body = job.spec as ExportJobSpec;
- const spec = body.spec;
- if (!spec || !Array.isArray(spec.clips)) throw new Error("Malformed export spec.");
- const projectId = job.projectId ?? spec.projectId;
+ const stored = job.spec as ExportJobSpec;
+ const projectId = job.projectId ?? stored.spec?.projectId;
if (!projectId) throw new Error("Export job has no project.");
- const mode: "export" | "preview" | "card" =
- job.kind === "preview" || spec.target === "preview"
- ? "preview"
- : job.kind === "card" || spec.target === "card"
- ? "card"
- : "export";
- const preview = mode === "preview";
+ if (!stored.fromDoc && (!stored.spec || !Array.isArray(stored.spec.clips))) {
+ throw new Error("Malformed export spec.");
+ }
return inWorkDir(async (work) => {
+ // The pipeline reads stills out of this dir; a doc-built spec paints its
+ // own straight into it, so it exists before the spec does.
+ handle.tmpDir = path.join(work, "overlays");
+ await mkdir(handle.tmpDir, { recursive: true });
+ const body: ExportJobSpec = stored.fromDoc
+ ? {
+ ...stored,
+ spec: (await buildDocExportSpec(
+ runnerSession(job),
+ projectId,
+ isDocExportPreset(stored.fromDoc.preset) ? stored.fromDoc.preset : "original",
+ handle.tmpDir
+ )) as ExportSpec,
+ }
+ : stored;
+ const spec = body.spec;
+ const mode: "export" | "preview" | "card" =
+ job.kind === "preview" || spec.target === "preview"
+ ? "preview"
+ : job.kind === "card" || spec.target === "card"
+ ? "card"
+ : "export";
+ const preview = mode === "preview";
const mediaDir = await stageJobMedia(job, body, projectId, work, handle);
const outName = mode === "export" ? job.outName?.trim() || "export.mp4" : `${mode}.mp4`;
diff --git a/site/src/cut/worker/session.ts b/site/src/cut/worker/session.ts
new file mode 100644
index 00000000..d67a6778
--- /dev/null
+++ b/site/src/cut/worker/session.ts
@@ -0,0 +1,31 @@
+import { CUT_HOSTED_ORIGIN } from "../lib/hosts";
+import type { HeadlessSession } from "../lib/headless/bind";
+import type { ClaimedJob } from "./db";
+
+// How a worker speaks for the account whose job it claimed: the shared runner
+// grant plus the job's user. Every hosted call a job makes rides this, so the
+// work spends that user's credits and writes that user's project, exactly as
+// the page would.
+
+const CLIENT_ID = "donkey-cut-runner";
+
+export function runnerSession(job: ClaimedJob): HeadlessSession {
+ const secret = process.env.CUT_RUNNER_SECRET;
+ if (secret)
+ return {
+ base: CUT_HOSTED_ORIGIN,
+ headers: {
+ "x-donkey-client-id": CLIENT_ID,
+ "x-donkey-runner-secret": secret,
+ "x-donkey-runner-user": job.userId,
+ },
+ };
+ if (process.env.NODE_ENV === "production")
+ throw new Error("CUT_RUNNER_SECRET is not set — the runner cannot call the hosted API.");
+ // A dev worker run by hand against the dev server: the bypass grant, the
+ // same one the eval scripts use.
+ return {
+ base: CUT_HOSTED_ORIGIN,
+ headers: { "x-donkey-client-id": CLIENT_ID, "x-donkey-dev-auth-bypass": "1" },
+ };
+}
diff --git a/site/src/cut/worker/turnJob.ts b/site/src/cut/worker/turnJob.ts
index 4654560b..3be6afd7 100644
--- a/site/src/cut/worker/turnJob.ts
+++ b/site/src/cut/worker/turnJob.ts
@@ -1,12 +1,12 @@
import type { UIMessage } from "ai";
import { geminiModelRoles } from "@/lib/inference/gemini-models";
import { openCloudProject, pushCloudProject } from "../lib/headless/docSession";
-import { CUT_HOSTED_ORIGIN } from "../lib/hosts";
import { dropPiSession, streamCutChat } from "../lib/pi/cutAgent";
import { headlessDeps, type HeadlessSession } from "../lib/pi/serverDeps";
import type { RenderHandle } from "../server/exportPipeline";
import type { AgentTurnSpec } from "../server/cloud/turns";
import type { ClaimedJob } from "./db";
+import { runnerSession } from "./session";
// An agent turn, executed where the tab used to be: open the project doc into
// the editor store, run the production chat loop with the headless deps, push
@@ -15,7 +15,6 @@ import type { ClaimedJob } from "./db";
// shared secret plus the job's user — so the turn spends that user's credits
// and writes that user's project, exactly as the page would.
-const CLIENT_ID = "donkey-cut-runner";
/** A stored thread as the AI panel keeps it; the runner appends whole
* messages and touches nothing else in the payload. */
@@ -27,26 +26,6 @@ interface StoredThreadPayload {
[key: string]: unknown;
}
-function runnerSession(job: ClaimedJob): HeadlessSession {
- const secret = process.env.CUT_RUNNER_SECRET;
- if (secret)
- return {
- base: CUT_HOSTED_ORIGIN,
- headers: {
- "x-donkey-client-id": CLIENT_ID,
- "x-donkey-runner-secret": secret,
- "x-donkey-runner-user": job.userId,
- },
- };
- if (process.env.NODE_ENV === "production")
- throw new Error("CUT_RUNNER_SECRET is not set — the runner cannot call the hosted API.");
- // A dev worker run by hand against the dev server: the bypass grant, the
- // same one the eval scripts use.
- return {
- base: CUT_HOSTED_ORIGIN,
- headers: { "x-donkey-client-id": CLIENT_ID, "x-donkey-dev-auth-bypass": "1" },
- };
-}
/** Record the finished exchange on the project's stored thread: the turn's
* own messages plus the assistant reply, merged over whatever the thread