Skip to content

Open source CapCut alternative.

TypeScript

feat(site): headless runner seam for the cut editor [rebuild]

David Lea6bce19cb209
diff --git a/site/package-lock.json b/site/package-lock.json
index f26773e5..797546da 100644
--- a/site/package-lock.json
+++ b/site/package-lock.json
@@ -58,6 +58,7 @@
"recharts": "^3.8.0",
"resend": "^6.18.1",
"shadcn": "^4.7.0",
+ "skia-canvas": "^3.0.8",
"stripe": "^22.2.0",
"svix": "^1.99.1",
"tailwind-merge": "^3.6.0",
@@ -7915,7 +7916,6 @@
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
"integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==",
- "devOptional": true,
"license": "Apache-2.0",
"engines": {
"node": ">=8"
@@ -9498,6 +9498,26 @@
"dev": true,
"license": "ISC"
},
+ "node_modules/follow-redirects": {
+ "version": "1.16.0",
+ "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
+ "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
+ "funding": [
+ {
+ "type": "individual",
+ "url": "https://github.com/sponsors/RubenVerborgh"
+ }
+ ],
+ "license": "MIT",
+ "engines": {
+ "node": ">=4.0"
+ },
+ "peerDependenciesMeta": {
+ "debug": {
+ "optional": true
+ }
+ }
+ },
"node_modules/for-each": {
"version": "0.3.5",
"resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz",
@@ -13491,6 +13511,12 @@
"node": ">=6"
}
},
+ "node_modules/parenthesis": {
+ "version": "3.1.8",
+ "resolved": "https://registry.npmjs.org/parenthesis/-/parenthesis-3.1.8.tgz",
+ "integrity": "sha512-KF/U8tk54BgQewkJPvB4s/US3VQY68BRDpH638+7O/n58TpnwiwnOtGIOsT2/i+M78s61BBpeC83STB88d8sqw==",
+ "license": "MIT"
+ },
"node_modules/parse-entities": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz",
@@ -15425,6 +15451,19 @@
"integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==",
"license": "MIT"
},
+ "node_modules/skia-canvas": {
+ "version": "3.0.8",
+ "resolved": "https://registry.npmjs.org/skia-canvas/-/skia-canvas-3.0.8.tgz",
+ "integrity": "sha512-FSYKxp8Ng2vOeeOBiyPhnn6ui6FirPJXMyjk4PKl8N/OWzVrkMawUgY9zubIWHMdYtyWFn0gfX3QlRwg6HBmdg==",
+ "hasInstallScript": true,
+ "license": "MIT",
+ "dependencies": {
+ "detect-libc": "^2.1.1",
+ "follow-redirects": "^1.15.11",
+ "https-proxy-agent": "^7.0.6",
+ "string-split-by": "^1.0.0"
+ }
+ },
"node_modules/socket.io": {
"version": "4.8.3",
"resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.3.tgz",
@@ -15621,6 +15660,15 @@
"integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==",
"license": "MIT"
},
+ "node_modules/string-split-by": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/string-split-by/-/string-split-by-1.0.0.tgz",
+ "integrity": "sha512-KaJKY+hfpzNyet/emP81PJA9hTVSfxNLS9SFTWxdCnnW1/zOOwiV248+EfoX7IQFcBaOp4G5YE6xTJMF+pLg6A==",
+ "license": "MIT",
+ "dependencies": {
+ "parenthesis": "^3.1.5"
+ }
+ },
"node_modules/string-width": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz",
diff --git a/site/package.json b/site/package.json
index a8b6bde1..a3119c38 100644
--- a/site/package.json
+++ b/site/package.json
@@ -19,9 +19,10 @@
"prisma:prod": "dotenv -e .env -- prisma db push",
"engine:dev": "bun run src/cut/engine/main.ts",
"engine:build": "bash scripts/build-cut-engine.sh",
- "worker:build": "bun build --target=node --minify --external=pg-native src/cut/worker/main.ts --outfile dist/cut-worker/main.js",
+ "worker:build": "bun build --target=node --minify --external=pg-native --external=skia-canvas --loader .css:text src/cut/worker/main.ts --outfile dist/cut-worker/main.js",
"bench:watch": "bun run scripts/bench-watch.ts",
"eval:cut-chat": "bun run scripts/eval-cut-chat.ts",
+ "eval:cut-headless": "bun run scripts/eval-cut-headless.ts",
"email:dev": "email dev --dir src/emails --port 3001",
"test": "bun test src packages/effects-kit/src",
"test:genvideo": "bun run scripts/genvideo-selftest.ts",
@@ -74,6 +75,7 @@
"recharts": "^3.8.0",
"resend": "^6.18.1",
"shadcn": "^4.7.0",
+ "skia-canvas": "^3.0.8",
"stripe": "^22.2.0",
"svix": "^1.99.1",
"tailwind-merge": "^3.6.0",
diff --git a/site/scripts/eval-cut-headless.ts b/site/scripts/eval-cut-headless.ts
new file mode 100644
index 00000000..05a003e1
--- /dev/null
+++ b/site/scripts/eval-cut-headless.ts
@@ -0,0 +1,118 @@
+#!/usr/bin/env bun
+/**
+ * Headless turn smoke: one real chat turn through the production loop with no
+ * page attached. Seeds a cloud project on the dev server, opens its document
+ * into the editor store, runs streamCutChat on the headless deps, pushes the
+ * store back through the versioned PUT, and asserts the edit landed in the
+ * stored document.
+ *
+ * Run with the site dev server up:
+ * bun run scripts/eval-cut-headless.ts [--base http://localhost:3000] [--keep]
+ *
+ * Auth is the dev bypass header (a scripts-only grant), so runs are
+ * dev-server-only and spend no credits beyond the model call.
+ */
+
+import type { UIMessage } from "ai";
+import { openCloudProject, pushCloudProject } from "../src/cut/lib/headless/docSession";
+import { dropPiSession, streamCutChat } from "../src/cut/lib/pi/cutAgent";
+import { headlessDeps, type HeadlessSession } from "../src/cut/lib/pi/serverDeps";
+import { geminiModelRoles } from "../src/lib/inference/gemini-models";
+
+const arg = (name: string): string | undefined => {
+ const i = process.argv.indexOf(`--${name}`);
+ return i >= 0 ? process.argv[i + 1] : undefined;
+};
+const base = arg("base") ?? "http://localhost:3000";
+const keep = process.argv.includes("--keep");
+
+const session: HeadlessSession = {
+ base,
+ headers: {
+ "x-donkey-client-id": "donkey-cut-eval",
+ "x-donkey-dev-auth-bypass": "1",
+ },
+};
+
+const api = (path: string, init?: RequestInit) =>
+ fetch(base + path, {
+ ...init,
+ headers: { "Content-Type": "application/json", ...session.headers, ...init?.headers },
+ });
+
+const fail = (msg: string): never => {
+ // Throw so the seeded project's finally-block DELETE still runs; the
+ // exit code carries the failure.
+ process.exitCode = 1;
+ throw new Error(`FAIL ${msg}`);
+};
+
+// 1) Seed an empty cloud project.
+const created = await api("/api/cut-cloud/projects", {
+ method: "POST",
+ body: JSON.stringify({ name: `headless-eval-${Date.now()}` }),
+});
+if (!created.ok) fail(`could not create project (${created.status}) — is the dev server up?`);
+const projectId = ((await created.json()) as { id: string }).id;
+console.log(`seeded project ${projectId}`);
+
+try {
+ // 2) Open its doc into the editor store.
+ const doc = await openCloudProject(session, projectId);
+ console.log(`opened at version ${doc.version}`);
+
+ // 3) One real edit turn through the production loop.
+ const ask = 'Add a title that says "HEADLESS" at the very start of the video.';
+ const messages = [
+ { id: "m1", role: "user", parts: [{ type: "text", text: ask }] },
+ ] as unknown as UIMessage[];
+ const threadId = `headless-eval-${projectId}`;
+ const calls: string[] = [];
+ const deps = headlessDeps(session);
+ const exec = deps.execTool;
+ deps.execTool = async (name, args) => {
+ calls.push(name);
+ return exec(name, args);
+ };
+
+ let reply = "";
+ let streamError: string | null = null;
+ const reader = streamCutChat({
+ threadId,
+ model: geminiModelRoles.chat,
+ messages,
+ deps,
+ }).getReader();
+ try {
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ const chunk = value as unknown as { type: string; delta?: string; errorText?: string };
+ if (chunk.type === "text-delta" && chunk.delta) reply += chunk.delta;
+ if (chunk.type === "error" && chunk.errorText) streamError = chunk.errorText;
+ }
+ } finally {
+ dropPiSession(threadId);
+ }
+ if (streamError) fail(`stream error: ${streamError}`);
+ console.log(`turn ran ${calls.length} tool calls [${calls.join(", ")}]`);
+ console.log(`reply: ${reply.trim().slice(0, 200)}`);
+
+ // 4) Push the store back up and re-read the stored document.
+ const openedVersion = doc.version;
+ const pushed = await pushCloudProject(session, doc);
+ console.log(`pushed as version ${pushed.version}`);
+ if (pushed.version === openedVersion) fail("the versioned PUT did not advance the doc version");
+ const readBack = await api(`/api/cut-cloud/projects/${projectId}`);
+ if (!readBack.ok) fail(`could not re-read project (${readBack.status})`);
+ const stored = (await readBack.json()) as {
+ overlays?: { text?: unknown; start?: number }[];
+ };
+ const title = (stored.overlays ?? []).find(
+ (o) => typeof o.text === "string" && o.text.toUpperCase().includes("HEADLESS")
+ );
+ if (!title) fail(`no HEADLESS title in the stored doc — overlays: ${JSON.stringify(stored.overlays)}`);
+ console.log(`PASS stored doc carries the title at ${title!.start}s`);
+} finally {
+ if (!keep) await api(`/api/cut-cloud/projects/${projectId}`, { method: "DELETE" });
+}
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 63bafbc0..8671a921 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -2354,6 +2354,35 @@ class ToolError extends Error {}
* Execute an assistant tool call against the live editor store.
* Returns a small JSON-safe result; throws ToolError with a readable message.
*/
+/** Tools whose whole effect is the editor UI in an open tab — panel focus,
+ * scroll, playback, the preview snapshot. A headless session answers them
+ * with a typed no-op so the model keeps its footing. */
+export const UI_TOOLS: ReadonlySet<string> = new Set([
+ "capture_frame",
+ "set_side_panel",
+ "set_view",
+ "open_export",
+ "set_playing",
+]);
+
+/** Tools that decode or rasterize media with browser APIs today — frame
+ * grabs, audio scans, image decodes, the transcription mixdown. A headless
+ * session refuses them with a typed error until the runner grows ffmpeg
+ * equivalents. Membership is about page decoding; tools that only call
+ * servers run headless through the bound transports and stay out. */
+export const BROWSER_MEDIA_TOOLS: ReadonlySet<string> = new Set([
+ "watch_video",
+ "listen_audio",
+ "detect_silence",
+ "refine_speech_cuts",
+ "freeze_frame",
+ "create_sticker",
+ "subtitles_generate",
+ "captions_generate",
+ "subtitles_from_visuals",
+ "stock_add",
+]);
+
export async function runAiTool(
name: string,
input: Record<string, unknown>
diff --git a/site/src/cut/lib/backend/cloud.ts b/site/src/cut/lib/backend/cloud.ts
index 9e08dd3a..948459bd 100644
--- a/site/src/cut/lib/backend/cloud.ts
+++ b/site/src/cut/lib/backend/cloud.ts
@@ -8,6 +8,22 @@ import type { CutBackend } from "./types";
const cloudPath = (path: string) => path.replace(/^\/api\/cut\//, "/api/cut-cloud/");
+// A headless session's transport: an absolute origin and auth headers folded
+// into every cloud call. The page binds nothing — its calls are same-origin
+// and the session cookie rides on its own.
+let session: { base: string; headers: Record<string, string> } | null = null;
+export function bindCloudSession(base: string, headers: Record<string, string>) {
+ session = { base: base.replace(/\/$/, ""), headers };
+}
+
+function cloudRequest(path: string, init?: RequestInit): Promise<Response> {
+ if (!session) return fetch(path, init);
+ return fetch(session.base + path, {
+ ...init,
+ headers: { ...session.headers, ...(init?.headers as Record<string, string> | undefined) },
+ });
+}
+
// Cloud project docs are versioned for lost-write detection: GET hands the
// current version out in a header, PUT sends it back as ?v= and gets the
// incremented one in its body. The map lives in the transport so call sites
@@ -58,11 +74,11 @@ const PROJECT_DOC = /^\/api\/cut\/projects\/(?!folders$)([^/?]+)$/;
async function cloudFetch(path: string, init?: RequestInit): Promise<Response> {
const doc = PROJECT_DOC.exec(path);
const method = (init?.method ?? "GET").toUpperCase();
- if (!doc || (method !== "GET" && method !== "PUT")) return fetch(cloudPath(path), init);
+ if (!doc || (method !== "GET" && method !== "PUT")) return cloudRequest(cloudPath(path), init);
const projectId = decodeURIComponent(doc[1]);
if (method === "GET") {
- const req = fetch(cloudPath(path), init);
+ const req = cloudRequest(cloudPath(path), init);
const tracked = req.then(
(res) => {
if (res.ok) noteVersion(projectId, res.headers.get("x-cut-doc-version"));
@@ -88,7 +104,7 @@ async function putDoc(projectId: string, path: string, init?: RequestInit): Prom
// A pinned base outranks the map; with neither, an unversioned PUT is the
// first save and succeeds unconditionally.
const v = basePins.get(projectId) ?? docVersions.get(projectId);
- const res = await fetch(cloudPath(path) + (v ? `?v=${encodeURIComponent(v)}` : ""), init);
+ const res = await cloudRequest(cloudPath(path) + (v ? `?v=${encodeURIComponent(v)}` : ""), init);
if (res.ok || res.status === 409) basePins.delete(projectId);
if (res.status === 409) {
const body = (await res
@@ -96,11 +112,12 @@ async function putDoc(projectId: string, path: string, init?: RequestInit): Prom
.json()
.catch(() => null)) as { doc?: unknown; version?: number | string } | null;
noteVersion(projectId, body?.version);
- window.dispatchEvent(
- new CustomEvent("cut-cloud-doc-conflict", {
- detail: { projectId, doc: body?.doc, version: body?.version },
- })
- );
+ if (typeof window !== "undefined")
+ window.dispatchEvent(
+ new CustomEvent("cut-cloud-doc-conflict", {
+ detail: { projectId, doc: body?.doc, version: body?.version },
+ })
+ );
} else if (res.ok) {
const body = (await res
.clone()
diff --git a/site/src/cut/lib/genScene.ts b/site/src/cut/lib/genScene.ts
index c3afaaee..5c50341b 100644
--- a/site/src/cut/lib/genScene.ts
+++ b/site/src/cut/lib/genScene.ts
@@ -99,6 +99,10 @@ useEditor.subscribe((s, prev) => {
// mounts. loadProject flips `loaded` false→true in the same set() that
// fills genvideo, so the edge fires once per open.
if (!s.loaded || prev.loaded || !s.projectId) return;
+ // A headless process (the turn runner, an eval) opens projects to run one
+ // job; resuming a persisted plan there would spend credits on a render the
+ // page is already resuming for its user.
+ if (typeof window === "undefined") return;
// A shared view shows the owner's persisted run as data only — never
// adopt or resume it (resuming would render on the viewer's account).
if (s.readOnly) return;
diff --git a/site/src/cut/lib/generate.ts b/site/src/cut/lib/generate.ts
index 3b4800e3..ed296127 100644
--- a/site/src/cut/lib/generate.ts
+++ b/site/src/cut/lib/generate.ts
@@ -16,7 +16,7 @@ import {
upsertRenderInDoc,
} from "./genvideo/docWriter";
import { CUT_APP_BASE } from "./nav";
-import { hostedPost } from "./hosted";
+import { hostedFetch, hostedPost } from "./hosted";
import { enrichAsset, importFileToProject, uploadProjectImage } from "./media";
import { refsToInlineImages, videoSafeInline, visualRefs, type InlineImage } from "./refMedia";
import { useGenNotify } from "./genNotify";
@@ -711,7 +711,7 @@ export const useGenerate = create<GenerateState>((set, get) => {
},
probeNow: () => {
- probing ??= fetch("/api/auth/get-session", { cache: "no-store" })
+ probing ??= hostedFetch("/api/auth/get-session", { cache: "no-store" })
.then((r) => (r.ok ? r.json() : null))
.then((s) => {
const signedIn = Boolean((s as { user?: unknown } | null)?.user);
diff --git a/site/src/cut/lib/headless/bind.ts b/site/src/cut/lib/headless/bind.ts
new file mode 100644
index 00000000..ce3f5fdc
--- /dev/null
+++ b/site/src/cut/lib/headless/bind.ts
@@ -0,0 +1,23 @@
+import { bindCutMode } from "../backend";
+import { bindCloudSession } from "../backend/cloud";
+import { bindHostedSession } from "../hosted";
+
+// One call wires a headless process to the hosted site as one user: the
+// backend mode pins to cloud, and both transports — the cut-cloud driver and
+// the hosted inference chokepoint — fold the session's origin and auth
+// headers into every request. After this, the same lib code the page runs
+// (tools, generation, media import) reaches the right server on its own.
+
+export interface HeadlessSession {
+ /** Absolute origin of the hosted site, e.g. https://donkeycut.com. */
+ base: string;
+ /** Auth folded into every request: a session cookie, the runner grant, or
+ * the dev bypass. */
+ headers: Record<string, string>;
+}
+
+export function bindHeadlessSession(session: HeadlessSession): void {
+ bindCutMode("cloud");
+ bindCloudSession(session.base, session.headers);
+ bindHostedSession(session.base, session.headers);
+}
diff --git a/site/src/cut/lib/headless/docSession.ts b/site/src/cut/lib/headless/docSession.ts
new file mode 100644
index 00000000..f8d46086
--- /dev/null
+++ b/site/src/cut/lib/headless/docSession.ts
@@ -0,0 +1,80 @@
+import type { HeadlessSession } from "./bind";
+import { serializeDoc, useEditor } from "../store";
+import type { MediaAsset, ProjectDoc } from "../types";
+
+// A headless run's hold on one cloud project: read the document, hydrate the
+// editor store with it, and push the store's persistable slice back through
+// the same versioned PUT the page's autosave uses. The version is required at
+// both ends — the open fails without one, and the push always sends ?v= — so
+// a concurrent writer is answered with a 409 the caller must handle.
+
+export interface HeadlessDocSession {
+ projectId: string;
+ /** The doc version the store's contents are built on. */
+ version: string;
+}
+
+const projectPath = (s: HeadlessSession, projectId: string) =>
+ `${s.base}/api/cut-cloud/projects/${encodeURIComponent(projectId)}`;
+
+/** Fetch a cloud project's document and open it in the editor store. */
+export async function openCloudProject(
+ s: HeadlessSession,
+ projectId: string
+): Promise<HeadlessDocSession> {
+ const res = await fetch(projectPath(s, projectId), { headers: s.headers });
+ if (!res.ok) throw new Error(`Could not read project ${projectId} (${res.status}).`);
+ const version = res.headers.get("x-cut-doc-version");
+ if (!version)
+ throw new Error("The project read carried no doc version; refusing an unversioned session.");
+ const doc = (await res.json()) as ProjectDoc;
+ // Media rides signed R2 URLs: tool paths that read asset bytes (attachment
+ // refs, sticker decodes) fetch them bare, and the /media route would answer
+ // those unauthenticated fetches 401. The route URL stays the fallback for
+ // anything the mint misses.
+ const fileNames = (doc.assets ?? []).map((a) => a.fileName);
+ const signed = new Map<string, string>();
+ if (fileNames.length > 0) {
+ const mint = await fetch(`${s.base}/api/cut-cloud/media/presign-get`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json", ...s.headers },
+ body: JSON.stringify({ items: fileNames.map((fileName) => ({ projectId, fileName })) }),
+ }).catch(() => null);
+ if (mint?.ok) {
+ const body = (await mint.json()) as { urls?: { fileName: string; url: string }[] };
+ for (const u of body.urls ?? []) signed.set(u.fileName, u.url);
+ }
+ }
+ const assets: MediaAsset[] = (doc.assets ?? []).map((a) => ({
+ ...a,
+ url: signed.get(a.fileName) ?? `${projectPath(s, projectId)}/media/${encodeURIComponent(a.fileName)}`,
+ }));
+ await useEditor.getState().openProjectDoc(projectId, doc, assets);
+ return { projectId, version };
+}
+
+/** Push the store's persistable slice back up on the session's version. A 409
+ * means another writer moved the document past this session's base; the
+ * caller decides whether to reopen and redo or surface the conflict. */
+export async function pushCloudProject(
+ s: HeadlessSession,
+ session: HeadlessDocSession
+): Promise<HeadlessDocSession> {
+ const state = useEditor.getState();
+ if (state.projectId !== session.projectId)
+ throw new Error("The store holds a different project than this session opened.");
+ const res = await fetch(
+ `${projectPath(s, session.projectId)}?v=${encodeURIComponent(session.version)}`,
+ {
+ method: "PUT",
+ headers: { "Content-Type": "application/json", ...s.headers },
+ body: JSON.stringify(serializeDoc(state)),
+ }
+ );
+ if (res.status === 409)
+ throw new Error("The project changed under this session — another writer holds a newer version.");
+ if (!res.ok) throw new Error(`Could not save project ${session.projectId} (${res.status}).`);
+ const body = (await res.json()) as { version?: number };
+ if (body.version !== undefined) session.version = String(body.version);
+ return session;
+}
diff --git a/site/src/cut/lib/hosted.ts b/site/src/cut/lib/hosted.ts
index 1fdc47d0..1b94e6cb 100644
--- a/site/src/cut/lib/hosted.ts
+++ b/site/src/cut/lib/hosted.ts
@@ -51,7 +51,7 @@ let rechecking: Promise<void> | null = null;
* tab up after the user already paid. Any failure leaves the flag as is. */
export function recheckCredits(): Promise<void> {
if (!useOutOfCredits.getState().out) return Promise.resolve();
- rechecking ??= fetch("/api/credits/balance", { cache: "no-store" })
+ rechecking ??= hostedFetch("/api/credits/balance", { cache: "no-store" })
.then(async (res) => {
if (!res.ok) return;
const body = (await res.json()) as { balanceMicros?: string };
@@ -95,6 +95,23 @@ export function useCreditsRecheck(): void {
const OFFLOAD_ABOVE_BYTES = 1024 * 1024;
const MAX_BODY_BYTES = 4 * 1024 * 1024;
+/** A headless session's transport: an absolute origin and auth headers folded
+ * into every hosted call. In the page this stays unbound; same-origin calls
+ * carry the session cookie on their own. */
+let session: { base: string; headers: Record<string, string> } | null = null;
+export function bindHostedSession(base: string, headers: Record<string, string>) {
+ session = { base: base.replace(/\/$/, ""), headers };
+}
+
+/** Fetch a hosted route through the bound session when one exists. Every
+ * hosted-route call in lib code goes through here so a headless process
+ * reaches the right server with its auth. */
+export const hostedFetch = (path: string, init?: RequestInit): Promise<Response> =>
+ fetch((session?.base ?? "") + path, {
+ ...init,
+ headers: { ...session?.headers, ...(init?.headers as Record<string, string> | undefined) },
+ });
+
/** POST one of Donkey's hosted inference routes with the user's session. */
export const hostedPost = async (path: string, body: unknown, signal?: AbortSignal) => {
let payload = JSON.stringify(body);
@@ -104,7 +121,7 @@ export const hostedPost = async (path: string, body: unknown, signal?: AbortSign
if (new Blob([payload]).size > MAX_BODY_BYTES) {
throw new Error("That request carries too much attached media to send. Use fewer references.");
}
- const res = await fetch(path, {
+ const res = await hostedFetch(path, {
method: "POST",
headers: { "Content-Type": "application/json", "x-donkey-client-id": CLIENT_ID },
body: payload,
diff --git a/site/src/cut/lib/hostedBlobs.ts b/site/src/cut/lib/hostedBlobs.ts
index 4445a99b..3e569e31 100644
--- a/site/src/cut/lib/hostedBlobs.ts
+++ b/site/src/cut/lib/hostedBlobs.ts
@@ -20,6 +20,7 @@
// out as it would have: storage is how a large request succeeds, never the
// reason a small one fails.
import { bytesFromBase64 } from "./bytes";
+import { hostedFetch } from "./hosted";
/** Keys a hosted payload carries base64 media under, and the media type to
* assume when the part does not name one. */
@@ -100,7 +101,7 @@ async function claimKeys(blobs: Claimable[], clientId: string): Promise<Claim[]>
const claimed = await Promise.all(
batches.map(async (batch) => {
const unavailable = batch.map(() => ({ skipped: "unavailable" }));
- const res = await fetch("/api/inference/uploads", {
+ const res = await hostedFetch("/api/inference/uploads", {
method: "POST",
headers: { "Content-Type": "application/json", "x-donkey-client-id": clientId },
body: JSON.stringify({ blobs: batch }),
diff --git a/site/src/cut/lib/pi/debris.ts b/site/src/cut/lib/pi/debris.ts
new file mode 100644
index 00000000..dbfa99af
--- /dev/null
+++ b/site/src/cut/lib/pi/debris.ts
@@ -0,0 +1,28 @@
+import { parkedTransitions, totalDuration, track0Clips, useEditor } from "../store";
+
+const r1 = (x: number) => Math.round(x * 10) / 10;
+
+/** The project's current debris, one line each: transition bars playing
+ * nothing, and items stranded past the end of the video. Read from the live
+ * editor store, so it serves the page and a headless session alike. */
+export function currentDebris(): string[] {
+ const s = useEditor.getState();
+ const lines: string[] = [];
+ for (const t of parkedTransitions(s.clips, s.transitions)) {
+ lines.push(
+ `parked transition ${t.id} (${t.style}, ${r1(t.seconds)}s) at ${r1(t.start)}s — lines up with no cut, plays nothing`
+ );
+ }
+ const end = totalDuration(track0Clips(s.clips));
+ if (end > 0) {
+ for (const o of s.overlays) {
+ if (o.start >= end)
+ lines.push(`overlay ${o.id} ("${"text" in o && o.text ? String(o.text).slice(0, 30) : o.kind}") starts at ${r1(o.start)}s, past the video's end at ${r1(end)}s`);
+ }
+ for (const a of s.audioClips) {
+ if (a.start >= end)
+ lines.push(`soundtrack clip ${a.id} starts at ${r1(a.start)}s, past the video's end at ${r1(end)}s`);
+ }
+ }
+ return lines;
+}
diff --git a/site/src/cut/lib/pi/prodDeps.ts b/site/src/cut/lib/pi/prodDeps.ts
index 44957ba2..c895d5ec 100644
--- a/site/src/cut/lib/pi/prodDeps.ts
+++ b/site/src/cut/lib/pi/prodDeps.ts
@@ -8,34 +8,8 @@ import { normalizeRef } from "../assetRef";
import { NO_CREDITS_MESSAGE, useGenerate } from "../generate";
import { hostedPost } from "../hosted";
import { refsToParts } from "../refMedia";
-import { parkedTransitions, totalDuration, track0Clips, useEditor } from "../store";
import type { CutAgentDeps } from "./cutAgent";
-
-const r1 = (x: number) => Math.round(x * 10) / 10;
-
-/** The project's current debris, one line each: transition bars playing
- * nothing, and items stranded past the end of the video. */
-function currentDebris(): string[] {
- const s = useEditor.getState();
- const lines: string[] = [];
- for (const t of parkedTransitions(s.clips, s.transitions)) {
- lines.push(
- `parked transition ${t.id} (${t.style}, ${r1(t.seconds)}s) at ${r1(t.start)}s — lines up with no cut, plays nothing`
- );
- }
- const end = totalDuration(track0Clips(s.clips));
- if (end > 0) {
- for (const o of s.overlays) {
- if (o.start >= end)
- lines.push(`overlay ${o.id} ("${"text" in o && o.text ? String(o.text).slice(0, 30) : o.kind}") starts at ${r1(o.start)}s, past the video's end at ${r1(end)}s`);
- }
- for (const a of s.audioClips) {
- if (a.start >= end)
- lines.push(`soundtrack clip ${a.id} starts at ${r1(a.start)}s, past the video's end at ${r1(end)}s`);
- }
- }
- return lines;
-}
+import { currentDebris } from "./debris";
// The live editor's wiring for the pi chat loop. This module carries the
// browser-only graph (the editor store, hosted auth, media resolution), so
diff --git a/site/src/cut/lib/pi/serverDeps.ts b/site/src/cut/lib/pi/serverDeps.ts
new file mode 100644
index 00000000..9966ebff
--- /dev/null
+++ b/site/src/cut/lib/pi/serverDeps.ts
@@ -0,0 +1,54 @@
+import { geminiModelRoles } from "@/lib/inference/gemini-models";
+import { AI_SKILL_INDEX, AI_SKILLS } from "@/cut/server/ai/catalog";
+import { buildAiContext } from "../aiContext";
+import { BROWSER_MEDIA_TOOLS, runAiTool, UI_TOOLS } from "../aiTools";
+import { normalizeRef } from "../assetRef";
+import { NO_CREDITS_MESSAGE } from "../generate";
+import { hostedPost } from "../hosted";
+import { bindHeadlessSession, type HeadlessSession } from "../headless/bind";
+import { refsToParts } from "../refMedia";
+import type { CutAgentDeps } from "./cutAgent";
+import { currentDebris } from "./debris";
+
+// The headless wiring for the pi chat loop. The runner opens a project into
+// the store (openProjectDoc), runs the turn with these deps, and pushes the
+// doc back up. Every transport rides the session's origin and auth headers.
+
+export type { HeadlessSession };
+
+export function headlessDeps(session: HeadlessSession): CutAgentDeps {
+ bindHeadlessSession(session);
+ return {
+ post: (payload, signal) => hostedPost("/api/inference/responses", payload, signal),
+ execTool: async (name, args) => {
+ if (name === "list_skills") return { skills: AI_SKILL_INDEX };
+ if (name === "read_skill") {
+ const doc = AI_SKILLS[String(args.name ?? "")];
+ if (!doc) throw new Error(`No such skill. Available: ${AI_SKILL_INDEX.join(", ")}`);
+ return doc;
+ }
+ if (UI_TOOLS.has(name))
+ return {
+ noEditor: true,
+ note: "No editor page is attached to this session, so this tool had no effect. The project itself is unchanged — keep working from the editor state.",
+ };
+ if (BROWSER_MEDIA_TOOLS.has(name))
+ throw new Error(
+ "This tool reads media through the editor page and is unavailable in this session. Work from the editor state and the transcript instead."
+ );
+ return runAiTool(name, args);
+ },
+ models: {
+ simple: geminiModelRoles.chatSimple,
+ complex: geminiModelRoles.chat,
+ gate: geminiModelRoles.fastDecision,
+ },
+ buildContext: () => buildAiContext(),
+ resolveRefs: async (meta) => {
+ const refs = meta.map(normalizeRef).filter((r) => r !== null);
+ return (await refsToParts(refs)).parts;
+ },
+ debris: currentDebris,
+ noCreditsMessage: NO_CREDITS_MESSAGE,
+ };
+}
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index fd680b68..d4a7ed5c 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -260,6 +260,14 @@ export interface EditorState {
* the stored copy has moved on, so the head-start snapshot is skipped and
* only the live document is applied. */
loadProject: (id: string, opts?: { inPlace?: boolean }) => Promise<void>;
+ /** Apply one project document to the store: the legacy migrations and the
+ * state swap, shared by every hydration path — the snapshot paint, the live
+ * load, a conflict reload, and a headless open. */
+ applyDocState: (doc: Partial<ProjectDoc>, assets: MediaAsset[], ui: ProjectUiState) => void;
+ /** Open a project from an already-fetched document. The headless runner's
+ * open: resets the store, drains pending doc writes, and hydrates straight
+ * from the given document. */
+ openProjectDoc: (id: string, doc: Partial<ProjectDoc>, assets: MediaAsset[]) => Promise<void>;
setProjectName: (name: string) => void;
setSaveState: (s: SaveState) => void;
/** Enter read-only shared mode; call before loadProject. */
@@ -1209,6 +1217,57 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
}));
};
+ /** Reset to an empty, loading project and drain any background doc writes
+ * still queued for it, so no open ever reads a half-written document. */
+ const resetForOpen = async (id: string) => {
+ history.length = 0;
+ future.length = 0;
+ pending = null;
+ // A fresh project owns no live run — any prior run's render-owned ids are
+ // stale, and the loaded clips are ordinary, fully-undoable content.
+ genClipIds.clear();
+ genAudioIds.clear();
+ hydrating = true;
+ set({
+ projectId: id,
+ loaded: false,
+ loadError: null,
+ saveState: "saved",
+ resumePush: false,
+ assets: [],
+ clips: [],
+ transitions: [],
+ audioClips: [],
+ overlays: [],
+ templates: [],
+ aspect: "9:16",
+ aspectTouched: false,
+ fadeIn: 0,
+ fadeOut: 0,
+ selection: null,
+ multiSelection: [],
+ playing: false,
+ previewStopAt: null,
+ subtitles: emptySubtitles(),
+ subtitleLane: 0,
+ subtitleStatus: "idle",
+ subtitleError: null,
+ exportOpen: false,
+ genvideo: undefined,
+ renders: [],
+ firstOpen: undefined,
+ });
+ hydrating = false;
+ // A background scene run may still be writing this project's doc — drain
+ // its queued writes so the open never reads a half-written doc. Ordering
+ // matters: projectId is set (loaded false) BEFORE this await, so a write
+ // arriving during the drain waits for the load (projectWriteMode) instead
+ // of queueing a doc write the drain would miss — nothing can land between
+ // the drain and the fetch that follows. Lazy import: docWriter reads store
+ // helpers, so a static import would be a cycle.
+ await import("./genvideo/docWriter").then((m) => m.docWriterIdle(id)).catch(() => {});
+ };
+
return {
projectId: null,
projectName: "",
@@ -1258,156 +1317,13 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
// clicking back in — is not a re-read, and gets the snapshot's head
// start like any other open.
const inPlace = opts?.inPlace === true;
- history.length = 0;
- future.length = 0;
- pending = null;
- // A fresh project owns no live run — any prior run's render-owned ids are
- // stale, and the loaded clips are ordinary, fully-undoable content.
- genClipIds.clear();
- genAudioIds.clear();
- hydrating = true;
- set({
- projectId: id,
- loaded: false,
- loadError: null,
- saveState: "saved",
- resumePush: false,
- assets: [],
- clips: [],
- transitions: [],
- audioClips: [],
- overlays: [],
- templates: [],
- aspect: "9:16",
- aspectTouched: false,
- fadeIn: 0,
- fadeOut: 0,
- selection: null,
- multiSelection: [],
- playing: false,
- previewStopAt: null,
- subtitles: emptySubtitles(),
- subtitleLane: 0,
- subtitleStatus: "idle",
- subtitleError: null,
- exportOpen: false,
- genvideo: undefined,
- renders: [],
- firstOpen: undefined,
- });
- hydrating = false;
- // A background scene run may still be writing this project's doc — drain
- // its queued writes so the load never reads a half-written doc. Ordering
- // matters: projectId is set (loaded false) BEFORE this await, so a write
- // arriving during the drain waits for the load (projectWriteMode) instead
- // of queueing a doc write the drain would miss — nothing can land between
- // the drain and the fetch below. Lazy import: docWriter reads store
- // helpers, so a static import would be a cycle.
- await import("./genvideo/docWriter").then((m) => m.docWriterIdle(id)).catch(() => {});
+ await resetForOpen(id);
// One shape of hydration, used by both the snapshot painted below and
// the live document that replaces it, so the legacy migrations happen
// once per document rather than once per code path.
- const hydrate = (doc: Partial<ProjectDoc>, assets: MediaAsset[], ui: ProjectUiState) => {
- const docClips = doc.clips ?? [];
- // Older docs stored video track 0 packed (array order implied the
- // position); bake explicit starts in once so every clip is free-placed.
- const legacy = (docClips as LegacyClip[]).some((c) => typeof c.start !== "number");
- const folded = (legacy ? packStarts(docClips as LegacyClip[]) : docClips).map((c) => ({
- ...c,
- track: c.track ?? 0,
- }));
- // Older docs kept tracks other than 0 in a separate `overlayClips` array;
- // fold them into the one clip list (each already carries its `track`).
- // Entries whose id already sits in `clips` are the same clip persisted
- // twice by a version-skewed save (an older engine keeps overlayClips
- // after a merged client writes the folded list) — keep the folded copy.
- // Entries with track 0 were unreachable dead data under the split shape
- // (never rendered, never played); promoting them would insert them into
- // track 0's sequence, so they stay dropped.
- const seen = new Set(folded.map((c) => c.id));
- const legacyLayers = (doc.overlayClips ?? []).filter(
- (c) => c.track !== 0 && !seen.has(c.id)
- );
- // Tracks number 0..N bottom-up. Docs saved when tracks could go
- // negative (backdrop rows below the spine) lift wholesale so the
- // lowest row becomes track 0 — the bottom row is the spine now.
- const joined = [...folded, ...legacyLayers];
- const lift = Math.max(0, ...joined.map((c) => -c.track));
- const lifted = lift ? joined.map((c) => ({ ...c, track: c.track + lift })) : joined;
- // Stamp `kind: "text"` on pre-union titles so every in-memory element
- // carries its discriminant; the serializer strips it back. Effects
- // saved onto a shared row move to one of their own, and a clip graded
- // back when a look was a clip property gets that grade as an element
- // over it — so a project made before either rule reads like a new one.
- // The behind-speaker boolean becomes an inverted subject mask on load,
- // so one mask model covers it everywhere in memory and on save.
- const stamped = normalizeElementLanes(
- migrateBehindSubject(stampOverlayKinds(doc.overlays ?? []))
- );
- const subtitles = doc.subtitles ?? emptySubtitles();
- // Docs saved when edge transition styles existed convert them into the
- // equivalent clip animations, and docs saved when a transition was a
- // physical overlap pull their intruding clips apart — clips never
- // overlap, whatever wrote the file. Pulling them apart lengthens the
- // cut, so the whole document goes through it together.
- const merged = separateOverlaps({
- clips: migrateLegacyTransitions(lifted),
- audioClips: doc.audioClips ?? [],
- overlays: stamped,
- cues: subtitles.cues,
- });
- const withLooks =
- liftClipLooks(merged.clips, merged.overlays, getClipSpans(merged.clips, assets)) ?? {
- clips: merged.clips,
- overlays: merged.overlays,
- };
- hydrating = true;
- set({
- projectName: doc.name ?? "",
- assets,
- clips: withLooks.clips,
- // Bars from the doc, plus one adopted for each transition/animation
- // a pre-bar doc stored as a clip field.
- transitions: adoptTransitionFields(
- withLooks.clips,
- sanitizeTransitions(doc.transitions)
- ),
- audioClips: merged.audioClips,
- overlays: withLooks.overlays,
- templates: doc.templates ?? [],
- aspect: normalizeAspect(doc.aspect) ?? "9:16",
- aspectTouched: doc.aspect !== undefined,
- fadeIn: doc.fadeIn ?? 0,
- fadeOut: doc.fadeOut ?? 0,
- // View state lives in IndexedDB; doc.ui covers projects saved
- // before the move.
- pxPerSec: Math.max(12, Math.min(800, ui.pxPerSec ?? doc.ui?.pxPerSec ?? 60)),
- timelineH: Math.max(
- TIMELINE_H_MIN,
- Math.min(timelineHMax(), ui.timelineH ?? TIMELINE_H_DEFAULT)
- ),
- publish: {
- caption: doc.publish?.caption ?? "",
- tags: doc.publish?.tags ?? "",
- soundTitle: doc.publish?.soundTitle ?? "",
- handle: doc.publish?.handle ?? "",
- },
- notes: {
- text: doc.notes?.text ?? "",
- publishedAt: doc.notes?.publishedAt ?? "",
- links: doc.notes?.links ?? [],
- },
- subtitles: { ...subtitles, cues: merged.cues },
- subtitleStatus: merged.cues.length > 0 ? "ready" : "idle",
- genvideo: doc.genvideo ?? undefined,
- renders: Array.isArray(doc.renders) ? doc.renders : [],
- firstOpen: doc.firstOpen,
- loaded: true,
- loadEpoch: get().loadEpoch + 1,
- });
- hydrating = false;
- };
+ const hydrate = (doc: Partial<ProjectDoc>, assets: MediaAsset[], ui: ProjectUiState) =>
+ get().applyDocState(doc, assets, ui);
// The live document goes out first, so everything below is a head start
// on it and never a substitute for it.
@@ -1543,6 +1459,117 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {
}
},
+ applyDocState: (doc, assets, ui) => {
+ const docClips = doc.clips ?? [];
+ // Older docs stored video track 0 packed (array order implied the
+ // position); bake explicit starts in once so every clip is free-placed.
+ const legacy = (docClips as LegacyClip[]).some((c) => typeof c.start !== "number");
+ const folded = (legacy ? packStarts(docClips as LegacyClip[]) : docClips).map((c) => ({
+ ...c,
+ track: c.track ?? 0,
+ }));
+ // Older docs kept tracks other than 0 in a separate `overlayClips` array;
+ // fold them into the one clip list (each already carries its `track`).
+ // Entries whose id already sits in `clips` are the same clip persisted
+ // twice by a version-skewed save (an older engine keeps overlayClips
+ // after a merged client writes the folded list) — keep the folded copy.
+ // Entries with track 0 were unreachable dead data under the split shape
+ // (never rendered, never played); promoting them would insert them into
+ // track 0's sequence, so they stay dropped.
+ const seen = new Set(folded.map((c) => c.id));
+ const legacyLayers = (doc.overlayClips ?? []).filter(
+ (c) => c.track !== 0 && !seen.has(c.id)
+ );
+ // Tracks number 0..N bottom-up. Docs saved when tracks could go
+ // negative (backdrop rows below the spine) lift wholesale so the
+ // lowest row becomes track 0 — the bottom row is the spine now.
+ const joined = [...folded, ...legacyLayers];
+ const lift = Math.max(0, ...joined.map((c) => -c.track));
+ const lifted = lift ? joined.map((c) => ({ ...c, track: c.track + lift })) : joined;
+ // Stamp `kind: "text"` on pre-union titles so every in-memory element
+ // carries its discriminant; the serializer strips it back. Effects
+ // saved onto a shared row move to one of their own, and a clip graded
+ // back when a look was a clip property gets that grade as an element
+ // over it — so a project made before either rule reads like a new one.
+ // The behind-speaker boolean becomes an inverted subject mask on load,
+ // so one mask model covers it everywhere in memory and on save.
+ const stamped = normalizeElementLanes(
+ migrateBehindSubject(stampOverlayKinds(doc.overlays ?? []))
+ );
+ const subtitles = doc.subtitles ?? emptySubtitles();
+ // Docs saved when edge transition styles existed convert them into the
+ // equivalent clip animations, and docs saved when a transition was a
+ // physical overlap pull their intruding clips apart — clips never
+ // overlap, whatever wrote the file. Pulling them apart lengthens the
+ // cut, so the whole document goes through it together.
+ const merged = separateOverlaps({
+ clips: migrateLegacyTransitions(lifted),
+ audioClips: doc.audioClips ?? [],
+ overlays: stamped,
+ cues: subtitles.cues,
+ });
+ const withLooks =
+ liftClipLooks(merged.clips, merged.overlays, getClipSpans(merged.clips, assets)) ?? {
+ clips: merged.clips,
+ overlays: merged.overlays,
+ };
+ hydrating = true;
+ try {
+ set({
+ projectName: doc.name ?? "",
+ assets,
+ clips: withLooks.clips,
+ // Bars from the doc, plus one adopted for each transition/animation
+ // a pre-bar doc stored as a clip field.
+ transitions: adoptTransitionFields(
+ withLooks.clips,
+ sanitizeTransitions(doc.transitions)
+ ),
+ audioClips: merged.audioClips,
+ overlays: withLooks.overlays,
+ templates: doc.templates ?? [],
+ aspect: normalizeAspect(doc.aspect) ?? "9:16",
+ aspectTouched: doc.aspect !== undefined,
+ fadeIn: doc.fadeIn ?? 0,
+ fadeOut: doc.fadeOut ?? 0,
+ // View state lives in IndexedDB; doc.ui covers projects saved
+ // before the move.
+ pxPerSec: Math.max(12, Math.min(800, ui.pxPerSec ?? doc.ui?.pxPerSec ?? 60)),
+ timelineH: Math.max(
+ TIMELINE_H_MIN,
+ Math.min(timelineHMax(), ui.timelineH ?? TIMELINE_H_DEFAULT)
+ ),
+ publish: {
+ caption: doc.publish?.caption ?? "",
+ tags: doc.publish?.tags ?? "",
+ soundTitle: doc.publish?.soundTitle ?? "",
+ handle: doc.publish?.handle ?? "",
+ },
+ notes: {
+ text: doc.notes?.text ?? "",
+ publishedAt: doc.notes?.publishedAt ?? "",
+ links: doc.notes?.links ?? [],
+ },
+ subtitles: { ...subtitles, cues: merged.cues },
+ subtitleStatus: merged.cues.length > 0 ? "ready" : "idle",
+ genvideo: doc.genvideo ?? undefined,
+ renders: Array.isArray(doc.renders) ? doc.renders : [],
+ firstOpen: doc.firstOpen,
+ loaded: true,
+ loadEpoch: get().loadEpoch + 1,
+ });
+ } finally {
+ // Reset on every path: a doc bad enough to throw inside set() must
+ // never leave the store treating ordinary writes as hydration.
+ hydrating = false;
+ }
+ },
+
+ openProjectDoc: async (id, doc, assets) => {
+ await resetForOpen(id);
+ get().applyDocState(doc, assets, {});
+ },
+
setProjectName: (name) => set({ projectName: name }),
setSaveState: (s) => set({ saveState: s }),
setSharedView: (features) => set({ readOnly: true, sharedFeatures: features }),