Open source CapCut alternative.
TypeScript
feat(site): headless runner seam for the cut editor [rebuild]
David Lea6bce19cb209
diff --git a/site/package-lock.json b/site/package-lock.jsonindex f26773e5..797546da 100644--- a/site/package-lock.json+++ b/site/package-lock.json@@ -58,6 +58,7 @@"recharts": "^3.8.0","resend": "^6.18.1","shadcn": "^4.7.0",+ "skia-canvas": "^3.0.8","stripe": "^22.2.0","svix": "^1.99.1","tailwind-merge": "^3.6.0",@@ -7915,7 +7916,6 @@"version": "2.1.2","resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz","integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==",- "devOptional": true,"license": "Apache-2.0","engines": {"node": ">=8"@@ -9498,6 +9498,26 @@"dev": true,"license": "ISC"},+ "node_modules/follow-redirects": {+ "version": "1.16.0",+ "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",+ "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",+ "funding": [+ {+ "type": "individual",+ "url": "https://github.com/sponsors/RubenVerborgh"+ }+ ],+ "license": "MIT",+ "engines": {+ "node": ">=4.0"+ },+ "peerDependenciesMeta": {+ "debug": {+ "optional": true+ }+ }+ },"node_modules/for-each": {"version": "0.3.5","resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz",@@ -13491,6 +13511,12 @@"node": ">=6"}},+ "node_modules/parenthesis": {+ "version": "3.1.8",+ "resolved": "https://registry.npmjs.org/parenthesis/-/parenthesis-3.1.8.tgz",+ "integrity": "sha512-KF/U8tk54BgQewkJPvB4s/US3VQY68BRDpH638+7O/n58TpnwiwnOtGIOsT2/i+M78s61BBpeC83STB88d8sqw==",+ "license": "MIT"+ },"node_modules/parse-entities": {"version": "4.0.2","resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz",@@ -15425,6 +15451,19 @@"integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==","license": "MIT"},+ "node_modules/skia-canvas": {+ "version": "3.0.8",+ "resolved": "https://registry.npmjs.org/skia-canvas/-/skia-canvas-3.0.8.tgz",+ "integrity": "sha512-FSYKxp8Ng2vOeeOBiyPhnn6ui6FirPJXMyjk4PKl8N/OWzVrkMawUgY9zubIWHMdYtyWFn0gfX3QlRwg6HBmdg==",+ "hasInstallScript": true,+ "license": "MIT",+ "dependencies": {+ "detect-libc": "^2.1.1",+ "follow-redirects": "^1.15.11",+ "https-proxy-agent": "^7.0.6",+ "string-split-by": "^1.0.0"+ }+ },"node_modules/socket.io": {"version": "4.8.3","resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.3.tgz",@@ -15621,6 +15660,15 @@"integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==","license": "MIT"},+ "node_modules/string-split-by": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/string-split-by/-/string-split-by-1.0.0.tgz",+ "integrity": "sha512-KaJKY+hfpzNyet/emP81PJA9hTVSfxNLS9SFTWxdCnnW1/zOOwiV248+EfoX7IQFcBaOp4G5YE6xTJMF+pLg6A==",+ "license": "MIT",+ "dependencies": {+ "parenthesis": "^3.1.5"+ }+ },"node_modules/string-width": {"version": "7.2.0","resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz",diff --git a/site/package.json b/site/package.jsonindex a8b6bde1..a3119c38 100644--- a/site/package.json+++ b/site/package.json@@ -19,9 +19,10 @@"prisma:prod": "dotenv -e .env -- prisma db push","engine:dev": "bun run src/cut/engine/main.ts","engine:build": "bash scripts/build-cut-engine.sh",- "worker:build": "bun build --target=node --minify --external=pg-native src/cut/worker/main.ts --outfile dist/cut-worker/main.js",+ "worker:build": "bun build --target=node --minify --external=pg-native --external=skia-canvas --loader .css:text src/cut/worker/main.ts --outfile dist/cut-worker/main.js","bench:watch": "bun run scripts/bench-watch.ts","eval:cut-chat": "bun run scripts/eval-cut-chat.ts",+ "eval:cut-headless": "bun run scripts/eval-cut-headless.ts","email:dev": "email dev --dir src/emails --port 3001","test": "bun test src packages/effects-kit/src","test:genvideo": "bun run scripts/genvideo-selftest.ts",@@ -74,6 +75,7 @@"recharts": "^3.8.0","resend": "^6.18.1","shadcn": "^4.7.0",+ "skia-canvas": "^3.0.8","stripe": "^22.2.0","svix": "^1.99.1","tailwind-merge": "^3.6.0",diff --git a/site/scripts/eval-cut-headless.ts b/site/scripts/eval-cut-headless.tsnew file mode 100644index 00000000..05a003e1--- /dev/null+++ b/site/scripts/eval-cut-headless.ts@@ -0,0 +1,118 @@+#!/usr/bin/env bun+/**+ * Headless turn smoke: one real chat turn through the production loop with no+ * page attached. Seeds a cloud project on the dev server, opens its document+ * into the editor store, runs streamCutChat on the headless deps, pushes the+ * store back through the versioned PUT, and asserts the edit landed in the+ * stored document.+ *+ * Run with the site dev server up:+ * bun run scripts/eval-cut-headless.ts [--base http://localhost:3000] [--keep]+ *+ * Auth is the dev bypass header (a scripts-only grant), so runs are+ * dev-server-only and spend no credits beyond the model call.+ */++import type { UIMessage } from "ai";+import { openCloudProject, pushCloudProject } from "../src/cut/lib/headless/docSession";+import { dropPiSession, streamCutChat } from "../src/cut/lib/pi/cutAgent";+import { headlessDeps, type HeadlessSession } from "../src/cut/lib/pi/serverDeps";+import { geminiModelRoles } from "../src/lib/inference/gemini-models";++const arg = (name: string): string | undefined => {+ const i = process.argv.indexOf(`--${name}`);+ return i >= 0 ? process.argv[i + 1] : undefined;+};+const base = arg("base") ?? "http://localhost:3000";+const keep = process.argv.includes("--keep");++const session: HeadlessSession = {+ base,+ headers: {+ "x-donkey-client-id": "donkey-cut-eval",+ "x-donkey-dev-auth-bypass": "1",+ },+};++const api = (path: string, init?: RequestInit) =>+ fetch(base + path, {+ ...init,+ headers: { "Content-Type": "application/json", ...session.headers, ...init?.headers },+ });++const fail = (msg: string): never => {+ // Throw so the seeded project's finally-block DELETE still runs; the+ // exit code carries the failure.+ process.exitCode = 1;+ throw new Error(`FAIL ${msg}`);+};++// 1) Seed an empty cloud project.+const created = await api("/api/cut-cloud/projects", {+ method: "POST",+ body: JSON.stringify({ name: `headless-eval-${Date.now()}` }),+});+if (!created.ok) fail(`could not create project (${created.status}) — is the dev server up?`);+const projectId = ((await created.json()) as { id: string }).id;+console.log(`seeded project ${projectId}`);++try {+ // 2) Open its doc into the editor store.+ const doc = await openCloudProject(session, projectId);+ console.log(`opened at version ${doc.version}`);++ // 3) One real edit turn through the production loop.+ const ask = 'Add a title that says "HEADLESS" at the very start of the video.';+ const messages = [+ { id: "m1", role: "user", parts: [{ type: "text", text: ask }] },+ ] as unknown as UIMessage[];+ const threadId = `headless-eval-${projectId}`;+ const calls: string[] = [];+ const deps = headlessDeps(session);+ const exec = deps.execTool;+ deps.execTool = async (name, args) => {+ calls.push(name);+ return exec(name, args);+ };++ let reply = "";+ let streamError: string | null = null;+ const reader = streamCutChat({+ threadId,+ model: geminiModelRoles.chat,+ messages,+ deps,+ }).getReader();+ try {+ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ const chunk = value as unknown as { type: string; delta?: string; errorText?: string };+ if (chunk.type === "text-delta" && chunk.delta) reply += chunk.delta;+ if (chunk.type === "error" && chunk.errorText) streamError = chunk.errorText;+ }+ } finally {+ dropPiSession(threadId);+ }+ if (streamError) fail(`stream error: ${streamError}`);+ console.log(`turn ran ${calls.length} tool calls [${calls.join(", ")}]`);+ console.log(`reply: ${reply.trim().slice(0, 200)}`);++ // 4) Push the store back up and re-read the stored document.+ const openedVersion = doc.version;+ const pushed = await pushCloudProject(session, doc);+ console.log(`pushed as version ${pushed.version}`);+ if (pushed.version === openedVersion) fail("the versioned PUT did not advance the doc version");+ const readBack = await api(`/api/cut-cloud/projects/${projectId}`);+ if (!readBack.ok) fail(`could not re-read project (${readBack.status})`);+ const stored = (await readBack.json()) as {+ overlays?: { text?: unknown; start?: number }[];+ };+ const title = (stored.overlays ?? []).find(+ (o) => typeof o.text === "string" && o.text.toUpperCase().includes("HEADLESS")+ );+ if (!title) fail(`no HEADLESS title in the stored doc — overlays: ${JSON.stringify(stored.overlays)}`);+ console.log(`PASS stored doc carries the title at ${title!.start}s`);+} finally {+ if (!keep) await api(`/api/cut-cloud/projects/${projectId}`, { method: "DELETE" });+}diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 63bafbc0..8671a921 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -2354,6 +2354,35 @@ class ToolError extends Error {}* Execute an assistant tool call against the live editor store.* Returns a small JSON-safe result; throws ToolError with a readable message.*/+/** Tools whose whole effect is the editor UI in an open tab — panel focus,+ * scroll, playback, the preview snapshot. A headless session answers them+ * with a typed no-op so the model keeps its footing. */+export const UI_TOOLS: ReadonlySet<string> = new Set([+ "capture_frame",+ "set_side_panel",+ "set_view",+ "open_export",+ "set_playing",+]);++/** Tools that decode or rasterize media with browser APIs today — frame+ * grabs, audio scans, image decodes, the transcription mixdown. A headless+ * session refuses them with a typed error until the runner grows ffmpeg+ * equivalents. Membership is about page decoding; tools that only call+ * servers run headless through the bound transports and stay out. */+export const BROWSER_MEDIA_TOOLS: ReadonlySet<string> = new Set([+ "watch_video",+ "listen_audio",+ "detect_silence",+ "refine_speech_cuts",+ "freeze_frame",+ "create_sticker",+ "subtitles_generate",+ "captions_generate",+ "subtitles_from_visuals",+ "stock_add",+]);+export async function runAiTool(name: string,input: Record<string, unknown>diff --git a/site/src/cut/lib/backend/cloud.ts b/site/src/cut/lib/backend/cloud.tsindex 9e08dd3a..948459bd 100644--- a/site/src/cut/lib/backend/cloud.ts+++ b/site/src/cut/lib/backend/cloud.ts@@ -8,6 +8,22 @@ import type { CutBackend } from "./types";const cloudPath = (path: string) => path.replace(/^\/api\/cut\//, "/api/cut-cloud/");+// A headless session's transport: an absolute origin and auth headers folded+// into every cloud call. The page binds nothing — its calls are same-origin+// and the session cookie rides on its own.+let session: { base: string; headers: Record<string, string> } | null = null;+export function bindCloudSession(base: string, headers: Record<string, string>) {+ session = { base: base.replace(/\/$/, ""), headers };+}++function cloudRequest(path: string, init?: RequestInit): Promise<Response> {+ if (!session) return fetch(path, init);+ return fetch(session.base + path, {+ ...init,+ headers: { ...session.headers, ...(init?.headers as Record<string, string> | undefined) },+ });+}+// Cloud project docs are versioned for lost-write detection: GET hands the// current version out in a header, PUT sends it back as ?v= and gets the// incremented one in its body. The map lives in the transport so call sites@@ -58,11 +74,11 @@ const PROJECT_DOC = /^\/api\/cut\/projects\/(?!folders$)([^/?]+)$/;async function cloudFetch(path: string, init?: RequestInit): Promise<Response> {const doc = PROJECT_DOC.exec(path);const method = (init?.method ?? "GET").toUpperCase();- if (!doc || (method !== "GET" && method !== "PUT")) return fetch(cloudPath(path), init);+ if (!doc || (method !== "GET" && method !== "PUT")) return cloudRequest(cloudPath(path), init);const projectId = decodeURIComponent(doc[1]);if (method === "GET") {- const req = fetch(cloudPath(path), init);+ const req = cloudRequest(cloudPath(path), init);const tracked = req.then((res) => {if (res.ok) noteVersion(projectId, res.headers.get("x-cut-doc-version"));@@ -88,7 +104,7 @@ async function putDoc(projectId: string, path: string, init?: RequestInit): Prom// A pinned base outranks the map; with neither, an unversioned PUT is the// first save and succeeds unconditionally.const v = basePins.get(projectId) ?? docVersions.get(projectId);- const res = await fetch(cloudPath(path) + (v ? `?v=${encodeURIComponent(v)}` : ""), init);+ const res = await cloudRequest(cloudPath(path) + (v ? `?v=${encodeURIComponent(v)}` : ""), init);if (res.ok || res.status === 409) basePins.delete(projectId);if (res.status === 409) {const body = (await res@@ -96,11 +112,12 @@ async function putDoc(projectId: string, path: string, init?: RequestInit): Prom.json().catch(() => null)) as { doc?: unknown; version?: number | string } | null;noteVersion(projectId, body?.version);- window.dispatchEvent(- new CustomEvent("cut-cloud-doc-conflict", {- detail: { projectId, doc: body?.doc, version: body?.version },- })- );+ if (typeof window !== "undefined")+ window.dispatchEvent(+ new CustomEvent("cut-cloud-doc-conflict", {+ detail: { projectId, doc: body?.doc, version: body?.version },+ })+ );} else if (res.ok) {const body = (await res.clone()diff --git a/site/src/cut/lib/genScene.ts b/site/src/cut/lib/genScene.tsindex c3afaaee..5c50341b 100644--- a/site/src/cut/lib/genScene.ts+++ b/site/src/cut/lib/genScene.ts@@ -99,6 +99,10 @@ useEditor.subscribe((s, prev) => {// mounts. loadProject flips `loaded` false→true in the same set() that// fills genvideo, so the edge fires once per open.if (!s.loaded || prev.loaded || !s.projectId) return;+ // A headless process (the turn runner, an eval) opens projects to run one+ // job; resuming a persisted plan there would spend credits on a render the+ // page is already resuming for its user.+ if (typeof window === "undefined") return;// A shared view shows the owner's persisted run as data only — never// adopt or resume it (resuming would render on the viewer's account).if (s.readOnly) return;diff --git a/site/src/cut/lib/generate.ts b/site/src/cut/lib/generate.tsindex 3b4800e3..ed296127 100644--- a/site/src/cut/lib/generate.ts+++ b/site/src/cut/lib/generate.ts@@ -16,7 +16,7 @@ import {upsertRenderInDoc,} from "./genvideo/docWriter";import { CUT_APP_BASE } from "./nav";-import { hostedPost } from "./hosted";+import { hostedFetch, hostedPost } from "./hosted";import { enrichAsset, importFileToProject, uploadProjectImage } from "./media";import { refsToInlineImages, videoSafeInline, visualRefs, type InlineImage } from "./refMedia";import { useGenNotify } from "./genNotify";@@ -711,7 +711,7 @@ export const useGenerate = create<GenerateState>((set, get) => {},probeNow: () => {- probing ??= fetch("/api/auth/get-session", { cache: "no-store" })+ probing ??= hostedFetch("/api/auth/get-session", { cache: "no-store" }).then((r) => (r.ok ? r.json() : null)).then((s) => {const signedIn = Boolean((s as { user?: unknown } | null)?.user);diff --git a/site/src/cut/lib/headless/bind.ts b/site/src/cut/lib/headless/bind.tsnew file mode 100644index 00000000..ce3f5fdc--- /dev/null+++ b/site/src/cut/lib/headless/bind.ts@@ -0,0 +1,23 @@+import { bindCutMode } from "../backend";+import { bindCloudSession } from "../backend/cloud";+import { bindHostedSession } from "../hosted";++// One call wires a headless process to the hosted site as one user: the+// backend mode pins to cloud, and both transports — the cut-cloud driver and+// the hosted inference chokepoint — fold the session's origin and auth+// headers into every request. After this, the same lib code the page runs+// (tools, generation, media import) reaches the right server on its own.++export interface HeadlessSession {+ /** Absolute origin of the hosted site, e.g. https://donkeycut.com. */+ base: string;+ /** Auth folded into every request: a session cookie, the runner grant, or+ * the dev bypass. */+ headers: Record<string, string>;+}++export function bindHeadlessSession(session: HeadlessSession): void {+ bindCutMode("cloud");+ bindCloudSession(session.base, session.headers);+ bindHostedSession(session.base, session.headers);+}diff --git a/site/src/cut/lib/headless/docSession.ts b/site/src/cut/lib/headless/docSession.tsnew file mode 100644index 00000000..f8d46086--- /dev/null+++ b/site/src/cut/lib/headless/docSession.ts@@ -0,0 +1,80 @@+import type { HeadlessSession } from "./bind";+import { serializeDoc, useEditor } from "../store";+import type { MediaAsset, ProjectDoc } from "../types";++// A headless run's hold on one cloud project: read the document, hydrate the+// editor store with it, and push the store's persistable slice back through+// the same versioned PUT the page's autosave uses. The version is required at+// both ends — the open fails without one, and the push always sends ?v= — so+// a concurrent writer is answered with a 409 the caller must handle.++export interface HeadlessDocSession {+ projectId: string;+ /** The doc version the store's contents are built on. */+ version: string;+}++const projectPath = (s: HeadlessSession, projectId: string) =>+ `${s.base}/api/cut-cloud/projects/${encodeURIComponent(projectId)}`;++/** Fetch a cloud project's document and open it in the editor store. */+export async function openCloudProject(+ s: HeadlessSession,+ projectId: string+): Promise<HeadlessDocSession> {+ const res = await fetch(projectPath(s, projectId), { headers: s.headers });+ if (!res.ok) throw new Error(`Could not read project ${projectId} (${res.status}).`);+ const version = res.headers.get("x-cut-doc-version");+ if (!version)+ throw new Error("The project read carried no doc version; refusing an unversioned session.");+ const doc = (await res.json()) as ProjectDoc;+ // Media rides signed R2 URLs: tool paths that read asset bytes (attachment+ // refs, sticker decodes) fetch them bare, and the /media route would answer+ // those unauthenticated fetches 401. The route URL stays the fallback for+ // anything the mint misses.+ const fileNames = (doc.assets ?? []).map((a) => a.fileName);+ const signed = new Map<string, string>();+ if (fileNames.length > 0) {+ const mint = await fetch(`${s.base}/api/cut-cloud/media/presign-get`, {+ method: "POST",+ headers: { "Content-Type": "application/json", ...s.headers },+ body: JSON.stringify({ items: fileNames.map((fileName) => ({ projectId, fileName })) }),+ }).catch(() => null);+ if (mint?.ok) {+ const body = (await mint.json()) as { urls?: { fileName: string; url: string }[] };+ for (const u of body.urls ?? []) signed.set(u.fileName, u.url);+ }+ }+ const assets: MediaAsset[] = (doc.assets ?? []).map((a) => ({+ ...a,+ url: signed.get(a.fileName) ?? `${projectPath(s, projectId)}/media/${encodeURIComponent(a.fileName)}`,+ }));+ await useEditor.getState().openProjectDoc(projectId, doc, assets);+ return { projectId, version };+}++/** Push the store's persistable slice back up on the session's version. A 409+ * means another writer moved the document past this session's base; the+ * caller decides whether to reopen and redo or surface the conflict. */+export async function pushCloudProject(+ s: HeadlessSession,+ session: HeadlessDocSession+): Promise<HeadlessDocSession> {+ const state = useEditor.getState();+ if (state.projectId !== session.projectId)+ throw new Error("The store holds a different project than this session opened.");+ const res = await fetch(+ `${projectPath(s, session.projectId)}?v=${encodeURIComponent(session.version)}`,+ {+ method: "PUT",+ headers: { "Content-Type": "application/json", ...s.headers },+ body: JSON.stringify(serializeDoc(state)),+ }+ );+ if (res.status === 409)+ throw new Error("The project changed under this session — another writer holds a newer version.");+ if (!res.ok) throw new Error(`Could not save project ${session.projectId} (${res.status}).`);+ const body = (await res.json()) as { version?: number };+ if (body.version !== undefined) session.version = String(body.version);+ return session;+}diff --git a/site/src/cut/lib/hosted.ts b/site/src/cut/lib/hosted.tsindex 1fdc47d0..1b94e6cb 100644--- a/site/src/cut/lib/hosted.ts+++ b/site/src/cut/lib/hosted.ts@@ -51,7 +51,7 @@ let rechecking: Promise<void> | null = null;* tab up after the user already paid. Any failure leaves the flag as is. */export function recheckCredits(): Promise<void> {if (!useOutOfCredits.getState().out) return Promise.resolve();- rechecking ??= fetch("/api/credits/balance", { cache: "no-store" })+ rechecking ??= hostedFetch("/api/credits/balance", { cache: "no-store" }).then(async (res) => {if (!res.ok) return;const body = (await res.json()) as { balanceMicros?: string };@@ -95,6 +95,23 @@ export function useCreditsRecheck(): void {const OFFLOAD_ABOVE_BYTES = 1024 * 1024;const MAX_BODY_BYTES = 4 * 1024 * 1024;+/** A headless session's transport: an absolute origin and auth headers folded+ * into every hosted call. In the page this stays unbound; same-origin calls+ * carry the session cookie on their own. */+let session: { base: string; headers: Record<string, string> } | null = null;+export function bindHostedSession(base: string, headers: Record<string, string>) {+ session = { base: base.replace(/\/$/, ""), headers };+}++/** Fetch a hosted route through the bound session when one exists. Every+ * hosted-route call in lib code goes through here so a headless process+ * reaches the right server with its auth. */+export const hostedFetch = (path: string, init?: RequestInit): Promise<Response> =>+ fetch((session?.base ?? "") + path, {+ ...init,+ headers: { ...session?.headers, ...(init?.headers as Record<string, string> | undefined) },+ });+/** POST one of Donkey's hosted inference routes with the user's session. */export const hostedPost = async (path: string, body: unknown, signal?: AbortSignal) => {let payload = JSON.stringify(body);@@ -104,7 +121,7 @@ export const hostedPost = async (path: string, body: unknown, signal?: AbortSignif (new Blob([payload]).size > MAX_BODY_BYTES) {throw new Error("That request carries too much attached media to send. Use fewer references.");}- const res = await fetch(path, {+ const res = await hostedFetch(path, {method: "POST",headers: { "Content-Type": "application/json", "x-donkey-client-id": CLIENT_ID },body: payload,diff --git a/site/src/cut/lib/hostedBlobs.ts b/site/src/cut/lib/hostedBlobs.tsindex 4445a99b..3e569e31 100644--- a/site/src/cut/lib/hostedBlobs.ts+++ b/site/src/cut/lib/hostedBlobs.ts@@ -20,6 +20,7 @@// out as it would have: storage is how a large request succeeds, never the// reason a small one fails.import { bytesFromBase64 } from "./bytes";+import { hostedFetch } from "./hosted";/** Keys a hosted payload carries base64 media under, and the media type to* assume when the part does not name one. */@@ -100,7 +101,7 @@ async function claimKeys(blobs: Claimable[], clientId: string): Promise<Claim[]>const claimed = await Promise.all(batches.map(async (batch) => {const unavailable = batch.map(() => ({ skipped: "unavailable" }));- const res = await fetch("/api/inference/uploads", {+ const res = await hostedFetch("/api/inference/uploads", {method: "POST",headers: { "Content-Type": "application/json", "x-donkey-client-id": clientId },body: JSON.stringify({ blobs: batch }),diff --git a/site/src/cut/lib/pi/debris.ts b/site/src/cut/lib/pi/debris.tsnew file mode 100644index 00000000..dbfa99af--- /dev/null+++ b/site/src/cut/lib/pi/debris.ts@@ -0,0 +1,28 @@+import { parkedTransitions, totalDuration, track0Clips, useEditor } from "../store";++const r1 = (x: number) => Math.round(x * 10) / 10;++/** The project's current debris, one line each: transition bars playing+ * nothing, and items stranded past the end of the video. Read from the live+ * editor store, so it serves the page and a headless session alike. */+export function currentDebris(): string[] {+ const s = useEditor.getState();+ const lines: string[] = [];+ for (const t of parkedTransitions(s.clips, s.transitions)) {+ lines.push(+ `parked transition ${t.id} (${t.style}, ${r1(t.seconds)}s) at ${r1(t.start)}s — lines up with no cut, plays nothing`+ );+ }+ const end = totalDuration(track0Clips(s.clips));+ if (end > 0) {+ for (const o of s.overlays) {+ if (o.start >= end)+ lines.push(`overlay ${o.id} ("${"text" in o && o.text ? String(o.text).slice(0, 30) : o.kind}") starts at ${r1(o.start)}s, past the video's end at ${r1(end)}s`);+ }+ for (const a of s.audioClips) {+ if (a.start >= end)+ lines.push(`soundtrack clip ${a.id} starts at ${r1(a.start)}s, past the video's end at ${r1(end)}s`);+ }+ }+ return lines;+}diff --git a/site/src/cut/lib/pi/prodDeps.ts b/site/src/cut/lib/pi/prodDeps.tsindex 44957ba2..c895d5ec 100644--- a/site/src/cut/lib/pi/prodDeps.ts+++ b/site/src/cut/lib/pi/prodDeps.ts@@ -8,34 +8,8 @@ import { normalizeRef } from "../assetRef";import { NO_CREDITS_MESSAGE, useGenerate } from "../generate";import { hostedPost } from "../hosted";import { refsToParts } from "../refMedia";-import { parkedTransitions, totalDuration, track0Clips, useEditor } from "../store";import type { CutAgentDeps } from "./cutAgent";--const r1 = (x: number) => Math.round(x * 10) / 10;--/** The project's current debris, one line each: transition bars playing- * nothing, and items stranded past the end of the video. */-function currentDebris(): string[] {- const s = useEditor.getState();- const lines: string[] = [];- for (const t of parkedTransitions(s.clips, s.transitions)) {- lines.push(- `parked transition ${t.id} (${t.style}, ${r1(t.seconds)}s) at ${r1(t.start)}s — lines up with no cut, plays nothing`- );- }- const end = totalDuration(track0Clips(s.clips));- if (end > 0) {- for (const o of s.overlays) {- if (o.start >= end)- lines.push(`overlay ${o.id} ("${"text" in o && o.text ? String(o.text).slice(0, 30) : o.kind}") starts at ${r1(o.start)}s, past the video's end at ${r1(end)}s`);- }- for (const a of s.audioClips) {- if (a.start >= end)- lines.push(`soundtrack clip ${a.id} starts at ${r1(a.start)}s, past the video's end at ${r1(end)}s`);- }- }- return lines;-}+import { currentDebris } from "./debris";// The live editor's wiring for the pi chat loop. This module carries the// browser-only graph (the editor store, hosted auth, media resolution), sodiff --git a/site/src/cut/lib/pi/serverDeps.ts b/site/src/cut/lib/pi/serverDeps.tsnew file mode 100644index 00000000..9966ebff--- /dev/null+++ b/site/src/cut/lib/pi/serverDeps.ts@@ -0,0 +1,54 @@+import { geminiModelRoles } from "@/lib/inference/gemini-models";+import { AI_SKILL_INDEX, AI_SKILLS } from "@/cut/server/ai/catalog";+import { buildAiContext } from "../aiContext";+import { BROWSER_MEDIA_TOOLS, runAiTool, UI_TOOLS } from "../aiTools";+import { normalizeRef } from "../assetRef";+import { NO_CREDITS_MESSAGE } from "../generate";+import { hostedPost } from "../hosted";+import { bindHeadlessSession, type HeadlessSession } from "../headless/bind";+import { refsToParts } from "../refMedia";+import type { CutAgentDeps } from "./cutAgent";+import { currentDebris } from "./debris";++// The headless wiring for the pi chat loop. The runner opens a project into+// the store (openProjectDoc), runs the turn with these deps, and pushes the+// doc back up. Every transport rides the session's origin and auth headers.++export type { HeadlessSession };++export function headlessDeps(session: HeadlessSession): CutAgentDeps {+ bindHeadlessSession(session);+ return {+ post: (payload, signal) => hostedPost("/api/inference/responses", payload, signal),+ execTool: async (name, args) => {+ if (name === "list_skills") return { skills: AI_SKILL_INDEX };+ if (name === "read_skill") {+ const doc = AI_SKILLS[String(args.name ?? "")];+ if (!doc) throw new Error(`No such skill. Available: ${AI_SKILL_INDEX.join(", ")}`);+ return doc;+ }+ if (UI_TOOLS.has(name))+ return {+ noEditor: true,+ note: "No editor page is attached to this session, so this tool had no effect. The project itself is unchanged — keep working from the editor state.",+ };+ if (BROWSER_MEDIA_TOOLS.has(name))+ throw new Error(+ "This tool reads media through the editor page and is unavailable in this session. Work from the editor state and the transcript instead."+ );+ return runAiTool(name, args);+ },+ models: {+ simple: geminiModelRoles.chatSimple,+ complex: geminiModelRoles.chat,+ gate: geminiModelRoles.fastDecision,+ },+ buildContext: () => buildAiContext(),+ resolveRefs: async (meta) => {+ const refs = meta.map(normalizeRef).filter((r) => r !== null);+ return (await refsToParts(refs)).parts;+ },+ debris: currentDebris,+ noCreditsMessage: NO_CREDITS_MESSAGE,+ };+}diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex fd680b68..d4a7ed5c 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -260,6 +260,14 @@ export interface EditorState {* the stored copy has moved on, so the head-start snapshot is skipped and* only the live document is applied. */loadProject: (id: string, opts?: { inPlace?: boolean }) => Promise<void>;+ /** Apply one project document to the store: the legacy migrations and the+ * state swap, shared by every hydration path — the snapshot paint, the live+ * load, a conflict reload, and a headless open. */+ applyDocState: (doc: Partial<ProjectDoc>, assets: MediaAsset[], ui: ProjectUiState) => void;+ /** Open a project from an already-fetched document. The headless runner's+ * open: resets the store, drains pending doc writes, and hydrates straight+ * from the given document. */+ openProjectDoc: (id: string, doc: Partial<ProjectDoc>, assets: MediaAsset[]) => Promise<void>;setProjectName: (name: string) => void;setSaveState: (s: SaveState) => void;/** Enter read-only shared mode; call before loadProject. */@@ -1209,6 +1217,57 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {}));};+ /** Reset to an empty, loading project and drain any background doc writes+ * still queued for it, so no open ever reads a half-written document. */+ const resetForOpen = async (id: string) => {+ history.length = 0;+ future.length = 0;+ pending = null;+ // A fresh project owns no live run — any prior run's render-owned ids are+ // stale, and the loaded clips are ordinary, fully-undoable content.+ genClipIds.clear();+ genAudioIds.clear();+ hydrating = true;+ set({+ projectId: id,+ loaded: false,+ loadError: null,+ saveState: "saved",+ resumePush: false,+ assets: [],+ clips: [],+ transitions: [],+ audioClips: [],+ overlays: [],+ templates: [],+ aspect: "9:16",+ aspectTouched: false,+ fadeIn: 0,+ fadeOut: 0,+ selection: null,+ multiSelection: [],+ playing: false,+ previewStopAt: null,+ subtitles: emptySubtitles(),+ subtitleLane: 0,+ subtitleStatus: "idle",+ subtitleError: null,+ exportOpen: false,+ genvideo: undefined,+ renders: [],+ firstOpen: undefined,+ });+ hydrating = false;+ // A background scene run may still be writing this project's doc — drain+ // its queued writes so the open never reads a half-written doc. Ordering+ // matters: projectId is set (loaded false) BEFORE this await, so a write+ // arriving during the drain waits for the load (projectWriteMode) instead+ // of queueing a doc write the drain would miss — nothing can land between+ // the drain and the fetch that follows. Lazy import: docWriter reads store+ // helpers, so a static import would be a cycle.+ await import("./genvideo/docWriter").then((m) => m.docWriterIdle(id)).catch(() => {});+ };+return {projectId: null,projectName: "",@@ -1258,156 +1317,13 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {// clicking back in — is not a re-read, and gets the snapshot's head// start like any other open.const inPlace = opts?.inPlace === true;- history.length = 0;- future.length = 0;- pending = null;- // A fresh project owns no live run — any prior run's render-owned ids are- // stale, and the loaded clips are ordinary, fully-undoable content.- genClipIds.clear();- genAudioIds.clear();- hydrating = true;- set({- projectId: id,- loaded: false,- loadError: null,- saveState: "saved",- resumePush: false,- assets: [],- clips: [],- transitions: [],- audioClips: [],- overlays: [],- templates: [],- aspect: "9:16",- aspectTouched: false,- fadeIn: 0,- fadeOut: 0,- selection: null,- multiSelection: [],- playing: false,- previewStopAt: null,- subtitles: emptySubtitles(),- subtitleLane: 0,- subtitleStatus: "idle",- subtitleError: null,- exportOpen: false,- genvideo: undefined,- renders: [],- firstOpen: undefined,- });- hydrating = false;- // A background scene run may still be writing this project's doc — drain- // its queued writes so the load never reads a half-written doc. Ordering- // matters: projectId is set (loaded false) BEFORE this await, so a write- // arriving during the drain waits for the load (projectWriteMode) instead- // of queueing a doc write the drain would miss — nothing can land between- // the drain and the fetch below. Lazy import: docWriter reads store- // helpers, so a static import would be a cycle.- await import("./genvideo/docWriter").then((m) => m.docWriterIdle(id)).catch(() => {});+ await resetForOpen(id);// One shape of hydration, used by both the snapshot painted below and// the live document that replaces it, so the legacy migrations happen// once per document rather than once per code path.- const hydrate = (doc: Partial<ProjectDoc>, assets: MediaAsset[], ui: ProjectUiState) => {- const docClips = doc.clips ?? [];- // Older docs stored video track 0 packed (array order implied the- // position); bake explicit starts in once so every clip is free-placed.- const legacy = (docClips as LegacyClip[]).some((c) => typeof c.start !== "number");- const folded = (legacy ? packStarts(docClips as LegacyClip[]) : docClips).map((c) => ({- ...c,- track: c.track ?? 0,- }));- // Older docs kept tracks other than 0 in a separate `overlayClips` array;- // fold them into the one clip list (each already carries its `track`).- // Entries whose id already sits in `clips` are the same clip persisted- // twice by a version-skewed save (an older engine keeps overlayClips- // after a merged client writes the folded list) — keep the folded copy.- // Entries with track 0 were unreachable dead data under the split shape- // (never rendered, never played); promoting them would insert them into- // track 0's sequence, so they stay dropped.- const seen = new Set(folded.map((c) => c.id));- const legacyLayers = (doc.overlayClips ?? []).filter(- (c) => c.track !== 0 && !seen.has(c.id)- );- // Tracks number 0..N bottom-up. Docs saved when tracks could go- // negative (backdrop rows below the spine) lift wholesale so the- // lowest row becomes track 0 — the bottom row is the spine now.- const joined = [...folded, ...legacyLayers];- const lift = Math.max(0, ...joined.map((c) => -c.track));- const lifted = lift ? joined.map((c) => ({ ...c, track: c.track + lift })) : joined;- // Stamp `kind: "text"` on pre-union titles so every in-memory element- // carries its discriminant; the serializer strips it back. Effects- // saved onto a shared row move to one of their own, and a clip graded- // back when a look was a clip property gets that grade as an element- // over it — so a project made before either rule reads like a new one.- // The behind-speaker boolean becomes an inverted subject mask on load,- // so one mask model covers it everywhere in memory and on save.- const stamped = normalizeElementLanes(- migrateBehindSubject(stampOverlayKinds(doc.overlays ?? []))- );- const subtitles = doc.subtitles ?? emptySubtitles();- // Docs saved when edge transition styles existed convert them into the- // equivalent clip animations, and docs saved when a transition was a- // physical overlap pull their intruding clips apart — clips never- // overlap, whatever wrote the file. Pulling them apart lengthens the- // cut, so the whole document goes through it together.- const merged = separateOverlaps({- clips: migrateLegacyTransitions(lifted),- audioClips: doc.audioClips ?? [],- overlays: stamped,- cues: subtitles.cues,- });- const withLooks =- liftClipLooks(merged.clips, merged.overlays, getClipSpans(merged.clips, assets)) ?? {- clips: merged.clips,- overlays: merged.overlays,- };- hydrating = true;- set({- projectName: doc.name ?? "",- assets,- clips: withLooks.clips,- // Bars from the doc, plus one adopted for each transition/animation- // a pre-bar doc stored as a clip field.- transitions: adoptTransitionFields(- withLooks.clips,- sanitizeTransitions(doc.transitions)- ),- audioClips: merged.audioClips,- overlays: withLooks.overlays,- templates: doc.templates ?? [],- aspect: normalizeAspect(doc.aspect) ?? "9:16",- aspectTouched: doc.aspect !== undefined,- fadeIn: doc.fadeIn ?? 0,- fadeOut: doc.fadeOut ?? 0,- // View state lives in IndexedDB; doc.ui covers projects saved- // before the move.- pxPerSec: Math.max(12, Math.min(800, ui.pxPerSec ?? doc.ui?.pxPerSec ?? 60)),- timelineH: Math.max(- TIMELINE_H_MIN,- Math.min(timelineHMax(), ui.timelineH ?? TIMELINE_H_DEFAULT)- ),- publish: {- caption: doc.publish?.caption ?? "",- tags: doc.publish?.tags ?? "",- soundTitle: doc.publish?.soundTitle ?? "",- handle: doc.publish?.handle ?? "",- },- notes: {- text: doc.notes?.text ?? "",- publishedAt: doc.notes?.publishedAt ?? "",- links: doc.notes?.links ?? [],- },- subtitles: { ...subtitles, cues: merged.cues },- subtitleStatus: merged.cues.length > 0 ? "ready" : "idle",- genvideo: doc.genvideo ?? undefined,- renders: Array.isArray(doc.renders) ? doc.renders : [],- firstOpen: doc.firstOpen,- loaded: true,- loadEpoch: get().loadEpoch + 1,- });- hydrating = false;- };+ const hydrate = (doc: Partial<ProjectDoc>, assets: MediaAsset[], ui: ProjectUiState) =>+ get().applyDocState(doc, assets, ui);// The live document goes out first, so everything below is a head start// on it and never a substitute for it.@@ -1543,6 +1459,117 @@ export const useEditor = create<EditorState>((baseSet, get, api) => {}},+ applyDocState: (doc, assets, ui) => {+ const docClips = doc.clips ?? [];+ // Older docs stored video track 0 packed (array order implied the+ // position); bake explicit starts in once so every clip is free-placed.+ const legacy = (docClips as LegacyClip[]).some((c) => typeof c.start !== "number");+ const folded = (legacy ? packStarts(docClips as LegacyClip[]) : docClips).map((c) => ({+ ...c,+ track: c.track ?? 0,+ }));+ // Older docs kept tracks other than 0 in a separate `overlayClips` array;+ // fold them into the one clip list (each already carries its `track`).+ // Entries whose id already sits in `clips` are the same clip persisted+ // twice by a version-skewed save (an older engine keeps overlayClips+ // after a merged client writes the folded list) — keep the folded copy.+ // Entries with track 0 were unreachable dead data under the split shape+ // (never rendered, never played); promoting them would insert them into+ // track 0's sequence, so they stay dropped.+ const seen = new Set(folded.map((c) => c.id));+ const legacyLayers = (doc.overlayClips ?? []).filter(+ (c) => c.track !== 0 && !seen.has(c.id)+ );+ // Tracks number 0..N bottom-up. Docs saved when tracks could go+ // negative (backdrop rows below the spine) lift wholesale so the+ // lowest row becomes track 0 — the bottom row is the spine now.+ const joined = [...folded, ...legacyLayers];+ const lift = Math.max(0, ...joined.map((c) => -c.track));+ const lifted = lift ? joined.map((c) => ({ ...c, track: c.track + lift })) : joined;+ // Stamp `kind: "text"` on pre-union titles so every in-memory element+ // carries its discriminant; the serializer strips it back. Effects+ // saved onto a shared row move to one of their own, and a clip graded+ // back when a look was a clip property gets that grade as an element+ // over it — so a project made before either rule reads like a new one.+ // The behind-speaker boolean becomes an inverted subject mask on load,+ // so one mask model covers it everywhere in memory and on save.+ const stamped = normalizeElementLanes(+ migrateBehindSubject(stampOverlayKinds(doc.overlays ?? []))+ );+ const subtitles = doc.subtitles ?? emptySubtitles();+ // Docs saved when edge transition styles existed convert them into the+ // equivalent clip animations, and docs saved when a transition was a+ // physical overlap pull their intruding clips apart — clips never+ // overlap, whatever wrote the file. Pulling them apart lengthens the+ // cut, so the whole document goes through it together.+ const merged = separateOverlaps({+ clips: migrateLegacyTransitions(lifted),+ audioClips: doc.audioClips ?? [],+ overlays: stamped,+ cues: subtitles.cues,+ });+ const withLooks =+ liftClipLooks(merged.clips, merged.overlays, getClipSpans(merged.clips, assets)) ?? {+ clips: merged.clips,+ overlays: merged.overlays,+ };+ hydrating = true;+ try {+ set({+ projectName: doc.name ?? "",+ assets,+ clips: withLooks.clips,+ // Bars from the doc, plus one adopted for each transition/animation+ // a pre-bar doc stored as a clip field.+ transitions: adoptTransitionFields(+ withLooks.clips,+ sanitizeTransitions(doc.transitions)+ ),+ audioClips: merged.audioClips,+ overlays: withLooks.overlays,+ templates: doc.templates ?? [],+ aspect: normalizeAspect(doc.aspect) ?? "9:16",+ aspectTouched: doc.aspect !== undefined,+ fadeIn: doc.fadeIn ?? 0,+ fadeOut: doc.fadeOut ?? 0,+ // View state lives in IndexedDB; doc.ui covers projects saved+ // before the move.+ pxPerSec: Math.max(12, Math.min(800, ui.pxPerSec ?? doc.ui?.pxPerSec ?? 60)),+ timelineH: Math.max(+ TIMELINE_H_MIN,+ Math.min(timelineHMax(), ui.timelineH ?? TIMELINE_H_DEFAULT)+ ),+ publish: {+ caption: doc.publish?.caption ?? "",+ tags: doc.publish?.tags ?? "",+ soundTitle: doc.publish?.soundTitle ?? "",+ handle: doc.publish?.handle ?? "",+ },+ notes: {+ text: doc.notes?.text ?? "",+ publishedAt: doc.notes?.publishedAt ?? "",+ links: doc.notes?.links ?? [],+ },+ subtitles: { ...subtitles, cues: merged.cues },+ subtitleStatus: merged.cues.length > 0 ? "ready" : "idle",+ genvideo: doc.genvideo ?? undefined,+ renders: Array.isArray(doc.renders) ? doc.renders : [],+ firstOpen: doc.firstOpen,+ loaded: true,+ loadEpoch: get().loadEpoch + 1,+ });+ } finally {+ // Reset on every path: a doc bad enough to throw inside set() must+ // never leave the store treating ordinary writes as hydration.+ hydrating = false;+ }+ },++ openProjectDoc: async (id, doc, assets) => {+ await resetForOpen(id);+ get().applyDocState(doc, assets, {});+ },+setProjectName: (name) => set({ projectName: name }),setSaveState: (s) => set({ saveState: s }),setSharedView: (features) => set({ readOnly: true, sharedFeatures: features }),