Open source CapCut alternative.
TypeScript
refactor(cut): the engine's detached tool notes live in one page-side module [rebuild]
David Le54fe68b305eb
diff --git a/site/src/cut/lib/chatResume.test.ts b/site/src/cut/lib/chatResume.test.tsnew file mode 100644index 00000000..d480378f--- /dev/null+++ b/site/src/cut/lib/chatResume.test.ts@@ -0,0 +1,67 @@+import { expect, test } from "bun:test";+import type { UIMessage } from "ai";+import { DETACHED_MEDIA_ERROR, DETACHED_UI_NOTE, INTERRUPTED_ERROR, isResumeMessage, resumePrompt, scrubLostTurn, unfinishedAsk } from "./chatResume";++const user = (id: string, text: string, metadata?: Record<string, unknown>): UIMessage =>+ ({ id, role: "user", parts: [{ type: "text", text }], ...(metadata ? { metadata } : {}) }) as UIMessage;+const reply = (id: string, parts: UIMessage["parts"] = [{ type: "text", text: "Done." }]): UIMessage => ({ id, role: "assistant", parts });+const tool = (state: string, extra: Record<string, unknown>) =>+ ({ type: "tool-add_clip", toolCallId: "call", state, input: {}, ...extra }) as unknown as UIMessage["parts"][number];++test("an ask with no reply resumes", () => {+ expect(unfinishedAsk([user("ask", "cut the pauses")], undefined, undefined)?.id).toBe("ask");+});++test("a reply cut off before it completed resumes its ask", () => {+ const messages = [user("ask", "cut the pauses"), reply("partial", [tool("output-error", { errorText: "Interrupted." })])];+ expect(unfinishedAsk(messages, "earlier", undefined)?.id).toBe("ask");+});++test("a completed reply settles the thread", () => {+ expect(unfinishedAsk([user("ask", "cut the pauses"), reply("done")], "done", undefined)).toBeUndefined();+});++test("a stopped or failed turn waits for the user", () => {+ const messages = [user("ask", "cut the pauses"), reply("partial")];+ expect(unfinishedAsk(messages, undefined, "stopped")).toBeUndefined();+ expect(unfinishedAsk([user("ask", "cut the pauses")], undefined, "failed")).toBeUndefined();+});++test("a completed reply that ran with no tab attached resumes on return", () => {+ const detached = [user("ask", "make a thumbnail"), reply("done", [tool("output-error", { errorText: DETACHED_MEDIA_ERROR })])];+ expect(unfinishedAsk(detached, "done", undefined)?.id).toBe("ask");+ const noEffect = [user("ask", "open captions"), reply("done", [tool("output-available", { output: { noEditor: true, note: DETACHED_UI_NOTE } })])];+ expect(unfinishedAsk(noEffect, "done", undefined)?.id).toBe("ask");+ const ordinary = [user("ask", "add it"), reply("done", [tool("output-error", { errorText: "No clip with that id." })])];+ expect(unfinishedAsk(ordinary, "done", undefined)).toBeUndefined();+});++test("a continuation that itself got cut off resumes the original ask", () => {+ const messages = [user("ask", "cut the pauses"), reply("partial"), user("again", "continue", { resume: true })];+ expect(isResumeMessage(messages[2])).toBe(true);+ expect(unfinishedAsk(messages, undefined, undefined)?.id).toBe("ask");+});++test("the continuation quotes the ask", () => {+ expect(resumePrompt(user("ask", "cut the pauses"))).toContain('"""\ncut the pauses\n"""');+ expect(resumePrompt(user("ask", ""))).toContain("only attachments");+});++test("scrubbing a lost turn keeps only the tool calls that landed", () => {+ const landed = tool("output-available", { output: { ok: true } });+ const messages = [+ user("earlier", "first ask"),+ reply("old", [tool("output-error", { errorText: INTERRUPTED_ERROR })]),+ user("ask", "cut the pauses"),+ reply("partial", [{ type: "text", text: "Cutting" }, landed, tool("output-error", { errorText: INTERRUPTED_ERROR })]),+ reply("empty", [{ type: "text", text: "I could not, no page was attached." }, tool("output-error", { errorText: DETACHED_MEDIA_ERROR })]),+ ];+ const scrubbed = scrubLostTurn(messages, "ask");+ expect(scrubbed.map((m) => m.id)).toEqual(["earlier", "old", "ask", "partial"]);+ expect(scrubbed[3].parts).toEqual([landed]);+ expect(scrubLostTurn(messages, "missing")).toBe(messages);+});++test("the continuation asks for a reply that says nothing about the interruption", () => {+ expect(resumePrompt(user("ask", "cut the pauses"))).toContain("say nothing about the page going away");+});diff --git a/site/src/cut/lib/chatResume.ts b/site/src/cut/lib/chatResume.tsnew file mode 100644index 00000000..a8084d60--- /dev/null+++ b/site/src/cut/lib/chatResume.ts@@ -0,0 +1,96 @@+import type { UIMessage } from "ai";++// A turn the page lost — a reload, a closed tab, a dropped connection, or a+// tool that needed the page while no tab was attached — resumes itself when+// the thread next opens. The panel reconnects to the engine's journal where+// one exists, and past that sends the model a continuation quoting the ask.++/** How many continuations one ask gets before the thread waits for the user. */+export const RESUME_LIMIT = 2;++/** The error a saved tool call settles to when the page died before its result landed. */+export const INTERRUPTED_ERROR = "Interrupted.";++/** What the engine answers for tools that need the page while no tab is+ * attached. A returning tab reads these off the finished reply and picks the+ * turn back up. */+export const DETACHED_UI_NOTE =+ "No editor page is attached to this session, so this tool had no effect. The project itself is unchanged — keep working from the editor state.";+export const DETACHED_MEDIA_ERROR =+ "This tool reads media through the editor page and is unavailable while no tab is attached. Work from the editor state instead.";+export const DETACHED_SESSION_ERROR =+ "This tool runs with the editor page's hosted sign-in and is unavailable while no tab is attached. Finish the edit from the project state; the user can run it from the editor.";++/** How the newest turn ended when it did not finish. Either way the thread+ * holds for the user; a turn with no verdict was cut off and resumes. */+export type TurnSettled = "stopped" | "failed";++export type ResumeMetadata = { resume: true; attachments?: unknown[] };++export function isResumeMessage(message: UIMessage): boolean {+ return message.role === "user" && (message.metadata as { resume?: boolean } | undefined)?.resume === true;+}++type ToolPart = { type: string; state?: string; errorText?: string; output?: unknown };++const isToolPart = (part: { type: string }): part is ToolPart =>+ part.type === "dynamic-tool" || part.type.startsWith("tool-");++/** A tool call the lost page never answered, or one that failed for want of a page. */+function lostToolCall(part: ToolPart): boolean {+ if (part.state === "output-error")+ return part.errorText === INTERRUPTED_ERROR || part.errorText === DETACHED_MEDIA_ERROR || part.errorText === DETACHED_SESSION_ERROR;+ if (part.state === "output-available")+ return !!part.output && typeof part.output === "object" && (part.output as { noEditor?: boolean }).noEditor === true;+ return false;+}++/** A reply that ran with no tab attached and hit a tool that needed one. */+export function detachedToolFailed(message: UIMessage): boolean {+ return message.parts.some((part) => isToolPart(part) && part.errorText !== INTERRUPTED_ERROR && lostToolCall(part));+}++/** The transcript with the lost turn's failure taken out, ready for the+ * continuation to finish the job in its place: the cut-off reply keeps only+ * the tool calls that landed, and a reply left with nothing goes entirely. */+export function scrubLostTurn(messages: UIMessage[], askId: string): UIMessage[] {+ const from = messages.findIndex((message) => message.id === askId);+ if (from < 0) return messages;+ return messages.flatMap((message, index) => {+ if (index <= from || message.role !== "assistant") return [message];+ const parts = message.parts.filter((part) => isToolPart(part) && part.state === "output-available" && !lostToolCall(part));+ return parts.length === 0 ? [] : [{ ...message, parts }];+ });+}++/** The ask the thread still owes a finished reply to; undefined once the+ * newest turn completed, or when the user stopped it or it failed outright. */+export function unfinishedAsk(+ messages: UIMessage[],+ completedMessageId: string | undefined,+ settled: TurnSettled | undefined,+): UIMessage | undefined {+ const last = messages.at(-1);+ const ask = messages.findLast((message) => message.role === "user" && !isResumeMessage(message));+ if (!last || !ask) return undefined;+ if (last.role === "assistant" && last.id === completedMessageId) return detachedToolFailed(last) ? ask : undefined;+ if (settled) return undefined;+ return ask;+}++export function askText(ask: UIMessage): string {+ return ask.parts.map((part) => (part.type === "text" ? part.text : "")).join("").trim();+}++/** The continuation a resumed turn opens with. It carries the ask itself, so+ * a provider session that never received it still knows the job. */+export function resumePrompt(ask: UIMessage): string {+ const text = askText(ask);+ return [+ "The editor page went away before your reply to this message finished:",+ "",+ text ? `"""\n${text}\n"""` : "(the message carried only attachments)",+ "",+ "Continue from where that turn stopped. The editor state below is current: edits that already landed are in it, renders still running show under `renders` (wait_for_renders lands them), and anything the ask needs that is missing you do now. Finish the ask, then reply as if this were one uninterrupted turn: say what is done, and say nothing about the page going away, a reload, or an interruption.",+ ].join("\n");+}diff --git a/site/src/cut/lib/pi/serverDeps.ts b/site/src/cut/lib/pi/serverDeps.tsindex b664c171..686e35d2 100644--- a/site/src/cut/lib/pi/serverDeps.ts+++ b/site/src/cut/lib/pi/serverDeps.ts@@ -3,6 +3,7 @@ import { AI_SKILL_INDEX, AI_SKILLS } from "@/cut/server/ai/catalog";import { buildAiContext } from "../aiContext";import { MEDIA_RUNTIME_TOOLS, runAiTool, UI_TOOLS } from "../aiTools";import { normalizeRef } from "../assetRef";+import { DETACHED_UI_NOTE } from "../chatResume";import { NO_CREDITS_MESSAGE } from "../credits";import { hostedPost } from "../hosted";import { bindHeadlessSession, type HeadlessSession } from "../headless/bind";@@ -28,11 +29,7 @@ export function headlessDeps(session: HeadlessSession): CutAgentDeps {if (!doc) throw new Error(`No such skill. Available: ${AI_SKILL_INDEX.join(", ")}`);return doc;}- if (UI_TOOLS.has(name))- return {- noEditor: true,- note: "No editor page is attached to this session, so this tool had no effect. The project itself is unchanged — keep working from the editor state.",- };+ if (UI_TOOLS.has(name)) return { noEditor: true, note: DETACHED_UI_NOTE };// The worker installs canvas and decoders at startup and these run as// written. A process that came up without them says so plainly.if (MEDIA_RUNTIME_TOOLS.has(name)) {diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.tsindex 4ae47d87..14f19ffd 100644--- a/site/src/cut/server/ai/headlessTools.ts+++ b/site/src/cut/server/ai/headlessTools.ts@@ -1,5 +1,6 @@import { copyFile } from "node:fs/promises";import { runAiTool, UI_TOOLS } from "../../lib/aiTools";+import { DETACHED_MEDIA_ERROR, DETACHED_SESSION_ERROR, DETACHED_UI_NOTE } from "../../lib/chatResume";import { mp4NameFor, setHostMediaStore } from "../../lib/mediaConvert";import { setHostDocStore } from "../../lib/projectReference";import { serializeDoc, useEditor } from "../../lib/store";@@ -144,21 +145,10 @@ async function executeHeadlessTool(): Promise<{ output?: unknown; errorText?: string }> {if (UI_TOOLS.has(toolName))return {- output: {- noEditor: true,- note: "No editor page is attached to this session, so this tool had no effect. The project itself is unchanged — keep working from the editor state.",- },- };- if (PAGE_MEDIA_TOOLS.has(toolName))- return {- errorText:- "This tool reads media through the editor page and is unavailable while no tab is attached. Work from the editor state instead.",- };- if (PAGE_SESSION_TOOLS.has(toolName))- return {- errorText:- "This tool runs with the editor page's hosted sign-in and is unavailable while no tab is attached. Finish the edit from the project state; the user can run it from the editor.",+ output: { noEditor: true, note: DETACHED_UI_NOTE },};+ if (PAGE_MEDIA_TOOLS.has(toolName)) return { errorText: DETACHED_MEDIA_ERROR };+ if (PAGE_SESSION_TOOLS.has(toolName)) return { errorText: DETACHED_SESSION_ERROR };const doc = await ensureOpen(projectId);if (!doc) return { errorText: "Project not found on this Mac." };try {