Skip to content

Open source CapCut alternative.

TypeScript

refactor(cut): the engine's detached tool notes live in one page-side module [rebuild]

David Le54fe68b305eb
diff --git a/site/src/cut/lib/chatResume.test.ts b/site/src/cut/lib/chatResume.test.ts
new file mode 100644
index 00000000..d480378f
--- /dev/null
+++ b/site/src/cut/lib/chatResume.test.ts
@@ -0,0 +1,67 @@
+import { expect, test } from "bun:test";
+import type { UIMessage } from "ai";
+import { DETACHED_MEDIA_ERROR, DETACHED_UI_NOTE, INTERRUPTED_ERROR, isResumeMessage, resumePrompt, scrubLostTurn, unfinishedAsk } from "./chatResume";
+
+const user = (id: string, text: string, metadata?: Record<string, unknown>): UIMessage =>
+ ({ id, role: "user", parts: [{ type: "text", text }], ...(metadata ? { metadata } : {}) }) as UIMessage;
+const reply = (id: string, parts: UIMessage["parts"] = [{ type: "text", text: "Done." }]): UIMessage => ({ id, role: "assistant", parts });
+const tool = (state: string, extra: Record<string, unknown>) =>
+ ({ type: "tool-add_clip", toolCallId: "call", state, input: {}, ...extra }) as unknown as UIMessage["parts"][number];
+
+test("an ask with no reply resumes", () => {
+ expect(unfinishedAsk([user("ask", "cut the pauses")], undefined, undefined)?.id).toBe("ask");
+});
+
+test("a reply cut off before it completed resumes its ask", () => {
+ const messages = [user("ask", "cut the pauses"), reply("partial", [tool("output-error", { errorText: "Interrupted." })])];
+ expect(unfinishedAsk(messages, "earlier", undefined)?.id).toBe("ask");
+});
+
+test("a completed reply settles the thread", () => {
+ expect(unfinishedAsk([user("ask", "cut the pauses"), reply("done")], "done", undefined)).toBeUndefined();
+});
+
+test("a stopped or failed turn waits for the user", () => {
+ const messages = [user("ask", "cut the pauses"), reply("partial")];
+ expect(unfinishedAsk(messages, undefined, "stopped")).toBeUndefined();
+ expect(unfinishedAsk([user("ask", "cut the pauses")], undefined, "failed")).toBeUndefined();
+});
+
+test("a completed reply that ran with no tab attached resumes on return", () => {
+ const detached = [user("ask", "make a thumbnail"), reply("done", [tool("output-error", { errorText: DETACHED_MEDIA_ERROR })])];
+ expect(unfinishedAsk(detached, "done", undefined)?.id).toBe("ask");
+ const noEffect = [user("ask", "open captions"), reply("done", [tool("output-available", { output: { noEditor: true, note: DETACHED_UI_NOTE } })])];
+ expect(unfinishedAsk(noEffect, "done", undefined)?.id).toBe("ask");
+ const ordinary = [user("ask", "add it"), reply("done", [tool("output-error", { errorText: "No clip with that id." })])];
+ expect(unfinishedAsk(ordinary, "done", undefined)).toBeUndefined();
+});
+
+test("a continuation that itself got cut off resumes the original ask", () => {
+ const messages = [user("ask", "cut the pauses"), reply("partial"), user("again", "continue", { resume: true })];
+ expect(isResumeMessage(messages[2])).toBe(true);
+ expect(unfinishedAsk(messages, undefined, undefined)?.id).toBe("ask");
+});
+
+test("the continuation quotes the ask", () => {
+ expect(resumePrompt(user("ask", "cut the pauses"))).toContain('"""\ncut the pauses\n"""');
+ expect(resumePrompt(user("ask", ""))).toContain("only attachments");
+});
+
+test("scrubbing a lost turn keeps only the tool calls that landed", () => {
+ const landed = tool("output-available", { output: { ok: true } });
+ const messages = [
+ user("earlier", "first ask"),
+ reply("old", [tool("output-error", { errorText: INTERRUPTED_ERROR })]),
+ user("ask", "cut the pauses"),
+ reply("partial", [{ type: "text", text: "Cutting" }, landed, tool("output-error", { errorText: INTERRUPTED_ERROR })]),
+ reply("empty", [{ type: "text", text: "I could not, no page was attached." }, tool("output-error", { errorText: DETACHED_MEDIA_ERROR })]),
+ ];
+ const scrubbed = scrubLostTurn(messages, "ask");
+ expect(scrubbed.map((m) => m.id)).toEqual(["earlier", "old", "ask", "partial"]);
+ expect(scrubbed[3].parts).toEqual([landed]);
+ expect(scrubLostTurn(messages, "missing")).toBe(messages);
+});
+
+test("the continuation asks for a reply that says nothing about the interruption", () => {
+ expect(resumePrompt(user("ask", "cut the pauses"))).toContain("say nothing about the page going away");
+});
diff --git a/site/src/cut/lib/chatResume.ts b/site/src/cut/lib/chatResume.ts
new file mode 100644
index 00000000..a8084d60
--- /dev/null
+++ b/site/src/cut/lib/chatResume.ts
@@ -0,0 +1,96 @@
+import type { UIMessage } from "ai";
+
+// A turn the page lost — a reload, a closed tab, a dropped connection, or a
+// tool that needed the page while no tab was attached — resumes itself when
+// the thread next opens. The panel reconnects to the engine's journal where
+// one exists, and past that sends the model a continuation quoting the ask.
+
+/** How many continuations one ask gets before the thread waits for the user. */
+export const RESUME_LIMIT = 2;
+
+/** The error a saved tool call settles to when the page died before its result landed. */
+export const INTERRUPTED_ERROR = "Interrupted.";
+
+/** What the engine answers for tools that need the page while no tab is
+ * attached. A returning tab reads these off the finished reply and picks the
+ * turn back up. */
+export const DETACHED_UI_NOTE =
+ "No editor page is attached to this session, so this tool had no effect. The project itself is unchanged — keep working from the editor state.";
+export const DETACHED_MEDIA_ERROR =
+ "This tool reads media through the editor page and is unavailable while no tab is attached. Work from the editor state instead.";
+export const DETACHED_SESSION_ERROR =
+ "This tool runs with the editor page's hosted sign-in and is unavailable while no tab is attached. Finish the edit from the project state; the user can run it from the editor.";
+
+/** How the newest turn ended when it did not finish. Either way the thread
+ * holds for the user; a turn with no verdict was cut off and resumes. */
+export type TurnSettled = "stopped" | "failed";
+
+export type ResumeMetadata = { resume: true; attachments?: unknown[] };
+
+export function isResumeMessage(message: UIMessage): boolean {
+ return message.role === "user" && (message.metadata as { resume?: boolean } | undefined)?.resume === true;
+}
+
+type ToolPart = { type: string; state?: string; errorText?: string; output?: unknown };
+
+const isToolPart = (part: { type: string }): part is ToolPart =>
+ part.type === "dynamic-tool" || part.type.startsWith("tool-");
+
+/** A tool call the lost page never answered, or one that failed for want of a page. */
+function lostToolCall(part: ToolPart): boolean {
+ if (part.state === "output-error")
+ return part.errorText === INTERRUPTED_ERROR || part.errorText === DETACHED_MEDIA_ERROR || part.errorText === DETACHED_SESSION_ERROR;
+ if (part.state === "output-available")
+ return !!part.output && typeof part.output === "object" && (part.output as { noEditor?: boolean }).noEditor === true;
+ return false;
+}
+
+/** A reply that ran with no tab attached and hit a tool that needed one. */
+export function detachedToolFailed(message: UIMessage): boolean {
+ return message.parts.some((part) => isToolPart(part) && part.errorText !== INTERRUPTED_ERROR && lostToolCall(part));
+}
+
+/** The transcript with the lost turn's failure taken out, ready for the
+ * continuation to finish the job in its place: the cut-off reply keeps only
+ * the tool calls that landed, and a reply left with nothing goes entirely. */
+export function scrubLostTurn(messages: UIMessage[], askId: string): UIMessage[] {
+ const from = messages.findIndex((message) => message.id === askId);
+ if (from < 0) return messages;
+ return messages.flatMap((message, index) => {
+ if (index <= from || message.role !== "assistant") return [message];
+ const parts = message.parts.filter((part) => isToolPart(part) && part.state === "output-available" && !lostToolCall(part));
+ return parts.length === 0 ? [] : [{ ...message, parts }];
+ });
+}
+
+/** The ask the thread still owes a finished reply to; undefined once the
+ * newest turn completed, or when the user stopped it or it failed outright. */
+export function unfinishedAsk(
+ messages: UIMessage[],
+ completedMessageId: string | undefined,
+ settled: TurnSettled | undefined,
+): UIMessage | undefined {
+ const last = messages.at(-1);
+ const ask = messages.findLast((message) => message.role === "user" && !isResumeMessage(message));
+ if (!last || !ask) return undefined;
+ if (last.role === "assistant" && last.id === completedMessageId) return detachedToolFailed(last) ? ask : undefined;
+ if (settled) return undefined;
+ return ask;
+}
+
+export function askText(ask: UIMessage): string {
+ return ask.parts.map((part) => (part.type === "text" ? part.text : "")).join("").trim();
+}
+
+/** The continuation a resumed turn opens with. It carries the ask itself, so
+ * a provider session that never received it still knows the job. */
+export function resumePrompt(ask: UIMessage): string {
+ const text = askText(ask);
+ return [
+ "The editor page went away before your reply to this message finished:",
+ "",
+ text ? `"""\n${text}\n"""` : "(the message carried only attachments)",
+ "",
+ "Continue from where that turn stopped. The editor state below is current: edits that already landed are in it, renders still running show under `renders` (wait_for_renders lands them), and anything the ask needs that is missing you do now. Finish the ask, then reply as if this were one uninterrupted turn: say what is done, and say nothing about the page going away, a reload, or an interruption.",
+ ].join("\n");
+}
diff --git a/site/src/cut/lib/pi/serverDeps.ts b/site/src/cut/lib/pi/serverDeps.ts
index b664c171..686e35d2 100644
--- a/site/src/cut/lib/pi/serverDeps.ts
+++ b/site/src/cut/lib/pi/serverDeps.ts
@@ -3,6 +3,7 @@ import { AI_SKILL_INDEX, AI_SKILLS } from "@/cut/server/ai/catalog";
import { buildAiContext } from "../aiContext";
import { MEDIA_RUNTIME_TOOLS, runAiTool, UI_TOOLS } from "../aiTools";
import { normalizeRef } from "../assetRef";
+import { DETACHED_UI_NOTE } from "../chatResume";
import { NO_CREDITS_MESSAGE } from "../credits";
import { hostedPost } from "../hosted";
import { bindHeadlessSession, type HeadlessSession } from "../headless/bind";
@@ -28,11 +29,7 @@ export function headlessDeps(session: HeadlessSession): CutAgentDeps {
if (!doc) throw new Error(`No such skill. Available: ${AI_SKILL_INDEX.join(", ")}`);
return doc;
}
- if (UI_TOOLS.has(name))
- return {
- noEditor: true,
- note: "No editor page is attached to this session, so this tool had no effect. The project itself is unchanged — keep working from the editor state.",
- };
+ if (UI_TOOLS.has(name)) return { noEditor: true, note: DETACHED_UI_NOTE };
// The worker installs canvas and decoders at startup and these run as
// written. A process that came up without them says so plainly.
if (MEDIA_RUNTIME_TOOLS.has(name)) {
diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.ts
index 4ae47d87..14f19ffd 100644
--- a/site/src/cut/server/ai/headlessTools.ts
+++ b/site/src/cut/server/ai/headlessTools.ts
@@ -1,5 +1,6 @@
import { copyFile } from "node:fs/promises";
import { runAiTool, UI_TOOLS } from "../../lib/aiTools";
+import { DETACHED_MEDIA_ERROR, DETACHED_SESSION_ERROR, DETACHED_UI_NOTE } from "../../lib/chatResume";
import { mp4NameFor, setHostMediaStore } from "../../lib/mediaConvert";
import { setHostDocStore } from "../../lib/projectReference";
import { serializeDoc, useEditor } from "../../lib/store";
@@ -144,21 +145,10 @@ async function executeHeadlessTool(
): Promise<{ output?: unknown; errorText?: string }> {
if (UI_TOOLS.has(toolName))
return {
- output: {
- noEditor: true,
- note: "No editor page is attached to this session, so this tool had no effect. The project itself is unchanged — keep working from the editor state.",
- },
- };
- if (PAGE_MEDIA_TOOLS.has(toolName))
- return {
- errorText:
- "This tool reads media through the editor page and is unavailable while no tab is attached. Work from the editor state instead.",
- };
- if (PAGE_SESSION_TOOLS.has(toolName))
- return {
- errorText:
- "This tool runs with the editor page's hosted sign-in and is unavailable while no tab is attached. Finish the edit from the project state; the user can run it from the editor.",
+ output: { noEditor: true, note: DETACHED_UI_NOTE },
};
+ if (PAGE_MEDIA_TOOLS.has(toolName)) return { errorText: DETACHED_MEDIA_ERROR };
+ if (PAGE_SESSION_TOOLS.has(toolName)) return { errorText: DETACHED_SESSION_ERROR };
const doc = await ensureOpen(projectId);
if (!doc) return { errorText: "Project not found on this Mac." };
try {