Skip to content

Open source CapCut alternative.

TypeScript

perf(site): cut chat runs the first round while the gate classifies

David Lea6b147107e76
diff --git a/site/src/cut/lib/pi/cutAgent.ts b/site/src/cut/lib/pi/cutAgent.ts
index ce52cf37..3c177b9a 100644
--- a/site/src/cut/lib/pi/cutAgent.ts
+++ b/site/src/cut/lib/pi/cutAgent.ts
@@ -2,7 +2,7 @@ import type { UIMessage, UIMessageChunk } from "ai";
import { Agent, type AgentMessage } from "@earendil-works/pi-agent-core";
import type { Message, UserMessage } from "@earendil-works/pi-ai";
import { AI_TOOLS, attachedAssetsBlock, systemPrompt } from "@/cut/server/ai/catalog";
-import { parseTurnIntent, TURN_INTENT_PROMPT, turnIntentInput, type TurnIntent } from "../turnIntent";
+import { parseTurnIntent, turnIntentInput, turnIntentPrompt, type TurnIntent } from "../turnIntent";
import { enforceContextBudget } from "./contextBudget";
import { donkeyModel } from "./donkeyModel";
import { ledgerText, recordCall, type LedgerRecord } from "./mutationLedger";
@@ -12,7 +12,8 @@ import { subscribeUiChunks } from "./uiChunks";
// The chat turn runner on the pi agent harness. Each turn: the intent gate
// classifies the newest message (chat verdicts withhold every tool, simple
-// verdicts downgrade the model), the Agent loops model calls and tool
+// verdicts downgrade the model and narrow the tool catalog to the areas the
+// verdict names), the Agent loops model calls and tool
// executions against the live editor store, and the events stream out as the
// UIMessageChunks AiPanel already consumes. The thread's LLM context is pi's
// own message list, kept per thread in the session registry — past tool calls
@@ -44,7 +45,8 @@ export interface CutAgentDeps {
/** Round-budget overrides (the eval exercises auto-continue with a tiny
* budget; production runs the defaults). */
limits?: { roundBudget?: number; maxExtensions?: number };
- /** Eval instrumentation. */
+ /** Timing instrumentation: the eval asserts on it, production logs it at
+ * debug level. */
hooks?: {
onGate?: (intent: TurnIntent, ms: number, skipped: boolean) => void;
/** One LLM round settled: wall time, and time to its first visible delta. */
@@ -281,6 +283,8 @@ function sessionFor(threadId: string, history: UIMessage[]): AgentMessage[] {
return [...stored, ...legacySession(history.slice(firstMissing))];
}
+const GATE_PROMPT = turnIntentPrompt();
+
/** The turn's gate and router (ported from the legacy loop): judge the newest
* message; "chat" withholds every tool, "simple" downgrades the model. A
* message with attachments is complex by construction. Fails open. */
@@ -304,7 +308,7 @@ async function classifyTurnIntent(
{
donkeyProvider: "gemini",
model: deps.models.gate,
- instructions: TURN_INTENT_PROMPT,
+ instructions: GATE_PROMPT,
input: turnIntentInput(turns),
},
abortSignal
@@ -362,170 +366,257 @@ export function streamCutChat({
emit({ type: "start" });
try {
const gateStart = performance.now();
- const intentPromise = classifyTurnIntent(messages, deps, abortSignal);
const lastUser = messages.findLast((m) => m.role === "user");
const promptPromise = buildPrompt(lastUser, deps);
- const { intent, skipped } = await intentPromise;
- deps.hooks?.onGate?.(intent, performance.now() - gateStart, skipped);
- const roundModel = intent === "simple" ? deps.models.simple : model;
-
- const session = sessionFor(threadId, messages.filter((m) => m !== lastUser));
+ const verdictPromise = classifyTurnIntent(messages, deps, abortSignal);
+ void verdictPromise.then(({ intent, skipped }) =>
+ deps.hooks?.onGate?.(intent, performance.now() - gateStart, skipped)
+ );
- // The scene-plan money gate, enforced structurally: a plan created
- // this turn cannot be approved this turn, whatever the model decides.
- let scenePlannedThisTurn = false;
const roundBudget = deps.limits?.roundBudget ?? ROUND_BUDGET;
const maxExtensions = deps.limits?.maxExtensions ?? MAX_EXTENSIONS;
const roundCeiling = roundBudget * (maxExtensions + 1);
- let rounds = 0;
- let extensions = 0;
- // Everything this turn ran, harvested off the tool results in code.
- const records: LedgerRecord[] = [];
- const agent = new Agent({
- initialState: {
- systemPrompt: systemPrompt(),
- model: donkeyModel(roundModel),
- messages: session,
- tools: intent === "chat" ? [] : toAgentTools(AI_TOOLS, deps.execTool),
- },
- streamFn: makeDonkeyStream({
- post: deps.post,
- onAuthFail: deps.onAuthFail,
- noCreditsMessage: deps.noCreditsMessage,
- }),
- transformContext: async (msgs) => {
- const out = enforceContextBudget(pruneStaleMedia(pruneStaleSnapshots(msgs)));
- // The turn ledger rides every call as an ephemeral tail message —
- // the reply-writing call always sees the current record, and the
- // stored session never carries it.
- const ledger = ledgerText(records, deps.debris?.() ?? []);
- if (!ledger) return out;
- return [...out, { role: "user", content: ledger, timestamp: 0 }];
- },
- toolExecution: "sequential",
- beforeToolCall: async ({ toolCall }) => {
- if (rounds >= roundCeiling)
- return {
- block: true,
- // The first blocked round leaves the model one round to write
- // its reply; a model that requests tools again instead has its
- // batch terminated, ending the turn.
- terminate: rounds > roundCeiling,
- reason:
- "Step ceiling reached — no more tool calls this turn. Reply now: what is done and what remains.",
- };
- if (toolCall.name === "approve_scene" && scenePlannedThisTurn)
- return {
- block: true,
- reason:
- "This plan landed this turn — the user hasn't answered yet. Ask them to confirm (they can also click Approve on the plan card), and call approve_scene only after they say yes in a later message.",
- };
- return undefined;
- },
- afterToolCall: async ({ toolCall, result, isError }) => {
- if (toolCall.name === "generate_scene" && !isError) scenePlannedThisTurn = true;
- const details = result.details as DonkeyToolDetails | undefined;
- const errorText = isError
- ? (result.content.find((c) => c.type === "text") as { text?: string } | undefined)
- ?.text || "failed"
- : undefined;
- recordCall(records, toolCall.name, details?.response, errorText);
- return undefined;
- },
- });
+ /** One full agent lifecycle: build, subscribe, prompt, run to idle.
+ * The caller decides what the run streams to (live or a buffer) and
+ * finalizes only the run it keeps. */
+ const runAgentTurn = async ({
+ roundModel,
+ withTools,
+ send,
+ executionGate,
+ onAgent,
+ }: {
+ roundModel: string;
+ withTools: boolean;
+ send: (chunk: Record<string, unknown>) => void;
+ /** Awaited before any tool executes; false ends the batch unrun. */
+ executionGate?: () => Promise<boolean>;
+ onAgent?: (agent: Agent) => void;
+ }) => {
+ // The scene-plan money gate, enforced structurally: a plan created
+ // this turn cannot be approved this turn, whatever the model decides.
+ let scenePlannedThisTurn = false;
+ let rounds = 0;
+ let extensions = 0;
+ // Everything this turn ran, harvested off the tool results in code.
+ const records: LedgerRecord[] = [];
+
+ const agent = new Agent({
+ initialState: {
+ systemPrompt: systemPrompt(),
+ model: donkeyModel(roundModel),
+ messages: sessionFor(threadId, messages.filter((m) => m !== lastUser)),
+ tools: withTools ? toAgentTools(AI_TOOLS, deps.execTool) : [],
+ },
+ streamFn: makeDonkeyStream({
+ post: deps.post,
+ onAuthFail: deps.onAuthFail,
+ noCreditsMessage: deps.noCreditsMessage,
+ }),
+ transformContext: async (msgs) => {
+ const out = enforceContextBudget(pruneStaleMedia(pruneStaleSnapshots(msgs)));
+ // The turn ledger rides every call as an ephemeral tail message —
+ // the reply-writing call always sees the current record, and the
+ // stored session never carries it.
+ const ledger = ledgerText(records, deps.debris?.() ?? []);
+ if (!ledger) return out;
+ return [...out, { role: "user", content: ledger, timestamp: 0 }];
+ },
+ toolExecution: "sequential",
+ beforeToolCall: async ({ toolCall }) => {
+ if (executionGate && !(await executionGate()))
+ return {
+ block: true,
+ terminate: true,
+ reason: "Rerouted — this turn is restarting on the full model.",
+ };
+ if (rounds >= roundCeiling)
+ return {
+ block: true,
+ // The first blocked round leaves the model one round to write
+ // its reply; a model that requests tools again instead has its
+ // batch terminated, ending the turn.
+ terminate: rounds > roundCeiling,
+ reason:
+ "Step ceiling reached — no more tool calls this turn. Reply now: what is done and what remains.",
+ };
+ if (toolCall.name === "approve_scene" && scenePlannedThisTurn)
+ return {
+ block: true,
+ reason:
+ "This plan landed this turn — the user hasn't answered yet. Ask them to confirm (they can also click Approve on the plan card), and call approve_scene only after they say yes in a later message.",
+ };
+ return undefined;
+ },
+ afterToolCall: async ({ toolCall, result, isError }) => {
+ if (toolCall.name === "generate_scene" && !isError) scenePlannedThisTurn = true;
+ const details = result.details as DonkeyToolDetails | undefined;
+ const errorText = isError
+ ? (result.content.find((c) => c.type === "text") as { text?: string } | undefined)
+ ?.text || "failed"
+ : undefined;
+ recordCall(records, toolCall.name, details?.response, errorText);
+ return undefined;
+ },
+ });
+ onAgent?.(agent);
- const unsubscribeUi = subscribeUiChunks(agent, emit);
- // Round timing for the eval: one assistant message = one LLM round.
- const onRound = deps.hooks?.onRound;
- let roundStart = 0;
- let roundFirstDelta: number | null = null;
- const unsubscribeTiming = !onRound
- ? () => {}
- : agent.subscribe((event) => {
- if (
- event.type === "message_start" &&
- (event.message as Message).role === "assistant"
- ) {
- roundStart = performance.now();
- roundFirstDelta = null;
- } else if (
- event.type === "message_update" &&
- event.assistantMessageEvent.type === "text_delta" &&
- roundFirstDelta === null &&
- event.assistantMessageEvent.delta.trim()
- ) {
- roundFirstDelta = performance.now() - roundStart;
- } else if (
- event.type === "message_end" &&
- (event.message as Message).role === "assistant" &&
- roundStart > 0
- ) {
- onRound(performance.now() - roundStart, roundFirstDelta);
- roundStart = 0;
- }
- });
- const unsubscribeTurns = agent.subscribe((event) => {
- if (event.type !== "turn_end") return;
- const msg = event.message as Message;
- if (msg.role !== "assistant" || !msg.content.some((c) => c.type === "toolCall")) return;
- rounds++;
- if (rounds % roundBudget === 0 && rounds < roundCeiling && extensions < maxExtensions) {
- extensions++;
- deps.hooks?.onExtension?.(extensions);
- agent.steer({
- role: "user",
- content: `[budget] Round budget auto-extended (${extensions} of ${maxExtensions}). Keep working; finish the job or report the concrete blocker.`,
- timestamp: Date.now(),
- });
+ const unsubscribeUi = subscribeUiChunks(agent, send);
+ // Round timing for the eval: one assistant message = one LLM round.
+ const onRound = deps.hooks?.onRound;
+ let roundStart = 0;
+ let roundFirstDelta: number | null = null;
+ const unsubscribeTiming = !onRound
+ ? () => {}
+ : agent.subscribe((event) => {
+ if (
+ event.type === "message_start" &&
+ (event.message as Message).role === "assistant"
+ ) {
+ roundStart = performance.now();
+ roundFirstDelta = null;
+ } else if (
+ event.type === "message_update" &&
+ event.assistantMessageEvent.type === "text_delta" &&
+ roundFirstDelta === null &&
+ event.assistantMessageEvent.delta.trim()
+ ) {
+ roundFirstDelta = performance.now() - roundStart;
+ } else if (
+ event.type === "message_end" &&
+ (event.message as Message).role === "assistant" &&
+ roundStart > 0
+ ) {
+ onRound(performance.now() - roundStart, roundFirstDelta);
+ roundStart = 0;
+ }
+ });
+ const unsubscribeTurns = agent.subscribe((event) => {
+ if (event.type !== "turn_end") return;
+ const msg = event.message as Message;
+ if (msg.role !== "assistant" || !msg.content.some((c) => c.type === "toolCall")) return;
+ rounds++;
+ if (rounds % roundBudget === 0 && rounds < roundCeiling && extensions < maxExtensions) {
+ extensions++;
+ deps.hooks?.onExtension?.(extensions);
+ agent.steer({
+ role: "user",
+ content: `[budget] Round budget auto-extended (${extensions} of ${maxExtensions}). Keep working; finish the job or report the concrete blocker.`,
+ timestamp: Date.now(),
+ });
+ }
+ });
+ const onAbort = () => agent.abort();
+ abortSignal?.addEventListener("abort", onAbort);
+
+ try {
+ const prompt = { ...(await promptPromise) };
+ // A signal that aborted during the gate/prompt awaits fired before
+ // the listener existed; nothing has run, so the turn just ends.
+ if (!abortSignal?.aborted) {
+ await agent.prompt(prompt);
+ await agent.waitForIdle();
+ }
+ } finally {
+ abortSignal?.removeEventListener("abort", onAbort);
+ unsubscribeUi();
+ unsubscribeTiming();
+ unsubscribeTurns();
}
- });
- const onAbort = () => agent.abort();
- abortSignal?.addEventListener("abort", onAbort);
+ return { agent, rounds, send };
+ };
- try {
- const prompt = await promptPromise;
- // A signal that aborted during the gate/prompt awaits fired before
- // the listener existed; nothing has run, so the turn just ends.
- if (!abortSignal?.aborted) {
- await agent.prompt(prompt);
- await agent.waitForIdle();
+ /** Save and sign off the one run this turn keeps. */
+ const finalize = (run: {
+ agent: Agent;
+ rounds: number;
+ send: (chunk: Record<string, unknown>) => void;
+ }) => {
+ sessions.set(threadId, sanitizeSession(run.agent.state.messages));
+ // The step-limit handoff guarantee: a turn that ends at the ceiling
+ // with the model still requesting tools closes with readable text —
+ // the terminated batch alone would leave the reply on blocked tool
+ // chips with no sign-off.
+ if (run.rounds > roundCeiling && !abortSignal?.aborted) {
+ const last = [...run.agent.state.messages]
+ .reverse()
+ .find((m) => (m as Message).role === "assistant") as Message | undefined;
+ const spoke =
+ !!last &&
+ Array.isArray(last.content) &&
+ last.content.some((c) => c.type === "text" && c.text.trim());
+ if (!spoke) {
+ const id = crypto.randomUUID();
+ run.send({ type: "text-start", id });
+ run.send({
+ type: "text-delta",
+ id,
+ delta:
+ 'Paused at this turn\'s step limit with work still open. Say "keep going" to continue.',
+ });
+ run.send({ type: "text-end", id });
+ }
}
- } finally {
- abortSignal?.removeEventListener("abort", onAbort);
- unsubscribeUi();
- unsubscribeTiming();
- unsubscribeTurns();
- }
+ const errorMessage = run.agent.state.errorMessage;
+ if (errorMessage && !abortSignal?.aborted) {
+ run.send({ type: "error", errorText: errorMessage });
+ }
+ };
- sessions.set(threadId, sanitizeSession(agent.state.messages));
- // The step-limit handoff guarantee: a turn that ends at the ceiling
- // with the model still requesting tools closes with readable text —
- // the terminated batch alone would leave the reply on blocked tool
- // chips with no sign-off.
- if (rounds > roundCeiling && !abortSignal?.aborted) {
- const last = [...agent.state.messages]
- .reverse()
- .find((m) => (m as Message).role === "assistant") as Message | undefined;
- const spoke =
- !!last &&
- Array.isArray(last.content) &&
- last.content.some((c) => c.type === "text" && c.text.trim());
- if (!spoke) {
- const id = crypto.randomUUID();
- emit({ type: "text-start", id });
- emit({
- type: "text-delta",
- id,
- delta:
- 'Paused at this turn\'s step limit with work still open. Say "keep going" to continue.',
- });
- emit({ type: "text-end", id });
+ // Speculative first round: the turn goes out immediately on the light
+ // model with the full catalog while the gate classifies in parallel.
+ // Tool execution waits on the verdict and UI chunks buffer until it
+ // lands, so a chat or complex verdict aborts the run with nothing
+ // shown and nothing executed, and the turn restarts routed — while a
+ // simple verdict (the common case) keeps the run and pays no gate
+ // wall time at all. A message with attachments resolves complex
+ // instantly, so it runs routed from the start.
+ const attached = (lastUser?.metadata as { attachments?: unknown[] } | undefined)
+ ?.attachments;
+ let kept = false;
+ if (!(Array.isArray(attached) && attached.length > 0) && !abortSignal?.aborted) {
+ const buffer: Record<string, unknown>[] = [];
+ let mode: "buffering" | "live" | "discarded" = "buffering";
+ const send = (chunk: Record<string, unknown>) => {
+ if (mode === "live") emit(chunk);
+ else if (mode === "buffering") buffer.push(chunk);
+ };
+ let speculating: Agent | null = null;
+ const watcher = verdictPromise.then(({ intent }) => {
+ if (intent === "simple") {
+ mode = "live";
+ for (const c of buffer.splice(0)) emit(c);
+ return true;
+ }
+ mode = "discarded";
+ buffer.length = 0;
+ speculating?.abort();
+ return false;
+ });
+ const run = await runAgentTurn({
+ roundModel: deps.models.simple,
+ withTools: true,
+ send,
+ executionGate: async () => (await verdictPromise).intent === "simple",
+ onAgent: (a) => {
+ speculating = a;
+ },
+ });
+ if (await watcher) {
+ finalize(run);
+ kept = true;
}
}
- const errorMessage = agent.state.errorMessage;
- if (errorMessage && !abortSignal?.aborted) {
- emit({ type: "error", errorText: errorMessage });
+ if (!kept && !abortSignal?.aborted) {
+ const { intent } = await verdictPromise;
+ finalize(
+ await runAgentTurn({
+ roundModel: intent === "simple" ? deps.models.simple : model,
+ withTools: intent !== "chat",
+ send: emit,
+ })
+ );
}
} catch (err) {
if (!abortSignal?.aborted) {
diff --git a/site/src/cut/lib/pi/prodDeps.ts b/site/src/cut/lib/pi/prodDeps.ts
index f973c52a..44957ba2 100644
--- a/site/src/cut/lib/pi/prodDeps.ts
+++ b/site/src/cut/lib/pi/prodDeps.ts
@@ -67,5 +67,13 @@ export function productionDeps(): CutAgentDeps {
debris: currentDebris,
onAuthFail: () => useGenerate.getState().probe(),
noCreditsMessage: NO_CREDITS_MESSAGE,
+ hooks: {
+ onGate: (intent, ms, skipped) =>
+ console.debug(`[chat] gate ${intent} ${Math.round(ms)}ms${skipped ? " skipped" : ""}`),
+ onRound: (ms, firstDeltaMs) =>
+ console.debug(
+ `[chat] round ${Math.round(ms)}ms${firstDeltaMs === null ? "" : `, first delta ${Math.round(firstDeltaMs)}ms`}`
+ ),
+ },
};
}
diff --git a/site/src/cut/lib/turnIntent.ts b/site/src/cut/lib/turnIntent.ts
index 9217b28b..bb71e2cf 100644
--- a/site/src/cut/lib/turnIntent.ts
+++ b/site/src/cut/lib/turnIntent.ts
@@ -5,18 +5,23 @@
// filler that requests nothing) runs with no tool declarations at all, so
// unsolicited edits are impossible by construction. A "simple" turn (one
// self-contained ask) runs on the light chat model; a "complex" turn (a
-// composed job) runs on the full one.
+// composed job) runs on the full model. The turn runner overlaps this call
+// with the first model round and holds tool execution until the verdict
+// lands, so the gate costs a simple turn no wall time.
export type TurnIntent = "chat" | "simple" | "complex";
-export const TURN_INTENT_PROMPT = `You gate and route the tools of an AI assistant built into a video editor. Read the conversation and judge only its newest user message.
+/** The gate's instructions. */
+export function turnIntentPrompt(): string {
+ return `You gate an AI assistant built into a video editor. Read the conversation and judge only its newest user message.
Reply with exactly one word:
chat — pure social filler (a greeting, thanks, a sign-off, an acknowledgement) that requests nothing.
simple — one self-contained ask: a single edit, a single generation, or a question to answer. Terse follow-ups ("yes", "do it", "the second one") that confirm one pending action are simple.
-complex — a composed job: several edits across the timeline, cutting or reorganizing many clips, assembling media into a cut, or anything needing a plan across steps. An edit aimed at "this clip / that one / it" with no named target is also complex — resolving the referent takes the full model.
+complex — a composed job: several edits across the timeline, cutting or reorganizing many clips, assembling media into a cut, or anything needing a plan across steps. A sweep phrased as one ask still fans out into many cuts — removing filler words, tightening silences, cutting every pause — and is complex. An edit aimed at "this clip / that one / it" with no named target is also complex — resolving the referent takes the full model. So is an edit whose targets are picked by description — "the grey ones", "the short clips" — since deciding which items match takes the full model.
If unsure between chat and the others, reply simple. If unsure between simple and complex, reply complex.`;
+}
/** The conversation as plain text turns for the classifier — recent turns ride
* along so follow-ups keep their referent, and each is capped so a pasted wall
@@ -30,10 +35,10 @@ export function turnIntentInput(
.map((t) => ({ role: t.role, content: [{ text: t.text.slice(0, 2000) }] }));
}
-/** Deterministic read of the classifier's one-word verdict. Only a clear
- * "chat" withholds tools, and only a clear "simple" takes the light model —
- * a garbled reply runs the full model with tools, so a classifier hiccup
- * never blocks or downgrades a real request. */
+/** Deterministic read of the classifier's verdict line. Only a clear "chat"
+ * withholds tools, and only a clear "simple" takes the light model — a garbled
+ * reply runs the full model with every tool, so a classifier hiccup never
+ * blocks or downgrades a real request. */
export function parseTurnIntent(outputText: string | undefined): TurnIntent {
const t = (outputText ?? "").trim();
if (/^\W*chat\b/i.test(t)) return "chat";