Skip to content

Open source CapCut alternative.

TypeScript

fix(site): gate only finished CLI turns [rebuild]

David Le91bc705fa808
diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsx
index 0bb30e61..3eb14953 100644
--- a/site/src/cut/components/AiPanel.tsx
+++ b/site/src/cut/components/AiPanel.tsx
@@ -95,7 +95,7 @@ import {
useGenerate,
useSignedIn,
} from "@/cut/lib/generate";
-import { useCreditsRecheck, useOutOfCredits } from "@/cut/lib/hosted";
+import { knownBalance, useCreditsRecheck, useOutOfCredits } from "@/cut/lib/hosted";
import { cutChatLive, dropPiSession, foldIntoCutChat, hydratePiSession, readPiSession, streamCutChat, triageQueuedMessages } from "@/cut/lib/pi/cutAgent";
import { dueForRetriage, toolProgress } from "@/cut/lib/queueTriage";
import { registerQueueSink } from "@/cut/lib/chatQueue";
@@ -109,9 +109,10 @@ import { recoverSceneCall } from "@/cut/lib/chatRecovery";
import { replayChatStream } from "@/cut/lib/chatReplay";
import { INTERRUPTED_ERROR, isResumeMessage, RESUME_LIMIT, resumePrompt, scrubLostTurn, unfinishedAsk, type ResumeMetadata, type TurnSettled } from "@/cut/lib/chatResume";
import { answerEngineGate, claimEngineTool, cancelEngineChat, foldIntoEngineChat } from "@/cut/lib/engineChat";
+import { messageText } from "@/cut/lib/messageText";
import { EngineGate, engineInstant, handledSince, judgeEngineTurn } from "@/cut/lib/pi/engineTurn";
import type { EngineRoute } from "@/cut/server/ai/turnCatalog";
-import { fetchCreditBalance } from "@/queries/credits";
+import { creditBalanceQueryKey, fetchCreditBalance } from "@/queries/credits";
import { projectBackend } from "@/cut/lib/residency";
import { beginBrowserChat, browserChatRunning, cancelBrowserChat, watchBrowserChatTurns } from "@/cut/lib/browserChatTurns";
import { putCloudThread } from "@/cut/lib/chatCloud";
@@ -144,6 +145,13 @@ interface ModelsInfo {
providers: Record<string, { available: boolean; note: string; installed?: boolean }>;
}
+/** What an engine send carries past the messages: the editor snapshot the
+ * turn was judged against, and its route when it was judged. */
+interface EngineSend {
+ context: unknown;
+ route: EngineRoute | undefined;
+}
+
/** A saved chat thread, persisted per project in localStorage. */
interface ChatThread {
id: string;
@@ -524,7 +532,7 @@ export function AiPanel({
const asks: Record<string, string> = {};
for (const id of Object.keys(runningThreads)) {
const ask = threads.find((t) => t.id === id)?.messages.findLast((m) => m.role === "user");
- const text = ask?.parts.map((p) => (p.type === "text" ? p.text : "")).join("").trim();
+ const text = ask ? messageText(ask) : "";
if (text) asks[id] = text;
}
for (const [id, seed] of Object.entries(seeds)) asks[id] = seed.text;
@@ -902,18 +910,21 @@ function ChipsReveal({ open, children }: { open: boolean; children: ReactNode })
);
}
-/** Running sessions stay mounted for the lifetime of their editor. */
/** Whether this account's turns are judged: signed in, with credits. An
- * account with none chats on its own CLI model, which decides alone. */
+ * account with none chats on its own CLI model, which decides alone. A
+ * balance a charged call reported after the last read decides over it. */
async function accountJudges(queryClient: QueryClient): Promise<boolean> {
if (useGenerate.getState().signedIn !== true || useOutOfCredits.getState().out) return false;
try {
- return Number((await fetchCreditBalance(queryClient)).balanceMicros) > 0;
+ const read = await fetchCreditBalance(queryClient);
+ const readAt = queryClient.getQueryState(creditBalanceQueryKey)?.dataUpdatedAt ?? 0;
+ return knownBalance(read.balance, readAt) > 0;
} catch {
return false;
}
}
+/** Running sessions stay mounted for the lifetime of their editor. */
function ChatSession({
projectId,
threadId,
@@ -1054,7 +1065,6 @@ function ChatSession({
const queryClient = useQueryClient();
// The judged route the next engine send carries, and the quality gate of
// the engine turn in flight. Both stay empty for an unjudged turn.
- const engineRouteRef = useRef<EngineRoute | undefined>(undefined);
const engineGateRef = useRef<EngineGate | null>(null);
// Resume from the saved thread when this id exists in history.
const [initialThread] = useState<ChatThread | undefined>(() =>
@@ -1126,9 +1136,11 @@ function ChatSession({
// this Mac's engine — even for a cloud project — so the URL comes from
// localBackend (read after the origin resolves), which carries the
// account scope the engine requires on every data route.
- prepareSendMessagesRequest: async ({ messages }) => {
- // The route was judged in sendMessages; an unjudged turn (an account
- // with no credits) carries none and the model reads skills itself.
+ // sendMessages hands over the editor snapshot it judged against and
+ // the route; an unjudged turn (an account with no credits) carries
+ // none and the model reads skills itself.
+ prepareSendMessagesRequest: async ({ messages, body }) => {
+ const { context, route } = body as EngineSend;
await engineReady();
return {
api: localBackend.url("/api/cut/ai/chat"),
@@ -1137,9 +1149,9 @@ function ChatSession({
runtime: chatRuntime(),
messages,
model: currentModel(),
- context: productionDeps(projectId).buildContext(),
+ context,
providerSession: sessionFor(currentModel()),
- route: engineRouteRef.current,
+ route,
handled: handledSince(messages),
},
};
@@ -1175,20 +1187,21 @@ function ChatSession({
// An account with credits has the turn judged before the send: one
// known action runs here with no model round, and the rest go out
// routed, with the gate answering when the CLI signs off.
+ const deps = productionDeps(projectId, options.abortSignal);
const decision = (await accountJudges(queryClient))
- ? await judgeEngineTurn(options.messages, productionDeps(projectId), options.abortSignal)
+ ? await judgeEngineTurn(options.messages, deps, options.abortSignal)
: null;
if (decision?.instant) {
- const handled = await engineInstant(decision.instant, productionDeps(projectId, options.abortSignal));
+ const handled = await engineInstant(decision.instant, deps);
if (handled) {
setClientTools(true);
return handled;
}
}
- engineRouteRef.current = decision?.route;
const ask = options.messages.findLast((m) => m.role === "user" && !isResumeMessage(m));
- engineGateRef.current = decision?.route.gate ? new EngineGate(ask?.parts.map((p) => (p.type === "text" ? p.text : "")).join("").trim() ?? "") : null;
- return engine().sendMessages(options);
+ engineGateRef.current = decision?.route.gate ? new EngineGate(ask ? messageText(ask) : "") : null;
+ const send: EngineSend = { context: deps.buildContext(), route: decision?.route };
+ return engine().sendMessages({ ...options, body: { ...options.body, ...send } });
},
reconnectToStream: async (options) => {
if (provider(currentModel()) === "gemini") return Promise.resolve(null);
@@ -1241,7 +1254,7 @@ function ChatSession({
const thread: ChatThread = {
...previous,
id: threadId,
- title: firstUser?.parts.map((part) => part.type === "text" ? part.text : "").join("").trim().slice(0, 80) || "New chat",
+ title: (firstUser ? messageText(firstUser) : "").slice(0, 80) || "New chat",
updatedAt: Date.now(),
completedMessageId: completedMessageId.current,
turnSettled: turnSettled.current,
@@ -1274,8 +1287,10 @@ function ChatSession({
const gate = engineGateRef.current;
const sessionKey = sessionKeyRef.current;
const signal = toolAbort.current.signal;
+ if (!sessionKey) return;
void (async () => {
- const steer = gate ? await gate.steer(reply, productionDeps(projectId), signal).catch(() => null) : null;
+ const steer = gate ? await gate.steer(gateId, reply, productionDeps(projectId), signal).catch(() => null) : null;
+ if (steer === undefined) return;
await answerEngineGate(sessionKey, gateId, steer).catch(() => {});
})();
}
@@ -1467,12 +1482,7 @@ function ChatSession({
if (messages.length === 0) return;
if (receivedThread.current) { receivedThread.current = false; return; }
const firstUser = messages.find((m) => m.role === "user");
- const title =
- firstUser?.parts
- .map((p) => (p.type === "text" ? p.text : ""))
- .join("")
- .trim()
- .slice(0, 80) || "New chat";
+ const title = (firstUser ? messageText(firstUser) : "").slice(0, 80) || "New chat";
pendingThread.current = {
id: threadId,
title,
@@ -1866,7 +1876,7 @@ function ChatSession({
// reply keeps the calls that landed and nothing else.
setMessages((current) => scrubLostTurn(current, ask.id));
const metadata: ResumeMetadata = { resume: true, ...(live.length > 0 ? { attachments: live } : {}) };
- runningAskRef.current = ask.parts.map((p) => (p.type === "text" ? p.text : "")).join("").trim();
+ runningAskRef.current = messageText(ask);
dispatchChat(() => sendMessage({ text: resumePrompt(ask), metadata }));
}, [status, busy, requests, dispatchChat, readOnly, projectLoaded, currentAvailable, model, signedIn, info, reconnect, resumeOwed, resumeStream, setMessages, sendMessage, projectId, threadId]);
diff --git a/site/src/cut/lib/chatResume.ts b/site/src/cut/lib/chatResume.ts
index a8084d60..b9e7f4fb 100644
--- a/site/src/cut/lib/chatResume.ts
+++ b/site/src/cut/lib/chatResume.ts
@@ -1,4 +1,5 @@
import type { UIMessage } from "ai";
+import { messageText } from "./messageText";
// A turn the page lost — a reload, a closed tab, a dropped connection, or a
// tool that needed the page while no tab was attached — resumes itself when
@@ -78,14 +79,10 @@ export function unfinishedAsk(
return ask;
}
-export function askText(ask: UIMessage): string {
- return ask.parts.map((part) => (part.type === "text" ? part.text : "")).join("").trim();
-}
-
/** The continuation a resumed turn opens with. It carries the ask itself, so
* a provider session that never received it still knows the job. */
export function resumePrompt(ask: UIMessage): string {
- const text = askText(ask);
+ const text = messageText(ask);
return [
"The editor page went away before your reply to this message finished:",
"",
diff --git a/site/src/cut/lib/engineChat.ts b/site/src/cut/lib/engineChat.ts
index 08f6d863..ad655ca1 100644
--- a/site/src/cut/lib/engineChat.ts
+++ b/site/src/cut/lib/engineChat.ts
@@ -34,7 +34,7 @@ export async function foldIntoEngineChat(
/** Answer the engine's quality gate for a signing-off turn: the steer for its
* next pass, or null to let it close. A gate the engine no longer holds is
* dropped. */
-export async function answerEngineGate(sessionKey: string | null, gateId: string, steer: string | null): Promise<void> {
+export async function answerEngineGate(sessionKey: string, gateId: string, steer: string | null): Promise<void> {
await localBackend.fetch("/api/cut/ai/gate", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ sessionKey, gateId, steer }),
diff --git a/site/src/cut/lib/hosted.test.ts b/site/src/cut/lib/hosted.test.ts
new file mode 100644
index 00000000..8c6691ea
--- /dev/null
+++ b/site/src/cut/lib/hosted.test.ts
@@ -0,0 +1,24 @@
+import { describe, expect, test } from "bun:test";
+import { knownBalance, reportBalance, useHostedBalance } from "./hosted";
+
+// The balance the page decides from: a read from the balance route, or the
+// balance a charged call reported after it.
+
+describe("knownBalance", () => {
+ test("a balance reported after the read wins", () => {
+ useHostedBalance.setState({ balance: null, reportedAt: 0 });
+ const readAt = Date.now() - 1000;
+ reportBalance("0.000000");
+ expect(knownBalance("5.000000", readAt)).toBe(0);
+ });
+
+ test("a read newer than the reported balance wins", () => {
+ reportBalance("0.000000");
+ expect(knownBalance("10.000000", Date.now() + 1)).toBe(10);
+ });
+
+ test("with nothing reported the read stands", () => {
+ useHostedBalance.setState({ balance: null, reportedAt: 0 });
+ expect(knownBalance("2.500000", 1)).toBe(2.5);
+ });
+});
diff --git a/site/src/cut/lib/hosted.ts b/site/src/cut/lib/hosted.ts
index fa62bc5d..bad3cf74 100644
--- a/site/src/cut/lib/hosted.ts
+++ b/site/src/cut/lib/hosted.ts
@@ -47,11 +47,19 @@ function setOut(out: boolean) {
* readout follows every call without a request of its own. `settled` counts
* calls that answered with no balance, for which the readout asks the balance
* route once the charge has had a moment to land. */
-export const useHostedBalance = create<{ balance: string | null; settled: number }>(() => ({
+export const useHostedBalance = create<{ balance: string | null; reportedAt: number; settled: number }>(() => ({
balance: null,
+ reportedAt: 0,
settled: 0,
}));
+/** The account's balance in dollars: a read from the balance route at
+ * `readAt`, unless a charged call reported a balance after it. */
+export function knownBalance(read: string, readAt: number): number {
+ const { balance, reportedAt } = useHostedBalance.getState();
+ return Number(balance !== null && reportedAt > readAt ? balance : read);
+}
+
/** A hosted call that answered with no balance was charged after its
* response left; the re-read waits for the charge to land. */
const SETTLE_DELAY_MS = 1500;
@@ -73,7 +81,7 @@ export function reportBalance(balance: string | undefined) {
useHostedBalance.setState((s) => ({ settled: s.settled + 1 }));
return;
}
- useHostedBalance.setState({ balance });
+ useHostedBalance.setState({ balance, reportedAt: Date.now() });
}
async function noteBalance(res: Response) {
diff --git a/site/src/cut/lib/messageText.ts b/site/src/cut/lib/messageText.ts
new file mode 100644
index 00000000..b9f0e66a
--- /dev/null
+++ b/site/src/cut/lib/messageText.ts
@@ -0,0 +1,6 @@
+import type { UIMessage } from "ai";
+
+/** A chat message's text parts, joined and trimmed. */
+export function messageText(message: UIMessage): string {
+ return message.parts.map((p) => (p.type === "text" ? p.text : "")).join("").trim();
+}
diff --git a/site/src/cut/lib/pi/cutAgent.ts b/site/src/cut/lib/pi/cutAgent.ts
index d1d24013..4d3dfd46 100644
--- a/site/src/cut/lib/pi/cutAgent.ts
+++ b/site/src/cut/lib/pi/cutAgent.ts
@@ -29,6 +29,7 @@ import {
type ResolvedAction,
} from "../instantAction";
import { askJudge } from "../judge";
+import { messageText } from "../messageText";
import {
placeQueuedRows,
queueTriageQuestions,
@@ -39,7 +40,6 @@ import {
type TriageRow,
} from "../queueTriage";
import {
- editorSlice,
routeTurn,
TURN_JUDGE_QUESTIONS,
type CutJudgeSettings,
@@ -47,21 +47,13 @@ import {
type TurnJudgeAnswers,
type TurnRoute,
} from "../turnJudge";
-import {
- QUALITY_QUESTIONS,
- QUALITY_STEER_PREFIX,
- qualityState,
- qualityVerdict,
- recordLook,
- type QualityAnswers,
- type QualityStep,
- type WatchedSource,
-} from "../turnQuality";
+import { QUALITY_STEER_PREFIX, type QualityStep } from "../turnQuality";
import { enforceContextBudget } from "./contextBudget";
import { donkeyModel, type ChatThinkingLevel } from "./donkeyModel";
-import { isMutatingTool, ledgerText, recordCall, type LedgerRecord } from "./mutationLedger";
+import { ledgerText } from "./mutationLedger";
import { makeDonkeyStream, type DonkeyToolDetails, type PostFn, type WireCarrier, type WirePart } from "./donkeyStream";
import { toAgentTools, toToolResult, uiToolOutput, type ExecTool } from "./tools";
+import { TurnGate } from "./turnGate";
import { subscribeUiChunks } from "./uiChunks";
// The chat turn runner on the pi agent harness. Each turn: one typed judgment
@@ -406,10 +398,7 @@ function toolsRanIn(m: UIMessage): string[] {
function legacySession(history: UIMessage[]): AgentMessage[] {
const out: AgentMessage[] = [];
for (const m of history) {
- let text = m.parts
- .map((p) => (p.type === "text" ? p.text : ""))
- .join("")
- .trim();
+ let text = messageText(m);
if (m.role === "user") {
const meta = (m.metadata as { attachments?: unknown[] } | undefined)?.attachments;
if (Array.isArray(meta) && meta.length > 0) text += attachedAssetsBlock(meta);
@@ -577,10 +566,7 @@ async function buildPrompt(
deps: CutAgentDeps,
context: unknown = deps.buildContext()
): Promise<UserMessage & WireCarrier> {
- let text = (lastUser?.parts ?? [])
- .map((p) => (p.type === "text" ? p.text : ""))
- .join("")
- .trim();
+ let text = lastUser ? messageText(lastUser) : "";
const wireParts: WirePart[] = [];
const meta = (lastUser?.metadata as { attachments?: unknown[] } | undefined)?.attachments;
if (Array.isArray(meta) && meta.length > 0) {
@@ -685,43 +671,6 @@ async function runInstantAction(
return true;
}
-/** What the turn has looked at and run, as a mark. A turn the gate sent back
- * that returns with the same mark has stopped moving. */
-export function workMark(records: LedgerRecord[], looks: Map<string, WatchedSource>): string {
- return [
- records.length,
- ...[...looks.values()].map((l) => `${l.passes}:${l.coveredTo}:${l.observed.length}`),
- ].join("|");
-}
-
-/** The turn's own work, judged before it closes. Fails open: a judgment that
- * cannot be asked lets the turn sign off. */
-export async function gateVerdict(
- reply: string,
- request: string,
- records: LedgerRecord[],
- looks: Map<string, WatchedSource>,
- deps: CutAgentDeps,
- settings: CutJudgeSettings,
- abortSignal?: AbortSignal
-) {
- const work = {
- request,
- reply: reply.trim(),
- ran: [...new Set(records.filter((r) => !r.error).map((r) => r.name))],
- failed: [...new Set(records.filter((r) => r.error).map((r) => `${r.name} (${r.error})`))],
- mutated: records.some((r) => !r.error && isMutatingTool(r.name)),
- sources: [...looks.values()],
- editor: editorSlice(deps.buildContext()),
- };
- try {
- const { answers } = await askJudge(deps.judge, qualityState(work), QUALITY_QUESTIONS, abortSignal);
- return qualityVerdict(answers as unknown as QualityAnswers, work, settings);
- } catch {
- return null;
- }
-}
-
/** One chat turn on the pi harness, streamed as UI chunks. Same contract as
* the legacy streamGeminiChat, plus the thread id that keys the session. */
export function streamCutChat({
@@ -747,10 +696,7 @@ export function streamCutChat({
try {
const gateStart = performance.now();
const lastUser = messages.findLast((m) => m.role === "user");
- const askText = (lastUser?.parts ?? [])
- .map((p) => (p.type === "text" ? p.text : ""))
- .join("")
- .trim();
+ const askText = lastUser ? messageText(lastUser) : "";
const context = deps.buildContext();
const promptPromise = buildPrompt(lastUser, deps, context);
const settings = deps.judgeSettings ?? cutJudge();
@@ -805,16 +751,8 @@ export function streamCutChat({
let scenePlannedThisTurn = false;
let rounds = 0;
let extensions = 0;
- // Everything this turn ran, harvested off the tool results in code.
- const records: LedgerRecord[] = [];
- // What this turn has looked at, and how many times the quality gate
- // has already sent it back to work.
- const looks = new Map<string, WatchedSource>();
- let gateRounds = 0;
- // What the turn had looked at and run the last time the gate held
- // it. A turn that comes back with the same record has stopped
- // moving, and sending it back again would only spin.
- let gateMark = "";
+ // What this turn ran and looked at, judged when it signs off.
+ const gate = new TurnGate(askText);
// The catalog this run declares. The speculative run waits a bounded
// moment for the route; a route that has landed narrows the tools
@@ -877,7 +815,7 @@ export function streamCutChat({
// The skill the judge attached rides the same way, from the
// first round it is known for.
const skill = withTools && settings.skillSuggestion && settled ? skillRelevanceBlock(settled.skill) : null;
- const ledger = ledgerText(records, deps.debris?.() ?? []);
+ const ledger = ledgerText(gate.records, deps.debris?.() ?? []);
const tail = [skill, ledger].filter((t): t is string => !!t).join("\n\n");
if (!tail) return out;
return [...out, { role: "user", content: tail, timestamp: 0 }];
@@ -915,8 +853,7 @@ export function streamCutChat({
? (result.content.find((c) => c.type === "text") as { text?: string } | undefined)
?.text || "failed"
: undefined;
- recordCall(records, toolCall.name, details?.response, errorText);
- if (!isError) recordLook(looks, toolCall.name, details?.response);
+ gate.record(toolCall.name, details?.response, errorText);
return undefined;
},
// The quality gate: a round that asks for no tools is the turn
@@ -935,28 +872,16 @@ export function streamCutChat({
!withTools ||
abortSignal?.aborted ||
cancelled?.() ||
- gateRounds >= settings.qualityRounds ||
- !settings.qualityGate ||
rounds >= roundCeiling - 1 ||
(Array.isArray(msg.content) && msg.content.some((c) => c.type === "toolCall"))
)
return false;
- // The gate judges work grounded in footage: a turn that looked
- // at a source can be held to what it saw. A turn that opened no
- // source is measurable too when it changed nothing — an ask for
- // work answered in words leaves the project where it was, and
- // that is a fact the record carries.
- if (looks.size === 0 && records.some((r) => !r.error && isMutatingTool(r.name))) return false;
- const mark = workMark(records, looks);
- if (mark === gateMark) return false; // sent back once, nothing moved
const replyText = Array.isArray(msg.content)
? msg.content.map((c) => (c.type === "text" ? c.text : "")).join("")
: "";
- const verdict = await gateVerdict(replyText, askText, records, looks, deps, settings, abortSignal);
+ const verdict = await gate.hold(replyText, deps, settings, abortSignal);
if (!verdict) return false;
- gateMark = mark;
- gateRounds++;
- deps.hooks?.onQualityGate?.(verdict.step, gateRounds);
+ deps.hooks?.onQualityGate?.(verdict.step, gate.held);
agent.followUp({ role: "user", content: verdict.steer, timestamp: Date.now() });
return false;
},
diff --git a/site/src/cut/lib/pi/engineTurn.test.ts b/site/src/cut/lib/pi/engineTurn.test.ts
index 1dc65da2..3af8bd49 100644
--- a/site/src/cut/lib/pi/engineTurn.test.ts
+++ b/site/src/cut/lib/pi/engineTurn.test.ts
@@ -20,12 +20,13 @@ const WATCH_RESULT = {
source: { assetId: "a1", name: "reference.mp4", duration: 71.6 },
};
-function routeAnswers(areas: string[]) {
+function routeAnswers(areas: string[], intent: "chat" | "complex") {
+ const sure = (choice: string) => (choice === intent ? 0.9 : 0.05);
const out: Record<string, unknown> = {
intent: {
type: "choice",
- choice: "complex",
- probabilities: { chat: 0.02, simple: 0.08, complex: 0.9 },
+ choice: intent,
+ probabilities: { chat: sure("chat"), simple: sure("simple"), complex: sure("complex") },
confidence: 0.9,
},
skill: { type: "choice", choice: "none", probabilities: { none: 0.9 }, confidence: 0.9 },
@@ -50,7 +51,9 @@ function qualityAnswers(finished: number) {
};
}
-function deps(opts: { finished?: number; delayMs?: number; settings?: Partial<CutAgentDeps["judgeSettings"]> } = {}) {
+function deps(
+ opts: { finished?: number; delayMs?: number; intent?: "chat" | "complex"; settings?: Partial<CutAgentDeps["judgeSettings"]> } = {}
+) {
const asks: string[][] = [];
const d: CutAgentDeps = {
post: async () => new Response(null, { status: 500 }),
@@ -58,7 +61,7 @@ function deps(opts: { finished?: number; delayMs?: number; settings?: Partial<Cu
const questions = Object.keys((payload as { questions?: Record<string, unknown> }).questions ?? {});
asks.push(questions);
if (opts.delayMs) await new Promise((r) => setTimeout(r, opts.delayMs));
- const answers = questions.includes("finished") ? qualityAnswers(opts.finished ?? 0.9) : routeAnswers(["timeline"]);
+ const answers = questions.includes("finished") ? qualityAnswers(opts.finished ?? 0.9) : routeAnswers(["timeline"], opts.intent ?? "complex");
return new Response(JSON.stringify({ model: "jev-latest", answers, usage: {} }), {
status: 200,
headers: { "Content-Type": "application/json" },
@@ -86,6 +89,13 @@ describe("judgeEngineTurn", () => {
expect(decision?.instant).toBeNull();
});
+ test("a chat verdict carries no gate", async () => {
+ const { deps: d } = deps({ intent: "chat" });
+ const decision = await judgeEngineTurn([user("u1", "what does ripple delete do?")], d);
+ expect(decision?.route.intent).toBe("chat");
+ expect(decision?.route.gate).toBe(false);
+ });
+
test("skill suggestion off leaves the skill out of the route", async () => {
const { deps: d } = deps({ settings: { skillSuggestion: false } });
const decision = await judgeEngineTurn([user("u1", "tighten every cut")], d);
@@ -170,9 +180,9 @@ describe("EngineGate", () => {
const { deps: d, asks } = deps({ finished: 0.1 });
const gate = new EngineGate("what happens in this video?");
gate.record("watch_video", WATCH_RESULT, undefined);
- const steer = await gate.steer("It opens on a yellow title.", d);
+ const steer = await gate.steer("g1", "It opens on a yellow title.", d);
expect(steer).toContain("from=20");
- expect(await gate.steer("It opens on a yellow title.", d)).toBeNull();
+ expect(await gate.steer("g2", "It opens on a yellow title.", d)).toBeNull();
expect(asks.filter((q) => q.includes("finished")).length).toBe(1);
});
@@ -180,7 +190,7 @@ describe("EngineGate", () => {
const { deps: d, asks } = deps({ finished: 0.01 });
const gate = new EngineGate("mute the first clip");
gate.record("set_clip_muted", { id: "c1", muted: true }, undefined);
- expect(await gate.steer("Muted it.", d)).toBeNull();
+ expect(await gate.steer("g1", "Muted it.", d)).toBeNull();
expect(asks.length).toBe(0);
});
@@ -188,7 +198,17 @@ describe("EngineGate", () => {
const { deps: d, asks } = deps({ finished: 0.1, settings: { qualityGate: false } });
const gate = new EngineGate("what happens in this video?");
gate.record("watch_video", WATCH_RESULT, undefined);
- expect(await gate.steer("It opens on a yellow title.", d)).toBeNull();
+ expect(await gate.steer("g1", "It opens on a yellow title.", d)).toBeNull();
expect(asks.length).toBe(0);
});
+
+ test("a replayed gate the turn already answered is not judged again", async () => {
+ const { deps: d, asks } = deps({ finished: 0.1 });
+ const gate = new EngineGate("what happens in this video?");
+ gate.record("watch_video", WATCH_RESULT, undefined);
+ expect(await gate.steer("g1", "It opens on a yellow title.", d)).toContain("from=20");
+ gate.record("watch_video", { ...WATCH_RESULT, coveredTo: 40 }, undefined);
+ expect(await gate.steer("g1", "It opens on a yellow title.", d)).toBeUndefined();
+ expect(asks.filter((q) => q.includes("finished")).length).toBe(1);
+ });
});
diff --git a/site/src/cut/lib/pi/engineTurn.ts b/site/src/cut/lib/pi/engineTurn.ts
index 38f6b8dc..f7d69ce5 100644
--- a/site/src/cut/lib/pi/engineTurn.ts
+++ b/site/src/cut/lib/pi/engineTurn.ts
@@ -2,10 +2,10 @@ import type { UIMessage, UIMessageChunk } from "ai";
import type { EngineRoute, HandledAsk } from "@/cut/server/ai/turnCatalog";
import { cutJudge } from "../chatRuntime";
import type { ResolvedAction } from "../instantAction";
-import { recordLook, type WatchedSource } from "../turnQuality";
-import { gateVerdict, judgeTurnAndAction, playInstant, workMark, type CutAgentDeps } from "./cutAgent";
-import { isMutatingTool, recordCall, type LedgerRecord } from "./mutationLedger";
+import { messageText } from "../messageText";
+import { judgeTurnAndAction, playInstant, type CutAgentDeps } from "./cutAgent";
import { toToolResult } from "./tools";
+import { TurnGate } from "./turnGate";
// A Claude or Codex turn judged the way a Gemini turn is. The page asks the
// judge before the send: a turn settled to one known action runs in the
@@ -18,7 +18,7 @@ import { toToolResult } from "./tools";
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
/** What the judgment settled for an engine turn. */
-export interface EngineDecision {
+interface EngineDecision {
route: EngineRoute;
instant: ResolvedAction | null;
}
@@ -43,7 +43,9 @@ export async function judgeEngineTurn(
intent: route.intent,
areas: route.areas,
...(settings.skillSuggestion && { skill: route.skill }),
- gate: settings.qualityGate,
+ // A chat verdict runs with no tools, so it has no work to hold up: the
+ // hosted loop skips the gate for it too.
+ gate: settings.qualityGate && route.intent !== "chat",
},
instant: landed.instant,
};
@@ -87,7 +89,7 @@ export function handledSince(messages: UIMessage[]): HandledAsk[] {
break;
}
const before = messages[i - 1];
- const ask = before?.role === "user" ? before.parts.map((p) => (p.type === "text" ? p.text : "")).join("").trim() : "";
+ const ask = before?.role === "user" ? messageText(before) : "";
out.unshift({ ask, ...handled });
}
return out;
@@ -97,49 +99,33 @@ export function handledSince(messages: UIMessage[]): HandledAsk[] {
* The quality gate for one engine turn. The page executes every tool call
* the CLI makes, so it keeps the turn's record here; when the engine asks at
* sign-off, the record and the reply are judged the way the hosted loop
- * judges them, under the same rounds cap and the same stand-downs.
+ * judges them.
*/
export class EngineGate {
- private records: LedgerRecord[] = [];
- private looks = new Map<string, WatchedSource>();
- private rounds = 0;
- private mark = "";
+ private readonly gate: TurnGate;
+ // The gates this turn answered. A reconnect replays the turn's journal from
+ // its start, and an answered gate must not be judged or billed again.
+ private readonly answered = new Set<string>();
- constructor(private readonly ask: string) {}
+ constructor(ask: string) {
+ this.gate = new TurnGate(ask);
+ }
/** One tool call the page ran for the turn. */
record(name: string, output: unknown, errorText: string | undefined): void {
const response = errorText === undefined ? toToolResult(name, output).details?.response : undefined;
- recordCall(this.records, name, response, errorText);
- if (errorText === undefined) {
- recordLook(this.looks, name, response);
- }
+ this.gate.record(name, response, errorText);
}
- /** The steer for the turn's next pass, or null to let it close. */
- async steer(reply: string, deps: CutAgentDeps, signal?: AbortSignal): Promise<string | null> {
- const settings = deps.judgeSettings ?? cutJudge();
- if (!settings.qualityGate || this.rounds >= settings.qualityRounds) {
- return null;
- }
-
- // The hosted loop's stand-downs: a turn that changed the project without
- // looking at a source has nothing measurable, and one sent back that
- // returns with the same record has stopped moving.
- if (this.looks.size === 0 && this.records.some((r) => !r.error && isMutatingTool(r.name))) {
- return null;
- }
- const mark = workMark(this.records, this.looks);
- if (mark === this.mark) {
- return null;
+ /** The steer for the turn's next pass, null to let it close, or undefined
+ * for a gate this turn already answered. */
+ async steer(gateId: string, reply: string, deps: CutAgentDeps, signal?: AbortSignal): Promise<string | null | undefined> {
+ if (this.answered.has(gateId)) {
+ return undefined;
}
+ this.answered.add(gateId);
- const verdict = await gateVerdict(reply, this.ask, this.records, this.looks, deps, settings, signal);
- if (!verdict) {
- return null;
- }
- this.mark = mark;
- this.rounds++;
- return verdict.steer;
+ const verdict = await this.gate.hold(reply, deps, deps.judgeSettings ?? cutJudge(), signal);
+ return verdict?.steer ?? null;
}
}
diff --git a/site/src/cut/lib/pi/turnGate.ts b/site/src/cut/lib/pi/turnGate.ts
new file mode 100644
index 00000000..ece7b10c
--- /dev/null
+++ b/site/src/cut/lib/pi/turnGate.ts
@@ -0,0 +1,107 @@
+import { askJudge } from "../judge";
+import { editorSlice, type CutJudgeSettings } from "../turnJudge";
+import {
+ QUALITY_QUESTIONS,
+ qualityState,
+ qualityVerdict,
+ recordLook,
+ type QualityAnswers,
+ type WatchedSource,
+} from "../turnQuality";
+import type { CutAgentDeps } from "./cutAgent";
+import { isMutatingTool, recordCall, type LedgerRecord } from "./mutationLedger";
+
+// The quality gate one turn runs when it signs off, shared by the hosted loop
+// and the engine turns the page answers for. It keeps the turn's record —
+// what ran, what it watched — and judges that record and the closing reply
+// against the ask under the rounds cap and the stand-downs.
+
+/** What the turn has looked at and run, as a mark. A turn the gate sent back
+ * that returns with the same mark has stopped moving. */
+function workMark(records: LedgerRecord[], looks: Map<string, WatchedSource>): string {
+ return [
+ records.length,
+ ...[...looks.values()].map((l) => `${l.passes}:${l.coveredTo}:${l.observed.length}`),
+ ].join("|");
+}
+
+/** The turn's own work, judged before it closes. Fails open: a judgment that
+ * cannot be asked lets the turn sign off. */
+async function gateVerdict(
+ reply: string,
+ request: string,
+ records: LedgerRecord[],
+ looks: Map<string, WatchedSource>,
+ deps: CutAgentDeps,
+ settings: CutJudgeSettings,
+ abortSignal?: AbortSignal
+) {
+ const work = {
+ request,
+ reply: reply.trim(),
+ ran: [...new Set(records.filter((r) => !r.error).map((r) => r.name))],
+ failed: [...new Set(records.filter((r) => r.error).map((r) => `${r.name} (${r.error})`))],
+ mutated: records.some((r) => !r.error && isMutatingTool(r.name)),
+ sources: [...looks.values()],
+ editor: editorSlice(deps.buildContext()),
+ };
+ try {
+ const { answers } = await askJudge(deps.judge, qualityState(work), QUALITY_QUESTIONS, abortSignal);
+ return qualityVerdict(answers as unknown as QualityAnswers, work, settings);
+ } catch {
+ return null;
+ }
+}
+
+export class TurnGate {
+ /** Everything the turn ran, harvested off the tool results in code. */
+ readonly records: LedgerRecord[] = [];
+ private readonly looks = new Map<string, WatchedSource>();
+ private rounds = 0;
+ // What the turn had looked at and run the last time the gate held it.
+ private mark = "";
+
+ constructor(private readonly ask: string) {}
+
+ /** How many times the gate has sent the turn back. */
+ get held(): number {
+ return this.rounds;
+ }
+
+ /** One tool call the turn ran: its response, or the error it failed with. */
+ record(name: string, response: unknown, errorText: string | undefined): void {
+ recordCall(this.records, name, response, errorText);
+ if (errorText === undefined) {
+ recordLook(this.looks, name, response);
+ }
+ }
+
+ /** The verdict that sends the signing-off turn back to work, or null to let
+ * it close. */
+ async hold(reply: string, deps: CutAgentDeps, settings: CutJudgeSettings, signal?: AbortSignal) {
+ if (!settings.qualityGate || this.rounds >= settings.qualityRounds) {
+ return null;
+ }
+
+ // The gate judges work grounded in footage: a turn that looked at a
+ // source can be held to what it saw. A turn that opened no source is
+ // measurable too when it changed nothing — an ask for work answered in
+ // words leaves the project where it was. One sent back that returns with
+ // the same record has stopped moving, and sending it again would spin.
+ if (this.looks.size === 0 && this.records.some((r) => !r.error && isMutatingTool(r.name))) {
+ return null;
+ }
+ const mark = workMark(this.records, this.looks);
+ if (mark === this.mark) {
+ return null;
+ }
+
+ const verdict = await gateVerdict(reply, this.ask, this.records, this.looks, deps, settings, signal);
+ if (!verdict) {
+ return null;
+ }
+ this.mark = mark;
+ this.rounds++;
+ return verdict;
+ }
+}
diff --git a/site/src/cut/lib/turnJudge.ts b/site/src/cut/lib/turnJudge.ts
index 6bb2a1c8..a7330926 100644
--- a/site/src/cut/lib/turnJudge.ts
+++ b/site/src/cut/lib/turnJudge.ts
@@ -9,6 +9,7 @@ import {
TOOL_AREAS,
} from "@/cut/server/ai/catalog";
import { choice, noul, type ChoiceAnswer, type Entry, type NoulAnswer } from "./judge";
+import { messageText } from "./messageText";
// The chat harness's understanding boundary for one composer turn: one typed
// judgment over the newest message, the recent turns, and a thin slice of the
@@ -103,12 +104,7 @@ const RECENT_TURNS = 5;
* attached, and the parts of the editor snapshot that change what an ask
* means. No urls, no media. */
export function judgeTurnState(messages: UIMessage[], context: unknown): Entry {
- const text = (m: UIMessage) =>
- m.parts
- .map((p) => (p.type === "text" ? p.text : ""))
- .join("")
- .trim()
- .slice(0, TEXT_CAP);
+ const text = (m: UIMessage) => messageText(m).slice(0, TEXT_CAP);
const last = messages.findLast((m) => m.role === "user");
const before = last ? messages.slice(0, messages.indexOf(last)) : messages;
const recent = before
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 37b5ba00..28d28ebf 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -208,8 +208,8 @@ export function areaTools(areas: Iterable<string>): AiToolDef[] {
}
/** The tool a turn calls to widen its catalog when the routed areas left out
- * what the ask needs. Loop-local: it lives outside the catalog, executes in
- * the chat loop, and never reaches a host or the engine. */
+ * what the ask needs. It lives outside the catalog: the hosted loop runs it
+ * in the chat loop, and the engine runs it for a Claude or Codex turn. */
export const REQUEST_TOOLS_DEF: AiToolDef = {
name: "request_tools",
description:
diff --git a/site/src/cut/server/ai/turnCatalog.ts b/site/src/cut/server/ai/turnCatalog.ts
index 9346156d..aa0389f9 100644
--- a/site/src/cut/server/ai/turnCatalog.ts
+++ b/site/src/cut/server/ai/turnCatalog.ts
@@ -1,3 +1,4 @@
+import { z } from "zod";
import type { AiToolDef } from "@/cut/lib/aiToolDef";
import { AI_TOOLS, areaTools, CORE_TOOLS, REQUEST_TOOLS_DEF, TOOL_AREA_NAMES } from "./catalog";
@@ -5,32 +6,39 @@ import { AI_TOOLS, areaTools, CORE_TOOLS, REQUEST_TOOLS_DEF, TOOL_AREA_NAMES } f
// turn the way it judges a Gemini one and sends the route with the request;
// the MCP proxy lists this turn's catalog, a call outside it is refused, and
// `request_tools` widens it. A turn sent without a route (no credits to judge
-// with, or an older page) declares the whole catalog and the model decides.
+// with, or a judgment that missed its wait) declares the whole catalog and
+// the model decides.
-/** The judged route an engine turn carries. Mirrors the page's TurnRoute. */
-export interface EngineRoute {
- intent: "chat" | "simple" | "complex";
- areas: string[];
- /** The skill to attach: a name, null for none, absent when skill
- * suggestion is off. */
- skill?: string | null;
- /** The page answers the quality gate when the turn signs off. */
- gate: boolean;
-}
+/** The judged route an engine turn carries: the page's route plus whether
+ * the page answers the quality gate when the turn signs off. The skill is a
+ * name, null for none, absent when skill suggestion is off. */
+export const routeSchema = z.object({
+ intent: z.enum(["chat", "simple", "complex"]),
+ areas: z.array(z.string().max(100)).max(100),
+ skill: z.string().max(200).nullable().optional(),
+ gate: z.boolean(),
+});
-/** An ask the page settled to one known action and ran with no model round,
+/** Asks the page settled to one known action and ran with no model round,
* reported to the provider's session on its next turn. */
-export interface HandledAsk {
- ask: string;
- tool: string;
- args: unknown;
- say: string;
-}
+export const handledSchema = z
+ .array(
+ z.object({
+ ask: z.string().max(100_000),
+ tool: z.string().max(200),
+ args: z.unknown(),
+ say: z.string().max(10_000),
+ })
+ )
+ .max(50);
+
+export type EngineRoute = z.infer<typeof routeSchema>;
+export type HandledAsk = z.infer<typeof handledSchema>[number];
/** How the provider picks up a widened catalog. Claude refreshes its listing
* on the proxy's list_changed notice; Codex reads it once per run, so the
* turn resumes in a fresh run to see the new tools. */
-export type CatalogRefresh = "live" | "next-run";
+type CatalogRefresh = "live" | "next-run";
interface TurnTools {
/** The declared areas; null declares the whole catalog. */
diff --git a/site/src/cut/server/http/ai.ts b/site/src/cut/server/http/ai.ts
index 17469825..c4942e5b 100644
--- a/site/src/cut/server/http/ai.ts
+++ b/site/src/cut/server/http/ai.ts
@@ -24,8 +24,20 @@ import {
import { rewriteCaptions, translateCaptions } from "../ai/captions";
import { writeVisualCues, type VisualFrame } from "../ai/visualSubtitles";
import { AI_SKILL_INDEX, AI_TOOLS, attachedAssetsBlock, readSkill, skillRelevanceBlock, systemPrompt } from "../ai/catalog";
-import { declaresTool, declareTurn, dropTurn, takeWidened, turnTools, widenTurn, type EngineRoute, type HandledAsk } from "../ai/turnCatalog";
+import {
+ declaresTool,
+ declareTurn,
+ dropTurn,
+ handledSchema,
+ routeSchema,
+ takeWidened,
+ turnTools,
+ widenTurn,
+ type EngineRoute,
+ type HandledAsk,
+} from "../ai/turnCatalog";
import { STEP_BUDGET, stopText, turnClose, type TurnEnd } from "../../lib/turnBudget";
+import { messageText } from "../../lib/messageText";
import { codexCommand } from "../tool-path";
import { errorMessage } from "../util";
@@ -42,20 +54,6 @@ interface ChatBody {
handled?: HandledAsk[];
}
-const routeSchema = z.object({
- intent: z.enum(["chat", "simple", "complex"]),
- areas: z.array(z.string().max(100)).max(100),
- skill: z.string().max(200).nullable().optional(),
- gate: z.boolean(),
-});
-
-const handledSchema = z.array(z.object({
- ask: z.string().max(100_000),
- tool: z.string().max(200),
- args: z.unknown(),
- say: z.string().max(10_000),
-})).max(50);
-
// The steer a Codex turn resumes on once request_tools widened its catalog.
const WIDENED_STEER = "The tool areas you requested are declared now. Carry on with the request.";
@@ -84,10 +82,7 @@ function lastUserText(messages: UIMessage[]): string {
for (let i = messages.length - 1; i >= 0; i--) {
const m = messages[i];
if (m.role !== "user") continue;
- return m.parts
- .map((p) => (p.type === "text" ? p.text : ""))
- .join("")
- .trim();
+ return messageText(m);
}
return "";
}
@@ -324,13 +319,16 @@ async function runCodex(
ask = WIDENED_STEER;
continue;
}
- const steer = body.route?.gate ? await askPageGate(sessionKey, run.reply) : null;
+ // A failed run already showed its error and the page stopped reading:
+ // only a run that completed asks the gate, as on the Claude path.
+ const steer = run.end === "done" && body.route?.gate ? await askPageGate(sessionKey, run.reply) : null;
if (!steer || signal.aborted) return;
ask = steer;
}
}
-/** One `codex exec` run; returns the session it ran in and its last line. */
+/** One `codex exec` run; returns the session it ran in, its last line, and
+ * how it ended. */
async function codexRun(
emit: UIChunkWriter["write"],
prompt: string,
@@ -339,12 +337,13 @@ async function codexRun(
base: string,
sessionKey: string,
signal: AbortSignal
-): Promise<{ thread: string | undefined; reply: string }> {
+): Promise<{ thread: string | undefined; reply: string; end: TurnEnd }> {
signal.throwIfAborted();
const codex = await codexCommand();
const mcp = mcpCommand(base, sessionKey);
let thread = session;
let reply = "";
+ let end: TurnEnd = "done";
const args = ["exec"];
if (session) args.push("resume", session);
args.push("--json", "--skip-git-repo-check", "-m", model);
@@ -417,6 +416,7 @@ async function codexRun(
reply = ev.item.text;
say(ev.item.text);
} else if (ev.type === "error" || ev.type === "turn.failed") {
+ end = "failed";
emit({ type: "error", errorText: stopText(ev.error?.message ?? ev.message) });
if (ev.type === "turn.failed") settle();
} else if (ev.type === "turn.completed") {
@@ -450,12 +450,13 @@ async function codexRun(
// clap puts the real diagnostic on the first `error:` line; the tail is
// just Usage/`try '--help'` boilerplate. Surface the error line if present.
const detail = lines.find((l) => /^error[:\s]/i.test(l)) ?? lines.slice(-2).join(" ");
+ end = "failed";
emit({ type: "error", errorText: `Codex exited with code ${code}. ${detail}`.trim() });
}
resolve();
});
});
- return { thread, reply };
+ return { thread, reply, end };
}
/**