Skip to content

Open source CapCut alternative.

TypeScript

feat(site): the complex chat role runs at the low thinking budget

David Le88781b67328d
diff --git a/site/src/cut/lib/pi/cutAgent.ts b/site/src/cut/lib/pi/cutAgent.ts
index b8d7bbb0..043be475 100644
--- a/site/src/cut/lib/pi/cutAgent.ts
+++ b/site/src/cut/lib/pi/cutAgent.ts
@@ -4,7 +4,7 @@ import type { Message, UserMessage } from "@earendil-works/pi-ai";
import { AI_TOOLS, attachedAssetsBlock, systemPrompt } from "@/cut/server/ai/catalog";
import { parseTurnIntent, turnIntentInput, turnIntentPrompt, type TurnIntent } from "../turnIntent";
import { enforceContextBudget } from "./contextBudget";
-import { donkeyModel } from "./donkeyModel";
+import { donkeyModel, type ChatThinkingLevel } from "./donkeyModel";
import { ledgerText, recordCall, type LedgerRecord } from "./mutationLedger";
import { makeDonkeyStream, type DonkeyToolDetails, type PostFn, type WireCarrier, type WirePart } from "./donkeyStream";
import { toAgentTools, type ExecTool } from "./tools";
@@ -172,6 +172,14 @@ function sanitizeSession(messages: AgentMessage[]): AgentMessage[] {
);
}
+// The complex role's rounds are tool orchestration over state the mutation
+// ledger already grounds, so they run at the low thinking budget: at its
+// default the model re-reads state after every edit and each round takes about
+// half again as long, with no gain on the chat eval. The simple role keeps its
+// default (the low budget slows that model and costs it a case), and so does a
+// chat-verdict turn, which answers a question with no tools to orchestrate.
+const COMPLEX_THINKING_LEVEL: ChatThinkingLevel = "low";
+
export function hydratePiSession(threadId: string, messages: AgentMessage[] | undefined): void {
if (messages && !sessions.has(threadId)) keepSession(threadId, sanitizeSession(messages));
}
@@ -399,12 +407,14 @@ export function streamCutChat({
* finalizes only the run it keeps. */
const runAgentTurn = async ({
roundModel,
+ thinkingLevel,
withTools,
send,
executionGate,
onAgent,
}: {
roundModel: string;
+ thinkingLevel?: ChatThinkingLevel;
withTools: boolean;
send: (chunk: Record<string, unknown>) => void;
/** Awaited before any tool executes; false ends the batch unrun. */
@@ -422,7 +432,7 @@ export function streamCutChat({
const agent = new Agent({
initialState: {
systemPrompt: systemPrompt(),
- model: donkeyModel(roundModel),
+ model: donkeyModel(roundModel, thinkingLevel),
messages: sessionFor(threadId, messages.filter((m) => m !== lastUser)),
tools: withTools ? toAgentTools(AI_TOOLS, deps.execTool) : [],
},
@@ -630,6 +640,7 @@ export function streamCutChat({
finalize(
await runAgentTurn({
roundModel: intent === "simple" ? deps.models.simple : model,
+ thinkingLevel: intent === "complex" ? COMPLEX_THINKING_LEVEL : undefined,
withTools: intent !== "chat",
send: emit,
})
diff --git a/site/src/cut/lib/pi/donkeyModel.ts b/site/src/cut/lib/pi/donkeyModel.ts
index 1fc9e864..06a8911b 100644
--- a/site/src/cut/lib/pi/donkeyModel.ts
+++ b/site/src/cut/lib/pi/donkeyModel.ts
@@ -2,14 +2,23 @@ import type { Model } from "@earendil-works/pi-ai";
// The chat loop's models as pi Model objects. The custom api id routes every
// call through donkeyStream — pi's own providers never see these — so the only
-// fields that matter downstream are `id` (the hosted route's model string) and
-// the metadata pi reads for bookkeeping.
+// fields that matter downstream are `id` (the hosted route's model string),
+// the thinking level donkeyStream puts on the request, and the metadata pi
+// reads for bookkeeping.
export type DonkeyApi = "donkey-responses";
const ZERO_COST = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
-export function donkeyModel(id: string): Model<DonkeyApi> {
+/** Gemini's thinking budget for a round. Unset leaves the model's default. */
+export type ChatThinkingLevel = "low" | "medium" | "high";
+
+export interface DonkeyModel extends Model<DonkeyApi> {
+ thinkingLevel?: ChatThinkingLevel;
+}
+
+export function donkeyModel(id: string, thinkingLevel?: ChatThinkingLevel): DonkeyModel {
return {
+ ...(thinkingLevel ? { thinkingLevel } : {}),
id,
name: id,
api: "donkey-responses",
diff --git a/site/src/cut/lib/pi/donkeyStream.ts b/site/src/cut/lib/pi/donkeyStream.ts
index e6a60af6..da000aac 100644
--- a/site/src/cut/lib/pi/donkeyStream.ts
+++ b/site/src/cut/lib/pi/donkeyStream.ts
@@ -11,6 +11,7 @@ import {
type Usage,
} from "@earendil-works/pi-ai";
import { reportBalance } from "../hosted";
+import type { DonkeyModel } from "./donkeyModel";
// The transport adapter: pi Context in, Donkey's hosted Responses route out.
// Every model call the agent makes flows through here — pi's own providers are
@@ -261,6 +262,8 @@ export function makeDonkeyStream(deps: DonkeyStreamDeps) {
input: serializeMessages(context.messages),
stream: true,
};
+ const thinkingLevel = (model as DonkeyModel).thinkingLevel;
+ if (thinkingLevel) payload.thinking_level = thinkingLevel;
if (context.tools && context.tools.length > 0) {
payload.tools = context.tools.map((t) => ({
type: "function",