A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
preserve reasoning in refinement and side questions (#2130)
kt81cd5390dbc8
diff --git a/packages/ai/.changes/eng-6010-auxiliary-reasoning.md b/packages/ai/.changes/eng-6010-auxiliary-reasoning.mdnew file mode 100644index 0000000000..7d559fafe7--- /dev/null+++ b/packages/ai/.changes/eng-6010-auxiliary-reasoning.md@@ -0,0 +1 @@+- Fixed budget-based Claude requests sending thinking budgets below the provider minimum when the output limit is small.diff --git a/packages/ai/src/providers/simple-options.ts b/packages/ai/src/providers/simple-options.tsindex 51f5189074..56e01af1c9 100644--- a/packages/ai/src/providers/simple-options.ts+++ b/packages/ai/src/providers/simple-options.ts@@ -37,12 +37,16 @@ export function adjustMaxTokensForThinking(const budgets = { ...defaultBudgets, ...customBudgets };const minOutputTokens = 1024;+ const minThinkingTokens = 1024;const level = clampReasoning(reasoningLevel)!;- let thinkingBudget = budgets[level]!;+ let thinkingBudget = Math.max(minThinkingTokens, budgets[level]!);const maxTokens = Math.min(baseMaxTokens + thinkingBudget, modelMaxTokens);+ if (maxTokens <= minThinkingTokens) {+ throw new Error("Budget-based thinking requires at least 1024 thinking tokens plus room for the response");+ }if (maxTokens <= thinkingBudget) {- thinkingBudget = Math.max(0, maxTokens - minOutputTokens);+ thinkingBudget = Math.max(minThinkingTokens, maxTokens - minOutputTokens);}return { maxTokens, thinkingBudget };diff --git a/packages/ai/test/anthropic-minimum-thinking-budget.test.ts b/packages/ai/test/anthropic-minimum-thinking-budget.test.tsnew file mode 100644index 0000000000..b7632ba09d--- /dev/null+++ b/packages/ai/test/anthropic-minimum-thinking-budget.test.ts@@ -0,0 +1,41 @@+import { describe, expect, it } from "vitest";+import { getModel } from "../src/models.js";+import { streamSimpleAnthropic } from "../src/providers/anthropic.js";+import { adjustMaxTokensForThinking } from "../src/providers/simple-options.js";++describe("budget-based Anthropic thinking minimum", () => {+ it.each([1025, 1500, 2048, 4096])("sends a valid thinking budget under a %i-token ceiling", async (maxTokens) => {+ let payload: { max_tokens: number; thinking: { type: string; budget_tokens: number } } | undefined;+ const model = { ...getModel("anthropic", "claude-sonnet-4-5"), maxTokens };+ const stream = streamSimpleAnthropic(+ model,+ { messages: [{ role: "user", content: "Hello", timestamp: 1 }] },+ {+ apiKey: "dummy-key",+ reasoning: "low",+ maxTokens,+ onPayload(value) {+ payload = value as typeof payload;+ throw new Error("Captured before network dispatch");+ },+ },+ );+ await stream.result();+ expect(payload).toBeDefined();+ expect(payload!.thinking.type).toBe("enabled");+ expect(payload!.thinking.budget_tokens).toBeGreaterThanOrEqual(1024);+ expect(payload!.thinking.budget_tokens).toBeLessThan(payload!.max_tokens);+ expect(payload!.max_tokens).toBe(maxTokens);+ });++ it.each([1, 476, 1024])("rejects an impossible %i-token model limit before constructing a request", (maxTokens) => {+ expect(() => adjustMaxTokensForThinking(maxTokens, maxTokens, "low")).toThrow("thinking requires at least");+ });++ it("honors the thinking minimum for an undersized custom budget", () => {+ expect(adjustMaxTokensForThinking(4096, 8192, "low", { low: 476 })).toEqual({+ maxTokens: 5120,+ thinkingBudget: 1024,+ });+ });+});diff --git a/packages/coding-agent/.changes/eng-6010-auxiliary-reasoning.md b/packages/coding-agent/.changes/eng-6010-auxiliary-reasoning.mdnew file mode 100644index 0000000000..5d6ea142f8--- /dev/null+++ b/packages/coding-agent/.changes/eng-6010-auxiliary-reasoning.md@@ -0,0 +1,3 @@+- Fixed refinement and side questions disabling reasoning when the session uses it, while respecting the selected model's supported thinking levels.+- Removed fixed output caps from refinement with reasoning enabled, while respecting the model's output limit and reserving context space for the prompt.+- Fixed long refinements exceeding the context window or being rejected prematurely, keeping recent conversation text and space for the response.diff --git a/packages/coding-agent/src/core/refinement/refinement.ts b/packages/coding-agent/src/core/refinement/refinement.tsindex 5242819411..ce15ade8d8 100644--- a/packages/coding-agent/src/core/refinement/refinement.ts+++ b/packages/coding-agent/src/core/refinement/refinement.ts@@ -1,7 +1,7 @@import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync } from "node:fs";import { join } from "node:path";import type { AgentMessage, ThinkingLevel } from "@earendil-works/pi-agent-core";-import type { Model } from "@earendil-works/pi-ai";+import type { Api, Model } from "@earendil-works/pi-ai";import { completeSimple } from "@earendil-works/pi-ai";import { getAgentDir } from "../../config.js";import { realpathIfPresentSync, writeFileAtomicSync } from "../../utils/atomic-file.js";@@ -9,6 +9,7 @@ import { serializeConversation } from "../compaction/utils.js";import { convertToLlm } from "../messages.js";import { completeWithProviderRetry, type ProviderRetryPolicy } from "../provider-retry.js";import type { CustomEntry } from "../session-manager.js";+import { getAuxiliaryThinkingLevel } from "../thinking-levels.js";export const REFINEMENT_CUSTOM_TYPE = "prime-agent.refinement";@@ -177,24 +178,59 @@ Return JSON only:"instructions": "optional concise instructions for /refine if shouldRefine is true"}`;-/**- * Output budgets are derived from the selected model instead of fixed literals.- * /refine input scales with harness size (entry overview, refinement history, and- * the trajectory slice), so a constant output cap silently truncates exactly the- * large multi-edit proposals that matter most. Math.min keeps small models honest.- */+// These caps apply only with reasoning off; thinking and JSON otherwise share the model's output budget.const REFINEMENT_MAX_OUTPUT_TOKENS = 32_000;const AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS = 4_096;+const REFINEMENT_CONTEXT_OVERHEAD_TOKENS = 1_024;const TRUNCATED_JSON_ERROR ="the model stopped before completing its JSON object. This usually means the output budget was exhausted; retry with a smaller request.";-function refinementMaxOutputTokens(model: Model<any>): number {- return Math.min(model.maxTokens, REFINEMENT_MAX_OUTPUT_TOKENS);+function refinementInputTokenBound(text: string): number {+ // One token per UTF-8 byte bounds byte-based tokenizers, including dense or unusual text.+ return Buffer.byteLength(text, "utf8");}-function autoRefineReviewMaxOutputTokens(model: Model<any>): number {- return Math.min(model.maxTokens, AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS);+function refinementRequest(+ model: Model<Api>,+ systemPrompt: string,+ conversationText: string,+ buildPrompt: (conversation: string) => string,+ outputReserve: number,+): { model: Model<Api>; userPrompt: string } {+ const systemReserve = refinementInputTokenBound(systemPrompt) + REFINEMENT_CONTEXT_OVERHEAD_TOKENS;+ const inputBudget =+ model.contextWindow - Math.min(model.maxTokens, outputReserve, Math.floor(model.contextWindow / 2));+ let userPrompt = buildPrompt(conversationText);+ if (systemReserve + refinementInputTokenBound(userPrompt) > inputBudget && conversationText.length > 0) {+ const promptForLength = (length: number): string => {+ let start = conversationText.length - length;+ const first = conversationText.charCodeAt(start);+ if (first >= 0xdc00 && first <= 0xdfff) start++;+ return buildPrompt(+ `[Earlier conversation omitted to fit the model context.]\n${conversationText.slice(start)}`,+ );+ };+ let low = 0;+ let high = conversationText.length;+ while (low < high) {+ const length = Math.ceil((low + high) / 2);+ if (systemReserve + refinementInputTokenBound(promptForLength(length)) <= inputBudget) low = length;+ else high = length - 1;+ }+ userPrompt = promptForLength(low);+ }+ const maxTokens = Math.min(+ model.maxTokens,+ model.contextWindow - systemReserve - refinementInputTokenBound(userPrompt),+ );+ if (maxTokens <= 0) {+ throw new Error(+ "Refinement prompt leaves no room for output in the model's context window; retry with a smaller request.",+ );+ }+ // Bound the request's model ceiling too: some adapters add thinking tokens before clamping to it.+ return { model: { ...model, maxTokens }, userPrompt };}function now(): string {@@ -910,32 +946,43 @@ export async function planRefinement(const scopeInstruction = options.global? "Requested refinement scope: global. Only propose stable cross-session continual harness edits, durable user preferences, reusable skills/subagents, or explicitly project-qualified facts that should affect future Prime Agent sessions. Do not persist session-only progress, temporary blockers, or current-run coordination globally.": "Requested refinement scope: local. Prefer local continual harness edits for current task progress, temporary blockers, current-run coordination, and project facts that are not clearly reusable across Prime Agent sessions. Global entries in the overview are read-only context: do not propose update or delete edits for them; create a local entry instead if an override is needed.";- const userPrompt = [- `<current_harness_state>\n${overviewForPrompt(state)}\n</current_harness_state>`,- `<refinement_history>\n${historyForPrompt(history)}\n</refinement_history>`,- `<conversation>\n${conversationText}\n</conversation>`,- `<scope_policy>\n${scopeInstruction}\n</scope_policy>`,- options.instructions ? `<user_refine_instructions>\n${options.instructions}\n</user_refine_instructions>` : "",- "Return only JSON edits. If no useful edit is justified, return an empty edits array with a rationale.",- ]- .filter(Boolean)- .join("\n\n");+ const buildPrompt = (conversation: string): string =>+ [+ `<current_harness_state>\n${overviewForPrompt(state)}\n</current_harness_state>`,+ `<refinement_history>\n${historyForPrompt(history)}\n</refinement_history>`,+ `<conversation>\n${conversation}\n</conversation>`,+ `<scope_policy>\n${scopeInstruction}\n</scope_policy>`,+ options.instructions ? `<user_refine_instructions>\n${options.instructions}\n</user_refine_instructions>` : "",+ "Return only JSON edits. If no useful edit is justified, return an empty edits array with a rationale.",+ ]+ .filter(Boolean)+ .join("\n\n");+ const reasoning = getAuxiliaryThinkingLevel(model, thinkingLevel);+ const { model: requestModel, userPrompt } = refinementRequest(+ model,+ REFINEMENT_SYSTEM_PROMPT,+ conversationText,+ buildPrompt,+ reasoning === "off" ? REFINEMENT_MAX_OUTPUT_TOKENS : model.maxTokens,+ );+ const maxTokens =+ reasoning === "off" ? Math.min(requestModel.maxTokens, REFINEMENT_MAX_OUTPUT_TOKENS) : requestModel.maxTokens;- // /refine requires a parseable JSON object in the final text. Some reasoning-capable- // OpenAI-compatible models can spend the response on visible thinking and return no- // final text, which makes otherwise successful daemon /refine calls fail parsing.- // Keep the refinement request non-reasoning regardless of the interactive session- // thinking level so the model uses its output budget for the JSON object.- void thinkingLevel;const response = await completeWithProviderRetry(() =>completeSimple(- model,+ requestModel,{systemPrompt: REFINEMENT_SYSTEM_PROMPT,messages: [{ role: "user", content: [{ type: "text", text: userPrompt }], timestamp: Date.now() }],},- { maxTokens: refinementMaxOutputTokens(model), signal, apiKey, headers },+ {+ reasoning,+ maxTokens,+ signal,+ apiKey,+ headers,+ },),{ policy: options.retry, signal },);@@ -980,33 +1027,49 @@ export async function reviewAutoRefine(retry?: ProviderRetryPolicy,): Promise<AutoRefineReview> {const conversationText = serializeConversation(convertToLlm(messages)).slice(-40_000);- const userPrompt = [- `<trigger>+ const buildPrompt = (conversation: string): string =>+ [+ `<trigger>${context.reason}; ${context.turnsSinceLastReview} assistant turns since last auto-refine review</trigger>`,- `<current_harness_state>+ `<current_harness_state>${overviewForPrompt(state)}</current_harness_state>`,- `<refinement_history>+ `<refinement_history>${historyForPrompt(history)}</refinement_history>`,- `<conversation>-${conversationText}+ `<conversation>+${conversation}</conversation>`,- "Return shouldRefine=true when the trajectory contains evidence useful to this session's future turns. Prefer local harness edits for current task progress, temporary blockers, and current-run coordination. Ask for global refinement only for durable cross-session lessons or explicitly project-qualified facts likely to be reused in future sessions.",- ].join("\n\n");- // Auto-refine review requires parseable JSON. Keep it non-reasoning so- // reasoning-capable models use final text budget for the JSON object.- void thinkingLevel;+ "Return shouldRefine=true when the trajectory contains evidence useful to this session's future turns. Prefer local harness edits for current task progress, temporary blockers, and current-run coordination. Ask for global refinement only for durable cross-session lessons or explicitly project-qualified facts likely to be reused in future sessions.",+ ].join("\n\n");+ const reasoning = getAuxiliaryThinkingLevel(model, thinkingLevel);+ const { model: requestModel, userPrompt } = refinementRequest(+ model,+ AUTO_REFINE_REVIEW_SYSTEM_PROMPT,+ conversationText,+ buildPrompt,+ reasoning === "off" ? AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS : model.maxTokens,+ );+ const maxTokens =+ reasoning === "off"+ ? Math.min(requestModel.maxTokens, AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS)+ : requestModel.maxTokens;const response = await completeWithProviderRetry(() =>completeSimple(- model,+ requestModel,{systemPrompt: AUTO_REFINE_REVIEW_SYSTEM_PROMPT,messages: [{ role: "user", content: [{ type: "text", text: userPrompt }], timestamp: Date.now() }],},- { maxTokens: autoRefineReviewMaxOutputTokens(model), signal, apiKey, headers },+ {+ reasoning,+ maxTokens,+ signal,+ apiKey,+ headers,+ },),{ policy: retry, signal },);diff --git a/packages/coding-agent/src/core/side-question.ts b/packages/coding-agent/src/core/side-question.tsindex 80e561fbf0..1ab9ea1b6d 100644--- a/packages/coding-agent/src/core/side-question.ts+++ b/packages/coding-agent/src/core/side-question.ts@@ -6,6 +6,7 @@ import {type ProviderRetryPolicy,} from "./provider-retry.js";import { unwrapSemanticEdgeStreamFn } from "./semantic-edges.js";+import { getAuxiliaryThinkingLevel } from "./thinking-levels.js";export type SideQuestionStatus = "running" | "complete" | "cancelled" | "error";@@ -90,7 +91,7 @@ export function startSideQuestion(model,systemPrompt: parent.state.systemPrompt,messages: [...structuredClone(parent.state.messages), ...previousTurnMessages],- thinkingLevel: "off",+ thinkingLevel: getAuxiliaryThinkingLevel(model, parent.state.thinkingLevel),serviceTier: parent.state.serviceTier,tools: [],},diff --git a/packages/coding-agent/src/core/thinking-levels.ts b/packages/coding-agent/src/core/thinking-levels.tsindex e7c76805f5..6243b524d5 100644--- a/packages/coding-agent/src/core/thinking-levels.ts+++ b/packages/coding-agent/src/core/thinking-levels.ts@@ -1,3 +1,11 @@import type { ThinkingLevel } from "@earendil-works/pi-agent-core";+import { type Api, clampThinkingLevel, type Model } from "@earendil-works/pi-ai";export const THINKING_LEVELS: ThinkingLevel[] = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];++export function getAuxiliaryThinkingLevel(model: Model<Api>, thinkingLevel: ThinkingLevel = "low"): ThinkingLevel {+ // Keep auxiliary calls inexpensive without disabling an enabled session's reasoning.+ // Clamping may raise this preference when the model requires a higher effort.+ const preferred = thinkingLevel === "off" || thinkingLevel === "minimal" ? thinkingLevel : "low";+ return clampThinkingLevel(model, preferred);+}diff --git a/packages/coding-agent/test/refinement.test.ts b/packages/coding-agent/test/refinement.test.tsindex 19a7313975..d8e2731859 100644--- a/packages/coding-agent/test/refinement.test.ts+++ b/packages/coding-agent/test/refinement.test.ts@@ -981,7 +981,7 @@ describe("harness refinement", () => {expect(state.entries.prompt.base_system_prompt).toBeUndefined();});- it("requests JSON refinement without model reasoning even when session thinking is enabled", async () => {+ it("requests JSON refinement with low reasoning when session thinking is enabled", async () => {const state = loadHarnessState(makeTempDir());completeSimpleMock.mockResolvedValueOnce(assistantText(@@ -1031,11 +1031,11 @@ describe("harness refinement", () => {});// Budget is derived from the model (8192) rather than a fixed literal.expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({+ reasoning: "low",maxTokens: 8192,apiKey: "api-key",headers: { "x-test-header": "1" },});- expect(completeSimpleMock.mock.calls[0][2]).not.toHaveProperty("reasoning");expect(result.appliedEdits[0]).toMatchObject({action: "create",kind: "memory",diff --git a/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.ts b/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.tsnew file mode 100644index 0000000000..a6231804cd--- /dev/null+++ b/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.ts@@ -0,0 +1,140 @@+import type { ThinkingLevel } from "@earendil-works/pi-agent-core";+import { fauxAssistantMessage, type Model, type SimpleStreamOptions } from "@earendil-works/pi-ai";+import { afterEach, describe, expect, it } from "vitest";+import { type HarnessState, planRefinement, reviewAutoRefine } from "../../../src/core/refinement/refinement.js";+import { type SideQuestionEvent, startSideQuestion } from "../../../src/core/side-question.js";+import { createHarness, type Harness } from "../harness.js";++const harnesses: Harness[] = [];+const noRetry = { enabled: false, maxRetries: 0, baseDelayMs: 1, maxRetryDelayMs: 1 };+const proposal = { summary: "No change", rationale: "Fixture", expectedOutcome: "No change", edits: [] };+const review = { shouldRefine: false, rationale: "No reusable lesson" };++afterEach(() => {+ for (const harness of harnesses.splice(0)) harness.cleanup();+});++interface ReasoningCase {+ name: string;+ reasoning: boolean;+ parentLevel: ThinkingLevel;+ expectedLevel: ThinkingLevel;+ omitRefinementLevel?: boolean;+ thinkingLevelMap?: Model<string>["thinkingLevelMap"];+}++const cases: ReasoningCase[] = [+ {+ name: "enabled session with incomplete capability metadata",+ reasoning: true,+ parentLevel: "high",+ expectedLevel: "low",+ },+ { name: "minimal session", reasoning: true, parentLevel: "minimal", expectedLevel: "minimal" },+ { name: "explicit off supported", reasoning: true, parentLevel: "off", expectedLevel: "off" },+ { name: "non-reasoning model", reasoning: false, parentLevel: "high", expectedLevel: "off" },+ {+ name: "refinement caller without a thinking preference",+ reasoning: true,+ parentLevel: "high",+ expectedLevel: "low",+ omitRefinementLevel: true,+ },+ {+ name: "off and minimal unsupported",+ reasoning: true,+ parentLevel: "off",+ expectedLevel: "low",+ thinkingLevelMap: { off: null, minimal: null },+ },+ {+ name: "only high supported",+ reasoning: true,+ parentLevel: "high",+ expectedLevel: "high",+ thinkingLevelMap: { off: null, minimal: null, low: null, medium: null },+ },+];++describe("auxiliary reasoning settings", () => {+ it.each(cases)("completes refinement, review, and side questions: $name", async (testCase) => {+ const harness = await createHarness({+ models: [+ { id: "reasoning-contract", reasoning: testCase.reasoning, contextWindow: 262_144, maxTokens: 128_000 },+ ],+ });+ harnesses.push(harness);+ const parent = harness.session.agent;+ const model = harness.getModel();+ model.thinkingLevelMap = testCase.thinkingLevelMap;+ parent.state.thinkingLevel = testCase.parentLevel;+ const parentMessages = structuredClone(parent.state.messages);+ const observed: SimpleStreamOptions[] = [];+ harness.setResponses(+ [JSON.stringify(proposal), JSON.stringify(review), "Side answer"].map((text) => (_context, options) => {+ const request = options as SimpleStreamOptions;+ observed.push(request);+ if (request.reasoning !== testCase.expectedLevel) {+ return fauxAssistantMessage("", {+ stopReason: "error",+ errorMessage: `Unsupported auxiliary reasoning: ${request.reasoning}`,+ });+ }+ return fauxAssistantMessage([+ { type: "thinking", thinking: "Fixture reasoning" },+ { type: "text", text },+ ]);+ }),+ );+ const state: HarnessState = {+ schema: 1,+ entries: { prompt: {}, memory: {}, skill: {}, subagent: {} },+ refinements: [],+ };+ const messages = [{ role: "user" as const, content: "Remember the result", timestamp: 1 }];+ await expect(+ planRefinement(+ messages,+ state,+ [],+ model,+ "faux-key",+ { retry: noRetry },+ undefined,+ undefined,+ testCase.omitRefinementLevel ? undefined : testCase.parentLevel,+ ),+ ).resolves.toMatchObject({ proposal });+ await expect(+ reviewAutoRefine(+ messages,+ state,+ [],+ model,+ "faux-key",+ { reason: "turn_interval", turnsSinceLastReview: 5 },+ undefined,+ undefined,+ testCase.omitRefinementLevel ? undefined : testCase.parentLevel,+ noRetry,+ ),+ ).resolves.toMatchObject(review);+ const events: SideQuestionEvent[] = [];+ await startSideQuestion(+ parent,+ "side",+ "What happened?",+ (event) => {+ events.push(event);+ },+ [],+ noRetry,+ ).done;+ expect(events.at(-1)).toMatchObject({ status: "complete", answer: "Side answer" });+ expect(observed.map((options) => options.reasoning)).toEqual(Array(3).fill(testCase.expectedLevel));+ expect(observed[0].maxTokens).toBe(testCase.expectedLevel === "off" ? 32_000 : model.maxTokens);+ expect(observed[1].maxTokens).toBe(testCase.expectedLevel === "off" ? 4_096 : model.maxTokens);+ expect(parent.state.thinkingLevel).toBe(testCase.parentLevel);+ expect(parent.state.messages).toEqual(parentMessages);+ });+});diff --git a/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.ts b/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.tsnew file mode 100644index 0000000000..d3ec65b32b--- /dev/null+++ b/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.ts@@ -0,0 +1,264 @@+import type { ThinkingLevel } from "@earendil-works/pi-agent-core";+import {+ completeSimple,+ type FauxResponseFactory,+ fauxAssistantMessage,+ type SimpleStreamOptions,+} from "@earendil-works/pi-ai";+import { afterEach, describe, expect, it } from "vitest";+import { type HarnessState, planRefinement, reviewAutoRefine } from "../../../src/core/refinement/refinement.js";+import { createHarness, getMessageText, type Harness } from "../harness.js";++const harnesses: Harness[] = [];+const retry = { enabled: true, maxRetries: 2, baseDelayMs: 1, maxRetryDelayMs: 1 };+const proposal = { summary: "No change", rationale: "Fixture", expectedOutcome: "No change", edits: [] };+const review = { shouldRefine: false, rationale: "No reusable lesson" };+const paths = [+ { kind: "plan", formerCap: 32_000, expected: proposal },+ { kind: "review", formerCap: 4_096, expected: review },+] as const;++afterEach(() => {+ for (const harness of harnesses.splice(0)) harness.cleanup();+});++async function requestRefinement(+ harness: Harness,+ kind: "plan" | "review",+ content = "",+ thinkingLevel: ThinkingLevel = "high",+) {+ const state: HarnessState = {+ schema: 1,+ entries: { prompt: {}, memory: {}, skill: {}, subagent: {} },+ refinements: [],+ };+ const messages = content ? [{ role: "user" as const, content, timestamp: 1 }] : [];+ if (kind === "plan") {+ const result = await planRefinement(+ messages,+ state,+ [],+ harness.getModel(),+ "faux-key",+ { retry },+ undefined,+ undefined,+ thinkingLevel,+ );+ return result.proposal;+ }+ return reviewAutoRefine(+ messages,+ state,+ [],+ harness.getModel(),+ "faux-key",+ { reason: "turn_interval", turnsSinceLastReview: 5 },+ undefined,+ undefined,+ thinkingLevel,+ retry,+ );+}++function budgetedResponse(text: string, reasoningTokens: number, requests: SimpleStreamOptions[]): FauxResponseFactory {+ return (_context, options, _state, model) => {+ const request = options as SimpleStreamOptions;+ requests.push(request);+ // Synthetic tokens: one JSON character per token, after a fixed reasoning cost.+ const totalBudget = Math.min(request.maxTokens ?? 32_000, model.maxTokens);+ const visible = text.slice(0, Math.max(0, totalBudget - reasoningTokens));+ return fauxAssistantMessage(+ [+ { type: "thinking", thinking: "Condensed fixture reasoning" },+ ...(visible ? [{ type: "text" as const, text: visible }] : []),+ ],+ { stopReason: visible === text ? "stop" : "length" },+ );+ };+}++describe.each(paths)("$kind shared reasoning and JSON budget", ({ kind, formerCap, expected }) => {+ it.each(["off", "high"] as const)(+ "accepts a long English trajectory on a 64k model with reasoning %s",+ async (thinkingLevel) => {+ const harness = await createHarness({+ models: [{ id: "glm-context", reasoning: true, contextWindow: 65_536, maxTokens: 16_384 }],+ });+ harnesses.push(harness);+ const content = `${"The project now stores the result locally. ".repeat(2_000)}Latest fact: preserve local state.`;+ let observedPrompt = "";+ const requests: SimpleStreamOptions[] = [];+ harness.setResponses([+ (context, options) => {+ observedPrompt = context.messages.map(getMessageText).join("");+ requests.push(options as SimpleStreamOptions);+ return fauxAssistantMessage(JSON.stringify(expected));+ },+ ]);+ await expect(requestRefinement(harness, kind, content, thinkingLevel)).resolves.toEqual(expected);+ expect(observedPrompt).toContain(content.slice(-39_000));+ expect(requests[0].maxTokens).toBe(thinkingLevel === "off" ? Math.min(formerCap, 16_384) : 16_384);+ },+ );++ it.each(["off", "high"] as const)("accepts a full multibyte trajectory with reasoning %s", async (thinkingLevel) => {+ const harness = await createHarness({+ models: [{ id: "multibyte-context", reasoning: true, contextWindow: 200_000, maxTokens: 65_536 }],+ });+ harnesses.push(harness);+ const content = `${"项目记录".repeat(20_000)}最新事实`;+ harness.setResponses([fauxAssistantMessage(JSON.stringify(expected))]);+ await expect(requestRefinement(harness, kind, content, thinkingLevel)).resolves.toEqual(expected);+ });++ it.each(["界", "😀"])(+ "retains recent %s context and valid Unicode when shortening a dense trajectory",+ async (character) => {+ const harness = await createHarness({+ models: [{ id: "dense-context", reasoning: true, contextWindow: 32_768, maxTokens: 8_192 }],+ });+ harnesses.push(harness);+ const content = `${character.repeat(80_000)}Latest fact: retain the newest decision.`;+ let observedPrompt = "";+ let maxTokens: number | undefined;+ harness.setResponses([+ (context, options) => {+ observedPrompt = context.messages.map(getMessageText).join("");+ maxTokens = (options as SimpleStreamOptions).maxTokens;+ return fauxAssistantMessage(JSON.stringify(expected));+ },+ ]);+ await expect(requestRefinement(harness, kind, content)).resolves.toEqual(expected);+ expect(observedPrompt).toContain("Earlier conversation omitted");+ expect(observedPrompt).toContain("Latest fact: retain the newest decision.");+ expect(observedPrompt).toContain(kind === "plan" ? "<scope_policy>" : "<trigger>");+ expect([...observedPrompt].some((point) => point.length === 1 && /[\ud800-\udfff]/u.test(point))).toBe(false);+ expect(maxTokens).toBe(8_192);+ },+ );++ it("leaves at least 60k output tokens for a long English trajectory on a 128k model", async () => {+ const harness = await createHarness({+ models: [{ id: "wide-context", reasoning: true, contextWindow: 128_000, maxTokens: 100_000 }],+ });+ harnesses.push(harness);+ const requests: SimpleStreamOptions[] = [];+ harness.setResponses([budgetedResponse(JSON.stringify(expected), 60_000, requests)]);+ await expect(requestRefinement(harness, kind, "Keep the useful project facts. ".repeat(3_000))).resolves.toEqual(+ expected,+ );+ expect(requests[0].maxTokens).toBeGreaterThan(60_000);+ });+ it.each([0, 8])("completes when the former cap leaves only %i JSON tokens", async (visibleTokens) => {+ const harness = await createHarness({+ models: [{ id: "shared-output-budget", reasoning: true, maxTokens: 65_536 }],+ });+ harnesses.push(harness);+ const requests: SimpleStreamOptions[] = [];+ const response = budgetedResponse(JSON.stringify(expected), formerCap - visibleTokens, requests);+ harness.setResponses([response, response]);++ const control = await completeSimple(+ harness.getModel(),+ { messages: [] },+ {+ apiKey: "faux-key",+ reasoning: "low",+ maxTokens: formerCap,+ },+ );+ expect(control.stopReason).toBe("length");+ expect(+ control.content+ .filter((block) => block.type === "text")+ .map((block) => block.text)+ .join(""),+ ).toBe(JSON.stringify(expected).slice(0, visibleTokens));++ await expect(requestRefinement(harness, kind)).resolves.toEqual(expected);+ expect(requests.map((request) => request.maxTokens)).toEqual([formerCap, 65_536]);+ expect(requests.map((request) => request.reasoning)).toEqual(["low", "low"]);+ expect(harness.faux.state.callCount).toBe(2);+ });++ it("uses a small model's actual output limit", async () => {+ const harness = await createHarness({ models: [{ id: "small-budget", reasoning: true, maxTokens: 2_048 }] });+ harnesses.push(harness);+ const requests: SimpleStreamOptions[] = [];+ harness.setResponses([budgetedResponse(JSON.stringify(expected), 1_024, requests)]);++ await expect(requestRefinement(harness, kind)).resolves.toEqual(expected);+ expect(requests[0].maxTokens).toBe(2_048);+ expect(harness.faux.state.callCount).toBe(1);+ });++ it.each([+ { name: "empty trajectory", content: "", addedThinkingTokens: 0 },+ { name: "short English words", content: "a ".repeat(40_000), addedThinkingTokens: 0 },+ { name: "ASCII punctuation", content: "!@#$%^&*()".repeat(8_000), addedThinkingTokens: 0 },+ { name: "multibyte trajectory", content: "界".repeat(15_000), addedThinkingTokens: 0 },+ { name: "separate thinking allowance", content: "界".repeat(15_000), addedThinkingTokens: 16_384 },+ ])(+ "reserves prompt context for equal context and output limits: $name",+ async ({ content, addedThinkingTokens }) => {+ const harness = await createHarness({+ models: [{ id: "shared-context-limit", reasoning: true, contextWindow: 131_072, maxTokens: 131_072 }],+ });+ harnesses.push(harness);+ const originalModel = structuredClone(harness.getModel());+ const requests: SimpleStreamOptions[] = [];+ let totalTokens = 0;+ harness.setResponses([+ (context, options, _state, model) => {+ const request = options as SimpleStreamOptions;+ requests.push(request);+ // Synthetic tokenizer: English words, CJK characters, and punctuation, plus message framing.+ const inputTokens =+ [context.systemPrompt ?? "", ...context.messages.map(getMessageText)]+ .join("")+ .match(/[\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]/gu)!.length + 256;+ // Some adapters add thinking tokens before clamping to the supplied model ceiling.+ const wireMaxTokens = Math.min((request.maxTokens ?? 32_000) + addedThinkingTokens, model.maxTokens);+ totalTokens = inputTokens + wireMaxTokens;+ return fauxAssistantMessage(JSON.stringify(expected));+ },+ ]);++ await expect(requestRefinement(harness, kind, content)).resolves.toEqual(expected);+ expect(totalTokens).toBeLessThanOrEqual(originalModel.contextWindow);+ expect(requests[0].maxTokens).toBeGreaterThan(formerCap);+ expect(requests[0].reasoning).toBe("low");+ expect(harness.getModel()).toEqual(originalModel);+ expect(harness.faux.state.callCount).toBe(1);+ },+ );++ it("rejects a prompt with no reserved output space before calling the provider", async () => {+ const harness = await createHarness({+ models: [{ id: "no-context-space", reasoning: true, contextWindow: 1_024, maxTokens: 1_024 }],+ });+ harnesses.push(harness);+ harness.setResponses([fauxAssistantMessage(JSON.stringify(expected))]);++ await expect(requestRefinement(harness, kind, "界".repeat(1_024))).rejects.toThrow("no room for output");+ expect(harness.faux.state.callCount).toBe(0);+ });++ it.each([0, 8])(+ "reports exhaustion at the model limit with %i JSON tokens without retrying",+ async (visibleTokens) => {+ const harness = await createHarness({+ models: [{ id: "exhausted-budget", reasoning: true, maxTokens: 2_048 }],+ });+ harnesses.push(harness);+ const requests: SimpleStreamOptions[] = [];+ harness.setResponses([budgetedResponse(JSON.stringify(expected), 2_048 - visibleTokens, requests)]);++ await expect(requestRefinement(harness, kind)).rejects.toThrow("output budget was exhausted");+ expect(requests[0].maxTokens).toBe(2_048);+ expect(harness.faux.state.callCount).toBe(1);+ },+ );+});