Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

preserve reasoning in refinement and side questions (#2130)

kt81cd5390dbc8
diff --git a/packages/ai/.changes/eng-6010-auxiliary-reasoning.md b/packages/ai/.changes/eng-6010-auxiliary-reasoning.md
new file mode 100644
index 0000000000..7d559fafe7
--- /dev/null
+++ b/packages/ai/.changes/eng-6010-auxiliary-reasoning.md
@@ -0,0 +1 @@
+- Fixed budget-based Claude requests sending thinking budgets below the provider minimum when the output limit is small.
diff --git a/packages/ai/src/providers/simple-options.ts b/packages/ai/src/providers/simple-options.ts
index 51f5189074..56e01af1c9 100644
--- a/packages/ai/src/providers/simple-options.ts
+++ b/packages/ai/src/providers/simple-options.ts
@@ -37,12 +37,16 @@ export function adjustMaxTokensForThinking(
const budgets = { ...defaultBudgets, ...customBudgets };
const minOutputTokens = 1024;
+ const minThinkingTokens = 1024;
const level = clampReasoning(reasoningLevel)!;
- let thinkingBudget = budgets[level]!;
+ let thinkingBudget = Math.max(minThinkingTokens, budgets[level]!);
const maxTokens = Math.min(baseMaxTokens + thinkingBudget, modelMaxTokens);
+ if (maxTokens <= minThinkingTokens) {
+ throw new Error("Budget-based thinking requires at least 1024 thinking tokens plus room for the response");
+ }
if (maxTokens <= thinkingBudget) {
- thinkingBudget = Math.max(0, maxTokens - minOutputTokens);
+ thinkingBudget = Math.max(minThinkingTokens, maxTokens - minOutputTokens);
}
return { maxTokens, thinkingBudget };
diff --git a/packages/ai/test/anthropic-minimum-thinking-budget.test.ts b/packages/ai/test/anthropic-minimum-thinking-budget.test.ts
new file mode 100644
index 0000000000..b7632ba09d
--- /dev/null
+++ b/packages/ai/test/anthropic-minimum-thinking-budget.test.ts
@@ -0,0 +1,41 @@
+import { describe, expect, it } from "vitest";
+import { getModel } from "../src/models.js";
+import { streamSimpleAnthropic } from "../src/providers/anthropic.js";
+import { adjustMaxTokensForThinking } from "../src/providers/simple-options.js";
+
+describe("budget-based Anthropic thinking minimum", () => {
+ it.each([1025, 1500, 2048, 4096])("sends a valid thinking budget under a %i-token ceiling", async (maxTokens) => {
+ let payload: { max_tokens: number; thinking: { type: string; budget_tokens: number } } | undefined;
+ const model = { ...getModel("anthropic", "claude-sonnet-4-5"), maxTokens };
+ const stream = streamSimpleAnthropic(
+ model,
+ { messages: [{ role: "user", content: "Hello", timestamp: 1 }] },
+ {
+ apiKey: "dummy-key",
+ reasoning: "low",
+ maxTokens,
+ onPayload(value) {
+ payload = value as typeof payload;
+ throw new Error("Captured before network dispatch");
+ },
+ },
+ );
+ await stream.result();
+ expect(payload).toBeDefined();
+ expect(payload!.thinking.type).toBe("enabled");
+ expect(payload!.thinking.budget_tokens).toBeGreaterThanOrEqual(1024);
+ expect(payload!.thinking.budget_tokens).toBeLessThan(payload!.max_tokens);
+ expect(payload!.max_tokens).toBe(maxTokens);
+ });
+
+ it.each([1, 476, 1024])("rejects an impossible %i-token model limit before constructing a request", (maxTokens) => {
+ expect(() => adjustMaxTokensForThinking(maxTokens, maxTokens, "low")).toThrow("thinking requires at least");
+ });
+
+ it("honors the thinking minimum for an undersized custom budget", () => {
+ expect(adjustMaxTokensForThinking(4096, 8192, "low", { low: 476 })).toEqual({
+ maxTokens: 5120,
+ thinkingBudget: 1024,
+ });
+ });
+});
diff --git a/packages/coding-agent/.changes/eng-6010-auxiliary-reasoning.md b/packages/coding-agent/.changes/eng-6010-auxiliary-reasoning.md
new file mode 100644
index 0000000000..5d6ea142f8
--- /dev/null
+++ b/packages/coding-agent/.changes/eng-6010-auxiliary-reasoning.md
@@ -0,0 +1,3 @@
+- Fixed refinement and side questions disabling reasoning when the session uses it, while respecting the selected model's supported thinking levels.
+- Removed fixed output caps from refinement with reasoning enabled, while respecting the model's output limit and reserving context space for the prompt.
+- Fixed long refinements exceeding the context window or being rejected prematurely, keeping recent conversation text and space for the response.
diff --git a/packages/coding-agent/src/core/refinement/refinement.ts b/packages/coding-agent/src/core/refinement/refinement.ts
index 5242819411..ce15ade8d8 100644
--- a/packages/coding-agent/src/core/refinement/refinement.ts
+++ b/packages/coding-agent/src/core/refinement/refinement.ts
@@ -1,7 +1,7 @@
import { appendFileSync, existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
import { join } from "node:path";
import type { AgentMessage, ThinkingLevel } from "@earendil-works/pi-agent-core";
-import type { Model } from "@earendil-works/pi-ai";
+import type { Api, Model } from "@earendil-works/pi-ai";
import { completeSimple } from "@earendil-works/pi-ai";
import { getAgentDir } from "../../config.js";
import { realpathIfPresentSync, writeFileAtomicSync } from "../../utils/atomic-file.js";
@@ -9,6 +9,7 @@ import { serializeConversation } from "../compaction/utils.js";
import { convertToLlm } from "../messages.js";
import { completeWithProviderRetry, type ProviderRetryPolicy } from "../provider-retry.js";
import type { CustomEntry } from "../session-manager.js";
+import { getAuxiliaryThinkingLevel } from "../thinking-levels.js";
export const REFINEMENT_CUSTOM_TYPE = "prime-agent.refinement";
@@ -177,24 +178,59 @@ Return JSON only:
"instructions": "optional concise instructions for /refine if shouldRefine is true"
}`;
-/**
- * Output budgets are derived from the selected model instead of fixed literals.
- * /refine input scales with harness size (entry overview, refinement history, and
- * the trajectory slice), so a constant output cap silently truncates exactly the
- * large multi-edit proposals that matter most. Math.min keeps small models honest.
- */
+// These caps apply only with reasoning off; thinking and JSON otherwise share the model's output budget.
const REFINEMENT_MAX_OUTPUT_TOKENS = 32_000;
const AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS = 4_096;
+const REFINEMENT_CONTEXT_OVERHEAD_TOKENS = 1_024;
const TRUNCATED_JSON_ERROR =
"the model stopped before completing its JSON object. This usually means the output budget was exhausted; retry with a smaller request.";
-function refinementMaxOutputTokens(model: Model<any>): number {
- return Math.min(model.maxTokens, REFINEMENT_MAX_OUTPUT_TOKENS);
+function refinementInputTokenBound(text: string): number {
+ // One token per UTF-8 byte bounds byte-based tokenizers, including dense or unusual text.
+ return Buffer.byteLength(text, "utf8");
}
-function autoRefineReviewMaxOutputTokens(model: Model<any>): number {
- return Math.min(model.maxTokens, AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS);
+function refinementRequest(
+ model: Model<Api>,
+ systemPrompt: string,
+ conversationText: string,
+ buildPrompt: (conversation: string) => string,
+ outputReserve: number,
+): { model: Model<Api>; userPrompt: string } {
+ const systemReserve = refinementInputTokenBound(systemPrompt) + REFINEMENT_CONTEXT_OVERHEAD_TOKENS;
+ const inputBudget =
+ model.contextWindow - Math.min(model.maxTokens, outputReserve, Math.floor(model.contextWindow / 2));
+ let userPrompt = buildPrompt(conversationText);
+ if (systemReserve + refinementInputTokenBound(userPrompt) > inputBudget && conversationText.length > 0) {
+ const promptForLength = (length: number): string => {
+ let start = conversationText.length - length;
+ const first = conversationText.charCodeAt(start);
+ if (first >= 0xdc00 && first <= 0xdfff) start++;
+ return buildPrompt(
+ `[Earlier conversation omitted to fit the model context.]\n${conversationText.slice(start)}`,
+ );
+ };
+ let low = 0;
+ let high = conversationText.length;
+ while (low < high) {
+ const length = Math.ceil((low + high) / 2);
+ if (systemReserve + refinementInputTokenBound(promptForLength(length)) <= inputBudget) low = length;
+ else high = length - 1;
+ }
+ userPrompt = promptForLength(low);
+ }
+ const maxTokens = Math.min(
+ model.maxTokens,
+ model.contextWindow - systemReserve - refinementInputTokenBound(userPrompt),
+ );
+ if (maxTokens <= 0) {
+ throw new Error(
+ "Refinement prompt leaves no room for output in the model's context window; retry with a smaller request.",
+ );
+ }
+ // Bound the request's model ceiling too: some adapters add thinking tokens before clamping to it.
+ return { model: { ...model, maxTokens }, userPrompt };
}
function now(): string {
@@ -910,32 +946,43 @@ export async function planRefinement(
const scopeInstruction = options.global
? "Requested refinement scope: global. Only propose stable cross-session continual harness edits, durable user preferences, reusable skills/subagents, or explicitly project-qualified facts that should affect future Prime Agent sessions. Do not persist session-only progress, temporary blockers, or current-run coordination globally."
: "Requested refinement scope: local. Prefer local continual harness edits for current task progress, temporary blockers, current-run coordination, and project facts that are not clearly reusable across Prime Agent sessions. Global entries in the overview are read-only context: do not propose update or delete edits for them; create a local entry instead if an override is needed.";
- const userPrompt = [
- `<current_harness_state>\n${overviewForPrompt(state)}\n</current_harness_state>`,
- `<refinement_history>\n${historyForPrompt(history)}\n</refinement_history>`,
- `<conversation>\n${conversationText}\n</conversation>`,
- `<scope_policy>\n${scopeInstruction}\n</scope_policy>`,
- options.instructions ? `<user_refine_instructions>\n${options.instructions}\n</user_refine_instructions>` : "",
- "Return only JSON edits. If no useful edit is justified, return an empty edits array with a rationale.",
- ]
- .filter(Boolean)
- .join("\n\n");
+ const buildPrompt = (conversation: string): string =>
+ [
+ `<current_harness_state>\n${overviewForPrompt(state)}\n</current_harness_state>`,
+ `<refinement_history>\n${historyForPrompt(history)}\n</refinement_history>`,
+ `<conversation>\n${conversation}\n</conversation>`,
+ `<scope_policy>\n${scopeInstruction}\n</scope_policy>`,
+ options.instructions ? `<user_refine_instructions>\n${options.instructions}\n</user_refine_instructions>` : "",
+ "Return only JSON edits. If no useful edit is justified, return an empty edits array with a rationale.",
+ ]
+ .filter(Boolean)
+ .join("\n\n");
+ const reasoning = getAuxiliaryThinkingLevel(model, thinkingLevel);
+ const { model: requestModel, userPrompt } = refinementRequest(
+ model,
+ REFINEMENT_SYSTEM_PROMPT,
+ conversationText,
+ buildPrompt,
+ reasoning === "off" ? REFINEMENT_MAX_OUTPUT_TOKENS : model.maxTokens,
+ );
+ const maxTokens =
+ reasoning === "off" ? Math.min(requestModel.maxTokens, REFINEMENT_MAX_OUTPUT_TOKENS) : requestModel.maxTokens;
- // /refine requires a parseable JSON object in the final text. Some reasoning-capable
- // OpenAI-compatible models can spend the response on visible thinking and return no
- // final text, which makes otherwise successful daemon /refine calls fail parsing.
- // Keep the refinement request non-reasoning regardless of the interactive session
- // thinking level so the model uses its output budget for the JSON object.
- void thinkingLevel;
const response = await completeWithProviderRetry(
() =>
completeSimple(
- model,
+ requestModel,
{
systemPrompt: REFINEMENT_SYSTEM_PROMPT,
messages: [{ role: "user", content: [{ type: "text", text: userPrompt }], timestamp: Date.now() }],
},
- { maxTokens: refinementMaxOutputTokens(model), signal, apiKey, headers },
+ {
+ reasoning,
+ maxTokens,
+ signal,
+ apiKey,
+ headers,
+ },
),
{ policy: options.retry, signal },
);
@@ -980,33 +1027,49 @@ export async function reviewAutoRefine(
retry?: ProviderRetryPolicy,
): Promise<AutoRefineReview> {
const conversationText = serializeConversation(convertToLlm(messages)).slice(-40_000);
- const userPrompt = [
- `<trigger>
+ const buildPrompt = (conversation: string): string =>
+ [
+ `<trigger>
${context.reason}; ${context.turnsSinceLastReview} assistant turns since last auto-refine review
</trigger>`,
- `<current_harness_state>
+ `<current_harness_state>
${overviewForPrompt(state)}
</current_harness_state>`,
- `<refinement_history>
+ `<refinement_history>
${historyForPrompt(history)}
</refinement_history>`,
- `<conversation>
-${conversationText}
+ `<conversation>
+${conversation}
</conversation>`,
- "Return shouldRefine=true when the trajectory contains evidence useful to this session's future turns. Prefer local harness edits for current task progress, temporary blockers, and current-run coordination. Ask for global refinement only for durable cross-session lessons or explicitly project-qualified facts likely to be reused in future sessions.",
- ].join("\n\n");
- // Auto-refine review requires parseable JSON. Keep it non-reasoning so
- // reasoning-capable models use final text budget for the JSON object.
- void thinkingLevel;
+ "Return shouldRefine=true when the trajectory contains evidence useful to this session's future turns. Prefer local harness edits for current task progress, temporary blockers, and current-run coordination. Ask for global refinement only for durable cross-session lessons or explicitly project-qualified facts likely to be reused in future sessions.",
+ ].join("\n\n");
+ const reasoning = getAuxiliaryThinkingLevel(model, thinkingLevel);
+ const { model: requestModel, userPrompt } = refinementRequest(
+ model,
+ AUTO_REFINE_REVIEW_SYSTEM_PROMPT,
+ conversationText,
+ buildPrompt,
+ reasoning === "off" ? AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS : model.maxTokens,
+ );
+ const maxTokens =
+ reasoning === "off"
+ ? Math.min(requestModel.maxTokens, AUTO_REFINE_REVIEW_MAX_OUTPUT_TOKENS)
+ : requestModel.maxTokens;
const response = await completeWithProviderRetry(
() =>
completeSimple(
- model,
+ requestModel,
{
systemPrompt: AUTO_REFINE_REVIEW_SYSTEM_PROMPT,
messages: [{ role: "user", content: [{ type: "text", text: userPrompt }], timestamp: Date.now() }],
},
- { maxTokens: autoRefineReviewMaxOutputTokens(model), signal, apiKey, headers },
+ {
+ reasoning,
+ maxTokens,
+ signal,
+ apiKey,
+ headers,
+ },
),
{ policy: retry, signal },
);
diff --git a/packages/coding-agent/src/core/side-question.ts b/packages/coding-agent/src/core/side-question.ts
index 80e561fbf0..1ab9ea1b6d 100644
--- a/packages/coding-agent/src/core/side-question.ts
+++ b/packages/coding-agent/src/core/side-question.ts
@@ -6,6 +6,7 @@ import {
type ProviderRetryPolicy,
} from "./provider-retry.js";
import { unwrapSemanticEdgeStreamFn } from "./semantic-edges.js";
+import { getAuxiliaryThinkingLevel } from "./thinking-levels.js";
export type SideQuestionStatus = "running" | "complete" | "cancelled" | "error";
@@ -90,7 +91,7 @@ export function startSideQuestion(
model,
systemPrompt: parent.state.systemPrompt,
messages: [...structuredClone(parent.state.messages), ...previousTurnMessages],
- thinkingLevel: "off",
+ thinkingLevel: getAuxiliaryThinkingLevel(model, parent.state.thinkingLevel),
serviceTier: parent.state.serviceTier,
tools: [],
},
diff --git a/packages/coding-agent/src/core/thinking-levels.ts b/packages/coding-agent/src/core/thinking-levels.ts
index e7c76805f5..6243b524d5 100644
--- a/packages/coding-agent/src/core/thinking-levels.ts
+++ b/packages/coding-agent/src/core/thinking-levels.ts
@@ -1,3 +1,11 @@
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
+import { type Api, clampThinkingLevel, type Model } from "@earendil-works/pi-ai";
export const THINKING_LEVELS: ThinkingLevel[] = ["off", "minimal", "low", "medium", "high", "xhigh", "max"];
+
+export function getAuxiliaryThinkingLevel(model: Model<Api>, thinkingLevel: ThinkingLevel = "low"): ThinkingLevel {
+ // Keep auxiliary calls inexpensive without disabling an enabled session's reasoning.
+ // Clamping may raise this preference when the model requires a higher effort.
+ const preferred = thinkingLevel === "off" || thinkingLevel === "minimal" ? thinkingLevel : "low";
+ return clampThinkingLevel(model, preferred);
+}
diff --git a/packages/coding-agent/test/refinement.test.ts b/packages/coding-agent/test/refinement.test.ts
index 19a7313975..d8e2731859 100644
--- a/packages/coding-agent/test/refinement.test.ts
+++ b/packages/coding-agent/test/refinement.test.ts
@@ -981,7 +981,7 @@ describe("harness refinement", () => {
expect(state.entries.prompt.base_system_prompt).toBeUndefined();
});
- it("requests JSON refinement without model reasoning even when session thinking is enabled", async () => {
+ it("requests JSON refinement with low reasoning when session thinking is enabled", async () => {
const state = loadHarnessState(makeTempDir());
completeSimpleMock.mockResolvedValueOnce(
assistantText(
@@ -1031,11 +1031,11 @@ describe("harness refinement", () => {
});
// Budget is derived from the model (8192) rather than a fixed literal.
expect(completeSimpleMock.mock.calls[0][2]).toMatchObject({
+ reasoning: "low",
maxTokens: 8192,
apiKey: "api-key",
headers: { "x-test-header": "1" },
});
- expect(completeSimpleMock.mock.calls[0][2]).not.toHaveProperty("reasoning");
expect(result.appliedEdits[0]).toMatchObject({
action: "create",
kind: "memory",
diff --git a/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.ts b/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.ts
new file mode 100644
index 0000000000..a6231804cd
--- /dev/null
+++ b/packages/coding-agent/test/suite/regressions/6010-auxiliary-reasoning.test.ts
@@ -0,0 +1,140 @@
+import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
+import { fauxAssistantMessage, type Model, type SimpleStreamOptions } from "@earendil-works/pi-ai";
+import { afterEach, describe, expect, it } from "vitest";
+import { type HarnessState, planRefinement, reviewAutoRefine } from "../../../src/core/refinement/refinement.js";
+import { type SideQuestionEvent, startSideQuestion } from "../../../src/core/side-question.js";
+import { createHarness, type Harness } from "../harness.js";
+
+const harnesses: Harness[] = [];
+const noRetry = { enabled: false, maxRetries: 0, baseDelayMs: 1, maxRetryDelayMs: 1 };
+const proposal = { summary: "No change", rationale: "Fixture", expectedOutcome: "No change", edits: [] };
+const review = { shouldRefine: false, rationale: "No reusable lesson" };
+
+afterEach(() => {
+ for (const harness of harnesses.splice(0)) harness.cleanup();
+});
+
+interface ReasoningCase {
+ name: string;
+ reasoning: boolean;
+ parentLevel: ThinkingLevel;
+ expectedLevel: ThinkingLevel;
+ omitRefinementLevel?: boolean;
+ thinkingLevelMap?: Model<string>["thinkingLevelMap"];
+}
+
+const cases: ReasoningCase[] = [
+ {
+ name: "enabled session with incomplete capability metadata",
+ reasoning: true,
+ parentLevel: "high",
+ expectedLevel: "low",
+ },
+ { name: "minimal session", reasoning: true, parentLevel: "minimal", expectedLevel: "minimal" },
+ { name: "explicit off supported", reasoning: true, parentLevel: "off", expectedLevel: "off" },
+ { name: "non-reasoning model", reasoning: false, parentLevel: "high", expectedLevel: "off" },
+ {
+ name: "refinement caller without a thinking preference",
+ reasoning: true,
+ parentLevel: "high",
+ expectedLevel: "low",
+ omitRefinementLevel: true,
+ },
+ {
+ name: "off and minimal unsupported",
+ reasoning: true,
+ parentLevel: "off",
+ expectedLevel: "low",
+ thinkingLevelMap: { off: null, minimal: null },
+ },
+ {
+ name: "only high supported",
+ reasoning: true,
+ parentLevel: "high",
+ expectedLevel: "high",
+ thinkingLevelMap: { off: null, minimal: null, low: null, medium: null },
+ },
+];
+
+describe("auxiliary reasoning settings", () => {
+ it.each(cases)("completes refinement, review, and side questions: $name", async (testCase) => {
+ const harness = await createHarness({
+ models: [
+ { id: "reasoning-contract", reasoning: testCase.reasoning, contextWindow: 262_144, maxTokens: 128_000 },
+ ],
+ });
+ harnesses.push(harness);
+ const parent = harness.session.agent;
+ const model = harness.getModel();
+ model.thinkingLevelMap = testCase.thinkingLevelMap;
+ parent.state.thinkingLevel = testCase.parentLevel;
+ const parentMessages = structuredClone(parent.state.messages);
+ const observed: SimpleStreamOptions[] = [];
+ harness.setResponses(
+ [JSON.stringify(proposal), JSON.stringify(review), "Side answer"].map((text) => (_context, options) => {
+ const request = options as SimpleStreamOptions;
+ observed.push(request);
+ if (request.reasoning !== testCase.expectedLevel) {
+ return fauxAssistantMessage("", {
+ stopReason: "error",
+ errorMessage: `Unsupported auxiliary reasoning: ${request.reasoning}`,
+ });
+ }
+ return fauxAssistantMessage([
+ { type: "thinking", thinking: "Fixture reasoning" },
+ { type: "text", text },
+ ]);
+ }),
+ );
+ const state: HarnessState = {
+ schema: 1,
+ entries: { prompt: {}, memory: {}, skill: {}, subagent: {} },
+ refinements: [],
+ };
+ const messages = [{ role: "user" as const, content: "Remember the result", timestamp: 1 }];
+ await expect(
+ planRefinement(
+ messages,
+ state,
+ [],
+ model,
+ "faux-key",
+ { retry: noRetry },
+ undefined,
+ undefined,
+ testCase.omitRefinementLevel ? undefined : testCase.parentLevel,
+ ),
+ ).resolves.toMatchObject({ proposal });
+ await expect(
+ reviewAutoRefine(
+ messages,
+ state,
+ [],
+ model,
+ "faux-key",
+ { reason: "turn_interval", turnsSinceLastReview: 5 },
+ undefined,
+ undefined,
+ testCase.omitRefinementLevel ? undefined : testCase.parentLevel,
+ noRetry,
+ ),
+ ).resolves.toMatchObject(review);
+ const events: SideQuestionEvent[] = [];
+ await startSideQuestion(
+ parent,
+ "side",
+ "What happened?",
+ (event) => {
+ events.push(event);
+ },
+ [],
+ noRetry,
+ ).done;
+ expect(events.at(-1)).toMatchObject({ status: "complete", answer: "Side answer" });
+ expect(observed.map((options) => options.reasoning)).toEqual(Array(3).fill(testCase.expectedLevel));
+ expect(observed[0].maxTokens).toBe(testCase.expectedLevel === "off" ? 32_000 : model.maxTokens);
+ expect(observed[1].maxTokens).toBe(testCase.expectedLevel === "off" ? 4_096 : model.maxTokens);
+ expect(parent.state.thinkingLevel).toBe(testCase.parentLevel);
+ expect(parent.state.messages).toEqual(parentMessages);
+ });
+});
diff --git a/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.ts b/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.ts
new file mode 100644
index 0000000000..d3ec65b32b
--- /dev/null
+++ b/packages/coding-agent/test/suite/regressions/6010-refinement-output-budget.test.ts
@@ -0,0 +1,264 @@
+import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
+import {
+ completeSimple,
+ type FauxResponseFactory,
+ fauxAssistantMessage,
+ type SimpleStreamOptions,
+} from "@earendil-works/pi-ai";
+import { afterEach, describe, expect, it } from "vitest";
+import { type HarnessState, planRefinement, reviewAutoRefine } from "../../../src/core/refinement/refinement.js";
+import { createHarness, getMessageText, type Harness } from "../harness.js";
+
+const harnesses: Harness[] = [];
+const retry = { enabled: true, maxRetries: 2, baseDelayMs: 1, maxRetryDelayMs: 1 };
+const proposal = { summary: "No change", rationale: "Fixture", expectedOutcome: "No change", edits: [] };
+const review = { shouldRefine: false, rationale: "No reusable lesson" };
+const paths = [
+ { kind: "plan", formerCap: 32_000, expected: proposal },
+ { kind: "review", formerCap: 4_096, expected: review },
+] as const;
+
+afterEach(() => {
+ for (const harness of harnesses.splice(0)) harness.cleanup();
+});
+
+async function requestRefinement(
+ harness: Harness,
+ kind: "plan" | "review",
+ content = "",
+ thinkingLevel: ThinkingLevel = "high",
+) {
+ const state: HarnessState = {
+ schema: 1,
+ entries: { prompt: {}, memory: {}, skill: {}, subagent: {} },
+ refinements: [],
+ };
+ const messages = content ? [{ role: "user" as const, content, timestamp: 1 }] : [];
+ if (kind === "plan") {
+ const result = await planRefinement(
+ messages,
+ state,
+ [],
+ harness.getModel(),
+ "faux-key",
+ { retry },
+ undefined,
+ undefined,
+ thinkingLevel,
+ );
+ return result.proposal;
+ }
+ return reviewAutoRefine(
+ messages,
+ state,
+ [],
+ harness.getModel(),
+ "faux-key",
+ { reason: "turn_interval", turnsSinceLastReview: 5 },
+ undefined,
+ undefined,
+ thinkingLevel,
+ retry,
+ );
+}
+
+function budgetedResponse(text: string, reasoningTokens: number, requests: SimpleStreamOptions[]): FauxResponseFactory {
+ return (_context, options, _state, model) => {
+ const request = options as SimpleStreamOptions;
+ requests.push(request);
+ // Synthetic tokens: one JSON character per token, after a fixed reasoning cost.
+ const totalBudget = Math.min(request.maxTokens ?? 32_000, model.maxTokens);
+ const visible = text.slice(0, Math.max(0, totalBudget - reasoningTokens));
+ return fauxAssistantMessage(
+ [
+ { type: "thinking", thinking: "Condensed fixture reasoning" },
+ ...(visible ? [{ type: "text" as const, text: visible }] : []),
+ ],
+ { stopReason: visible === text ? "stop" : "length" },
+ );
+ };
+}
+
+describe.each(paths)("$kind shared reasoning and JSON budget", ({ kind, formerCap, expected }) => {
+ it.each(["off", "high"] as const)(
+ "accepts a long English trajectory on a 64k model with reasoning %s",
+ async (thinkingLevel) => {
+ const harness = await createHarness({
+ models: [{ id: "glm-context", reasoning: true, contextWindow: 65_536, maxTokens: 16_384 }],
+ });
+ harnesses.push(harness);
+ const content = `${"The project now stores the result locally. ".repeat(2_000)}Latest fact: preserve local state.`;
+ let observedPrompt = "";
+ const requests: SimpleStreamOptions[] = [];
+ harness.setResponses([
+ (context, options) => {
+ observedPrompt = context.messages.map(getMessageText).join("");
+ requests.push(options as SimpleStreamOptions);
+ return fauxAssistantMessage(JSON.stringify(expected));
+ },
+ ]);
+ await expect(requestRefinement(harness, kind, content, thinkingLevel)).resolves.toEqual(expected);
+ expect(observedPrompt).toContain(content.slice(-39_000));
+ expect(requests[0].maxTokens).toBe(thinkingLevel === "off" ? Math.min(formerCap, 16_384) : 16_384);
+ },
+ );
+
+ it.each(["off", "high"] as const)("accepts a full multibyte trajectory with reasoning %s", async (thinkingLevel) => {
+ const harness = await createHarness({
+ models: [{ id: "multibyte-context", reasoning: true, contextWindow: 200_000, maxTokens: 65_536 }],
+ });
+ harnesses.push(harness);
+ const content = `${"项目记录".repeat(20_000)}最新事实`;
+ harness.setResponses([fauxAssistantMessage(JSON.stringify(expected))]);
+ await expect(requestRefinement(harness, kind, content, thinkingLevel)).resolves.toEqual(expected);
+ });
+
+ it.each(["界", "😀"])(
+ "retains recent %s context and valid Unicode when shortening a dense trajectory",
+ async (character) => {
+ const harness = await createHarness({
+ models: [{ id: "dense-context", reasoning: true, contextWindow: 32_768, maxTokens: 8_192 }],
+ });
+ harnesses.push(harness);
+ const content = `${character.repeat(80_000)}Latest fact: retain the newest decision.`;
+ let observedPrompt = "";
+ let maxTokens: number | undefined;
+ harness.setResponses([
+ (context, options) => {
+ observedPrompt = context.messages.map(getMessageText).join("");
+ maxTokens = (options as SimpleStreamOptions).maxTokens;
+ return fauxAssistantMessage(JSON.stringify(expected));
+ },
+ ]);
+ await expect(requestRefinement(harness, kind, content)).resolves.toEqual(expected);
+ expect(observedPrompt).toContain("Earlier conversation omitted");
+ expect(observedPrompt).toContain("Latest fact: retain the newest decision.");
+ expect(observedPrompt).toContain(kind === "plan" ? "<scope_policy>" : "<trigger>");
+ expect([...observedPrompt].some((point) => point.length === 1 && /[\ud800-\udfff]/u.test(point))).toBe(false);
+ expect(maxTokens).toBe(8_192);
+ },
+ );
+
+ it("leaves at least 60k output tokens for a long English trajectory on a 128k model", async () => {
+ const harness = await createHarness({
+ models: [{ id: "wide-context", reasoning: true, contextWindow: 128_000, maxTokens: 100_000 }],
+ });
+ harnesses.push(harness);
+ const requests: SimpleStreamOptions[] = [];
+ harness.setResponses([budgetedResponse(JSON.stringify(expected), 60_000, requests)]);
+ await expect(requestRefinement(harness, kind, "Keep the useful project facts. ".repeat(3_000))).resolves.toEqual(
+ expected,
+ );
+ expect(requests[0].maxTokens).toBeGreaterThan(60_000);
+ });
+ it.each([0, 8])("completes when the former cap leaves only %i JSON tokens", async (visibleTokens) => {
+ const harness = await createHarness({
+ models: [{ id: "shared-output-budget", reasoning: true, maxTokens: 65_536 }],
+ });
+ harnesses.push(harness);
+ const requests: SimpleStreamOptions[] = [];
+ const response = budgetedResponse(JSON.stringify(expected), formerCap - visibleTokens, requests);
+ harness.setResponses([response, response]);
+
+ const control = await completeSimple(
+ harness.getModel(),
+ { messages: [] },
+ {
+ apiKey: "faux-key",
+ reasoning: "low",
+ maxTokens: formerCap,
+ },
+ );
+ expect(control.stopReason).toBe("length");
+ expect(
+ control.content
+ .filter((block) => block.type === "text")
+ .map((block) => block.text)
+ .join(""),
+ ).toBe(JSON.stringify(expected).slice(0, visibleTokens));
+
+ await expect(requestRefinement(harness, kind)).resolves.toEqual(expected);
+ expect(requests.map((request) => request.maxTokens)).toEqual([formerCap, 65_536]);
+ expect(requests.map((request) => request.reasoning)).toEqual(["low", "low"]);
+ expect(harness.faux.state.callCount).toBe(2);
+ });
+
+ it("uses a small model's actual output limit", async () => {
+ const harness = await createHarness({ models: [{ id: "small-budget", reasoning: true, maxTokens: 2_048 }] });
+ harnesses.push(harness);
+ const requests: SimpleStreamOptions[] = [];
+ harness.setResponses([budgetedResponse(JSON.stringify(expected), 1_024, requests)]);
+
+ await expect(requestRefinement(harness, kind)).resolves.toEqual(expected);
+ expect(requests[0].maxTokens).toBe(2_048);
+ expect(harness.faux.state.callCount).toBe(1);
+ });
+
+ it.each([
+ { name: "empty trajectory", content: "", addedThinkingTokens: 0 },
+ { name: "short English words", content: "a ".repeat(40_000), addedThinkingTokens: 0 },
+ { name: "ASCII punctuation", content: "!@#$%^&*()".repeat(8_000), addedThinkingTokens: 0 },
+ { name: "multibyte trajectory", content: "界".repeat(15_000), addedThinkingTokens: 0 },
+ { name: "separate thinking allowance", content: "界".repeat(15_000), addedThinkingTokens: 16_384 },
+ ])(
+ "reserves prompt context for equal context and output limits: $name",
+ async ({ content, addedThinkingTokens }) => {
+ const harness = await createHarness({
+ models: [{ id: "shared-context-limit", reasoning: true, contextWindow: 131_072, maxTokens: 131_072 }],
+ });
+ harnesses.push(harness);
+ const originalModel = structuredClone(harness.getModel());
+ const requests: SimpleStreamOptions[] = [];
+ let totalTokens = 0;
+ harness.setResponses([
+ (context, options, _state, model) => {
+ const request = options as SimpleStreamOptions;
+ requests.push(request);
+ // Synthetic tokenizer: English words, CJK characters, and punctuation, plus message framing.
+ const inputTokens =
+ [context.systemPrompt ?? "", ...context.messages.map(getMessageText)]
+ .join("")
+ .match(/[\u4e00-\u9fff]|[A-Za-z0-9_]+|[^\s]/gu)!.length + 256;
+ // Some adapters add thinking tokens before clamping to the supplied model ceiling.
+ const wireMaxTokens = Math.min((request.maxTokens ?? 32_000) + addedThinkingTokens, model.maxTokens);
+ totalTokens = inputTokens + wireMaxTokens;
+ return fauxAssistantMessage(JSON.stringify(expected));
+ },
+ ]);
+
+ await expect(requestRefinement(harness, kind, content)).resolves.toEqual(expected);
+ expect(totalTokens).toBeLessThanOrEqual(originalModel.contextWindow);
+ expect(requests[0].maxTokens).toBeGreaterThan(formerCap);
+ expect(requests[0].reasoning).toBe("low");
+ expect(harness.getModel()).toEqual(originalModel);
+ expect(harness.faux.state.callCount).toBe(1);
+ },
+ );
+
+ it("rejects a prompt with no reserved output space before calling the provider", async () => {
+ const harness = await createHarness({
+ models: [{ id: "no-context-space", reasoning: true, contextWindow: 1_024, maxTokens: 1_024 }],
+ });
+ harnesses.push(harness);
+ harness.setResponses([fauxAssistantMessage(JSON.stringify(expected))]);
+
+ await expect(requestRefinement(harness, kind, "界".repeat(1_024))).rejects.toThrow("no room for output");
+ expect(harness.faux.state.callCount).toBe(0);
+ });
+
+ it.each([0, 8])(
+ "reports exhaustion at the model limit with %i JSON tokens without retrying",
+ async (visibleTokens) => {
+ const harness = await createHarness({
+ models: [{ id: "exhausted-budget", reasoning: true, maxTokens: 2_048 }],
+ });
+ harnesses.push(harness);
+ const requests: SimpleStreamOptions[] = [];
+ harness.setResponses([budgetedResponse(JSON.stringify(expected), 2_048 - visibleTokens, requests)]);
+
+ await expect(requestRefinement(harness, kind)).rejects.toThrow("output budget was exhausted");
+ expect(requests[0].maxTokens).toBe(2_048);
+ expect(harness.faux.state.callCount).toBe(1);
+ },
+ );
+});