A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
[RSI, performance] compaction: summarize via auxiliaryModel without evicting the session prompt cache (#2411)
Seth Karten4f6df9d14385
diff --git a/packages/coding-agent/.changes/branch-summary-auxiliary-model.md b/packages/coding-agent/.changes/branch-summary-auxiliary-model.mdnew file mode 100644index 0000000000..1f69548989--- /dev/null+++ b/packages/coding-agent/.changes/branch-summary-auxiliary-model.md@@ -0,0 +1 @@+- Branch summaries now run through the `auxiliaryModel` setting, not just refinement and compaction passes. Branch summarization fires at a tree-navigation context boundary and the summarizer uses its own prompt prefix, so running it on the session model paid full input price at context peak for a one-off call outside the session's cached prefix. The summary falls back to the session model when the setting is unset, unusable, or its context window cannot hold the branch the summary covers plus the reserve the summary call needs.diff --git a/packages/coding-agent/.changes/compaction-summary-auxiliary-model.md b/packages/coding-agent/.changes/compaction-summary-auxiliary-model.mdnew file mode 100644index 0000000000..8d868da263--- /dev/null+++ b/packages/coding-agent/.changes/compaction-summary-auxiliary-model.md@@ -0,0 +1,2 @@+- Compaction summaries now run through the `auxiliaryModel` setting, not just refinement passes. Compaction fires at context peak and the summarizer uses its own prompt prefix (a different system prompt and no tools), so it cannot hit the session's cached prefix: on the session model the summary re-reads its whole input at peak price, plus a one-shot 1.25x cache write on Anthropic-style providers, and on OpenAI-style providers its divergent prefix still rides the session's `prompt_cache_key`, which is the documented way to depress hit rates. Routing through `auxiliaryModel` moves that call off the session model. The summary falls back to the session model when the setting is unset or unusable.+- Compaction summaries no longer request the session's thinking level. Summarizing is transcription rather than reasoning, and with no reasoning requested the summary call stays cheap and cannot trip an invalid reasoning effort on the summary model.diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.tsindex 388c29f2ab..d071bd4fc9 100644--- a/packages/coding-agent/src/core/agent-session.ts+++ b/packages/coding-agent/src/core/agent-session.ts@@ -110,7 +110,9 @@ import {calculateContextTokens,collectEntriesForBranchSummary,compact,+ estimateBranchSummaryRequestTokens,estimateContextTokens,+ estimateSummaryRequestTokens,generateBranchSummary,prepareCompaction,serializeConversation,@@ -8811,6 +8813,24 @@ export class AgentSession {if (extensionCompaction) {({ summary, firstKeptEntryId, tokensBefore, details, usage } = extensionCompaction);} else {+ // Compaction fires at context peak, and the summarizer runs with its own+ // prompt prefix (a different system prompt, no tools), so it cannot hit+ // the session's cached prefix: on the session model the summary re-reads+ // its whole input at peak price, and on OpenAI-style providers it rides+ // the session's prompt_cache_key with a divergent prefix, depressing+ // hit rates. Route summaries to the auxiliary model when one is configured.+ const summarization = (await this._resolveAuxiliaryModel(+ "compaction summary",+ { model, apiKey, headers },+ // The summary request serializes the whole conversation, so a+ // smaller auxiliary window must fall back to the session model+ // instead of failing over-limit and stranding the context.+ estimateSummaryRequestTokens(preparation, customInstructions),+ )) ?? {+ model,+ apiKey,+ headers,+ };// Each summary wire call gets its own request ID: split turns send two// different bodies, and one Idempotency-Key must never cover both. A slice// that succeeds on the wire stays uncommitted until the compaction itself@@ -8821,10 +8841,10 @@ export class AgentSession {): Promise<T> => {const requestId = this._semanticEdges.startCompactionRequest(semanticCompaction.compactionId);if (requestId === undefined) {- return call(headers);+ return call(summarization.headers);}try {- const result = await call({ ...headers, ...modelRequestHeaders(requestId) });+ const result = await call({ ...summarization.headers, ...modelRequestHeaders(requestId) });// A slice resolving after a sibling's rejection already settled the// compaction would push into a drained list and stay in-flight forever.if (compactionSettled) {@@ -8840,12 +8860,15 @@ export class AgentSession {};({ summary, firstKeptEntryId, tokensBefore, details, usage } = await compact(preparation,- model,- apiKey,- headers,+ summarization.model,+ summarization.apiKey,+ summarization.headers,customInstructions,signal,- this.thinkingLevel,+ // Summarizing is transcription, not reasoning: no thinking level is+ // requested, so the summary call stays cheap and cannot trip an invalid+ // reasoning effort for the summary model.+ undefined,summaryCall,providerRetryPolicy(this.settingsManager),this.sessionId,@@ -9355,23 +9378,36 @@ export class AgentSession {}/**- * Refinement passes (review and planning) run with their own prompts, so- * issuing them on the session model evicts the provider's prefix-cache entry- * for the session and forces a full context re-read on the next session- * request. Route them to the configured auxiliary model when it is set and- * usable; fall back to the session model otherwise.+ * Background LLM passes (refinement review and planning, compaction summaries,+ * branch summaries) run with their own prompts, so they cannot hit the+ * session's cached prefix: on the session model they re-read their whole input+ * at peak price, and on OpenAI-style providers a divergent prefix riding the+ * session's prompt_cache_key depresses hit rates. Route them to the configured+ * auxiliary model when it is set and usable; fall back to the session model+ * otherwise.+ *+ * Callers that already resolved the session request auth pass it as+ * `fallback` so the fallback path reuses it instead of resolving again.*/- private async _resolveRefinementModel(): Promise<- { model: Model<Api>; apiKey: string; headers?: Record<string, string> } | undefined- > {+ private async _resolveAuxiliaryModel(+ purpose: string,+ fallback?: { model: Model<Api>; apiKey: string; headers?: Record<string, string> },+ requiredContextTokens?: number,+ ): Promise<{ model: Model<Api>; apiKey: string; headers?: Record<string, string> } | undefined> {const sessionModel = this.model;if (!sessionModel) {- return undefined;+ return fallback;}- const selector = this.settingsManager.getAuxiliaryModel()?.trim().toLowerCase();- if (!selector || `${sessionModel.provider}/${sessionModel.id}`.toLowerCase() === selector) {+ const resolveSessionAuth = async () => {+ if (fallback) {+ return fallback;+ }const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(sessionModel);return { model: requestModel, apiKey, headers };+ };+ const selector = this.settingsManager.getAuxiliaryModel()?.trim().toLowerCase();+ if (!selector || `${sessionModel.provider}/${sessionModel.id}`.toLowerCase() === selector) {+ return await resolveSessionAuth();}try {const model = (await this._authenticatedRlmModels()).find(@@ -9381,13 +9417,27 @@ export class AgentSession {throw new Error(`model "${selector}" is unavailable, unauthenticated, or expired`);}const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(model);+ // Callers that know the size of the request they will issue pass it in:+ // an auxiliary model whose context window cannot hold that request+ // fails over-limit on the wire (e.g. a compaction summary covering the+ // whole conversation), leaving the caller unable to make progress. A+ // known window that is too small routes to the session model through the+ // warning below; an unknown window (<= 0) keeps the routing as-is+ // rather than guessing. The throw lands in the catch, whose logging+ // stays selector-only (CodeQL js/clear-text-logging).+ if (+ requiredContextTokens !== undefined &&+ requestModel.contextWindow > 0 &&+ requestModel.contextWindow < requiredContextTokens+ ) {+ throw new Error("auxiliary model context window is too small for the request");+ }return { model: requestModel, apiKey, headers };} catch {// Error details from the auth stack can embed credential material, so only// the selector is logged (CodeQL js/clear-text-logging).- console.warn(`Warning: auxiliaryModel "${selector}" unusable for refinement; using the session model.`);- const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(sessionModel);- return { model: requestModel, apiKey, headers };+ console.warn(`Warning: auxiliaryModel "${selector}" unusable for ${purpose}; using the session model.`);+ return await resolveSessionAuth();}}@@ -9395,7 +9445,7 @@ export class AgentSession {if (this._autoRefineReviewer) {return this._autoRefineReviewer(context, signal);}- const refinementModel = await this._resolveRefinementModel();+ const refinementModel = await this._resolveAuxiliaryModel("refinement");if (!refinementModel) {return { shouldRefine: false, rationale: "No model selected." };}@@ -9758,7 +9808,7 @@ export class AgentSession {throw new Error(formatNoModelSelectedMessage());}- const refinementModel = await this._resolveRefinementModel();+ const refinementModel = await this._resolveAuxiliaryModel("refinement");if (!refinementModel) {throw new Error(formatNoModelSelectedMessage());}@@ -14286,10 +14336,33 @@ export class AgentSession {if (options.summarize && entriesToSummarize.length > 0 && !extensionSummary) {const { apiKey, headers, requestModel: model } = await this._getRequiredRequestAuth(this.model!);const branchSummarySettings = this.settingsManager.getBranchSummarySettings();- const result = await generateBranchSummary(entriesToSummarize, {+ // Branch summary fires at a tree-navigation context boundary, and the+ // summarizer runs with its own prompt prefix (SUMMARIZATION_SYSTEM_PROMPT+ // plus the <conversation> wrapper), so it cannot hit the session's cached+ // prefix: on the session model the summary re-reads the whole branch at+ // peak price. Route it to the auxiliary model when one is configured.+ const summarization = (await this._resolveAuxiliaryModel(+ "branch summary",+ { model, apiKey, headers },+ // The estimator sizes the request the session model would issue for+ // the branch being left. A smaller auxiliary window must fall back+ // rather than truncate away branch context or fail over-limit and+ // strand the navigation.+ estimateBranchSummaryRequestTokens(entriesToSummarize, {+ contextWindow: model.contextWindow,+ reserveTokens: branchSummarySettings.reserveTokens,+ customInstructions,+ replaceInstructions,+ }),+ )) ?? {model,apiKey,headers,+ };+ const result = await generateBranchSummary(entriesToSummarize, {+ model: summarization.model,+ apiKey: summarization.apiKey,+ headers: summarization.headers,signal: this._branchSummaryAbortController.signal,sessionId: this.sessionId,customInstructions,diff --git a/packages/coding-agent/src/core/compaction/branch-summarization.ts b/packages/coding-agent/src/core/compaction/branch-summarization.tsindex af8ee00615..ccf93b84b0 100644--- a/packages/coding-agent/src/core/compaction/branch-summarization.ts+++ b/packages/coding-agent/src/core/compaction/branch-summarization.ts@@ -247,6 +247,40 @@ Use this EXACT format:Keep each section concise. Preserve exact file paths, function names, and error messages.`;+/** Completion budget for the branch summary wire call. */+const BRANCH_SUMMARY_MAX_TOKENS = 2048;++/**+ * Input-token budget the summarizer keeps: entries that do not fit the model's+ * window after the reserve are dropped. Shared by `generateBranchSummary` and+ * `estimateBranchSummaryRequestTokens` so both slice with the same budget.+ */+function branchSummaryTokenBudget(contextWindow: number | undefined, reserveTokens: number): number {+ return (contextWindow || 128000) - reserveTokens;+}++/**+ * Build the summarizer prompt for a serialized branch: the conversation in its+ * `<conversation>` wrapper plus the selected instructions. Shared with+ * `estimateBranchSummaryRequestTokens` so the estimate cannot drift from the+ * request `generateBranchSummary` issues.+ */+function buildBranchSummaryPrompt(+ conversationText: string,+ customInstructions?: string,+ replaceInstructions?: boolean,+): string {+ let instructions: string;+ if (replaceInstructions && customInstructions) {+ instructions = customInstructions;+ } else if (customInstructions) {+ instructions = `${BRANCH_SUMMARY_PROMPT}\n\nAdditional focus: ${customInstructions}`;+ } else {+ instructions = BRANCH_SUMMARY_PROMPT;+ }+ return `<conversation>\n${conversationText}\n</conversation>\n\n${instructions}`;+}+/*** Generate a summary of abandoned branch entries.*@@ -268,8 +302,7 @@ export async function generateBranchSummary(retry,reserveTokens = 16384,} = options;- const contextWindow = model.contextWindow || 128000;- const tokenBudget = contextWindow - reserveTokens;+ const tokenBudget = branchSummaryTokenBudget(model.contextWindow, reserveTokens);const { messages, fileOps } = prepareBranchEntries(entries, tokenBudget);@@ -278,17 +311,8 @@ export async function generateBranchSummary(return { summary: "No content to summarize" };}// Serialize before the LLM call so it summarizes rather than continues this branch.- const llmMessages = convertToLlm(messages);- const conversationText = serializeConversation(llmMessages);- let instructions: string;- if (replaceInstructions && customInstructions) {- instructions = customInstructions;- } else if (customInstructions) {- instructions = `${BRANCH_SUMMARY_PROMPT}\n\nAdditional focus: ${customInstructions}`;- } else {- instructions = BRANCH_SUMMARY_PROMPT;- }- const promptText = `<conversation>\n${conversationText}\n</conversation>\n\n${instructions}`;+ const conversationText = serializeConversation(convertToLlm(messages));+ const promptText = buildBranchSummaryPrompt(conversationText, customInstructions, replaceInstructions);const summarizationMessages = [{@@ -302,7 +326,7 @@ export async function generateBranchSummary(completeSimple(model,{ systemPrompt: SUMMARIZATION_SYSTEM_PROMPT, messages: summarizationMessages },- { apiKey, headers, sessionId, signal, maxTokens: 2048 },+ { apiKey, headers, sessionId, signal, maxTokens: BRANCH_SUMMARY_MAX_TOKENS },),{ policy: retry, signal },);@@ -328,3 +352,59 @@ export async function generateBranchSummary(usage: response.usage,};}++export interface EstimateBranchSummaryRequestTokensOptions {+ /** Context window of the model that would run the summary (default 128000) */+ contextWindow?: number;+ /** Tokens reserved for prompt + LLM response (default 16384) */+ reserveTokens?: number;+ /** Optional custom instructions for summarization */+ customInstructions?: string;+ /** If true, customInstructions replaces the default prompt instead of being appended */+ replaceInstructions?: boolean;+}++/**+ * Estimate the context window the branch summary needs, using the chars/4+ * heuristic this module already uses for pre-LLM token math. A model must hold+ * two things: the request body `generateBranchSummary` builds+ * (SUMMARIZATION_SYSTEM_PROMPT, the serialized branch inside its+ * `<conversation>` wrapper, and the completion budget) and the reserve the+ * branch call subtracts from its window via `branchSummaryTokenBudget`. The+ * window also sizes the input slice, so a model that covers only the request+ * body drops the oldest entries - or every entry, turning the summary into a+ * "No content to summarize" stub - instead of running the request the session+ * model would have run. A branch whose newest entry alone overflows the budget+ * slices to nothing, so no wire request is issued; the returned floor still+ * requires a window above the reserve, because a smaller window would slice+ * with a non-positive budget that prepareBranchEntries treats as unlimited.+ */+export function estimateBranchSummaryRequestTokens(+ entries: SessionEntry[],+ options: EstimateBranchSummaryRequestTokensOptions = {},+): number {+ const { contextWindow, reserveTokens = 16384, customInstructions, replaceInstructions } = options;+ // Mirrors generateBranchSummary: the same budget decides which entries fit.+ const tokenBudget = branchSummaryTokenBudget(contextWindow, reserveTokens);+ const { messages } = prepareBranchEntries(entries, tokenBudget);+ if (messages.length === 0) {+ // No wire request is issued for this shape, but a resolved model whose+ // window is at or below the reserve would slice with a non-positive+ // budget, which prepareBranchEntries treats as unlimited and would send+ // the whole branch over-limit. Require a window that keeps that budget+ // positive; larger windows slice their own budget and stay bounded.+ return reserveTokens + 1;+ }+ const promptText = buildBranchSummaryPrompt(+ serializeConversation(convertToLlm(messages)),+ customInstructions,+ replaceInstructions,+ );+ const promptTokens = Math.ceil(promptText.length / 4);+ // The completion budget and the input slice reserve are separate draws on the+ // same window, so the larger of the two decides whether the model fits.+ return Math.max(+ Math.ceil(SUMMARIZATION_SYSTEM_PROMPT.length / 4) + promptTokens + BRANCH_SUMMARY_MAX_TOKENS,+ promptTokens + reserveTokens,+ );+}diff --git a/packages/coding-agent/src/core/compaction/compaction.ts b/packages/coding-agent/src/core/compaction/compaction.tsindex a8216fe6d2..c7dee0df1f 100644--- a/packages/coding-agent/src/core/compaction/compaction.ts+++ b/packages/coding-agent/src/core/compaction/compaction.ts@@ -539,20 +539,16 @@ export async function generateSummary(sessionId?: string,recentStateAnchor?: string,): Promise<SummarySlice> {- const maxTokens = Math.floor(0.8 * reserveTokens);+ const maxTokens = historySummaryCompletionBudget(reserveTokens);- const basePrompt = buildSummarizationPrompt(customInstructions, previousSummary);// Serialize before the LLM call so it summarizes rather than continues this conversation.- const llmMessages = convertToLlm(currentMessages);- const conversationText = serializeConversation(llmMessages);- let promptText = `<conversation>\n${conversationText}\n</conversation>\n\n`;- if (previousSummary) {- promptText += `<previous-summary>\n${previousSummary}\n</previous-summary>\n\n`;- }- if (recentStateAnchor) {- promptText += `<recent-state-anchor>\nNewest assistant message that stays retained below the summary. The conversation to summarize is older than this anchor; the retained messages below are authoritative, so treat this anchor, not the conversation above, as the current state.\n\n${recentStateAnchor}\n</recent-state-anchor>\n\n`;- }- promptText += basePrompt;+ const conversationText = serializeConversation(convertToLlm(currentMessages));+ const promptText = buildHistorySummaryPrompt(+ conversationText,+ previousSummary,+ recentStateAnchor,+ customInstructions,+ );const summarizationMessages = [{@@ -717,6 +713,17 @@ function extractRecentStateAnchor(entries: SessionEntry[], keptStart: number, ke}return undefined;}+/**+ * The `<recent-state-anchor>` block `generateSummary` appends when the kept+ * tail has assistant text. The window estimator reuses it so its mirror of+ * the wire request stays exact.+ */+function recentStateAnchorBlock(recentStateAnchor?: string): string {+ return recentStateAnchor+ ? `<recent-state-anchor>\nNewest assistant message that stays retained below the summary. The conversation to summarize is older than this anchor; the retained messages below are authoritative, so treat this anchor, not the conversation above, as the current state.\n\n${recentStateAnchor}\n</recent-state-anchor>\n\n`+ : "";+}+const TURN_PREFIX_SUMMARIZATION_PROMPT = `This is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.Summarize the prefix to provide context for the retained suffix:@@ -732,6 +739,55 @@ Summarize the prefix to provide context for the retained suffix:Be concise. Focus on what's needed to understand the kept suffix.`;+/**+ * Completion budget for the history summary: 0.8 of the token reserve.+ * Shared with `estimateSummaryRequestTokens` so the window estimate cannot+ * drift from the wire request.+ */+function historySummaryCompletionBudget(reserveTokens: number): number {+ return Math.floor(0.8 * reserveTokens);+}++/**+ * Completion budget for the split-turn prefix summary: a tighter 0.5 draw on+ * the reserve. Shared with `estimateSummaryRequestTokens` so the window+ * estimate cannot drift from the wire request.+ */+function turnPrefixSummaryCompletionBudget(reserveTokens: number): number {+ return Math.floor(0.5 * reserveTokens);+}++/**+ * Build the history-summary prompt: the serialized conversation in its+ * `<conversation>` wrapper, the previous summary on iterative updates, the+ * recency anchor, and the summarization instructions. Shared with+ * `estimateSummaryRequestTokens` so the window estimate cannot drift from+ * the wire request `generateSummary` issues.+ */+function buildHistorySummaryPrompt(+ conversationText: string,+ previousSummary?: string,+ recentStateAnchor?: string,+ customInstructions?: string,+): string {+ let promptText = `<conversation>\n${conversationText}\n</conversation>\n\n`;+ if (previousSummary) {+ promptText += `<previous-summary>\n${previousSummary}\n</previous-summary>\n\n`;+ }+ promptText += recentStateAnchorBlock(recentStateAnchor);+ promptText += buildSummarizationPrompt(customInstructions, previousSummary);+ return promptText;+}++/**+ * Build the split-turn prefix-summary prompt. Shared with+ * `estimateSummaryRequestTokens` so the window estimate cannot drift from+ * the wire request `generateTurnPrefixSummary` issues.+ */+function buildTurnPrefixSummaryPrompt(conversationText: string): string {+ return `<conversation>\n${conversationText}\n</conversation>\n\n${TURN_PREFIX_SUMMARIZATION_PROMPT}`;+}+/*** Generate summaries for compaction using prepared data.* Returns CompactionResult - SessionManager adds uuid/parentUuid when saving.@@ -863,10 +919,10 @@ async function generateTurnPrefixSummary(retry?: ProviderRetryPolicy,sessionId?: string,): Promise<SummarySlice> {- const maxTokens = Math.floor(0.5 * reserveTokens); // Smaller budget for turn prefix+ const maxTokens = turnPrefixSummaryCompletionBudget(reserveTokens);const llmMessages = convertToLlm(messages);const conversationText = serializeConversation(llmMessages);- const promptText = `<conversation>\n${conversationText}\n</conversation>\n\n${TURN_PREFIX_SUMMARIZATION_PROMPT}`;+ const promptText = buildTurnPrefixSummaryPrompt(conversationText);const summarizationMessages = [{role: "user" as const,@@ -899,3 +955,51 @@ async function generateTurnPrefixSummary(usage: response.usage,};}++/**+ * Estimate the context window the summary model needs for the wire requests+ * `compact` builds from this preparation, using the chars/4 heuristic this+ * module already uses for pre-LLM token math. Builds the exact request+ * bodies through the same prompt builders and completion budgets as+ * `generateSummary` and the split turn's prefix summary (the serialized+ * conversation, plus the previous summary and recency anchor when set), so+ * the estimate cannot drift from the requests on the wire; both carry+ * SUMMARIZATION_SYSTEM_PROMPT as the system prompt, so its size counts too.+ * The largest slice wins: routing must fit every request the compaction will+ * issue, not the average. 0 means no summary request is applicable.+ */+export function estimateSummaryRequestTokens(preparation: CompactionPreparation, customInstructions?: string): number {+ const { messagesToSummarize, turnPrefixMessages, isSplitTurn, previousSummary, recentStateAnchor, settings } =+ preparation;+ const systemPromptTokens = Math.ceil(SUMMARIZATION_SYSTEM_PROMPT.length / 4);+ let required = 0;+ // compact() issues the history slice for every compaction except a split+ // turn with a non-empty prefix and nothing to summarize ("No prior history"+ // needs no wire call); a stale previousSummary alone never adds a request+ // compact() skips.+ const issuesHistoryCall = messagesToSummarize.length > 0 || !(isSplitTurn && turnPrefixMessages.length > 0);+ if (issuesHistoryCall) {+ const promptText = buildHistorySummaryPrompt(+ serializeConversation(convertToLlm(messagesToSummarize)),+ previousSummary,+ recentStateAnchor,+ customInstructions,+ );+ required = Math.max(+ required,+ systemPromptTokens + Math.ceil(promptText.length / 4) + historySummaryCompletionBudget(settings.reserveTokens),+ );+ }+ // A split turn's prefix summary is a separate request with its own body and+ // a smaller completion budget, so it can exceed the history slice.+ if (turnPrefixMessages.length > 0) {+ const promptText = buildTurnPrefixSummaryPrompt(serializeConversation(convertToLlm(turnPrefixMessages)));+ required = Math.max(+ required,+ systemPromptTokens ++ Math.ceil(promptText.length / 4) ++ turnPrefixSummaryCompletionBudget(settings.reserveTokens),+ );+ }+ return required;+}diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.tsindex e8ef970dae..5b343f08a8 100644--- a/packages/coding-agent/src/core/settings-manager.ts+++ b/packages/coding-agent/src/core/settings-manager.ts@@ -199,10 +199,13 @@ export interface Settings {subagentDefaultModel?: string; // "provider/id" for rlm.spawn without a pinned model; unset inherits the parent modelupdateChannel?: "stable" | "nightly"; // release channel for self-updates; unset follows the running versionrecentModels?: string[]; // "provider/id" keys, most-recently-used first- // "provider/id" for background LLM passes (refinement review and planning);- // unset falls back to the session model. Routing these to a different model- // keeps their different prompt prefixes from evicting the session's provider- // prefix-cache entry.+ // "provider/id" for background LLM passes (refinement review and planning,+ // compaction summaries, branch summaries); unset falls back to the session+ // model. These passes use their own prompt prefixes, so they can never hit+ // the session's cached prefix: on the session model they re-read their whole+ // input at peak price, and on OpenAI-style providers a divergent prefix+ // riding the session's prompt_cache_key depresses hit rates. Routing them+ // to a different model moves those calls off the session model.auxiliaryModel?: string;defaultThinkingLevel?: "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max";defaultServiceTier?: ServiceTier;@@ -825,9 +828,14 @@ export class SettingsManager {this.save();}+ /**+ * "provider/id" of the model that runs background LLM passes (refinement+ * review and planning, compaction summaries, branch summaries). Falls back to+ * the session model when unset, equal to the session model, or unusable.+ */getAuxiliaryModel(): string | undefined {// Hand-edited or corrupt settings files can persist non-string values; treat- // anything malformed as unset so refinement falls back to the session model.+ // anything malformed as unset so the pass falls back to the session model.const value = this.settings.auxiliaryModel;return typeof value === "string" ? value : undefined;}diff --git a/packages/coding-agent/test/compaction.test.ts b/packages/coding-agent/test/compaction.test.tsindex 4978fbc3db..7eee61800f 100644--- a/packages/coding-agent/test/compaction.test.ts+++ b/packages/coding-agent/test/compaction.test.ts@@ -1,17 +1,27 @@import type { AgentMessage } from "@earendil-works/pi-agent-core";-import type { AssistantMessage, Message, ToolCall, Usage } from "@earendil-works/pi-ai";+import type { AssistantMessage, Message, Model, ToolCall, Usage } from "@earendil-works/pi-ai";+import { fauxAssistantMessage } from "@earendil-works/pi-ai";import { readFileSync } from "fs";import { join } from "path";-import { beforeEach, describe, expect, it } from "vitest";+import { beforeEach, describe, expect, it, vi } from "vitest";import {type CompactionSettings,+ compact,DEFAULT_COMPACTION_SETTINGS,estimateContextTokens,+ estimateSummaryRequestTokens,findCutPoint,prepareCompaction,shouldCompact,} from "../src/core/compaction/index.js";-import { serializeConversation } from "../src/core/compaction/utils.js";+import { SUMMARIZATION_SYSTEM_PROMPT, serializeConversation } from "../src/core/compaction/utils.js";++const { completeSimpleMock } = vi.hoisted(() => ({ completeSimpleMock: vi.fn() }));+vi.mock("@earendil-works/pi-ai", async (importOriginal) => {+ const actual = await importOriginal<Record<string, unknown>>();+ return { ...actual, completeSimple: completeSimpleMock };+});+import {buildSessionContext,type CompactionEntry,@@ -340,6 +350,31 @@ describe("prepareCompaction recency anchor", () => {expect(preparation!.recentStateAnchor).toBe(anchor);expect(preparation!.previousSummary).toBe(keptSummary);});+ it("sizes the history request estimate with the recency anchor", () => {+ const preparation = anchoredPreparation("First summary", ["newest kept text"]);+ expect(estimateSummaryRequestTokens(preparation!)).toBeGreaterThan(+ estimateSummaryRequestTokens({ ...preparation!, recentStateAnchor: undefined }),+ );+ });+ it("sizes the window from the exact wire bodies compact() issues", async () => {+ const preparation = anchoredPreparation("First summary", ["newest kept text"]);+ completeSimpleMock.mockReset().mockResolvedValue(fauxAssistantMessage("Test summary"));+ const model = { provider: "faux", id: "faux-1" } as unknown as Model<string>;+ await compact(preparation!, model, "test-key");+ const calls = completeSimpleMock.mock.calls.filter(+ (call) => (call[1] as { systemPrompt?: string }).systemPrompt === SUMMARIZATION_SYSTEM_PROMPT,+ );+ expect(calls.length).toBeGreaterThan(0);+ const required = estimateSummaryRequestTokens(preparation!);+ for (const call of calls) {+ const text = (call[1] as unknown as { messages: { content: { text: string }[] }[] }).messages[0].content[0]+ .text;+ const { maxTokens = 0 } = call[2] as { maxTokens?: number };+ expect(required).toBeGreaterThanOrEqual(+ Math.ceil(SUMMARIZATION_SYSTEM_PROMPT.length / 4) + Math.ceil(text.length / 4) + maxTokens,+ );+ }+ });});// ============================================================================diff --git a/packages/coding-agent/test/suite/agent-session-branch-summary-auxiliary-model.test.ts b/packages/coding-agent/test/suite/agent-session-branch-summary-auxiliary-model.test.tsnew file mode 100644index 0000000000..77972d5aa5--- /dev/null+++ b/packages/coding-agent/test/suite/agent-session-branch-summary-auxiliary-model.test.ts@@ -0,0 +1,57 @@+import type * as PiAi from "@earendil-works/pi-ai";+import { fauxAssistantMessage } from "@earendil-works/pi-ai";+import { describe, expect, it, vi } from "vitest";+import { SUMMARIZATION_SYSTEM_PROMPT } from "../../src/core/compaction/utils.js";+import { createHarness } from "./harness.js";++const { completeSimpleMock } = vi.hoisted(() => ({ completeSimpleMock: vi.fn() }));+vi.mock("@earendil-works/pi-ai", async (importOriginal) => {+ const actual = await importOriginal<typeof PiAi>();+ return { ...actual, completeSimple: completeSimpleMock };+});+describe("AgentSession branch summary auxiliary model", () => {+ type Case = { auxiliaryModel?: string; window?: number; sessionWindow?: number; stub?: boolean };+ const cases: Array<[string, Case, string]> = [+ ["routes branch summaries to the auxiliary model", { auxiliaryModel: "faux/aux-model" }, "aux-model"],+ ["falls back when the auxiliary model is unusable", { auxiliaryModel: "faux/missing-model" }, "session-model"],+ ["falls back when the window is small", { auxiliaryModel: "faux/aux-model", window: 8192 }, "session-model"],+ ["falls back when only the body fits", { auxiliaryModel: "faux/aux-model", window: 30000 }, "session-model"],+ [+ "falls back when the reserve outgrows the window",+ { auxiliaryModel: "faux/aux-model", window: 8192, sessionWindow: 16400, stub: true },+ "session-model",+ ],+ ];+ it.each(cases)("%s", async (_label, options, expectedModel) => {+ completeSimpleMock.mockReset().mockResolvedValue(fauxAssistantMessage("Test summary"));+ const warnSpy = vi.spyOn(console, "warn");+ warnSpy.mockReset().mockImplementation(() => {});+ const harness = await createHarness({+ models: [+ { id: "session-model", name: "Session Model", contextWindow: options.sessionWindow },+ { id: "aux-model", name: "Aux Model", contextWindow: options.window },+ ],+ settings: { auxiliaryModel: options.auxiliaryModel },+ persistSession: true,+ });+ const turnText = "long branch turn text ".repeat(options.window ? 600 : 1);+ harness.setResponses([1, 2, 3].map(() => fauxAssistantMessage(turnText)));+ for (const _ of [1, 2, 3]) await harness.session.prompt(turnText);+ const [rootNode] = harness.sessionManager.getTree();+ const result = await harness.session.navigateTree(rootNode.entry.id, { summarize: true });+ const calls = completeSimpleMock.mock.calls.filter(+ (call) => (call[1] as { systemPrompt?: string }).systemPrompt === SUMMARIZATION_SYSTEM_PROMPT,+ );+ expect(calls.length).toBe(options.stub ? 0 : 1);+ expect(result.summaryEntry?.summary).toContain(options.stub ? "No content to summarize" : "Test summary");+ for (const call of calls) {+ expect(call[0]).toMatchObject({ provider: "faux", id: expectedModel });+ }+ const fallbackWarn = expectedModel === "session-model" ? options.auxiliaryModel : undefined;+ const expectedWarnings = fallbackWarn+ ? [`Warning: auxiliaryModel "${fallbackWarn}" unusable for branch summary; using the session model.`]+ : [];+ expect(warnSpy.mock.calls.map(([message]) => String(message))).toStrictEqual(expectedWarnings);+ harness.cleanup();+ });+});diff --git a/packages/coding-agent/test/suite/agent-session-compaction-auxiliary-model.test.ts b/packages/coding-agent/test/suite/agent-session-compaction-auxiliary-model.test.tsnew file mode 100644index 0000000000..4a53ef8aad--- /dev/null+++ b/packages/coding-agent/test/suite/agent-session-compaction-auxiliary-model.test.ts@@ -0,0 +1,53 @@+import type * as PiAi from "@earendil-works/pi-ai";+import { fauxAssistantMessage } from "@earendil-works/pi-ai";+import { describe, expect, it, vi } from "vitest";+import { SUMMARIZATION_SYSTEM_PROMPT } from "../../src/core/compaction/utils.js";+import { createHarness } from "./harness.js";++const { completeSimpleMock } = vi.hoisted(() => ({ completeSimpleMock: vi.fn() }));++vi.mock("@earendil-works/pi-ai", async (importOriginal) => {+ const actual = await importOriginal<typeof PiAi>();+ return { ...actual, completeSimple: completeSimpleMock };+});+describe("AgentSession compaction auxiliary model", () => {+ const cases: Array<[string, { auxiliaryModel?: string; window?: number }, string]> = [+ ["routes compaction summaries to the auxiliary model", { auxiliaryModel: "faux/aux-model" }, "aux-model"],+ ["falls back when the auxiliary model is unusable", { auxiliaryModel: "faux/missing-model" }, "session-model"],+ ["falls back when the window is too small", { auxiliaryModel: "faux/aux-model", window: 8192 }, "session-model"],+ ["falls back to the session model when no auxiliary model is configured", {}, "session-model"],+ ];+ it.each(cases)("%s", async (_label, options, expectedModel) => {+ completeSimpleMock.mockReset().mockResolvedValue(fauxAssistantMessage("Test summary"));+ const warnSpy = vi.spyOn(console, "warn");+ warnSpy.mockReset().mockImplementation(() => {});+ const harness = await createHarness({+ models: [+ { id: "session-model", name: "Session Model", reasoning: true },+ { id: "aux-model", name: "Aux Model", contextWindow: options.window },+ ],+ settings: { auxiliaryModel: options.auxiliaryModel, compaction: { keepRecentTokens: 1 } },+ persistSession: true,+ });+ harness.session.setThinkingLevel("medium");+ harness.setResponses([fauxAssistantMessage("one response"), fauxAssistantMessage("two response")]);+ await harness.session.prompt("one");+ await harness.session.prompt("two");+ const result = await harness.session.compact();+ const calls = completeSimpleMock.mock.calls.filter(+ (call) => (call[1] as { systemPrompt?: string }).systemPrompt === SUMMARIZATION_SYSTEM_PROMPT,+ );+ expect(calls.length).toBeGreaterThan(0);+ expect(result.summary).toContain("Test summary");+ for (const call of calls) {+ expect(call[0]).toMatchObject({ provider: "faux", id: expectedModel });+ expect(call[2]).not.toHaveProperty("reasoning");+ }+ const fallbackWarn = expectedModel === "session-model" ? options.auxiliaryModel : undefined;+ const expectedWarnings = fallbackWarn+ ? [`Warning: auxiliaryModel "${fallbackWarn}" unusable for compaction summary; using the session model.`]+ : [];+ expect(warnSpy.mock.calls.map(([message]) => String(message))).toStrictEqual(expectedWarnings);+ harness.cleanup();+ });+});