Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

fix(coding-agent): keep refinement passes from evicting the session prompt cache (#2220)

Seth Karten2f5cd0cc1631
diff --git a/packages/coding-agent/.changes/prompt-cache-miss-minimization.md b/packages/coding-agent/.changes/prompt-cache-miss-minimization.md
new file mode 100644
index 0000000000..9a2a4b8768
--- /dev/null
+++ b/packages/coding-agent/.changes/prompt-cache-miss-minimization.md
@@ -0,0 +1,2 @@
+- Added an `auxiliaryModel` setting (`"provider/id"`) that routes refinement LLM passes (auto-refine review and refinement planning) to a different model. These passes use their own prompt prefixes, so running them on the session model evicts the provider's prompt-cache entry for the session and forces a full context re-read on the next session request; the setting isolates those passes while falling back to the session model when unset or unusable.
+- Prime Inference `anthropic/*` models now send anthropic-style `cache_control` markers (system prompt, last tool, last conversation message) so prompt caching engages on gateways that pass them through to the upstream Anthropic API, matching the Anthropic cache pricing the catalog already applies to those entries.
diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts
index 7a854eb064..e11e325609 100644
--- a/packages/coding-agent/src/core/agent-session.ts
+++ b/packages/coding-agent/src/core/agent-session.ts
@@ -8785,23 +8785,59 @@ export class AgentSession {
}
}
+ /**
+ * Refinement passes (review and planning) run with their own prompts, so
+ * issuing them on the session model evicts the provider's prefix-cache entry
+ * for the session and forces a full context re-read on the next session
+ * request. Route them to the configured auxiliary model when it is set and
+ * usable; fall back to the session model otherwise.
+ */
+ private async _resolveRefinementModel(): Promise<
+ { model: Model<Api>; apiKey: string; headers?: Record<string, string> } | undefined
+ > {
+ const sessionModel = this.model;
+ if (!sessionModel) {
+ return undefined;
+ }
+ const selector = this.settingsManager.getAuxiliaryModel()?.trim().toLowerCase();
+ if (!selector || `${sessionModel.provider}/${sessionModel.id}`.toLowerCase() === selector) {
+ const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(sessionModel);
+ return { model: requestModel, apiKey, headers };
+ }
+ try {
+ const model = (await this._authenticatedRlmModels()).find(
+ (candidate) => `${candidate.provider}/${candidate.id}`.toLowerCase() === selector,
+ );
+ if (!model) {
+ throw new Error(`model "${selector}" is unavailable, unauthenticated, or expired`);
+ }
+ const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(model);
+ return { model: requestModel, apiKey, headers };
+ } catch {
+ // Error details from the auth stack can embed credential material, so only
+ // the selector is logged (CodeQL js/clear-text-logging).
+ console.warn(`Warning: auxiliaryModel "${selector}" unusable for refinement; using the session model.`);
+ const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(sessionModel);
+ return { model: requestModel, apiKey, headers };
+ }
+ }
+
private async _reviewAutoRefine(context: AutoRefineReviewRequest, signal?: AbortSignal): Promise<AutoRefineReview> {
if (this._autoRefineReviewer) {
return this._autoRefineReviewer(context, signal);
}
- const model = this.model;
- if (!model) {
+ const refinementModel = await this._resolveRefinementModel();
+ if (!refinementModel) {
return { shouldRefine: false, rationale: "No model selected." };
}
- const { apiKey, headers, requestModel } = await this._getRequiredRequestAuth(model);
return reviewAutoRefine(
this.agent.state.messages,
this._loadMergedHarnessState(),
this._loadRefinementHistory(),
- requestModel,
- apiKey,
+ refinementModel.model,
+ refinementModel.apiKey,
context,
- headers,
+ refinementModel.headers,
signal,
this.thinkingLevel,
providerRetryPolicy(this.settingsManager),
@@ -9040,7 +9076,10 @@ export class AgentSession {
throw new Error(formatNoModelSelectedMessage());
}
- const { apiKey, headers, requestModel: model } = await this._getRequiredRequestAuth(this.model);
+ const refinementModel = await this._resolveRefinementModel();
+ if (!refinementModel) {
+ throw new Error(formatNoModelSelectedMessage());
+ }
const globalHarnessStateDir = getGlobalHarnessStateDir();
const localHarnessStateDir = this._localHarnessStateDir();
const requestedScope = options.global ? "global" : "local";
@@ -9102,10 +9141,10 @@ export class AgentSession {
this.agent.state.messages,
planningState,
history,
- model,
- apiKey,
+ refinementModel.model,
+ refinementModel.apiKey,
{ ...options, retry: providerRetryPolicy(this.settingsManager) },
- headers,
+ refinementModel.headers,
signal,
this.thinkingLevel,
this.sessionId,
diff --git a/packages/coding-agent/src/core/prime-inference-model-catalog.ts b/packages/coding-agent/src/core/prime-inference-model-catalog.ts
index 8c9c2f6725..2b3e2861a6 100644
--- a/packages/coding-agent/src/core/prime-inference-model-catalog.ts
+++ b/packages/coding-agent/src/core/prime-inference-model-catalog.ts
@@ -48,6 +48,15 @@ export function buildPrimeInferenceModels(
if (!template && (!entry.contextWindow || !entry.maxTokens || entry.reasoning === undefined)) continue;
const contextWindow = entry.contextWindow ?? template?.contextWindow ?? 0;
const maxTokens = Math.min(entry.maxTokens ?? template?.maxTokens ?? 0, contextWindow);
+ const compat = structuredClone(template?.compat ?? DEFAULT_COMPAT);
+ // Anthropic models cache with explicit breakpoints, not automatic
+ // server-side prefix caching; cacheControlFormat makes the provider add
+ // anthropic-style cache_control markers for these entries. The catalog
+ // already prices anthropic/* with Anthropic cache economics (10% cache
+ // reads, 125% cache writes), so the wire format follows the pricing.
+ if (entry.id.toLowerCase().startsWith("anthropic/")) {
+ compat.cacheControlFormat = "anthropic";
+ }
models.push({
id: entry.id,
name: entry.name ?? template?.name ?? entry.id,
@@ -61,7 +70,7 @@ export function buildPrimeInferenceModels(
contextWindow,
maxTokens,
...(template?.featured ? { featured: true } : {}),
- compat: structuredClone(template?.compat ?? DEFAULT_COMPAT),
+ compat,
});
}
const minimumModels = options.minimumModels ?? Math.ceil(bundledModels.length * MIN_CATALOG_COVERAGE);
diff --git a/packages/coding-agent/src/core/settings-manager.ts b/packages/coding-agent/src/core/settings-manager.ts
index f5e08a14c2..35e6237541 100644
--- a/packages/coding-agent/src/core/settings-manager.ts
+++ b/packages/coding-agent/src/core/settings-manager.ts
@@ -145,6 +145,11 @@ export interface Settings {
defaultProvider?: string;
defaultModel?: string;
recentModels?: string[]; // "provider/id" keys, most-recently-used first
+ // "provider/id" for background LLM passes (refinement review and planning);
+ // unset falls back to the session model. Routing these to a different model
+ // keeps their different prompt prefixes from evicting the session's provider
+ // prefix-cache entry.
+ auxiliaryModel?: string;
defaultThinkingLevel?: "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
defaultServiceTier?: ServiceTier;
rlmMaxDepth?: number; // default for new sessions; unset falls through to RLM_MAX_DEPTH, then 2
@@ -746,6 +751,13 @@ export class SettingsManager {
this.save();
}
+ getAuxiliaryModel(): string | undefined {
+ // Hand-edited or corrupt settings files can persist non-string values; treat
+ // anything malformed as unset so refinement falls back to the session model.
+ const value = this.settings.auxiliaryModel;
+ return typeof value === "string" ? value : undefined;
+ }
+
getRecentModels(): string[] {
return this.settings.recentModels ?? [];
}
diff --git a/packages/coding-agent/test/prime-inference-model-catalog.test.ts b/packages/coding-agent/test/prime-inference-model-catalog.test.ts
index e5a033a00e..f461a247d8 100644
--- a/packages/coding-agent/test/prime-inference-model-catalog.test.ts
+++ b/packages/coding-agent/test/prime-inference-model-catalog.test.ts
@@ -87,6 +87,18 @@ describe("Prime Inference model catalog", () => {
expect(live).not.toHaveProperty("headers");
});
+ test("marks anthropic models for anthropic-style cache control and leaves others untouched", () => {
+ const models =
+ buildPrimeInferenceModels(
+ [model("anthropic/claude-opus-4.8"), model("vendor/model")],
+ [entry("anthropic/claude-opus-4.8"), entry("vendor/model")],
+ ) ?? [];
+ const anthropicModel = models.find((candidate) => candidate.id === "anthropic/claude-opus-4.8");
+ const vendorModel = models.find((candidate) => candidate.id === "vendor/model");
+ expect(anthropicModel?.compat?.cacheControlFormat).toBe("anthropic");
+ expect(vendorModel?.compat?.cacheControlFormat).toBeUndefined();
+ });
+
test("adds complete new models and skips incomplete unknown models", () => {
const models =
buildPrimeInferenceModels(
diff --git a/packages/coding-agent/test/settings-manager.test.ts b/packages/coding-agent/test/settings-manager.test.ts
index ffef822127..9079f851d7 100644
--- a/packages/coding-agent/test/settings-manager.test.ts
+++ b/packages/coding-agent/test/settings-manager.test.ts
@@ -276,6 +276,22 @@ describe("SettingsManager", () => {
});
});
+ describe("auxiliaryModel", () => {
+ it("returns a valid persisted selector", () => {
+ writeFileSync(join(agentDir, "settings.json"), JSON.stringify({ auxiliaryModel: "faux/aux-model" }));
+ const manager = SettingsManager.create(projectDir, agentDir);
+
+ expect(manager.getAuxiliaryModel()).toBe("faux/aux-model");
+ });
+
+ it("treats malformed persisted values as unset", () => {
+ writeFileSync(join(agentDir, "settings.json"), JSON.stringify({ auxiliaryModel: 42 }));
+ const manager = SettingsManager.create(projectDir, agentDir);
+
+ expect(manager.getAuxiliaryModel()).toBeUndefined();
+ });
+ });
+
describe("recentModels", () => {
it("records most-recently-used first, dedupes, and persists", async () => {
const manager = SettingsManager.create(projectDir, agentDir);
diff --git a/packages/coding-agent/test/suite/agent-session-refine-auxiliary-model.test.ts b/packages/coding-agent/test/suite/agent-session-refine-auxiliary-model.test.ts
new file mode 100644
index 0000000000..13aafab922
--- /dev/null
+++ b/packages/coding-agent/test/suite/agent-session-refine-auxiliary-model.test.ts
@@ -0,0 +1,109 @@
+import type * as PiAi from "@earendil-works/pi-ai";
+import type { AssistantMessage } from "@earendil-works/pi-ai";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const { completeSimpleMock } = vi.hoisted(() => ({
+ completeSimpleMock: vi.fn(),
+}));
+
+vi.mock("@earendil-works/pi-ai", async (importOriginal) => {
+ const actual = await importOriginal<typeof PiAi>();
+ return {
+ ...actual,
+ completeSimple: completeSimpleMock,
+ };
+});
+
+import { createHarness, type Harness } from "./harness.js";
+
+function assistantText(text: string): AssistantMessage {
+ return {
+ role: "assistant",
+ content: [{ type: "text", text }],
+ api: "faux",
+ provider: "faux",
+ model: "faux",
+ usage: {
+ input: 1,
+ output: 1,
+ cacheRead: 0,
+ cacheWrite: 0,
+ totalTokens: 2,
+ cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+ },
+ stopReason: "stop",
+ timestamp: Date.now(),
+ };
+}
+
+function emptyProposalResponse(): AssistantMessage {
+ return assistantText(
+ JSON.stringify({ summary: "no-op", rationale: "nothing to record", expectedOutcome: "none", edits: [] }),
+ );
+}
+
+describe("AgentSession refinement auxiliary model", () => {
+ const harnesses: Harness[] = [];
+
+ beforeEach(() => {
+ completeSimpleMock.mockReset();
+ completeSimpleMock.mockResolvedValue(emptyProposalResponse());
+ });
+
+ afterEach(() => {
+ for (const harness of harnesses.splice(0)) harness.cleanup();
+ });
+
+ async function createRefineHarness(settings?: { auxiliaryModel?: string }): Promise<Harness> {
+ const harness = await createHarness({
+ models: [
+ { id: "session-model", name: "Session Model" },
+ { id: "aux-model", name: "Aux Model" },
+ ],
+ settings,
+ persistSession: true,
+ });
+ harnesses.push(harness);
+ return harness;
+ }
+
+ it("routes refinement planning to the configured auxiliary model", async () => {
+ const harness = await createRefineHarness({ auxiliaryModel: "faux/aux-model" });
+ await harness.session.refine({ instructions: "record a note" });
+ expect(completeSimpleMock).toHaveBeenCalledTimes(1);
+ expect(completeSimpleMock.mock.calls[0][0]).toMatchObject({
+ provider: "faux",
+ id: "aux-model",
+ });
+ });
+
+ it("falls back to the session model when no auxiliary model is configured", async () => {
+ const harness = await createRefineHarness();
+ await harness.session.refine({ instructions: "record a note" });
+ expect(completeSimpleMock).toHaveBeenCalledTimes(1);
+ expect(completeSimpleMock.mock.calls[0][0]).toMatchObject({
+ provider: "faux",
+ id: "session-model",
+ });
+ });
+
+ it("falls back to the session model when the auxiliary model is unusable", async () => {
+ const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+ try {
+ const harness = await createRefineHarness({ auxiliaryModel: "faux/missing-model" });
+ await harness.session.refine({ instructions: "record a note" });
+ expect(completeSimpleMock).toHaveBeenCalledTimes(1);
+ expect(completeSimpleMock.mock.calls[0][0]).toMatchObject({
+ provider: "faux",
+ id: "session-model",
+ });
+ expect(warnSpy).toHaveBeenCalledTimes(1);
+ const [message] = warnSpy.mock.calls[0];
+ expect(message).toContain('auxiliaryModel "faux/missing-model" unusable for refinement');
+ // Caught error details can embed credential material, so they must not be logged.
+ expect(message).not.toContain("unavailable, unauthenticated, or expired");
+ } finally {
+ warnSpy.mockRestore();
+ }
+ });
+});