The Open Framework for Building Agents
fix(eve): prompt cache - preserve turn context across tool steps (#3089)
Andrew Barba461b578f219a
diff --git a/.changeset/stable-turn-context-prefix.md b/.changeset/stable-turn-context-prefix.mdnew file mode 100644index 000000000..62ec90ca2--- /dev/null+++ b/.changeset/stable-turn-context-prefix.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Keep framework context at a stable position throughout a turn so tool steps can reuse the existing prompt prefix instead of moving skill announcements behind new tool results.diff --git a/docs/instructions.mdx b/docs/instructions.mdxindex 6f3a33584..5f193235c 100644--- a/docs/instructions.mdx+++ b/docs/instructions.mdx@@ -86,7 +86,7 @@ See [Dynamic capabilities](./guides/dynamic-capabilities) for examples and failu[Compaction](./concepts/sessions-runs-and-streaming#compact-clear-and-reset) treats user-role instructions like ordinary conversation history, so a summary may replace their original text. Clear removes them with the rest of model-message history and does not rerun static or dynamic instructions. System-role instructions remain because they are outside history.-Keep system-role content stable and place it before frequently changing context when practical. That gives providers the best opportunity to reuse a prompt prefix, but cache behavior and billing remain provider-specific. User-role instructions preserve the normal append-only message prefix; eve does not promise a cache hit.+Keep system-role content stable and place it before frequently changing context when practical. That gives providers the best opportunity to reuse a prompt prefix, but cache behavior and billing remain provider-specific. User-role instructions preserve the normal append-only message prefix. Framework context such as dynamic skill announcements also keeps its position before the current request across tool steps, including when a durable step resumes. eve does not promise a cache hit.## Disclaimerdiff --git a/e2e/fixtures/agent-skills/agent/agent.ts b/e2e/fixtures/agent-skills/agent/agent.tsindex abfdd63f5..0089ad0ec 100644--- a/e2e/fixtures/agent-skills/agent/agent.ts+++ b/e2e/fixtures/agent-skills/agent/agent.ts@@ -1,5 +1,6 @@import { e2eAgentConfig } from "@eve-e2e/config";-import { defineAgent } from "eve";+import { defineAgent, defineDynamic } from "eve";+import { PREFIX_REQUEST, prefixModel } from "./lib/prompt-prefix";import type { MockModelRequest } from "eve/evals";const DYNAMIC_INSTRUCTIONS_TOKEN = "dynamic-instructions-ok-M3K8";@@ -11,7 +12,19 @@ function respond(request: MockModelRequest): string {return hasDynamicUserInstruction ? DYNAMIC_INSTRUCTIONS_TOKEN : "missing dynamic instructions";}+const { model, modelContextWindowTokens, ...config } = e2eAgentConfig({ mock: respond });+export default defineAgent({- ...e2eAgentConfig({ mock: respond }),+ ...config,+ model: defineDynamic({+ events: {+ "step.started": (_event, ctx) =>+ ctx.messages.some(+ (message) => message.role === "user" && message.content === PREFIX_REQUEST,+ )+ ? { model: prefixModel, modelContextWindowTokens: 1_000_000 }+ : { model, modelContextWindowTokens },+ },+ }),reasoning: "high",});diff --git a/e2e/fixtures/agent-skills/agent/lib/prompt-prefix.ts b/e2e/fixtures/agent-skills/agent/lib/prompt-prefix.tsnew file mode 100644index 000000000..18732278b--- /dev/null+++ b/e2e/fixtures/agent-skills/agent/lib/prompt-prefix.ts@@ -0,0 +1,24 @@+import { mockModel } from "eve/evals";+import { z } from "zod";++export const PREFIX_REQUEST = "Verify the durable prompt prefix.";+export const prefixSchema = z.array(+ z.object({ role: z.enum(["assistant", "system", "tool", "user"]), text: z.string() }),+);++export const prefixModel = mockModel({+ modelId: "prompt-prefix-check",+ respond(request) {+ const result = request.toolResults.find((entry) => entry.name === "capture_prompt");+ if (result === undefined) {+ return { toolCalls: [{ name: "capture_prompt", input: { prefix: request.messages } }] };+ }+ const prefix = prefixSchema.parse(result.output);+ const hasAnnouncement = prefix.some(+ (message) => message.role === "user" && message.text.startsWith("Available skills"),+ );+ const stable =+ JSON.stringify(request.messages.slice(0, prefix.length)) === JSON.stringify(prefix);+ return hasAnnouncement && stable ? "prompt-prefix-ok" : "prompt-prefix-changed";+ },+});diff --git a/e2e/fixtures/agent-skills/agent/tools/capture_prompt.ts b/e2e/fixtures/agent-skills/agent/tools/capture_prompt.tsnew file mode 100644index 000000000..d72b6830d--- /dev/null+++ b/e2e/fixtures/agent-skills/agent/tools/capture_prompt.ts@@ -0,0 +1,16 @@+import { defineDynamic, defineTool } from "eve/tools";+import { z } from "zod";+import { PREFIX_REQUEST, prefixSchema } from "../lib/prompt-prefix";++export default defineDynamic({+ events: {+ "step.started": (_event, ctx) =>+ ctx.messages.some((message) => message.role === "user" && message.content === PREFIX_REQUEST)+ ? defineTool({+ description: "Capture a prompt prefix for the durable cache regression eval.",+ inputSchema: z.object({ prefix: prefixSchema }),+ execute: async (input) => input.prefix,+ })+ : null,+ },+});diff --git a/e2e/fixtures/agent-skills/evals/skills/prompt-prefix.eval.ts b/e2e/fixtures/agent-skills/evals/skills/prompt-prefix.eval.tsnew file mode 100644index 000000000..5473715df--- /dev/null+++ b/e2e/fixtures/agent-skills/evals/skills/prompt-prefix.eval.ts@@ -0,0 +1,14 @@+import { defineEval } from "eve/evals";+import { PREFIX_REQUEST } from "../../agent/lib/prompt-prefix";++export default defineEval({+ description: "Dynamic skill announcements preserve the prompt prefix across durable tool steps.",+ async test(t) {+ const first = await t.send("Say ready.");+ first.expectOk();+ const checked = await t.send(PREFIX_REQUEST);+ checked.expectOk();+ checked.calledTool("capture_prompt");+ checked.messageIncludes("prompt-prefix-ok");+ },+});diff --git a/e2e/fixtures/agent-skills/package.json b/e2e/fixtures/agent-skills/package.jsonindex d77014d60..e2e90715a 100644--- a/e2e/fixtures/agent-skills/package.json+++ b/e2e/fixtures/agent-skills/package.json@@ -17,6 +17,7 @@},"devDependencies": {"@types/node": "catalog:",- "typescript": "catalog:"+ "typescript": "catalog:",+ "zod": "catalog:"}}diff --git a/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts b/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.tsindex b026bef84..0a7830e41 100644--- a/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts+++ b/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts@@ -1,3 +1,4 @@+import { setTurnClientContextState } from "#harness/turn-client-context.js";import { jsonSchema, type LanguageModel, type ModelMessage, simulateReadableStream } from "ai";import { MockLanguageModelV4 } from "ai/test";import { describe, expect, it, vi } from "vitest";@@ -833,71 +834,85 @@ describe("tool loop generate approval resume (real AI SDK)", () => {// appended as a user message after the approval response. The AI SDK reads// approvals only from the tail tool message, so the approved tool never ran// and the provider rejected the prompt with a tool call that had no output.- it.each([- { key: PendingSkillAnnouncementKey, label: "dynamic skill announcement" },- { key: TurnTaskStateKey, label: "task state" },- ])("executes the approved tool when $label is injected on the resume step", async ({ key }) => {- const siblingCall = {- input: { command: "whoami" },- toolCallId: "call-sibling",- toolName: "bash",- type: "tool-call" as const,- };- const siblingResult = {- output: { type: "text" as const, value: "eve" },- toolCallId: siblingCall.toolCallId,- toolName: "bash",- type: "tool-result" as const,- };- const session = appendPendingInputBatch({- requests: [pendingApprovalInputRequest],- // The parked shape when a gated call shares a step with an ungated one.- responseMessages: [- { content: [toolCall, approvalRequest, siblingCall], role: "assistant" },- { content: [siblingResult], role: "tool" },- ],- session: createBaseSession(),- });- const ctx = new ContextContainer();- ctx.set(key, "[Runtime context]\nInjected on the resume step.");- const execute = vi.fn(async () => "/workspace");- const model = createModel();+ it.each(+ [+ { key: PendingSkillAnnouncementKey, label: "dynamic skill announcement" },+ { key: TurnTaskStateKey, label: "task state" },+ ].flatMap((context) => [false, true].map((restoredAnchor) => ({ ...context, restoredAnchor }))),+ )(+ "executes the approved tool when $label is injected on the resume step (restored anchor: $restoredAnchor)",+ async ({ key, restoredAnchor }) => {+ const siblingCall = {+ input: { command: "whoami" },+ toolCallId: "call-sibling",+ toolName: "bash",+ type: "tool-call" as const,+ };+ const siblingResult = {+ output: { type: "text" as const, value: "eve" },+ toolCallId: siblingCall.toolCallId,+ toolName: "bash",+ type: "tool-result" as const,+ };+ const session = appendPendingInputBatch({+ requests: [pendingApprovalInputRequest],+ // The parked shape when a gated call shares a step with an ungated one.+ responseMessages: [+ { content: [toolCall, approvalRequest, siblingCall], role: "assistant" },+ { content: [siblingResult], role: "tool" },+ ],+ session: createBaseSession(),+ });+ const ctx = new ContextContainer();+ ctx.set(key, "[Runtime context]\nInjected on the resume step.");+ const execute = vi.fn(async () => "/workspace");+ const model = createModel();- const result = await contextStorage.run(ctx, () =>- createToolLoopHarness(createConfig(model, execute))(- setHarnessEmissionState(session, {- sequence: 1,- sessionStarted: true,- stepIndex: 1,- turnId: "turn-1",- }),- { inputResponses: [{ optionId: "approve", requestId: approvalRequest.approvalId }] },- ),- );+ const result = await contextStorage.run(ctx, () =>+ createToolLoopHarness(createConfig(model, execute))(+ setHarnessEmissionState(+ restoredAnchor+ ? setTurnClientContextState(session, {+ insertionIndex: 0,+ messages: [],+ turnId: "turn-1",+ })+ : session,+ {+ sequence: 1,+ sessionStarted: true,+ stepIndex: 1,+ turnId: "turn-1",+ },+ ),+ { inputResponses: [{ optionId: "approve", requestId: approvalRequest.approvalId }] },+ ),+ );- expect(execute).toHaveBeenCalledExactlyOnceWith(- toolCall.input,- expect.objectContaining({ toolCallId: toolCall.toolCallId }),- );+ expect(execute).toHaveBeenCalledExactlyOnceWith(+ toolCall.input,+ expect.objectContaining({ toolCallId: toolCall.toolCallId }),+ );- const providerPrompt = model.doGenerateCalls[0]?.prompt ?? [];- const answered = new Set<string>();- const called: string[] = [];- for (const message of providerPrompt) {- if (!Array.isArray(message.content)) continue;- for (const part of message.content) {- if (part.type === "tool-call") called.push(part.toolCallId);- if (part.type === "tool-result") answered.add(part.toolCallId);+ const providerPrompt = model.doGenerateCalls[0]?.prompt ?? [];+ const answered = new Set<string>();+ const called: string[] = [];+ for (const message of providerPrompt) {+ if (!Array.isArray(message.content)) continue;+ for (const part of message.content) {+ if (part.type === "tool-call") called.push(part.toolCallId);+ if (part.type === "tool-result") answered.add(part.toolCallId);+ }}- }- expect(called).toEqual([toolCall.toolCallId, siblingCall.toolCallId]);- expect(called.filter((id) => !answered.has(id))).toEqual([]);- expect(providerPrompt.at(-1)?.role).toBe("tool");- expect(result.session.history.at(-1)).toMatchObject({- content: [{ text: "The command returned /workspace.", type: "text" }],- role: "assistant",- });- });+ expect(called).toEqual([toolCall.toolCallId, siblingCall.toolCallId]);+ expect(called.filter((id) => !answered.has(id))).toEqual([]);+ expect(providerPrompt.at(-1)?.role).toBe("tool");+ expect(result.session.history.at(-1)).toMatchObject({+ content: [{ text: "The command returned /workspace.", type: "text" }],+ role: "assistant",+ });+ },+ );// Acceptance gate for the HITL non-blocking plan (research/hitl-request-lifecycle.md):// once messages run as normal turns while an approval is open, the approval batch isdiff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.tsindex 20196a0c6..abfdba2a7 100644--- a/packages/eve/src/harness/tool-loop.test.ts+++ b/packages/eve/src/harness/tool-loop.test.ts@@ -12255,6 +12255,78 @@ describe("createToolLoopHarness", () => {});});+ it.each(["plain", "client context", "compaction", "projected history"])(+ "keeps framework context before the turn input across durable steps (%s)",+ async (scenario) => {+ const withClientContext = scenario === "client context";+ if (scenario === "compaction") {+ vi.mocked(shouldCompact).mockReturnValueOnce(true);+ vi.mocked(compactMessages).mockImplementationOnce(async (messages) => messages.slice(2));+ }+ const toolCall = {+ type: "tool-call" as const,+ toolCallId: "cache-call",+ toolName: "add",+ input: { a: 20, b: 22 },+ };+ const toolResult = {+ type: "tool-result" as const,+ toolCallId: "cache-call",+ toolName: "add",+ output: "42",+ };+ setupMockAgent({+ finishReason: "tool-calls",+ response: {+ messages: [+ { role: "assistant", content: [toolCall] },+ { role: "tool", content: [toolResult] },+ ],+ },+ text: "",+ toolCalls: [toolCall],+ toolResults: [{ ...toolResult, input: toolCall.input }],+ });+ const ctx = new ContextContainer();+ ctx.set(TurnTaskStateKey, "Task status: analysis in progress");+ const runStep = createToolLoopHarness(+ createTestConfig("conversation", undefined, {+ historyProjector:+ scenario === "projected history"+ ? ({ messages }) =>+ messages.filter((message) => message.content !== "Previous answer")+ : undefined,+ }),+ );+ const initial = setHarnessEmissionState(+ createTestSession({+ history: [+ { role: "user", content: "Previous request" },+ { role: "assistant", content: "Previous answer" },+ ],+ }),+ { sessionStarted: true, sequence: 1, stepIndex: 0, turnId: "turn_1" },+ );+ const input = { context: ["Current channel context"], message: "Add 20 and 22." };+ const first = await contextStorage.run(ctx, () =>+ runStep(+ initial,+ withClientContext ? attachClientContext(input, ["Client context"]) : input,+ ),+ );+ expect(first.next).toBe(runStep);+ const firstPrompt = structuredClone(getLastAgentSettings().messages);+ setupMockAgent(defaultModelResult());+ const restored = JSON.parse(JSON.stringify(first.session)) as HarnessSession;+ await contextStorage.run(ctx, () => runStep(restored));+ const nextPrompt = getLastAgentSettings().messages;+ expect(nextPrompt.slice(0, firstPrompt.length)).toEqual(firstPrompt);+ expect(+ nextPrompt.filter((message) => message.content === "Task status: analysis in progress"),+ ).toHaveLength(1);+ },+ );+it("keeps ephemeral client context out of compaction and its token baseline", async () => {vi.mocked(shouldCompact).mockReturnValueOnce(true);vi.mocked(compactMessages).mockImplementationOnce(async (messages) => [...messages]);diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex d30fdf520..85e61d906 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -1067,10 +1067,13 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {// Keep the insertion point stable when a later durable step reconstructs// the model-only prompt, preserving the full prompt prefix within the turn.let turnClientContext = storedClientContext;- if (clientContext !== undefined) {+ if (+ clientContext !== undefined ||+ (storedClientContext === undefined && preparedTurnInput.length > 0)+ ) {turnClientContext = {insertionIndex: storedClientContext?.insertionIndex ?? messages.length,- messages: [...clientContext],+ messages: clientContext ?? storedClientContext?.messages ?? [],turnId,};}@@ -1208,7 +1211,10 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {// Insert that context before the current delivery so the triggering user// message remains the model's latest request.const currentMessages = createCurrentMessages(projectedMessages, {- currentTurnMessages: preparedTurnInput,+ currentTurnMessages:+ turnClientContext === undefined+ ? preparedTurnInput+ : messages.slice(turnClientContext.insertionIndex),});if (ctx !== undefined) {currentMessages.addSystem(buildDynamicInstructionMessages(ctx));diff --git a/packages/eve/src/harness/turn-client-context.ts b/packages/eve/src/harness/turn-client-context.tsindex aba8bad9f..79c4705aa 100644--- a/packages/eve/src/harness/turn-client-context.ts+++ b/packages/eve/src/harness/turn-client-context.ts@@ -3,6 +3,7 @@ import type { HarnessSession, SessionStateMap } from "#harness/types.js";const TURN_CLIENT_CONTEXT_STATE_KEY = "eve.harness.turnClientContext";export interface TurnClientContextState {+ /** Durable turn boundary shared by client context and framework announcements. */readonly insertionIndex: number;readonly messages: readonly string[];readonly turnId: string;diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex d0b90a5a2..2a771f5fa 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -886,6 +886,9 @@ importers:typescript:specifier: 'catalog:'version: 7.0.2+ zod:+ specifier: 'catalog:'+ version: 4.5.4e2e/fixtures/agent-subagents:dependencies: