Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): prompt cache - preserve turn context across tool steps (#3089)

Andrew Barba461b578f219a
diff --git a/.changeset/stable-turn-context-prefix.md b/.changeset/stable-turn-context-prefix.md
new file mode 100644
index 000000000..62ec90ca2
--- /dev/null
+++ b/.changeset/stable-turn-context-prefix.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Keep framework context at a stable position throughout a turn so tool steps can reuse the existing prompt prefix instead of moving skill announcements behind new tool results.
diff --git a/docs/instructions.mdx b/docs/instructions.mdx
index 6f3a33584..5f193235c 100644
--- a/docs/instructions.mdx
+++ b/docs/instructions.mdx
@@ -86,7 +86,7 @@ See [Dynamic capabilities](./guides/dynamic-capabilities) for examples and failu
[Compaction](./concepts/sessions-runs-and-streaming#compact-clear-and-reset) treats user-role instructions like ordinary conversation history, so a summary may replace their original text. Clear removes them with the rest of model-message history and does not rerun static or dynamic instructions. System-role instructions remain because they are outside history.
-Keep system-role content stable and place it before frequently changing context when practical. That gives providers the best opportunity to reuse a prompt prefix, but cache behavior and billing remain provider-specific. User-role instructions preserve the normal append-only message prefix; eve does not promise a cache hit.
+Keep system-role content stable and place it before frequently changing context when practical. That gives providers the best opportunity to reuse a prompt prefix, but cache behavior and billing remain provider-specific. User-role instructions preserve the normal append-only message prefix. Framework context such as dynamic skill announcements also keeps its position before the current request across tool steps, including when a durable step resumes. eve does not promise a cache hit.
## Disclaimer
diff --git a/e2e/fixtures/agent-skills/agent/agent.ts b/e2e/fixtures/agent-skills/agent/agent.ts
index abfdd63f5..0089ad0ec 100644
--- a/e2e/fixtures/agent-skills/agent/agent.ts
+++ b/e2e/fixtures/agent-skills/agent/agent.ts
@@ -1,5 +1,6 @@
import { e2eAgentConfig } from "@eve-e2e/config";
-import { defineAgent } from "eve";
+import { defineAgent, defineDynamic } from "eve";
+import { PREFIX_REQUEST, prefixModel } from "./lib/prompt-prefix";
import type { MockModelRequest } from "eve/evals";
const DYNAMIC_INSTRUCTIONS_TOKEN = "dynamic-instructions-ok-M3K8";
@@ -11,7 +12,19 @@ function respond(request: MockModelRequest): string {
return hasDynamicUserInstruction ? DYNAMIC_INSTRUCTIONS_TOKEN : "missing dynamic instructions";
}
+const { model, modelContextWindowTokens, ...config } = e2eAgentConfig({ mock: respond });
+
export default defineAgent({
- ...e2eAgentConfig({ mock: respond }),
+ ...config,
+ model: defineDynamic({
+ events: {
+ "step.started": (_event, ctx) =>
+ ctx.messages.some(
+ (message) => message.role === "user" && message.content === PREFIX_REQUEST,
+ )
+ ? { model: prefixModel, modelContextWindowTokens: 1_000_000 }
+ : { model, modelContextWindowTokens },
+ },
+ }),
reasoning: "high",
});
diff --git a/e2e/fixtures/agent-skills/agent/lib/prompt-prefix.ts b/e2e/fixtures/agent-skills/agent/lib/prompt-prefix.ts
new file mode 100644
index 000000000..18732278b
--- /dev/null
+++ b/e2e/fixtures/agent-skills/agent/lib/prompt-prefix.ts
@@ -0,0 +1,24 @@
+import { mockModel } from "eve/evals";
+import { z } from "zod";
+
+export const PREFIX_REQUEST = "Verify the durable prompt prefix.";
+export const prefixSchema = z.array(
+ z.object({ role: z.enum(["assistant", "system", "tool", "user"]), text: z.string() }),
+);
+
+export const prefixModel = mockModel({
+ modelId: "prompt-prefix-check",
+ respond(request) {
+ const result = request.toolResults.find((entry) => entry.name === "capture_prompt");
+ if (result === undefined) {
+ return { toolCalls: [{ name: "capture_prompt", input: { prefix: request.messages } }] };
+ }
+ const prefix = prefixSchema.parse(result.output);
+ const hasAnnouncement = prefix.some(
+ (message) => message.role === "user" && message.text.startsWith("Available skills"),
+ );
+ const stable =
+ JSON.stringify(request.messages.slice(0, prefix.length)) === JSON.stringify(prefix);
+ return hasAnnouncement && stable ? "prompt-prefix-ok" : "prompt-prefix-changed";
+ },
+});
diff --git a/e2e/fixtures/agent-skills/agent/tools/capture_prompt.ts b/e2e/fixtures/agent-skills/agent/tools/capture_prompt.ts
new file mode 100644
index 000000000..d72b6830d
--- /dev/null
+++ b/e2e/fixtures/agent-skills/agent/tools/capture_prompt.ts
@@ -0,0 +1,16 @@
+import { defineDynamic, defineTool } from "eve/tools";
+import { z } from "zod";
+import { PREFIX_REQUEST, prefixSchema } from "../lib/prompt-prefix";
+
+export default defineDynamic({
+ events: {
+ "step.started": (_event, ctx) =>
+ ctx.messages.some((message) => message.role === "user" && message.content === PREFIX_REQUEST)
+ ? defineTool({
+ description: "Capture a prompt prefix for the durable cache regression eval.",
+ inputSchema: z.object({ prefix: prefixSchema }),
+ execute: async (input) => input.prefix,
+ })
+ : null,
+ },
+});
diff --git a/e2e/fixtures/agent-skills/evals/skills/prompt-prefix.eval.ts b/e2e/fixtures/agent-skills/evals/skills/prompt-prefix.eval.ts
new file mode 100644
index 000000000..5473715df
--- /dev/null
+++ b/e2e/fixtures/agent-skills/evals/skills/prompt-prefix.eval.ts
@@ -0,0 +1,14 @@
+import { defineEval } from "eve/evals";
+import { PREFIX_REQUEST } from "../../agent/lib/prompt-prefix";
+
+export default defineEval({
+ description: "Dynamic skill announcements preserve the prompt prefix across durable tool steps.",
+ async test(t) {
+ const first = await t.send("Say ready.");
+ first.expectOk();
+ const checked = await t.send(PREFIX_REQUEST);
+ checked.expectOk();
+ checked.calledTool("capture_prompt");
+ checked.messageIncludes("prompt-prefix-ok");
+ },
+});
diff --git a/e2e/fixtures/agent-skills/package.json b/e2e/fixtures/agent-skills/package.json
index d77014d60..e2e90715a 100644
--- a/e2e/fixtures/agent-skills/package.json
+++ b/e2e/fixtures/agent-skills/package.json
@@ -17,6 +17,7 @@
},
"devDependencies": {
"@types/node": "catalog:",
- "typescript": "catalog:"
+ "typescript": "catalog:",
+ "zod": "catalog:"
}
}
diff --git a/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts b/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts
index b026bef84..0a7830e41 100644
--- a/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts
+++ b/packages/eve/src/harness/tool-loop-generate-approval-resume.integration.test.ts
@@ -1,3 +1,4 @@
+import { setTurnClientContextState } from "#harness/turn-client-context.js";
import { jsonSchema, type LanguageModel, type ModelMessage, simulateReadableStream } from "ai";
import { MockLanguageModelV4 } from "ai/test";
import { describe, expect, it, vi } from "vitest";
@@ -833,71 +834,85 @@ describe("tool loop generate approval resume (real AI SDK)", () => {
// appended as a user message after the approval response. The AI SDK reads
// approvals only from the tail tool message, so the approved tool never ran
// and the provider rejected the prompt with a tool call that had no output.
- it.each([
- { key: PendingSkillAnnouncementKey, label: "dynamic skill announcement" },
- { key: TurnTaskStateKey, label: "task state" },
- ])("executes the approved tool when $label is injected on the resume step", async ({ key }) => {
- const siblingCall = {
- input: { command: "whoami" },
- toolCallId: "call-sibling",
- toolName: "bash",
- type: "tool-call" as const,
- };
- const siblingResult = {
- output: { type: "text" as const, value: "eve" },
- toolCallId: siblingCall.toolCallId,
- toolName: "bash",
- type: "tool-result" as const,
- };
- const session = appendPendingInputBatch({
- requests: [pendingApprovalInputRequest],
- // The parked shape when a gated call shares a step with an ungated one.
- responseMessages: [
- { content: [toolCall, approvalRequest, siblingCall], role: "assistant" },
- { content: [siblingResult], role: "tool" },
- ],
- session: createBaseSession(),
- });
- const ctx = new ContextContainer();
- ctx.set(key, "[Runtime context]\nInjected on the resume step.");
- const execute = vi.fn(async () => "/workspace");
- const model = createModel();
+ it.each(
+ [
+ { key: PendingSkillAnnouncementKey, label: "dynamic skill announcement" },
+ { key: TurnTaskStateKey, label: "task state" },
+ ].flatMap((context) => [false, true].map((restoredAnchor) => ({ ...context, restoredAnchor }))),
+ )(
+ "executes the approved tool when $label is injected on the resume step (restored anchor: $restoredAnchor)",
+ async ({ key, restoredAnchor }) => {
+ const siblingCall = {
+ input: { command: "whoami" },
+ toolCallId: "call-sibling",
+ toolName: "bash",
+ type: "tool-call" as const,
+ };
+ const siblingResult = {
+ output: { type: "text" as const, value: "eve" },
+ toolCallId: siblingCall.toolCallId,
+ toolName: "bash",
+ type: "tool-result" as const,
+ };
+ const session = appendPendingInputBatch({
+ requests: [pendingApprovalInputRequest],
+ // The parked shape when a gated call shares a step with an ungated one.
+ responseMessages: [
+ { content: [toolCall, approvalRequest, siblingCall], role: "assistant" },
+ { content: [siblingResult], role: "tool" },
+ ],
+ session: createBaseSession(),
+ });
+ const ctx = new ContextContainer();
+ ctx.set(key, "[Runtime context]\nInjected on the resume step.");
+ const execute = vi.fn(async () => "/workspace");
+ const model = createModel();
- const result = await contextStorage.run(ctx, () =>
- createToolLoopHarness(createConfig(model, execute))(
- setHarnessEmissionState(session, {
- sequence: 1,
- sessionStarted: true,
- stepIndex: 1,
- turnId: "turn-1",
- }),
- { inputResponses: [{ optionId: "approve", requestId: approvalRequest.approvalId }] },
- ),
- );
+ const result = await contextStorage.run(ctx, () =>
+ createToolLoopHarness(createConfig(model, execute))(
+ setHarnessEmissionState(
+ restoredAnchor
+ ? setTurnClientContextState(session, {
+ insertionIndex: 0,
+ messages: [],
+ turnId: "turn-1",
+ })
+ : session,
+ {
+ sequence: 1,
+ sessionStarted: true,
+ stepIndex: 1,
+ turnId: "turn-1",
+ },
+ ),
+ { inputResponses: [{ optionId: "approve", requestId: approvalRequest.approvalId }] },
+ ),
+ );
- expect(execute).toHaveBeenCalledExactlyOnceWith(
- toolCall.input,
- expect.objectContaining({ toolCallId: toolCall.toolCallId }),
- );
+ expect(execute).toHaveBeenCalledExactlyOnceWith(
+ toolCall.input,
+ expect.objectContaining({ toolCallId: toolCall.toolCallId }),
+ );
- const providerPrompt = model.doGenerateCalls[0]?.prompt ?? [];
- const answered = new Set<string>();
- const called: string[] = [];
- for (const message of providerPrompt) {
- if (!Array.isArray(message.content)) continue;
- for (const part of message.content) {
- if (part.type === "tool-call") called.push(part.toolCallId);
- if (part.type === "tool-result") answered.add(part.toolCallId);
+ const providerPrompt = model.doGenerateCalls[0]?.prompt ?? [];
+ const answered = new Set<string>();
+ const called: string[] = [];
+ for (const message of providerPrompt) {
+ if (!Array.isArray(message.content)) continue;
+ for (const part of message.content) {
+ if (part.type === "tool-call") called.push(part.toolCallId);
+ if (part.type === "tool-result") answered.add(part.toolCallId);
+ }
}
- }
- expect(called).toEqual([toolCall.toolCallId, siblingCall.toolCallId]);
- expect(called.filter((id) => !answered.has(id))).toEqual([]);
- expect(providerPrompt.at(-1)?.role).toBe("tool");
- expect(result.session.history.at(-1)).toMatchObject({
- content: [{ text: "The command returned /workspace.", type: "text" }],
- role: "assistant",
- });
- });
+ expect(called).toEqual([toolCall.toolCallId, siblingCall.toolCallId]);
+ expect(called.filter((id) => !answered.has(id))).toEqual([]);
+ expect(providerPrompt.at(-1)?.role).toBe("tool");
+ expect(result.session.history.at(-1)).toMatchObject({
+ content: [{ text: "The command returned /workspace.", type: "text" }],
+ role: "assistant",
+ });
+ },
+ );
// Acceptance gate for the HITL non-blocking plan (research/hitl-request-lifecycle.md):
// once messages run as normal turns while an approval is open, the approval batch is
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index 20196a0c6..abfdba2a7 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -12255,6 +12255,78 @@ describe("createToolLoopHarness", () => {
});
});
+ it.each(["plain", "client context", "compaction", "projected history"])(
+ "keeps framework context before the turn input across durable steps (%s)",
+ async (scenario) => {
+ const withClientContext = scenario === "client context";
+ if (scenario === "compaction") {
+ vi.mocked(shouldCompact).mockReturnValueOnce(true);
+ vi.mocked(compactMessages).mockImplementationOnce(async (messages) => messages.slice(2));
+ }
+ const toolCall = {
+ type: "tool-call" as const,
+ toolCallId: "cache-call",
+ toolName: "add",
+ input: { a: 20, b: 22 },
+ };
+ const toolResult = {
+ type: "tool-result" as const,
+ toolCallId: "cache-call",
+ toolName: "add",
+ output: "42",
+ };
+ setupMockAgent({
+ finishReason: "tool-calls",
+ response: {
+ messages: [
+ { role: "assistant", content: [toolCall] },
+ { role: "tool", content: [toolResult] },
+ ],
+ },
+ text: "",
+ toolCalls: [toolCall],
+ toolResults: [{ ...toolResult, input: toolCall.input }],
+ });
+ const ctx = new ContextContainer();
+ ctx.set(TurnTaskStateKey, "Task status: analysis in progress");
+ const runStep = createToolLoopHarness(
+ createTestConfig("conversation", undefined, {
+ historyProjector:
+ scenario === "projected history"
+ ? ({ messages }) =>
+ messages.filter((message) => message.content !== "Previous answer")
+ : undefined,
+ }),
+ );
+ const initial = setHarnessEmissionState(
+ createTestSession({
+ history: [
+ { role: "user", content: "Previous request" },
+ { role: "assistant", content: "Previous answer" },
+ ],
+ }),
+ { sessionStarted: true, sequence: 1, stepIndex: 0, turnId: "turn_1" },
+ );
+ const input = { context: ["Current channel context"], message: "Add 20 and 22." };
+ const first = await contextStorage.run(ctx, () =>
+ runStep(
+ initial,
+ withClientContext ? attachClientContext(input, ["Client context"]) : input,
+ ),
+ );
+ expect(first.next).toBe(runStep);
+ const firstPrompt = structuredClone(getLastAgentSettings().messages);
+ setupMockAgent(defaultModelResult());
+ const restored = JSON.parse(JSON.stringify(first.session)) as HarnessSession;
+ await contextStorage.run(ctx, () => runStep(restored));
+ const nextPrompt = getLastAgentSettings().messages;
+ expect(nextPrompt.slice(0, firstPrompt.length)).toEqual(firstPrompt);
+ expect(
+ nextPrompt.filter((message) => message.content === "Task status: analysis in progress"),
+ ).toHaveLength(1);
+ },
+ );
+
it("keeps ephemeral client context out of compaction and its token baseline", async () => {
vi.mocked(shouldCompact).mockReturnValueOnce(true);
vi.mocked(compactMessages).mockImplementationOnce(async (messages) => [...messages]);
diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts
index d30fdf520..85e61d906 100644
--- a/packages/eve/src/harness/tool-loop.ts
+++ b/packages/eve/src/harness/tool-loop.ts
@@ -1067,10 +1067,13 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
// Keep the insertion point stable when a later durable step reconstructs
// the model-only prompt, preserving the full prompt prefix within the turn.
let turnClientContext = storedClientContext;
- if (clientContext !== undefined) {
+ if (
+ clientContext !== undefined ||
+ (storedClientContext === undefined && preparedTurnInput.length > 0)
+ ) {
turnClientContext = {
insertionIndex: storedClientContext?.insertionIndex ?? messages.length,
- messages: [...clientContext],
+ messages: clientContext ?? storedClientContext?.messages ?? [],
turnId,
};
}
@@ -1208,7 +1211,10 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
// Insert that context before the current delivery so the triggering user
// message remains the model's latest request.
const currentMessages = createCurrentMessages(projectedMessages, {
- currentTurnMessages: preparedTurnInput,
+ currentTurnMessages:
+ turnClientContext === undefined
+ ? preparedTurnInput
+ : messages.slice(turnClientContext.insertionIndex),
});
if (ctx !== undefined) {
currentMessages.addSystem(buildDynamicInstructionMessages(ctx));
diff --git a/packages/eve/src/harness/turn-client-context.ts b/packages/eve/src/harness/turn-client-context.ts
index aba8bad9f..79c4705aa 100644
--- a/packages/eve/src/harness/turn-client-context.ts
+++ b/packages/eve/src/harness/turn-client-context.ts
@@ -3,6 +3,7 @@ import type { HarnessSession, SessionStateMap } from "#harness/types.js";
const TURN_CLIENT_CONTEXT_STATE_KEY = "eve.harness.turnClientContext";
export interface TurnClientContextState {
+ /** Durable turn boundary shared by client context and framework announcements. */
readonly insertionIndex: number;
readonly messages: readonly string[];
readonly turnId: string;
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d0b90a5a2..2a771f5fa 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -886,6 +886,9 @@ importers:
typescript:
specifier: 'catalog:'
version: 7.0.2
+ zod:
+ specifier: 'catalog:'
+ version: 4.5.4
e2e/fixtures/agent-subagents:
dependencies: