Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(e2e): remove flaky prompt cache fixture (#2924)

Andrew Barbad7071258064a
diff --git a/e2e/README.md b/e2e/README.md
index b260f40e8..81be80e88 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -142,13 +142,6 @@ alias env (local matrix, plain `eve eval --strict`) the eval skips.
Most fixture agents and their configured judges resolve `EVE_E2E_MODEL`
through `@eve-e2e/config`, defaulting to `openai/gpt-5.6-sol` for local runs.
-`agent-prompt-cache` is the one fixture that authors a direct
-`@ai-sdk/anthropic` model instance instead of a gateway model id: its eval
-asserts the harness's Anthropic cache-breakpoint placement, which only runs on
-that path. It uses the matrix model when it is an Anthropic model and otherwise
-falls back to `anthropic/claude-opus-5`. The instance points at the AI
-Gateway's Anthropic-compatible Messages endpoint so it uses the same
-`AI_GATEWAY_API_KEY` credential as every other fixture.
`agent-workflow-stress` uses eve's `mockModel` fixture helper so its 100-turn
runs stay fast and deterministic. Its concurrent and sequential evals cover
high-volume session execution and repeated session resumption respectively.
diff --git a/e2e/fixtures/agent-prompt-cache/agent/agent.ts b/e2e/fixtures/agent-prompt-cache/agent/agent.ts
deleted file mode 100644
index d37bcb18f..000000000
--- a/e2e/fixtures/agent-prompt-cache/agent/agent.ts
+++ /dev/null
@@ -1,57 +0,0 @@
-import { createAnthropic } from "@ai-sdk/anthropic";
-import { e2eAgentConfig, MOCK_MODEL_SENTINEL } from "@eve-e2e/config";
-import { type AgentDefinition, defineAgent } from "eve";
-import { vercelOidc } from "eve/agents/auth";
-
-/**
- * Prompt-cache e2e fixture.
- *
- * The harness only places explicit cache breakpoints when the model is a
- * provider instance with an Anthropic provider name (`anthropic-direct` in
- * `detectPromptCachePath`). Gateway model id strings take `gateway-auto`,
- * where the gateway caches on its own and the breakpoint code never runs,
- * so a breakpoint regression could not fail an eval there. That is why this
- * fixture authors a direct `@ai-sdk/anthropic` instance while every other
- * fixture uses a gateway string.
- *
- * CI has no `ANTHROPIC_API_KEY`, so the instance points at the AI Gateway's
- * Anthropic-compatible Messages endpoint, which honors `cache_control` and
- * passes `cache_read_input_tokens` / `cache_creation_input_tokens` through
- * unchanged. The same AI Gateway credential as every other fixture is sent
- * as a bearer token: `AI_GATEWAY_API_KEY` takes precedence, with request-scoped
- * Vercel OIDC as the fallback.
- */
-const resolveVercelOidc = vercelOidc();
-const matrixModel = process.env.EVE_E2E_MODEL ?? "anthropic/claude-opus-5";
-const promptCacheModel = matrixModel.startsWith("anthropic/")
- ? matrixModel
- : "anthropic/claude-opus-5";
-
-const anthropic = createAnthropic({
- baseURL: "https://ai-gateway.vercel.sh/v1",
- // The fetch hook replaces this placeholder with a current Gateway credential.
- authToken: "resolved-per-request",
- fetch: async (input, init) => {
- const headers = new Headers(init?.headers);
- const apiKey = process.env.AI_GATEWAY_API_KEY?.trim();
- const authorization = apiKey
- ? `Bearer ${apiKey}`
- : (await resolveVercelOidc()).headers.authorization;
-
- headers.set("authorization", authorization);
- return fetch(input, { ...init, headers });
- },
-});
-
-const agent: AgentDefinition = defineAgent({
- // Harness config wires the workflow world and supplies the mock model in
- // the world suites; real-model runs need the direct Anthropic instance so
- // explicit cache breakpoints stay under test (see the note above).
- ...e2eAgentConfig(),
- ...(process.env.EVE_E2E_MODEL === MOCK_MODEL_SENTINEL
- ? {}
- : { model: anthropic(promptCacheModel) }),
- modelContextWindowTokens: 200_000,
-});
-
-export default agent;
diff --git a/e2e/fixtures/agent-prompt-cache/agent/instructions.md b/e2e/fixtures/agent-prompt-cache/agent/instructions.md
deleted file mode 100644
index eb9129510..000000000
--- a/e2e/fixtures/agent-prompt-cache/agent/instructions.md
+++ /dev/null
@@ -1,174 +0,0 @@
-# District archive assistant
-
-You answer questions about the district archive. Use the `fetch-archive-page`
-tool to retrieve archive pages. Fetch pages strictly one at a time: issue one
-tool call, wait for its result, and only then request the next page. Never
-request more than one page in a single step. Answer tersely.
-
-## Shelf catalog
-
-The catalog below maps shelves to volumes. It exists partly as reference data
-and partly to keep the system prompt above Anthropic's minimum cacheable
-prefix length (2,048 tokens for Haiku-class models), which the prompt-cache
-eval in this fixture depends on.
-
-- Shelf 001 stores volume "Beta Ledger 1" covering reservoir levels for district 2.
-- Shelf 002 stores volume "Gamma Ledger 2" covering canal maintenance logs for district 3.
-- Shelf 003 stores volume "Delta Ledger 3" covering well surveys for district 4.
-- Shelf 004 stores volume "Epsilon Ledger 4" covering flood bulletins for district 5.
-- Shelf 005 stores volume "Zeta Ledger 5" covering irrigation permits for district 6.
-- Shelf 006 stores volume "Eta Ledger 6" covering drainage inspections for district 7.
-- Shelf 007 stores volume "Theta Ledger 7" covering aquifer readings for district 8.
-- Shelf 008 stores volume "Iota Ledger 8" covering rainfall records for district 9.
-- Shelf 009 stores volume "Kappa Ledger 9" covering reservoir levels for district 10.
-- Shelf 010 stores volume "Lambda Ledger 10" covering canal maintenance logs for district 11.
-- Shelf 011 stores volume "Mu Ledger 11" covering well surveys for district 12.
-- Shelf 012 stores volume "Alpha Ledger 12" covering flood bulletins for district 13.
-- Shelf 013 stores volume "Beta Ledger 13" covering irrigation permits for district 14.
-- Shelf 014 stores volume "Gamma Ledger 14" covering drainage inspections for district 15.
-- Shelf 015 stores volume "Delta Ledger 15" covering aquifer readings for district 16.
-- Shelf 016 stores volume "Epsilon Ledger 16" covering rainfall records for district 17.
-- Shelf 017 stores volume "Zeta Ledger 17" covering reservoir levels for district 18.
-- Shelf 018 stores volume "Eta Ledger 18" covering canal maintenance logs for district 19.
-- Shelf 019 stores volume "Theta Ledger 19" covering well surveys for district 20.
-- Shelf 020 stores volume "Iota Ledger 20" covering flood bulletins for district 21.
-- Shelf 021 stores volume "Kappa Ledger 21" covering irrigation permits for district 22.
-- Shelf 022 stores volume "Lambda Ledger 22" covering drainage inspections for district 23.
-- Shelf 023 stores volume "Mu Ledger 23" covering aquifer readings for district 24.
-- Shelf 024 stores volume "Alpha Ledger 24" covering rainfall records for district 25.
-- Shelf 025 stores volume "Beta Ledger 25" covering reservoir levels for district 26.
-- Shelf 026 stores volume "Gamma Ledger 26" covering canal maintenance logs for district 27.
-- Shelf 027 stores volume "Delta Ledger 27" covering well surveys for district 28.
-- Shelf 028 stores volume "Epsilon Ledger 28" covering flood bulletins for district 29.
-- Shelf 029 stores volume "Zeta Ledger 29" covering irrigation permits for district 30.
-- Shelf 030 stores volume "Eta Ledger 30" covering drainage inspections for district 31.
-- Shelf 031 stores volume "Theta Ledger 31" covering aquifer readings for district 32.
-- Shelf 032 stores volume "Iota Ledger 32" covering rainfall records for district 33.
-- Shelf 033 stores volume "Kappa Ledger 33" covering reservoir levels for district 34.
-- Shelf 034 stores volume "Lambda Ledger 34" covering canal maintenance logs for district 35.
-- Shelf 035 stores volume "Mu Ledger 35" covering well surveys for district 36.
-- Shelf 036 stores volume "Alpha Ledger 36" covering flood bulletins for district 37.
-- Shelf 037 stores volume "Beta Ledger 37" covering irrigation permits for district 38.
-- Shelf 038 stores volume "Gamma Ledger 38" covering drainage inspections for district 39.
-- Shelf 039 stores volume "Delta Ledger 39" covering aquifer readings for district 40.
-- Shelf 040 stores volume "Epsilon Ledger 40" covering rainfall records for district 1.
-- Shelf 041 stores volume "Zeta Ledger 41" covering reservoir levels for district 2.
-- Shelf 042 stores volume "Eta Ledger 42" covering canal maintenance logs for district 3.
-- Shelf 043 stores volume "Theta Ledger 43" covering well surveys for district 4.
-- Shelf 044 stores volume "Iota Ledger 44" covering flood bulletins for district 5.
-- Shelf 045 stores volume "Kappa Ledger 45" covering irrigation permits for district 6.
-- Shelf 046 stores volume "Lambda Ledger 46" covering drainage inspections for district 7.
-- Shelf 047 stores volume "Mu Ledger 47" covering aquifer readings for district 8.
-- Shelf 048 stores volume "Alpha Ledger 48" covering rainfall records for district 9.
-- Shelf 049 stores volume "Beta Ledger 49" covering reservoir levels for district 10.
-- Shelf 050 stores volume "Gamma Ledger 50" covering canal maintenance logs for district 11.
-- Shelf 051 stores volume "Delta Ledger 51" covering well surveys for district 12.
-- Shelf 052 stores volume "Epsilon Ledger 52" covering flood bulletins for district 13.
-- Shelf 053 stores volume "Zeta Ledger 53" covering irrigation permits for district 14.
-- Shelf 054 stores volume "Eta Ledger 54" covering drainage inspections for district 15.
-- Shelf 055 stores volume "Theta Ledger 55" covering aquifer readings for district 16.
-- Shelf 056 stores volume "Iota Ledger 56" covering rainfall records for district 17.
-- Shelf 057 stores volume "Kappa Ledger 57" covering reservoir levels for district 18.
-- Shelf 058 stores volume "Lambda Ledger 58" covering canal maintenance logs for district 19.
-- Shelf 059 stores volume "Mu Ledger 59" covering well surveys for district 20.
-- Shelf 060 stores volume "Alpha Ledger 60" covering flood bulletins for district 21.
-- Shelf 061 stores volume "Beta Ledger 61" covering irrigation permits for district 22.
-- Shelf 062 stores volume "Gamma Ledger 62" covering drainage inspections for district 23.
-- Shelf 063 stores volume "Delta Ledger 63" covering aquifer readings for district 24.
-- Shelf 064 stores volume "Epsilon Ledger 64" covering rainfall records for district 25.
-- Shelf 065 stores volume "Zeta Ledger 65" covering reservoir levels for district 26.
-- Shelf 066 stores volume "Eta Ledger 66" covering canal maintenance logs for district 27.
-- Shelf 067 stores volume "Theta Ledger 67" covering well surveys for district 28.
-- Shelf 068 stores volume "Iota Ledger 68" covering flood bulletins for district 29.
-- Shelf 069 stores volume "Kappa Ledger 69" covering irrigation permits for district 30.
-- Shelf 070 stores volume "Lambda Ledger 70" covering drainage inspections for district 31.
-- Shelf 071 stores volume "Mu Ledger 71" covering aquifer readings for district 32.
-- Shelf 072 stores volume "Alpha Ledger 72" covering rainfall records for district 33.
-- Shelf 073 stores volume "Beta Ledger 73" covering reservoir levels for district 34.
-- Shelf 074 stores volume "Gamma Ledger 74" covering canal maintenance logs for district 35.
-- Shelf 075 stores volume "Delta Ledger 75" covering well surveys for district 36.
-- Shelf 076 stores volume "Epsilon Ledger 76" covering flood bulletins for district 37.
-- Shelf 077 stores volume "Zeta Ledger 77" covering irrigation permits for district 38.
-- Shelf 078 stores volume "Eta Ledger 78" covering drainage inspections for district 39.
-- Shelf 079 stores volume "Theta Ledger 79" covering aquifer readings for district 40.
-- Shelf 080 stores volume "Iota Ledger 80" covering rainfall records for district 1.
-- Shelf 081 stores volume "Kappa Ledger 81" covering reservoir levels for district 2.
-- Shelf 082 stores volume "Lambda Ledger 82" covering canal maintenance logs for district 3.
-- Shelf 083 stores volume "Mu Ledger 83" covering well surveys for district 4.
-- Shelf 084 stores volume "Alpha Ledger 84" covering flood bulletins for district 5.
-- Shelf 085 stores volume "Beta Ledger 85" covering irrigation permits for district 6.
-- Shelf 086 stores volume "Gamma Ledger 86" covering drainage inspections for district 7.
-- Shelf 087 stores volume "Delta Ledger 87" covering aquifer readings for district 8.
-- Shelf 088 stores volume "Epsilon Ledger 88" covering rainfall records for district 9.
-- Shelf 089 stores volume "Zeta Ledger 89" covering reservoir levels for district 10.
-- Shelf 090 stores volume "Eta Ledger 90" covering canal maintenance logs for district 11.
-- Shelf 091 stores volume "Theta Ledger 91" covering well surveys for district 12.
-- Shelf 092 stores volume "Iota Ledger 92" covering flood bulletins for district 13.
-- Shelf 093 stores volume "Kappa Ledger 93" covering irrigation permits for district 14.
-- Shelf 094 stores volume "Lambda Ledger 94" covering drainage inspections for district 15.
-- Shelf 095 stores volume "Mu Ledger 95" covering aquifer readings for district 16.
-- Shelf 096 stores volume "Alpha Ledger 96" covering rainfall records for district 17.
-- Shelf 097 stores volume "Beta Ledger 97" covering reservoir levels for district 18.
-- Shelf 098 stores volume "Gamma Ledger 98" covering canal maintenance logs for district 19.
-- Shelf 099 stores volume "Delta Ledger 99" covering well surveys for district 20.
-- Shelf 100 stores volume "Epsilon Ledger 100" covering flood bulletins for district 21.
-- Shelf 101 stores volume "Zeta Ledger 101" covering irrigation permits for district 22.
-- Shelf 102 stores volume "Eta Ledger 102" covering drainage inspections for district 23.
-- Shelf 103 stores volume "Theta Ledger 103" covering aquifer readings for district 24.
-- Shelf 104 stores volume "Iota Ledger 104" covering rainfall records for district 25.
-- Shelf 105 stores volume "Kappa Ledger 105" covering reservoir levels for district 26.
-- Shelf 106 stores volume "Lambda Ledger 106" covering canal maintenance logs for district 27.
-- Shelf 107 stores volume "Mu Ledger 107" covering well surveys for district 28.
-- Shelf 108 stores volume "Alpha Ledger 108" covering flood bulletins for district 29.
-- Shelf 109 stores volume "Beta Ledger 109" covering irrigation permits for district 30.
-- Shelf 110 stores volume "Gamma Ledger 110" covering drainage inspections for district 31.
-- Shelf 111 stores volume "Delta Ledger 111" covering aquifer readings for district 32.
-- Shelf 112 stores volume "Epsilon Ledger 112" covering rainfall records for district 33.
-- Shelf 113 stores volume "Zeta Ledger 113" covering reservoir levels for district 34.
-- Shelf 114 stores volume "Eta Ledger 114" covering canal maintenance logs for district 35.
-- Shelf 115 stores volume "Theta Ledger 115" covering well surveys for district 36.
-- Shelf 116 stores volume "Iota Ledger 116" covering flood bulletins for district 37.
-- Shelf 117 stores volume "Kappa Ledger 117" covering irrigation permits for district 38.
-- Shelf 118 stores volume "Lambda Ledger 118" covering drainage inspections for district 39.
-- Shelf 119 stores volume "Mu Ledger 119" covering aquifer readings for district 40.
-- Shelf 120 stores volume "Alpha Ledger 120" covering rainfall records for district 1.
-- Shelf 121 stores volume "Beta Ledger 121" covering reservoir levels for district 2.
-- Shelf 122 stores volume "Gamma Ledger 122" covering canal maintenance logs for district 3.
-- Shelf 123 stores volume "Delta Ledger 123" covering well surveys for district 4.
-- Shelf 124 stores volume "Epsilon Ledger 124" covering flood bulletins for district 5.
-- Shelf 125 stores volume "Zeta Ledger 125" covering irrigation permits for district 6.
-- Shelf 126 stores volume "Eta Ledger 126" covering drainage inspections for district 7.
-- Shelf 127 stores volume "Theta Ledger 127" covering aquifer readings for district 8.
-- Shelf 128 stores volume "Iota Ledger 128" covering rainfall records for district 9.
-- Shelf 129 stores volume "Kappa Ledger 129" covering reservoir levels for district 10.
-- Shelf 130 stores volume "Lambda Ledger 130" covering canal maintenance logs for district 11.
-- Shelf 131 stores volume "Mu Ledger 131" covering well surveys for district 12.
-- Shelf 132 stores volume "Alpha Ledger 132" covering flood bulletins for district 13.
-- Shelf 133 stores volume "Beta Ledger 133" covering irrigation permits for district 14.
-- Shelf 134 stores volume "Gamma Ledger 134" covering drainage inspections for district 15.
-- Shelf 135 stores volume "Delta Ledger 135" covering aquifer readings for district 16.
-- Shelf 136 stores volume "Epsilon Ledger 136" covering rainfall records for district 17.
-- Shelf 137 stores volume "Zeta Ledger 137" covering reservoir levels for district 18.
-- Shelf 138 stores volume "Eta Ledger 138" covering canal maintenance logs for district 19.
-- Shelf 139 stores volume "Theta Ledger 139" covering well surveys for district 20.
-- Shelf 140 stores volume "Iota Ledger 140" covering flood bulletins for district 21.
-- Shelf 141 stores volume "Kappa Ledger 141" covering irrigation permits for district 22.
-- Shelf 142 stores volume "Lambda Ledger 142" covering drainage inspections for district 23.
-- Shelf 143 stores volume "Mu Ledger 143" covering aquifer readings for district 24.
-- Shelf 144 stores volume "Alpha Ledger 144" covering rainfall records for district 25.
-- Shelf 145 stores volume "Beta Ledger 145" covering reservoir levels for district 26.
-- Shelf 146 stores volume "Gamma Ledger 146" covering canal maintenance logs for district 27.
-- Shelf 147 stores volume "Delta Ledger 147" covering well surveys for district 28.
-- Shelf 148 stores volume "Epsilon Ledger 148" covering flood bulletins for district 29.
-- Shelf 149 stores volume "Zeta Ledger 149" covering irrigation permits for district 30.
-- Shelf 150 stores volume "Eta Ledger 150" covering drainage inspections for district 31.
-- Shelf 151 stores volume "Theta Ledger 151" covering aquifer readings for district 32.
-- Shelf 152 stores volume "Iota Ledger 152" covering rainfall records for district 33.
-- Shelf 153 stores volume "Kappa Ledger 153" covering reservoir levels for district 34.
-- Shelf 154 stores volume "Lambda Ledger 154" covering canal maintenance logs for district 35.
-- Shelf 155 stores volume "Mu Ledger 155" covering well surveys for district 36.
-- Shelf 156 stores volume "Alpha Ledger 156" covering flood bulletins for district 37.
-- Shelf 157 stores volume "Beta Ledger 157" covering irrigation permits for district 38.
-- Shelf 158 stores volume "Gamma Ledger 158" covering drainage inspections for district 39.
-- Shelf 159 stores volume "Delta Ledger 159" covering aquifer readings for district 40.
-- Shelf 160 stores volume "Epsilon Ledger 160" covering rainfall records for district 1.
diff --git a/e2e/fixtures/agent-prompt-cache/agent/tools/fetch-archive-page.ts b/e2e/fixtures/agent-prompt-cache/agent/tools/fetch-archive-page.ts
deleted file mode 100644
index 9ac710f2f..000000000
--- a/e2e/fixtures/agent-prompt-cache/agent/tools/fetch-archive-page.ts
+++ /dev/null
@@ -1,26 +0,0 @@
-import { defineTool } from "eve/tools";
-import { z } from "zod";
-
-/**
- * Returns one deterministic ~2,500-token page of archive rows. The payload
- * is intentionally large: the prompt-cache eval measures whether these tool
- * results are cached in the same request that first carries them, and small
- * results would drown the signal in per-request framing tokens.
- */
-export default defineTool({
- description:
- "Returns one page of the district archive. Call once per page number and wait for the result before requesting the next page.",
- inputSchema: z.object({
- page: z.number().int().min(1).max(9).describe("Archive page number to fetch."),
- }),
- async execute({ page }) {
- const rows: string[] = [];
- for (let row = 1; row <= 220; row++) {
- rows.push(
- `page ${page} row ${String(row).padStart(3, "0")}: district ${((page * 7 + row) % 40) + 1} ` +
- `rainfall ${(page * 31 + row * 13) % 200} mm, reservoir ${(page * 17 + row * 11) % 100}% full`,
- );
- }
- return { page, rowCount: rows.length, content: rows.join("\n") };
- },
-});
diff --git a/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts b/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts
deleted file mode 100644
index 943c028ce..000000000
--- a/e2e/fixtures/agent-prompt-cache/evals/evals.config.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { e2eJudgeModel } from "@eve-e2e/config";
-import { defineEvalConfig } from "eve/evals";
-
-/** Judge is unused here; assertions are numeric. Kept for config parity. */
-export default defineEvalConfig({
- judge: { model: e2eJudgeModel() },
-});
diff --git a/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts b/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts
deleted file mode 100644
index 243f38084..000000000
--- a/e2e/fixtures/agent-prompt-cache/evals/prompt-cache/input-cache-rate.eval.ts
+++ /dev/null
@@ -1,76 +0,0 @@
-import { defineEval } from "eve/evals";
-import { satisfies } from "eve/evals/expect";
-
-/**
- * Proves the harness's Anthropic cache breakpoints keep a regular
- * (non-compacted) tool session almost fully served from the prompt cache.
- *
- * The metric is the input-cache rate: of the prompt tokens the model had
- * already seen on a previous step (everything except first-time cache
- * writes), how many were read from cache?
- *
- * rate = Σ cache_read / (Σ cache_read + Σ uncached_input)
- * uncached_input = input_total − cache_read − cache_write
- *
- * With breakpoints on the last message of every request, each step's new
- * content is written to the cache in the request that first carries it, so
- * uncached input stays at a few framing tokens per step and the rate lands
- * above 99%. When the final breakpoint lags one message (the regression this
- * eval guards against), every tool result is billed uncached once before it
- * enters the cache, and the rate collapses to roughly 45–60% — see
- * `packages/eve/src/harness/prompt-cache-accounting.test.ts` for the
- * trace-level accounting.
- */
-export default defineEval({
- tags: ["real-model"],
- description:
- "Anthropic-direct prompt caching: input-cache rate stays above 99% across a multi-step tool session.",
- async test(t) {
- const first = await t.send(
- "Fetch archive pages 1, 2, and 3 using the fetch-archive-page tool, strictly one page per " +
- 'tool call, waiting for each result before the next call. Then reply with exactly "PAGES LOADED".',
- );
- first.expectOk();
- first.calledTool("fetch-archive-page");
-
- const second = await t.send(
- 'Now fetch archive page 4 the same way, then reply with exactly "DONE".',
- );
- second.expectOk();
- second.calledTool("fetch-archive-page");
-
- let cacheRead = 0;
- let uncachedInput = 0;
- let stepCount = 0;
- for (const event of t.events) {
- if (event.type !== "step.completed") continue;
- const usage = event.data.usage;
- if (usage === undefined) continue;
- stepCount += 1;
- const read = usage.cacheReadTokens ?? 0;
- const uncached = (usage.inputTokens ?? 0) - read - (usage.cacheWriteTokens ?? 0);
- cacheRead += read;
- uncachedInput += Math.max(0, uncached);
- t.log(
- `step ${stepCount}: input=${usage.inputTokens ?? 0} read=${read} ` +
- `write=${usage.cacheWriteTokens ?? 0} uncached=${uncached}`,
- );
- }
-
- const rate = cacheRead / (cacheRead + uncachedInput);
- t.log(
- `input-cache rate: ${(rate * 100).toFixed(2)}% ` +
- `(read=${cacheRead} uncached=${uncachedInput} over ${stepCount} steps)`,
- );
-
- // A multi-step session is required for the metric to mean anything.
- t.check(
- stepCount,
- satisfies((steps: number) => steps >= 4, "session ran at least 4 model steps"),
- );
- t.check(
- rate,
- satisfies((value: number) => value > 0.99, "input-cache rate above 99%"),
- );
- },
-});
diff --git a/e2e/fixtures/agent-prompt-cache/package.json b/e2e/fixtures/agent-prompt-cache/package.json
deleted file mode 100644
index ea4fa6e3b..000000000
--- a/e2e/fixtures/agent-prompt-cache/package.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
- "name": "agent-prompt-cache",
- "version": "0.0.0",
- "private": true,
- "type": "module",
- "scripts": {
- "build": "eve build",
- "dev": "eve dev",
- "start": "eve start",
- "typecheck": "eve build && tsc",
- "test:e2e": "eve eval --strict"
- },
- "dependencies": {
- "@ai-sdk/anthropic": "catalog:",
- "@eve-e2e/config": "workspace:*",
- "@workflow/world-postgres": "catalog:",
- "eve": "workspace:*",
- "zod": "catalog:"
- },
- "devDependencies": {
- "@types/node": "catalog:",
- "typescript": "catalog:"
- },
- "e2e": {
- "modelMatrix": "full"
- }
-}
diff --git a/e2e/fixtures/agent-prompt-cache/tsconfig.json b/e2e/fixtures/agent-prompt-cache/tsconfig.json
deleted file mode 100644
index 33033274f..000000000
--- a/e2e/fixtures/agent-prompt-cache/tsconfig.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
- "compilerOptions": {
- "target": "ES2022",
- "module": "NodeNext",
- "moduleResolution": "NodeNext",
- "outDir": "dist",
- "rootDir": ".",
- "strict": true,
- "esModuleInterop": true,
- "skipLibCheck": true,
- "forceConsistentCasingInFileNames": true,
- "declaration": true,
- "noEmit": true,
- "types": ["node"]
- },
- "include": ["agent/**/*.ts", "evals/**/*.ts"]
-}
diff --git a/packages/eve/src/harness/prompt-cache-accounting.test.ts b/packages/eve/src/harness/prompt-cache-accounting.test.ts
index 9be92b7dd..a2431aae7 100644
--- a/packages/eve/src/harness/prompt-cache-accounting.test.ts
+++ b/packages/eve/src/harness/prompt-cache-accounting.test.ts
@@ -444,7 +444,6 @@ describe("grep-heavy coding session", () => {
for (const turn of usage) {
expect(turn.input).toBe(0);
}
- // Mirrors the agent-prompt-cache e2e gate.
expect(inputCacheRate(usage)).toBeGreaterThan(0.99);
});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index dd95d938b..3e9cd3716 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -812,31 +812,6 @@ importers:
specifier: 'catalog:'
version: 7.0.2
- e2e/fixtures/agent-prompt-cache:
- dependencies:
- '@ai-sdk/anthropic':
- specifier: 'catalog:'
- version: 4.0.36([email protected])
- '@eve-e2e/config':
- specifier: workspace:*
- version: link:../e2e-config
- '@workflow/world-postgres':
- specifier: 'catalog:'
- eve:
- specifier: workspace:*
- version: link:../../../packages/eve
- zod:
- specifier: 'catalog:'
- version: 4.5.4
- devDependencies:
- '@types/node':
- specifier: 'catalog:'
- version: 25.9.1
- typescript:
- specifier: 'catalog:'
- version: 7.0.2
-
e2e/fixtures/agent-schedules:
dependencies:
'@eve-e2e/config':