The Open Framework for Building Agents
test(eve): cover progress preservation across compaction (#755)
Rui72ccdc0ec536
diff --git a/.changeset/calm-compaction-checkpoints.md b/.changeset/calm-compaction-checkpoints.mdnew file mode 100644index 000000000..b3617d387--- /dev/null+++ b/.changeset/calm-compaction-checkpoints.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Compaction now reserves room for its checkpoint prompt before reaching the configured threshold. The prompt asks the compaction model to distinguish completed work from remaining work, and later compactions receive the previous checkpoint intact instead of truncating it with ordinary transcript text.diff --git a/docs/agent-config.md b/docs/agent-config.mdindex 8fcb33e47..e6da47ecd 100644--- a/docs/agent-config.md+++ b/docs/agent-config.md@@ -106,7 +106,7 @@ which levels are available and how they map to provider-native settings. Use## Compaction-Compaction summarizes older turns as you approach the context window. It's on by default, so you only tune when it kicks in. Lower `thresholdPercent` to compact sooner:+Compaction summarizes older turns as you approach the context window. It's on by default, so you only tune when it kicks in. eve adds the estimated fixed checkpoint-prompt envelope to the trigger count, so compaction starts sooner than the conversation-only estimate. Lower `thresholdPercent` to compact sooner:```ts title="agent/agent.ts"export default defineAgent({diff --git a/docs/concepts/default-harness.md b/docs/concepts/default-harness.mdindex 8d519fac7..c680253f6 100644--- a/docs/concepts/default-harness.md+++ b/docs/concepts/default-harness.md@@ -7,7 +7,7 @@ The default harness is eve's built-in agent loop. It manages model calls, compac## Compaction-The harness keeps a long session from overflowing the model's context window. Once the conversation crosses a fraction of the window (`thresholdPercent`, `0.9` by default), it summarizes the older turns into a compact form and keeps going. The summary uses the active turn model unless you override it. Tune when and how it kicks in under [`compaction`](../agent-config#compaction) in `agent.ts`:+The harness keeps a long session from overflowing the model's context window. Before comparing the conversation with `thresholdPercent` (`0.9` by default), it adds the estimated fixed envelope of the checkpoint prompt used for compaction. It then summarizes the older turns and keeps going. The prompt asks the compaction model to distinguish completed progress and decisions from remaining work and to retain the constraints, preferences, data, and references needed to continue. When eve compacts again, it passes the previous checkpoint separately and without the transcript's per-message truncation, then replaces it with the updated checkpoint. The summary uses the active turn model unless you override it. Tune when and how it kicks in under [`compaction`](../agent-config#compaction) in `agent.ts`:```ts title="agent/agent.ts"export default defineAgent({diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/agent.tsnew file mode 100644index 000000000..62a0b8da5--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/agent.ts@@ -0,0 +1,9 @@+import { createCompactionRegressionAgent } from "@eve/e2e-compaction-regression-shared/agent";+import type { AgentDefinition } from "eve";++const agent: AgentDefinition = createCompactionRegressionAgent({+ compactionModel: "openai/gpt-5.6-sol",+ modelFamily: "gpt-5.6",+});++export default agent;diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/instructions.md b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/instructions.mdnew file mode 100644index 000000000..b296ae3dc--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/instructions.md@@ -0,0 +1,14 @@+# Compaction regression fixture++Follow the user's requested regression case exactly.++When a tool reports `completed: true` or returns a `completionMarker`, that work+is complete. Call that tool no more than once. Report the marker without+restarting the work after context compaction.++Completed evidence takes precedence over a pending todo. If a compacted summary+says a work unit is complete but the todo still shows it as pending, do not run+the work again.++If a tool returns `hardStop: true`, call no more tools and report its+`completionMarker` immediately.diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/advance-checkpoint.ts b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/advance-checkpoint.tsnew file mode 100644index 000000000..e7c2434b0--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/advance-checkpoint.ts@@ -0,0 +1 @@+export { default } from "@eve/e2e-compaction-regression-shared/tools/advance-checkpoint";diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/inspect-repository.ts b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/inspect-repository.tsnew file mode 100644index 000000000..a90064462--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/inspect-repository.ts@@ -0,0 +1 @@+export { default } from "@eve/e2e-compaction-regression-shared/tools/inspect-repository";diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/perform-source-analysis.ts b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/perform-source-analysis.tsnew file mode 100644index 000000000..9859f5c2f--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/agent/tools/perform-source-analysis.ts@@ -0,0 +1 @@+export { default } from "@eve/e2e-compaction-regression-shared/tools/perform-source-analysis";diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/evals/evals.config.ts b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/evals/evals.config.tsnew file mode 100644index 000000000..b159ca9a0--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/evals/evals.config.ts@@ -0,0 +1,6 @@+import { defineEvalConfig } from "eve/evals";++export default defineEvalConfig({+ maxConcurrency: 1,+ timeoutMs: 180_000,+});diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/evals/redundant-tool-calls/gpt-5.6.eval.ts b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/evals/redundant-tool-calls/gpt-5.6.eval.tsnew file mode 100644index 000000000..fd0aaf09d--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/evals/redundant-tool-calls/gpt-5.6.eval.ts@@ -0,0 +1,9 @@+import { defineEval } from "eve/evals";+import { testRedundantToolCalls } from "@eve/e2e-compaction-regression-shared/evals";++export default defineEval({+ description: "GPT-5.6 does not repeat an identical successful call after compaction.",+ async test(t) {+ await testRedundantToolCalls(t, "gpt-5.6");+ },+});diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/evals/stale-todo-work/gpt-5.6.eval.ts b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/evals/stale-todo-work/gpt-5.6.eval.tsnew file mode 100644index 000000000..689751f30--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/evals/stale-todo-work/gpt-5.6.eval.ts@@ -0,0 +1,9 @@+import { defineEval } from "eve/evals";+import { testStaleTodoWork } from "@eve/e2e-compaction-regression-shared/evals";++export default defineEval({+ description: "GPT-5.6 does not repeat completed work because a todo stayed pending.",+ async test(t) {+ await testStaleTodoWork(t, "gpt-5.6");+ },+});diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/package.json b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/package.jsonnew file mode 100644index 000000000..84b2f40e2--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/package.json@@ -0,0 +1,25 @@+{+ "name": "agent-compaction-regressions-gpt-5-6",+ "version": "0.0.0",+ "private": true,+ "type": "module",+ "scripts": {+ "build": "eve build",+ "dev": "eve dev",+ "start": "eve start",+ "typecheck": "eve build && tsc",+ "test:e2e": "eve eval --strict"+ },+ "dependencies": {+ "@eve/e2e-compaction-regression-shared": "workspace:*",+ "@opentelemetry/core": "2.6.1",+ "@opentelemetry/sdk-trace-base": "2.6.1",+ "@vercel/otel": "2.1.2",+ "eve": "workspace:*",+ "zod": "catalog:"+ },+ "devDependencies": {+ "@types/node": "catalog:",+ "typescript": "catalog:"+ }+}diff --git a/e2e/fixtures/agent-compaction-regressions-gpt-5-6/tsconfig.json b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/tsconfig.jsonnew file mode 100644index 000000000..85e3eceb8--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-gpt-5-6/tsconfig.json@@ -0,0 +1,17 @@+{+ "compilerOptions": {+ "target": "ES2022",+ "module": "esnext",+ "moduleResolution": "bundler",+ "outDir": "dist",+ "rootDir": ".",+ "strict": true,+ "esModuleInterop": true,+ "skipLibCheck": true,+ "forceConsistentCasingInFileNames": true,+ "declaration": true,+ "noEmit": true,+ "types": ["node"]+ },+ "include": ["agent/**/*.ts", "evals/**/*.ts"]+}diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/agent.tsnew file mode 100644index 000000000..d53a7276b--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/agent.ts@@ -0,0 +1,9 @@+import { createCompactionRegressionAgent } from "@eve/e2e-compaction-regression-shared/agent";+import type { AgentDefinition } from "eve";++const agent: AgentDefinition = createCompactionRegressionAgent({+ compactionModel: "anthropic/claude-opus-4.8",+ modelFamily: "opus-4.8",+});++export default agent;diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/instructions.md b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/instructions.mdnew file mode 100644index 000000000..b296ae3dc--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/instructions.md@@ -0,0 +1,14 @@+# Compaction regression fixture++Follow the user's requested regression case exactly.++When a tool reports `completed: true` or returns a `completionMarker`, that work+is complete. Call that tool no more than once. Report the marker without+restarting the work after context compaction.++Completed evidence takes precedence over a pending todo. If a compacted summary+says a work unit is complete but the todo still shows it as pending, do not run+the work again.++If a tool returns `hardStop: true`, call no more tools and report its+`completionMarker` immediately.diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/advance-checkpoint.ts b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/advance-checkpoint.tsnew file mode 100644index 000000000..e7c2434b0--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/advance-checkpoint.ts@@ -0,0 +1 @@+export { default } from "@eve/e2e-compaction-regression-shared/tools/advance-checkpoint";diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/inspect-repository.ts b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/inspect-repository.tsnew file mode 100644index 000000000..a90064462--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/inspect-repository.ts@@ -0,0 +1 @@+export { default } from "@eve/e2e-compaction-regression-shared/tools/inspect-repository";diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/perform-source-analysis.ts b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/perform-source-analysis.tsnew file mode 100644index 000000000..9859f5c2f--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/agent/tools/perform-source-analysis.ts@@ -0,0 +1 @@+export { default } from "@eve/e2e-compaction-regression-shared/tools/perform-source-analysis";diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/evals/evals.config.ts b/e2e/fixtures/agent-compaction-regressions-opus-4-8/evals/evals.config.tsnew file mode 100644index 000000000..b159ca9a0--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/evals/evals.config.ts@@ -0,0 +1,6 @@+import { defineEvalConfig } from "eve/evals";++export default defineEvalConfig({+ maxConcurrency: 1,+ timeoutMs: 180_000,+});diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/evals/redundant-tool-calls/opus-4.8.eval.ts b/e2e/fixtures/agent-compaction-regressions-opus-4-8/evals/redundant-tool-calls/opus-4.8.eval.tsnew file mode 100644index 000000000..5a5614d7a--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/evals/redundant-tool-calls/opus-4.8.eval.ts@@ -0,0 +1,9 @@+import { testRedundantToolCalls } from "@eve/e2e-compaction-regression-shared/evals";+import { defineEval } from "eve/evals";++export default defineEval({+ description: "Claude Opus 4.8 does not repeat an identical successful call after compaction.",+ async test(t) {+ await testRedundantToolCalls(t, "opus-4.8");+ },+});diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/evals/stale-todo-work/opus-4.8.eval.ts b/e2e/fixtures/agent-compaction-regressions-opus-4-8/evals/stale-todo-work/opus-4.8.eval.tsnew file mode 100644index 000000000..e1493c2ef--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/evals/stale-todo-work/opus-4.8.eval.ts@@ -0,0 +1,9 @@+import { testStaleTodoWork } from "@eve/e2e-compaction-regression-shared/evals";+import { defineEval } from "eve/evals";++export default defineEval({+ description: "Claude Opus 4.8 does not repeat completed work because a todo stayed pending.",+ async test(t) {+ await testStaleTodoWork(t, "opus-4.8");+ },+});diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/package.json b/e2e/fixtures/agent-compaction-regressions-opus-4-8/package.jsonnew file mode 100644index 000000000..bc9fed894--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/package.json@@ -0,0 +1,25 @@+{+ "name": "agent-compaction-regressions-opus-4-8",+ "version": "0.0.0",+ "private": true,+ "type": "module",+ "scripts": {+ "build": "eve build",+ "dev": "eve dev",+ "start": "eve start",+ "typecheck": "eve build && tsc",+ "test:e2e": "eve eval --strict"+ },+ "dependencies": {+ "@eve/e2e-compaction-regression-shared": "workspace:*",+ "@opentelemetry/core": "2.6.1",+ "@opentelemetry/sdk-trace-base": "2.6.1",+ "@vercel/otel": "2.1.2",+ "eve": "workspace:*",+ "zod": "catalog:"+ },+ "devDependencies": {+ "@types/node": "catalog:",+ "typescript": "catalog:"+ }+}diff --git a/e2e/fixtures/agent-compaction-regressions-opus-4-8/tsconfig.json b/e2e/fixtures/agent-compaction-regressions-opus-4-8/tsconfig.jsonnew file mode 100644index 000000000..85e3eceb8--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-opus-4-8/tsconfig.json@@ -0,0 +1,17 @@+{+ "compilerOptions": {+ "target": "ES2022",+ "module": "esnext",+ "moduleResolution": "bundler",+ "outDir": "dist",+ "rootDir": ".",+ "strict": true,+ "esModuleInterop": true,+ "skipLibCheck": true,+ "forceConsistentCasingInFileNames": true,+ "declaration": true,+ "noEmit": true,+ "types": ["node"]+ },+ "include": ["agent/**/*.ts", "evals/**/*.ts"]+}diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/agent.tsnew file mode 100644index 000000000..6841e0910--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/agent.ts@@ -0,0 +1,9 @@+import { createCompactionRegressionAgent } from "@eve/e2e-compaction-regression-shared/agent";+import type { AgentDefinition } from "eve";++const agent: AgentDefinition = createCompactionRegressionAgent({+ compactionModel: "anthropic/claude-sonnet-5",+ modelFamily: "sonnet-5",+});++export default agent;diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/instructions.md b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/instructions.mdnew file mode 100644index 000000000..b296ae3dc--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/instructions.md@@ -0,0 +1,14 @@+# Compaction regression fixture++Follow the user's requested regression case exactly.++When a tool reports `completed: true` or returns a `completionMarker`, that work+is complete. Call that tool no more than once. Report the marker without+restarting the work after context compaction.++Completed evidence takes precedence over a pending todo. If a compacted summary+says a work unit is complete but the todo still shows it as pending, do not run+the work again.++If a tool returns `hardStop: true`, call no more tools and report its+`completionMarker` immediately.diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/advance-checkpoint.ts b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/advance-checkpoint.tsnew file mode 100644index 000000000..e7c2434b0--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/advance-checkpoint.ts@@ -0,0 +1 @@+export { default } from "@eve/e2e-compaction-regression-shared/tools/advance-checkpoint";diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/inspect-repository.ts b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/inspect-repository.tsnew file mode 100644index 000000000..a90064462--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/inspect-repository.ts@@ -0,0 +1 @@+export { default } from "@eve/e2e-compaction-regression-shared/tools/inspect-repository";diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/perform-source-analysis.ts b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/perform-source-analysis.tsnew file mode 100644index 000000000..9859f5c2f--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/agent/tools/perform-source-analysis.ts@@ -0,0 +1 @@+export { default } from "@eve/e2e-compaction-regression-shared/tools/perform-source-analysis";diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/evals/evals.config.ts b/e2e/fixtures/agent-compaction-regressions-sonnet-5/evals/evals.config.tsnew file mode 100644index 000000000..b159ca9a0--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/evals/evals.config.ts@@ -0,0 +1,6 @@+import { defineEvalConfig } from "eve/evals";++export default defineEvalConfig({+ maxConcurrency: 1,+ timeoutMs: 180_000,+});diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/evals/redundant-tool-calls/sonnet-5.eval.ts b/e2e/fixtures/agent-compaction-regressions-sonnet-5/evals/redundant-tool-calls/sonnet-5.eval.tsnew file mode 100644index 000000000..4bf1c8835--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/evals/redundant-tool-calls/sonnet-5.eval.ts@@ -0,0 +1,9 @@+import { testRedundantToolCalls } from "@eve/e2e-compaction-regression-shared/evals";+import { defineEval } from "eve/evals";++export default defineEval({+ description: "Claude Sonnet 5 does not repeat an identical successful call after compaction.",+ async test(t) {+ await testRedundantToolCalls(t, "sonnet-5");+ },+});diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/evals/stale-todo-work/sonnet-5.eval.ts b/e2e/fixtures/agent-compaction-regressions-sonnet-5/evals/stale-todo-work/sonnet-5.eval.tsnew file mode 100644index 000000000..3f76c4d02--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/evals/stale-todo-work/sonnet-5.eval.ts@@ -0,0 +1,9 @@+import { testStaleTodoWork } from "@eve/e2e-compaction-regression-shared/evals";+import { defineEval } from "eve/evals";++export default defineEval({+ description: "Claude Sonnet 5 does not repeat completed work because a todo stayed pending.",+ async test(t) {+ await testStaleTodoWork(t, "sonnet-5");+ },+});diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/package.json b/e2e/fixtures/agent-compaction-regressions-sonnet-5/package.jsonnew file mode 100644index 000000000..2ea9748b5--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/package.json@@ -0,0 +1,25 @@+{+ "name": "agent-compaction-regressions-sonnet-5",+ "version": "0.0.0",+ "private": true,+ "type": "module",+ "scripts": {+ "build": "eve build",+ "dev": "eve dev",+ "start": "eve start",+ "typecheck": "eve build && tsc",+ "test:e2e": "eve eval --strict"+ },+ "dependencies": {+ "@eve/e2e-compaction-regression-shared": "workspace:*",+ "@opentelemetry/core": "2.6.1",+ "@opentelemetry/sdk-trace-base": "2.6.1",+ "@vercel/otel": "2.1.2",+ "eve": "workspace:*",+ "zod": "catalog:"+ },+ "devDependencies": {+ "@types/node": "catalog:",+ "typescript": "catalog:"+ }+}diff --git a/e2e/fixtures/agent-compaction-regressions-sonnet-5/tsconfig.json b/e2e/fixtures/agent-compaction-regressions-sonnet-5/tsconfig.jsonnew file mode 100644index 000000000..85e3eceb8--- /dev/null+++ b/e2e/fixtures/agent-compaction-regressions-sonnet-5/tsconfig.json@@ -0,0 +1,17 @@+{+ "compilerOptions": {+ "target": "ES2022",+ "module": "esnext",+ "moduleResolution": "bundler",+ "outDir": "dist",+ "rootDir": ".",+ "strict": true,+ "esModuleInterop": true,+ "skipLibCheck": true,+ "forceConsistentCasingInFileNames": true,+ "declaration": true,+ "noEmit": true,+ "types": ["node"]+ },+ "include": ["agent/**/*.ts", "evals/**/*.ts"]+}diff --git a/e2e/fixtures/compaction-regression-shared/package.json b/e2e/fixtures/compaction-regression-shared/package.jsonnew file mode 100644index 000000000..7c3e6963c--- /dev/null+++ b/e2e/fixtures/compaction-regression-shared/package.json@@ -0,0 +1,17 @@+{+ "name": "@eve/e2e-compaction-regression-shared",+ "version": "0.0.0",+ "private": true,+ "type": "module",+ "exports": {+ "./agent": "./src/agent.ts",+ "./evals": "./src/evals.ts",+ "./tools/advance-checkpoint": "./src/advance-checkpoint.ts",+ "./tools/inspect-repository": "./src/inspect-repository.ts",+ "./tools/perform-source-analysis": "./src/perform-source-analysis.ts"+ },+ "dependencies": {+ "eve": "workspace:*",+ "zod": "catalog:"+ }+}diff --git a/e2e/fixtures/compaction-regression-shared/src/advance-checkpoint.ts b/e2e/fixtures/compaction-regression-shared/src/advance-checkpoint.tsnew file mode 100644index 000000000..d61f1c769--- /dev/null+++ b/e2e/fixtures/compaction-regression-shared/src/advance-checkpoint.ts@@ -0,0 +1,30 @@+import { defineState } from "eve/context";+import { defineTool } from "eve/tools";+import { z } from "zod";++import { SECOND_CHECKPOINT_MARKER } from "./constants";++const invocationCount = defineState("compaction-regression.advance-checkpoint", () => 0);+const modelFamilySchema = z.enum(["gpt-5.6", "opus-4.8", "sonnet-5"]);++export default defineTool({+ description:+ "Test-only second-compaction trigger tool. Records a completed fixture work unit and adds enough evidence for the harness to cross the compaction threshold again.",+ inputSchema: z.object({+ modelFamily: modelFamilySchema,+ regressionCase: z.enum(["redundant-tool-calls", "stale-todo-work"]),+ }),+ async execute(input) {+ const attempt = invocationCount.get() + 1;+ invocationCount.update(() => attempt);++ return {+ checkpointMarker: SECOND_CHECKPOINT_MARKER,+ completed: true,+ modelFamily: input.modelFamily,+ regressionCase: input.regressionCase,+ attempt,+ evidencePadding: "second checkpoint evidence ".repeat(100),+ };+ },+});diff --git a/e2e/fixtures/compaction-regression-shared/src/agent.ts b/e2e/fixtures/compaction-regression-shared/src/agent.tsnew file mode 100644index 000000000..b951f5020--- /dev/null+++ b/e2e/fixtures/compaction-regression-shared/src/agent.ts@@ -0,0 +1,150 @@+import { defineAgent, type AgentDefinition } from "eve";+import { mockModel, type MockModelRequest } from "eve/evals";++import { SECOND_CHECKPOINT_MARKER } from "./constants";++const TEST_CONTEXT_WINDOW_TOKENS = 32_000;+const MAX_TOOL_CALLS = 10;++export type ModelFamily = "gpt-5.6" | "opus-4.8" | "sonnet-5";+type RegressionCase = "redundant-tool-calls" | "stale-todo-work";++interface ActiveRegression {+ readonly regressionCase: RegressionCase;+}++export function createCompactionRegressionAgent(input: {+ readonly compactionModel: string;+ readonly modelFamily: ModelFamily;+}): AgentDefinition {+ let activeRegression: ActiveRegression | undefined;+ const checkpointAdvanceCallCounts = new Map<RegressionCase, number>();+ const toolCallCounts = new Map<RegressionCase, number>();+ const taskModel = mockModel({+ modelId: `compaction-regression-task-model-${input.modelFamily}`,+ respond(request) {+ const initialRegression = findInitialRegression(request, input.modelFamily);+ if (+ initialRegression !== undefined &&+ activeRegression?.regressionCase !== initialRegression.regressionCase+ ) {+ activeRegression = initialRegression;+ checkpointAdvanceCallCounts.set(initialRegression.regressionCase, 0);+ toolCallCounts.set(initialRegression.regressionCase, 0);+ }++ if (activeRegression === undefined) {+ throw new Error("Compaction regression task model received no case marker.");+ }++ const regression = activeRegression;+ const marker = completionMarker(regression.regressionCase);++ // These are fixture markers, not compaction protocol fields. `marker` records the+ // regression work tool; `SECOND_CHECKPOINT_MARKER` records the test-only tool+ // whose output makes the harness cross the compaction threshold a second time.+ if (checkpointContains(request.messages, marker)) {+ if (checkpointContains(request.messages, SECOND_CHECKPOINT_MARKER)) {+ return `Done: ${marker}; ${SECOND_CHECKPOINT_MARKER}`;+ }++ const advanceCalls = checkpointAdvanceCallCounts.get(regression.regressionCase) ?? 0;+ if (advanceCalls >= MAX_TOOL_CALLS) {+ return `Hard stop after ${MAX_TOOL_CALLS} checkpoint advances: ${marker}`;+ }++ checkpointAdvanceCallCounts.set(regression.regressionCase, advanceCalls + 1);+ return {+ toolCalls: [+ {+ id: `advance-checkpoint-${advanceCalls + 1}`,+ input: {+ modelFamily: input.modelFamily,+ regressionCase: regression.regressionCase,+ },+ name: "advance-checkpoint",+ },+ ],+ };+ }++ const completedCalls = toolCallCounts.get(regression.regressionCase) ?? 0;+ if (completedCalls >= MAX_TOOL_CALLS) {+ return `Hard stop after ${MAX_TOOL_CALLS} calls: ${marker}`;+ }++ const attempt = completedCalls + 1;+ toolCallCounts.set(regression.regressionCase, attempt);++ return regression.regressionCase === "redundant-tool-calls"+ ? {+ toolCalls: [+ {+ id: `inspect-repository-${attempt}`,+ input: { modelFamily: input.modelFamily, scope: "repository" },+ name: "inspect-repository",+ },+ ],+ }+ : {+ toolCalls: [+ {+ id: `perform-source-analysis-${attempt}`,+ input: { approach: `attempt-${attempt}`, modelFamily: input.modelFamily },+ name: "perform-source-analysis",+ },+ ],+ };+ },+ });++ return defineAgent({+ model: taskModel,+ modelContextWindowTokens: TEST_CONTEXT_WINDOW_TOKENS,+ compaction: {+ model: input.compactionModel,+ modelContextWindowTokens: TEST_CONTEXT_WINDOW_TOKENS,+ thresholdPercent: 0.02,+ },+ limits: {+ maxInputTokensPerSession: 100_000,+ },+ });+}++function findInitialRegression(+ request: MockModelRequest,+ modelFamily: ModelFamily,+): ActiveRegression | undefined {+ for (const message of request.userMessages) {+ if (!message.includes(`[model: ${modelFamily}]`)) continue;++ const regressionCase = regressionCaseFromText(message);+ if (regressionCase !== undefined) return { regressionCase };+ }++ return undefined;+}++function regressionCaseFromText(text: string): RegressionCase | undefined {+ if (text.includes("[case: redundant-tool-calls]")) return "redundant-tool-calls";+ if (text.includes("[case: stale-todo-work]")) return "stale-todo-work";+ return undefined;+}++function completionMarker(regressionCase: RegressionCase): string {+ return regressionCase === "redundant-tool-calls"+ ? "REPOSITORY_INSPECTION_COMPLETE"+ : "SOURCE_ANALYSIS_COMPLETE";+}++function checkpointContains(messages: MockModelRequest["messages"], marker: string): boolean {+ return messages.some((message, index) => {+ if (message.role !== "user" || message.text !== "Summary of our conversation so far:") {+ return false;+ }++ const checkpoint = messages[index + 1];+ return checkpoint?.role === "assistant" && checkpoint.text.includes(marker);+ });+}diff --git a/e2e/fixtures/compaction-regression-shared/src/constants.ts b/e2e/fixtures/compaction-regression-shared/src/constants.tsnew file mode 100644index 000000000..9773bae5d--- /dev/null+++ b/e2e/fixtures/compaction-regression-shared/src/constants.ts@@ -0,0 +1,2 @@+/** Fixture evidence emitted by the test-only second-compaction trigger tool. */+export const SECOND_CHECKPOINT_MARKER = "SECOND_CHECKPOINT_READY";diff --git a/e2e/fixtures/compaction-regression-shared/src/evals.ts b/e2e/fixtures/compaction-regression-shared/src/evals.tsnew file mode 100644index 000000000..a2d670472--- /dev/null+++ b/e2e/fixtures/compaction-regression-shared/src/evals.ts@@ -0,0 +1,62 @@+import type { EveEvalContext } from "eve/evals";++import type { ModelFamily } from "./agent";+import { SECOND_CHECKPOINT_MARKER } from "./constants";++export async function testRedundantToolCalls(+ t: EveEvalContext,+ modelFamily: ModelFamily,+): Promise<void> {+ const turn = await t.send(+ [+ `[model: ${modelFamily}]`,+ "[case: redundant-tool-calls]",+ "Call inspect-repository exactly once with scope repository.",+ "After it succeeds, report REPOSITORY_INSPECTION_COMPLETE and call no more tools.",+ ].join("\n"),+ );++ turn.expectOk();+ t.succeeded();+ t.calledTool("inspect-repository", {+ count: 1,+ input: { scope: "repository" },+ output: { completed: true, completionMarker: "REPOSITORY_INSPECTION_COMPLETE" },+ });+ t.calledTool("advance-checkpoint", {+ count: 1,+ output: { checkpointMarker: SECOND_CHECKPOINT_MARKER, completed: true },+ });+ t.event("compaction.completed", { count: 2 });+ t.messageIncludes("REPOSITORY_INSPECTION_COMPLETE");+ t.messageIncludes(SECOND_CHECKPOINT_MARKER);+}++export async function testStaleTodoWork(+ t: EveEvalContext,+ modelFamily: ModelFamily,+): Promise<void> {+ const turn = await t.send(+ [+ `[model: ${modelFamily}]`,+ "[case: stale-todo-work]",+ "Call perform-source-analysis exactly once with approach initial.",+ "The tool deliberately leaves its completed work in a pending todo.",+ "After it succeeds, report SOURCE_ANALYSIS_COMPLETE and call no more tools.",+ ].join("\n"),+ );++ turn.expectOk();+ t.succeeded();+ t.calledTool("perform-source-analysis", {+ count: 1,+ output: { completed: true, workUnit: "source-analysis" },+ });+ t.calledTool("advance-checkpoint", {+ count: 1,+ output: { checkpointMarker: SECOND_CHECKPOINT_MARKER, completed: true },+ });+ t.event("compaction.completed", { count: 2 });+ t.messageIncludes("SOURCE_ANALYSIS_COMPLETE");+ t.messageIncludes(SECOND_CHECKPOINT_MARKER);+}diff --git a/e2e/fixtures/compaction-regression-shared/src/inspect-repository.ts b/e2e/fixtures/compaction-regression-shared/src/inspect-repository.tsnew file mode 100644index 000000000..61a1e072e--- /dev/null+++ b/e2e/fixtures/compaction-regression-shared/src/inspect-repository.ts@@ -0,0 +1,30 @@+import { defineState } from "eve/context";+import { defineTool } from "eve/tools";+import { z } from "zod";++const completionMarker = "REPOSITORY_INSPECTION_COMPLETE";+const invocationCount = defineState("compaction-regression.inspect-repository", () => 0);+const modelFamilySchema = z.enum(["gpt-5.6", "opus-4.8", "sonnet-5"]);++export default defineTool({+ description:+ "Compaction regression tool. Inspect the repository exactly once when the user requests the redundant-tool-calls case.",+ inputSchema: z.object({+ modelFamily: modelFamilySchema,+ scope: z.literal("repository"),+ }),+ async execute(input) {+ const attempt = invocationCount.get() + 1;+ invocationCount.update(() => attempt);++ return {+ modelFamily: input.modelFamily,+ completed: true,+ completionMarker,+ workUnit: "repository-inspection",+ hardStop: attempt >= 10,+ attempt,+ evidencePadding: "repository inspection evidence ".repeat(100),+ };+ },+});diff --git a/e2e/fixtures/compaction-regression-shared/src/perform-source-analysis.ts b/e2e/fixtures/compaction-regression-shared/src/perform-source-analysis.tsnew file mode 100644index 000000000..25848bbfe--- /dev/null+++ b/e2e/fixtures/compaction-regression-shared/src/perform-source-analysis.ts@@ -0,0 +1,38 @@+import { defineState } from "eve/context";+import { defineTool } from "eve/tools";+import { todo } from "eve/tools/defaults";+import { z } from "zod";++const completionMarker = "SOURCE_ANALYSIS_COMPLETE";+const invocationCount = defineState("compaction-regression.perform-source-analysis", () => 0);+const modelFamilySchema = z.enum(["gpt-5.6", "opus-4.8", "sonnet-5"]);++export default defineTool({+ description:+ "Compaction regression tool. Complete source analysis exactly once when the user requests the stale-todo-work case.",+ inputSchema: z.object({+ approach: z.string().min(1),+ modelFamily: modelFamilySchema,+ }),+ async execute(input, ctx) {+ const attempt = invocationCount.get() + 1;+ invocationCount.update(() => attempt);+ await todo.execute(+ {+ todos: [{ content: "Complete source analysis", priority: "high", status: "pending" }],+ },+ ctx,+ );++ return {+ modelFamily: input.modelFamily,+ completed: true,+ completionMarker,+ workUnit: "source-analysis",+ hardStop: attempt >= 10,+ attempt,+ approach: input.approach,+ evidencePadding: "source analysis evidence ".repeat(100),+ };+ },+});diff --git a/packages/eve/src/harness/compaction-prompt.test.ts b/packages/eve/src/harness/compaction-prompt.test.tsnew file mode 100644index 000000000..293c47af9--- /dev/null+++ b/packages/eve/src/harness/compaction-prompt.test.ts@@ -0,0 +1,61 @@+import type { ModelMessage } from "ai";+import { describe, expect, it } from "vitest";++import { COMPACTION_PROMPT_ENVELOPE, createCompactionPrompt } from "#harness/compaction-prompt.js";++describe("createCompactionPrompt", () => {+ it("preserves the previous checkpoint without applying transcript truncation", () => {+ const markerAfterTextLimit = "CRITICAL_STATE_AFTER_280_CHARACTERS";+ const previousCheckpoint = `${"completed work ".repeat(24)}${markerAfterTextLimit}`;++ const result = createCompactionPrompt({+ messages: [{ content: "New evidence", role: "user" }],+ previousCheckpoint,+ });++ expect(result.system).toBe(COMPACTION_PROMPT_ENVELOPE.system);+ expect(result.prompt).toContain(`<previous-checkpoint>\n${previousCheckpoint}`);+ expect(result.prompt).toContain(markerAfterTextLimit);+ });++ it("summarizes structured tool messages without dumping raw JSON", () => {+ const messages: ModelMessage[] = [+ {+ content: [+ {+ input: { query: "debug" },+ toolCallId: "call-1",+ toolName: "search",+ type: "tool-call",+ },+ ],+ role: "assistant",+ },+ {+ content: [+ {+ output: {+ type: "json",+ value: ["alpha", "beta", "gamma", "delta"],+ },+ toolCallId: "call-1",+ toolName: "search",+ type: "tool-result",+ },+ ],+ role: "tool",+ },+ ];++ const result = createCompactionPrompt({ messages, previousCheckpoint: undefined });++ expect(result.prompt).toContain("Conversation transcript:");+ expect(result.prompt).toContain("### assistant");+ expect(result.prompt).toContain("Called search with object(query=debug)");+ expect(result.prompt).toContain(+ "Tool search returned object(type=json, value=array(4: alpha, beta, gamma, …))",+ );+ expect(result.prompt).not.toContain('{"query"');+ expect(result.prompt).not.toContain('{"items"');+ });+});diff --git a/packages/eve/src/harness/compaction-prompt.ts b/packages/eve/src/harness/compaction-prompt.tsnew file mode 100644index 000000000..9f3ce0773--- /dev/null+++ b/packages/eve/src/harness/compaction-prompt.ts@@ -0,0 +1,181 @@+import type { ModelMessage } from "ai";++export const COMPACTION_CHECKPOINT_MARKER = "Summary of our conversation so far:";++const COMPACTION_SYSTEM_PROMPT = `You are performing a CONTEXT CHECKPOINT COMPACTION. Create a handoff summary for another LLM that will resume the task.++Include:++- Current progress and key decisions made+- Important context, constraints, or user preferences+- What remains to be done, with clear next steps+- Any critical data, examples, or references needed to continue++Be concise, structured, and focused on helping the next LLM seamlessly continue the work. Write in the same language as the conversation. Do not continue the conversation, answer its questions, or invent facts. Only output the handoff summary.`;++const COMPACTION_CHECKPOINT_PROMPT = `Update the previous checkpoint with the newer information in the conversation. If there is no previous checkpoint, create one from the conversation.++Make completed work explicit so the next model does not repeat it. Keep completed work separate from current and remaining work, and do not describe completed work as pending unless later messages show it must be redone. Preserve exact file paths, function names, commands, error messages, identifiers, and measured values when they are needed to continue.`;++const COMPACTION_TEXT_LIMIT = 280;+const COMPACTION_COLLECTION_LIMIT = 3;++interface CompactionTranscriptMessage {+ readonly content: string;+ readonly role: ModelMessage["role"];+}++export interface CompactionPrompt {+ readonly prompt: string;+ readonly system: string;+}++/** Static prompt text added around checkpoint and conversation content. */+export const COMPACTION_PROMPT_ENVELOPE = {+ prompt: formatCompactionPrompt({ previousCheckpoint: "", transcript: "" }),+ system: COMPACTION_SYSTEM_PROMPT,+} satisfies CompactionPrompt;++/** Builds the compaction model input from framework-owned checkpoint state and older messages. */+export function createCompactionPrompt(input: {+ readonly messages: readonly ModelMessage[];+ readonly previousCheckpoint: string | undefined;+}): CompactionPrompt {+ const transcript = input.messages.map((message) => ({+ content: summarizeCompactionMessageContent(message),+ role: message.role,+ }));++ return {+ prompt: formatCompactionPrompt({+ previousCheckpoint: input.previousCheckpoint?.trim() ?? "(none)",+ transcript: formatCompactionTranscript(transcript),+ }),+ system: COMPACTION_SYSTEM_PROMPT,+ };+}++function formatCompactionPrompt(input: {+ readonly previousCheckpoint: string;+ readonly transcript: string;+}): string {+ return `<previous-checkpoint>+${input.previousCheckpoint}+</previous-checkpoint>++<conversation>+Conversation transcript:+${input.transcript}+</conversation>++${COMPACTION_CHECKPOINT_PROMPT}`;+}++function formatCompactionTranscript(messages: readonly CompactionTranscriptMessage[]): string {+ const sections = messages+ .filter((message) => message.content.trim().length > 0)+ .map((message) => `### ${message.role}\n${message.content.trim()}`);++ return sections.length === 0 ? "(empty)" : sections.join("\n\n");+}++function summarizeCompactionMessageContent(message: ModelMessage): string {+ if (typeof message.content === "string") {+ return summarizeText(message.content);+ }++ return message.content+ .map((part) => summarizeCompactionContentPart(part))+ .filter((summary) => summary.length > 0)+ .join("\n")+ .trim();+}++type ModelMessageContentPart = Exclude<ModelMessage["content"], string>[number];++function summarizeCompactionContentPart(part: ModelMessageContentPart): string {+ switch (part.type) {+ case "text":+ return summarizeText(part.text);+ case "reasoning":+ return "";+ case "file":+ return part.filename+ ? `Attached file ${part.filename} (${part.mediaType})`+ : `Attached file attachment (${part.mediaType})`;+ case "tool-call":+ return summarizeToolCallPart(part);+ case "tool-result":+ return summarizeToolResultPart(part);+ default:+ return "";+ }+}++function summarizeToolCallPart(part: { toolName: string; input?: unknown }): string {+ const input = part.input !== undefined ? summarizeCompactValue(part.input) : "";+ return input ? `Called ${part.toolName} with ${input}` : `Called ${part.toolName}`;+}++function summarizeToolResultPart(part: {+ toolName: string;+ output?: unknown;+ isError?: boolean;+}): string {+ const output = part.output !== undefined ? summarizeCompactValue(part.output) : "";+ const status = part.isError ? "errored" : "returned";+ return output ? `Tool ${part.toolName} ${status} ${output}` : `Tool ${part.toolName} ${status}`;+}++function summarizeCompactValue(value: unknown, depth = 0): string {+ if (value === null) return "null";+ if (value === undefined) return "";+ if (typeof value === "string") return summarizeText(value);+ if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {+ return String(value);+ }++ if (Array.isArray(value)) {+ if (value.length === 0) {+ return "array(0)";+ }++ if (depth >= 2) {+ return `array(${value.length})`;+ }++ const entries = value+ .slice(0, COMPACTION_COLLECTION_LIMIT)+ .map((item) => summarizeCompactValue(item, depth + 1));+ const suffix = value.length > COMPACTION_COLLECTION_LIMIT ? ", …" : "";+ return `array(${value.length}: ${entries.join(", ")}${suffix})`;+ }++ if (typeof value === "object") {+ const entries = Object.entries(value as Record<string, unknown>);+ if (entries.length === 0) {+ return "object(0)";+ }++ if (depth >= 2) {+ return `object(${entries.length} keys)`;+ }++ const rendered = entries+ .slice(0, COMPACTION_COLLECTION_LIMIT)+ .map(([key, nested]) => `${key}=${summarizeCompactValue(nested, depth + 1)}`);+ const suffix = entries.length > COMPACTION_COLLECTION_LIMIT ? ", …" : "";+ return `object(${rendered.join(", ")}${suffix})`;+ }++ return "";+}++function summarizeText(value: string): string {+ const normalized = value.replace(/\s+/g, " ").trim();+ if (normalized.length <= COMPACTION_TEXT_LIMIT) {+ return normalized;+ }++ return `${normalized.slice(0, COMPACTION_TEXT_LIMIT).trimEnd()}…`;+}diff --git a/packages/eve/src/harness/compaction.test.ts b/packages/eve/src/harness/compaction.test.tsindex e91916cde..abde90235 100644--- a/packages/eve/src/harness/compaction.test.ts+++ b/packages/eve/src/harness/compaction.test.ts@@ -1,6 +1,7 @@import type { ModelMessage } from "ai";import { afterEach, describe, expect, it, vi } from "vitest";+import { COMPACTION_PROMPT_ENVELOPE } from "#harness/compaction-prompt.js";import {compactMessages,estimateTokens,@@ -157,13 +158,45 @@ describe("getInputTokenCount", () => {describe("shouldCompact", () => {it("returns false when under threshold", () => {const messages: ModelMessage[] = [{ content: "short", role: "user" }];- expect(shouldCompact(messages, config)).toBe(false);+ expect(shouldCompact(messages, { ...config, threshold: 1_000 })).toBe(false);});it("returns true when over threshold", () => {const messages: ModelMessage[] = [{ content: "a".repeat(500), role: "user" }];expect(shouldCompact(messages, config)).toBe(true);});++ it("uses the fixed prompt envelope in threshold accounting", () => {+ const messages: ModelMessage[] = [{ content: "Continue the investigation.", role: "user" }];+ const compaction: CompactionConfig = {+ lastKnownInputTokens: 200,+ lastKnownPromptMessageCount: messages.length,+ recentWindowSize: 2,+ threshold: 1_000,+ };+ const activeInputTokens = getInputTokenCount(messages, compaction);+ const promptEnvelopeTokens = estimateTokens([+ { content: COMPACTION_PROMPT_ENVELOPE.system, role: "system" },+ { content: COMPACTION_PROMPT_ENVELOPE.prompt, role: "user" },+ ] satisfies ModelMessage[]);++ expect(+ shouldCompact(messages, {+ ...compaction,+ threshold: activeInputTokens + promptEnvelopeTokens,+ }),+ ).toBe(false);+ expect(+ shouldCompact(messages, {+ ...compaction,+ threshold: activeInputTokens + promptEnvelopeTokens - 1,+ }),+ ).toBe(true);+ });++ it("does not compact an empty history based on prompt overhead alone", () => {+ expect(shouldCompact([], { ...config, threshold: 0 })).toBe(false);+ });});describe("resolveCompactionModel", () => {@@ -220,6 +253,60 @@ describe("resolveCompactionModel", () => {});describe("compactMessages", () => {+ it("carries a prior checkpoint into the next compaction without truncating it", async () => {+ const { generateText } = await import("ai");+ const markerAfterTextLimit = "CRITICAL_STATE_AFTER_280_CHARACTERS";+ const previousCheckpoint = `${"completed work ".repeat(24)}${markerAfterTextLimit}`;++ vi.mocked(generateText).mockResolvedValue({+ text: "Updated checkpoint",+ } as Awaited<ReturnType<typeof generateText>>);++ const messages: ModelMessage[] = [+ { content: "Summary of our conversation so far:", role: "user" },+ { content: previousCheckpoint, role: "assistant" },+ { content: "new older context", role: "user" },+ { content: "new older response", role: "assistant" },+ { content: "recent question", role: "user" },+ { content: "recent answer", role: "assistant" },+ ];++ const model = {} as Parameters<typeof compactMessages>[1];+ await compactMessages(messages, model, {+ recentWindowSize: 2,+ threshold: 100_000,+ });++ const call = vi.mocked(generateText).mock.calls[0]?.[0];+ expect(call?.prompt).toContain(previousCheckpoint);+ expect(call?.prompt).toContain(markerAfterTextLimit);+ });++ it("replaces a prior checkpoint instead of retaining it in the recent window", async () => {+ const { generateText } = await import("ai");+ const previousCheckpoint = "Previous checkpoint";++ vi.mocked(generateText).mockResolvedValue({+ text: "Updated checkpoint",+ } as Awaited<ReturnType<typeof generateText>>);++ const messages: ModelMessage[] = [+ { content: "Summary of our conversation so far:", role: "user" },+ { content: previousCheckpoint, role: "assistant" },+ { content: "new evidence", role: "user" },+ { content: "latest response", role: "assistant" },+ ];++ const model = {} as Parameters<typeof compactMessages>[1];+ const result = await compactMessages(messages, model, {+ recentWindowSize: 10,+ threshold: 100_000,+ });++ expect(result.filter((message) => message.content === previousCheckpoint)).toHaveLength(0);+ expect(result.filter((message) => message.content === "Updated checkpoint")).toHaveLength(1);+ });+it("summarizes older messages and keeps recent window", async () => {const { generateText } = await import("ai");@@ -472,59 +559,6 @@ describe("compactMessages", () => {);});- it("summarizes structured tool messages without dumping raw JSON", async () => {- const { generateText } = await import("ai");-- vi.mocked(generateText).mockResolvedValue({- text: "Summary of prior context",- } as Awaited<ReturnType<typeof generateText>>);-- const messages: ModelMessage[] = [- { content: "old message 1", role: "user" },- {- content: [- {- input: { query: "debug" },- toolCallId: "call-1",- toolName: "search",- type: "tool-call",- },- ],- role: "assistant",- },- {- content: [- {- output: {- type: "json",- value: ["alpha", "beta", "gamma", "delta"],- },- toolCallId: "call-1",- toolName: "search",- type: "tool-result",- },- ],- role: "tool",- },- { content: "recent 1", role: "user" },- { content: "recent 2", role: "assistant" },- ];-- const model = {} as Parameters<typeof compactMessages>[1];- await compactMessages(messages, model, config);-- const call = vi.mocked(generateText).mock.calls[0]?.[0];- expect(call?.system).toContain("conversation summarizer");- expect(call?.prompt).toContain("Conversation transcript:");- expect(call?.prompt).toContain("### assistant");- expect(call?.prompt).toContain("Called search with object(query=debug)");- expect(call?.prompt).toContain(- "Tool search returned object(type=json, value=array(4: alpha, beta, gamma, …))",- );- expect(call?.prompt).not.toContain('{"query"');- expect(call?.prompt).not.toContain('{"items"');- });-it("appends synthetic user message when recent window trails with assistant", async () => {const { generateText } = await import("ai");diff --git a/packages/eve/src/harness/compaction.ts b/packages/eve/src/harness/compaction.tsindex 648a4884f..d37044a81 100644--- a/packages/eve/src/harness/compaction.ts+++ b/packages/eve/src/harness/compaction.ts@@ -1,25 +1,14 @@import { generateText, type LanguageModel, type ModelMessage, type TelemetryOptions } from "ai";+import {+ COMPACTION_CHECKPOINT_MARKER,+ COMPACTION_PROMPT_ENVELOPE,+ createCompactionPrompt,+} from "#harness/compaction-prompt.js";import type { RuntimeModelReference } from "#runtime/agent/bootstrap.js";import type { CompactionConfig, ToolLoopHarnessConfig } from "#harness/types.js";-const COMPACTION_SYSTEM_PROMPT = [- "You are a conversation summarizer.",- "Write a concise but useful summary for continuing the work.",- "Preserve the goal, important instructions, technical decisions, discoveries, open work, and relevant tool results.",- "Use the same language as the conversation.",- "Prefer short labeled sections such as Goal, Instructions, Discoveries, Accomplished, and Next steps when helpful.",- "Do not answer questions or invent facts.",-].join(" ");-const COMPACTION_SUMMARY_RESERVE_TOKENS = 2_048;-const COMPACTION_TEXT_LIMIT = 280;-const COMPACTION_COLLECTION_LIMIT = 3;--interface CompactionTranscriptMessage {- readonly content: string;- readonly role: ModelMessage["role"];-}/*** Element type of a non-string `ModelMessage.content` array.@@ -38,6 +27,11 @@ export function estimateTokens(value: unknown): number {return JSON.stringify(value).length / 4;}+const COMPACTION_PROMPT_OVERHEAD_TOKENS = estimateTokens([+ { content: COMPACTION_PROMPT_ENVELOPE.system, role: "system" },+ { content: COMPACTION_PROMPT_ENVELOPE.prompt, role: "user" },+] satisfies ModelMessage[]);+/*** Best available input-token count: the model-reported count from the last* step, plus a rough character-based estimate of whatever messages have been@@ -64,13 +58,17 @@ export function getInputTokenCount(}/**- * Returns true when the message history exceeds the compaction threshold.+ * Returns true when the message history and fixed compaction-prompt envelope+ * exceed the compaction threshold.*/export function shouldCompact(messages: readonly ModelMessage[],config: CompactionConfig,): boolean {- return getInputTokenCount(messages, config) > config.threshold;+ return (+ messages.length > 0 &&+ getInputTokenCount(messages, config) + COMPACTION_PROMPT_OVERHEAD_TOKENS > config.threshold+ );}/**@@ -113,26 +111,24 @@ export async function compactMessages(headers?: Record<string, string>,abortSignal?: AbortSignal,): Promise<ModelMessage[]> {- let keep = selectRecentWindowSize(messages, config);+ const { conversation, previousCheckpoint } = extractPreviousCheckpoint(messages);+ let keep = selectRecentWindowSize(conversation, config);while (true) {- const { older, recent } = splitMessagesForCompaction(messages, keep);- if (older.length === 0) {+ const { older, recent } = splitMessagesForCompaction(conversation, keep);+ if (older.length === 0 && previousCheckpoint === undefined) {return keepNonToolResultMessages(recent);}- const prunedOlder: CompactionTranscriptMessage[] = older.map((message) => ({- content: summarizeCompactionMessageContent(message),- role: message.role,- }));+ const summaryPrompt = createCompactionPrompt({ messages: older, previousCheckpoint });const result = await generateText({abortSignal,headers,model,- prompt: formatCompactionPrompt(prunedOlder),+ prompt: summaryPrompt.prompt,providerOptions,- system: COMPACTION_SYSTEM_PROMPT,+ system: summaryPrompt.system,telemetry: telemetry ? { ...telemetry, functionId: "eve.compaction" } : undefined,temperature: 0,});@@ -154,7 +150,7 @@ export async function compactMessages(: [];const compacted: ModelMessage[] = [- { content: "Summary of our conversation so far:", role: "user" },+ { content: COMPACTION_CHECKPOINT_MARKER, role: "user" },{ content: result.text, role: "assistant" },...keptTail,...trailingAssistantGuard,@@ -168,6 +164,26 @@ export async function compactMessages(}}+function extractPreviousCheckpoint(messages: readonly ModelMessage[]): {+ readonly conversation: ModelMessage[];+ readonly previousCheckpoint: string | undefined;+} {+ const marker = messages[0];+ const checkpoint = messages[1];+ if (+ marker?.role !== "user" ||+ marker.content !== COMPACTION_CHECKPOINT_MARKER ||+ checkpoint?.role !== "assistant"+ ) {+ return { conversation: [...messages], previousCheckpoint: undefined };+ }++ return {+ conversation: messages.slice(2),+ previousCheckpoint: assistantMessageText(checkpoint),+ };+}+/*** Returns the kept tail for a compacted history: recent messages with tool* activity removed. Tool-result messages are dropped, and assistant messages are@@ -269,114 +285,3 @@ function splitMessagesForCompaction(recent: messages.slice(-keep),};}--function formatCompactionPrompt(messages: readonly CompactionTranscriptMessage[]): string {- const sections = messages- .filter((message) => message.content.trim().length > 0)- .map((message) => `### ${message.role}\n${message.content.trim()}`);-- if (sections.length === 0) {- return "Summarize the conversation so far.";- }-- return ["Conversation transcript:", ...sections].join("\n\n");-}--function summarizeCompactionMessageContent(message: ModelMessage): string {- if (typeof message.content === "string") {- return summarizeText(message.content);- }-- return message.content- .map((part) => summarizeCompactionContentPart(part))- .filter((summary) => summary.length > 0)- .join("\n")- .trim();-}--function summarizeCompactionContentPart(part: ModelMessageContentPart): string {- switch (part.type) {- case "text":- return summarizeText(part.text);- case "reasoning":- return "";- case "file":- return part.filename- ? `Attached file ${part.filename} (${part.mediaType})`- : `Attached file attachment (${part.mediaType})`;- case "tool-call":- return summarizeToolCallPart(part);- case "tool-result":- return summarizeToolResultPart(part);- default:- return "";- }-}--function summarizeToolCallPart(part: { toolName: string; input?: unknown }): string {- const input = part.input !== undefined ? summarizeCompactValue(part.input) : "";- return input ? `Called ${part.toolName} with ${input}` : `Called ${part.toolName}`;-}--function summarizeToolResultPart(part: {- toolName: string;- output?: unknown;- isError?: boolean;-}): string {- const output = part.output !== undefined ? summarizeCompactValue(part.output) : "";- const status = part.isError ? "errored" : "returned";- return output ? `Tool ${part.toolName} ${status} ${output}` : `Tool ${part.toolName} ${status}`;-}--function summarizeCompactValue(value: unknown, depth = 0): string {- if (value === null) return "null";- if (value === undefined) return "";- if (typeof value === "string") return summarizeText(value);- if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {- return String(value);- }-- if (Array.isArray(value)) {- if (value.length === 0) {- return "array(0)";- }-- if (depth >= 2) {- return `array(${value.length})`;- }-- const entries = value- .slice(0, COMPACTION_COLLECTION_LIMIT)- .map((item) => summarizeCompactValue(item, depth + 1));- const suffix = value.length > COMPACTION_COLLECTION_LIMIT ? ", …" : "";- return `array(${value.length}: ${entries.join(", ")}${suffix})`;- }-- if (typeof value === "object") {- const entries = Object.entries(value as Record<string, unknown>);- if (entries.length === 0) {- return "object(0)";- }-- if (depth >= 2) {- return `object(${entries.length} keys)`;- }-- const rendered = entries- .slice(0, COMPACTION_COLLECTION_LIMIT)- .map(([key, nested]) => `${key}=${summarizeCompactValue(nested, depth + 1)}`);- const suffix = entries.length > COMPACTION_COLLECTION_LIMIT ? ", …" : "";- return `object(${rendered.join(", ")}${suffix})`;- }-- return "";-}--function summarizeText(value: string): string {- const normalized = value.replace(/\s+/g, " ").trim();- if (normalized.length <= COMPACTION_TEXT_LIMIT) {- return normalized;- }-- return `${normalized.slice(0, COMPACTION_TEXT_LIMIT).trimEnd()}…`;-}diff --git a/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts b/packages/eve/src/harness/tool-loop-compaction-accounting.test.tsindex ea63d0498..2b92774a2 100644--- a/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts+++ b/packages/eve/src/harness/tool-loop-compaction-accounting.test.ts@@ -206,7 +206,7 @@ describe("tool-loop structured compaction accounting", () => {createTestSession({compaction: {recentWindowSize: 10,- threshold: 101,+ threshold: 500,},}),{ message: "Compute something" },diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 541fa2df3..298710f9a 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -621,6 +621,99 @@ importers:specifier: 'catalog:'version: 7.0.2+ e2e/fixtures/agent-compaction-regressions-gpt-5-6:+ dependencies:+ '@eve/e2e-compaction-regression-shared':+ specifier: workspace:*+ version: link:../compaction-regression-shared+ '@opentelemetry/core':+ specifier: 2.6.1+ version: 2.6.1(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace-base':+ specifier: 2.6.1+ version: 2.6.1(@opentelemetry/[email protected])+ '@vercel/otel':+ specifier: 2.1.2+ version: 2.1.2(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))+ eve:+ specifier: workspace:*+ version: link:../../../packages/eve+ zod:+ specifier: 'catalog:'+ version: 4.4.3+ devDependencies:+ '@types/node':+ specifier: 'catalog:'+ version: 25.9.1+ typescript:+ specifier: 'catalog:'+ version: 7.0.2++ e2e/fixtures/agent-compaction-regressions-opus-4-8:+ dependencies:+ '@eve/e2e-compaction-regression-shared':+ specifier: workspace:*+ version: link:../compaction-regression-shared+ '@opentelemetry/core':+ specifier: 2.6.1+ version: 2.6.1(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace-base':+ specifier: 2.6.1+ version: 2.6.1(@opentelemetry/[email protected])+ '@vercel/otel':+ specifier: 2.1.2+ version: 2.1.2(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))+ eve:+ specifier: workspace:*+ version: link:../../../packages/eve+ zod:+ specifier: 'catalog:'+ version: 4.4.3+ devDependencies:+ '@types/node':+ specifier: 'catalog:'+ version: 25.9.1+ typescript:+ specifier: 'catalog:'+ version: 7.0.2++ e2e/fixtures/agent-compaction-regressions-sonnet-5:+ dependencies:+ '@eve/e2e-compaction-regression-shared':+ specifier: workspace:*+ version: link:../compaction-regression-shared+ '@opentelemetry/core':+ specifier: 2.6.1+ version: 2.6.1(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace-base':+ specifier: 2.6.1+ version: 2.6.1(@opentelemetry/[email protected])+ '@vercel/otel':+ specifier: 2.1.2+ version: 2.1.2(@opentelemetry/[email protected])(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))+ eve:+ specifier: workspace:*+ version: link:../../../packages/eve+ zod:+ specifier: 'catalog:'+ version: 4.4.3+ devDependencies:+ '@types/node':+ specifier: 'catalog:'+ version: 25.9.1+ typescript:+ specifier: 'catalog:'+ version: 7.0.2++ e2e/fixtures/compaction-regression-shared:+ dependencies:+ eve:+ specifier: workspace:*+ version: link:../../../packages/eve+ zod:+ specifier: 'catalog:'+ version: 4.4.3+e2e/fixtures/agent-model:dependencies:eve: