Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): compaction - retain measured pressure when trimming history (#3094)

Andrew Barba0d19fa3783e4
diff --git a/.changeset/measured-compaction-pressure.md b/.changeset/measured-compaction-pressure.md
new file mode 100644
index 000000000..c26a60219
--- /dev/null
+++ b/.changeset/measured-compaction-pressure.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Fix repeated ineffective compaction when provider-reported input tokens exceed the character estimate. Tool-result trimming now accounts for measured context pressure and falls back to summarization when it cannot free enough space.
diff --git a/docs/concepts/default-harness.md b/docs/concepts/default-harness.md
index ce5bb6b6b..5e9adb9bb 100644
--- a/docs/concepts/default-harness.md
+++ b/docs/concepts/default-harness.md
@@ -18,6 +18,12 @@ export default defineAgent({
});
```
+Before summarizing, eve tries to shorten oversized tool results in older history.
+It checks whether that reduction is sufficient using the last provider-reported
+input token count plus an estimate of new messages. A smaller character estimate
+alone cannot satisfy compaction triggered by a higher provider count. If trimming
+cannot free enough space, eve summarizes the older history.
+
Compaction also preserves the framework's own tool state automatically. It resets read-before-write tracking (so a write afterward re-reads the file whose read evidence was summarized away) and re-injects the active todo list, so the model keeps its task list across the summary. There is no per-tool hook to configure.
First-class [memory](../memory) participates in a separate lifecycle. eve asks
diff --git a/e2e/fixtures/agent-compaction-measured/agent/agent.ts b/e2e/fixtures/agent-compaction-measured/agent/agent.ts
new file mode 100644
index 000000000..be503e49f
--- /dev/null
+++ b/e2e/fixtures/agent-compaction-measured/agent/agent.ts
@@ -0,0 +1,36 @@
+import { e2eAgentConfig } from "@eve-e2e/config";
+import { defineAgent } from "eve";
+import { mockModel } from "eve/evals";
+
+const model = mockModel({
+ modelId: "measured-compaction-model",
+ respond(request) {
+ if (
+ request.messages.some(
+ (entry) => entry.role === "system" && entry.text.includes("CONTEXT CHECKPOINT COMPACTION"),
+ )
+ ) {
+ return "Recorded the earlier evidence. Continue with the latest request.";
+ }
+ const message = request.lastUserMessage ?? "";
+ if (message.includes("verify")) {
+ return {
+ text: request.messages.some((entry) => entry.text === "Summary of our conversation so far:")
+ ? "MEASURED_COMPACTION_OK"
+ : "MEASURED_COMPACTION_MISSING",
+ usage: { inputTokens: 1, outputTokens: 1 },
+ };
+ }
+ return {
+ text: "Evidence recorded.",
+ usage: { inputTokens: message.includes("report high usage") ? 30_000 : 1, outputTokens: 1 },
+ };
+ },
+});
+
+export default defineAgent({
+ ...e2eAgentConfig(),
+ model,
+ modelContextWindowTokens: 100_000,
+ compaction: { thresholdPercent: 0.2 },
+});
diff --git a/e2e/fixtures/agent-compaction-measured/agent/instructions.md b/e2e/fixtures/agent-compaction-measured/agent/instructions.md
new file mode 100644
index 000000000..2a13b2f78
--- /dev/null
+++ b/e2e/fixtures/agent-compaction-measured/agent/instructions.md
@@ -0,0 +1 @@
+Record the supplied evidence and confirm the latest request.
diff --git a/e2e/fixtures/agent-compaction-measured/evals/evals.config.ts b/e2e/fixtures/agent-compaction-measured/evals/evals.config.ts
new file mode 100644
index 000000000..e2bfb2f36
--- /dev/null
+++ b/e2e/fixtures/agent-compaction-measured/evals/evals.config.ts
@@ -0,0 +1,3 @@
+import { defineEvalConfig } from "eve/evals";
+
+export default defineEvalConfig({ timeoutMs: 180_000 });
diff --git a/e2e/fixtures/agent-compaction-measured/evals/measured-tokens.eval.ts b/e2e/fixtures/agent-compaction-measured/evals/measured-tokens.eval.ts
new file mode 100644
index 000000000..001712ba2
--- /dev/null
+++ b/e2e/fixtures/agent-compaction-measured/evals/measured-tokens.eval.ts
@@ -0,0 +1,21 @@
+import { defineEval } from "eve/evals";
+
+const MEASURED_TOKENS_CASE = "[case: measured-tokens]";
+
+export default defineEval({
+ description:
+ "Provider-measured context pressure produces a summary even when the character estimate is small.",
+ async test(t) {
+ for (let index = 0; index < 6; index += 1) {
+ const seed = await t.send(`${MEASURED_TOKENS_CASE} record evidence ${index}.`);
+ seed.expectOk();
+ seed.notEvent("compaction.completed");
+ }
+ const measured = await t.send(`${MEASURED_TOKENS_CASE} report high usage.`);
+ measured.expectOk();
+ const compacted = await t.send(`${MEASURED_TOKENS_CASE} verify the checkpoint.`);
+ compacted.expectOk();
+ compacted.event("compaction.completed");
+ compacted.messageIncludes("MEASURED_COMPACTION_OK");
+ },
+});
diff --git a/e2e/fixtures/agent-compaction-measured/package.json b/e2e/fixtures/agent-compaction-measured/package.json
new file mode 100644
index 000000000..cc6973fd5
--- /dev/null
+++ b/e2e/fixtures/agent-compaction-measured/package.json
@@ -0,0 +1,22 @@
+{
+ "name": "agent-compaction-measured",
+ "version": "0.0.0",
+ "private": true,
+ "type": "module",
+ "scripts": {
+ "build": "eve build",
+ "dev": "eve dev",
+ "start": "eve start",
+ "typecheck": "eve build && tsc",
+ "test:e2e": "eve eval --strict"
+ },
+ "dependencies": {
+ "@eve-e2e/config": "workspace:*",
+ "@workflow/world-postgres": "catalog:",
+ "eve": "workspace:*"
+ },
+ "devDependencies": {
+ "@types/node": "catalog:",
+ "typescript": "catalog:"
+ }
+}
diff --git a/e2e/fixtures/agent-compaction-measured/tsconfig.json b/e2e/fixtures/agent-compaction-measured/tsconfig.json
new file mode 100644
index 000000000..85e3eceb8
--- /dev/null
+++ b/e2e/fixtures/agent-compaction-measured/tsconfig.json
@@ -0,0 +1,17 @@
+{
+ "compilerOptions": {
+ "target": "ES2022",
+ "module": "esnext",
+ "moduleResolution": "bundler",
+ "outDir": "dist",
+ "rootDir": ".",
+ "strict": true,
+ "esModuleInterop": true,
+ "skipLibCheck": true,
+ "forceConsistentCasingInFileNames": true,
+ "declaration": true,
+ "noEmit": true,
+ "types": ["node"]
+ },
+ "include": ["agent/**/*.ts", "evals/**/*.ts"]
+}
diff --git a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts
index bfdfcb2c9..1b56229ea 100644
--- a/e2e/fixtures/agent-compaction-regressions/agent/agent.ts
+++ b/e2e/fixtures/agent-compaction-regressions/agent/agent.ts
@@ -122,6 +122,9 @@ const taskModel = mockModel({
toolCallCounts.set(regressionCase, contentOutputCalls + 1);
return {
+ // This case isolates cap-only rewriting: the file supplies the
+ // pressure, while the measured-token fixture covers a large baseline.
+ usage: { inputTokens: 1 },
toolCalls: [
{
id: "emit-compaction-content-1",
diff --git a/packages/eve/src/harness/compaction.test.ts b/packages/eve/src/harness/compaction.test.ts
index 995ccbdb7..2939ef8a5 100644
--- a/packages/eve/src/harness/compaction.test.ts
+++ b/packages/eve/src/harness/compaction.test.ts
@@ -352,6 +352,8 @@ async function compact(
} as Awaited<ReturnType<typeof generateText>>);
const compactionConfig: CompactionConfig = {
+ lastKnownInputTokens: overrides.lastKnownInputTokens,
+ lastKnownPromptMessageCount: overrides.lastKnownPromptMessageCount,
recentWindowSize: overrides.recentWindowSize ?? 4,
threshold: overrides.threshold ?? ROOMY,
};
@@ -366,6 +368,73 @@ async function compact(
}
describe("compactMessages: tool-result cap heuristic", () => {
+ it.each([false, true])(
+ "summarizes provider-measured pressure when capping is insufficient (tool output: %s)",
+ async (withToolOutput) => {
+ const messages = [
+ ...checkpointHead("Previous investigation checkpoint."),
+ user("Dense multilingual context ".repeat(30)),
+ ...(withToolOutput
+ ? toolExchange({ callId: "call-0", payloadChars: 4_000 })
+ : [assistant("Evidence recorded.")]),
+ user("Continue the investigation."),
+ ];
+ const measuredConfig = {
+ lastKnownInputTokens: 30_000,
+ lastKnownPromptMessageCount: messages.length - 1,
+ recentWindowSize: 1,
+ threshold: 20_000,
+ };
+ expect(shouldCompact(messages, measuredConfig)).toBe(true);
+ expect(estimateTokens(messages) + ENVELOPE_TOKENS).toBeLessThan(measuredConfig.threshold);
+
+ const { result, summarizer } = await compact(messages, measuredConfig);
+
+ expect(summarizer).toHaveBeenCalledOnce();
+ expect(result[1]).toEqual(assistant("checkpoint text"));
+ expect(result.at(-1)).toEqual(messages.at(-1));
+ expect(result).toHaveLength(3);
+ expect(shouldCompact(result, { recentWindowSize: 1, threshold: 20_000 })).toBe(false);
+ },
+ );
+
+ it("accepts capping that frees enough space against the measured input count", async () => {
+ const messages = [
+ user("Investigate."),
+ ...toolExchange({ callId: "large", payloadChars: 100_000 }),
+ user("Continue."),
+ ];
+ const measuredConfig = {
+ lastKnownInputTokens: 30_000,
+ lastKnownPromptMessageCount: messages.length,
+ recentWindowSize: 1,
+ threshold: 20_000,
+ };
+ expect(shouldCompact(messages, measuredConfig)).toBe(true);
+ const { result, summarizer } = await compact(messages, measuredConfig);
+ expect(summarizer).not.toHaveBeenCalled();
+ expect(result).toHaveLength(messages.length);
+ expect(JSON.stringify(result)).toContain("Truncated by eve");
+ });
+
+ it.each([undefined, -1, 100])(
+ "ignores unusable prompt counts (%s) when evaluating capping",
+ async (lastKnownPromptMessageCount) => {
+ const messages = [
+ user("Investigate."),
+ ...toolExchange({ callId: "large", payloadChars: 4_000 }),
+ user("Continue."),
+ ];
+ const { summarizer } = await compact(messages, {
+ lastKnownInputTokens: 30_000,
+ lastKnownPromptMessageCount,
+ recentWindowSize: 1,
+ threshold: 20_000,
+ });
+ expect(summarizer).not.toHaveBeenCalled();
+ },
+ );
+
it("caps oversized older tool results in place without calling the summarizer", async () => {
const [call, resultMsg] = toolExchange({
callId: "call-0",
diff --git a/packages/eve/src/harness/compaction.ts b/packages/eve/src/harness/compaction.ts
index 29fa9bcca..fe83ebf06 100644
--- a/packages/eve/src/harness/compaction.ts
+++ b/packages/eve/src/harness/compaction.ts
@@ -102,6 +102,7 @@ interface CompactionHeuristicInput {
readonly older: readonly ModelMessage[];
readonly previousCheckpoint: string | undefined;
readonly recent: readonly ModelMessage[];
+ readonly tokenEstimateAdjustment: number;
}
/**
@@ -146,7 +147,12 @@ function toolResultCapHeuristic(input: CompactionHeuristicInput): CompactionHeur
// capping can be a near no-op when the older region holds few large
// results, and accepting one on a looser ruler would let shouldCompact
// re-fire every step without compaction ever making progress.
- const evaluation = evaluateThreshold(capped, input.config, "should-compact");
+ const evaluation = evaluateThreshold(
+ capped,
+ input.config,
+ "should-compact",
+ input.tokenEstimateAdjustment,
+ );
return evaluation.type === "within-limit"
? { messages: capped, type: "within-limit" }
: { type: "insufficient" };
@@ -162,9 +168,10 @@ function evaluateThreshold(
messages: readonly ModelMessage[],
config: CompactionConfig,
ruler: "estimate" | "should-compact",
+ tokenEstimateAdjustment = 0,
): { readonly estimatedTokens: number; readonly type: "over-limit" | "within-limit" } {
const overhead = ruler === "should-compact" ? COMPACTION_PROMPT_OVERHEAD_TOKENS : 0;
- const estimatedTokens = estimateTokens(messages) + overhead;
+ const estimatedTokens = estimateTokens(messages) + overhead + tokenEstimateAdjustment;
return {
estimatedTokens,
type: estimatedTokens <= config.threshold ? "within-limit" : "over-limit",
@@ -197,8 +204,22 @@ export async function compactMessages(
return keepNonToolResultMessages(recent);
}
+ // Capping preserves most of the measured prompt. Retain any known
+ // underestimate while crediting only the estimated tokens it removes.
+ // A new summary replaces that prompt, so it uses its own estimate below.
+ const tokenEstimateAdjustment = Math.max(
+ 0,
+ getInputTokenCount(messages, config) - estimateTokens(messages),
+ );
for (const heuristic of COMPACTION_HEURISTICS) {
- const outcome = heuristic({ config, conversation, older, previousCheckpoint, recent });
+ const outcome = heuristic({
+ config,
+ conversation,
+ older,
+ previousCheckpoint,
+ recent,
+ tokenEstimateAdjustment,
+ });
if (outcome.type === "within-limit") {
return outcome.messages;
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 2a771f5fa..dcabcb7d7 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -679,6 +679,25 @@ importers:
specifier: 'catalog:'
version: 7.0.2
+ e2e/fixtures/agent-compaction-measured:
+ dependencies:
+ '@eve-e2e/config':
+ specifier: workspace:*
+ version: link:../e2e-config
+ '@workflow/world-postgres':
+ specifier: 'catalog:'
+ eve:
+ specifier: workspace:*
+ version: link:../../../packages/eve
+ devDependencies:
+ '@types/node':
+ specifier: 'catalog:'
+ version: 24.13.3
+ typescript:
+ specifier: 'catalog:'
+ version: 7.0.2
+
e2e/fixtures/agent-compaction-regressions:
dependencies:
'@eve-e2e/config':