The Open Framework for Building Agents
fix(eve): keep the head and tail of tool results capped by compaction (#4773)
roboeve[bot]a3654ef7e8fc
diff --git a/.changeset/compaction-head-tail-cap.md b/.changeset/compaction-head-tail-cap.mdnew file mode 100644index 000000000..b8bdd9a1e--- /dev/null+++ b/.changeset/compaction-head-tail-cap.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Compaction now keeps the start and end of an oversized tool result instead of only its start, so failures, errors, and the `bash` exit code at the end of the output survive.diff --git a/packages/eve/src/harness/compaction/engine.test.ts b/packages/eve/src/harness/compaction/engine.test.tsindex 501f1bd8d..7c571d4a1 100644--- a/packages/eve/src/harness/compaction/engine.test.ts+++ b/packages/eve/src/harness/compaction/engine.test.ts@@ -619,27 +619,49 @@ describe("compactMessages: tool-result cap heuristic", () => {expect(second[2]).toEqual(capped);});- it("does not split a UTF-16 surrogate pair when capping tool results", async () => {- // JSON.stringify of this output places π₯'s high surrogate at index 1886,- // the last unit of the 1887-unit cut the annotation leaves of the- // 2000-unit TRANSCRIPT_PAYLOAD_LIMIT.- const content = `${"x".repeat(1851)}π₯${"y".repeat(500)}`;+ it("does not split a UTF-16 surrogate pair at either cut when capping tool results", async () => {+ // Astral characters across the whole cut window: both the head and the+ // tail cut fall inside surrogate pairs at one of the two offsets.+ for (const offset of [0, 1]) {+ const content = `${"x".repeat(offset)}${"π₯".repeat(2_000)}`;+ const { result } = await compact(+ [+ user("investigate"),+ ...toolResultExchange({ type: "json", value: { content } }),+ user("next"),+ ],+ { recentWindowSize: 1 },+ );+ const value = toolResultValue(result[2]);+ expect(value.startsWith("[Truncated by eve")).toBe(true);+ expect(value.length).toBeLessThanOrEqual(2_000);+ expect(value).toBe(value.toWellFormed());+ expect(value).toContain("chars omitted");+ expect(value.endsWith('π₯"}}')).toBe(true);+ }+ });++ it("keeps the failure and exit code of a large bash result when capping", async () => {+ const stdout = `${"β passing test\n".repeat(300)}FAIL src/auth.test.ts\nTest Files 1 failed`;const call: ModelMessage = {- content: [{ input: {}, toolCallId: "call-0", toolName: "grep", type: "tool-call" }],+ content: [{ input: {}, toolCallId: "call-0", toolName: "bash", type: "tool-call" }],role: "assistant",};const resultMsg: ModelMessage = {content: [{- output: { type: "json", value: { content } },+ output: {+ type: "json",+ value: { stderr: "", stdout, truncated: false, exitCode: 1, status: "completed" },+ },toolCallId: "call-0",- toolName: "grep",+ toolName: "bash",type: "tool-result",},],role: "tool",};- const messages = [user("investigate"), call, resultMsg, user("what did you find?")];+ const messages = [user("run the tests"), call, resultMsg, user("what failed?")];const { result, summarizer } = await compact(messages, { recentWindowSize: 1 });@@ -650,11 +672,11 @@ describe("compactMessages: tool-result cap heuristic", () => {typeof output === "object" && output !== null && "value" in output? String(output.value): "";- expect(value).toContain("Truncated by eve");- expect(value).toBe(value.toWellFormed());- expect(/[\uD800-\uDBFF](?![\uDC00-\uDFFF])/.test(value)).toBe(false);- expect(value).not.toContain("π₯");- expect(value).toContain("x".repeat(1851));+ expect(value.startsWith("[Truncated by eve")).toBe(true);+ expect(value).toContain("β passing test");+ expect(value).toContain("chars omitted");+ expect(value).toContain(String.raw`FAIL src/auth.test.ts\nTest Files 1 failed`);+ expect(value).toContain('"exitCode":1,"status":"completed"}');});it.each([@@ -684,8 +706,11 @@ describe("compactMessages: tool-result cap heuristic", () => {});it.each([- ["plain JSON", { content: `${"x".repeat(1_700)}KEPT-END-7781${"z".repeat(4_000)}` }],- ["escape-heavy JSON", { content: `${'q"\\'.repeat(300)}KEPT-END-7781${"z".repeat(4_000)}` }],+ ["plain JSON", { content: `KEPT-HEAD-7781${"z".repeat(4_000)}KEPT-TAIL-7781` }],+ [+ "escape-heavy JSON",+ { content: `KEPT-HEAD-7781${"z".repeat(4_000)}${'q"\\'.repeat(300)}KEPT-TAIL-7781` },+ ],])("keeps the whole capped %s content when a later compaction summarizes",async (_name, value) => {@@ -696,7 +721,9 @@ describe("compactMessages: tool-result cap heuristic", () => {];const { result: first } = await compact(history, { recentWindowSize: 1 });const cappedValue = toolResultValue(first[2]);- expect(cappedValue).toContain("KEPT-END-7781");+ expect(cappedValue).toContain("KEPT-HEAD-7781");+ expect(cappedValue.split("chars omitted").length - 1).toBe(1);+ expect(cappedValue.endsWith('KEPT-TAIL-7781"}}')).toBe(true);const summarize = summarizeWith("forced checkpoint");await compactMessages(first, { recentWindowSize: 1, threshold: ROOMY }, summarize, true);diff --git a/packages/eve/src/harness/compaction/engine.ts b/packages/eve/src/harness/compaction/engine.tsindex 13acb6f10..936f11ce0 100644--- a/packages/eve/src/harness/compaction/engine.ts+++ b/packages/eve/src/harness/compaction/engine.ts@@ -1,12 +1,12 @@import type { ModelMessage } from "ai";import {+ clipMiddleUtf16Safe,COMPACTION_CHECKPOINT_MARKER,COMPACTION_PROMPT_ENVELOPE,COMPACTION_RESUMPTION_MESSAGE,createCompactionPrompt,serializeToolResultOutput,- sliceUtf16Safe,stubContentOutputFileParts,TRANSCRIPT_PAYLOAD_LIMIT,} from "#harness/compaction/prompt.js";@@ -266,8 +266,8 @@ const CAPPED_RESULT_PREFIX = `${CAPPED_RESULT_ANNOTATION}\n\n`;* tool_use/tool_result pairing untouched. The cap matches the transcript* payload clip, so a later summarization sees the same content the model* kept β capping never destroys material the summarizer would need, unlike- * dropping results outright. The annotation leads the value so it survives- * prefix-capped renderings.+ * dropping results outright. The cap keeps the head and tail of the output,+ * where failures and exit codes usually are.** Capping is idempotent: the capped value, annotation included, fits the* same limit it was measured against, and a text output is measured by its@@ -302,7 +302,7 @@ function capToolResults(messages: readonly ModelMessage[]): ModelMessage[] {...part,output: {type: "text" as const,- value: `${CAPPED_RESULT_PREFIX}${sliceUtf16Safe(serialized, TRANSCRIPT_PAYLOAD_LIMIT - CAPPED_RESULT_PREFIX.length)}`,+ value: `${CAPPED_RESULT_PREFIX}${clipMiddleUtf16Safe(serialized, TRANSCRIPT_PAYLOAD_LIMIT - CAPPED_RESULT_PREFIX.length)}`,},};});diff --git a/packages/eve/src/harness/compaction/prompt.test.ts b/packages/eve/src/harness/compaction/prompt.test.tsindex 22794a746..8b085ba8d 100644--- a/packages/eve/src/harness/compaction/prompt.test.ts+++ b/packages/eve/src/harness/compaction/prompt.test.ts@@ -2,6 +2,7 @@ import type { ModelMessage } from "ai";import { describe, expect, it } from "vitest";import {+ clipMiddleUtf16Safe,COMPACTION_PROMPT_ENVELOPE,createCompactionPrompt,sliceUtf16Safe,@@ -87,6 +88,59 @@ describe("createCompactionPrompt", () => {expect(result.prompt).toContain("β¦");});+ it("keeps the end of an oversized tool result in the transcript", () => {+ const stdout = `${"β passing test\n".repeat(300)}FAIL src/auth.test.ts\nTest Files 1 failed`;+ const messages: ModelMessage[] = [+ {+ content: [+ {+ output: {+ type: "json",+ value: { stderr: "", stdout, exitCode: 1, status: "completed" },+ },+ toolCallId: "call-1",+ toolName: "bash",+ type: "tool-result",+ },+ ],+ role: "tool",+ },+ ];++ const { prompt } = createCompactionPrompt({ messages, previousCheckpoint: undefined });++ expect(prompt).toContain("β passing test");+ expect(prompt).toContain("chars omitted");+ expect(prompt).toContain('Test Files 1 failed","exitCode":1,"status":"completed"}');+ });++ it.each([+ ["text", { type: "text", value: `${"β passing test\n".repeat(300)}FAIL src/auth.test.ts` }],+ [+ "content",+ {+ type: "content",+ value: [{ text: `${"β passing test\n".repeat(300)}FAIL src/auth.test.ts`, type: "text" }],+ },+ ],+ ])(+ "keeps the head and tail of an oversized %s tool output in the transcript",+ (_name, output) => {+ const messages = [+ {+ content: [{ output, toolCallId: "call-1", toolName: "bash", type: "tool-result" }],+ role: "tool",+ },+ ] as ModelMessage[];++ const { prompt } = createCompactionPrompt({ messages, previousCheckpoint: undefined });++ expect(prompt).toContain("Tool bash returned β passing test");+ expect(prompt.split("chars omitted").length - 1).toBe(1);+ expect(prompt).toContain("β passing test FAIL src/auth.test.ts");+ },+ );+it("stubs content-output file parts instead of reproducing base64", () => {const base64 = "iVBORw0KGgo".repeat(500);const messages: ModelMessage[] = [@@ -221,3 +275,29 @@ describe("sliceUtf16Safe", () => {expect(sliced).toBe(sliced.toWellFormed());});});++describe("clipMiddleUtf16Safe", () => {+ it("returns values within the limit unchanged", () => {+ expect(clipMiddleUtf16Safe("short", TRANSCRIPT_PAYLOAD_LIMIT)).toBe("short");+ });++ it("keeps both ends and counts the omitted middle within the limit", () => {+ const value = `HEAD${"m".repeat(10_000)}TAIL`;+ const clipped = clipMiddleUtf16Safe(value, TRANSCRIPT_PAYLOAD_LIMIT);++ expect(clipped.length).toBeLessThanOrEqual(TRANSCRIPT_PAYLOAD_LIMIT);+ expect(clipped.startsWith("HEAD")).toBe(true);+ expect(clipped.endsWith("TAIL")).toBe(true);+ const omitted = Number(/\[(\d+) chars omitted\]/.exec(clipped)?.[1]);+ const marker = ` β¦ [${omitted} chars omitted] β¦ `;+ expect(clipped.length - marker.length + omitted).toBe(value.length);+ });++ it.each([0, 1])("never splits a surrogate pair at either cut (offset %s)", (offset) => {+ const value = "x".repeat(offset) + "π₯".repeat(2_000);+ const clipped = clipMiddleUtf16Safe(value, TRANSCRIPT_PAYLOAD_LIMIT);++ expect(clipped.length).toBeLessThanOrEqual(TRANSCRIPT_PAYLOAD_LIMIT);+ expect(clipped).toBe(clipped.toWellFormed());+ });+});diff --git a/packages/eve/src/harness/compaction/prompt.ts b/packages/eve/src/harness/compaction/prompt.tsindex 95659ec09..db0b6963c 100644--- a/packages/eve/src/harness/compaction/prompt.ts+++ b/packages/eve/src/harness/compaction/prompt.ts@@ -214,13 +214,13 @@ function renderTranscriptToolResult(function renderToolResultOutput(value: unknown, limit: number): string {const parts = contentToolOutputParts(value);if (parts === undefined) {- return capText(serializeToolResultOutput(value), limit);+ return capPayload(serializeToolResultOutput(value), limit);}const rendered = parts.map((part) => renderContentToolOutputPart(part, limit)).filter((entry) => entry.length > 0).join(" ");- return capText(rendered, limit);+ return capPayload(rendered, limit);}/**@@ -319,7 +319,7 @@ function renderToolCall(part: { toolName: string; input?: unknown }, limit: numbfunction renderPayload(value: unknown, limit: number): string {if (value === undefined) return "";- return capText(JSON.stringify(value) ?? "", limit);+ return capPayload(JSON.stringify(value) ?? "", limit);}// One summarized-attachment surface for the checkpoint model, whether the@@ -350,6 +350,35 @@ export function sliceUtf16Safe(value: string, limit: number): string {return value.slice(0, end);}+/**+ * Keeps the head and tail of an oversized tool payload, never exceeding+ * `limit`. Tool output often ends with what matters most: the failing test,+ * the error, or the `exitCode` and `status` that `bash` serializes last.+ */+export function clipMiddleUtf16Safe(value: string, limit: number): string {+ if (value.length <= limit) {+ return value;+ }+ // Sized for the largest possible count so the real marker always fits.+ const kept = limit - elisionMarker(value.length).length;+ const head = sliceUtf16Safe(value, Math.ceil(kept * 0.6));+ let tailStart = value.length - (kept - head.length);+ const first = value.charCodeAt(tailStart);+ if (first >= 0xdc00 && first <= 0xdfff) {+ tailStart += 1;+ }+ const tail = value.slice(tailStart);+ return `${head}${elisionMarker(tailStart - head.length)}${tail}`;+}++function elisionMarker(omitted: number): string {+ return ` β¦ [${omitted} chars omitted] β¦ `;+}++function capPayload(value: string, limit: number): string {+ return clipMiddleUtf16Safe(value.replace(/\s+/g, " ").trim(), limit);+}+function capText(value: string, limit: number): string {const normalized = value.replace(/\s+/g, " ").trim();if (normalized.length <= limit) {