Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): only suppress standalone empty-delivery markers (#3160)

Ruib3ce5104b3eb
diff --git a/.changeset/standalone-empty-delivery.md b/.changeset/standalone-empty-delivery.md
new file mode 100644
index 000000000..945bb7b07
--- /dev/null
+++ b/.changeset/standalone-empty-delivery.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Only suppress a response when the empty-delivery marker is the entire response, apart from surrounding whitespace. Replies that quote or explain the marker are now delivered and retained in conversation history.
diff --git a/docs/concepts/sessions-runs-and-streaming.md b/docs/concepts/sessions-runs-and-streaming.md
index 75df34661..e3f851309 100644
--- a/docs/concepts/sessions-runs-and-streaming.md
+++ b/docs/concepts/sessions-runs-and-streaming.md
@@ -95,6 +95,8 @@ Note: consider the privacy, confidentiality, and user-experience implications fo
`message.completed` can fire more than once in a turn: the agent often emits interim assistant text before a tool call. To tell tool-call narration from a terminal reply, check `message.completed.data.finishReason`. `step.completed.data.finishReason` mirrors the step outcome, and usage lives on `step.completed`.
+When a task explicitly requires conditional delivery and there is nothing new to report, the agent can finish with exactly `<eve-empty-delivery/>`. eve emits `message.completed` with `message: null` for that intentional silence. The marker must be the entire response, apart from surrounding whitespace; the HTML-escaped form `&lt;eve-empty-delivery/&gt;` is also accepted. A response that quotes the marker in prose or code is delivered normally.
+
A delegated subagent publishes progress on its own child-session stream. The parent emits `subagent.called` with a `childSessionId`, which a client uses to attach. `subagent.completed` carries a working task receipt after admission; later updates and outcomes arrive as task-triggered `message.received` notifications.
`step.failed` and `turn.failed` carry `{ code, message, details? }` for the failed fragment or turn, and `session.failed` is the terminal session-level variant. `turn.cancelled` is not a failure: the cancelled turn ends without any failure event, `session.waiting` follows, and the session accepts the next message normally. Whatever the turn streamed before cancellation stays on the stream. Durable history keeps the accepted user input and previously settled work, but discards incomplete assistant output and unfinished tool state. When a turn requested an output schema, the finalized payload lands on `result.completed` as `data.result` before the turn boundary. `authorization.required` carries the sign-in challenge (`data.authorization` may include `url`, `userCode`, `expiresAt`, `instructions`), and `authorization.completed` carries `data.outcome` (`"authorized" | "declined" | "failed" | "timed-out"`).
diff --git a/e2e/fixtures/agent-background-tools/agent/agent.ts b/e2e/fixtures/agent-background-tools/agent/agent.ts
index ef0641fce..b4a74e2c5 100644
--- a/e2e/fixtures/agent-background-tools/agent/agent.ts
+++ b/e2e/fixtures/agent-background-tools/agent/agent.ts
@@ -10,6 +10,17 @@ const EMPTY_DELIVERY_SENTINEL = "<eve-empty-delivery/>";
function respond(request: MockModelRequest): MockModelResponse | string {
const message = [...request.userMessages].reverse().find((entry) => entry.trim() !== "") ?? "";
+ const examplePrefix = "Alice is documenting conditional delivery. Return exactly this example:\n";
+ if (message.startsWith(examplePrefix)) {
+ return message.slice(examplePrefix.length);
+ }
+ if (message === "Repeat your previous assistant response verbatim.") {
+ return (
+ [...request.messages].reverse().find((entry) => entry.role === "assistant")?.text ??
+ "No previous assistant response."
+ );
+ }
+
if (request.userMessages.some((entry) => entry.includes(SCHEDULED))) {
return respondScheduled(request);
}
diff --git a/e2e/fixtures/agent-background-tools/evals/empty-delivery-marker.eval.ts b/e2e/fixtures/agent-background-tools/evals/empty-delivery-marker.eval.ts
new file mode 100644
index 000000000..fe943ecaf
--- /dev/null
+++ b/e2e/fixtures/agent-background-tools/evals/empty-delivery-marker.eval.ts
@@ -0,0 +1,47 @@
+import { defineEval } from "eve/evals";
+import { equals } from "eve/evals/expect";
+
+export default defineEval({
+ description:
+ "Quoted delivery markers remain visible and in the next turn's context; standalone markers finish silently.",
+
+ async test(t) {
+ const examplePrefix =
+ "Alice is documenting conditional delivery. Return exactly this example:\n";
+
+ for (const marker of ["<eve-empty-delivery/>", "&lt;eve-empty-delivery/&gt;"]) {
+ const message = `The pending-task instruction requires \`${marker}\` and no other text.`;
+ const explanation = await t.send(`${examplePrefix}${message}`);
+ explanation.expectOk();
+ await t.require(explanation.message, equals(message));
+ explanation.event("message.completed", {
+ data: (data) => data.message === message,
+ count: 1,
+ });
+ explanation.notEvent("message.completed", {
+ data: (data) => data.message === null,
+ });
+
+ const recalled = await t.send("Repeat your previous assistant response verbatim.");
+ recalled.expectOk();
+ await t.require(recalled.message, equals(message));
+
+ const silent = await t.send(`${examplePrefix} \n${marker}\t `);
+ silent.expectOk();
+ await t.require(silent.message, equals(undefined));
+ silent.event("message.completed", {
+ data: (data) => data.message === null,
+ count: 1,
+ });
+ silent.notEvent("message.completed", {
+ data: (data) => data.message !== null,
+ });
+ }
+
+ const followUp = await t.send(`${examplePrefix}The documentation is ready.`);
+ followUp.expectOk();
+ await t.require(followUp.message, equals("The documentation is ready."));
+ t.noFailedActions();
+ t.succeeded();
+ },
+});
diff --git a/packages/eve/src/harness/emission.test.ts b/packages/eve/src/harness/emission.test.ts
index 662aef596..aca22ae4e 100644
--- a/packages/eve/src/harness/emission.test.ts
+++ b/packages/eve/src/harness/emission.test.ts
@@ -275,30 +275,31 @@ describe("emitStreamContent empty delivery", () => {
);
});
- it("skips delivery when the sentinel appears anywhere in the final message", async () => {
- const emit = createEmitStub();
+ it.each([EMPTY_DELIVERY_SENTINEL, "&lt;eve-empty-delivery/&gt;"])(
+ "delivers explanations that quote %s across streamed deltas",
+ async (sentinel) => {
+ const emit = createEmitStub();
+ const deltas = ["The pending-task instruction requires `", sentinel, "` and no other text."];
+ const message = deltas.join("");
- await emitStreamContent(
- emit,
- EMISSION_STATE,
- streamOf([
- {
- id: "text-1",
- text: `Internal preamble ${EMPTY_DELIVERY_SENTINEL} trailing text`,
- type: "text-delta",
- },
- { finishReason: "stop", type: "finish-step" },
- ] as TextStreamPart<ToolSet>[]),
- );
+ await emitStreamContent(
+ emit,
+ EMISSION_STATE,
+ streamOf([
+ ...deltas.map((text) => ({ id: "text-1", text, type: "text-delta" })),
+ { finishReason: "stop", type: "finish-step" },
+ ] as TextStreamPart<ToolSet>[]),
+ );
- const events = vi.mocked(emit).mock.calls.map(([event]) => event);
- expect(events.at(-1)).toEqual(
- expect.objectContaining({
- data: expect.objectContaining({ message: null }),
- type: "message.completed",
- }),
- );
- });
+ const events = vi.mocked(emit).mock.calls.map(([event]) => event);
+ expect(events.at(-1)).toEqual(
+ expect.objectContaining({
+ data: expect.objectContaining({ message }),
+ type: "message.completed",
+ }),
+ );
+ },
+ );
});
describe("emitStreamContent action requests", () => {
diff --git a/packages/eve/src/harness/tool-loop.test.ts b/packages/eve/src/harness/tool-loop.test.ts
index c3013355d..b0a168380 100644
--- a/packages/eve/src/harness/tool-loop.test.ts
+++ b/packages/eve/src/harness/tool-loop.test.ts
@@ -985,19 +985,20 @@ describe("createToolLoopHarness", () => {
["literal", EMPTY_DELIVERY_SENTINEL],
["HTML-escaped", "&lt;eve-empty-delivery/&gt;"],
])(
- "parks without delivery when a terminal response contains the %s sentinel",
+ "parks without delivery when a terminal response is only the %s sentinel",
async (_, sentinel) => {
+ const message = ` \n${sentinel}\t `;
setupMockAgent({
finishReason: "stop",
response: {
messages: [
{
- content: `internal ${sentinel} trailing`,
+ content: message,
role: "assistant",
},
],
},
- text: `internal ${sentinel} trailing`,
+ text: message,
toolCalls: [],
toolResults: [],
});
@@ -1019,6 +1020,43 @@ describe("createToolLoopHarness", () => {
},
);
+ it.each([EMPTY_DELIVERY_SENTINEL, "&lt;eve-empty-delivery/&gt;"])(
+ "delivers and persists explanations that quote %s",
+ async (sentinel) => {
+ const message = `The pending-task instruction requires \`${sentinel}\` and no other text.`;
+ setupMockAgent({
+ finishReason: "stop",
+ response: { messages: [{ content: message, role: "assistant" }] },
+ text: message,
+ toolCalls: [],
+ toolResults: [],
+ });
+
+ const { emit, events } = createEventCollector();
+ const runStep = createToolLoopHarness(createTestConfig("conversation", emit));
+ const result = await runStep(createTestSession(), { message: "Hi" });
+
+ expect(result.next).toBeNull();
+ expect(result.session.history).toEqual([
+ { content: "Hi", role: "user" },
+ { content: message, role: "assistant" },
+ ]);
+ expect(vi.mocked(ToolLoopAgent).mock.calls.length).toBe(1);
+ expect(events).toContainEqual(
+ expect.objectContaining({
+ data: expect.objectContaining({ message }),
+ type: "message.completed",
+ }),
+ );
+ expect(events).not.toContainEqual(
+ expect.objectContaining({
+ data: expect.objectContaining({ message: null }),
+ type: "message.completed",
+ }),
+ );
+ },
+ );
+
it("keeps executable tools directly available to the model", async () => {
setupMockAgent({
finishReason: "stop",
diff --git a/packages/eve/src/shared/empty-delivery.test.ts b/packages/eve/src/shared/empty-delivery.test.ts
index b8b4c78d1..852cdde1a 100644
--- a/packages/eve/src/shared/empty-delivery.test.ts
+++ b/packages/eve/src/shared/empty-delivery.test.ts
@@ -3,21 +3,28 @@ import { describe, expect, it } from "vitest";
import { EMPTY_DELIVERY_SENTINEL, hasEmptyDeliverySentinel } from "#shared/empty-delivery.js";
describe("hasEmptyDeliverySentinel", () => {
- it("recognizes the exact sentinel", () => {
- expect(hasEmptyDeliverySentinel(EMPTY_DELIVERY_SENTINEL)).toBe(true);
- });
-
- it("recognizes an HTML-escaped sentinel", () => {
- expect(hasEmptyDeliverySentinel("&lt;eve-empty-delivery/&gt;")).toBe(true);
- });
+ describe.each([EMPTY_DELIVERY_SENTINEL, "&lt;eve-empty-delivery/&gt;"])("%s", (sentinel) => {
+ it("recognizes a standalone sentinel with optional surrounding whitespace", () => {
+ expect(hasEmptyDeliverySentinel(sentinel)).toBe(true);
+ expect(hasEmptyDeliverySentinel(` \n${sentinel}\t `)).toBe(true);
+ });
- it("recognizes the sentinel anywhere in the response", () => {
- expect(hasEmptyDeliverySentinel(`before ${EMPTY_DELIVERY_SENTINEL} after`)).toBe(true);
+ it.each([
+ `before ${sentinel}`,
+ `${sentinel} after`,
+ `before ${sentinel} after`,
+ `\`${sentinel}\``,
+ `\`\`\`\n${sentinel}\n\`\`\``,
+ `${sentinel}\n${sentinel}`,
+ ])("preserves message content: %s", (message) => {
+ expect(hasEmptyDeliverySentinel(message)).toBe(false);
+ });
});
it("rejects absent, empty, and partial sentinels", () => {
expect(hasEmptyDeliverySentinel("<eve-empty-delivery>")).toBe(false);
expect(hasEmptyDeliverySentinel("")).toBe(false);
+ expect(hasEmptyDeliverySentinel(" \n\t")).toBe(false);
expect(hasEmptyDeliverySentinel(null)).toBe(false);
expect(hasEmptyDeliverySentinel(undefined)).toBe(false);
});
diff --git a/packages/eve/src/shared/empty-delivery.ts b/packages/eve/src/shared/empty-delivery.ts
index 95100b561..c452af763 100644
--- a/packages/eve/src/shared/empty-delivery.ts
+++ b/packages/eve/src/shared/empty-delivery.ts
@@ -4,8 +4,6 @@ const HTML_ESCAPED_EMPTY_DELIVERY_SENTINEL = "&lt;eve-empty-delivery/&gt;";
export const CONDITIONAL_DELIVERY_INSTRUCTION = `Conditional delivery\nOnly when the current task explicitly makes delivery conditional and there is nothing new to report, reply with exactly ${EMPTY_DELIVERY_SENTINEL} and no other text. This includes results already delivered or incorporated into an earlier response; do not send an acknowledgement of that redundancy. Do not use this marker for ordinary conversations, after input or approval responses, or to omit commentary from a response that still needs delivery. Never return an empty response; use the marker to intentionally deliver nothing.`;
export function hasEmptyDeliverySentinel(text: string | null | undefined): boolean {
- return (
- text?.includes(EMPTY_DELIVERY_SENTINEL) === true ||
- text?.includes(HTML_ESCAPED_EMPTY_DELIVERY_SENTINEL) === true
- );
+ const trimmed = text?.trim();
+ return trimmed === EMPTY_DELIVERY_SENTINEL || trimmed === HTML_ESCAPED_EMPTY_DELIVERY_SENTINEL;
}