Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): ensure text approvals are processed properly for budget prompts (#4375)

Ben Pankow8df6c8365ef8
diff --git a/.changeset/budget-prompt-typed-approval-queued-message.md b/.changeset/budget-prompt-typed-approval-queued-message.md
new file mode 100644
index 000000000..509f96ed5
--- /dev/null
+++ b/.changeset/budget-prompt-typed-approval-queued-message.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Typing "approve" on a session budget prompt now resumes the session when you had already sent another message while the prompt was open. Before, eve joined your queued message and the approval into a single message that matched neither option, so the agent stayed blocked. The queued message now runs after the approval.
diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.ts
index b41f98b8a..be95bb3e6 100644
--- a/packages/eve/src/harness/input-requests.ts
+++ b/packages/eve/src/harness/input-requests.ts
@@ -167,6 +167,20 @@ export function resolvePendingInput(input: {
}
}
+/**
+ * Answers an open budget prompt from new input's text before that input joins
+ * a message deferred behind the prompt. Joined, the text no longer matches an
+ * option, so the answer would queue behind the message it should release.
+ */
+export function answerSessionLimitFromText(
+ session: HarnessSession,
+ stepInput: StepInput | undefined,
+): StepInput | undefined {
+ if (stepInput === undefined || getDeferredStepInput(session) === undefined) return stepInput;
+ const limitBatch = getPendingInputBatches(session.state).find(isSessionLimitInputBatch);
+ return limitBatch === undefined ? stepInput : resolveTextMessageInput(limitBatch, stepInput);
+}
+
type PendingInputRoute =
| { readonly batch: PendingInputBatch; readonly kind: "session-limit" }
| { readonly kind: "approvals" };
diff --git a/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts b/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts
index 2bd2cdae1..c7379e0ef 100644
--- a/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts
+++ b/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts
@@ -381,6 +381,23 @@ it("keeps a message waiting behind a budget prompt and runs it after the grant",
).toEqual(["Say hello.", "Read the status.", "Also say goodbye."]);
});
+it("runs a message waiting behind a budget prompt after a typed approval", async () => {
+ const f = fixture("message-behind-session-limit-text", false, 1);
+ f.script.push("Initial text.");
+ await f.drive({ message: "Say hello." });
+ await f.drive({ message: "Read the status." });
+ expect((await f.drive({ message: "Also say goodbye." })).held).toEqual({ kind: "request" });
+
+ f.script.push(calls("read"), "FINAL");
+ await f.drive({ message: "approve" });
+ expect(f.executions).toEqual(["read"]);
+ expect(
+ f.events
+ .filter((event) => event.type === "message.received")
+ .map((event) => event.data.message),
+ ).toEqual(["Say hello.", "Read the status.", "Also say goodbye."]);
+});
+
it("reaches the next budget prompt after a grant without an older approval [control]", async () => {
const f = fixture("control-session-limit", false, 1);
f.script.push("Initial text.");
diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.ts
index 8d519b749..800a69911 100644
--- a/packages/eve/src/harness/tool-loop.ts
+++ b/packages/eve/src/harness/tool-loop.ts
@@ -136,6 +136,7 @@ import {
import { coordinateApprovalDelivery } from "#harness/approval-delivery-coordinator.js";
import type { InstrumentationAttempt, InstrumentationStepScope } from "#instrumentation/runtime.js";
import {
+ answerSessionLimitFromText,
consumeDeferredStepInput,
getApprovedTools,
getPendingInputRequestIds,
@@ -650,7 +651,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {
// replay only after they finish.
const stepInput =
pendingCoordination === undefined
- ? consumeDeferredStepInput({ input, session })
+ ? consumeDeferredStepInput({ input: answerSessionLimitFromText(session, input), session })
: { input, session };
session = stepInput.session;
const coordination = await resolvePendingCoordination({
diff --git a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
index e4fb3174f..a2ffb984c 100644
--- a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
+++ b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
@@ -13,6 +13,8 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform
✅ passes · ❌ broken · — not supported
+## Questions
+
| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |
| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| a rendered question shows every option a person can choose | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ |
@@ -23,6 +25,11 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform
| pressing an option of an answered question sends it to the agent as new input | —<sup>[3](#note-3)</sup> | —<sup>[4](#note-4)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| pressing options of two pending questions answers each with its own option | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+
+## Tool approvals
+
+| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |
+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ |
| pressing Approve runs the gated tool | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| pressing Approve twice runs the gated tool once | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
@@ -31,6 +38,11 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform
| pressing Cancel stops the gated tool without running it | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| a text reply of approve runs the gated tool | —<sup>[5](#note-5)</sup> | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| a text reply of cancel stops the gated tool without running it | —<sup>[5](#note-5)</sup> | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+
+## Answered prompts
+
+| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |
+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| pressing an option clears the question's buttons | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| answering a question by text clears its buttons | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[6](#note-6)</sup> | ❌<sup>[6](#note-6)</sup> | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| pressing an option names who answered on the question | —<sup>[7](#note-7)</sup> | —<sup>[8](#note-8)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ❌<sup>[9](#note-9)</sup> | | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
@@ -39,20 +51,30 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform
| approving by text clears the approval's buttons | —<sup>[5](#note-5)</sup> | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| pressing Approve names who approved on the approval | —<sup>[7](#note-7)</sup> | —<sup>[8](#note-8)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| approving by text names who approved on the approval | —<sup>[5](#note-5)</sup> | —<sup>[8](#note-8)</sup> | ❌<sup>[9](#note-9)</sup> | ❌<sup>[9](#note-9)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[10](#note-10)</sup> | ❌<sup>[10](#note-10)</sup> | ❌<sup>[9](#note-9)</sup> | ❌<sup>[9](#note-9)</sup> | ❌<sup>[9](#note-9)</sup> | ❌<sup>[9](#note-9)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+
+## Budget prompts
+
+| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |
+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
| running out of budget opens budget prompt | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ |
| pressing approve on budget prompt allows agent to continue | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| reply of approve on budget prompt allows agent to continue | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| pressing stop on budget prompt halts work, next message asks again | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
| reply of stop on budget prompt halts work, next message asks again | ❌<sup>[11](#note-11)</sup> | —<sup>[11](#note-11)</sup> | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| query sent during budget prompt answered after budget approval | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> |
-| a sign-in names the service and shows its sign-in link | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ✅ | ✅ |
-| a sign-in shows its confirmation code | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ✅ | ✅ |
-| only the person signing in sees the sign-in link and code | —<sup>[16](#note-16)</sup> | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | ❌<sup>[17](#note-17)</sup> | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | ✅ | ❌<sup>[18](#note-18)</sup> | —<sup>[16](#note-16)</sup> | —<sup>[16](#note-16)</sup> |
-| a sign-in without a link shows its instructions | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |
+| query sent during budget prompt answered after budget approval | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+
+## Sign-ins
+
+| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |
+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
+| a sign-in names the service and shows its sign-in link | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |
+| a sign-in shows its confirmation code | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |
+| only the person signing in sees the sign-in link and code | —<sup>[15](#note-15)</sup> | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ❌<sup>[16](#note-16)</sup> | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ❌<sup>[17](#note-17)</sup> | —<sup>[15](#note-15)</sup> | —<sup>[15](#note-15)</sup> |
+| a sign-in without a link shows its instructions | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ |
| after signing in, the agent carries on with the request | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ |
-| completing a sign-in tells the person it succeeded | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |
+| completing a sign-in tells the person it succeeded | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ |
| message after ignored sign-in gets an answer, signing in late doesn't run | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| message after ignored sign-in tells user it was cancelled | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |
+| message after ignored sign-in tells user it was cancelled | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ |
## Notes
@@ -67,10 +89,9 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform
9. <a id="note-9"></a>a resolved prompt doesn't say who answered; input.resolved carries no responder
10. <a id="note-10"></a>the card loses its buttons after a typed approval but doesn't say who approved
11. <a id="note-11"></a>a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen
-12. <a id="note-12"></a>eve coalesces the queued reply with the later approve, which then matches no option
-13. <a id="note-13"></a>outside a DM the bot says to continue in a direct message but never sends one
-14. <a id="note-14"></a>the channel has no default sign-in renderer
-15. <a id="note-15"></a>the rule applies only where the conversation is shared or private, and this one is public
-16. <a id="note-16"></a>the rule applies only where the conversation is public or shared, and this one is private
-17. <a id="note-17"></a>the sign-in prompt, link included, is posted to the whole thread
-18. <a id="note-18"></a>the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified
+12. <a id="note-12"></a>outside a DM the bot says to continue in a direct message but never sends one
+13. <a id="note-13"></a>the channel has no default sign-in renderer
+14. <a id="note-14"></a>the rule applies only where the conversation is shared or private, and this one is public
+15. <a id="note-15"></a>the rule applies only where the conversation is public or shared, and this one is private
+16. <a id="note-16"></a>the sign-in prompt, link included, is posted to the whole thread
+17. <a id="note-17"></a>the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified
diff --git a/packages/eve/src/internal/testing/channel-conformance/conformance.ts b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
index 627b4676f..bd1e605dd 100644
--- a/packages/eve/src/internal/testing/channel-conformance/conformance.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
@@ -4,6 +4,7 @@ import {
type ContractRule,
type HitlRule,
hitlContract,
+ hitlContractSections,
} from "#internal/testing/channel-conformance/contract.js";
import {
type ChannelCapability,
@@ -98,14 +99,6 @@ interface ConformanceChannel {
readonly unsupported?: Partial<Record<HitlRule, string>>;
}
-/** A message sent during a budget prompt is queued, and every later message queues behind it. */
-const QUEUED_BUDGET_REPLY = {
- "query sent during budget prompt answered after budget approval": {
- reason: "eve coalesces the queued reply with the later approve, which then matches no option",
- symptom: /Timed out waiting for a reply to "Carol wants the review by Friday\."/u,
- },
-} satisfies Partial<Record<HitlRule, BrokenCell>>;
-
const SIGN_IN_NOT_SHOWN = /Timed out waiting for the bot to show/u;
/** The channel has no default `authorization.required` renderer, so a sign-in shows nothing. */
@@ -156,7 +149,6 @@ const CHAT_SDK_BROKEN = {
"questionPress",
"questionText",
]),
- ...QUEUED_BUDGET_REPLY,
};
const DISCORD_BROKEN = {
@@ -170,7 +162,6 @@ const DISCORD_BROKEN = {
};
const SLACK_BROKEN = {
- ...QUEUED_BUDGET_REPLY,
...staleAnsweredPrompts(
"only the button interaction handler edits a question; a typed answer leaves it",
["questionText"],
@@ -183,7 +174,6 @@ const SLACK_BROKEN = {
const TEAMS_BROKEN = {
...unnamedAnsweredPrompts(UNNAMED_RESPONDER, ["approvalText", "questionPress", "questionText"]),
- ...QUEUED_BUDGET_REPLY,
} satisfies Partial<Record<HitlRule, BrokenCell>>;
/** The sign-in prompt, link included, goes to the whole thread. */
@@ -201,7 +191,6 @@ const TELEGRAM_BROKEN = {
"questionPress",
"questionText",
]),
- ...QUEUED_BUDGET_REPLY,
};
const TUI_TYPED_APPROVAL =
@@ -225,7 +214,7 @@ const WEB_CHAT_SINGLE_PERSON =
const hitlConformance = {
"chat-sdk": [
{ driver: chatSdkDriver, broken: { ...CHAT_SDK_BROKEN, ...SIGN_IN_ONLY_IN_DMS } },
- { driver: chatSdkTextDriver, broken: QUEUED_BUDGET_REPLY },
+ { driver: chatSdkTextDriver },
],
"chat-sdk-dm": [{ dm: true, driver: () => chatSdkDriver("private"), broken: CHAT_SDK_BROKEN }],
discord: [{ driver: discordDriver, broken: DISCORD_BROKEN }],
@@ -234,7 +223,6 @@ const hitlConformance = {
{
driver: githubDriver,
broken: {
- ...QUEUED_BUDGET_REPLY,
...noSignInRenderer(
"a sign-in without a link shows its instructions",
"completing a sign-in tells the person it succeeded",
@@ -247,7 +235,6 @@ const hitlConformance = {
{
driver: linearDriver,
broken: {
- ...QUEUED_BUDGET_REPLY,
"only the person signing in sees the sign-in link and code": {
reason:
"the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified",
@@ -256,9 +243,9 @@ const hitlConformance = {
},
},
],
- linq: [{ driver: linqDriver, broken: { ...QUEUED_BUDGET_REPLY, ...SIGN_IN_ONLY_IN_DMS } }],
- "linq-dm": [{ dm: true, driver: () => linqDriver("private"), broken: QUEUED_BUDGET_REPLY }],
- photon: [{ driver: photonDriver, broken: QUEUED_BUDGET_REPLY }],
+ linq: [{ driver: linqDriver, broken: SIGN_IN_ONLY_IN_DMS }],
+ "linq-dm": [{ dm: true, driver: () => linqDriver("private") }],
+ photon: [{ driver: photonDriver }],
slack: [{ driver: slackDriver, broken: SLACK_BROKEN }],
"slack-dm": [{ dm: true, driver: () => slackDriver("private"), broken: SLACK_BROKEN }],
teams: [{ driver: teamsDriver, broken: { ...TEAMS_BROKEN, ...SIGN_IN_LINK_POSTED_TO_THREAD } }],
@@ -269,7 +256,6 @@ const hitlConformance = {
{
driver: tuiDriver,
broken: {
- ...QUEUED_BUDGET_REPLY,
...budgetPromptNotShown(
"a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen",
"pressing stop on budget prompt halts work, next message asks again",
@@ -292,14 +278,12 @@ const hitlConformance = {
twilio: [
{
driver: twilioDriver,
- broken: QUEUED_BUDGET_REPLY,
},
],
"web-chat": [
{
driver: webChatDriver,
broken: {
- ...QUEUED_BUDGET_REPLY,
...budgetPromptNotShown(
"a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen",
"pressing stop on budget prompt halts work, next message asks again",
@@ -444,13 +428,21 @@ export function renderHitlConformanceMatrix(): string {
const index = notes.get(note)!;
return `${MATRIX_SYMBOLS[cell.kind]}<sup>[${index}](#note-${index})</sup>`;
};
- const rows = hitlContract.map((rule) =>
- row([
- // Non-breaking spaces keep each rule on one line; GitHub scrolls the table instead.
- escape(rule.rule).replaceAll(" ", "\u00a0"),
- ...entries.map((entry) => shown(entry, rule)),
- ]),
- );
+ // One table per section, each repeating the column header so it reads on its own.
+ const tables = hitlContractSections.flatMap((section) => [
+ `## ${section.title}`,
+ "",
+ row(["Rule", ...entries.map((entry) => `\`${nameOf(entry)}\``)]),
+ row(["---", ...entries.map(() => ":---:")]),
+ ...section.rules.map((rule) =>
+ row([
+ // Non-breaking spaces keep each rule on one line; GitHub scrolls the table instead.
+ escape(rule.rule).replaceAll(" ", "\u00a0"),
+ ...entries.map((entry) => shown(entry, rule)),
+ ]),
+ ),
+ "",
+ ]);
return [
"# HITL conformance matrix",
"",
@@ -467,10 +459,7 @@ export function renderHitlConformanceMatrix(): string {
"",
"✅ passes · ❌ broken · — not supported",
"",
- row(["Rule", ...entries.map((entry) => `\`${nameOf(entry)}\``)]),
- row(["---", ...entries.map(() => ":---:")]),
- ...rows,
- "",
+ ...tables,
"## Notes",
"",
...[...notes].map(([note, index]) => `${index}. <a id="note-${index}"></a>${note}`),
diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.ts
index ccdeca027..a50093524 100644
--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts
@@ -238,7 +238,7 @@ async function abandonSignIn(conversation: ChannelConversation) {
await conversation.say(CHANGE_OF_PLANS);
}
-export const hitlContract = [
+const questionRules = [
{
rule: "a rendered question shows every option a person can choose",
source: "docs/tools/human-in-the-loop.md#questions",
@@ -340,6 +340,9 @@ export const hitlContract = [
await conversation.waitForReplyTo("Saturday");
},
},
+] as const satisfies readonly ContractRule[];
+
+const approvalRules = [
{
rule: "a tool approval shows a choice to approve and one to cancel",
source: "docs/tools/human-in-the-loop.md#approvals",
@@ -447,6 +450,9 @@ export const hitlContract = [
await expectNotDeployed(conversation);
},
},
+] as const satisfies readonly ContractRule[];
+
+const answeredPromptRules = [
{
rule: "pressing an option clears the question's buttons",
source: "#1 (Slack's answered question card)",
@@ -523,6 +529,9 @@ export const hitlContract = [
await expectResponderNamed(conversation, APPROVAL_PROMPT);
},
},
+] as const satisfies readonly ContractRule[];
+
+const budgetRules = [
{
rule: "running out of budget opens budget prompt",
source: "docs/agent-config.md#runtime-limits",
@@ -603,6 +612,9 @@ export const hitlContract = [
await conversation.waitForReplyTo(LATER_MESSAGE);
},
},
+] as const satisfies readonly ContractRule[];
+
+const signInRules = [
{
rule: "a sign-in names the service and shows its sign-in link",
source: "docs/connections/overview.mdx#self-hosted-interactive-oauth",
@@ -711,4 +723,19 @@ export const hitlContract = [
},
] as const satisfies readonly ContractRule[];
-export type HitlRule = (typeof hitlContract)[number]["rule"];
+/** The contract's rules, grouped by the kind of prompt they cover. */
+export const hitlContractSections = [
+ { title: "Questions", rules: questionRules },
+ { title: "Tool approvals", rules: approvalRules },
+ { title: "Answered prompts", rules: answeredPromptRules },
+ { title: "Budget prompts", rules: budgetRules },
+ { title: "Sign-ins", rules: signInRules },
+] as const;
+
+export const hitlContract = hitlContractSections.flatMap(
+ (section): readonly HitlContractRule[] => section.rules,
+);
+
+type HitlContractRule = (typeof hitlContractSections)[number]["rules"][number];
+
+export type HitlRule = HitlContractRule["rule"];