The Open Framework for Building Agents
fix(eve): ensure text approvals are processed properly for budget prompts (#4375)
Ben Pankow8df6c8365ef8
diff --git a/.changeset/budget-prompt-typed-approval-queued-message.md b/.changeset/budget-prompt-typed-approval-queued-message.mdnew file mode 100644index 000000000..509f96ed5--- /dev/null+++ b/.changeset/budget-prompt-typed-approval-queued-message.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Typing "approve" on a session budget prompt now resumes the session when you had already sent another message while the prompt was open. Before, eve joined your queued message and the approval into a single message that matched neither option, so the agent stayed blocked. The queued message now runs after the approval.diff --git a/packages/eve/src/harness/input-requests.ts b/packages/eve/src/harness/input-requests.tsindex b41f98b8a..be95bb3e6 100644--- a/packages/eve/src/harness/input-requests.ts+++ b/packages/eve/src/harness/input-requests.ts@@ -167,6 +167,20 @@ export function resolvePendingInput(input: {}}+/**+ * Answers an open budget prompt from new input's text before that input joins+ * a message deferred behind the prompt. Joined, the text no longer matches an+ * option, so the answer would queue behind the message it should release.+ */+export function answerSessionLimitFromText(+ session: HarnessSession,+ stepInput: StepInput | undefined,+): StepInput | undefined {+ if (stepInput === undefined || getDeferredStepInput(session) === undefined) return stepInput;+ const limitBatch = getPendingInputBatches(session.state).find(isSessionLimitInputBatch);+ return limitBatch === undefined ? stepInput : resolveTextMessageInput(limitBatch, stepInput);+}+type PendingInputRoute =| { readonly batch: PendingInputBatch; readonly kind: "session-limit" }| { readonly kind: "approvals" };diff --git a/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts b/packages/eve/src/harness/issue-3494-adversarial.integration.test.tsindex 2bd2cdae1..c7379e0ef 100644--- a/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts+++ b/packages/eve/src/harness/issue-3494-adversarial.integration.test.ts@@ -381,6 +381,23 @@ it("keeps a message waiting behind a budget prompt and runs it after the grant",).toEqual(["Say hello.", "Read the status.", "Also say goodbye."]);});+it("runs a message waiting behind a budget prompt after a typed approval", async () => {+ const f = fixture("message-behind-session-limit-text", false, 1);+ f.script.push("Initial text.");+ await f.drive({ message: "Say hello." });+ await f.drive({ message: "Read the status." });+ expect((await f.drive({ message: "Also say goodbye." })).held).toEqual({ kind: "request" });++ f.script.push(calls("read"), "FINAL");+ await f.drive({ message: "approve" });+ expect(f.executions).toEqual(["read"]);+ expect(+ f.events+ .filter((event) => event.type === "message.received")+ .map((event) => event.data.message),+ ).toEqual(["Say hello.", "Read the status.", "Also say goodbye."]);+});+it("reaches the next budget prompt after a grant without an older approval [control]", async () => {const f = fixture("control-session-limit", false, 1);f.script.push("Initial text.");diff --git a/packages/eve/src/harness/tool-loop.ts b/packages/eve/src/harness/tool-loop.tsindex 8d519b749..800a69911 100644--- a/packages/eve/src/harness/tool-loop.ts+++ b/packages/eve/src/harness/tool-loop.ts@@ -136,6 +136,7 @@ import {import { coordinateApprovalDelivery } from "#harness/approval-delivery-coordinator.js";import type { InstrumentationAttempt, InstrumentationStepScope } from "#instrumentation/runtime.js";import {+ answerSessionLimitFromText,consumeDeferredStepInput,getApprovedTools,getPendingInputRequestIds,@@ -650,7 +651,7 @@ export function createToolLoopHarness(config: ToolLoopHarnessConfig): StepFn {// replay only after they finish.const stepInput =pendingCoordination === undefined- ? consumeDeferredStepInput({ input, session })+ ? consumeDeferredStepInput({ input: answerSessionLimitFromText(session, input), session }): { input, session };session = stepInput.session;const coordination = await resolvePendingCoordination({diff --git a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md b/packages/eve/src/internal/testing/channel-conformance/MATRIX.mdindex e4fb3174f..a2ffb984c 100644--- a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md+++ b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md@@ -13,6 +13,8 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform✅ passes · ❌ broken · — not supported+## Questions+| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` || --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: || a rendered question shows every option a person can choose | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ |@@ -23,6 +25,11 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform| pressing an option of an answered question sends it to the agent as new input | —<sup>[3](#note-3)</sup> | —<sup>[4](#note-4)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || pressing options of two pending questions answers each with its own option | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |++## Tool approvals++| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: || a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ || pressing Approve runs the gated tool | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || pressing Approve twice runs the gated tool once | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |@@ -31,6 +38,11 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform| pressing Cancel stops the gated tool without running it | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || a text reply of approve runs the gated tool | —<sup>[5](#note-5)</sup> | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ || a text reply of cancel stops the gated tool without running it | —<sup>[5](#note-5)</sup> | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |++## Answered prompts++| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: || pressing an option clears the question's buttons | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || answering a question by text clears its buttons | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[6](#note-6)</sup> | ❌<sup>[6](#note-6)</sup> | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || pressing an option names who answered on the question | —<sup>[7](#note-7)</sup> | —<sup>[8](#note-8)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ❌<sup>[9](#note-9)</sup> | | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |@@ -39,20 +51,30 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform| approving by text clears the approval's buttons | —<sup>[5](#note-5)</sup> | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || pressing Approve names who approved on the approval | —<sup>[7](#note-7)</sup> | —<sup>[8](#note-8)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || approving by text names who approved on the approval | —<sup>[5](#note-5)</sup> | —<sup>[8](#note-8)</sup> | ❌<sup>[9](#note-9)</sup> | ❌<sup>[9](#note-9)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[10](#note-10)</sup> | ❌<sup>[10](#note-10)</sup> | ❌<sup>[9](#note-9)</sup> | ❌<sup>[9](#note-9)</sup> | ❌<sup>[9](#note-9)</sup> | ❌<sup>[9](#note-9)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |++## Budget prompts++| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: || running out of budget opens budget prompt | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ || pressing approve on budget prompt allows agent to continue | ✅ | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || reply of approve on budget prompt allows agent to continue | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ || pressing stop on budget prompt halts work, next message asks again | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> || reply of stop on budget prompt halts work, next message asks again | ❌<sup>[11](#note-11)</sup> | —<sup>[11](#note-11)</sup> | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| query sent during budget prompt answered after budget approval | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> | ❌<sup>[12](#note-12)</sup> |-| a sign-in names the service and shows its sign-in link | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ✅ | ✅ |-| a sign-in shows its confirmation code | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ✅ | ✅ |-| only the person signing in sees the sign-in link and code | —<sup>[16](#note-16)</sup> | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | ❌<sup>[17](#note-17)</sup> | —<sup>[16](#note-16)</sup> | ✅ | —<sup>[16](#note-16)</sup> | ✅ | ❌<sup>[18](#note-18)</sup> | —<sup>[16](#note-16)</sup> | —<sup>[16](#note-16)</sup> |-| a sign-in without a link shows its instructions | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |+| query sent during budget prompt answered after budget approval | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |++## Sign-ins++| Rule | `tui` | `web chat` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `photon` | `twilio` |+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |+| a sign-in names the service and shows its sign-in link | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |+| a sign-in shows its confirmation code | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |+| only the person signing in sees the sign-in link and code | —<sup>[15](#note-15)</sup> | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ❌<sup>[16](#note-16)</sup> | —<sup>[15](#note-15)</sup> | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ❌<sup>[17](#note-17)</sup> | —<sup>[15](#note-15)</sup> | —<sup>[15](#note-15)</sup> |+| a sign-in without a link shows its instructions | ✅ | ✅ | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ❌<sup>[12](#note-12)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ || after signing in, the agent carries on with the request | ✅ | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ | ✅ |-| completing a sign-in tells the person it succeeded | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |+| completing a sign-in tells the person it succeeded | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ || message after ignored sign-in gets an answer, signing in late doesn't run | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| message after ignored sign-in tells user it was cancelled | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ |+| message after ignored sign-in tells user it was cancelled | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ |## Notes@@ -67,10 +89,9 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform9. <a id="note-9"></a>a resolved prompt doesn't say who answered; input.resolved carries no responder10. <a id="note-10"></a>the card loses its buttons after a typed approval but doesn't say who approved11. <a id="note-11"></a>a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen-12. <a id="note-12"></a>eve coalesces the queued reply with the later approve, which then matches no option-13. <a id="note-13"></a>outside a DM the bot says to continue in a direct message but never sends one-14. <a id="note-14"></a>the channel has no default sign-in renderer-15. <a id="note-15"></a>the rule applies only where the conversation is shared or private, and this one is public-16. <a id="note-16"></a>the rule applies only where the conversation is public or shared, and this one is private-17. <a id="note-17"></a>the sign-in prompt, link included, is posted to the whole thread-18. <a id="note-18"></a>the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified+12. <a id="note-12"></a>outside a DM the bot says to continue in a direct message but never sends one+13. <a id="note-13"></a>the channel has no default sign-in renderer+14. <a id="note-14"></a>the rule applies only where the conversation is shared or private, and this one is public+15. <a id="note-15"></a>the rule applies only where the conversation is public or shared, and this one is private+16. <a id="note-16"></a>the sign-in prompt, link included, is posted to the whole thread+17. <a id="note-17"></a>the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverifieddiff --git a/packages/eve/src/internal/testing/channel-conformance/conformance.ts b/packages/eve/src/internal/testing/channel-conformance/conformance.tsindex 627b4676f..bd1e605dd 100644--- a/packages/eve/src/internal/testing/channel-conformance/conformance.ts+++ b/packages/eve/src/internal/testing/channel-conformance/conformance.ts@@ -4,6 +4,7 @@ import {type ContractRule,type HitlRule,hitlContract,+ hitlContractSections,} from "#internal/testing/channel-conformance/contract.js";import {type ChannelCapability,@@ -98,14 +99,6 @@ interface ConformanceChannel {readonly unsupported?: Partial<Record<HitlRule, string>>;}-/** A message sent during a budget prompt is queued, and every later message queues behind it. */-const QUEUED_BUDGET_REPLY = {- "query sent during budget prompt answered after budget approval": {- reason: "eve coalesces the queued reply with the later approve, which then matches no option",- symptom: /Timed out waiting for a reply to "Carol wants the review by Friday\."/u,- },-} satisfies Partial<Record<HitlRule, BrokenCell>>;-const SIGN_IN_NOT_SHOWN = /Timed out waiting for the bot to show/u;/** The channel has no default `authorization.required` renderer, so a sign-in shows nothing. */@@ -156,7 +149,6 @@ const CHAT_SDK_BROKEN = {"questionPress","questionText",]),- ...QUEUED_BUDGET_REPLY,};const DISCORD_BROKEN = {@@ -170,7 +162,6 @@ const DISCORD_BROKEN = {};const SLACK_BROKEN = {- ...QUEUED_BUDGET_REPLY,...staleAnsweredPrompts("only the button interaction handler edits a question; a typed answer leaves it",["questionText"],@@ -183,7 +174,6 @@ const SLACK_BROKEN = {const TEAMS_BROKEN = {...unnamedAnsweredPrompts(UNNAMED_RESPONDER, ["approvalText", "questionPress", "questionText"]),- ...QUEUED_BUDGET_REPLY,} satisfies Partial<Record<HitlRule, BrokenCell>>;/** The sign-in prompt, link included, goes to the whole thread. */@@ -201,7 +191,6 @@ const TELEGRAM_BROKEN = {"questionPress","questionText",]),- ...QUEUED_BUDGET_REPLY,};const TUI_TYPED_APPROVAL =@@ -225,7 +214,7 @@ const WEB_CHAT_SINGLE_PERSON =const hitlConformance = {"chat-sdk": [{ driver: chatSdkDriver, broken: { ...CHAT_SDK_BROKEN, ...SIGN_IN_ONLY_IN_DMS } },- { driver: chatSdkTextDriver, broken: QUEUED_BUDGET_REPLY },+ { driver: chatSdkTextDriver },],"chat-sdk-dm": [{ dm: true, driver: () => chatSdkDriver("private"), broken: CHAT_SDK_BROKEN }],discord: [{ driver: discordDriver, broken: DISCORD_BROKEN }],@@ -234,7 +223,6 @@ const hitlConformance = {{driver: githubDriver,broken: {- ...QUEUED_BUDGET_REPLY,...noSignInRenderer("a sign-in without a link shows its instructions","completing a sign-in tells the person it succeeded",@@ -247,7 +235,6 @@ const hitlConformance = {{driver: linearDriver,broken: {- ...QUEUED_BUDGET_REPLY,"only the person signing in sees the sign-in link and code": {reason:"the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified",@@ -256,9 +243,9 @@ const hitlConformance = {},},],- linq: [{ driver: linqDriver, broken: { ...QUEUED_BUDGET_REPLY, ...SIGN_IN_ONLY_IN_DMS } }],- "linq-dm": [{ dm: true, driver: () => linqDriver("private"), broken: QUEUED_BUDGET_REPLY }],- photon: [{ driver: photonDriver, broken: QUEUED_BUDGET_REPLY }],+ linq: [{ driver: linqDriver, broken: SIGN_IN_ONLY_IN_DMS }],+ "linq-dm": [{ dm: true, driver: () => linqDriver("private") }],+ photon: [{ driver: photonDriver }],slack: [{ driver: slackDriver, broken: SLACK_BROKEN }],"slack-dm": [{ dm: true, driver: () => slackDriver("private"), broken: SLACK_BROKEN }],teams: [{ driver: teamsDriver, broken: { ...TEAMS_BROKEN, ...SIGN_IN_LINK_POSTED_TO_THREAD } }],@@ -269,7 +256,6 @@ const hitlConformance = {{driver: tuiDriver,broken: {- ...QUEUED_BUDGET_REPLY,...budgetPromptNotShown("a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen","pressing stop on budget prompt halts work, next message asks again",@@ -292,14 +278,12 @@ const hitlConformance = {twilio: [{driver: twilioDriver,- broken: QUEUED_BUDGET_REPLY,},],"web-chat": [{driver: webChatDriver,broken: {- ...QUEUED_BUDGET_REPLY,...budgetPromptNotShown("a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen","pressing stop on budget prompt halts work, next message asks again",@@ -444,13 +428,21 @@ export function renderHitlConformanceMatrix(): string {const index = notes.get(note)!;return `${MATRIX_SYMBOLS[cell.kind]}<sup>[${index}](#note-${index})</sup>`;};- const rows = hitlContract.map((rule) =>- row([- // Non-breaking spaces keep each rule on one line; GitHub scrolls the table instead.- escape(rule.rule).replaceAll(" ", "\u00a0"),- ...entries.map((entry) => shown(entry, rule)),- ]),- );+ // One table per section, each repeating the column header so it reads on its own.+ const tables = hitlContractSections.flatMap((section) => [+ `## ${section.title}`,+ "",+ row(["Rule", ...entries.map((entry) => `\`${nameOf(entry)}\``)]),+ row(["---", ...entries.map(() => ":---:")]),+ ...section.rules.map((rule) =>+ row([+ // Non-breaking spaces keep each rule on one line; GitHub scrolls the table instead.+ escape(rule.rule).replaceAll(" ", "\u00a0"),+ ...entries.map((entry) => shown(entry, rule)),+ ]),+ ),+ "",+ ]);return ["# HITL conformance matrix","",@@ -467,10 +459,7 @@ export function renderHitlConformanceMatrix(): string {"","✅ passes · ❌ broken · — not supported","",- row(["Rule", ...entries.map((entry) => `\`${nameOf(entry)}\``)]),- row(["---", ...entries.map(() => ":---:")]),- ...rows,- "",+ ...tables,"## Notes","",...[...notes].map(([note, index]) => `${index}. <a id="note-${index}"></a>${note}`),diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.tsindex ccdeca027..a50093524 100644--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts@@ -238,7 +238,7 @@ async function abandonSignIn(conversation: ChannelConversation) {await conversation.say(CHANGE_OF_PLANS);}-export const hitlContract = [+const questionRules = [{rule: "a rendered question shows every option a person can choose",source: "docs/tools/human-in-the-loop.md#questions",@@ -340,6 +340,9 @@ export const hitlContract = [await conversation.waitForReplyTo("Saturday");},},+] as const satisfies readonly ContractRule[];++const approvalRules = [{rule: "a tool approval shows a choice to approve and one to cancel",source: "docs/tools/human-in-the-loop.md#approvals",@@ -447,6 +450,9 @@ export const hitlContract = [await expectNotDeployed(conversation);},},+] as const satisfies readonly ContractRule[];++const answeredPromptRules = [{rule: "pressing an option clears the question's buttons",source: "#1 (Slack's answered question card)",@@ -523,6 +529,9 @@ export const hitlContract = [await expectResponderNamed(conversation, APPROVAL_PROMPT);},},+] as const satisfies readonly ContractRule[];++const budgetRules = [{rule: "running out of budget opens budget prompt",source: "docs/agent-config.md#runtime-limits",@@ -603,6 +612,9 @@ export const hitlContract = [await conversation.waitForReplyTo(LATER_MESSAGE);},},+] as const satisfies readonly ContractRule[];++const signInRules = [{rule: "a sign-in names the service and shows its sign-in link",source: "docs/connections/overview.mdx#self-hosted-interactive-oauth",@@ -711,4 +723,19 @@ export const hitlContract = [},] as const satisfies readonly ContractRule[];-export type HitlRule = (typeof hitlContract)[number]["rule"];+/** The contract's rules, grouped by the kind of prompt they cover. */+export const hitlContractSections = [+ { title: "Questions", rules: questionRules },+ { title: "Tool approvals", rules: approvalRules },+ { title: "Answered prompts", rules: answeredPromptRules },+ { title: "Budget prompts", rules: budgetRules },+ { title: "Sign-ins", rules: signInRules },+] as const;++export const hitlContract = hitlContractSections.flatMap(+ (section): readonly HitlContractRule[] => section.rules,+);++type HitlContractRule = (typeof hitlContractSections)[number]["rules"][number];++export type HitlRule = HitlContractRule["rule"];