The Open Framework for Building Agents
[6/7] test(eve): add budget and sign-in conformance rules (#4247)
Ben Pankow77874cd4c51c
diff --git a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md b/packages/eve/src/internal/testing/channel-conformance/MATRIX.mdindex d7d8299e6..9f9ddcd10 100644--- a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md+++ b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md@@ -11,143 +11,69 @@ recorded symptom. Regenerate it after changing either file:pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conformance/matrix -u```-✅ passes · ❌ broken · — not supported (the platform lacks a capability the rule-needs, or the client declines it below)+✅ passes · ❌ broken · — not supported-| Rule | `chat-sdk` | `chat-sdk-text` | `chat-sdk-dm` | `discord` | `discord-dm` | `github` | `linear` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `tui` | `twilio` |+| Rule | `tui` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `twilio` || --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |-| a rendered question shows every option a person can choose | ✅ | ✅ | — | ✅ | — | ✅ | ✅ | ✅ | — | ✅ | — | ✅ | — | ✅ | — | ✅ | ✅ |-| pressing a rendered option answers the pending question with that option | ✅ | — | ✅ | ✅ | ✅ | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — |-| a text reply matching an option answers the only pending question | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| a text reply that matches no option answers the question with the person's words | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| a text reply answers an open-ended question with the person's words | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| pressing an option of an answered question sends it to the agent as new input | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | — | — |-| pressing options of two pending questions answers each with its own option | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |-| a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | — | ✅ | — | ✅ | ✅ | ✅ | — | ✅ | — | ✅ | — | ✅ | — | ✅ | ✅ |-| pressing Approve runs the gated tool | ✅ | — | ✅ | ✅ | ✅ | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — |-| pressing Approve twice runs the gated tool once | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |-| pressing Approve on one of two pending approvals runs only that tool | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |-| answering an approval and a question pending together settles both | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |-| pressing Cancel stops the gated tool without running it | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |-| a text reply of approve runs the gated tool | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |-| a text reply of cancel stops the gated tool without running it | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |-| pressing an option clears the question's buttons | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |-| answering a question by text clears its buttons | ✅ | — | ✅ | — | — | — | — | — | — | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | — |-| pressing an option names who answered on the question | ❌ | — | — | ❌ | — | — | — | — | — | ✅ | — | ❌ | — | ❌ | — | — | — |-| answering a question by text names who answered on the question | ❌ | — | ❌ | — | — | — | — | — | — | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | — | — |-| pressing Approve clears the approval's buttons | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |-| approving by text clears the approval's buttons | ✅ | — | ✅ | — | — | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | — |-| pressing Approve names who approved on the approval | ❌ | — | — | ❌ | — | — | — | — | — | ✅ | — | ✅ | — | ❌ | — | — | — |-| approving by text names who approved on the approval | ❌ | — | ❌ | — | — | — | — | — | — | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | — | — |+| a rendered question shows every option a person can choose | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ |+| pressing a rendered option answers the pending question with that option | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| a text reply matching an option answers the only pending question | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| a text reply that matches no option answers the question with the person's words | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| a text reply answers an open-ended question with the person's words | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| pressing an option of an answered question sends it to the agent as new input | —<sup>[3](#note-3)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| pressing options of two pending questions answers each with its own option | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ |+| pressing Approve runs the gated tool | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| pressing Approve twice runs the gated tool once | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| pressing Approve on one of two pending approvals runs only that tool | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| answering an approval and a question pending together settles both | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| pressing Cancel stops the gated tool without running it | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| a text reply of approve runs the gated tool | —<sup>[4](#note-4)</sup> | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| a text reply of cancel stops the gated tool without running it | —<sup>[4](#note-4)</sup> | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| pressing an option clears the question's buttons | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| answering a question by text clears its buttons | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[5](#note-5)</sup> | ❌<sup>[5](#note-5)</sup> | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| pressing an option names who answered on the question | —<sup>[6](#note-6)</sup> | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ❌<sup>[7](#note-7)</sup> | | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| answering a question by text names who answered on the question | —<sup>[6](#note-6)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[5](#note-5)</sup> | ❌<sup>[5](#note-5)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| pressing Approve clears the approval's buttons | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| approving by text clears the approval's buttons | —<sup>[4](#note-4)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| pressing Approve names who approved on the approval | —<sup>[6](#note-6)</sup> | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| approving by text names who approved on the approval | —<sup>[4](#note-4)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[8](#note-8)</sup> | ❌<sup>[8](#note-8)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| running out of budget opens budget prompt | ✅ | ✅ | | ✅ | ❌<sup>[9](#note-9)</sup> | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ |+| pressing approve on budget prompt allows agent to continue | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| reply of approve on budget prompt allows agent to continue | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| pressing stop on budget prompt halts work, next message asks again | ❌<sup>[10](#note-10)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |+| reply of stop on budget prompt halts work, next message asks again | ❌<sup>[10](#note-10)</sup> | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| query sent during budget prompt answered after budget approval | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> |+| a sign-in names the service and shows its sign-in link | ❌<sup>[12](#note-12)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |+| a sign-in shows its confirmation code | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ❌<sup>[16](#note-16)</sup> | ❌<sup>[16](#note-16)</sup> | ✅ | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |+| only the person signing in sees the sign-in link and code | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | ❌<sup>[18](#note-18)</sup> | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | ✅ | ❌<sup>[19](#note-19)</sup> | —<sup>[17](#note-17)</sup> |+| a sign-in without a link shows its instructions | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ❌<sup>[20](#note-20)</sup> | ❌<sup>[20](#note-20)</sup> | ❌<sup>[21](#note-21)</sup> | ❌<sup>[21](#note-21)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |+| after signing in, the agent carries on with the request | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ |+| completing a sign-in tells the person it succeeded | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |+| message after ignored sign-in gets an answer, signing in late doesn't run | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| message after ignored sign-in tells user it was cancelled | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |-## Broken+## Notes-- **chat-sdk**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **chat-sdk**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **chat-sdk**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **chat-sdk**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **chat-sdk-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **chat-sdk-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **discord**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **discord**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **slack**, answering a question by text clears its buttons: only the button interaction handler edits a question; a typed answer leaves it-- **slack**, answering a question by text names who answered on the question: only the button interaction handler edits a question; a typed answer leaves it-- **slack**, approving by text names who approved on the approval: the card loses its buttons after a typed approval but doesn't say who approved-- **slack-dm**, answering a question by text clears its buttons: only the button interaction handler edits a question; a typed answer leaves it-- **slack-dm**, answering a question by text names who answered on the question: only the button interaction handler edits a question; a typed answer leaves it-- **slack-dm**, approving by text names who approved on the approval: the card loses its buttons after a typed approval but doesn't say who approved-- **teams**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **teams**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **teams**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **teams-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **teams-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **telegram**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **telegram**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **telegram**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **telegram**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **telegram-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder-- **telegram-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder--## Declined--- **chat-sdk-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **chat-sdk-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **discord-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **linq-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **slack-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **teams-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **telegram-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it-- **tui**, pressing an option of an answered question sends it to the agent as new input: an answered question's drawer closes, so nothing is left to press-- **tui**, a text reply of approve runs the gated tool: the approval drawer holds the keyboard; a person answers it with y or n-- **tui**, a text reply of cancel stops the gated tool without running it: the approval drawer holds the keyboard; a person answers it with y or n-- **tui**, pressing an option names who answered on the question: one person answers at their own terminal; there's nobody else to tell-- **tui**, answering a question by text names who answered on the question: one person answers at their own terminal; there's nobody else to tell-- **tui**, approving by text clears the approval's buttons: the approval drawer holds the keyboard; a person answers it with y or n-- **tui**, pressing Approve names who approved on the approval: one person answers at their own terminal; there's nobody else to tell-- **tui**, approving by text names who approved on the approval: the approval drawer holds the keyboard; a person answers it with y or n+1. <a id="note-1"></a>the platform has no buttons a person can press+2. <a id="note-2"></a>the platform has no plain-text replies+3. <a id="note-3"></a>an answered question's drawer closes, so nothing is left to press+4. <a id="note-4"></a>the approval drawer holds the keyboard; a person answers it with y or n+5. <a id="note-5"></a>only the button interaction handler edits a question; a typed answer leaves it+6. <a id="note-6"></a>one person answers at their own terminal; there's nobody else to tell+7. <a id="note-7"></a>a resolved prompt doesn't say who answered; input.resolved carries no responder+8. <a id="note-8"></a>the card loses its buttons after a typed approval but doesn't say who approved+9. <a id="note-9"></a>a budget prompt's request id overflows Discord's 100-character custom_id, so posting it throws+10. <a id="note-10"></a>a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen+11. <a id="note-11"></a>eve coalesces the queued reply with the later approve, which then matches no option+12. <a id="note-12"></a>the TUI labels a sign-in with the tool name, not the challenge's displayName+13. <a id="note-13"></a>outside a DM the bot says to continue in a direct message but never sends one+14. <a id="note-14"></a>the channel has no default sign-in renderer+15. <a id="note-15"></a>the rule applies only where the conversation is shared or private, and this one is public+16. <a id="note-16"></a>the Teams sign-in card omits the challenge's user code+17. <a id="note-17"></a>the rule applies only where the conversation is public or shared, and this one is private+18. <a id="note-18"></a>the sign-in prompt, link included, is posted to the whole thread+19. <a id="note-19"></a>the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified+20. <a id="note-20"></a>Slack sends the private sign-in prompt only for a challenge with a URL+21. <a id="note-21"></a>the Teams sign-in card omits the challenge's instructionsdiff --git a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.tsindex 43cade21f..0a3c7f0e3 100644--- a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts@@ -16,10 +16,18 @@ import {type PlatformCall,type Surface,numberedOptions,+ linkTargets,} from "#internal/testing/channel-conformance/harness.js";const ADAPTER = "conformance";const PERSON = { fullName: "Alice", isBot: false, isMe: false, userId: "alice", userName: "alice" };+/** The signed-in person a real integration would attach from its own user directory. */+const PERSON_AUTH = {+ attributes: {},+ authenticator: "conformance",+ principalId: "alice",+ principalType: "user",+};let nextThread = 0;interface CardNode {@@ -87,6 +95,7 @@ export function chatSdkDriver(surface: Exclude<Surface, "public"> = "shared"): Cconst card = cardOf(call.body as AdapterPostableMessage);return {id: messageIdOf(call),+ links: linkTargets(card),options: buttonsOf(call),text: card === undefined ? (driver.postedText(call) ?? "") : texts(card),};@@ -179,17 +188,17 @@ function chatSdkDriverWith(input: {});if (dm) {bridge.bot.onDirectMessage(async (thread, message) => {- await bridge.send(message.text, { auth: null, context: [], thread });+ await bridge.send(message.text, { auth: PERSON_AUTH, context: [], thread });});} else {// The wiring docs/channels/chat-sdk.mdx shows: a mention starts the session, and// subscribing lets the rest of the thread continue it without one.bridge.bot.onNewMention(async (thread, message) => {await thread.subscribe();- await bridge.send(message.text, { auth: null, context: [], thread });+ await bridge.send(message.text, { auth: PERSON_AUTH, context: [], thread });});bridge.bot.onSubscribedMessage(async (thread, message) => {- await bridge.send(message.text, { auth: null, context: [], thread });+ await bridge.send(message.text, { auth: PERSON_AUTH, context: [], thread });});}return bridge.channel;diff --git a/packages/eve/src/internal/testing/channel-conformance/conformance.ts b/packages/eve/src/internal/testing/channel-conformance/conformance.tsindex 8f598ce8d..d8a633141 100644--- a/packages/eve/src/internal/testing/channel-conformance/conformance.ts+++ b/packages/eve/src/internal/testing/channel-conformance/conformance.ts@@ -6,6 +6,7 @@ import {hitlContract,} from "#internal/testing/channel-conformance/contract.js";import {+ type ChannelCapability,type ChannelDriver,type ClientDriver,withChannelConversation,@@ -95,22 +96,88 @@ interface ConformanceChannel {readonly unsupported?: Partial<Record<HitlRule, string>>;}+/** A message sent during a budget prompt is queued, and every later message queues behind it. */+const QUEUED_BUDGET_REPLY = {+ "query sent during budget prompt answered after budget approval": {+ reason: "eve coalesces the queued reply with the later approve, which then matches no option",+ symptom: /Timed out waiting for a reply to "Carol wants the review by Friday\."/u,+ },+} satisfies Partial<Record<HitlRule, BrokenCell>>;++const SIGN_IN_NOT_SHOWN = /Timed out waiting for the bot to show/u;++/** The channel has no default `authorization.required` renderer, so a sign-in shows nothing. */+function noSignInRenderer(...rules: HitlRule[]): Partial<Record<HitlRule, BrokenCell>> {+ return Object.fromEntries(+ rules.map((rule) => [+ rule,+ { reason: "the channel has no default sign-in renderer", symptom: SIGN_IN_NOT_SHOWN },+ ]),+ );+}++/** Budget prompts in `rules` never show; see each caller for why. */+function budgetPromptNotShown(+ reason: string,+ ...rules: HitlRule[]+): Partial<Record<HitlRule, BrokenCell>> {+ return Object.fromEntries(+ rules.map((rule) => [+ rule,+ { reason, symptom: /Timed out waiting for one of the questions \["This session has hit/u },+ ]),+ );+}++/** Chat SDK's default sign-in, outside a DM, points the person at a DM it never sends. */+const SIGN_IN_ONLY_IN_DMS = Object.fromEntries(+ [+ "a sign-in names the service and shows its sign-in link",+ "a sign-in shows its confirmation code",+ "a sign-in without a link shows its instructions",+ ].map((rule) => [+ rule,+ {+ reason: "outside a DM the bot says to continue in a direct message but never sends one",+ symptom: SIGN_IN_NOT_SHOWN,+ },+ ]),+) satisfies Partial<Record<HitlRule, BrokenCell>>;+const UNNAMED_RESPONDER ="a resolved prompt doesn't say who answered; input.resolved carries no responder";-const CHAT_SDK_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [- "approvalPress",- "approvalText",- "questionPress",- "questionText",-]);+const CHAT_SDK_BROKEN = {+ ...unnamedAnsweredPrompts(UNNAMED_RESPONDER, [+ "approvalPress",+ "approvalText",+ "questionPress",+ "questionText",+ ]),+ ...QUEUED_BUDGET_REPLY,+};-const DISCORD_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [- "approvalPress",- "questionPress",-]);+const DISCORD_BROKEN = {+ ...unnamedAnsweredPrompts(UNNAMED_RESPONDER, ["approvalPress", "questionPress"]),+ ...budgetPromptNotShown(+ "a budget prompt's request id overflows Discord's 100-character custom_id, so posting it throws",+ "running out of budget opens budget prompt",+ "pressing approve on budget prompt allows agent to continue",+ ),+ ...noSignInRenderer(+ "a sign-in names the service and shows its sign-in link",+ "a sign-in shows its confirmation code",+ "a sign-in without a link shows its instructions",+ "completing a sign-in tells the person it succeeded",+ ),+};const SLACK_BROKEN = {+ ...QUEUED_BUDGET_REPLY,+ "a sign-in without a link shows its instructions": {+ reason: "Slack sends the private sign-in prompt only for a challenge with a URL",+ symptom: SIGN_IN_NOT_SHOWN,+ },...staleAnsweredPrompts("only the button interaction handler edits a question; a typed answer leaves it",["questionText"],@@ -121,18 +188,36 @@ const SLACK_BROKEN = {},} satisfies Partial<Record<HitlRule, BrokenCell>>;-const TEAMS_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [- "approvalText",- "questionPress",- "questionText",-]);+const TEAMS_BROKEN = {+ ...unnamedAnsweredPrompts(UNNAMED_RESPONDER, ["approvalText", "questionPress", "questionText"]),+ ...QUEUED_BUDGET_REPLY,+ "a sign-in shows its confirmation code": {+ reason: "the Teams sign-in card omits the challenge's user code",+ symptom: SIGN_IN_NOT_SHOWN,+ },+ "a sign-in without a link shows its instructions": {+ reason: "the Teams sign-in card omits the challenge's instructions",+ symptom: SIGN_IN_NOT_SHOWN,+ },+} satisfies Partial<Record<HitlRule, BrokenCell>>;-const TELEGRAM_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [- "approvalPress",- "approvalText",- "questionPress",- "questionText",-]);+/** The sign-in prompt, link included, goes to the whole thread. */+const SIGN_IN_LINK_POSTED_TO_THREAD = {+ "only the person signing in sees the sign-in link and code": {+ reason: "the sign-in prompt, link included, is posted to the whole thread",+ symptom: /a message everyone sees carried the sign-in (link|code)/u,+ },+} satisfies Partial<Record<HitlRule, BrokenCell>>;++const TELEGRAM_BROKEN = {+ ...unnamedAnsweredPrompts(UNNAMED_RESPONDER, [+ "approvalPress",+ "approvalText",+ "questionPress",+ "questionText",+ ]),+ ...QUEUED_BUDGET_REPLY,+};const TUI_TYPED_APPROVAL ="the approval drawer holds the keyboard; a person answers it with y or n";@@ -151,23 +236,62 @@ const TUI_SINGLE_PERSON = "one person answers at their own terminal; there's nob* failure (such as harness breakage) both turn it red.*/const hitlConformance = {- "chat-sdk": [{ driver: chatSdkDriver, broken: CHAT_SDK_BROKEN }, { driver: chatSdkTextDriver }],+ "chat-sdk": [+ { driver: chatSdkDriver, broken: { ...CHAT_SDK_BROKEN, ...SIGN_IN_ONLY_IN_DMS } },+ { driver: chatSdkTextDriver, broken: QUEUED_BUDGET_REPLY },+ ],"chat-sdk-dm": [{ dm: true, driver: () => chatSdkDriver("private"), broken: CHAT_SDK_BROKEN }],discord: [{ driver: discordDriver, broken: DISCORD_BROKEN }],"discord-dm": [{ dm: true, driver: () => discordDriver("private"), broken: DISCORD_BROKEN }],- github: [{ driver: githubDriver }],- linear: [{ driver: linearDriver }],- linq: [{ driver: linqDriver }],- "linq-dm": [{ dm: true, driver: () => linqDriver("private") }],+ github: [+ {+ driver: githubDriver,+ broken: {+ ...QUEUED_BUDGET_REPLY,+ ...noSignInRenderer(+ "a sign-in without a link shows its instructions",+ "completing a sign-in tells the person it succeeded",+ "message after ignored sign-in tells user it was cancelled",+ ),+ },+ },+ ],+ linear: [+ {+ driver: linearDriver,+ broken: {+ ...QUEUED_BUDGET_REPLY,+ "only the person signing in sees the sign-in link and code": {+ reason:+ "the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified",+ symptom: /a message everyone sees carried the sign-in (link|code)/u,+ },+ },+ },+ ],+ linq: [{ driver: linqDriver, broken: { ...QUEUED_BUDGET_REPLY, ...SIGN_IN_ONLY_IN_DMS } }],+ "linq-dm": [{ dm: true, driver: () => linqDriver("private"), broken: QUEUED_BUDGET_REPLY }],slack: [{ driver: slackDriver, broken: SLACK_BROKEN }],"slack-dm": [{ dm: true, driver: () => slackDriver("private"), broken: SLACK_BROKEN }],- teams: [{ driver: teamsDriver, broken: TEAMS_BROKEN }],+ teams: [{ driver: teamsDriver, broken: { ...TEAMS_BROKEN, ...SIGN_IN_LINK_POSTED_TO_THREAD } }],"teams-dm": [{ dm: true, driver: () => teamsDriver("private"), broken: TEAMS_BROKEN }],telegram: [{ driver: telegramDriver, broken: TELEGRAM_BROKEN }],"telegram-dm": [{ dm: true, driver: () => telegramDriver("private"), broken: TELEGRAM_BROKEN }],tui: [{driver: tuiDriver,+ broken: {+ ...QUEUED_BUDGET_REPLY,+ "a sign-in names the service and shows its sign-in link": {+ reason: "the TUI labels a sign-in with the tool name, not the challenge's displayName",+ symptom: /Timed out waiting for the bot to show Calendar/u,+ },+ ...budgetPromptNotShown(+ "a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen",+ "pressing stop on budget prompt halts work, next message asks again",+ "reply of stop on budget prompt halts work, next message asks again",+ ),+ },unsupported: {"pressing an option of an answered question sends it to the agent as new input":"an answered question's drawer closes, so nothing is left to press",@@ -181,7 +305,21 @@ const hitlConformance = {},},],- twilio: [{ driver: twilioDriver }],+ twilio: [+ {+ driver: twilioDriver,+ broken: {+ ...QUEUED_BUDGET_REPLY,+ ...noSignInRenderer(+ "a sign-in names the service and shows its sign-in link",+ "a sign-in shows its confirmation code",+ "a sign-in without a link shows its instructions",+ "completing a sign-in tells the person it succeeded",+ "message after ignored sign-in tells user it was cancelled",+ ),+ },+ },+ ],} satisfies Record<string, readonly ConformanceChannel[]>;/**@@ -193,20 +331,38 @@ const BROKEN_WAIT_TIMEOUT_MS = 3_000;/** How one channel or client stands on one rule. */type Cell =| { readonly kind: "pass" }- | { readonly kind: "unsupported"; readonly reason?: string }+ | { readonly kind: "unsupported"; readonly reason: string }| { readonly kind: "broken"; readonly broken: BrokenCell };+const CAPABILITY_NAMES: Record<ChannelCapability, string> = {+ buttons: "buttons a person can press",+ "text-replies": "plain-text replies",+};++function variesByConversation(rule: (typeof hitlContract)[number]): boolean {+ return (rule as ContractRule).variesByConversation === true;+}+function cellOf(entry: ConformanceChannel, rule: (typeof hitlContract)[number]): Cell {- if (entry.dm === true && (rule as ContractRule).variesByConversation !== true) {+ if (entry.dm === true && !variesByConversation(rule)) {return {kind: "unsupported",reason:"it doesn't vary between a shared thread and a DM, and the shared-thread column covers it",};}- const { capabilities } = entry.driver();- if (!rule.requires.every((capability) => capabilities.includes(capability))) {- return { kind: "unsupported" };+ const { capabilities, surface } = entry.driver();+ const missing = rule.requires.filter((capability) => !capabilities.includes(capability));+ if (missing.length > 0) {+ const names = missing.map((capability) => CAPABILITY_NAMES[capability]).join(" or ");+ return { kind: "unsupported", reason: `the platform has no ${names}` };+ }+ const { surfaces } = rule as ContractRule;+ if (surfaces !== undefined && !surfaces.includes(surface)) {+ return {+ kind: "unsupported",+ reason: `the rule applies only where the conversation is ${surfaces.join(" or ")}, and this one is ${surface}`,+ };}const declined = entry.unsupported?.[rule.rule];if (declined !== undefined) return { kind: "unsupported", reason: declined };@@ -226,11 +382,11 @@ export function describeHitlConformance(channel: keyof typeof hitlConformance):({ entry }) => {for (const rule of hitlContract) {const cell = cellOf(entry, rule);+ const { agent } = rule as ContractRule;const run = (options?: { readonly waitTimeoutMs: number }) =>- withChannelConversation(entry.driver(), (c) => rule.run(c), options);+ withChannelConversation(entry.driver(), (c) => rule.run(c), { ...agent, ...options });if (cell.kind === "unsupported") {- const why = cell.reason === undefined ? "" : `: ${cell.reason}`;- it.skip(`${rule.rule} (not supported${why})`, () => {});+ it.skip(`${rule.rule} (not supported: ${cell.reason})`, () => {});} else if (cell.kind === "pass") {it(rule.rule, () => run(), 60_000);} else {@@ -256,21 +412,46 @@ const MATRIX_SYMBOLS = { broken: "❌", pass: "✅", unsupported: "—" } as con* current whenever the suite passes.*/export function renderHitlConformanceMatrix(): string {- const entries = Object.values(hitlConformance).flatMap(+ const all = Object.values(hitlConformance).flatMap((group): readonly ConformanceChannel[] => group,);- const names = entries.map((entry) => entry.driver().name);+ const nameOf = (entry: ConformanceChannel) => entry.driver().name;+ const dmOf = (entry: ConformanceChannel) =>+ all.filter((dm) => dm.dm === true && nameOf(dm) === `${nameOf(entry)}-dm`);+ // The TUI, then Chat SDK's bridges, then channels with a DM column, then the+ // rest. Each DM column sits right after its channel's shared-thread column.+ const group = (entry: ConformanceChannel) => {+ if (nameOf(entry) === "tui") return 0;+ if (nameOf(entry).startsWith("chat-sdk")) return 1;+ return dmOf(entry).length > 0 ? 2 : 3;+ };+ const entries = all+ .filter((entry) => entry.dm !== true)+ .toSorted((a, b) => group(a) - group(b))+ .flatMap((entry) => [entry, ...dmOf(entry)]);const escape = (text: string) => text.replaceAll("|", "\\|");const row = (cells: readonly string[]) => `| ${cells.join(" | ")} |`;- const notes = (kind: "broken" | "unsupported") =>- entries.flatMap((entry, index) =>- hitlContract.flatMap((rule) => {- const cell = cellOf(entry, rule);- if (cell.kind !== kind) return [];- const reason = cell.kind === "broken" ? cell.broken.reason : cell.reason;- return reason === undefined ? [] : [`- **${names[index]}**, ${rule.rule}: ${reason}`];- }),- );+ // One note per distinct reason, numbered in reading order, so every cell+ // sharing a cause links to the same note. Plain anchors rather than Markdown+ // footnotes, which GitHub renders with links back up to every citing cell.+ const notes = new Map<string, number>();+ const shown = (entry: ConformanceChannel, rule: (typeof hitlContract)[number]) => {+ // A DM column leaves blank what only the shared-thread column runs.+ if (entry.dm === true && !variesByConversation(rule)) return "";+ const cell = cellOf(entry, rule);+ if (cell.kind === "pass") return MATRIX_SYMBOLS.pass;+ const note = cell.kind === "broken" ? cell.broken.reason : cell.reason;+ if (!notes.has(note)) notes.set(note, notes.size + 1);+ const index = notes.get(note)!;+ return `${MATRIX_SYMBOLS[cell.kind]}<sup>[${index}](#note-${index})</sup>`;+ };+ const rows = hitlContract.map((rule) =>+ row([+ // Non-breaking spaces keep each rule on one line; GitHub scrolls the table instead.+ escape(rule.rule).replaceAll(" ", "\u00a0"),+ ...entries.map((entry) => shown(entry, rule)),+ ]),+ );return ["# HITL conformance matrix","",@@ -285,26 +466,15 @@ export function renderHitlConformanceMatrix(): string {"pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conformance/matrix -u","```","",- "✅ passes · ❌ broken · — not supported (the platform lacks a capability the rule",- "needs, or the client declines it below)",- "",- row(["Rule", ...names.map((name) => `\`${name}\``)]),- row(["---", ...names.map(() => ":---:")]),- ...hitlContract.map((rule) =>- row([- // Non-breaking spaces keep each rule on one line; GitHub scrolls the table instead.- escape(rule.rule).replaceAll(" ", "\u00a0"),- ...entries.map((entry) => MATRIX_SYMBOLS[cellOf(entry, rule).kind]),- ]),- ),- "",- "## Broken",+ "✅ passes · ❌ broken · — not supported","",- ...notes("broken").map(escape),+ row(["Rule", ...entries.map((entry) => `\`${nameOf(entry)}\``)]),+ row(["---", ...entries.map(() => ":---:")]),+ ...rows,"",- "## Declined",+ "## Notes","",- ...notes("unsupported").map(escape),+ ...[...notes].map(([note, index]) => `${index}. <a id="note-${index}"></a>${note}`),"",].join("\n");}diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.tsindex 875c6b548..7778b12a5 100644--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts@@ -3,9 +3,13 @@ import { expect } from "vitest";import {type ChannelCapability,type ChannelConversation,+ type ConversationOptions,GATED_TOOL,+ PLAIN_TOOL,type RenderedOption,+ type Surface,SECOND_GATED_TOOL,+ SIGN_IN_TOOLS,TWO_QUESTIONS_TOOL,} from "#internal/testing/channel-conformance/harness.js";import {@@ -30,6 +34,10 @@ export interface ContractRule {* these; its shared-thread column covers the rest.*/readonly variesByConversation?: boolean;+ /** The surfaces the behavior is promised on; every surface when unset. */+ readonly surfaces?: readonly Surface[];+ /** How the test agent differs from the default for this rule. */+ readonly agent?: Pick<ConversationOptions, "limits">;run(conversation: ChannelConversation): Promise<void>;}@@ -155,6 +163,79 @@ function expectResponderNamed(conversation: ChannelConversation, prompt: string)).toBe(true);}+/**+ * A one-token input budget lets the model call that crosses it finish, so a+ * turn that calls a tool is held before the model reads the tool's result.+ */+const ONE_TOKEN_BUDGET = { limits: { maxInputTokensPerSession: 1 } } as const;+const BUDGET_PROMPT =+ "This session has hit the input-token limit (1) per session. This is a guardrail against " ++ "defective long-running sessions. If session activity looks fine, just approve to keep going.";+const LOOK_UP_NOTES = `Use ${PLAIN_TOOL} to find Bob's review notes.`;+const LATER_MESSAGE = "Carol wants the review by Friday.";++/** Starts a turn that spends the budget on its first model call and is held after its tool runs. */+async function exhaustBudget(conversation: ChannelConversation) {+ await conversation.say(LOOK_UP_NOTES);+ return await conversation.waitForQuestion(BUDGET_PROMPT);+}++/**+ * Once approved, the agent replies without redoing the work it did before the+ * prompt. A text answer's channel context (such as Telegram's message id)+ * reaches the model after the tool result, and the test model answers that+ * context instead of reporting the result, so this waits for any reply rather+ * than the tool's.+ */+async function expectWorkContinued(conversation: ChannelConversation) {+ await conversation.waitForReply();+ await conversation.waitForRest();+ expect(conversation.runsOf(PLAIN_TOOL), `${PLAIN_TOOL} ran again after approval`).toBe(1);+}++async function expectStoppedAndAskedAgain(conversation: ChannelConversation) {+ await conversation.say(LATER_MESSAGE);+ // Stopping keeps the session over budget, so the next message asks again.+ const options = await conversation.waitForQuestion(BUDGET_PROMPT);+ expect(options.map((option) => option.label).sort()).toEqual(["Approve", "Stop"]);+ expect(conversation.replyCount(), "the bot replied after Stop").toBe(0);+}++const CALENDAR = SIGN_IN_TOOLS.read_calendar;+const READ_CALENDAR = "Use read_calendar to check my week.";+const MAIL = SIGN_IN_TOOLS.read_mail;+const READ_MAIL = "Use read_mail to check my inbox.";+const CHANGE_OF_PLANS = "Alice decided to check her week later.";++async function expectSignInToolResult(conversation: ChannelConversation) {+ const output = await conversation.waitForToolResult("read_calendar");+ expect(output, `read_calendar returned ${JSON.stringify(output)}`).toEqual({ signedIn: true });+ expect(conversation.runsOf("read_calendar")).toBe(1);+}++/**+ * Asks for something that needs a sign-in, then opens the sign-in as a person+ * would. In a group the bot may first post a status with a button that sends+ * the sign-in to whoever asked, such as Telegram's Authorize.+ */+async function requestSignIn(conversation: ChannelConversation, message: string) {+ await conversation.say(message);+ await conversation.waitForSignIn();+ const open = conversation.shownOptions().find((option) => SIGN_IN_OPENERS.test(option.label));+ if (open !== undefined) await conversation.press(open);+}++const SIGN_IN_OPENERS = /^(?:authori[sz]e|sign in)$/iu;++/** Starts a calendar sign-in, then moves on with a new message before finishing it. */+async function abandonSignIn(conversation: ChannelConversation) {+ await conversation.say(READ_CALENDAR);+ await conversation.waitForSignIn();+ // A person moves on once they've seen the sign-in, which in a group they reply to.+ await conversation.waitForRest();+ await conversation.say(CHANGE_OF_PLANS);+}+export const hitlContract = [{rule: "a rendered question shows every option a person can choose",@@ -440,6 +521,192 @@ export const hitlContract = [expectResponderNamed(conversation, APPROVAL_PROMPT);},},+ {+ rule: "running out of budget opens budget prompt",+ source: "docs/agent-config.md#runtime-limits",+ requires: [],+ agent: ONE_TOKEN_BUDGET,+ async run(conversation) {+ const options = await exhaustBudget(conversation);+ const labels = options.map((option) => option.label).sort();+ expect(labels, `the budget prompt showed ${JSON.stringify(labels)}`).toEqual([+ "Approve",+ "Stop",+ ]);+ expect(conversation.replyCount(), "the agent replied before anyone approved").toBe(0);+ },+ },+ {+ rule: "pressing approve on budget prompt allows agent to continue",+ source: "docs/agent-config.md#runtime-limits",+ requires: ["buttons"],+ agent: ONE_TOKEN_BUDGET,+ async run(conversation) {+ const options = await exhaustBudget(conversation);+ const approve = options.find((option) => option.label === "Approve");+ expect(approve, "an Approve option to press").toBeDefined();+ await conversation.press(approve!);+ await expectWorkContinued(conversation);+ },+ },+ {+ rule: "reply of approve on budget prompt allows agent to continue",+ source: "docs/agent-config.md#runtime-limits",+ requires: ["text-replies"],+ variesByConversation: true,+ agent: ONE_TOKEN_BUDGET,+ async run(conversation) {+ await exhaustBudget(conversation);+ await conversation.say("approve");+ await expectWorkContinued(conversation);+ },+ },+ {+ rule: "pressing stop on budget prompt halts work, next message asks again",+ source: "docs/agent-config.md#runtime-limits",+ requires: ["buttons", "text-replies"],+ variesByConversation: true,+ agent: ONE_TOKEN_BUDGET,+ async run(conversation) {+ const options = await exhaustBudget(conversation);+ const stop = options.find((option) => option.label === "Stop");+ expect(stop, "a Stop option to press").toBeDefined();+ await conversation.press(stop!);+ await expectStoppedAndAskedAgain(conversation);+ },+ },+ {+ rule: "reply of stop on budget prompt halts work, next message asks again",+ source: "docs/agent-config.md#runtime-limits",+ requires: ["text-replies"],+ variesByConversation: true,+ agent: ONE_TOKEN_BUDGET,+ async run(conversation) {+ await exhaustBudget(conversation);+ await conversation.say("stop");+ await expectStoppedAndAskedAgain(conversation);+ },+ },+ {+ rule: "query sent during budget prompt answered after budget approval",+ source: "docs/agent-config.md#runtime-limits",+ requires: ["text-replies"],+ variesByConversation: true,+ agent: ONE_TOKEN_BUDGET,+ async run(conversation) {+ await exhaustBudget(conversation);+ await conversation.say(LATER_MESSAGE);+ // Had the message closed or replaced the prompt, approve would be held behind it.+ await conversation.say("approve");+ await conversation.waitForReplyTo(LATER_MESSAGE);+ },+ },+ {+ rule: "a sign-in names the service and shows its sign-in link",+ source: "docs/connections/overview.mdx#self-hosted-interactive-oauth",+ requires: [],+ variesByConversation: true,+ // A public surface has nowhere private to show a link.+ surfaces: ["shared", "private"],+ async run(conversation) {+ await requestSignIn(conversation, READ_CALENDAR);+ const shown = await conversation.waitForShown(CALENDAR.url);+ // Slack names the service in the shared status and links privately, so accept either.+ await conversation.waitForShown(CALENDAR.displayName);+ expect(shown, "the sign-in link was shown").toContain(CALENDAR.url);+ },+ },+ {+ rule: "a sign-in shows its confirmation code",+ source: "docs/connections/overview.mdx#self-hosted-interactive-oauth",+ requires: [],+ variesByConversation: true,+ surfaces: ["shared", "private"],+ async run(conversation) {+ await requestSignIn(conversation, READ_CALENDAR);+ await conversation.waitForShown(CALENDAR.userCode);+ },+ },+ {+ rule: "only the person signing in sees the sign-in link and code",+ source:+ "docs/channels/slack.mdx#render-and-decode-hitl-controls-yourself (a sign-in challenge is a credential)",+ requires: [],+ variesByConversation: true,+ surfaces: ["public", "shared"],+ async run(conversation) {+ await requestSignIn(conversation, READ_CALENDAR);+ await conversation.waitForRest();+ const shared = conversation.sharedText();+ expect(shared, "a message everyone sees carried the sign-in link").not.toContain(+ CALENDAR.url,+ );+ expect(shared, "a message everyone sees carried the sign-in code").not.toContain(+ CALENDAR.userCode,+ );+ },+ },+ {+ rule: "a sign-in without a link shows its instructions",+ source: "docs/connections/overview.mdx#self-hosted-interactive-oauth",+ requires: [],+ variesByConversation: true,+ async run(conversation) {+ await requestSignIn(conversation, READ_MAIL);+ await conversation.waitForShown(MAIL.instructions);+ },+ },+ {+ rule: "after signing in, the agent carries on with the request",+ source: "docs/connections/overview.mdx#interactive-oauth-via-vercel-connect",+ requires: [],+ async run(conversation) {+ await conversation.say(READ_CALENDAR);+ await conversation.waitForSignIn();+ expect(conversation.runsOf("read_calendar"), "read_calendar ran before sign-in").toBe(0);+ await conversation.completeSignIn();+ await expectSignInToolResult(conversation);+ },+ },+ {+ rule: "completing a sign-in tells the person it succeeded",+ source: "docs/connections/overview.mdx#interactive-oauth-via-vercel-connect",+ requires: [],+ variesByConversation: true,+ async run(conversation) {+ await conversation.say(READ_CALENDAR);+ await conversation.waitForSignIn();+ await conversation.completeSignIn();+ await conversation.waitForShown(/\b(?:authorized|complete|connected)\b/iu);+ },+ },+ {+ rule: "message after ignored sign-in gets an answer, signing in late doesn't run",+ source: "docs/connections/overview.mdx#interactive-oauth-via-vercel-connect",+ requires: ["text-replies"],+ variesByConversation: true,+ async run(conversation) {+ await abandonSignIn(conversation);+ await conversation.waitForReplyTo(CHANGE_OF_PLANS);+ // Finishing the abandoned sign-in afterwards must not run the tool.+ await conversation.completeSignIn();+ await conversation.waitForRest();+ expect(+ conversation.runsOf("read_calendar"),+ "read_calendar ran after its sign-in was cancelled",+ ).toBe(0);+ },+ },+ {+ rule: "message after ignored sign-in tells user it was cancelled",+ source: "docs/connections/overview.mdx#interactive-oauth-via-vercel-connect",+ requires: ["text-replies"],+ variesByConversation: true,+ async run(conversation) {+ await abandonSignIn(conversation);+ await conversation.waitForShown(/\b(?:cancel|declin)/iu);+ },+ },] as const satisfies readonly ContractRule[];export type HitlRule = (typeof hitlContract)[number]["rule"];diff --git a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts b/packages/eve/src/internal/testing/channel-conformance/discord-driver.tsindex 15737b0fa..5e632a60c 100644--- a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts@@ -7,6 +7,7 @@ import {type RenderedOption,type Surface,recordingFetch,+ linkTargets,} from "#internal/testing/channel-conformance/harness.js";let nextChannel = 0;@@ -145,6 +146,7 @@ export function discordDriver(surface: Exclude<Surface, "public"> = "shared"): Cif (!isMessageWrite(call)) return undefined;return {id: (call.response as { readonly id: string }).id,+ links: linkTargets((call.body as { readonly components?: unknown }).components),options: choicesOf(call),text: (call.body as { readonly content?: string }).content ?? "",};diff --git a/packages/eve/src/internal/testing/channel-conformance/harness.ts b/packages/eve/src/internal/testing/channel-conformance/harness.tsindex a4f605acf..42405caed 100644--- a/packages/eve/src/internal/testing/channel-conformance/harness.ts+++ b/packages/eve/src/internal/testing/channel-conformance/harness.ts@@ -8,7 +8,6 @@ import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-aimport { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent-cache.js";import { createAttachSessionFn, type Session } from "#channel/session.js";import { attachRouteSessionCreator } from "#internal/nitro/routes/channel-route-context.js";-import { none } from "#public/channels/auth.js";import { eveChannel } from "#public/channels/eve.js";import { z } from "#compiled/zod/index.js";import { always } from "#tools/approval/policies.js";@@ -17,6 +16,15 @@ import { askQuestion } from "#tools/provided/ask-question.js";import { defineWorkflowTool } from "#public/tools/index.js";import { askDayAndTimeWorkflow } from "#internal/testing/channel-conformance/two-questions-workflow.js";import { getWorld } from "#internal/workflow/runtime.js";+import type { MessageStreamEvent } from "#protocol/message.js";+import {+ type ConnectionAuthorizationChallenge,+ ConnectionAuthorizationRequiredError,+} from "#connections/errors.js";+import { handleConnectionCallbackRequest } from "#execution/connections/callback-route.js";+import type { RouteContext } from "#public/definitions/channel.js";+import type { AgentLimitsDefinition } from "#shared/agent-definition.js";+import { defineInteractiveAuthorization } from "#shared/connection-types.js";/** One outbound call a channel made to its platform API. */export interface PlatformCall {@@ -50,6 +58,30 @@ export interface ShownMessage {readonly text: string;/** The choices the message still lets a person press. */readonly options: readonly RenderedOption[];+ /** Where the message's link buttons lead, such as a sign-in URL. */+ readonly links?: readonly string[];+ /**+ * Only the person sees it, though others can see the conversation, such as+ * Slack's ephemeral messages.+ */+ readonly onlyPerson?: boolean;+}++/** What a client shows a person, and whether anyone else in the conversation sees it too. */+export interface ShownText {+ readonly text: string;+ readonly onlyPerson: boolean;+ /** The choices it lets a person press. */+ readonly options: readonly RenderedOption[];+}++/** Every `url` string anywhere in a message payload: the targets of its link buttons. */+export function linkTargets(value: unknown): string[] {+ if (Array.isArray(value)) return value.flatMap(linkTargets);+ if (typeof value !== "object" || value === null) return [];+ return Object.entries(value).flatMap(([key, child]) =>+ key === "url" && typeof child === "string" ? [child] : linkTargets(child),+ );}/**@@ -109,7 +141,8 @@ export interface ChannelDriver {/*** The message one outbound call posts or edits, as a person sees it* afterward. Required with the `buttons` capability, since rules check how- * an answered prompt's message changes.+ * an answered prompt's message changes. Without it, what a person sees is+ * {@link postedText}.*/shownMessage?(call: PlatformCall): ShownMessage | undefined;/** How the person driving the conversation appears in the platform's text, in any form. */@@ -141,6 +174,23 @@ export interface ChannelConversation {waitForToolResult(tool: string): Promise<unknown>;/** Waits until the bot replies to input that carried `text`, however the channel framed it. */waitForReplyTo(text: string): Promise<void>;+ /** How many test-model replies, plain or reporting a tool result, the bot has shown. */+ replyCount(): number;+ /** Waits for the bot to show a test-model reply, plain or reporting a tool result. */+ waitForReply(): Promise<void>;+ /** Waits for the bot to show text matching `pattern`, returning the text that matched. */+ waitForShown(pattern: string | RegExp): Promise<string>;+ /** Everything the bot has shown that everyone in the conversation sees, joined. */+ sharedText(): string;+ /** Every choice the bot's messages let a person press now, newest message first. */+ shownOptions(): readonly RenderedOption[];+ /** Waits for the turn to hold for a sign-in, however the channel shows it. */+ waitForSignIn(): Promise<void>;+ /**+ * Finishes the oldest unfinished sign-in as the provider would: by+ * redirecting the person's browser to eve's callback URL with a code.+ */+ completeSignIn(): Promise<void>;/** Waits until the bot's reply shows `tool` ran or was denied. */waitForToolOutcome(tool: string): Promise<ToolOutcome>;/** Waits until every session has stopped working and waits only on a person. */@@ -155,6 +205,7 @@ export interface ChannelConversation {shownPrompt(prompt: string): ShownMessage;/** How the person appears in the platform's text, in any form. */readonly personShownAs: readonly string[];+ runsOf(tool: CountedTool): number;}/**@@ -190,6 +241,8 @@ export interface ClientView {shownPrompt?(prompt: string): ShownMessage;/** How the person appears in the client's text, in any form. */readonly personShownAs?: readonly string[];+ /** Everything the client shows now that a person can read or open, one entry per message. */+ shown(): readonly ShownText[];/** What the client shows now, for timeout errors. */describe(): string;/** Stops the client and releases anything it holds, such as its event stream. */@@ -229,6 +282,50 @@ export const SECOND_GATED_TOOL = "publish_notes";export type GatedTool = typeof GATED_TOOL | typeof SECOND_GATED_TOOL;+/** The test agent's plain tool: it runs without asking anyone. */+export const PLAIN_TOOL = "look_up_notes";++/** The test agent's tools that need the person to sign in to a provider before they run. */+export const SIGN_IN_TOOLS = {+ /** A browser sign-in that also shows a confirmation code. */+ read_calendar: {+ displayName: "Calendar",+ instructions: "Sign in to let the agent read your calendar.",+ url: "https://idp.example/authorize?client_id=eve-conformance",+ userCode: "WDJB-MJHT",+ },+ /** A sign-in confirmed out of band, with nothing to open. */+ read_mail: {+ displayName: "Mail",+ instructions: "Approve the sign-in request in the Mail app on your phone.",+ },+} as const satisfies Record<string, ConnectionAuthorizationChallenge>;++export type SignInTool = keyof typeof SIGN_IN_TOOLS;++/** A tool whose runs a rule can count. */+export type CountedTool = GatedTool | SignInTool | typeof PLAIN_TOOL;++/** The code the fake provider hands back when a person finishes signing in. */+const SIGN_IN_CODE = "conformance-code";++/**+ * The person a client driver's eve channel authenticates. Sign-ins are+ * user-scoped, so the channel must map its caller to a user principal.+ */+const CLIENT_PERSON = {+ attributes: {},+ authenticator: "conformance",+ principalId: "alice",+ principalType: "user",+} as const;++export interface ConversationOptions {+ /** The test agent's session usage limits. */+ readonly limits?: AgentLimitsDefinition;+ readonly waitTimeoutMs?: number;+}+/** The test agent's tool that asks {@link DAY_PROMPT} and {@link TIME_PROMPT} at once. */export const TWO_QUESTIONS_TOOL = "plan_review";@@ -261,13 +358,14 @@ export function recordingFetch(export async function withChannelConversation(driver: ChannelDriver | ClientDriver,body: (conversation: ChannelConversation) => Promise<void>,- options: { readonly waitTimeoutMs?: number } = {},+ options: ConversationOptions = {},): Promise<void> {const waitTimeoutMs = options.waitTimeoutMs ?? WAIT_TIMEOUT_MS;const wait: Wait = (label, select, describe) =>poll(`${label} on ${driver.name}`, select, describe, waitTimeoutMs);if (isClientDriver(driver)) {- await converse(driver.name, "eve", eveChannel({ auth: none() }), body, wait, (dispatch) =>+ const channel = eveChannel({ auth: () => CLIENT_PERSON });+ await converse(driver.name, "eve", channel, body, options, wait, (dispatch) =>openClient(driver, dispatch, wait),);return;@@ -275,7 +373,7 @@ export async function withChannelConversation(const calls: PlatformCall[] = [];try {const channel = driver.createChannel((call) => void calls.push(call));- await converse(driver.name, driver.name, channel, body, wait, async (dispatch) =>+ await converse(driver.name, driver.name, channel, body, options, wait, async (dispatch) =>webhookView(driver, calls, dispatch, wait),);} finally {@@ -318,6 +416,8 @@ function webhookView(if (!response.ok) throw new Error(`${driver.name} webhook answered ${response.status}.`);}const describe = () => `Platform calls:\n${JSON.stringify(calls, null, 2)}`;+ /** Where each prompt may next appear: after the call it was last found in. */+ const promptsFrom = new Map<string, number>();return {say: (text) => post(driver.message(text)),@@ -326,10 +426,15 @@ function webhookView(wait(`one of the questions ${JSON.stringify(prompts)}`,() => {- for (const call of calls) {+ for (const [index, call] of calls.entries()) {for (const prompt of prompts) {+ if (index < (promptsFrom.get(prompt) ?? 0)) continue;const options = driver.findOptions(call, prompt);- if (options !== undefined) return { options, prompt };+ if (options === undefined) continue;+ // After a prompt is asked, an optionless match is an edit of the answered message.+ if (promptsFrom.has(prompt) && options.length === 0) continue;+ promptsFrom.set(prompt, index + 1);+ return { options, prompt };}}return undefined;@@ -351,6 +456,16 @@ function webhookView(.at(-1)!;},personShownAs: driver.personShownAs ?? [],+ shown: () =>+ calls.flatMap((call) => {+ const shown = driver.shownMessage?.(call);+ if (shown !== undefined) {+ const text = [shown.text, ...(shown.links ?? [])].join("\n");+ return [{ onlyPerson: shown.onlyPerson === true, options: shown.options, text }];+ }+ const text = driver.postedText(call);+ return text === undefined ? [] : [{ onlyPerson: false, options: [], text }];+ }),describe,close: async () => {},};@@ -392,16 +507,73 @@ async function converse(channelName: string,created: unknown,body: (conversation: ChannelConversation) => Promise<void>,+ options: ConversationOptions,wait: Wait,open: (dispatch: Dispatch) => Promise<ClientView>,): Promise<void> {if (!isCompiledChannel(created)) throw new Error(`${label} is not a compiled channel.`);const channel: CompiledChannel = created;- const runs: Record<GatedTool, number> = { [GATED_TOOL]: 0, [SECOND_GATED_TOOL]: 0 };+ const runs: Record<CountedTool, number> = {+ [GATED_TOOL]: 0,+ [SECOND_GATED_TOOL]: 0,+ [PLAIN_TOOL]: 0,+ read_calendar: 0,+ read_mail: 0,+ };+ /** eve's callback URL for each sign-in started, as the provider received it. */+ const signInCallbacks: string[] = [];++ function signInTool(name: SignInTool) {+ let token: string | undefined;+ const auth = defineInteractiveAuthorization({+ async getToken() {+ if (token === undefined) throw new ConnectionAuthorizationRequiredError(name);+ return { token };+ },+ async startAuthorization({ callbackUrl }) {+ signInCallbacks.push(callbackUrl);+ return { challenge: SIGN_IN_TOOLS[name] };+ },+ async completeAuthorization({ callback }) {+ if (callback.params.code !== SIGN_IN_CODE) throw new Error("Unexpected sign-in code.");+ token = `${name}-token`;+ return { token };+ },+ });+ return {+ logicalPath: `tools/${name}.ts`,+ loadNamespace: async () => ({+ default: defineTool({+ description: `Reads the person's ${SIGN_IN_TOOLS[name].displayName}. Only call when asked to use ${name}.`,+ async execute(_input, ctx) {+ await ctx.getToken(auth, { authKey: name });+ runs[name] += 1;+ return { signedIn: true };+ },+ inputSchema: z.object({}),+ }),+ }),+ };+ }const runtime = await createTestRuntime({- agent: { name: `${label}-hitl-conformance` },+ agent: { limits: options.limits, name: `${label}-hitl-conformance` },modules: [+ {+ logicalPath: `tools/${PLAIN_TOOL}.ts`,+ loadNamespace: async () => ({+ default: defineTool({+ description: `Looks up meeting notes. Only call when asked to use ${PLAIN_TOOL}.`,+ execute: async () => {+ runs[PLAIN_TOOL] += 1;+ return { notes: "Bob's review notes" };+ },+ inputSchema: z.object({}),+ }),+ }),+ },+ signInTool("read_calendar"),+ signInTool("read_mail"),{logicalPath: "tools/ask_question.ts",loadNamespace: async () => ({ default: askQuestion() }),@@ -515,6 +687,10 @@ async function converse(},view.describe,);+ const replyCount = () =>+ view.replies().filter((reply) => isMockReplyTo(reply, "") || reply.startsWith("Used "))+ .length;+ let signInsCompleted = 0;const conversation: ChannelConversation = {async say(text) {@@ -549,6 +725,52 @@ async function converse(if (output !== undefined) return { kind: "ran", output };return isMockDenialReply(reply) ? { kind: "denied" } : undefined;}),+ replyCount,+ waitForReply: async () =>+ void (await wait("a reply", () => (replyCount() > 0 ? true : undefined), view.describe)),+ waitForShown: (pattern) =>+ wait(+ `the bot to show ${String(pattern)}`,+ () =>+ view+ .shown()+ .map(({ text }) => text)+ .find((text) =>+ typeof pattern === "string" ? text.includes(pattern) : pattern.test(text),+ ),+ view.describe,+ ),+ shownOptions: () =>+ view+ .shown()+ .toReversed()+ .flatMap(({ options }) => options),+ sharedText: () =>+ view+ .shown()+ .filter((shown) => !shown.onlyPerson)+ .map(({ text }) => text)+ .join("\n"),+ waitForSignIn: async () =>+ void (await wait(+ "the turn to hold for a sign-in",+ async () => {+ for (const session of sessions.values()) {+ if (await holdsFor(session, isSignIn)) return true;+ }+ return undefined;+ },+ () => "",+ )),+ async completeSignIn() {+ const callbackUrl = await wait(+ "a sign-in callback URL",+ () => signInCallbacks.at(signInsCompleted),+ () => "",+ );+ signInsCompleted += 1;+ await deliverSignInCallback(callbackUrl);+ },runsOf: (tool) => runs[tool],waitForRest: () => waitForRest([...sessions.values()], wait),shownPrompt(prompt) {@@ -588,7 +810,7 @@ async function converse(`the turn to hold for "${prompt}"`,async () => {for (const session of sessions.values()) {- if (await holdsFor(session, prompt)) return true;+ if (await holdsFor(session, asks(prompt))) return true;}return undefined;},@@ -707,27 +929,41 @@ async function waitForRest(sessions: readonly Session[], wait: Wait): Promise<vo);}+/** Whether an event asks the person `prompt`. */+const asks = (prompt: string) => (event: MessageStreamEvent) =>+ event.type === "input.requested" &&+ event.data.requests.some((request) => request.prompt === prompt);++/** Whether an event asks the person to sign in. */+const isSignIn = (event: MessageStreamEvent) => event.type === "authorization.required";+/**- * Whether the turn held for input after it last asked `prompt`. The turn emits- * `turn.waiting` after each request, or once after requests raised together.+ * Whether the session held for the person after it last emitted an event+ * `asked` matches. A question, approval, or sign-in parks the open turn+ * (`turn.waiting`) after each request, or once after requests raised+ * together; a session-limit prompt ends the turn (`session.waiting`).*/-async function holdsFor(session: Session, prompt: string): Promise<boolean> {+async function holdsFor(+ session: Session,+ asked: (event: MessageStreamEvent) => boolean,+): Promise<boolean> {const tail = await session.getStreamTailIndex();if (tail < 0) return false;const reader = (await session.getEventStream({ startIndex: 0 })).getReader();- let asked = false;+ let seen = false;let held = false;try {for (let index = 0; index <= tail; index += 1) {const { done, value } = await reader.read();if (done) break;- if (- value.type === "input.requested" &&- value.data.requests.some((request) => request.prompt === prompt)- ) {- asked = true;+ if (asked(value)) {+ seen = true;held = false;- } else if (asked && value.type === "turn.waiting" && value.data.on === "input") {+ } else if (+ seen &&+ ((value.type === "turn.waiting" && value.data.on === "input") ||+ value.type === "session.waiting")+ ) {held = true;}}@@ -805,6 +1041,30 @@ function isMockReplyTo(text: string, message: string): boolean {return text.startsWith("Bootstrap reply") && text.includes(message);}+/**+ * Delivers the provider's redirect to eve's framework callback route. A+ * callback for a sign-in eve no longer waits on is answered `404`, which a+ * person would see as an error page; rules assert what follows instead.+ */+async function deliverSignInCallback(callbackUrl: string): Promise<void> {+ const url = new URL(callbackUrl);+ url.searchParams.set("code", SIGN_IN_CODE);+ const segments = url.pathname.split("/");+ const at = segments.lastIndexOf("callback");+ const [name, attemptId, token] = [at - 1, at + 1, at + 2].map((index) =>+ decodeURIComponent(segments[index] ?? ""),+ );+ const context: RouteContext = {+ params: { attemptId, name, token } as Record<string, string>,+ requestIp: null,+ waitUntil: () => {},+ };+ const response = await handleConnectionCallbackRequest(new Request(url), context);+ if (!response.ok && response.status !== 404) {+ throw new Error(`The sign-in callback answered ${response.status}.`);+ }+}+/** The test model reports a denied call's `execution-denied` result in its reply. */function isMockDenialReply(text: string): boolean {return text.includes('"type":"execution-denied"');diff --git a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linear-driver.tsindex e27f2a747..dff6b580e 100644--- a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts@@ -3,6 +3,7 @@ import {type PlatformCall,type RenderedOption,recordingFetch,+ linkTargets,} from "#internal/testing/channel-conformance/harness.js";import { linearChannel } from "#public/channels/linear/index.js";import { signLinearWebhookBody } from "#public/channels/linear/verify.js";@@ -112,11 +113,31 @@ export function linearDriver(): ChannelDriver {.variables?.input;return input?.content?.type === "response" ? input.content.body : undefined;},+ shownMessage(call) {+ if (call.method !== "AgentActivityCreate") return undefined;+ const input = (call.body as { readonly variables?: { readonly input?: LinearActivityInput } })+ .variables?.input;+ if (input?.content === undefined) return undefined;+ const response = call.response as {+ readonly data: {+ readonly agentActivityCreate: { readonly agentActivity: { readonly id: string } };+ };+ };+ return {+ id: response.data.agentActivityCreate.agentActivity.id,+ // An `auth` elicitation's sign-in link rides in its signal metadata.+ links: linkTargets(input.signalMetadata),+ // The driver can't press Linear's native select, so nothing here is pressable.+ options: [],+ text: input.content.body,+ };+ },};}interface LinearActivityInput {readonly content?: { readonly body: string; readonly type: string };+ readonly signalMetadata?: unknown;}function parseVisibleOptions(body: string): RenderedOption[] {diff --git a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linq-driver.tsindex 68e1a03d9..f76c82d15 100644--- a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts@@ -23,6 +23,8 @@ interface LinqPart {interface LinqMessageBody {readonly message?: { readonly parts?: readonly LinqPart[] };+ /** An edit's replacement text. */+ readonly text?: string;}/** Drives the real Linq adapter through signed webhooks, in a group chat by default or 1:1. */@@ -31,6 +33,7 @@ export function linqDriver(surface: Exclude<Surface, "public"> = "shared"): ChannextChat += 1;const chatId = `linq-conformance-chat-${nextChat}`;let messageId = 0;+ let outbound = 0;let restoreFetch: (() => void) | undefined;function signedMessage(text: string): Request {@@ -72,10 +75,13 @@ export function linqDriver(surface: Exclude<Surface, "public"> = "shared"): Chanconst fakeFetch = recordingFetch(record, async (request) => {const url = new URL(request.url);const bodyText = await request.text();+ outbound += 1;+ const id = `linq-outbound-${outbound}`;return {body: bodyText === "" ? {} : JSON.parse(bodyText),method: `${request.method} ${url.pathname}`,- response: {},+ // The adapter reads the sent or edited message's id back to edit it later.+ response: { chat_id: chatId, id, message: { id } },};});// The adapter accepts no fetch option, so route only its test host globally and preserve other fetches.@@ -110,9 +116,12 @@ export function linqDriver(surface: Exclude<Surface, "public"> = "shared"): Chan};}+/** Text the bot sent (`POST …/messages`) or edited a message to (`PATCH /messages/:id`). */function postedText(call: PlatformCall): string | undefined {+ const body = call.body as LinqMessageBody;+ if (call.method.startsWith("PATCH ") && call.method.includes("/messages/")) return body.text;if (!call.method.endsWith("/messages")) return undefined;- const parts = (call.body as LinqMessageBody).message?.parts;+ const parts = body.message?.parts;return parts?.filter((part) => part.type === "text").map((part) => part.value ?? "")diff --git a/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts b/packages/eve/src/internal/testing/channel-conformance/slack-driver.tsindex f6ed510e4..f57f5aa7e 100644--- a/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts@@ -8,6 +8,7 @@ import {type PlatformCall,type RenderedOption,recordingFetch,+ linkTargets,} from "#internal/testing/channel-conformance/harness.js";import { decodeSlackApiBody } from "#internal/testing/slack-api-body.js";@@ -45,6 +46,8 @@ interface PressHandle {interface SlackMessageBody {readonly blocks?: readonly unknown[];+ /** Replies post Markdown here, with `text` as the plain fallback. */+ readonly markdown_text?: string;readonly text?: string;readonly ts?: string;}@@ -161,8 +164,10 @@ export function slackDriver(surface: Exclude<Surface, "public"> = "shared"): Chaconst id = messageTsOf(call);return {id,+ links: linkTargets(body.blocks ?? []),+ onlyPerson: call.method === "chat.postEphemeral",options: hitlOptions(body.blocks ?? [], id),- text: [body.text ?? "", ...blockTexts(body.blocks ?? [])].join("\n"),+ text: [body.markdown_text ?? body.text ?? "", ...blockTexts(body.blocks ?? [])].join("\n"),};},personShownAs: [`<@${PERSON}>`],@@ -195,8 +200,13 @@ export function slackDriver(surface: Exclude<Surface, "public"> = "shared"): Cha};}+/** A message write a person sees; an ephemeral one only they see, such as a private sign-in. */function isMessageWrite(call: PlatformCall): boolean {- return call.method === "chat.postMessage" || call.method === "chat.update";+ return (+ call.method === "chat.postMessage" ||+ call.method === "chat.postEphemeral" ||+ call.method === "chat.update"+ );}/** A post's ts comes back from Slack; an update names the ts it rewrites. */diff --git a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts b/packages/eve/src/internal/testing/channel-conformance/teams-driver.tsindex 4314b1d33..dc3776f55 100644--- a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts@@ -5,6 +5,7 @@ import {type RenderedOption,type Surface,recordingFetch,+ linkTargets,} from "#internal/testing/channel-conformance/harness.js";let nextConversation = 0;@@ -132,6 +133,7 @@ export function teamsDriver(surface: Exclude<Surface, "public"> = "shared"): Cha);return {id: activityIdOf(call),+ links: linkTargets(adaptiveCard(body)?.actions),options: cardOptions(call),text: [body.text ?? "", ...texts].join("\n"),};diff --git a/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts b/packages/eve/src/internal/testing/channel-conformance/telegram-driver.tsindex b704bfe1c..20f317a5b 100644--- a/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts@@ -5,6 +5,7 @@ import {type RenderedOption,type Surface,recordingFetch,+ linkTargets,} from "#internal/testing/channel-conformance/harness.js";const SECRET = "telegram-conformance-secret";@@ -17,6 +18,8 @@ interface InlineButton {}interface MessageBody {+ /** Shows the message to one member of a group only. */+ readonly ephemeral_message_parameters?: unknown;readonly message_id?: number;readonly reply_markup?: { readonly inline_keyboard?: readonly InlineButton[][] };readonly text?: string;@@ -138,6 +141,8 @@ export function telegramDriver(surface: Exclude<Surface, "public"> = "shared"):if (!MESSAGE_WRITES.has(call.method)) return undefined;return {id: String(messageIdOf(call)),+ links: linkTargets((call.body as { readonly reply_markup?: unknown }).reply_markup),+ onlyPerson: (call.body as MessageBody).ephemeral_message_parameters !== undefined,// Telegram drops a message's inline keyboard when an edit omits `reply_markup`.options: inlineOptions(call),text: (call.body as MessageBody).text ?? "",diff --git a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts b/packages/eve/src/internal/testing/channel-conformance/tui-driver.tsindex 63fdd4520..80b9cfbb8 100644--- a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts@@ -26,8 +26,8 @@ const FREEFORM_ROW = "Type your own answer…";export function tuiDriver(): ClientDriver {return {name: "tui",- capabilities: ["buttons", "text-replies"],// A local terminal shows only the person at it.+ capabilities: ["buttons", "text-replies"],surface: "private",async open(host, wait) {// Wide enough that no reply wraps, so each one stays on one screen row.@@ -98,6 +98,7 @@ export function tuiDriver(): ClientDriver {focusedRow(screen, prompt) === undefined ? [] : readOptions(screen, keyboard, prompt),text: screen.snapshot(),}),+ shown: () => [{ onlyPerson: true, options: [], text: screen.snapshot() }],describe,async close() {renderer.requestInterrupt();