Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

[6/7] test(eve): add budget and sign-in conformance rules (#4247)

Ben Pankow77874cd4c51c
diff --git a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
index d7d8299e6..9f9ddcd10 100644
--- a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
+++ b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
@@ -11,143 +11,69 @@ recorded symptom. Regenerate it after changing either file:
pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conformance/matrix -u
```
-✅ passes · ❌ broken · — not supported (the platform lacks a capability the rule
-needs, or the client declines it below)
+✅ passes · ❌ broken · — not supported
-| Rule | `chat-sdk` | `chat-sdk-text` | `chat-sdk-dm` | `discord` | `discord-dm` | `github` | `linear` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `tui` | `twilio` |
+| Rule | `tui` | `chat-sdk` | `chat-sdk-dm` | `chat-sdk-text` | `discord` | `discord-dm` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `github` | `linear` | `twilio` |
| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
-| a rendered question shows every option a person can choose | ✅ | ✅ | — | ✅ | — | ✅ | ✅ | ✅ | — | ✅ | — | ✅ | — | ✅ | — | ✅ | ✅ |
-| pressing a rendered option answers the pending question with that option | ✅ | — | ✅ | ✅ | ✅ | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — |
-| a text reply matching an option answers the only pending question | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| a text reply that matches no option answers the question with the person's words | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| a text reply answers an open-ended question with the person's words | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| pressing an option of an answered question sends it to the agent as new input | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | — | — |
-| pressing options of two pending questions answers each with its own option | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
-| a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | — | ✅ | — | ✅ | ✅ | ✅ | — | ✅ | — | ✅ | — | ✅ | — | ✅ | ✅ |
-| pressing Approve runs the gated tool | ✅ | — | ✅ | ✅ | ✅ | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — |
-| pressing Approve twice runs the gated tool once | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
-| pressing Approve on one of two pending approvals runs only that tool | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
-| answering an approval and a question pending together settles both | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
-| pressing Cancel stops the gated tool without running it | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
-| a text reply of approve runs the gated tool | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |
-| a text reply of cancel stops the gated tool without running it | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |
-| pressing an option clears the question's buttons | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
-| answering a question by text clears its buttons | ✅ | — | ✅ | — | — | — | — | — | — | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | — |
-| pressing an option names who answered on the question | ❌ | — | — | ❌ | — | — | — | — | — | ✅ | — | ❌ | — | ❌ | — | — | — |
-| answering a question by text names who answered on the question | ❌ | — | ❌ | — | — | — | — | — | — | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | — | — |
-| pressing Approve clears the approval's buttons | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
-| approving by text clears the approval's buttons | ✅ | — | ✅ | — | — | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | — |
-| pressing Approve names who approved on the approval | ❌ | — | — | ❌ | — | — | — | — | — | ✅ | — | ✅ | — | ❌ | — | — | — |
-| approving by text names who approved on the approval | ❌ | — | ❌ | — | — | — | — | — | — | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | — | — |
+| a rendered question shows every option a person can choose | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ |
+| pressing a rendered option answers the pending question with that option | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| a text reply matching an option answers the only pending question | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| a text reply that matches no option answers the question with the person's words | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| a text reply answers an open-ended question with the person's words | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| pressing an option of an answered question sends it to the agent as new input | —<sup>[3](#note-3)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| pressing options of two pending questions answers each with its own option | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ |
+| pressing Approve runs the gated tool | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| pressing Approve twice runs the gated tool once | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| pressing Approve on one of two pending approvals runs only that tool | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| answering an approval and a question pending together settles both | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| pressing Cancel stops the gated tool without running it | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| a text reply of approve runs the gated tool | —<sup>[4](#note-4)</sup> | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| a text reply of cancel stops the gated tool without running it | —<sup>[4](#note-4)</sup> | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| pressing an option clears the question's buttons | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| answering a question by text clears its buttons | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[5](#note-5)</sup> | ❌<sup>[5](#note-5)</sup> | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| pressing an option names who answered on the question | —<sup>[6](#note-6)</sup> | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ❌<sup>[7](#note-7)</sup> | | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| answering a question by text names who answered on the question | —<sup>[6](#note-6)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[5](#note-5)</sup> | ❌<sup>[5](#note-5)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| pressing Approve clears the approval's buttons | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ✅ | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| approving by text clears the approval's buttons | —<sup>[4](#note-4)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| pressing Approve names who approved on the approval | —<sup>[6](#note-6)</sup> | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ❌<sup>[7](#note-7)</sup> | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| approving by text names who approved on the approval | —<sup>[4](#note-4)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ❌<sup>[8](#note-8)</sup> | ❌<sup>[8](#note-8)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | ❌<sup>[7](#note-7)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| running out of budget opens budget prompt | ✅ | ✅ | | ✅ | ❌<sup>[9](#note-9)</sup> | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ |
+| pressing approve on budget prompt allows agent to continue | ✅ | ✅ | | —<sup>[1](#note-1)</sup> | ❌<sup>[9](#note-9)</sup> | | —<sup>[1](#note-1)</sup> | | ✅ | | ✅ | | ✅ | | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| reply of approve on budget prompt allows agent to continue | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| pressing stop on budget prompt halts work, next message asks again | ❌<sup>[10](#note-10)</sup> | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> | —<sup>[1](#note-1)</sup> |
+| reply of stop on budget prompt halts work, next message asks again | ❌<sup>[10](#note-10)</sup> | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| query sent during budget prompt answered after budget approval | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> | ❌<sup>[11](#note-11)</sup> |
+| a sign-in names the service and shows its sign-in link | ❌<sup>[12](#note-12)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |
+| a sign-in shows its confirmation code | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ✅ | ❌<sup>[16](#note-16)</sup> | ❌<sup>[16](#note-16)</sup> | ✅ | ✅ | —<sup>[15](#note-15)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |
+| only the person signing in sees the sign-in link and code | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | ❌<sup>[18](#note-18)</sup> | —<sup>[17](#note-17)</sup> | ✅ | —<sup>[17](#note-17)</sup> | ✅ | ❌<sup>[19](#note-19)</sup> | —<sup>[17](#note-17)</sup> |
+| a sign-in without a link shows its instructions | ✅ | ❌<sup>[13](#note-13)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ❌<sup>[13](#note-13)</sup> | ✅ | ❌<sup>[20](#note-20)</sup> | ❌<sup>[20](#note-20)</sup> | ❌<sup>[21](#note-21)</sup> | ❌<sup>[21](#note-21)</sup> | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |
+| after signing in, the agent carries on with the request | ✅ | ✅ | | ✅ | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | | ✅ | ✅ | ✅ |
+| completing a sign-in tells the person it succeeded | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ❌<sup>[14](#note-14)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |
+| message after ignored sign-in gets an answer, signing in late doesn't run | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| message after ignored sign-in tells user it was cancelled | ✅ | ✅ | ✅ | ✅ | —<sup>[2](#note-2)</sup> | —<sup>[2](#note-2)</sup> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌<sup>[14](#note-14)</sup> | ✅ | ❌<sup>[14](#note-14)</sup> |
-## Broken
+## Notes
-- **chat-sdk**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **chat-sdk**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **chat-sdk**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **chat-sdk**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **chat-sdk-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **chat-sdk-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **discord**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **discord**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **slack**, answering a question by text clears its buttons: only the button interaction handler edits a question; a typed answer leaves it
-- **slack**, answering a question by text names who answered on the question: only the button interaction handler edits a question; a typed answer leaves it
-- **slack**, approving by text names who approved on the approval: the card loses its buttons after a typed approval but doesn't say who approved
-- **slack-dm**, answering a question by text clears its buttons: only the button interaction handler edits a question; a typed answer leaves it
-- **slack-dm**, answering a question by text names who answered on the question: only the button interaction handler edits a question; a typed answer leaves it
-- **slack-dm**, approving by text names who approved on the approval: the card loses its buttons after a typed approval but doesn't say who approved
-- **teams**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **teams**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **teams**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **teams-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **teams-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **telegram**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **telegram**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **telegram**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **telegram**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **telegram-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
-- **telegram-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
-
-## Declined
-
-- **chat-sdk-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **chat-sdk-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **discord-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **linq-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **slack-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **teams-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **telegram-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
-- **tui**, pressing an option of an answered question sends it to the agent as new input: an answered question's drawer closes, so nothing is left to press
-- **tui**, a text reply of approve runs the gated tool: the approval drawer holds the keyboard; a person answers it with y or n
-- **tui**, a text reply of cancel stops the gated tool without running it: the approval drawer holds the keyboard; a person answers it with y or n
-- **tui**, pressing an option names who answered on the question: one person answers at their own terminal; there's nobody else to tell
-- **tui**, answering a question by text names who answered on the question: one person answers at their own terminal; there's nobody else to tell
-- **tui**, approving by text clears the approval's buttons: the approval drawer holds the keyboard; a person answers it with y or n
-- **tui**, pressing Approve names who approved on the approval: one person answers at their own terminal; there's nobody else to tell
-- **tui**, approving by text names who approved on the approval: the approval drawer holds the keyboard; a person answers it with y or n
+1. <a id="note-1"></a>the platform has no buttons a person can press
+2. <a id="note-2"></a>the platform has no plain-text replies
+3. <a id="note-3"></a>an answered question's drawer closes, so nothing is left to press
+4. <a id="note-4"></a>the approval drawer holds the keyboard; a person answers it with y or n
+5. <a id="note-5"></a>only the button interaction handler edits a question; a typed answer leaves it
+6. <a id="note-6"></a>one person answers at their own terminal; there's nobody else to tell
+7. <a id="note-7"></a>a resolved prompt doesn't say who answered; input.resolved carries no responder
+8. <a id="note-8"></a>the card loses its buttons after a typed approval but doesn't say who approved
+9. <a id="note-9"></a>a budget prompt's request id overflows Discord's 100-character custom_id, so posting it throws
+10. <a id="note-10"></a>a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen
+11. <a id="note-11"></a>eve coalesces the queued reply with the later approve, which then matches no option
+12. <a id="note-12"></a>the TUI labels a sign-in with the tool name, not the challenge's displayName
+13. <a id="note-13"></a>outside a DM the bot says to continue in a direct message but never sends one
+14. <a id="note-14"></a>the channel has no default sign-in renderer
+15. <a id="note-15"></a>the rule applies only where the conversation is shared or private, and this one is public
+16. <a id="note-16"></a>the Teams sign-in card omits the challenge's user code
+17. <a id="note-17"></a>the rule applies only where the conversation is public or shared, and this one is private
+18. <a id="note-18"></a>the sign-in prompt, link included, is posted to the whole thread
+19. <a id="note-19"></a>the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified
+20. <a id="note-20"></a>Slack sends the private sign-in prompt only for a challenge with a URL
+21. <a id="note-21"></a>the Teams sign-in card omits the challenge's instructions
diff --git a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts
index 43cade21f..0a3c7f0e3 100644
--- a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts
@@ -16,10 +16,18 @@ import {
type PlatformCall,
type Surface,
numberedOptions,
+ linkTargets,
} from "#internal/testing/channel-conformance/harness.js";
const ADAPTER = "conformance";
const PERSON = { fullName: "Alice", isBot: false, isMe: false, userId: "alice", userName: "alice" };
+/** The signed-in person a real integration would attach from its own user directory. */
+const PERSON_AUTH = {
+ attributes: {},
+ authenticator: "conformance",
+ principalId: "alice",
+ principalType: "user",
+};
let nextThread = 0;
interface CardNode {
@@ -87,6 +95,7 @@ export function chatSdkDriver(surface: Exclude<Surface, "public"> = "shared"): C
const card = cardOf(call.body as AdapterPostableMessage);
return {
id: messageIdOf(call),
+ links: linkTargets(card),
options: buttonsOf(call),
text: card === undefined ? (driver.postedText(call) ?? "") : texts(card),
};
@@ -179,17 +188,17 @@ function chatSdkDriverWith(input: {
});
if (dm) {
bridge.bot.onDirectMessage(async (thread, message) => {
- await bridge.send(message.text, { auth: null, context: [], thread });
+ await bridge.send(message.text, { auth: PERSON_AUTH, context: [], thread });
});
} else {
// The wiring docs/channels/chat-sdk.mdx shows: a mention starts the session, and
// subscribing lets the rest of the thread continue it without one.
bridge.bot.onNewMention(async (thread, message) => {
await thread.subscribe();
- await bridge.send(message.text, { auth: null, context: [], thread });
+ await bridge.send(message.text, { auth: PERSON_AUTH, context: [], thread });
});
bridge.bot.onSubscribedMessage(async (thread, message) => {
- await bridge.send(message.text, { auth: null, context: [], thread });
+ await bridge.send(message.text, { auth: PERSON_AUTH, context: [], thread });
});
}
return bridge.channel;
diff --git a/packages/eve/src/internal/testing/channel-conformance/conformance.ts b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
index 8f598ce8d..d8a633141 100644
--- a/packages/eve/src/internal/testing/channel-conformance/conformance.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
@@ -6,6 +6,7 @@ import {
hitlContract,
} from "#internal/testing/channel-conformance/contract.js";
import {
+ type ChannelCapability,
type ChannelDriver,
type ClientDriver,
withChannelConversation,
@@ -95,22 +96,88 @@ interface ConformanceChannel {
readonly unsupported?: Partial<Record<HitlRule, string>>;
}
+/** A message sent during a budget prompt is queued, and every later message queues behind it. */
+const QUEUED_BUDGET_REPLY = {
+ "query sent during budget prompt answered after budget approval": {
+ reason: "eve coalesces the queued reply with the later approve, which then matches no option",
+ symptom: /Timed out waiting for a reply to "Carol wants the review by Friday\."/u,
+ },
+} satisfies Partial<Record<HitlRule, BrokenCell>>;
+
+const SIGN_IN_NOT_SHOWN = /Timed out waiting for the bot to show/u;
+
+/** The channel has no default `authorization.required` renderer, so a sign-in shows nothing. */
+function noSignInRenderer(...rules: HitlRule[]): Partial<Record<HitlRule, BrokenCell>> {
+ return Object.fromEntries(
+ rules.map((rule) => [
+ rule,
+ { reason: "the channel has no default sign-in renderer", symptom: SIGN_IN_NOT_SHOWN },
+ ]),
+ );
+}
+
+/** Budget prompts in `rules` never show; see each caller for why. */
+function budgetPromptNotShown(
+ reason: string,
+ ...rules: HitlRule[]
+): Partial<Record<HitlRule, BrokenCell>> {
+ return Object.fromEntries(
+ rules.map((rule) => [
+ rule,
+ { reason, symptom: /Timed out waiting for one of the questions \["This session has hit/u },
+ ]),
+ );
+}
+
+/** Chat SDK's default sign-in, outside a DM, points the person at a DM it never sends. */
+const SIGN_IN_ONLY_IN_DMS = Object.fromEntries(
+ [
+ "a sign-in names the service and shows its sign-in link",
+ "a sign-in shows its confirmation code",
+ "a sign-in without a link shows its instructions",
+ ].map((rule) => [
+ rule,
+ {
+ reason: "outside a DM the bot says to continue in a direct message but never sends one",
+ symptom: SIGN_IN_NOT_SHOWN,
+ },
+ ]),
+) satisfies Partial<Record<HitlRule, BrokenCell>>;
+
const UNNAMED_RESPONDER =
"a resolved prompt doesn't say who answered; input.resolved carries no responder";
-const CHAT_SDK_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
- "approvalPress",
- "approvalText",
- "questionPress",
- "questionText",
-]);
+const CHAT_SDK_BROKEN = {
+ ...unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
+ "approvalPress",
+ "approvalText",
+ "questionPress",
+ "questionText",
+ ]),
+ ...QUEUED_BUDGET_REPLY,
+};
-const DISCORD_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
- "approvalPress",
- "questionPress",
-]);
+const DISCORD_BROKEN = {
+ ...unnamedAnsweredPrompts(UNNAMED_RESPONDER, ["approvalPress", "questionPress"]),
+ ...budgetPromptNotShown(
+ "a budget prompt's request id overflows Discord's 100-character custom_id, so posting it throws",
+ "running out of budget opens budget prompt",
+ "pressing approve on budget prompt allows agent to continue",
+ ),
+ ...noSignInRenderer(
+ "a sign-in names the service and shows its sign-in link",
+ "a sign-in shows its confirmation code",
+ "a sign-in without a link shows its instructions",
+ "completing a sign-in tells the person it succeeded",
+ ),
+};
const SLACK_BROKEN = {
+ ...QUEUED_BUDGET_REPLY,
+ "a sign-in without a link shows its instructions": {
+ reason: "Slack sends the private sign-in prompt only for a challenge with a URL",
+ symptom: SIGN_IN_NOT_SHOWN,
+ },
...staleAnsweredPrompts(
"only the button interaction handler edits a question; a typed answer leaves it",
["questionText"],
@@ -121,18 +188,36 @@ const SLACK_BROKEN = {
},
} satisfies Partial<Record<HitlRule, BrokenCell>>;
-const TEAMS_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
- "approvalText",
- "questionPress",
- "questionText",
-]);
+const TEAMS_BROKEN = {
+ ...unnamedAnsweredPrompts(UNNAMED_RESPONDER, ["approvalText", "questionPress", "questionText"]),
+ ...QUEUED_BUDGET_REPLY,
+ "a sign-in shows its confirmation code": {
+ reason: "the Teams sign-in card omits the challenge's user code",
+ symptom: SIGN_IN_NOT_SHOWN,
+ },
+ "a sign-in without a link shows its instructions": {
+ reason: "the Teams sign-in card omits the challenge's instructions",
+ symptom: SIGN_IN_NOT_SHOWN,
+ },
+} satisfies Partial<Record<HitlRule, BrokenCell>>;
-const TELEGRAM_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
- "approvalPress",
- "approvalText",
- "questionPress",
- "questionText",
-]);
+/** The sign-in prompt, link included, goes to the whole thread. */
+const SIGN_IN_LINK_POSTED_TO_THREAD = {
+ "only the person signing in sees the sign-in link and code": {
+ reason: "the sign-in prompt, link included, is posted to the whole thread",
+ symptom: /a message everyone sees carried the sign-in (link|code)/u,
+ },
+} satisfies Partial<Record<HitlRule, BrokenCell>>;
+
+const TELEGRAM_BROKEN = {
+ ...unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
+ "approvalPress",
+ "approvalText",
+ "questionPress",
+ "questionText",
+ ]),
+ ...QUEUED_BUDGET_REPLY,
+};
const TUI_TYPED_APPROVAL =
"the approval drawer holds the keyboard; a person answers it with y or n";
@@ -151,23 +236,62 @@ const TUI_SINGLE_PERSON = "one person answers at their own terminal; there's nob
* failure (such as harness breakage) both turn it red.
*/
const hitlConformance = {
- "chat-sdk": [{ driver: chatSdkDriver, broken: CHAT_SDK_BROKEN }, { driver: chatSdkTextDriver }],
+ "chat-sdk": [
+ { driver: chatSdkDriver, broken: { ...CHAT_SDK_BROKEN, ...SIGN_IN_ONLY_IN_DMS } },
+ { driver: chatSdkTextDriver, broken: QUEUED_BUDGET_REPLY },
+ ],
"chat-sdk-dm": [{ dm: true, driver: () => chatSdkDriver("private"), broken: CHAT_SDK_BROKEN }],
discord: [{ driver: discordDriver, broken: DISCORD_BROKEN }],
"discord-dm": [{ dm: true, driver: () => discordDriver("private"), broken: DISCORD_BROKEN }],
- github: [{ driver: githubDriver }],
- linear: [{ driver: linearDriver }],
- linq: [{ driver: linqDriver }],
- "linq-dm": [{ dm: true, driver: () => linqDriver("private") }],
+ github: [
+ {
+ driver: githubDriver,
+ broken: {
+ ...QUEUED_BUDGET_REPLY,
+ ...noSignInRenderer(
+ "a sign-in without a link shows its instructions",
+ "completing a sign-in tells the person it succeeded",
+ "message after ignored sign-in tells user it was cancelled",
+ ),
+ },
+ },
+ ],
+ linear: [
+ {
+ driver: linearDriver,
+ broken: {
+ ...QUEUED_BUDGET_REPLY,
+ "only the person signing in sees the sign-in link and code": {
+ reason:
+ "the code is in the elicitation body the whole issue sees; who sees the auth signal's link is unverified",
+ symptom: /a message everyone sees carried the sign-in (link|code)/u,
+ },
+ },
+ },
+ ],
+ linq: [{ driver: linqDriver, broken: { ...QUEUED_BUDGET_REPLY, ...SIGN_IN_ONLY_IN_DMS } }],
+ "linq-dm": [{ dm: true, driver: () => linqDriver("private"), broken: QUEUED_BUDGET_REPLY }],
slack: [{ driver: slackDriver, broken: SLACK_BROKEN }],
"slack-dm": [{ dm: true, driver: () => slackDriver("private"), broken: SLACK_BROKEN }],
- teams: [{ driver: teamsDriver, broken: TEAMS_BROKEN }],
+ teams: [{ driver: teamsDriver, broken: { ...TEAMS_BROKEN, ...SIGN_IN_LINK_POSTED_TO_THREAD } }],
"teams-dm": [{ dm: true, driver: () => teamsDriver("private"), broken: TEAMS_BROKEN }],
telegram: [{ driver: telegramDriver, broken: TELEGRAM_BROKEN }],
"telegram-dm": [{ dm: true, driver: () => telegramDriver("private"), broken: TELEGRAM_BROKEN }],
tui: [
{
driver: tuiDriver,
+ broken: {
+ ...QUEUED_BUDGET_REPLY,
+ "a sign-in names the service and shows its sign-in link": {
+ reason: "the TUI labels a sign-in with the tool name, not the challenge's displayName",
+ symptom: /Timed out waiting for the bot to show Calendar/u,
+ },
+ ...budgetPromptNotShown(
+ "a re-raised budget prompt keeps its request id, and eve/client ignores ids it has seen",
+ "pressing stop on budget prompt halts work, next message asks again",
+ "reply of stop on budget prompt halts work, next message asks again",
+ ),
+ },
unsupported: {
"pressing an option of an answered question sends it to the agent as new input":
"an answered question's drawer closes, so nothing is left to press",
@@ -181,7 +305,21 @@ const hitlConformance = {
},
},
],
- twilio: [{ driver: twilioDriver }],
+ twilio: [
+ {
+ driver: twilioDriver,
+ broken: {
+ ...QUEUED_BUDGET_REPLY,
+ ...noSignInRenderer(
+ "a sign-in names the service and shows its sign-in link",
+ "a sign-in shows its confirmation code",
+ "a sign-in without a link shows its instructions",
+ "completing a sign-in tells the person it succeeded",
+ "message after ignored sign-in tells user it was cancelled",
+ ),
+ },
+ },
+ ],
} satisfies Record<string, readonly ConformanceChannel[]>;
/**
@@ -193,20 +331,38 @@ const BROKEN_WAIT_TIMEOUT_MS = 3_000;
/** How one channel or client stands on one rule. */
type Cell =
| { readonly kind: "pass" }
- | { readonly kind: "unsupported"; readonly reason?: string }
+ | { readonly kind: "unsupported"; readonly reason: string }
| { readonly kind: "broken"; readonly broken: BrokenCell };
+const CAPABILITY_NAMES: Record<ChannelCapability, string> = {
+ buttons: "buttons a person can press",
+ "text-replies": "plain-text replies",
+};
+
+function variesByConversation(rule: (typeof hitlContract)[number]): boolean {
+ return (rule as ContractRule).variesByConversation === true;
+}
+
function cellOf(entry: ConformanceChannel, rule: (typeof hitlContract)[number]): Cell {
- if (entry.dm === true && (rule as ContractRule).variesByConversation !== true) {
+ if (entry.dm === true && !variesByConversation(rule)) {
return {
kind: "unsupported",
reason:
"it doesn't vary between a shared thread and a DM, and the shared-thread column covers it",
};
}
- const { capabilities } = entry.driver();
- if (!rule.requires.every((capability) => capabilities.includes(capability))) {
- return { kind: "unsupported" };
+ const { capabilities, surface } = entry.driver();
+ const missing = rule.requires.filter((capability) => !capabilities.includes(capability));
+ if (missing.length > 0) {
+ const names = missing.map((capability) => CAPABILITY_NAMES[capability]).join(" or ");
+ return { kind: "unsupported", reason: `the platform has no ${names}` };
+ }
+ const { surfaces } = rule as ContractRule;
+ if (surfaces !== undefined && !surfaces.includes(surface)) {
+ return {
+ kind: "unsupported",
+ reason: `the rule applies only where the conversation is ${surfaces.join(" or ")}, and this one is ${surface}`,
+ };
}
const declined = entry.unsupported?.[rule.rule];
if (declined !== undefined) return { kind: "unsupported", reason: declined };
@@ -226,11 +382,11 @@ export function describeHitlConformance(channel: keyof typeof hitlConformance):
({ entry }) => {
for (const rule of hitlContract) {
const cell = cellOf(entry, rule);
+ const { agent } = rule as ContractRule;
const run = (options?: { readonly waitTimeoutMs: number }) =>
- withChannelConversation(entry.driver(), (c) => rule.run(c), options);
+ withChannelConversation(entry.driver(), (c) => rule.run(c), { ...agent, ...options });
if (cell.kind === "unsupported") {
- const why = cell.reason === undefined ? "" : `: ${cell.reason}`;
- it.skip(`${rule.rule} (not supported${why})`, () => {});
+ it.skip(`${rule.rule} (not supported: ${cell.reason})`, () => {});
} else if (cell.kind === "pass") {
it(rule.rule, () => run(), 60_000);
} else {
@@ -256,21 +412,46 @@ const MATRIX_SYMBOLS = { broken: "❌", pass: "✅", unsupported: "—" } as con
* current whenever the suite passes.
*/
export function renderHitlConformanceMatrix(): string {
- const entries = Object.values(hitlConformance).flatMap(
+ const all = Object.values(hitlConformance).flatMap(
(group): readonly ConformanceChannel[] => group,
);
- const names = entries.map((entry) => entry.driver().name);
+ const nameOf = (entry: ConformanceChannel) => entry.driver().name;
+ const dmOf = (entry: ConformanceChannel) =>
+ all.filter((dm) => dm.dm === true && nameOf(dm) === `${nameOf(entry)}-dm`);
+ // The TUI, then Chat SDK's bridges, then channels with a DM column, then the
+ // rest. Each DM column sits right after its channel's shared-thread column.
+ const group = (entry: ConformanceChannel) => {
+ if (nameOf(entry) === "tui") return 0;
+ if (nameOf(entry).startsWith("chat-sdk")) return 1;
+ return dmOf(entry).length > 0 ? 2 : 3;
+ };
+ const entries = all
+ .filter((entry) => entry.dm !== true)
+ .toSorted((a, b) => group(a) - group(b))
+ .flatMap((entry) => [entry, ...dmOf(entry)]);
const escape = (text: string) => text.replaceAll("|", "\\|");
const row = (cells: readonly string[]) => `| ${cells.join(" | ")} |`;
- const notes = (kind: "broken" | "unsupported") =>
- entries.flatMap((entry, index) =>
- hitlContract.flatMap((rule) => {
- const cell = cellOf(entry, rule);
- if (cell.kind !== kind) return [];
- const reason = cell.kind === "broken" ? cell.broken.reason : cell.reason;
- return reason === undefined ? [] : [`- **${names[index]}**, ${rule.rule}: ${reason}`];
- }),
- );
+ // One note per distinct reason, numbered in reading order, so every cell
+ // sharing a cause links to the same note. Plain anchors rather than Markdown
+ // footnotes, which GitHub renders with links back up to every citing cell.
+ const notes = new Map<string, number>();
+ const shown = (entry: ConformanceChannel, rule: (typeof hitlContract)[number]) => {
+ // A DM column leaves blank what only the shared-thread column runs.
+ if (entry.dm === true && !variesByConversation(rule)) return "";
+ const cell = cellOf(entry, rule);
+ if (cell.kind === "pass") return MATRIX_SYMBOLS.pass;
+ const note = cell.kind === "broken" ? cell.broken.reason : cell.reason;
+ if (!notes.has(note)) notes.set(note, notes.size + 1);
+ const index = notes.get(note)!;
+ return `${MATRIX_SYMBOLS[cell.kind]}<sup>[${index}](#note-${index})</sup>`;
+ };
+ const rows = hitlContract.map((rule) =>
+ row([
+ // Non-breaking spaces keep each rule on one line; GitHub scrolls the table instead.
+ escape(rule.rule).replaceAll(" ", "\u00a0"),
+ ...entries.map((entry) => shown(entry, rule)),
+ ]),
+ );
return [
"# HITL conformance matrix",
"",
@@ -285,26 +466,15 @@ export function renderHitlConformanceMatrix(): string {
"pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conformance/matrix -u",
"```",
"",
- "✅ passes · ❌ broken · — not supported (the platform lacks a capability the rule",
- "needs, or the client declines it below)",
- "",
- row(["Rule", ...names.map((name) => `\`${name}\``)]),
- row(["---", ...names.map(() => ":---:")]),
- ...hitlContract.map((rule) =>
- row([
- // Non-breaking spaces keep each rule on one line; GitHub scrolls the table instead.
- escape(rule.rule).replaceAll(" ", "\u00a0"),
- ...entries.map((entry) => MATRIX_SYMBOLS[cellOf(entry, rule).kind]),
- ]),
- ),
- "",
- "## Broken",
+ "✅ passes · ❌ broken · — not supported",
"",
- ...notes("broken").map(escape),
+ row(["Rule", ...entries.map((entry) => `\`${nameOf(entry)}\``)]),
+ row(["---", ...entries.map(() => ":---:")]),
+ ...rows,
"",
- "## Declined",
+ "## Notes",
"",
- ...notes("unsupported").map(escape),
+ ...[...notes].map(([note, index]) => `${index}. <a id="note-${index}"></a>${note}`),
"",
].join("\n");
}
diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.ts
index 875c6b548..7778b12a5 100644
--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts
@@ -3,9 +3,13 @@ import { expect } from "vitest";
import {
type ChannelCapability,
type ChannelConversation,
+ type ConversationOptions,
GATED_TOOL,
+ PLAIN_TOOL,
type RenderedOption,
+ type Surface,
SECOND_GATED_TOOL,
+ SIGN_IN_TOOLS,
TWO_QUESTIONS_TOOL,
} from "#internal/testing/channel-conformance/harness.js";
import {
@@ -30,6 +34,10 @@ export interface ContractRule {
* these; its shared-thread column covers the rest.
*/
readonly variesByConversation?: boolean;
+ /** The surfaces the behavior is promised on; every surface when unset. */
+ readonly surfaces?: readonly Surface[];
+ /** How the test agent differs from the default for this rule. */
+ readonly agent?: Pick<ConversationOptions, "limits">;
run(conversation: ChannelConversation): Promise<void>;
}
@@ -155,6 +163,79 @@ function expectResponderNamed(conversation: ChannelConversation, prompt: string)
).toBe(true);
}
+/**
+ * A one-token input budget lets the model call that crosses it finish, so a
+ * turn that calls a tool is held before the model reads the tool's result.
+ */
+const ONE_TOKEN_BUDGET = { limits: { maxInputTokensPerSession: 1 } } as const;
+const BUDGET_PROMPT =
+ "This session has hit the input-token limit (1) per session. This is a guardrail against " +
+ "defective long-running sessions. If session activity looks fine, just approve to keep going.";
+const LOOK_UP_NOTES = `Use ${PLAIN_TOOL} to find Bob's review notes.`;
+const LATER_MESSAGE = "Carol wants the review by Friday.";
+
+/** Starts a turn that spends the budget on its first model call and is held after its tool runs. */
+async function exhaustBudget(conversation: ChannelConversation) {
+ await conversation.say(LOOK_UP_NOTES);
+ return await conversation.waitForQuestion(BUDGET_PROMPT);
+}
+
+/**
+ * Once approved, the agent replies without redoing the work it did before the
+ * prompt. A text answer's channel context (such as Telegram's message id)
+ * reaches the model after the tool result, and the test model answers that
+ * context instead of reporting the result, so this waits for any reply rather
+ * than the tool's.
+ */
+async function expectWorkContinued(conversation: ChannelConversation) {
+ await conversation.waitForReply();
+ await conversation.waitForRest();
+ expect(conversation.runsOf(PLAIN_TOOL), `${PLAIN_TOOL} ran again after approval`).toBe(1);
+}
+
+async function expectStoppedAndAskedAgain(conversation: ChannelConversation) {
+ await conversation.say(LATER_MESSAGE);
+ // Stopping keeps the session over budget, so the next message asks again.
+ const options = await conversation.waitForQuestion(BUDGET_PROMPT);
+ expect(options.map((option) => option.label).sort()).toEqual(["Approve", "Stop"]);
+ expect(conversation.replyCount(), "the bot replied after Stop").toBe(0);
+}
+
+const CALENDAR = SIGN_IN_TOOLS.read_calendar;
+const READ_CALENDAR = "Use read_calendar to check my week.";
+const MAIL = SIGN_IN_TOOLS.read_mail;
+const READ_MAIL = "Use read_mail to check my inbox.";
+const CHANGE_OF_PLANS = "Alice decided to check her week later.";
+
+async function expectSignInToolResult(conversation: ChannelConversation) {
+ const output = await conversation.waitForToolResult("read_calendar");
+ expect(output, `read_calendar returned ${JSON.stringify(output)}`).toEqual({ signedIn: true });
+ expect(conversation.runsOf("read_calendar")).toBe(1);
+}
+
+/**
+ * Asks for something that needs a sign-in, then opens the sign-in as a person
+ * would. In a group the bot may first post a status with a button that sends
+ * the sign-in to whoever asked, such as Telegram's Authorize.
+ */
+async function requestSignIn(conversation: ChannelConversation, message: string) {
+ await conversation.say(message);
+ await conversation.waitForSignIn();
+ const open = conversation.shownOptions().find((option) => SIGN_IN_OPENERS.test(option.label));
+ if (open !== undefined) await conversation.press(open);
+}
+
+const SIGN_IN_OPENERS = /^(?:authori[sz]e|sign in)$/iu;
+
+/** Starts a calendar sign-in, then moves on with a new message before finishing it. */
+async function abandonSignIn(conversation: ChannelConversation) {
+ await conversation.say(READ_CALENDAR);
+ await conversation.waitForSignIn();
+ // A person moves on once they've seen the sign-in, which in a group they reply to.
+ await conversation.waitForRest();
+ await conversation.say(CHANGE_OF_PLANS);
+}
+
export const hitlContract = [
{
rule: "a rendered question shows every option a person can choose",
@@ -440,6 +521,192 @@ export const hitlContract = [
expectResponderNamed(conversation, APPROVAL_PROMPT);
},
},
+ {
+ rule: "running out of budget opens budget prompt",
+ source: "docs/agent-config.md#runtime-limits",
+ requires: [],
+ agent: ONE_TOKEN_BUDGET,
+ async run(conversation) {
+ const options = await exhaustBudget(conversation);
+ const labels = options.map((option) => option.label).sort();
+ expect(labels, `the budget prompt showed ${JSON.stringify(labels)}`).toEqual([
+ "Approve",
+ "Stop",
+ ]);
+ expect(conversation.replyCount(), "the agent replied before anyone approved").toBe(0);
+ },
+ },
+ {
+ rule: "pressing approve on budget prompt allows agent to continue",
+ source: "docs/agent-config.md#runtime-limits",
+ requires: ["buttons"],
+ agent: ONE_TOKEN_BUDGET,
+ async run(conversation) {
+ const options = await exhaustBudget(conversation);
+ const approve = options.find((option) => option.label === "Approve");
+ expect(approve, "an Approve option to press").toBeDefined();
+ await conversation.press(approve!);
+ await expectWorkContinued(conversation);
+ },
+ },
+ {
+ rule: "reply of approve on budget prompt allows agent to continue",
+ source: "docs/agent-config.md#runtime-limits",
+ requires: ["text-replies"],
+ variesByConversation: true,
+ agent: ONE_TOKEN_BUDGET,
+ async run(conversation) {
+ await exhaustBudget(conversation);
+ await conversation.say("approve");
+ await expectWorkContinued(conversation);
+ },
+ },
+ {
+ rule: "pressing stop on budget prompt halts work, next message asks again",
+ source: "docs/agent-config.md#runtime-limits",
+ requires: ["buttons", "text-replies"],
+ variesByConversation: true,
+ agent: ONE_TOKEN_BUDGET,
+ async run(conversation) {
+ const options = await exhaustBudget(conversation);
+ const stop = options.find((option) => option.label === "Stop");
+ expect(stop, "a Stop option to press").toBeDefined();
+ await conversation.press(stop!);
+ await expectStoppedAndAskedAgain(conversation);
+ },
+ },
+ {
+ rule: "reply of stop on budget prompt halts work, next message asks again",
+ source: "docs/agent-config.md#runtime-limits",
+ requires: ["text-replies"],
+ variesByConversation: true,
+ agent: ONE_TOKEN_BUDGET,
+ async run(conversation) {
+ await exhaustBudget(conversation);
+ await conversation.say("stop");
+ await expectStoppedAndAskedAgain(conversation);
+ },
+ },
+ {
+ rule: "query sent during budget prompt answered after budget approval",
+ source: "docs/agent-config.md#runtime-limits",
+ requires: ["text-replies"],
+ variesByConversation: true,
+ agent: ONE_TOKEN_BUDGET,
+ async run(conversation) {
+ await exhaustBudget(conversation);
+ await conversation.say(LATER_MESSAGE);
+ // Had the message closed or replaced the prompt, approve would be held behind it.
+ await conversation.say("approve");
+ await conversation.waitForReplyTo(LATER_MESSAGE);
+ },
+ },
+ {
+ rule: "a sign-in names the service and shows its sign-in link",
+ source: "docs/connections/overview.mdx#self-hosted-interactive-oauth",
+ requires: [],
+ variesByConversation: true,
+ // A public surface has nowhere private to show a link.
+ surfaces: ["shared", "private"],
+ async run(conversation) {
+ await requestSignIn(conversation, READ_CALENDAR);
+ const shown = await conversation.waitForShown(CALENDAR.url);
+ // Slack names the service in the shared status and links privately, so accept either.
+ await conversation.waitForShown(CALENDAR.displayName);
+ expect(shown, "the sign-in link was shown").toContain(CALENDAR.url);
+ },
+ },
+ {
+ rule: "a sign-in shows its confirmation code",
+ source: "docs/connections/overview.mdx#self-hosted-interactive-oauth",
+ requires: [],
+ variesByConversation: true,
+ surfaces: ["shared", "private"],
+ async run(conversation) {
+ await requestSignIn(conversation, READ_CALENDAR);
+ await conversation.waitForShown(CALENDAR.userCode);
+ },
+ },
+ {
+ rule: "only the person signing in sees the sign-in link and code",
+ source:
+ "docs/channels/slack.mdx#render-and-decode-hitl-controls-yourself (a sign-in challenge is a credential)",
+ requires: [],
+ variesByConversation: true,
+ surfaces: ["public", "shared"],
+ async run(conversation) {
+ await requestSignIn(conversation, READ_CALENDAR);
+ await conversation.waitForRest();
+ const shared = conversation.sharedText();
+ expect(shared, "a message everyone sees carried the sign-in link").not.toContain(
+ CALENDAR.url,
+ );
+ expect(shared, "a message everyone sees carried the sign-in code").not.toContain(
+ CALENDAR.userCode,
+ );
+ },
+ },
+ {
+ rule: "a sign-in without a link shows its instructions",
+ source: "docs/connections/overview.mdx#self-hosted-interactive-oauth",
+ requires: [],
+ variesByConversation: true,
+ async run(conversation) {
+ await requestSignIn(conversation, READ_MAIL);
+ await conversation.waitForShown(MAIL.instructions);
+ },
+ },
+ {
+ rule: "after signing in, the agent carries on with the request",
+ source: "docs/connections/overview.mdx#interactive-oauth-via-vercel-connect",
+ requires: [],
+ async run(conversation) {
+ await conversation.say(READ_CALENDAR);
+ await conversation.waitForSignIn();
+ expect(conversation.runsOf("read_calendar"), "read_calendar ran before sign-in").toBe(0);
+ await conversation.completeSignIn();
+ await expectSignInToolResult(conversation);
+ },
+ },
+ {
+ rule: "completing a sign-in tells the person it succeeded",
+ source: "docs/connections/overview.mdx#interactive-oauth-via-vercel-connect",
+ requires: [],
+ variesByConversation: true,
+ async run(conversation) {
+ await conversation.say(READ_CALENDAR);
+ await conversation.waitForSignIn();
+ await conversation.completeSignIn();
+ await conversation.waitForShown(/\b(?:authorized|complete|connected)\b/iu);
+ },
+ },
+ {
+ rule: "message after ignored sign-in gets an answer, signing in late doesn't run",
+ source: "docs/connections/overview.mdx#interactive-oauth-via-vercel-connect",
+ requires: ["text-replies"],
+ variesByConversation: true,
+ async run(conversation) {
+ await abandonSignIn(conversation);
+ await conversation.waitForReplyTo(CHANGE_OF_PLANS);
+ // Finishing the abandoned sign-in afterwards must not run the tool.
+ await conversation.completeSignIn();
+ await conversation.waitForRest();
+ expect(
+ conversation.runsOf("read_calendar"),
+ "read_calendar ran after its sign-in was cancelled",
+ ).toBe(0);
+ },
+ },
+ {
+ rule: "message after ignored sign-in tells user it was cancelled",
+ source: "docs/connections/overview.mdx#interactive-oauth-via-vercel-connect",
+ requires: ["text-replies"],
+ variesByConversation: true,
+ async run(conversation) {
+ await abandonSignIn(conversation);
+ await conversation.waitForShown(/\b(?:cancel|declin)/iu);
+ },
+ },
] as const satisfies readonly ContractRule[];
export type HitlRule = (typeof hitlContract)[number]["rule"];
diff --git a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts b/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts
index 15737b0fa..5e632a60c 100644
--- a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts
@@ -7,6 +7,7 @@ import {
type RenderedOption,
type Surface,
recordingFetch,
+ linkTargets,
} from "#internal/testing/channel-conformance/harness.js";
let nextChannel = 0;
@@ -145,6 +146,7 @@ export function discordDriver(surface: Exclude<Surface, "public"> = "shared"): C
if (!isMessageWrite(call)) return undefined;
return {
id: (call.response as { readonly id: string }).id,
+ links: linkTargets((call.body as { readonly components?: unknown }).components),
options: choicesOf(call),
text: (call.body as { readonly content?: string }).content ?? "",
};
diff --git a/packages/eve/src/internal/testing/channel-conformance/harness.ts b/packages/eve/src/internal/testing/channel-conformance/harness.ts
index a4f605acf..42405caed 100644
--- a/packages/eve/src/internal/testing/channel-conformance/harness.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/harness.ts
@@ -8,7 +8,6 @@ import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-a
import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent-cache.js";
import { createAttachSessionFn, type Session } from "#channel/session.js";
import { attachRouteSessionCreator } from "#internal/nitro/routes/channel-route-context.js";
-import { none } from "#public/channels/auth.js";
import { eveChannel } from "#public/channels/eve.js";
import { z } from "#compiled/zod/index.js";
import { always } from "#tools/approval/policies.js";
@@ -17,6 +16,15 @@ import { askQuestion } from "#tools/provided/ask-question.js";
import { defineWorkflowTool } from "#public/tools/index.js";
import { askDayAndTimeWorkflow } from "#internal/testing/channel-conformance/two-questions-workflow.js";
import { getWorld } from "#internal/workflow/runtime.js";
+import type { MessageStreamEvent } from "#protocol/message.js";
+import {
+ type ConnectionAuthorizationChallenge,
+ ConnectionAuthorizationRequiredError,
+} from "#connections/errors.js";
+import { handleConnectionCallbackRequest } from "#execution/connections/callback-route.js";
+import type { RouteContext } from "#public/definitions/channel.js";
+import type { AgentLimitsDefinition } from "#shared/agent-definition.js";
+import { defineInteractiveAuthorization } from "#shared/connection-types.js";
/** One outbound call a channel made to its platform API. */
export interface PlatformCall {
@@ -50,6 +58,30 @@ export interface ShownMessage {
readonly text: string;
/** The choices the message still lets a person press. */
readonly options: readonly RenderedOption[];
+ /** Where the message's link buttons lead, such as a sign-in URL. */
+ readonly links?: readonly string[];
+ /**
+ * Only the person sees it, though others can see the conversation, such as
+ * Slack's ephemeral messages.
+ */
+ readonly onlyPerson?: boolean;
+}
+
+/** What a client shows a person, and whether anyone else in the conversation sees it too. */
+export interface ShownText {
+ readonly text: string;
+ readonly onlyPerson: boolean;
+ /** The choices it lets a person press. */
+ readonly options: readonly RenderedOption[];
+}
+
+/** Every `url` string anywhere in a message payload: the targets of its link buttons. */
+export function linkTargets(value: unknown): string[] {
+ if (Array.isArray(value)) return value.flatMap(linkTargets);
+ if (typeof value !== "object" || value === null) return [];
+ return Object.entries(value).flatMap(([key, child]) =>
+ key === "url" && typeof child === "string" ? [child] : linkTargets(child),
+ );
}
/**
@@ -109,7 +141,8 @@ export interface ChannelDriver {
/**
* The message one outbound call posts or edits, as a person sees it
* afterward. Required with the `buttons` capability, since rules check how
- * an answered prompt's message changes.
+ * an answered prompt's message changes. Without it, what a person sees is
+ * {@link postedText}.
*/
shownMessage?(call: PlatformCall): ShownMessage | undefined;
/** How the person driving the conversation appears in the platform's text, in any form. */
@@ -141,6 +174,23 @@ export interface ChannelConversation {
waitForToolResult(tool: string): Promise<unknown>;
/** Waits until the bot replies to input that carried `text`, however the channel framed it. */
waitForReplyTo(text: string): Promise<void>;
+ /** How many test-model replies, plain or reporting a tool result, the bot has shown. */
+ replyCount(): number;
+ /** Waits for the bot to show a test-model reply, plain or reporting a tool result. */
+ waitForReply(): Promise<void>;
+ /** Waits for the bot to show text matching `pattern`, returning the text that matched. */
+ waitForShown(pattern: string | RegExp): Promise<string>;
+ /** Everything the bot has shown that everyone in the conversation sees, joined. */
+ sharedText(): string;
+ /** Every choice the bot's messages let a person press now, newest message first. */
+ shownOptions(): readonly RenderedOption[];
+ /** Waits for the turn to hold for a sign-in, however the channel shows it. */
+ waitForSignIn(): Promise<void>;
+ /**
+ * Finishes the oldest unfinished sign-in as the provider would: by
+ * redirecting the person's browser to eve's callback URL with a code.
+ */
+ completeSignIn(): Promise<void>;
/** Waits until the bot's reply shows `tool` ran or was denied. */
waitForToolOutcome(tool: string): Promise<ToolOutcome>;
/** Waits until every session has stopped working and waits only on a person. */
@@ -155,6 +205,7 @@ export interface ChannelConversation {
shownPrompt(prompt: string): ShownMessage;
/** How the person appears in the platform's text, in any form. */
readonly personShownAs: readonly string[];
+ runsOf(tool: CountedTool): number;
}
/**
@@ -190,6 +241,8 @@ export interface ClientView {
shownPrompt?(prompt: string): ShownMessage;
/** How the person appears in the client's text, in any form. */
readonly personShownAs?: readonly string[];
+ /** Everything the client shows now that a person can read or open, one entry per message. */
+ shown(): readonly ShownText[];
/** What the client shows now, for timeout errors. */
describe(): string;
/** Stops the client and releases anything it holds, such as its event stream. */
@@ -229,6 +282,50 @@ export const SECOND_GATED_TOOL = "publish_notes";
export type GatedTool = typeof GATED_TOOL | typeof SECOND_GATED_TOOL;
+/** The test agent's plain tool: it runs without asking anyone. */
+export const PLAIN_TOOL = "look_up_notes";
+
+/** The test agent's tools that need the person to sign in to a provider before they run. */
+export const SIGN_IN_TOOLS = {
+ /** A browser sign-in that also shows a confirmation code. */
+ read_calendar: {
+ displayName: "Calendar",
+ instructions: "Sign in to let the agent read your calendar.",
+ url: "https://idp.example/authorize?client_id=eve-conformance",
+ userCode: "WDJB-MJHT",
+ },
+ /** A sign-in confirmed out of band, with nothing to open. */
+ read_mail: {
+ displayName: "Mail",
+ instructions: "Approve the sign-in request in the Mail app on your phone.",
+ },
+} as const satisfies Record<string, ConnectionAuthorizationChallenge>;
+
+export type SignInTool = keyof typeof SIGN_IN_TOOLS;
+
+/** A tool whose runs a rule can count. */
+export type CountedTool = GatedTool | SignInTool | typeof PLAIN_TOOL;
+
+/** The code the fake provider hands back when a person finishes signing in. */
+const SIGN_IN_CODE = "conformance-code";
+
+/**
+ * The person a client driver's eve channel authenticates. Sign-ins are
+ * user-scoped, so the channel must map its caller to a user principal.
+ */
+const CLIENT_PERSON = {
+ attributes: {},
+ authenticator: "conformance",
+ principalId: "alice",
+ principalType: "user",
+} as const;
+
+export interface ConversationOptions {
+ /** The test agent's session usage limits. */
+ readonly limits?: AgentLimitsDefinition;
+ readonly waitTimeoutMs?: number;
+}
+
/** The test agent's tool that asks {@link DAY_PROMPT} and {@link TIME_PROMPT} at once. */
export const TWO_QUESTIONS_TOOL = "plan_review";
@@ -261,13 +358,14 @@ export function recordingFetch(
export async function withChannelConversation(
driver: ChannelDriver | ClientDriver,
body: (conversation: ChannelConversation) => Promise<void>,
- options: { readonly waitTimeoutMs?: number } = {},
+ options: ConversationOptions = {},
): Promise<void> {
const waitTimeoutMs = options.waitTimeoutMs ?? WAIT_TIMEOUT_MS;
const wait: Wait = (label, select, describe) =>
poll(`${label} on ${driver.name}`, select, describe, waitTimeoutMs);
if (isClientDriver(driver)) {
- await converse(driver.name, "eve", eveChannel({ auth: none() }), body, wait, (dispatch) =>
+ const channel = eveChannel({ auth: () => CLIENT_PERSON });
+ await converse(driver.name, "eve", channel, body, options, wait, (dispatch) =>
openClient(driver, dispatch, wait),
);
return;
@@ -275,7 +373,7 @@ export async function withChannelConversation(
const calls: PlatformCall[] = [];
try {
const channel = driver.createChannel((call) => void calls.push(call));
- await converse(driver.name, driver.name, channel, body, wait, async (dispatch) =>
+ await converse(driver.name, driver.name, channel, body, options, wait, async (dispatch) =>
webhookView(driver, calls, dispatch, wait),
);
} finally {
@@ -318,6 +416,8 @@ function webhookView(
if (!response.ok) throw new Error(`${driver.name} webhook answered ${response.status}.`);
}
const describe = () => `Platform calls:\n${JSON.stringify(calls, null, 2)}`;
+ /** Where each prompt may next appear: after the call it was last found in. */
+ const promptsFrom = new Map<string, number>();
return {
say: (text) => post(driver.message(text)),
@@ -326,10 +426,15 @@ function webhookView(
wait(
`one of the questions ${JSON.stringify(prompts)}`,
() => {
- for (const call of calls) {
+ for (const [index, call] of calls.entries()) {
for (const prompt of prompts) {
+ if (index < (promptsFrom.get(prompt) ?? 0)) continue;
const options = driver.findOptions(call, prompt);
- if (options !== undefined) return { options, prompt };
+ if (options === undefined) continue;
+ // After a prompt is asked, an optionless match is an edit of the answered message.
+ if (promptsFrom.has(prompt) && options.length === 0) continue;
+ promptsFrom.set(prompt, index + 1);
+ return { options, prompt };
}
}
return undefined;
@@ -351,6 +456,16 @@ function webhookView(
.at(-1)!;
},
personShownAs: driver.personShownAs ?? [],
+ shown: () =>
+ calls.flatMap((call) => {
+ const shown = driver.shownMessage?.(call);
+ if (shown !== undefined) {
+ const text = [shown.text, ...(shown.links ?? [])].join("\n");
+ return [{ onlyPerson: shown.onlyPerson === true, options: shown.options, text }];
+ }
+ const text = driver.postedText(call);
+ return text === undefined ? [] : [{ onlyPerson: false, options: [], text }];
+ }),
describe,
close: async () => {},
};
@@ -392,16 +507,73 @@ async function converse(
channelName: string,
created: unknown,
body: (conversation: ChannelConversation) => Promise<void>,
+ options: ConversationOptions,
wait: Wait,
open: (dispatch: Dispatch) => Promise<ClientView>,
): Promise<void> {
if (!isCompiledChannel(created)) throw new Error(`${label} is not a compiled channel.`);
const channel: CompiledChannel = created;
- const runs: Record<GatedTool, number> = { [GATED_TOOL]: 0, [SECOND_GATED_TOOL]: 0 };
+ const runs: Record<CountedTool, number> = {
+ [GATED_TOOL]: 0,
+ [SECOND_GATED_TOOL]: 0,
+ [PLAIN_TOOL]: 0,
+ read_calendar: 0,
+ read_mail: 0,
+ };
+ /** eve's callback URL for each sign-in started, as the provider received it. */
+ const signInCallbacks: string[] = [];
+
+ function signInTool(name: SignInTool) {
+ let token: string | undefined;
+ const auth = defineInteractiveAuthorization({
+ async getToken() {
+ if (token === undefined) throw new ConnectionAuthorizationRequiredError(name);
+ return { token };
+ },
+ async startAuthorization({ callbackUrl }) {
+ signInCallbacks.push(callbackUrl);
+ return { challenge: SIGN_IN_TOOLS[name] };
+ },
+ async completeAuthorization({ callback }) {
+ if (callback.params.code !== SIGN_IN_CODE) throw new Error("Unexpected sign-in code.");
+ token = `${name}-token`;
+ return { token };
+ },
+ });
+ return {
+ logicalPath: `tools/${name}.ts`,
+ loadNamespace: async () => ({
+ default: defineTool({
+ description: `Reads the person's ${SIGN_IN_TOOLS[name].displayName}. Only call when asked to use ${name}.`,
+ async execute(_input, ctx) {
+ await ctx.getToken(auth, { authKey: name });
+ runs[name] += 1;
+ return { signedIn: true };
+ },
+ inputSchema: z.object({}),
+ }),
+ }),
+ };
+ }
const runtime = await createTestRuntime({
- agent: { name: `${label}-hitl-conformance` },
+ agent: { limits: options.limits, name: `${label}-hitl-conformance` },
modules: [
+ {
+ logicalPath: `tools/${PLAIN_TOOL}.ts`,
+ loadNamespace: async () => ({
+ default: defineTool({
+ description: `Looks up meeting notes. Only call when asked to use ${PLAIN_TOOL}.`,
+ execute: async () => {
+ runs[PLAIN_TOOL] += 1;
+ return { notes: "Bob's review notes" };
+ },
+ inputSchema: z.object({}),
+ }),
+ }),
+ },
+ signInTool("read_calendar"),
+ signInTool("read_mail"),
{
logicalPath: "tools/ask_question.ts",
loadNamespace: async () => ({ default: askQuestion() }),
@@ -515,6 +687,10 @@ async function converse(
},
view.describe,
);
+ const replyCount = () =>
+ view.replies().filter((reply) => isMockReplyTo(reply, "") || reply.startsWith("Used "))
+ .length;
+ let signInsCompleted = 0;
const conversation: ChannelConversation = {
async say(text) {
@@ -549,6 +725,52 @@ async function converse(
if (output !== undefined) return { kind: "ran", output };
return isMockDenialReply(reply) ? { kind: "denied" } : undefined;
}),
+ replyCount,
+ waitForReply: async () =>
+ void (await wait("a reply", () => (replyCount() > 0 ? true : undefined), view.describe)),
+ waitForShown: (pattern) =>
+ wait(
+ `the bot to show ${String(pattern)}`,
+ () =>
+ view
+ .shown()
+ .map(({ text }) => text)
+ .find((text) =>
+ typeof pattern === "string" ? text.includes(pattern) : pattern.test(text),
+ ),
+ view.describe,
+ ),
+ shownOptions: () =>
+ view
+ .shown()
+ .toReversed()
+ .flatMap(({ options }) => options),
+ sharedText: () =>
+ view
+ .shown()
+ .filter((shown) => !shown.onlyPerson)
+ .map(({ text }) => text)
+ .join("\n"),
+ waitForSignIn: async () =>
+ void (await wait(
+ "the turn to hold for a sign-in",
+ async () => {
+ for (const session of sessions.values()) {
+ if (await holdsFor(session, isSignIn)) return true;
+ }
+ return undefined;
+ },
+ () => "",
+ )),
+ async completeSignIn() {
+ const callbackUrl = await wait(
+ "a sign-in callback URL",
+ () => signInCallbacks.at(signInsCompleted),
+ () => "",
+ );
+ signInsCompleted += 1;
+ await deliverSignInCallback(callbackUrl);
+ },
runsOf: (tool) => runs[tool],
waitForRest: () => waitForRest([...sessions.values()], wait),
shownPrompt(prompt) {
@@ -588,7 +810,7 @@ async function converse(
`the turn to hold for "${prompt}"`,
async () => {
for (const session of sessions.values()) {
- if (await holdsFor(session, prompt)) return true;
+ if (await holdsFor(session, asks(prompt))) return true;
}
return undefined;
},
@@ -707,27 +929,41 @@ async function waitForRest(sessions: readonly Session[], wait: Wait): Promise<vo
);
}
+/** Whether an event asks the person `prompt`. */
+const asks = (prompt: string) => (event: MessageStreamEvent) =>
+ event.type === "input.requested" &&
+ event.data.requests.some((request) => request.prompt === prompt);
+
+/** Whether an event asks the person to sign in. */
+const isSignIn = (event: MessageStreamEvent) => event.type === "authorization.required";
+
/**
- * Whether the turn held for input after it last asked `prompt`. The turn emits
- * `turn.waiting` after each request, or once after requests raised together.
+ * Whether the session held for the person after it last emitted an event
+ * `asked` matches. A question, approval, or sign-in parks the open turn
+ * (`turn.waiting`) after each request, or once after requests raised
+ * together; a session-limit prompt ends the turn (`session.waiting`).
*/
-async function holdsFor(session: Session, prompt: string): Promise<boolean> {
+async function holdsFor(
+ session: Session,
+ asked: (event: MessageStreamEvent) => boolean,
+): Promise<boolean> {
const tail = await session.getStreamTailIndex();
if (tail < 0) return false;
const reader = (await session.getEventStream({ startIndex: 0 })).getReader();
- let asked = false;
+ let seen = false;
let held = false;
try {
for (let index = 0; index <= tail; index += 1) {
const { done, value } = await reader.read();
if (done) break;
- if (
- value.type === "input.requested" &&
- value.data.requests.some((request) => request.prompt === prompt)
- ) {
- asked = true;
+ if (asked(value)) {
+ seen = true;
held = false;
- } else if (asked && value.type === "turn.waiting" && value.data.on === "input") {
+ } else if (
+ seen &&
+ ((value.type === "turn.waiting" && value.data.on === "input") ||
+ value.type === "session.waiting")
+ ) {
held = true;
}
}
@@ -805,6 +1041,30 @@ function isMockReplyTo(text: string, message: string): boolean {
return text.startsWith("Bootstrap reply") && text.includes(message);
}
+/**
+ * Delivers the provider's redirect to eve's framework callback route. A
+ * callback for a sign-in eve no longer waits on is answered `404`, which a
+ * person would see as an error page; rules assert what follows instead.
+ */
+async function deliverSignInCallback(callbackUrl: string): Promise<void> {
+ const url = new URL(callbackUrl);
+ url.searchParams.set("code", SIGN_IN_CODE);
+ const segments = url.pathname.split("/");
+ const at = segments.lastIndexOf("callback");
+ const [name, attemptId, token] = [at - 1, at + 1, at + 2].map((index) =>
+ decodeURIComponent(segments[index] ?? ""),
+ );
+ const context: RouteContext = {
+ params: { attemptId, name, token } as Record<string, string>,
+ requestIp: null,
+ waitUntil: () => {},
+ };
+ const response = await handleConnectionCallbackRequest(new Request(url), context);
+ if (!response.ok && response.status !== 404) {
+ throw new Error(`The sign-in callback answered ${response.status}.`);
+ }
+}
+
/** The test model reports a denied call's `execution-denied` result in its reply. */
function isMockDenialReply(text: string): boolean {
return text.includes('"type":"execution-denied"');
diff --git a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts
index e27f2a747..dff6b580e 100644
--- a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts
@@ -3,6 +3,7 @@ import {
type PlatformCall,
type RenderedOption,
recordingFetch,
+ linkTargets,
} from "#internal/testing/channel-conformance/harness.js";
import { linearChannel } from "#public/channels/linear/index.js";
import { signLinearWebhookBody } from "#public/channels/linear/verify.js";
@@ -112,11 +113,31 @@ export function linearDriver(): ChannelDriver {
.variables?.input;
return input?.content?.type === "response" ? input.content.body : undefined;
},
+ shownMessage(call) {
+ if (call.method !== "AgentActivityCreate") return undefined;
+ const input = (call.body as { readonly variables?: { readonly input?: LinearActivityInput } })
+ .variables?.input;
+ if (input?.content === undefined) return undefined;
+ const response = call.response as {
+ readonly data: {
+ readonly agentActivityCreate: { readonly agentActivity: { readonly id: string } };
+ };
+ };
+ return {
+ id: response.data.agentActivityCreate.agentActivity.id,
+ // An `auth` elicitation's sign-in link rides in its signal metadata.
+ links: linkTargets(input.signalMetadata),
+ // The driver can't press Linear's native select, so nothing here is pressable.
+ options: [],
+ text: input.content.body,
+ };
+ },
};
}
interface LinearActivityInput {
readonly content?: { readonly body: string; readonly type: string };
+ readonly signalMetadata?: unknown;
}
function parseVisibleOptions(body: string): RenderedOption[] {
diff --git a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts
index 68e1a03d9..f76c82d15 100644
--- a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts
@@ -23,6 +23,8 @@ interface LinqPart {
interface LinqMessageBody {
readonly message?: { readonly parts?: readonly LinqPart[] };
+ /** An edit's replacement text. */
+ readonly text?: string;
}
/** Drives the real Linq adapter through signed webhooks, in a group chat by default or 1:1. */
@@ -31,6 +33,7 @@ export function linqDriver(surface: Exclude<Surface, "public"> = "shared"): Chan
nextChat += 1;
const chatId = `linq-conformance-chat-${nextChat}`;
let messageId = 0;
+ let outbound = 0;
let restoreFetch: (() => void) | undefined;
function signedMessage(text: string): Request {
@@ -72,10 +75,13 @@ export function linqDriver(surface: Exclude<Surface, "public"> = "shared"): Chan
const fakeFetch = recordingFetch(record, async (request) => {
const url = new URL(request.url);
const bodyText = await request.text();
+ outbound += 1;
+ const id = `linq-outbound-${outbound}`;
return {
body: bodyText === "" ? {} : JSON.parse(bodyText),
method: `${request.method} ${url.pathname}`,
- response: {},
+ // The adapter reads the sent or edited message's id back to edit it later.
+ response: { chat_id: chatId, id, message: { id } },
};
});
// The adapter accepts no fetch option, so route only its test host globally and preserve other fetches.
@@ -110,9 +116,12 @@ export function linqDriver(surface: Exclude<Surface, "public"> = "shared"): Chan
};
}
+/** Text the bot sent (`POST …/messages`) or edited a message to (`PATCH /messages/:id`). */
function postedText(call: PlatformCall): string | undefined {
+ const body = call.body as LinqMessageBody;
+ if (call.method.startsWith("PATCH ") && call.method.includes("/messages/")) return body.text;
if (!call.method.endsWith("/messages")) return undefined;
- const parts = (call.body as LinqMessageBody).message?.parts;
+ const parts = body.message?.parts;
return parts
?.filter((part) => part.type === "text")
.map((part) => part.value ?? "")
diff --git a/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts b/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts
index f6ed510e4..f57f5aa7e 100644
--- a/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts
@@ -8,6 +8,7 @@ import {
type PlatformCall,
type RenderedOption,
recordingFetch,
+ linkTargets,
} from "#internal/testing/channel-conformance/harness.js";
import { decodeSlackApiBody } from "#internal/testing/slack-api-body.js";
@@ -45,6 +46,8 @@ interface PressHandle {
interface SlackMessageBody {
readonly blocks?: readonly unknown[];
+ /** Replies post Markdown here, with `text` as the plain fallback. */
+ readonly markdown_text?: string;
readonly text?: string;
readonly ts?: string;
}
@@ -161,8 +164,10 @@ export function slackDriver(surface: Exclude<Surface, "public"> = "shared"): Cha
const id = messageTsOf(call);
return {
id,
+ links: linkTargets(body.blocks ?? []),
+ onlyPerson: call.method === "chat.postEphemeral",
options: hitlOptions(body.blocks ?? [], id),
- text: [body.text ?? "", ...blockTexts(body.blocks ?? [])].join("\n"),
+ text: [body.markdown_text ?? body.text ?? "", ...blockTexts(body.blocks ?? [])].join("\n"),
};
},
personShownAs: [`<@${PERSON}>`],
@@ -195,8 +200,13 @@ export function slackDriver(surface: Exclude<Surface, "public"> = "shared"): Cha
};
}
+/** A message write a person sees; an ephemeral one only they see, such as a private sign-in. */
function isMessageWrite(call: PlatformCall): boolean {
- return call.method === "chat.postMessage" || call.method === "chat.update";
+ return (
+ call.method === "chat.postMessage" ||
+ call.method === "chat.postEphemeral" ||
+ call.method === "chat.update"
+ );
}
/** A post's ts comes back from Slack; an update names the ts it rewrites. */
diff --git a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts b/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts
index 4314b1d33..dc3776f55 100644
--- a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts
@@ -5,6 +5,7 @@ import {
type RenderedOption,
type Surface,
recordingFetch,
+ linkTargets,
} from "#internal/testing/channel-conformance/harness.js";
let nextConversation = 0;
@@ -132,6 +133,7 @@ export function teamsDriver(surface: Exclude<Surface, "public"> = "shared"): Cha
);
return {
id: activityIdOf(call),
+ links: linkTargets(adaptiveCard(body)?.actions),
options: cardOptions(call),
text: [body.text ?? "", ...texts].join("\n"),
};
diff --git a/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts b/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts
index b704bfe1c..20f317a5b 100644
--- a/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts
@@ -5,6 +5,7 @@ import {
type RenderedOption,
type Surface,
recordingFetch,
+ linkTargets,
} from "#internal/testing/channel-conformance/harness.js";
const SECRET = "telegram-conformance-secret";
@@ -17,6 +18,8 @@ interface InlineButton {
}
interface MessageBody {
+ /** Shows the message to one member of a group only. */
+ readonly ephemeral_message_parameters?: unknown;
readonly message_id?: number;
readonly reply_markup?: { readonly inline_keyboard?: readonly InlineButton[][] };
readonly text?: string;
@@ -138,6 +141,8 @@ export function telegramDriver(surface: Exclude<Surface, "public"> = "shared"):
if (!MESSAGE_WRITES.has(call.method)) return undefined;
return {
id: String(messageIdOf(call)),
+ links: linkTargets((call.body as { readonly reply_markup?: unknown }).reply_markup),
+ onlyPerson: (call.body as MessageBody).ephemeral_message_parameters !== undefined,
// Telegram drops a message's inline keyboard when an edit omits `reply_markup`.
options: inlineOptions(call),
text: (call.body as MessageBody).text ?? "",
diff --git a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts b/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts
index 63fdd4520..80b9cfbb8 100644
--- a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts
@@ -26,8 +26,8 @@ const FREEFORM_ROW = "Type your own answer…";
export function tuiDriver(): ClientDriver {
return {
name: "tui",
- capabilities: ["buttons", "text-replies"],
// A local terminal shows only the person at it.
+ capabilities: ["buttons", "text-replies"],
surface: "private",
async open(host, wait) {
// Wide enough that no reply wraps, so each one stays on one screen row.
@@ -98,6 +98,7 @@ export function tuiDriver(): ClientDriver {
focusedRow(screen, prompt) === undefined ? [] : readOptions(screen, keyboard, prompt),
text: screen.snapshot(),
}),
+ shown: () => [{ onlyPerson: true, options: [], text: screen.snapshot() }],
describe,
async close() {
renderer.requestInterrupt();