Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

[5/7] test(eve): run the HITL conformance suite in shared threads and DMs (#4253)

Ben Pankow886ee32bd6f0
diff --git a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
index 8c0d336b1..d7d8299e6 100644
--- a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
+++ b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md
@@ -14,32 +14,32 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform
✅ passes · ❌ broken · — not supported (the platform lacks a capability the rule
needs, or the client declines it below)
-| Rule | `chat-sdk` | `chat-sdk-text` | `discord` | `github` | `linear` | `linq` | `slack` | `teams` | `telegram` | `tui` | `twilio` |
-| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
-| a rendered question shows every option a person can choose | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| pressing a rendered option answers the pending question with that option | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |
-| a text reply matching an option answers the only pending question | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| a text reply that matches no option answers the question with the person's words | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| a text reply answers an open-ended question with the person's words | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| pressing an option of an answered question sends it to the agent as new input | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | — | — |
-| pressing options of two pending questions answers each with its own option | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |
-| a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-| pressing Approve runs the gated tool | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |
-| pressing Approve twice runs the gated tool once | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |
-| pressing Approve on one of two pending approvals runs only that tool | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |
-| answering an approval and a question pending together settles both | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |
-| pressing Cancel stops the gated tool without running it | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |
-| a text reply of approve runs the gated tool | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |
-| a text reply of cancel stops the gated tool without running it | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |
-| pressing an option clears the question's buttons | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |
-| answering a question by text clears its buttons | ✅ | — | — | — | — | — | ❌ | ✅ | ✅ | ✅ | — |
-| pressing an option names who answered on the question | ❌ | — | ❌ | — | — | — | ✅ | ❌ | ❌ | — | — |
-| answering a question by text names who answered on the question | ❌ | — | — | — | — | — | ❌ | ❌ | ❌ | — | — |
-| pressing Approve clears the approval's buttons | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |
-| approving by text clears the approval's buttons | ✅ | — | — | — | — | — | ✅ | ✅ | ✅ | — | — |
-| pressing Approve names who approved on the approval | ❌ | — | ❌ | — | — | — | ✅ | ✅ | ❌ | — | — |
-| approving by text names who approved on the approval | ❌ | — | — | — | — | — | ❌ | ❌ | ❌ | — | — |
+| Rule | `chat-sdk` | `chat-sdk-text` | `chat-sdk-dm` | `discord` | `discord-dm` | `github` | `linear` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `tui` | `twilio` |
+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |
+| a rendered question shows every option a person can choose | ✅ | ✅ | — | ✅ | — | ✅ | ✅ | ✅ | — | ✅ | — | ✅ | — | ✅ | — | ✅ | ✅ |
+| pressing a rendered option answers the pending question with that option | ✅ | — | ✅ | ✅ | ✅ | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — |
+| a text reply matching an option answers the only pending question | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| a text reply that matches no option answers the question with the person's words | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| a text reply answers an open-ended question with the person's words | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| pressing an option of an answered question sends it to the agent as new input | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | — | — |
+| pressing options of two pending questions answers each with its own option | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
+| a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | — | ✅ | — | ✅ | ✅ | ✅ | — | ✅ | — | ✅ | — | ✅ | — | ✅ | ✅ |
+| pressing Approve runs the gated tool | ✅ | — | ✅ | ✅ | ✅ | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — |
+| pressing Approve twice runs the gated tool once | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
+| pressing Approve on one of two pending approvals runs only that tool | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
+| answering an approval and a question pending together settles both | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
+| pressing Cancel stops the gated tool without running it | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
+| a text reply of approve runs the gated tool | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |
+| a text reply of cancel stops the gated tool without running it | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |
+| pressing an option clears the question's buttons | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
+| answering a question by text clears its buttons | ✅ | — | ✅ | — | — | — | — | — | — | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | — |
+| pressing an option names who answered on the question | ❌ | — | — | ❌ | — | — | — | — | — | ✅ | — | ❌ | — | ❌ | — | — | — |
+| answering a question by text names who answered on the question | ❌ | — | ❌ | — | — | — | — | — | — | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | — | — |
+| pressing Approve clears the approval's buttons | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |
+| approving by text clears the approval's buttons | ✅ | — | ✅ | — | — | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | — |
+| pressing Approve names who approved on the approval | ❌ | — | — | ❌ | — | — | — | — | — | ✅ | — | ✅ | — | ❌ | — | — | — |
+| approving by text names who approved on the approval | ❌ | — | ❌ | — | — | — | — | — | — | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | — | — |
## Broken
@@ -47,21 +47,102 @@ needs, or the client declines it below)
- **chat-sdk**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **chat-sdk**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **chat-sdk**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
+- **chat-sdk-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
+- **chat-sdk-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **discord**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **discord**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **slack**, answering a question by text clears its buttons: only the button interaction handler edits a question; a typed answer leaves it
- **slack**, answering a question by text names who answered on the question: only the button interaction handler edits a question; a typed answer leaves it
- **slack**, approving by text names who approved on the approval: the card loses its buttons after a typed approval but doesn't say who approved
+- **slack-dm**, answering a question by text clears its buttons: only the button interaction handler edits a question; a typed answer leaves it
+- **slack-dm**, answering a question by text names who answered on the question: only the button interaction handler edits a question; a typed answer leaves it
+- **slack-dm**, approving by text names who approved on the approval: the card loses its buttons after a typed approval but doesn't say who approved
- **teams**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **teams**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **teams**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
+- **teams-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
+- **teams-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **telegram**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **telegram**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **telegram**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
- **telegram**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
+- **telegram-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder
+- **telegram-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder
## Declined
+- **chat-sdk-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **chat-sdk-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **discord-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **linq-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **slack-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **teams-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
+- **telegram-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it
- **tui**, pressing an option of an answered question sends it to the agent as new input: an answered question's drawer closes, so nothing is left to press
- **tui**, a text reply of approve runs the gated tool: the approval drawer holds the keyboard; a person answers it with y or n
- **tui**, a text reply of cancel stops the gated tool without running it: the approval drawer holds the keyboard; a person answers it with y or n
diff --git a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts
index f35b16ffa..43cade21f 100644
--- a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts
@@ -14,6 +14,7 @@ import { createMemoryState } from "#compiled/@chat-adapter/state-memory/index.js
import {
type ChannelDriver,
type PlatformCall,
+ type Surface,
numberedOptions,
} from "#internal/testing/channel-conformance/harness.js";
@@ -60,16 +61,21 @@ function buttonsOf(call: PlatformCall, prompt?: string): { handle: PressHandle;
}
/**
- * Drives `chatSdkChannel` with a card-capable direct-message adapter: one
- * thread, no streaming, and every message handed to eve with an empty
- * `context`. The fake adapter is the platform: it reads inbound JSON and
- * records every post and edit.
+ * Drives `chatSdkChannel` with a card-capable adapter: one thread, in a
+ * workspace channel by default or a direct message, no streaming, and every
+ * message handed to eve with an empty `context`. The fake adapter is the
+ * platform: it reads inbound JSON and records every post and edit.
*/
-export function chatSdkDriver(): ChannelDriver {
- const driver = chatSdkDriverWith({ name: "chat-sdk", render: (posted) => posted });
+export function chatSdkDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {
+ const driver = chatSdkDriverWith({
+ name: surface === "private" ? "chat-sdk-dm" : "chat-sdk",
+ render: (posted) => posted,
+ surface,
+ });
return {
...driver,
capabilities: ["buttons", "text-replies"],
+ surface,
findOptions(call, prompt) {
if (!isPost(call)) return undefined;
const card = cardOf(call.body as AdapterPostableMessage);
@@ -110,10 +116,12 @@ export function chatSdkTextDriver(): ChannelDriver {
const driver = chatSdkDriverWith({
name: "chat-sdk-text",
render: (posted) => converter.renderPostable(posted),
+ surface: "private",
});
return {
...driver,
capabilities: ["text-replies"],
+ surface: "private",
findOptions(call, prompt) {
if (!isPost(call) || typeof call.body !== "string" || !call.body.includes(prompt)) {
return undefined;
@@ -139,7 +147,8 @@ class PlainTextConverter extends BaseFormatConverter {
function chatSdkDriverWith(input: {
readonly name: string;
readonly render: (posted: AdapterPostableMessage) => unknown;
-}): Omit<ChannelDriver, "capabilities" | "findOptions" | "press"> & {
+ readonly surface: Exclude<Surface, "public">;
+}): Omit<ChannelDriver, "capabilities" | "findOptions" | "press" | "surface"> & {
inbound(body: Inbound): Request;
} {
nextThread += 1;
@@ -158,7 +167,8 @@ function chatSdkDriverWith(input: {
name: input.name,
inbound,
createChannel(record) {
- const adapter = fakeAdapter(threadId, record, () => (sequence += 1), input.render);
+ const dm = input.surface === "private";
+ const adapter = fakeAdapter(threadId, record, () => (sequence += 1), input.render, dm);
const bridge = chatSdkChannel({
adapters: { [ADAPTER]: adapter },
concurrency: "concurrent",
@@ -167,9 +177,21 @@ function chatSdkDriverWith(input: {
streaming: false,
userName: "eve",
});
- bridge.bot.onDirectMessage(async (thread, message) => {
- await bridge.send(message.text, { auth: null, context: [], thread });
- });
+ if (dm) {
+ bridge.bot.onDirectMessage(async (thread, message) => {
+ await bridge.send(message.text, { auth: null, context: [], thread });
+ });
+ } else {
+ // The wiring docs/channels/chat-sdk.mdx shows: a mention starts the session, and
+ // subscribing lets the rest of the thread continue it without one.
+ bridge.bot.onNewMention(async (thread, message) => {
+ await thread.subscribe();
+ await bridge.send(message.text, { auth: null, context: [], thread });
+ });
+ bridge.bot.onSubscribedMessage(async (thread, message) => {
+ await bridge.send(message.text, { auth: null, context: [], thread });
+ });
+ }
return bridge.channel;
},
message: (text) => inbound({ kind: "message", text }),
@@ -193,7 +215,9 @@ function fakeAdapter(
record: (call: PlatformCall) => void,
nextId: () => number,
render: (posted: AdapterPostableMessage) => unknown,
+ dm: boolean,
): Adapter {
+ const visibility = dm ? ("private" as const) : ("workspace" as const);
let chat: ChatInstance | null = null;
const self = {
name: ADAPTER,
@@ -221,7 +245,8 @@ function fakeAdapter(
await chat?.processMessage(
adapter,
threadId,
- inboundMessage(threadId, id, body.text),
+ // A person mentions the bot to start a channel thread; Chat routes the rest by subscription.
+ inboundMessage(threadId, id, body.text, !dm),
options,
);
}
@@ -230,16 +255,16 @@ function fakeAdapter(
channelIdFromThreadId: () => threadId,
decodeThreadId: (id: string) => ({ threadId: id }),
encodeThreadId: (input: { threadId: string }) => input.threadId,
- getChannelVisibility: () => "private" as const,
- isDM: () => true,
+ getChannelVisibility: () => visibility,
+ isDM: () => dm,
parseMessage: (raw: { text?: string }) => inboundMessage(threadId, "parsed", raw.text ?? ""),
renderFormatted: () => "",
fetchMessages: async () => ({ messages: [] }),
fetchThread: async (id: string) => ({
channelId: threadId,
- channelVisibility: "private" as const,
+ channelVisibility: visibility,
id,
- isDM: true,
+ isDM: dm,
metadata: {},
}),
async postMessage(id: string, posted: AdapterPostableMessage) {
@@ -261,13 +286,13 @@ function fakeAdapter(
return adapter;
}
-function inboundMessage(threadId: string, id: string, text: string): Message {
+function inboundMessage(threadId: string, id: string, text: string, isMention = false): Message {
return new Message({
attachments: [],
author: PERSON,
formatted: parseMarkdown(text),
id,
- isMention: false,
+ isMention,
metadata: { dateSent: new Date("2026-01-01T00:00:00.000Z"), edited: false },
raw: { text },
text,
diff --git a/packages/eve/src/internal/testing/channel-conformance/conformance.ts b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
index 98cbd779a..8f598ce8d 100644
--- a/packages/eve/src/internal/testing/channel-conformance/conformance.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
@@ -1,6 +1,10 @@
import { describe, expect, it } from "vitest";
-import { type HitlRule, hitlContract } from "#internal/testing/channel-conformance/contract.js";
+import {
+ type ContractRule,
+ type HitlRule,
+ hitlContract,
+} from "#internal/testing/channel-conformance/contract.js";
import {
type ChannelDriver,
type ClientDriver,
@@ -81,11 +85,55 @@ function unnamedAnsweredPrompts(
interface ConformanceChannel {
readonly driver: () => ChannelDriver | ClientDriver;
+ /**
+ * A DM column beside the channel's shared-thread column. It runs only rules
+ * whose behavior varies by conversation; the shared column covers the rest.
+ */
+ readonly dm?: true;
readonly broken?: Partial<Record<HitlRule, BrokenCell>>;
/** Rules this client deliberately doesn't offer, with why, beyond what its capabilities rule out. */
readonly unsupported?: Partial<Record<HitlRule, string>>;
}
+const UNNAMED_RESPONDER =
+ "a resolved prompt doesn't say who answered; input.resolved carries no responder";
+
+const CHAT_SDK_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
+ "approvalPress",
+ "approvalText",
+ "questionPress",
+ "questionText",
+]);
+
+const DISCORD_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
+ "approvalPress",
+ "questionPress",
+]);
+
+const SLACK_BROKEN = {
+ ...staleAnsweredPrompts(
+ "only the button interaction handler edits a question; a typed answer leaves it",
+ ["questionText"],
+ ),
+ "approving by text names who approved on the approval": {
+ reason: "the card loses its buttons after a typed approval but doesn't say who approved",
+ symptom: /the answered prompt never names who answered/,
+ },
+} satisfies Partial<Record<HitlRule, BrokenCell>>;
+
+const TEAMS_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
+ "approvalText",
+ "questionPress",
+ "questionText",
+]);
+
+const TELEGRAM_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [
+ "approvalPress",
+ "approvalText",
+ "questionPress",
+ "questionText",
+]);
+
const TUI_TYPED_APPROVAL =
"the approval drawer holds the keyboard; a person answers it with y or n";
const TUI_SINGLE_PERSON = "one person answers at their own terminal; there's nobody else to tell";
@@ -103,61 +151,20 @@ const TUI_SINGLE_PERSON = "one person answers at their own terminal; there's nob
* failure (such as harness breakage) both turn it red.
*/
const hitlConformance = {
- "chat-sdk": [
- {
- driver: chatSdkDriver,
- broken: unnamedAnsweredPrompts(
- "a resolved prompt doesn't say who answered; input.resolved carries no responder",
- ["approvalPress", "approvalText", "questionPress", "questionText"],
- ),
- },
- { driver: chatSdkTextDriver },
- ],
- discord: [
- {
- driver: discordDriver,
- broken: unnamedAnsweredPrompts(
- "a resolved prompt doesn't say who answered; input.resolved carries no responder",
- ["approvalPress", "questionPress"],
- ),
- },
- ],
+ "chat-sdk": [{ driver: chatSdkDriver, broken: CHAT_SDK_BROKEN }, { driver: chatSdkTextDriver }],
+ "chat-sdk-dm": [{ dm: true, driver: () => chatSdkDriver("private"), broken: CHAT_SDK_BROKEN }],
+ discord: [{ driver: discordDriver, broken: DISCORD_BROKEN }],
+ "discord-dm": [{ dm: true, driver: () => discordDriver("private"), broken: DISCORD_BROKEN }],
github: [{ driver: githubDriver }],
linear: [{ driver: linearDriver }],
linq: [{ driver: linqDriver }],
- slack: [
- {
- driver: slackDriver,
- broken: {
- ...staleAnsweredPrompts(
- "only the button interaction handler edits a question; a typed answer leaves it",
- ["questionText"],
- ),
- "approving by text names who approved on the approval": {
- reason: "the card loses its buttons after a typed approval but doesn't say who approved",
- symptom: /the answered prompt never names who answered/,
- },
- },
- },
- ],
- teams: [
- {
- driver: teamsDriver,
- broken: unnamedAnsweredPrompts(
- "a resolved prompt doesn't say who answered; input.resolved carries no responder",
- ["approvalText", "questionPress", "questionText"],
- ),
- },
- ],
- telegram: [
- {
- driver: telegramDriver,
- broken: unnamedAnsweredPrompts(
- "a resolved prompt doesn't say who answered; input.resolved carries no responder",
- ["approvalPress", "approvalText", "questionPress", "questionText"],
- ),
- },
- ],
+ "linq-dm": [{ dm: true, driver: () => linqDriver("private") }],
+ slack: [{ driver: slackDriver, broken: SLACK_BROKEN }],
+ "slack-dm": [{ dm: true, driver: () => slackDriver("private"), broken: SLACK_BROKEN }],
+ teams: [{ driver: teamsDriver, broken: TEAMS_BROKEN }],
+ "teams-dm": [{ dm: true, driver: () => teamsDriver("private"), broken: TEAMS_BROKEN }],
+ telegram: [{ driver: telegramDriver, broken: TELEGRAM_BROKEN }],
+ "telegram-dm": [{ dm: true, driver: () => telegramDriver("private"), broken: TELEGRAM_BROKEN }],
tui: [
{
driver: tuiDriver,
@@ -190,6 +197,13 @@ type Cell =
| { readonly kind: "broken"; readonly broken: BrokenCell };
function cellOf(entry: ConformanceChannel, rule: (typeof hitlContract)[number]): Cell {
+ if (entry.dm === true && (rule as ContractRule).variesByConversation !== true) {
+ return {
+ kind: "unsupported",
+ reason:
+ "it doesn't vary between a shared thread and a DM, and the shared-thread column covers it",
+ };
+ }
const { capabilities } = entry.driver();
if (!rule.requires.every((capability) => capabilities.includes(capability))) {
return { kind: "unsupported" };
diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.ts
index 4da04aacb..875c6b548 100644
--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts
@@ -23,6 +23,13 @@ export interface ContractRule {
/** Where the behavior is promised: a docs anchor or the PR that introduced it. */
readonly source: string;
readonly requires: readonly ChannelCapability[];
+ /**
+ * The behavior can differ between a shared thread and a DM, such as a typed
+ * answer that must mention the bot in a channel, or a press whose presser a
+ * platform names differently in a server. A channel's DM column runs only
+ * these; its shared-thread column covers the rest.
+ */
+ readonly variesByConversation?: boolean;
run(conversation: ChannelConversation): Promise<void>;
}
@@ -165,6 +172,7 @@ export const hitlContract = [
rule: "pressing a rendered option answers the pending question with that option",
source: "docs/tools/human-in-the-loop.md#answering-from-a-client-or-channel",
requires: ["buttons"],
+ variesByConversation: true,
async run(conversation) {
const options = await askWhichDay(conversation);
const saturday = options.find((option) => option.label === "Saturday");
@@ -177,6 +185,7 @@ export const hitlContract = [
rule: "a text reply matching an option answers the only pending question",
source: "docs/tools/human-in-the-loop.md#how-pause-and-resume-works",
requires: ["text-replies"],
+ variesByConversation: true,
async run(conversation) {
await askWhichDay(conversation);
await conversation.say("Saturday");
@@ -187,6 +196,7 @@ export const hitlContract = [
rule: "a text reply that matches no option answers the question with the person's words",
source: "docs/tools/human-in-the-loop.md#questions",
requires: ["text-replies"],
+ variesByConversation: true,
async run(conversation) {
await askWhichDay(conversation);
await conversation.say(OWN_WORDS);
@@ -197,6 +207,7 @@ export const hitlContract = [
rule: "a text reply answers an open-ended question with the person's words",
source: "docs/tools/human-in-the-loop.md#questions",
requires: ["text-replies"],
+ variesByConversation: true,
async run(conversation) {
await conversation.say(ASK_OPEN);
await conversation.waitForQuestion(OPEN_PROMPT);
@@ -236,6 +247,7 @@ export const hitlContract = [
rule: "a text reply matching an option does not answer either of two pending questions",
source: "docs/tools/human-in-the-loop.md#how-pause-and-resume-works",
requires: ["text-replies"],
+ variesByConversation: true,
async run(conversation) {
await conversation.say(PLAN_REVIEW);
await conversation.waitForRequest(DAY_PROMPT);
@@ -268,6 +280,7 @@ export const hitlContract = [
rule: "pressing Approve runs the gated tool",
source: "docs/tools/human-in-the-loop.md#approvals",
requires: ["buttons"],
+ variesByConversation: true,
async run(conversation) {
const options = await askToDeploy(conversation);
const approve = options.find((option) => APPROVE_LABELS.includes(option.label));
@@ -333,6 +346,7 @@ export const hitlContract = [
rule: "a text reply of approve runs the gated tool",
source: "docs/tools/human-in-the-loop.md#how-pause-and-resume-works",
requires: ["text-replies"],
+ variesByConversation: true,
async run(conversation) {
await askToDeploy(conversation);
await conversation.say("approve");
@@ -343,6 +357,7 @@ export const hitlContract = [
rule: "a text reply of cancel stops the gated tool without running it",
source: "docs/tools/human-in-the-loop.md#how-pause-and-resume-works",
requires: ["text-replies"],
+ variesByConversation: true,
async run(conversation) {
await askToDeploy(conversation);
await conversation.say("cancel");
@@ -362,6 +377,7 @@ export const hitlContract = [
rule: "answering a question by text clears its buttons",
source: "#1 (Slack's answered question card)",
requires: ["buttons", "text-replies"],
+ variesByConversation: true,
async run(conversation) {
await answerSaturday(conversation, "text");
expectButtonsCleared(conversation, PROMPT);
@@ -380,6 +396,7 @@ export const hitlContract = [
rule: "answering a question by text names who answered on the question",
source: "#1 (Slack's answered question card)",
requires: ["buttons", "text-replies"],
+ variesByConversation: true,
async run(conversation) {
await answerSaturday(conversation, "text");
expectResponderNamed(conversation, PROMPT);
@@ -398,6 +415,7 @@ export const hitlContract = [
rule: "approving by text clears the approval's buttons",
source: "#2212 (Slack's settled approval card)",
requires: ["buttons", "text-replies"],
+ variesByConversation: true,
async run(conversation) {
await approveDeploy(conversation, "text");
expectButtonsCleared(conversation, APPROVAL_PROMPT);
@@ -416,6 +434,7 @@ export const hitlContract = [
rule: "approving by text names who approved on the approval",
source: "#2212 (Slack's settled approval card)",
requires: ["buttons", "text-replies"],
+ variesByConversation: true,
async run(conversation) {
await approveDeploy(conversation, "text");
expectResponderNamed(conversation, APPROVAL_PROMPT);
diff --git a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts b/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts
index e440a6d5d..15737b0fa 100644
--- a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts
@@ -5,6 +5,7 @@ import {
type ChannelDriver,
type PlatformCall,
type RenderedOption,
+ type Surface,
recordingFetch,
} from "#internal/testing/channel-conformance/harness.js";
@@ -35,8 +36,16 @@ function testKeys(): { privateKey: KeyObject; publicKeyHex: string } {
};
}
-/** Drives Discord's signed application-command and component interaction webhooks. */
-export function discordDriver(): ChannelDriver {
+/**
+ * Drives Discord's signed application-command and component interaction
+ * webhooks, in a server text channel by default or in a DM.
+ */
+export function discordDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {
+ const dm = surface === "private";
+ // In a server, Discord names the actor as a member; in a DM, as a user.
+ const where = dm
+ ? { channel: { type: 1 }, user: PERSON }
+ : { channel: { type: 0 }, guild_id: "G_CONFORMANCE", member: { roles: [], user: PERSON } };
nextChannel += 1;
const channelId = `C_CONFORMANCE_${nextChannel}`;
const { privateKey, publicKeyHex } = testKeys();
@@ -100,8 +109,9 @@ export function discordDriver(): ChannelDriver {
}
return {
- name: "discord",
+ name: dm ? "discord-dm" : "discord",
capabilities: ["buttons"],
+ surface,
createChannel: (record) =>
discordChannel({
api: { fetch: recordingFetch(record, decode) },
@@ -111,14 +121,13 @@ export function discordDriver(): ChannelDriver {
const id = nextInteraction();
return signed(
JSON.stringify({
+ ...where,
application_id: "APP1",
- channel: { type: 1 },
channel_id: channelId,
data: { name: "ask", options: [{ name: "message", type: 3, value: text }] },
id,
token: `tok-${id}`,
type: 2,
- user: PERSON,
version: 1,
}),
);
@@ -155,10 +164,10 @@ export function discordDriver(): ChannelDriver {
? { component_type: 2, custom_id: handle.customId }
: { component_type: 3, custom_id: handle.customId, values: [handle.value] },
id,
+ ...where,
message: { id: handle.messageId },
token: `tok-${id}`,
type: 3,
- user: PERSON,
version: 1,
}),
);
diff --git a/packages/eve/src/internal/testing/channel-conformance/github-driver.ts b/packages/eve/src/internal/testing/channel-conformance/github-driver.ts
index 96aad056c..a1da551be 100644
--- a/packages/eve/src/internal/testing/channel-conformance/github-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/github-driver.ts
@@ -64,6 +64,7 @@ export function githubDriver(): ChannelDriver {
return {
name: "github",
capabilities: ["text-replies"],
+ surface: "public",
createChannel: (record) =>
githubChannel({
api: { fetch: recordingFetch(record, decode) },
diff --git a/packages/eve/src/internal/testing/channel-conformance/harness.ts b/packages/eve/src/internal/testing/channel-conformance/harness.ts
index 3dbd4193f..a4f605acf 100644
--- a/packages/eve/src/internal/testing/channel-conformance/harness.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/harness.ts
@@ -62,6 +62,19 @@ export type ChannelCapability =
/** A person can send a plain-text message to the conversation. */
| "text-replies";
+/**
+ * Who can see the conversation a driver holds. Anything meant for the person
+ * alone must reach them privately on a shared surface and can't be shown at
+ * all on a public one.
+ */
+export type Surface =
+ /** Anyone, such as a GitHub issue. */
+ | "public"
+ /** Members of a channel, group, or workspace, such as a Slack channel thread. Most agents run here. */
+ | "shared"
+ /** Only the person, such as a direct message or a local terminal. */
+ | "private";
+
/**
* Teaches the HITL conformance suite to speak one channel's platform protocol.
*
@@ -72,6 +85,7 @@ export type ChannelCapability =
export interface ChannelDriver {
readonly name: string;
readonly capabilities: readonly ChannelCapability[];
+ readonly surface: Surface;
/**
* Builds the channel against a fake platform that reports each outbound call
* to `record`. HTTP platforms use {@link recordingFetch}.
@@ -151,6 +165,7 @@ export interface ChannelConversation {
export interface ClientDriver {
readonly name: string;
readonly capabilities: readonly ChannelCapability[];
+ readonly surface: Surface;
/**
* Starts the client against the agent at `host`. The global `fetch` serves
* that origin from the eve channel's routes until the client closes.
@@ -502,7 +517,10 @@ async function converse(
);
const conversation: ChannelConversation = {
- say: (text) => view.say(text),
+ async say(text) {
+ await waitForStepsToFinish([...sessions.values()], wait);
+ await view.say(text);
+ },
press: (option) => view.press(option),
async waitForQuestion(prompt) {
const { options } = await view.waitForQuestion([prompt]);
@@ -603,20 +621,48 @@ const TERMINAL_STEP_STATUSES = new Set(["completed", "failed", "cancelled"]);
* still running. A turn that starts after the first cancel needs another.
*/
async function cancelUntilResting(session: Session): Promise<void> {
- const world = await getWorld();
const deadline = Date.now() + WAIT_TIMEOUT_MS;
+ let last: string | undefined;
+ let running: string[] = [];
while (Date.now() < deadline) {
await session.cancel();
const tail = await session.getStreamTailIndex();
const reader = (await session.getEventStream({ startIndex: tail })).getReader();
- const last = await reader.read().finally(() => reader.cancel());
- if (last.value?.type === "session.waiting") {
- const steps = await world.steps.list({ resolveData: "none", runId: session.id });
- if (steps.data.every((step) => TERMINAL_STEP_STATUSES.has(step.status))) return;
- }
+ last = (await reader.read().finally(() => reader.cancel())).value?.type;
+ running = await runningSteps(session);
+ if (last === "session.waiting" && running.length === 0) return;
await new Promise((resolve) => setTimeout(resolve, 25));
}
- throw new Error(`Timed out waiting for session ${session.id} to rest.`);
+ throw new Error(
+ `Timed out waiting for session ${session.id} to rest. Its stream ends with ${last ?? "nothing"}` +
+ (running.length === 0 ? "." : `, and these steps are still running: ${running.join(", ")}.`),
+ );
+}
+
+/** The names of `session`'s steps that haven't finished. */
+async function runningSteps(session: Session): Promise<string[]> {
+ const world = await getWorld();
+ const steps = await world.steps.list({ resolveData: "none", runId: session.id });
+ return steps.data
+ .filter((step) => !TERMINAL_STEP_STATUSES.has(step.status))
+ .map((step) => `${step.stepName} (${step.status})`);
+}
+
+/**
+ * Waits until no session has a step running. A person writes once the bot has
+ * finished, and a channel claims the address of a message it posted, such as
+ * the Telegram message a person replies to, only once the step that posted it
+ * commits. Writing sooner starts a second session instead of continuing this one.
+ */
+async function waitForStepsToFinish(sessions: readonly Session[], wait: Wait): Promise<void> {
+ await wait(
+ "every session's steps to finish",
+ async () => {
+ const running = await Promise.all(sessions.map(runningSteps));
+ return running.every((steps) => steps.length === 0) ? true : undefined;
+ },
+ () => "",
+ );
}
/** How long a session's stream must stay unchanged to count as resting. */
diff --git a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts
index 937cc73c1..e27f2a747 100644
--- a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts
@@ -63,6 +63,8 @@ export function linearDriver(): ChannelDriver {
return {
name: "linear",
capabilities: ["text-replies"],
+ // An agent session lives on an issue the whole workspace can see.
+ surface: "shared",
createChannel: (record) =>
linearChannel({
api: { fetch: recordingFetch(record, decode) },
diff --git a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts
index a18d1c8a3..68e1a03d9 100644
--- a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts
@@ -4,6 +4,7 @@ import { linqChannel } from "#public/channels/linq/index.js";
import {
type ChannelDriver,
type PlatformCall,
+ type Surface,
numberedOptions,
recordingFetch,
} from "#internal/testing/channel-conformance/harness.js";
@@ -24,8 +25,9 @@ interface LinqMessageBody {
readonly message?: { readonly parts?: readonly LinqPart[] };
}
-/** Drives the real Linq adapter through a signed direct-message webhook. */
-export function linqDriver(): ChannelDriver {
+/** Drives the real Linq adapter through signed webhooks, in a group chat by default or 1:1. */
+export function linqDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {
+ const group = surface === "shared";
nextChat += 1;
const chatId = `linq-conformance-chat-${nextChat}`;
let messageId = 0;
@@ -35,7 +37,8 @@ export function linqDriver(): ChannelDriver {
messageId += 1;
const body = JSON.stringify({
data: {
- chat: { id: chatId, is_group: false },
+ // Linq's bridge hears every group message; a mention would route past its handler.
+ chat: { id: chatId, is_group: group },
direction: "inbound",
id: `linq-inbound-${messageId}`,
parts: [{ type: "text", value: text }],
@@ -61,8 +64,9 @@ export function linqDriver(): ChannelDriver {
}
return {
- name: "linq",
+ name: group ? "linq" : "linq-dm",
capabilities: ["text-replies"],
+ surface,
createChannel(record) {
const previousFetch = globalThis.fetch;
const fakeFetch = recordingFetch(record, async (request) => {
diff --git a/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts b/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts
index 3654a3af0..f6ed510e4 100644
--- a/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts
@@ -4,6 +4,7 @@ import { slackChannel } from "#public/channels/slack/index.js";
import { HITL_ACTION_PREFIX } from "#public/channels/slack/hitl.js";
import {
type ChannelDriver,
+ type Surface,
type PlatformCall,
type RenderedOption,
recordingFetch,
@@ -12,6 +13,7 @@ import { decodeSlackApiBody } from "#internal/testing/slack-api-body.js";
const SIGNING_SECRET = "slack-conformance-secret";
const PERSON = "U_ALICE";
+const BOT = "U_EVE";
let nextChannel = 0;
const TEAM = "T01";
@@ -47,11 +49,15 @@ interface SlackMessageBody {
readonly ts?: string;
}
-/** Drives the Slack channel through its Events API and interactivity webhooks in a DM. */
-export function slackDriver(): ChannelDriver {
- // A fresh DM per driver keeps each test's session apart in the shared workflow world.
+/**
+ * Drives the Slack channel through its Events API and interactivity webhooks,
+ * in a public channel thread by default or in a DM.
+ */
+export function slackDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {
+ const dm = surface === "private";
+ // A fresh channel per driver keeps each test's session apart in the shared workflow world.
nextChannel += 1;
- const CHANNEL = `D${String(nextChannel).padStart(3, "0")}`;
+ const CHANNEL = `${dm ? "D" : "C"}${String(nextChannel).padStart(3, "0")}`;
let sequence = 0;
// Every message is a reply in the thread the first one starts.
const threadTs = "1700000000.000001";
@@ -91,14 +97,15 @@ export function slackDriver(): ChannelDriver {
ok: true,
team_id: TEAM,
ts,
- user_id: "U_EVE",
+ user_id: BOT,
},
};
}
return {
- name: "slack",
+ name: dm ? "slack-dm" : "slack",
capabilities: ["buttons", "text-replies"],
+ surface,
createChannel: (record) =>
slackChannel({
api: { fetch: recordingFetch(record, decode) },
@@ -108,18 +115,26 @@ export function slackDriver(): ChannelDriver {
const ts = threadStarted ? nextTs() : threadTs;
const thread = threadStarted ? { thread_ts: threadTs } : {};
threadStarted = true;
+ // In a channel the default policy hears only mentions, so a person mentions the bot each
+ // time. On its own line, so the test model's line-based directives still read the message.
+ const event = dm
+ ? { channel_type: "im", text, type: "message" }
+ : { channel_type: "channel", text: `${text}\n<@${BOT}>`, type: "app_mention" };
return signed(
JSON.stringify({
event: {
...thread,
+ ...event,
channel: CHANNEL,
- channel_type: "im",
event_ts: ts,
- text,
ts,
- type: "message",
user: PERSON,
},
+ // Slack names the installation that received the event, which is how eve knows its
+ // own bot user, e.g. to strip that mention from a typed answer.
+ authorizations: [
+ { is_bot: true, is_enterprise_install: false, team_id: TEAM, user_id: BOT },
+ ],
event_id: `Ev${ts}`,
team_id: TEAM,
type: "event_callback",
diff --git a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts b/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts
index 325a3f671..4314b1d33 100644
--- a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts
@@ -3,6 +3,7 @@ import {
type ChannelDriver,
type PlatformCall,
type RenderedOption,
+ type Surface,
recordingFetch,
} from "#internal/testing/channel-conformance/harness.js";
@@ -11,6 +12,7 @@ const TENANT = "TENANT";
const PERSON = { id: "USER", name: "Alice" } as const;
const BOT = { id: "BOT", name: "eve Bot" } as const;
const SERVICE_URL = "https://smba.example.test/teams";
+const MENTION = `<at>${BOT.name}</at>`;
interface Choice {
readonly title: string;
@@ -47,11 +49,24 @@ interface PressHandle {
readonly value?: string;
}
-/** Drives Teams' Bot Framework webhook and Connector API in a personal chat. */
-export function teamsDriver(): ChannelDriver {
+/**
+ * Drives Teams' Bot Framework webhook and Connector API, in a team channel's
+ * thread by default or in a personal chat.
+ */
+export function teamsDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {
+ const personal = surface === "private";
// Conversation ids are part of Teams' continuation token, so isolate each driver instance.
nextConversation += 1;
- const conversationId = `CONV-${nextConversation}`;
+ // A channel thread's conversation id names the post that started it.
+ const conversationId = personal
+ ? `CONV-${nextConversation}`
+ : `CONV-${nextConversation};messageid=MSG-1`;
+ const conversation = personal
+ ? { conversationType: "personal", id: conversationId }
+ : { conversationType: "channel", id: conversationId };
+ const channelData = personal
+ ? { tenant: { id: TENANT } }
+ : { channel: { id: "CHANNEL" }, team: { id: "TEAM" }, tenant: { id: TENANT } };
let activityId = 0;
function nextActivityId(): string {
@@ -62,8 +77,8 @@ export function teamsDriver(): ChannelDriver {
function activity(payload: Record<string, unknown>): Request {
return new Request("https://agent.example.com/eve/v1/teams", {
body: JSON.stringify({
- channelData: { tenant: { id: TENANT } },
- conversation: { id: conversationId, conversationType: "personal" },
+ channelData,
+ conversation,
from: PERSON,
id: `MSG-${activityId + 1}`,
recipient: BOT,
@@ -85,14 +100,25 @@ export function teamsDriver(): ChannelDriver {
}
return {
- name: "teams",
+ name: personal ? "teams-dm" : "teams",
capabilities: ["buttons", "text-replies"],
+ surface,
createChannel: (record) =>
teamsChannel({
api: { fetch: recordingFetch(record, decode) },
credentials: { tokenProvider: () => "test-token", webhookVerifier: () => true },
}),
- message: (text) => activity({ text, type: "message" }),
+ message: (text) =>
+ personal
+ ? activity({ text, type: "message" })
+ : // In a channel the default policy hears only mentions, so a person mentions the bot
+ // each time, on its own line so the test model's line-based directives still read it.
+ activity({
+ entities: [{ mentioned: BOT, text: MENTION, type: "mention" }],
+ text: `${text}\n${MENTION}`,
+ textFormat: "xml",
+ type: "message",
+ }),
findOptions(call, prompt) {
const body = call.body as ActivityBody;
if (body.type !== "message" || !body.text?.includes(prompt)) return undefined;
diff --git a/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts b/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts
index adeadee8a..b704bfe1c 100644
--- a/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts
@@ -3,10 +3,12 @@ import {
type ChannelDriver,
type PlatformCall,
type RenderedOption,
+ type Surface,
recordingFetch,
} from "#internal/testing/channel-conformance/harness.js";
const SECRET = "telegram-conformance-secret";
+const BOT = { first_name: "eve", id: 99, is_bot: true, username: "testbot" } as const;
let nextChatId = 1000;
interface InlineButton {
@@ -50,14 +52,37 @@ function inlineOptions(call: PlatformCall): RenderedOption[] {
);
}
-/** Drives the Telegram channel through its Bot API webhook in a private chat. */
-export function telegramDriver(): ChannelDriver {
+/**
+ * Drives the Telegram channel through its Bot API webhook, in a supergroup by
+ * default or in a private chat.
+ */
+export function telegramDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {
+ const group = surface === "shared";
// A fresh chat per driver keeps each test's session apart in the shared workflow world.
nextChatId += 1;
- const CHAT = { id: nextChatId, type: "private" } as const;
+ const CHAT = group
+ ? { id: -nextChatId, title: "Release crew", type: "supergroup" }
+ : { id: nextChatId, type: "private" };
const PERSON = { first_name: "Alice", id: nextChatId, is_bot: false } as const;
let updateId = 0;
let messageId = 0;
+ /** The bot's latest message, which a person in a group replies to so the bot hears them. */
+ let lastBotMessage: number | undefined;
+
+ /**
+ * A group's default policy hears only mentions and replies to the bot, and a
+ * mention starts a new conversation, so a person mentions the bot once and
+ * then replies to its latest message. The mention sits on its own line so
+ * the test model's line-based directives still read the message.
+ */
+ function addressed(text: string): Record<string, unknown> {
+ if (!group) return { text };
+ if (lastBotMessage === undefined) return { text: `${text}\n@${BOT.username}` };
+ return {
+ reply_to_message: { chat: CHAT, date: 0, from: BOT, message_id: lastBotMessage },
+ text,
+ };
+ }
function update(payload: Record<string, unknown>): Request {
updateId += 1;
@@ -75,6 +100,7 @@ export function telegramDriver(): ChannelDriver {
const method = new URL(request.url).pathname.split("/").at(-1)!;
const text = await request.text();
messageId += 1;
+ if (method === "sendMessage") lastBotMessage = messageId;
return {
body: text === "" ? {} : JSON.parse(text),
method,
@@ -83,15 +109,25 @@ export function telegramDriver(): ChannelDriver {
}
return {
- name: "telegram",
+ name: group ? "telegram" : "telegram-dm",
capabilities: ["buttons", "text-replies"],
+ surface,
createChannel: (record) =>
telegramChannel({
api: { fetch: recordingFetch(record, decode) },
+ botUsername: BOT.username,
credentials: { botToken: "bot-token", webhookSecretToken: SECRET },
}),
message: (text) =>
- update({ message: { chat: CHAT, date: 0, from: PERSON, message_id: 1000 + updateId, text } }),
+ update({
+ message: {
+ chat: CHAT,
+ date: 0,
+ from: PERSON,
+ message_id: 1000 + updateId,
+ ...addressed(text),
+ },
+ }),
findOptions(call, prompt) {
const body = call.body as MessageBody;
if (call.method !== "sendMessage" || body.text?.includes(prompt) !== true) return undefined;
diff --git a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts b/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts
index 46e5aedd0..63fdd4520 100644
--- a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts
@@ -27,6 +27,8 @@ export function tuiDriver(): ClientDriver {
return {
name: "tui",
capabilities: ["buttons", "text-replies"],
+ // A local terminal shows only the person at it.
+ surface: "private",
async open(host, wait) {
// Wide enough that no reply wraps, so each one stays on one screen row.
const screen = new MockScreen({ columns: 1000, rows: 60 });
diff --git a/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts b/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts
index 7b9a86e99..ecbffb6d4 100644
--- a/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts
@@ -48,6 +48,7 @@ export function twilioDriver(): ChannelDriver {
return {
name: "twilio",
capabilities: ["text-replies"],
+ surface: "private",
createChannel: (record) =>
twilioChannel({
allowFrom: "*",
diff --git a/packages/eve/src/public/channels/chat-sdk/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/chat-sdk/hitl-conformance-dm.integration.test.ts
new file mode 100644
index 000000000..e880c1d3d
--- /dev/null
+++ b/packages/eve/src/public/channels/chat-sdk/hitl-conformance-dm.integration.test.ts
@@ -0,0 +1,3 @@
+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";
+
+describeHitlConformance("chat-sdk-dm");
diff --git a/packages/eve/src/public/channels/discord/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/discord/hitl-conformance-dm.integration.test.ts
new file mode 100644
index 000000000..af54ab402
--- /dev/null
+++ b/packages/eve/src/public/channels/discord/hitl-conformance-dm.integration.test.ts
@@ -0,0 +1,3 @@
+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";
+
+describeHitlConformance("discord-dm");
diff --git a/packages/eve/src/public/channels/linq/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/linq/hitl-conformance-dm.integration.test.ts
new file mode 100644
index 000000000..ed2b482b4
--- /dev/null
+++ b/packages/eve/src/public/channels/linq/hitl-conformance-dm.integration.test.ts
@@ -0,0 +1,3 @@
+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";
+
+describeHitlConformance("linq-dm");
diff --git a/packages/eve/src/public/channels/slack/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/slack/hitl-conformance-dm.integration.test.ts
new file mode 100644
index 000000000..554f6c4fe
--- /dev/null
+++ b/packages/eve/src/public/channels/slack/hitl-conformance-dm.integration.test.ts
@@ -0,0 +1,3 @@
+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";
+
+describeHitlConformance("slack-dm");
diff --git a/packages/eve/src/public/channels/teams/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/teams/hitl-conformance-dm.integration.test.ts
new file mode 100644
index 000000000..bd460bb34
--- /dev/null
+++ b/packages/eve/src/public/channels/teams/hitl-conformance-dm.integration.test.ts
@@ -0,0 +1,3 @@
+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";
+
+describeHitlConformance("teams-dm");
diff --git a/packages/eve/src/public/channels/telegram/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/telegram/hitl-conformance-dm.integration.test.ts
new file mode 100644
index 000000000..6c5231115
--- /dev/null
+++ b/packages/eve/src/public/channels/telegram/hitl-conformance-dm.integration.test.ts
@@ -0,0 +1,3 @@
+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";
+
+describeHitlConformance("telegram-dm");
diff --git a/packages/eve/src/public/channels/telegram/hitl.test.ts b/packages/eve/src/public/channels/telegram/hitl.test.ts
index c40ca8652..ce025f2fc 100644
--- a/packages/eve/src/public/channels/telegram/hitl.test.ts
+++ b/packages/eve/src/public/channels/telegram/hitl.test.ts
@@ -2,13 +2,11 @@ import { describe, expect, it } from "vitest";
import {
TELEGRAM_CALLBACK_RESPONSE_PREFIX,
- TELEGRAM_HITL_CALLBACK_PREFIX,
TELEGRAM_REPLY_RESPONSE_PREFIX,
isTelegramSyntheticResponse,
registerTelegramFreeformPrompt,
renderTelegramInputRequest,
resolveTelegramInputResponses,
- telegramCallbackInputResponse,
telegramReplyInputResponse,
type TelegramHitlState,
} from "#public/channels/telegram/hitl.js";
@@ -63,28 +61,6 @@ describe("renderTelegramInputRequest", () => {
});
describe("resolveTelegramInputResponses", () => {
- it("maps a compact callback id to its request on every press", () => {
- const state: TelegramHitlState = {
- hitlCallbacks: {
- [`${TELEGRAM_HITL_CALLBACK_PREFIX}0`]: {
- optionId: "approve",
- requestId: "call_1",
- },
- },
- };
-
- expect(
- resolveTelegramInputResponses(state, [
- telegramCallbackInputResponse(`${TELEGRAM_HITL_CALLBACK_PREFIX}0`),
- ]),
- ).toEqual([{ optionId: "approve", requestId: "call_1" }]);
- expect(
- resolveTelegramInputResponses(state, [
- telegramCallbackInputResponse(`${TELEGRAM_HITL_CALLBACK_PREFIX}0`),
- ]),
- ).toEqual([{ optionId: "approve", requestId: "call_1" }]);
- });
-
it("maps replies to ForceReply prompts back to freeform answers", () => {
const state: TelegramHitlState = {};
registerTelegramFreeformPrompt(state, { messageId: "99", requestId: "call_1" });
diff --git a/packages/eve/vitest.integration.config.ts b/packages/eve/vitest.integration.config.ts
index 8695edf59..0f61179cd 100644
--- a/packages/eve/vitest.integration.config.ts
+++ b/packages/eve/vitest.integration.config.ts
@@ -32,7 +32,12 @@ export default defineConfig({
},
test: {
environment: "node",
- exclude: ["**/node_modules/**", "test/scenarios/**"],
+ exclude: [
+ "**/node_modules/**",
+ "test/scenarios/**",
+ // Channel conformance checks platform protocol, not the OS, and is slowest on Windows runners.
+ ...(process.platform === "win32" ? ["**/hitl-conformance*.integration.test.ts"] : []),
+ ],
globalSetup: ["./test/setup/clear-workflow-cache.ts"],
include: ["src/**/*.integration.test.ts", "test/**/*.integration.test.ts"],
setupFiles: ["./test/setup/mock-ai-gateway.ts"],