The Open Framework for Building Agents
[5/7] test(eve): run the HITL conformance suite in shared threads and DMs (#4253)
Ben Pankow886ee32bd6f0
diff --git a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md b/packages/eve/src/internal/testing/channel-conformance/MATRIX.mdindex 8c0d336b1..d7d8299e6 100644--- a/packages/eve/src/internal/testing/channel-conformance/MATRIX.md+++ b/packages/eve/src/internal/testing/channel-conformance/MATRIX.md@@ -14,32 +14,32 @@ pnpm --filter eve exec vitest run --config vitest.unit.config.ts channel-conform✅ passes · ❌ broken · — not supported (the platform lacks a capability the ruleneeds, or the client declines it below)-| Rule | `chat-sdk` | `chat-sdk-text` | `discord` | `github` | `linear` | `linq` | `slack` | `teams` | `telegram` | `tui` | `twilio` |-| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |-| a rendered question shows every option a person can choose | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| pressing a rendered option answers the pending question with that option | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |-| a text reply matching an option answers the only pending question | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| a text reply that matches no option answers the question with the person's words | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| a text reply answers an open-ended question with the person's words | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| pressing an option of an answered question sends it to the agent as new input | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | — | — |-| pressing options of two pending questions answers each with its own option | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |-| a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |-| pressing Approve runs the gated tool | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |-| pressing Approve twice runs the gated tool once | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |-| pressing Approve on one of two pending approvals runs only that tool | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |-| answering an approval and a question pending together settles both | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |-| pressing Cancel stops the gated tool without running it | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |-| a text reply of approve runs the gated tool | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |-| a text reply of cancel stops the gated tool without running it | ✅ | ✅ | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |-| pressing an option clears the question's buttons | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |-| answering a question by text clears its buttons | ✅ | — | — | — | — | — | ❌ | ✅ | ✅ | ✅ | — |-| pressing an option names who answered on the question | ❌ | — | ❌ | — | — | — | ✅ | ❌ | ❌ | — | — |-| answering a question by text names who answered on the question | ❌ | — | — | — | — | — | ❌ | ❌ | ❌ | — | — |-| pressing Approve clears the approval's buttons | ✅ | — | ✅ | — | — | — | ✅ | ✅ | ✅ | ✅ | — |-| approving by text clears the approval's buttons | ✅ | — | — | — | — | — | ✅ | ✅ | ✅ | — | — |-| pressing Approve names who approved on the approval | ❌ | — | ❌ | — | — | — | ✅ | ✅ | ❌ | — | — |-| approving by text names who approved on the approval | ❌ | — | — | — | — | — | ❌ | ❌ | ❌ | — | — |+| Rule | `chat-sdk` | `chat-sdk-text` | `chat-sdk-dm` | `discord` | `discord-dm` | `github` | `linear` | `linq` | `linq-dm` | `slack` | `slack-dm` | `teams` | `teams-dm` | `telegram` | `telegram-dm` | `tui` | `twilio` |+| --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |+| a rendered question shows every option a person can choose | ✅ | ✅ | — | ✅ | — | ✅ | ✅ | ✅ | — | ✅ | — | ✅ | — | ✅ | — | ✅ | ✅ |+| pressing a rendered option answers the pending question with that option | ✅ | — | ✅ | ✅ | ✅ | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — |+| a text reply matching an option answers the only pending question | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| a text reply that matches no option answers the question with the person's words | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| a text reply answers an open-ended question with the person's words | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| pressing an option of an answered question sends it to the agent as new input | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | — | — |+| pressing options of two pending questions answers each with its own option | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |+| a text reply matching an option does not answer either of two pending questions | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |+| a tool approval shows a choice to approve and one to cancel | ✅ | ✅ | — | ✅ | — | ✅ | ✅ | ✅ | — | ✅ | — | ✅ | — | ✅ | — | ✅ | ✅ |+| pressing Approve runs the gated tool | ✅ | — | ✅ | ✅ | ✅ | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — |+| pressing Approve twice runs the gated tool once | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |+| pressing Approve on one of two pending approvals runs only that tool | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |+| answering an approval and a question pending together settles both | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |+| pressing Cancel stops the gated tool without running it | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |+| a text reply of approve runs the gated tool | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |+| a text reply of cancel stops the gated tool without running it | ✅ | ✅ | ✅ | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | ✅ |+| pressing an option clears the question's buttons | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |+| answering a question by text clears its buttons | ✅ | — | ✅ | — | — | — | — | — | — | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | — |+| pressing an option names who answered on the question | ❌ | — | — | ❌ | — | — | — | — | — | ✅ | — | ❌ | — | ❌ | — | — | — |+| answering a question by text names who answered on the question | ❌ | — | ❌ | — | — | — | — | — | — | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | — | — |+| pressing Approve clears the approval's buttons | ✅ | — | — | ✅ | — | — | — | — | — | ✅ | — | ✅ | — | ✅ | — | ✅ | — |+| approving by text clears the approval's buttons | ✅ | — | ✅ | — | — | — | — | — | — | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | — | — |+| pressing Approve names who approved on the approval | ❌ | — | — | ❌ | — | — | — | — | — | ✅ | — | ✅ | — | ❌ | — | — | — |+| approving by text names who approved on the approval | ❌ | — | ❌ | — | — | — | — | — | — | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | — | — |## Broken@@ -47,21 +47,102 @@ needs, or the client declines it below)- **chat-sdk**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder- **chat-sdk**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder- **chat-sdk**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder+- **chat-sdk-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder+- **chat-sdk-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder- **discord**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder- **discord**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder- **slack**, answering a question by text clears its buttons: only the button interaction handler edits a question; a typed answer leaves it- **slack**, answering a question by text names who answered on the question: only the button interaction handler edits a question; a typed answer leaves it- **slack**, approving by text names who approved on the approval: the card loses its buttons after a typed approval but doesn't say who approved+- **slack-dm**, answering a question by text clears its buttons: only the button interaction handler edits a question; a typed answer leaves it+- **slack-dm**, answering a question by text names who answered on the question: only the button interaction handler edits a question; a typed answer leaves it+- **slack-dm**, approving by text names who approved on the approval: the card loses its buttons after a typed approval but doesn't say who approved- **teams**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder- **teams**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder- **teams**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder+- **teams-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder+- **teams-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder- **telegram**, pressing an option names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder- **telegram**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder- **telegram**, pressing Approve names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder- **telegram**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder+- **telegram-dm**, answering a question by text names who answered on the question: a resolved prompt doesn't say who answered; input.resolved carries no responder+- **telegram-dm**, approving by text names who approved on the approval: a resolved prompt doesn't say who answered; input.resolved carries no responder## Declined+- **chat-sdk-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **chat-sdk-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **discord-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **linq-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **slack-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **teams-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, a rendered question shows every option a person can choose: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, pressing an option of an answered question sends it to the agent as new input: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, pressing options of two pending questions answers each with its own option: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, a tool approval shows a choice to approve and one to cancel: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, pressing Approve twice runs the gated tool once: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, pressing Approve on one of two pending approvals runs only that tool: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, answering an approval and a question pending together settles both: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, pressing Cancel stops the gated tool without running it: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, pressing an option clears the question's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, pressing an option names who answered on the question: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, pressing Approve clears the approval's buttons: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it+- **telegram-dm**, pressing Approve names who approved on the approval: it doesn't vary between a shared thread and a DM, and the shared-thread column covers it- **tui**, pressing an option of an answered question sends it to the agent as new input: an answered question's drawer closes, so nothing is left to press- **tui**, a text reply of approve runs the gated tool: the approval drawer holds the keyboard; a person answers it with y or n- **tui**, a text reply of cancel stops the gated tool without running it: the approval drawer holds the keyboard; a person answers it with y or ndiff --git a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.tsindex f35b16ffa..43cade21f 100644--- a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts@@ -14,6 +14,7 @@ import { createMemoryState } from "#compiled/@chat-adapter/state-memory/index.jsimport {type ChannelDriver,type PlatformCall,+ type Surface,numberedOptions,} from "#internal/testing/channel-conformance/harness.js";@@ -60,16 +61,21 @@ function buttonsOf(call: PlatformCall, prompt?: string): { handle: PressHandle;}/**- * Drives `chatSdkChannel` with a card-capable direct-message adapter: one- * thread, no streaming, and every message handed to eve with an empty- * `context`. The fake adapter is the platform: it reads inbound JSON and- * records every post and edit.+ * Drives `chatSdkChannel` with a card-capable adapter: one thread, in a+ * workspace channel by default or a direct message, no streaming, and every+ * message handed to eve with an empty `context`. The fake adapter is the+ * platform: it reads inbound JSON and records every post and edit.*/-export function chatSdkDriver(): ChannelDriver {- const driver = chatSdkDriverWith({ name: "chat-sdk", render: (posted) => posted });+export function chatSdkDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {+ const driver = chatSdkDriverWith({+ name: surface === "private" ? "chat-sdk-dm" : "chat-sdk",+ render: (posted) => posted,+ surface,+ });return {...driver,capabilities: ["buttons", "text-replies"],+ surface,findOptions(call, prompt) {if (!isPost(call)) return undefined;const card = cardOf(call.body as AdapterPostableMessage);@@ -110,10 +116,12 @@ export function chatSdkTextDriver(): ChannelDriver {const driver = chatSdkDriverWith({name: "chat-sdk-text",render: (posted) => converter.renderPostable(posted),+ surface: "private",});return {...driver,capabilities: ["text-replies"],+ surface: "private",findOptions(call, prompt) {if (!isPost(call) || typeof call.body !== "string" || !call.body.includes(prompt)) {return undefined;@@ -139,7 +147,8 @@ class PlainTextConverter extends BaseFormatConverter {function chatSdkDriverWith(input: {readonly name: string;readonly render: (posted: AdapterPostableMessage) => unknown;-}): Omit<ChannelDriver, "capabilities" | "findOptions" | "press"> & {+ readonly surface: Exclude<Surface, "public">;+}): Omit<ChannelDriver, "capabilities" | "findOptions" | "press" | "surface"> & {inbound(body: Inbound): Request;} {nextThread += 1;@@ -158,7 +167,8 @@ function chatSdkDriverWith(input: {name: input.name,inbound,createChannel(record) {- const adapter = fakeAdapter(threadId, record, () => (sequence += 1), input.render);+ const dm = input.surface === "private";+ const adapter = fakeAdapter(threadId, record, () => (sequence += 1), input.render, dm);const bridge = chatSdkChannel({adapters: { [ADAPTER]: adapter },concurrency: "concurrent",@@ -167,9 +177,21 @@ function chatSdkDriverWith(input: {streaming: false,userName: "eve",});- bridge.bot.onDirectMessage(async (thread, message) => {- await bridge.send(message.text, { auth: null, context: [], thread });- });+ if (dm) {+ bridge.bot.onDirectMessage(async (thread, message) => {+ await bridge.send(message.text, { auth: null, context: [], thread });+ });+ } else {+ // The wiring docs/channels/chat-sdk.mdx shows: a mention starts the session, and+ // subscribing lets the rest of the thread continue it without one.+ bridge.bot.onNewMention(async (thread, message) => {+ await thread.subscribe();+ await bridge.send(message.text, { auth: null, context: [], thread });+ });+ bridge.bot.onSubscribedMessage(async (thread, message) => {+ await bridge.send(message.text, { auth: null, context: [], thread });+ });+ }return bridge.channel;},message: (text) => inbound({ kind: "message", text }),@@ -193,7 +215,9 @@ function fakeAdapter(record: (call: PlatformCall) => void,nextId: () => number,render: (posted: AdapterPostableMessage) => unknown,+ dm: boolean,): Adapter {+ const visibility = dm ? ("private" as const) : ("workspace" as const);let chat: ChatInstance | null = null;const self = {name: ADAPTER,@@ -221,7 +245,8 @@ function fakeAdapter(await chat?.processMessage(adapter,threadId,- inboundMessage(threadId, id, body.text),+ // A person mentions the bot to start a channel thread; Chat routes the rest by subscription.+ inboundMessage(threadId, id, body.text, !dm),options,);}@@ -230,16 +255,16 @@ function fakeAdapter(channelIdFromThreadId: () => threadId,decodeThreadId: (id: string) => ({ threadId: id }),encodeThreadId: (input: { threadId: string }) => input.threadId,- getChannelVisibility: () => "private" as const,- isDM: () => true,+ getChannelVisibility: () => visibility,+ isDM: () => dm,parseMessage: (raw: { text?: string }) => inboundMessage(threadId, "parsed", raw.text ?? ""),renderFormatted: () => "",fetchMessages: async () => ({ messages: [] }),fetchThread: async (id: string) => ({channelId: threadId,- channelVisibility: "private" as const,+ channelVisibility: visibility,id,- isDM: true,+ isDM: dm,metadata: {},}),async postMessage(id: string, posted: AdapterPostableMessage) {@@ -261,13 +286,13 @@ function fakeAdapter(return adapter;}-function inboundMessage(threadId: string, id: string, text: string): Message {+function inboundMessage(threadId: string, id: string, text: string, isMention = false): Message {return new Message({attachments: [],author: PERSON,formatted: parseMarkdown(text),id,- isMention: false,+ isMention,metadata: { dateSent: new Date("2026-01-01T00:00:00.000Z"), edited: false },raw: { text },text,diff --git a/packages/eve/src/internal/testing/channel-conformance/conformance.ts b/packages/eve/src/internal/testing/channel-conformance/conformance.tsindex 98cbd779a..8f598ce8d 100644--- a/packages/eve/src/internal/testing/channel-conformance/conformance.ts+++ b/packages/eve/src/internal/testing/channel-conformance/conformance.ts@@ -1,6 +1,10 @@import { describe, expect, it } from "vitest";-import { type HitlRule, hitlContract } from "#internal/testing/channel-conformance/contract.js";+import {+ type ContractRule,+ type HitlRule,+ hitlContract,+} from "#internal/testing/channel-conformance/contract.js";import {type ChannelDriver,type ClientDriver,@@ -81,11 +85,55 @@ function unnamedAnsweredPrompts(interface ConformanceChannel {readonly driver: () => ChannelDriver | ClientDriver;+ /**+ * A DM column beside the channel's shared-thread column. It runs only rules+ * whose behavior varies by conversation; the shared column covers the rest.+ */+ readonly dm?: true;readonly broken?: Partial<Record<HitlRule, BrokenCell>>;/** Rules this client deliberately doesn't offer, with why, beyond what its capabilities rule out. */readonly unsupported?: Partial<Record<HitlRule, string>>;}+const UNNAMED_RESPONDER =+ "a resolved prompt doesn't say who answered; input.resolved carries no responder";++const CHAT_SDK_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [+ "approvalPress",+ "approvalText",+ "questionPress",+ "questionText",+]);++const DISCORD_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [+ "approvalPress",+ "questionPress",+]);++const SLACK_BROKEN = {+ ...staleAnsweredPrompts(+ "only the button interaction handler edits a question; a typed answer leaves it",+ ["questionText"],+ ),+ "approving by text names who approved on the approval": {+ reason: "the card loses its buttons after a typed approval but doesn't say who approved",+ symptom: /the answered prompt never names who answered/,+ },+} satisfies Partial<Record<HitlRule, BrokenCell>>;++const TEAMS_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [+ "approvalText",+ "questionPress",+ "questionText",+]);++const TELEGRAM_BROKEN = unnamedAnsweredPrompts(UNNAMED_RESPONDER, [+ "approvalPress",+ "approvalText",+ "questionPress",+ "questionText",+]);+const TUI_TYPED_APPROVAL ="the approval drawer holds the keyboard; a person answers it with y or n";const TUI_SINGLE_PERSON = "one person answers at their own terminal; there's nobody else to tell";@@ -103,61 +151,20 @@ const TUI_SINGLE_PERSON = "one person answers at their own terminal; there's nob* failure (such as harness breakage) both turn it red.*/const hitlConformance = {- "chat-sdk": [- {- driver: chatSdkDriver,- broken: unnamedAnsweredPrompts(- "a resolved prompt doesn't say who answered; input.resolved carries no responder",- ["approvalPress", "approvalText", "questionPress", "questionText"],- ),- },- { driver: chatSdkTextDriver },- ],- discord: [- {- driver: discordDriver,- broken: unnamedAnsweredPrompts(- "a resolved prompt doesn't say who answered; input.resolved carries no responder",- ["approvalPress", "questionPress"],- ),- },- ],+ "chat-sdk": [{ driver: chatSdkDriver, broken: CHAT_SDK_BROKEN }, { driver: chatSdkTextDriver }],+ "chat-sdk-dm": [{ dm: true, driver: () => chatSdkDriver("private"), broken: CHAT_SDK_BROKEN }],+ discord: [{ driver: discordDriver, broken: DISCORD_BROKEN }],+ "discord-dm": [{ dm: true, driver: () => discordDriver("private"), broken: DISCORD_BROKEN }],github: [{ driver: githubDriver }],linear: [{ driver: linearDriver }],linq: [{ driver: linqDriver }],- slack: [- {- driver: slackDriver,- broken: {- ...staleAnsweredPrompts(- "only the button interaction handler edits a question; a typed answer leaves it",- ["questionText"],- ),- "approving by text names who approved on the approval": {- reason: "the card loses its buttons after a typed approval but doesn't say who approved",- symptom: /the answered prompt never names who answered/,- },- },- },- ],- teams: [- {- driver: teamsDriver,- broken: unnamedAnsweredPrompts(- "a resolved prompt doesn't say who answered; input.resolved carries no responder",- ["approvalText", "questionPress", "questionText"],- ),- },- ],- telegram: [- {- driver: telegramDriver,- broken: unnamedAnsweredPrompts(- "a resolved prompt doesn't say who answered; input.resolved carries no responder",- ["approvalPress", "approvalText", "questionPress", "questionText"],- ),- },- ],+ "linq-dm": [{ dm: true, driver: () => linqDriver("private") }],+ slack: [{ driver: slackDriver, broken: SLACK_BROKEN }],+ "slack-dm": [{ dm: true, driver: () => slackDriver("private"), broken: SLACK_BROKEN }],+ teams: [{ driver: teamsDriver, broken: TEAMS_BROKEN }],+ "teams-dm": [{ dm: true, driver: () => teamsDriver("private"), broken: TEAMS_BROKEN }],+ telegram: [{ driver: telegramDriver, broken: TELEGRAM_BROKEN }],+ "telegram-dm": [{ dm: true, driver: () => telegramDriver("private"), broken: TELEGRAM_BROKEN }],tui: [{driver: tuiDriver,@@ -190,6 +197,13 @@ type Cell =| { readonly kind: "broken"; readonly broken: BrokenCell };function cellOf(entry: ConformanceChannel, rule: (typeof hitlContract)[number]): Cell {+ if (entry.dm === true && (rule as ContractRule).variesByConversation !== true) {+ return {+ kind: "unsupported",+ reason:+ "it doesn't vary between a shared thread and a DM, and the shared-thread column covers it",+ };+ }const { capabilities } = entry.driver();if (!rule.requires.every((capability) => capabilities.includes(capability))) {return { kind: "unsupported" };diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.tsindex 4da04aacb..875c6b548 100644--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts@@ -23,6 +23,13 @@ export interface ContractRule {/** Where the behavior is promised: a docs anchor or the PR that introduced it. */readonly source: string;readonly requires: readonly ChannelCapability[];+ /**+ * The behavior can differ between a shared thread and a DM, such as a typed+ * answer that must mention the bot in a channel, or a press whose presser a+ * platform names differently in a server. A channel's DM column runs only+ * these; its shared-thread column covers the rest.+ */+ readonly variesByConversation?: boolean;run(conversation: ChannelConversation): Promise<void>;}@@ -165,6 +172,7 @@ export const hitlContract = [rule: "pressing a rendered option answers the pending question with that option",source: "docs/tools/human-in-the-loop.md#answering-from-a-client-or-channel",requires: ["buttons"],+ variesByConversation: true,async run(conversation) {const options = await askWhichDay(conversation);const saturday = options.find((option) => option.label === "Saturday");@@ -177,6 +185,7 @@ export const hitlContract = [rule: "a text reply matching an option answers the only pending question",source: "docs/tools/human-in-the-loop.md#how-pause-and-resume-works",requires: ["text-replies"],+ variesByConversation: true,async run(conversation) {await askWhichDay(conversation);await conversation.say("Saturday");@@ -187,6 +196,7 @@ export const hitlContract = [rule: "a text reply that matches no option answers the question with the person's words",source: "docs/tools/human-in-the-loop.md#questions",requires: ["text-replies"],+ variesByConversation: true,async run(conversation) {await askWhichDay(conversation);await conversation.say(OWN_WORDS);@@ -197,6 +207,7 @@ export const hitlContract = [rule: "a text reply answers an open-ended question with the person's words",source: "docs/tools/human-in-the-loop.md#questions",requires: ["text-replies"],+ variesByConversation: true,async run(conversation) {await conversation.say(ASK_OPEN);await conversation.waitForQuestion(OPEN_PROMPT);@@ -236,6 +247,7 @@ export const hitlContract = [rule: "a text reply matching an option does not answer either of two pending questions",source: "docs/tools/human-in-the-loop.md#how-pause-and-resume-works",requires: ["text-replies"],+ variesByConversation: true,async run(conversation) {await conversation.say(PLAN_REVIEW);await conversation.waitForRequest(DAY_PROMPT);@@ -268,6 +280,7 @@ export const hitlContract = [rule: "pressing Approve runs the gated tool",source: "docs/tools/human-in-the-loop.md#approvals",requires: ["buttons"],+ variesByConversation: true,async run(conversation) {const options = await askToDeploy(conversation);const approve = options.find((option) => APPROVE_LABELS.includes(option.label));@@ -333,6 +346,7 @@ export const hitlContract = [rule: "a text reply of approve runs the gated tool",source: "docs/tools/human-in-the-loop.md#how-pause-and-resume-works",requires: ["text-replies"],+ variesByConversation: true,async run(conversation) {await askToDeploy(conversation);await conversation.say("approve");@@ -343,6 +357,7 @@ export const hitlContract = [rule: "a text reply of cancel stops the gated tool without running it",source: "docs/tools/human-in-the-loop.md#how-pause-and-resume-works",requires: ["text-replies"],+ variesByConversation: true,async run(conversation) {await askToDeploy(conversation);await conversation.say("cancel");@@ -362,6 +377,7 @@ export const hitlContract = [rule: "answering a question by text clears its buttons",source: "#1 (Slack's answered question card)",requires: ["buttons", "text-replies"],+ variesByConversation: true,async run(conversation) {await answerSaturday(conversation, "text");expectButtonsCleared(conversation, PROMPT);@@ -380,6 +396,7 @@ export const hitlContract = [rule: "answering a question by text names who answered on the question",source: "#1 (Slack's answered question card)",requires: ["buttons", "text-replies"],+ variesByConversation: true,async run(conversation) {await answerSaturday(conversation, "text");expectResponderNamed(conversation, PROMPT);@@ -398,6 +415,7 @@ export const hitlContract = [rule: "approving by text clears the approval's buttons",source: "#2212 (Slack's settled approval card)",requires: ["buttons", "text-replies"],+ variesByConversation: true,async run(conversation) {await approveDeploy(conversation, "text");expectButtonsCleared(conversation, APPROVAL_PROMPT);@@ -416,6 +434,7 @@ export const hitlContract = [rule: "approving by text names who approved on the approval",source: "#2212 (Slack's settled approval card)",requires: ["buttons", "text-replies"],+ variesByConversation: true,async run(conversation) {await approveDeploy(conversation, "text");expectResponderNamed(conversation, APPROVAL_PROMPT);diff --git a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts b/packages/eve/src/internal/testing/channel-conformance/discord-driver.tsindex e440a6d5d..15737b0fa 100644--- a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts@@ -5,6 +5,7 @@ import {type ChannelDriver,type PlatformCall,type RenderedOption,+ type Surface,recordingFetch,} from "#internal/testing/channel-conformance/harness.js";@@ -35,8 +36,16 @@ function testKeys(): { privateKey: KeyObject; publicKeyHex: string } {};}-/** Drives Discord's signed application-command and component interaction webhooks. */-export function discordDriver(): ChannelDriver {+/**+ * Drives Discord's signed application-command and component interaction+ * webhooks, in a server text channel by default or in a DM.+ */+export function discordDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {+ const dm = surface === "private";+ // In a server, Discord names the actor as a member; in a DM, as a user.+ const where = dm+ ? { channel: { type: 1 }, user: PERSON }+ : { channel: { type: 0 }, guild_id: "G_CONFORMANCE", member: { roles: [], user: PERSON } };nextChannel += 1;const channelId = `C_CONFORMANCE_${nextChannel}`;const { privateKey, publicKeyHex } = testKeys();@@ -100,8 +109,9 @@ export function discordDriver(): ChannelDriver {}return {- name: "discord",+ name: dm ? "discord-dm" : "discord",capabilities: ["buttons"],+ surface,createChannel: (record) =>discordChannel({api: { fetch: recordingFetch(record, decode) },@@ -111,14 +121,13 @@ export function discordDriver(): ChannelDriver {const id = nextInteraction();return signed(JSON.stringify({+ ...where,application_id: "APP1",- channel: { type: 1 },channel_id: channelId,data: { name: "ask", options: [{ name: "message", type: 3, value: text }] },id,token: `tok-${id}`,type: 2,- user: PERSON,version: 1,}),);@@ -155,10 +164,10 @@ export function discordDriver(): ChannelDriver {? { component_type: 2, custom_id: handle.customId }: { component_type: 3, custom_id: handle.customId, values: [handle.value] },id,+ ...where,message: { id: handle.messageId },token: `tok-${id}`,type: 3,- user: PERSON,version: 1,}),);diff --git a/packages/eve/src/internal/testing/channel-conformance/github-driver.ts b/packages/eve/src/internal/testing/channel-conformance/github-driver.tsindex 96aad056c..a1da551be 100644--- a/packages/eve/src/internal/testing/channel-conformance/github-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/github-driver.ts@@ -64,6 +64,7 @@ export function githubDriver(): ChannelDriver {return {name: "github",capabilities: ["text-replies"],+ surface: "public",createChannel: (record) =>githubChannel({api: { fetch: recordingFetch(record, decode) },diff --git a/packages/eve/src/internal/testing/channel-conformance/harness.ts b/packages/eve/src/internal/testing/channel-conformance/harness.tsindex 3dbd4193f..a4f605acf 100644--- a/packages/eve/src/internal/testing/channel-conformance/harness.ts+++ b/packages/eve/src/internal/testing/channel-conformance/harness.ts@@ -62,6 +62,19 @@ export type ChannelCapability =/** A person can send a plain-text message to the conversation. */| "text-replies";+/**+ * Who can see the conversation a driver holds. Anything meant for the person+ * alone must reach them privately on a shared surface and can't be shown at+ * all on a public one.+ */+export type Surface =+ /** Anyone, such as a GitHub issue. */+ | "public"+ /** Members of a channel, group, or workspace, such as a Slack channel thread. Most agents run here. */+ | "shared"+ /** Only the person, such as a direct message or a local terminal. */+ | "private";+/*** Teaches the HITL conformance suite to speak one channel's platform protocol.*@@ -72,6 +85,7 @@ export type ChannelCapability =export interface ChannelDriver {readonly name: string;readonly capabilities: readonly ChannelCapability[];+ readonly surface: Surface;/*** Builds the channel against a fake platform that reports each outbound call* to `record`. HTTP platforms use {@link recordingFetch}.@@ -151,6 +165,7 @@ export interface ChannelConversation {export interface ClientDriver {readonly name: string;readonly capabilities: readonly ChannelCapability[];+ readonly surface: Surface;/*** Starts the client against the agent at `host`. The global `fetch` serves* that origin from the eve channel's routes until the client closes.@@ -502,7 +517,10 @@ async function converse();const conversation: ChannelConversation = {- say: (text) => view.say(text),+ async say(text) {+ await waitForStepsToFinish([...sessions.values()], wait);+ await view.say(text);+ },press: (option) => view.press(option),async waitForQuestion(prompt) {const { options } = await view.waitForQuestion([prompt]);@@ -603,20 +621,48 @@ const TERMINAL_STEP_STATUSES = new Set(["completed", "failed", "cancelled"]);* still running. A turn that starts after the first cancel needs another.*/async function cancelUntilResting(session: Session): Promise<void> {- const world = await getWorld();const deadline = Date.now() + WAIT_TIMEOUT_MS;+ let last: string | undefined;+ let running: string[] = [];while (Date.now() < deadline) {await session.cancel();const tail = await session.getStreamTailIndex();const reader = (await session.getEventStream({ startIndex: tail })).getReader();- const last = await reader.read().finally(() => reader.cancel());- if (last.value?.type === "session.waiting") {- const steps = await world.steps.list({ resolveData: "none", runId: session.id });- if (steps.data.every((step) => TERMINAL_STEP_STATUSES.has(step.status))) return;- }+ last = (await reader.read().finally(() => reader.cancel())).value?.type;+ running = await runningSteps(session);+ if (last === "session.waiting" && running.length === 0) return;await new Promise((resolve) => setTimeout(resolve, 25));}- throw new Error(`Timed out waiting for session ${session.id} to rest.`);+ throw new Error(+ `Timed out waiting for session ${session.id} to rest. Its stream ends with ${last ?? "nothing"}` ++ (running.length === 0 ? "." : `, and these steps are still running: ${running.join(", ")}.`),+ );+}++/** The names of `session`'s steps that haven't finished. */+async function runningSteps(session: Session): Promise<string[]> {+ const world = await getWorld();+ const steps = await world.steps.list({ resolveData: "none", runId: session.id });+ return steps.data+ .filter((step) => !TERMINAL_STEP_STATUSES.has(step.status))+ .map((step) => `${step.stepName} (${step.status})`);+}++/**+ * Waits until no session has a step running. A person writes once the bot has+ * finished, and a channel claims the address of a message it posted, such as+ * the Telegram message a person replies to, only once the step that posted it+ * commits. Writing sooner starts a second session instead of continuing this one.+ */+async function waitForStepsToFinish(sessions: readonly Session[], wait: Wait): Promise<void> {+ await wait(+ "every session's steps to finish",+ async () => {+ const running = await Promise.all(sessions.map(runningSteps));+ return running.every((steps) => steps.length === 0) ? true : undefined;+ },+ () => "",+ );}/** How long a session's stream must stay unchanged to count as resting. */diff --git a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linear-driver.tsindex 937cc73c1..e27f2a747 100644--- a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts@@ -63,6 +63,8 @@ export function linearDriver(): ChannelDriver {return {name: "linear",capabilities: ["text-replies"],+ // An agent session lives on an issue the whole workspace can see.+ surface: "shared",createChannel: (record) =>linearChannel({api: { fetch: recordingFetch(record, decode) },diff --git a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linq-driver.tsindex a18d1c8a3..68e1a03d9 100644--- a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts@@ -4,6 +4,7 @@ import { linqChannel } from "#public/channels/linq/index.js";import {type ChannelDriver,type PlatformCall,+ type Surface,numberedOptions,recordingFetch,} from "#internal/testing/channel-conformance/harness.js";@@ -24,8 +25,9 @@ interface LinqMessageBody {readonly message?: { readonly parts?: readonly LinqPart[] };}-/** Drives the real Linq adapter through a signed direct-message webhook. */-export function linqDriver(): ChannelDriver {+/** Drives the real Linq adapter through signed webhooks, in a group chat by default or 1:1. */+export function linqDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {+ const group = surface === "shared";nextChat += 1;const chatId = `linq-conformance-chat-${nextChat}`;let messageId = 0;@@ -35,7 +37,8 @@ export function linqDriver(): ChannelDriver {messageId += 1;const body = JSON.stringify({data: {- chat: { id: chatId, is_group: false },+ // Linq's bridge hears every group message; a mention would route past its handler.+ chat: { id: chatId, is_group: group },direction: "inbound",id: `linq-inbound-${messageId}`,parts: [{ type: "text", value: text }],@@ -61,8 +64,9 @@ export function linqDriver(): ChannelDriver {}return {- name: "linq",+ name: group ? "linq" : "linq-dm",capabilities: ["text-replies"],+ surface,createChannel(record) {const previousFetch = globalThis.fetch;const fakeFetch = recordingFetch(record, async (request) => {diff --git a/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts b/packages/eve/src/internal/testing/channel-conformance/slack-driver.tsindex 3654a3af0..f6ed510e4 100644--- a/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/slack-driver.ts@@ -4,6 +4,7 @@ import { slackChannel } from "#public/channels/slack/index.js";import { HITL_ACTION_PREFIX } from "#public/channels/slack/hitl.js";import {type ChannelDriver,+ type Surface,type PlatformCall,type RenderedOption,recordingFetch,@@ -12,6 +13,7 @@ import { decodeSlackApiBody } from "#internal/testing/slack-api-body.js";const SIGNING_SECRET = "slack-conformance-secret";const PERSON = "U_ALICE";+const BOT = "U_EVE";let nextChannel = 0;const TEAM = "T01";@@ -47,11 +49,15 @@ interface SlackMessageBody {readonly ts?: string;}-/** Drives the Slack channel through its Events API and interactivity webhooks in a DM. */-export function slackDriver(): ChannelDriver {- // A fresh DM per driver keeps each test's session apart in the shared workflow world.+/**+ * Drives the Slack channel through its Events API and interactivity webhooks,+ * in a public channel thread by default or in a DM.+ */+export function slackDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {+ const dm = surface === "private";+ // A fresh channel per driver keeps each test's session apart in the shared workflow world.nextChannel += 1;- const CHANNEL = `D${String(nextChannel).padStart(3, "0")}`;+ const CHANNEL = `${dm ? "D" : "C"}${String(nextChannel).padStart(3, "0")}`;let sequence = 0;// Every message is a reply in the thread the first one starts.const threadTs = "1700000000.000001";@@ -91,14 +97,15 @@ export function slackDriver(): ChannelDriver {ok: true,team_id: TEAM,ts,- user_id: "U_EVE",+ user_id: BOT,},};}return {- name: "slack",+ name: dm ? "slack-dm" : "slack",capabilities: ["buttons", "text-replies"],+ surface,createChannel: (record) =>slackChannel({api: { fetch: recordingFetch(record, decode) },@@ -108,18 +115,26 @@ export function slackDriver(): ChannelDriver {const ts = threadStarted ? nextTs() : threadTs;const thread = threadStarted ? { thread_ts: threadTs } : {};threadStarted = true;+ // In a channel the default policy hears only mentions, so a person mentions the bot each+ // time. On its own line, so the test model's line-based directives still read the message.+ const event = dm+ ? { channel_type: "im", text, type: "message" }+ : { channel_type: "channel", text: `${text}\n<@${BOT}>`, type: "app_mention" };return signed(JSON.stringify({event: {...thread,+ ...event,channel: CHANNEL,- channel_type: "im",event_ts: ts,- text,ts,- type: "message",user: PERSON,},+ // Slack names the installation that received the event, which is how eve knows its+ // own bot user, e.g. to strip that mention from a typed answer.+ authorizations: [+ { is_bot: true, is_enterprise_install: false, team_id: TEAM, user_id: BOT },+ ],event_id: `Ev${ts}`,team_id: TEAM,type: "event_callback",diff --git a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts b/packages/eve/src/internal/testing/channel-conformance/teams-driver.tsindex 325a3f671..4314b1d33 100644--- a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts@@ -3,6 +3,7 @@ import {type ChannelDriver,type PlatformCall,type RenderedOption,+ type Surface,recordingFetch,} from "#internal/testing/channel-conformance/harness.js";@@ -11,6 +12,7 @@ const TENANT = "TENANT";const PERSON = { id: "USER", name: "Alice" } as const;const BOT = { id: "BOT", name: "eve Bot" } as const;const SERVICE_URL = "https://smba.example.test/teams";+const MENTION = `<at>${BOT.name}</at>`;interface Choice {readonly title: string;@@ -47,11 +49,24 @@ interface PressHandle {readonly value?: string;}-/** Drives Teams' Bot Framework webhook and Connector API in a personal chat. */-export function teamsDriver(): ChannelDriver {+/**+ * Drives Teams' Bot Framework webhook and Connector API, in a team channel's+ * thread by default or in a personal chat.+ */+export function teamsDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {+ const personal = surface === "private";// Conversation ids are part of Teams' continuation token, so isolate each driver instance.nextConversation += 1;- const conversationId = `CONV-${nextConversation}`;+ // A channel thread's conversation id names the post that started it.+ const conversationId = personal+ ? `CONV-${nextConversation}`+ : `CONV-${nextConversation};messageid=MSG-1`;+ const conversation = personal+ ? { conversationType: "personal", id: conversationId }+ : { conversationType: "channel", id: conversationId };+ const channelData = personal+ ? { tenant: { id: TENANT } }+ : { channel: { id: "CHANNEL" }, team: { id: "TEAM" }, tenant: { id: TENANT } };let activityId = 0;function nextActivityId(): string {@@ -62,8 +77,8 @@ export function teamsDriver(): ChannelDriver {function activity(payload: Record<string, unknown>): Request {return new Request("https://agent.example.com/eve/v1/teams", {body: JSON.stringify({- channelData: { tenant: { id: TENANT } },- conversation: { id: conversationId, conversationType: "personal" },+ channelData,+ conversation,from: PERSON,id: `MSG-${activityId + 1}`,recipient: BOT,@@ -85,14 +100,25 @@ export function teamsDriver(): ChannelDriver {}return {- name: "teams",+ name: personal ? "teams-dm" : "teams",capabilities: ["buttons", "text-replies"],+ surface,createChannel: (record) =>teamsChannel({api: { fetch: recordingFetch(record, decode) },credentials: { tokenProvider: () => "test-token", webhookVerifier: () => true },}),- message: (text) => activity({ text, type: "message" }),+ message: (text) =>+ personal+ ? activity({ text, type: "message" })+ : // In a channel the default policy hears only mentions, so a person mentions the bot+ // each time, on its own line so the test model's line-based directives still read it.+ activity({+ entities: [{ mentioned: BOT, text: MENTION, type: "mention" }],+ text: `${text}\n${MENTION}`,+ textFormat: "xml",+ type: "message",+ }),findOptions(call, prompt) {const body = call.body as ActivityBody;if (body.type !== "message" || !body.text?.includes(prompt)) return undefined;diff --git a/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts b/packages/eve/src/internal/testing/channel-conformance/telegram-driver.tsindex adeadee8a..b704bfe1c 100644--- a/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/telegram-driver.ts@@ -3,10 +3,12 @@ import {type ChannelDriver,type PlatformCall,type RenderedOption,+ type Surface,recordingFetch,} from "#internal/testing/channel-conformance/harness.js";const SECRET = "telegram-conformance-secret";+const BOT = { first_name: "eve", id: 99, is_bot: true, username: "testbot" } as const;let nextChatId = 1000;interface InlineButton {@@ -50,14 +52,37 @@ function inlineOptions(call: PlatformCall): RenderedOption[] {);}-/** Drives the Telegram channel through its Bot API webhook in a private chat. */-export function telegramDriver(): ChannelDriver {+/**+ * Drives the Telegram channel through its Bot API webhook, in a supergroup by+ * default or in a private chat.+ */+export function telegramDriver(surface: Exclude<Surface, "public"> = "shared"): ChannelDriver {+ const group = surface === "shared";// A fresh chat per driver keeps each test's session apart in the shared workflow world.nextChatId += 1;- const CHAT = { id: nextChatId, type: "private" } as const;+ const CHAT = group+ ? { id: -nextChatId, title: "Release crew", type: "supergroup" }+ : { id: nextChatId, type: "private" };const PERSON = { first_name: "Alice", id: nextChatId, is_bot: false } as const;let updateId = 0;let messageId = 0;+ /** The bot's latest message, which a person in a group replies to so the bot hears them. */+ let lastBotMessage: number | undefined;++ /**+ * A group's default policy hears only mentions and replies to the bot, and a+ * mention starts a new conversation, so a person mentions the bot once and+ * then replies to its latest message. The mention sits on its own line so+ * the test model's line-based directives still read the message.+ */+ function addressed(text: string): Record<string, unknown> {+ if (!group) return { text };+ if (lastBotMessage === undefined) return { text: `${text}\n@${BOT.username}` };+ return {+ reply_to_message: { chat: CHAT, date: 0, from: BOT, message_id: lastBotMessage },+ text,+ };+ }function update(payload: Record<string, unknown>): Request {updateId += 1;@@ -75,6 +100,7 @@ export function telegramDriver(): ChannelDriver {const method = new URL(request.url).pathname.split("/").at(-1)!;const text = await request.text();messageId += 1;+ if (method === "sendMessage") lastBotMessage = messageId;return {body: text === "" ? {} : JSON.parse(text),method,@@ -83,15 +109,25 @@ export function telegramDriver(): ChannelDriver {}return {- name: "telegram",+ name: group ? "telegram" : "telegram-dm",capabilities: ["buttons", "text-replies"],+ surface,createChannel: (record) =>telegramChannel({api: { fetch: recordingFetch(record, decode) },+ botUsername: BOT.username,credentials: { botToken: "bot-token", webhookSecretToken: SECRET },}),message: (text) =>- update({ message: { chat: CHAT, date: 0, from: PERSON, message_id: 1000 + updateId, text } }),+ update({+ message: {+ chat: CHAT,+ date: 0,+ from: PERSON,+ message_id: 1000 + updateId,+ ...addressed(text),+ },+ }),findOptions(call, prompt) {const body = call.body as MessageBody;if (call.method !== "sendMessage" || body.text?.includes(prompt) !== true) return undefined;diff --git a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts b/packages/eve/src/internal/testing/channel-conformance/tui-driver.tsindex 46e5aedd0..63fdd4520 100644--- a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts@@ -27,6 +27,8 @@ export function tuiDriver(): ClientDriver {return {name: "tui",capabilities: ["buttons", "text-replies"],+ // A local terminal shows only the person at it.+ surface: "private",async open(host, wait) {// Wide enough that no reply wraps, so each one stays on one screen row.const screen = new MockScreen({ columns: 1000, rows: 60 });diff --git a/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts b/packages/eve/src/internal/testing/channel-conformance/twilio-driver.tsindex 7b9a86e99..ecbffb6d4 100644--- a/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts@@ -48,6 +48,7 @@ export function twilioDriver(): ChannelDriver {return {name: "twilio",capabilities: ["text-replies"],+ surface: "private",createChannel: (record) =>twilioChannel({allowFrom: "*",diff --git a/packages/eve/src/public/channels/chat-sdk/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/chat-sdk/hitl-conformance-dm.integration.test.tsnew file mode 100644index 000000000..e880c1d3d--- /dev/null+++ b/packages/eve/src/public/channels/chat-sdk/hitl-conformance-dm.integration.test.ts@@ -0,0 +1,3 @@+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";++describeHitlConformance("chat-sdk-dm");diff --git a/packages/eve/src/public/channels/discord/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/discord/hitl-conformance-dm.integration.test.tsnew file mode 100644index 000000000..af54ab402--- /dev/null+++ b/packages/eve/src/public/channels/discord/hitl-conformance-dm.integration.test.ts@@ -0,0 +1,3 @@+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";++describeHitlConformance("discord-dm");diff --git a/packages/eve/src/public/channels/linq/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/linq/hitl-conformance-dm.integration.test.tsnew file mode 100644index 000000000..ed2b482b4--- /dev/null+++ b/packages/eve/src/public/channels/linq/hitl-conformance-dm.integration.test.ts@@ -0,0 +1,3 @@+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";++describeHitlConformance("linq-dm");diff --git a/packages/eve/src/public/channels/slack/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/slack/hitl-conformance-dm.integration.test.tsnew file mode 100644index 000000000..554f6c4fe--- /dev/null+++ b/packages/eve/src/public/channels/slack/hitl-conformance-dm.integration.test.ts@@ -0,0 +1,3 @@+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";++describeHitlConformance("slack-dm");diff --git a/packages/eve/src/public/channels/teams/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/teams/hitl-conformance-dm.integration.test.tsnew file mode 100644index 000000000..bd460bb34--- /dev/null+++ b/packages/eve/src/public/channels/teams/hitl-conformance-dm.integration.test.ts@@ -0,0 +1,3 @@+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";++describeHitlConformance("teams-dm");diff --git a/packages/eve/src/public/channels/telegram/hitl-conformance-dm.integration.test.ts b/packages/eve/src/public/channels/telegram/hitl-conformance-dm.integration.test.tsnew file mode 100644index 000000000..6c5231115--- /dev/null+++ b/packages/eve/src/public/channels/telegram/hitl-conformance-dm.integration.test.ts@@ -0,0 +1,3 @@+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";++describeHitlConformance("telegram-dm");diff --git a/packages/eve/src/public/channels/telegram/hitl.test.ts b/packages/eve/src/public/channels/telegram/hitl.test.tsindex c40ca8652..ce025f2fc 100644--- a/packages/eve/src/public/channels/telegram/hitl.test.ts+++ b/packages/eve/src/public/channels/telegram/hitl.test.ts@@ -2,13 +2,11 @@ import { describe, expect, it } from "vitest";import {TELEGRAM_CALLBACK_RESPONSE_PREFIX,- TELEGRAM_HITL_CALLBACK_PREFIX,TELEGRAM_REPLY_RESPONSE_PREFIX,isTelegramSyntheticResponse,registerTelegramFreeformPrompt,renderTelegramInputRequest,resolveTelegramInputResponses,- telegramCallbackInputResponse,telegramReplyInputResponse,type TelegramHitlState,} from "#public/channels/telegram/hitl.js";@@ -63,28 +61,6 @@ describe("renderTelegramInputRequest", () => {});describe("resolveTelegramInputResponses", () => {- it("maps a compact callback id to its request on every press", () => {- const state: TelegramHitlState = {- hitlCallbacks: {- [`${TELEGRAM_HITL_CALLBACK_PREFIX}0`]: {- optionId: "approve",- requestId: "call_1",- },- },- };-- expect(- resolveTelegramInputResponses(state, [- telegramCallbackInputResponse(`${TELEGRAM_HITL_CALLBACK_PREFIX}0`),- ]),- ).toEqual([{ optionId: "approve", requestId: "call_1" }]);- expect(- resolveTelegramInputResponses(state, [- telegramCallbackInputResponse(`${TELEGRAM_HITL_CALLBACK_PREFIX}0`),- ]),- ).toEqual([{ optionId: "approve", requestId: "call_1" }]);- });-it("maps replies to ForceReply prompts back to freeform answers", () => {const state: TelegramHitlState = {};registerTelegramFreeformPrompt(state, { messageId: "99", requestId: "call_1" });diff --git a/packages/eve/vitest.integration.config.ts b/packages/eve/vitest.integration.config.tsindex 8695edf59..0f61179cd 100644--- a/packages/eve/vitest.integration.config.ts+++ b/packages/eve/vitest.integration.config.ts@@ -32,7 +32,12 @@ export default defineConfig({},test: {environment: "node",- exclude: ["**/node_modules/**", "test/scenarios/**"],+ exclude: [+ "**/node_modules/**",+ "test/scenarios/**",+ // Channel conformance checks platform protocol, not the OS, and is slowest on Windows runners.+ ...(process.platform === "win32" ? ["**/hitl-conformance*.integration.test.ts"] : []),+ ],globalSetup: ["./test/setup/clear-workflow-cache.ts"],include: ["src/**/*.integration.test.ts", "test/**/*.integration.test.ts"],setupFiles: ["./test/setup/mock-ai-gateway.ts"],