The Open Framework for Building Agents
[4/6] test(eve): add conformance drivers for the major channels (#4163)
Ben Pankowbd5afcf706e3
diff --git a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.tsindex d88ece98f..80a48a68b 100644--- a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts+++ b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts@@ -2,11 +2,13 @@ import { chatSdkChannel } from "#public/channels/chat-sdk/index.js";import {type Adapter,type AdapterPostableMessage,+ BaseFormatConverter,type ChatInstance,Message,type StateAdapter,type WebhookOptions,parseMarkdown,+ toPlainText,} from "#compiled/chat/index.js";import { createMemoryState } from "#compiled/@chat-adapter/state-memory/index.js";import type { ChannelDriver, PlatformCall } from "#internal/testing/channel-conformance/harness.js";@@ -32,10 +34,76 @@ type Inbound =* Drives `chatSdkChannel` with a card-capable direct-message adapter: one* thread, no streaming, and every message handed to eve with an empty* `context`. The fake adapter is the platform: it reads inbound JSON and- * records every post and edit. Text-only adapters such as Linq and Photon- * flatten cards, so they need their own drivers.+ * records every post and edit.*/export function chatSdkDriver(): ChannelDriver {+ const driver = chatSdkDriverWith({ name: "chat-sdk", render: (posted) => posted });+ return {+ ...driver,+ capabilities: ["buttons", "text-replies"],+ findOptions(call, prompt) {+ if (!isPost(call)) return undefined;+ const card = cardOf(call.body as AdapterPostableMessage);+ if (card === undefined || !texts(card).includes(prompt)) return undefined;+ return nodes(card)+ .filter((node) => node.type === "button" && node.id !== undefined)+ .map((button) => ({ handle: button, label: button.label ?? "" }));+ },+ press(option) {+ const button = option.handle as CardNode;+ return driver.inbound({ actionId: button.id!, kind: "action", value: button.value });+ },+ };+}++/**+ * The same bridge behind a text-only adapter, as Photon iMessage's behaves:+ * every post reaches the person as the `chat` package's default plain-text+ * rendering, so a card shows only its fallback text and has nothing to press.+ * Photon's real adapter sends over gRPC, so this is the closest in-process+ * stand-in; it covers eve's bridge and the SDK fallback, not Photon itself.+ */+export function chatSdkTextDriver(): ChannelDriver {+ const converter = new PlainTextConverter();+ const driver = chatSdkDriverWith({+ name: "chat-sdk-text",+ render: (posted) => converter.renderPostable(posted),+ });+ return {+ ...driver,+ capabilities: ["text-replies"],+ findOptions(call, prompt) {+ if (!isPost(call) || typeof call.body !== "string" || !call.body.includes(prompt)) {+ return undefined;+ }+ // #3715's fallback lists each choice as `"<id>" (<label>)`.+ return [...call.body.matchAll(/"[^"]+" \(([^)]+)\)/gu)].map(([, label]) => ({+ handle: label,+ label: label!,+ }));+ },+ press() {+ throw new Error("A text-only Chat SDK adapter has nothing to press.");+ },+ };+}++class PlainTextConverter extends BaseFormatConverter {+ fromAst(ast: Parameters<BaseFormatConverter["fromAst"]>[0]): string {+ return toPlainText(ast);+ }++ toAst(text: string) {+ return parseMarkdown(text);+ }+}++function chatSdkDriverWith(input: {+ readonly name: string;+ readonly render: (posted: AdapterPostableMessage) => unknown;+}): Omit<ChannelDriver, "capabilities" | "findOptions" | "press"> & {+ inbound(body: Inbound): Request;+} {nextThread += 1;const threadId = `${ADAPTER}:D${nextThread}`;let sequence = 0;@@ -49,11 +117,12 @@ export function chatSdkDriver(): ChannelDriver {}return {- name: "chat-sdk",- capabilities: ["buttons", "text-replies"],+ name: input.name,+ inbound,createChannel(record) {+ const adapter = fakeAdapter(threadId, record, () => (sequence += 1), input.render);const bridge = chatSdkChannel({- adapters: { [ADAPTER]: fakeAdapter(threadId, record, () => (sequence += 1)) },+ adapters: { [ADAPTER]: adapter },concurrency: "concurrent",routes: { [ADAPTER]: `/eve/v1/${ADAPTER}` },state: createMemoryState() as StateAdapter,@@ -66,21 +135,8 @@ export function chatSdkDriver(): ChannelDriver {return bridge.channel;},message: (text) => inbound({ kind: "message", text }),- findOptions(call, prompt) {- if (call.method !== "postMessage" && call.method !== "editMessage") return undefined;- const card = cardOf(call.body as AdapterPostableMessage);- if (card === undefined || !texts(card).includes(prompt)) return undefined;- const buttons = nodes(card).filter((node) => node.type === "button" && node.id !== undefined);- return buttons.length === 0- ? undefined- : buttons.map((button) => ({ handle: button, label: button.label ?? "" }));- },- press(option) {- const button = option.handle as CardNode;- return inbound({ actionId: button.id!, kind: "action", value: button.value });- },postedText(call: PlatformCall) {- if (call.method !== "postMessage" && call.method !== "editMessage") return undefined;+ if (!isPost(call)) return undefined;const posted = call.body as AdapterPostableMessage;if (typeof posted === "string") return posted;if ("markdown" in posted) return posted.markdown;@@ -90,10 +146,15 @@ export function chatSdkDriver(): ChannelDriver {};}+function isPost(call: PlatformCall): boolean {+ return call.method === "postMessage" || call.method === "editMessage";+}+function fakeAdapter(threadId: string,record: (call: PlatformCall) => void,nextId: () => number,+ render: (posted: AdapterPostableMessage) => unknown,): Adapter {let chat: ChatInstance | null = null;const self = {@@ -145,11 +206,11 @@ function fakeAdapter(}),async postMessage(id: string, posted: AdapterPostableMessage) {const messageId = `posted-${nextId()}`;- record({ body: posted, method: "postMessage", response: { id: messageId } });+ record({ body: render(posted), method: "postMessage", response: { id: messageId } });return { id: messageId, raw: posted, threadId: id };},async editMessage(id: string, messageId: string, posted: AdapterPostableMessage) {- record({ body: posted, method: "editMessage", response: { id: messageId } });+ record({ body: render(posted), method: "editMessage", response: { id: messageId } });return { id: messageId, raw: posted, threadId: id };},async addReaction() {},diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.tsindex a1e8c9ce0..ae2c79eed 100644--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts@@ -24,9 +24,7 @@ const PROMPT = "Which day works for the review?";async function askWhichDay(conversation: ChannelConversation) {await conversation.say(ASK);- const options = await conversation.waitForQuestion(PROMPT);- expect(options.map((option) => option.label)).toEqual(["Saturday", "Sunday"]);- return options;+ return await conversation.waitForQuestion(PROMPT);}function expectAnsweredSaturday(output: unknown) {@@ -38,12 +36,26 @@ function expectAnsweredSaturday(output: unknown) {}export const hitlContract = [+ {+ rule: "a rendered question shows every option a person can choose",+ source: "docs/tools/human-in-the-loop.md#questions",+ requires: [],+ async run(conversation) {+ const options = await askWhichDay(conversation);+ expect(+ options.map((option) => option.label),+ `the question showed ${JSON.stringify(options.map((option) => option.label))}`,+ ).toEqual(["Saturday", "Sunday"]);+ },+ },{rule: "pressing a rendered option answers the pending question with that option",source: "docs/tools/human-in-the-loop.md#answering-from-a-client-or-channel",requires: ["buttons"],async run(conversation) {- const [saturday] = await askWhichDay(conversation);+ const options = await askWhichDay(conversation);+ const saturday = options.find((option) => option.label === "Saturday");+ expect(saturday, "a Saturday option to press").toBeDefined();await conversation.press(saturday!);expectAnsweredSaturday(await conversation.waitForToolResult("ask_question"));},diff --git a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts b/packages/eve/src/internal/testing/channel-conformance/discord-driver.tsnew file mode 100644index 000000000..f772043f9--- /dev/null+++ b/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts@@ -0,0 +1,155 @@+import { generateKeyPairSync, sign, type KeyObject } from "node:crypto";++import { discordChannel } from "#public/channels/discord/index.js";+import {+ type ChannelDriver,+ type PlatformCall,+ type RenderedOption,+ recordingFetch,+} from "#internal/testing/channel-conformance/harness.js";++let nextChannel = 0;++/** A message component eve renders for a choice: a button (type 2) or a select menu (type 3). */+interface DiscordComponent {+ readonly custom_id?: string;+ readonly label?: string;+ readonly options?: readonly { readonly label: string; readonly value: string }[];+ readonly type?: number;+}++interface PressHandle {+ readonly customId: string;+ readonly messageId: string;+ /** Set for a select menu option; a button press carries no value. */+ readonly value?: string;+}++function testKeys(): { privateKey: KeyObject; publicKeyHex: string } {+ const { privateKey, publicKey } = generateKeyPairSync("ed25519");+ const der = publicKey.export({ format: "der", type: "spki" });+ return {+ privateKey,+ publicKeyHex: Buffer.from(der).subarray(-32).toString("hex"),+ };+}++/** Drives Discord's signed application-command and component interaction webhooks. */+export function discordDriver(): ChannelDriver {+ nextChannel += 1;+ const channelId = `C_CONFORMANCE_${nextChannel}`;+ const { privateKey, publicKeyHex } = testKeys();+ let interactionId = 0;+ let messageId = 0;++ function signed(body: string): Request {+ const timestamp = String(Math.floor(Date.now() / 1000));+ const signature = sign(null, Buffer.from(`${timestamp}${body}`), privateKey).toString("hex");+ return new Request("https://agent.example.com/eve/v1/discord", {+ body,+ headers: {+ "content-type": "application/json",+ "x-signature-ed25519": signature,+ "x-signature-timestamp": timestamp,+ },+ method: "POST",+ });+ }++ async function decode(request: Request): Promise<PlatformCall> {+ const text = await request.text();+ const body = text === "" ? {} : JSON.parse(text);+ const path = new URL(request.url).pathname;+ if (path.endsWith("/typing")) {+ return { body, method: "POST", response: {} };+ }+ messageId += 1;+ return {+ body,+ method: request.method,+ response: { channel_id: channelId, id: `M_CONFORMANCE_${messageId}` },+ };+ }++ function nextInteraction(): string {+ interactionId += 1;+ return `I_CONFORMANCE_${interactionId}`;+ }++ return {+ name: "discord",+ capabilities: ["buttons"],+ createChannel: (record) =>+ discordChannel({+ api: { fetch: recordingFetch(record, decode) },+ credentials: { applicationId: "APP1", botToken: "bot-token", publicKey: publicKeyHex },+ }),+ message: (text) => {+ const id = nextInteraction();+ return signed(+ JSON.stringify({+ application_id: "APP1",+ channel: { type: 1 },+ channel_id: channelId,+ data: { name: "ask", options: [{ name: "message", type: 3, value: text }] },+ id,+ token: `tok-${id}`,+ type: 2,+ user: { id: "U_CONFORMANCE", username: "alice" },+ version: 1,+ }),+ );+ },+ findOptions(call, prompt) {+ if (call.method !== "PATCH") return undefined;+ const body = call.body as {+ readonly components?: readonly { readonly components?: readonly DiscordComponent[] }[];+ readonly content?: string;+ };+ if (body.content?.includes(prompt) !== true) return undefined;+ const messageId = (call.response as { readonly id?: string }).id ?? "";+ return (body.components ?? []).flatMap((row) =>+ (row.components ?? []).flatMap((component) => visibleChoices(component, messageId)),+ );+ },+ press(option) {+ const handle = option.handle as PressHandle;+ const id = nextInteraction();+ return signed(+ JSON.stringify({+ application_id: "APP1",+ channel_id: channelId,+ data:+ handle.value === undefined+ ? { component_type: 2, custom_id: handle.customId }+ : { component_type: 3, custom_id: handle.customId, values: [handle.value] },+ id,+ message: { id: handle.messageId },+ token: `tok-${id}`,+ type: 3,+ user: { id: "U_CONFORMANCE", username: "alice" },+ version: 1,+ }),+ );+ },+ postedText(call) {+ if (call.method !== "POST" && call.method !== "PATCH") return undefined;+ return (call.body as { readonly content?: string }).content;+ },+ };+}++function visibleChoices(component: DiscordComponent, messageId: string): RenderedOption[] {+ if (component.custom_id === undefined) return [];+ const customId = component.custom_id;+ if (component.type === 2 && component.label !== undefined) {+ return [{ handle: { customId, messageId } satisfies PressHandle, label: component.label }];+ }+ if (component.type === 3) {+ return (component.options ?? []).map((option) => ({+ handle: { customId, messageId, value: option.value } satisfies PressHandle,+ label: option.label,+ }));+ }+ return [];+}diff --git a/packages/eve/src/internal/testing/channel-conformance/github-driver.ts b/packages/eve/src/internal/testing/channel-conformance/github-driver.tsnew file mode 100644index 000000000..96aad056c--- /dev/null+++ b/packages/eve/src/internal/testing/channel-conformance/github-driver.ts@@ -0,0 +1,92 @@+import { githubChannel } from "#public/channels/github/index.js";+import { signGitHubWebhookBody } from "#public/channels/github/verify.js";+import {+ type ChannelDriver,+ type PlatformCall,+ type RenderedOption,+ recordingFetch,+} from "#internal/testing/channel-conformance/harness.js";++const SECRET = "github-conformance-secret";+let nextConversation = 0;++/** Drives GitHub issue comments through the signed App webhook route. */+export function githubDriver(): ChannelDriver {+ // Repository and issue identities are the continuation key, so isolate each driver instance.+ nextConversation += 1;+ const repositoryId = 10_000 + nextConversation;+ const issueNumber = 100 + nextConversation;+ let delivery = 0;+ let commentId = 0;++ function message(text: string): Request {+ delivery += 1;+ commentId += 1;+ const payload = {+ action: "created",+ comment: {+ body: `@testbot ${text}`,+ id: commentId,+ user: { id: 1, login: "octocat", type: "User" },+ },+ installation: { id: 55 },+ issue: { number: issueNumber },+ repository: {+ full_name: "vercel/eve",+ id: repositoryId,+ name: "eve",+ owner: { login: "vercel" },+ },+ sender: { id: 1, login: "octocat", type: "User" },+ };+ const body = JSON.stringify(payload);+ return new Request("https://agent.example.com/eve/v1/github", {+ body,+ headers: {+ "content-type": "application/json",+ "x-github-delivery": `github-conformance-${delivery}`,+ "x-github-event": "issue_comment",+ "x-hub-signature-256": signGitHubWebhookBody(body, SECRET),+ },+ method: "POST",+ });+ }++ async function decode(request: Request): Promise<PlatformCall> {+ const bodyText = await request.text();+ return {+ body: bodyText === "" ? {} : JSON.parse(bodyText),+ method: new URL(request.url).pathname,+ response: { id: 50_000 + commentId },+ };+ }++ return {+ name: "github",+ capabilities: ["text-replies"],+ createChannel: (record) =>+ githubChannel({+ api: { fetch: recordingFetch(record, decode) },+ botName: "testbot",+ credentials: { installationToken: "ghs_test", webhookSecret: SECRET },+ progress: { reactions: false },+ }),+ message,+ findOptions(call, prompt) {+ if (!call.method.endsWith(`/issues/${issueNumber}/comments`)) return undefined;+ const body = call.body as { readonly body?: string };+ if (body.body?.includes(prompt) !== true) return undefined;+ return [...(body.body.matchAll(/^\s*(\d+)\.\s*(.+?)\s*$/gmu) ?? [])].map((match) => ({+ handle: match[1],+ label: match[2]!.split(" - ", 1)[0]!,+ })) satisfies RenderedOption[];+ },+ press: () => {+ throw new Error("GitHub does not support pressing rendered options.");+ },+ postedText: (call) => {+ if (!call.method.endsWith(`/issues/${issueNumber}/comments`)) return undefined;+ return (call.body as { readonly body?: string }).body;+ },+ };+}diff --git a/packages/eve/src/internal/testing/channel-conformance/harness.ts b/packages/eve/src/internal/testing/channel-conformance/harness.tsindex 57d75d2fd..2059aa703 100644--- a/packages/eve/src/internal/testing/channel-conformance/harness.ts+++ b/packages/eve/src/internal/testing/channel-conformance/harness.ts@@ -47,9 +47,16 @@ export interface ChannelDriver {* to `record`. HTTP platforms use {@link recordingFetch}.*/createChannel(record: (call: PlatformCall) => void): unknown;+ /** Undoes anything `createChannel` installed outside the channel, such as a global `fetch`. */+ dispose?(): void;/** A webhook request carrying a person's message. */message(text: string): Request;- /** Options rendered in one outbound call for a question, or `undefined` when it isn't one. */+ /**+ * The options a person can see in one outbound call that posts the question:+ * `undefined` when the call isn't the question, `[]` when it shows no options.+ * Read what the platform displays (buttons, or labels in the text), not eve's+ * request metadata.+ */findOptions(call: PlatformCall, prompt: string): readonly RenderedOption[] | undefined;/** A webhook request pressing a rendered option. */press(option: RenderedOption): Request;@@ -69,7 +76,7 @@ export interface ChannelConversation {waitForToolResult(tool: string): Promise<unknown>;}-const WAIT_TIMEOUT_MS = 15_000;+const WAIT_TIMEOUT_MS = 10_000;/** A `fetch` for an HTTP platform API: `decode` turns each request into a call and its answer. */export function recordingFetch(@@ -86,13 +93,25 @@ export function recordingFetch(/*** Runs `body` against an agent with `ask_question` and `driver`'s channel. Every* interaction goes through the channel's real webhook routes; the only fake is- * the platform API behind the channel's injected `fetch`.+ * the platform behind the channel, usually its injected `fetch`.*/export async function withChannelConversation(driver: ChannelDriver,body: (conversation: ChannelConversation) => Promise<void>,): Promise<void> {const calls: PlatformCall[] = [];+ try {+ await converse(driver, calls, body);+ } finally {+ driver.dispose?.();+ }+}++async function converse(+ driver: ChannelDriver,+ calls: PlatformCall[],+ body: (conversation: ChannelConversation) => Promise<void>,+): Promise<void> {const created = driver.createChannel((call) => void calls.push(call));if (!isCompiledChannel(created)) throw new Error(`${driver.name} is not a compiled channel.`);const channel: CompiledChannel = created;diff --git a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linear-driver.tsnew file mode 100644index 000000000..937cc73c1--- /dev/null+++ b/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts@@ -0,0 +1,125 @@+import {+ type ChannelDriver,+ type PlatformCall,+ type RenderedOption,+ recordingFetch,+} from "#internal/testing/channel-conformance/harness.js";+import { linearChannel } from "#public/channels/linear/index.js";+import { signLinearWebhookBody } from "#public/channels/linear/verify.js";++const SECRET = "linear-conformance-secret";+let nextConversation = 0;++/** Drives Linear Agent Session webhooks and Agent Activity GraphQL mutations. */+export function linearDriver(): ChannelDriver {+ nextConversation += 1;+ const sessionId = `agent-session-conformance-${nextConversation}`;+ const issueId = `issue-conformance-${nextConversation}`;+ const issueIdentifier = `EVE-${nextConversation}`;+ let delivery = 0;+ let activity = 0;++ function webhook(payload: Record<string, unknown>): Request {+ const body = JSON.stringify({+ type: "AgentSessionEvent",+ webhookTimestamp: Date.now(),+ organizationId: "org_1",+ appUserId: "app_user_1",+ ...payload,+ });+ delivery += 1;+ return new Request("https://agent.example.com/eve/v1/linear", {+ body,+ headers: {+ "content-type": "application/json",+ "linear-delivery": `delivery-${delivery}`,+ "linear-event": "AgentSessionEvent",+ "linear-signature": signLinearWebhookBody(body, SECRET),+ },+ method: "POST",+ });+ }++ async function decode(request: Request): Promise<PlatformCall> {+ const body = JSON.parse(await request.text()) as {+ readonly query?: string;+ readonly variables?: unknown;+ };+ activity += 1;+ return {+ body,+ method: body.query?.includes("AgentActivityCreate") ? "AgentActivityCreate" : "GraphQL",+ response: {+ data: {+ agentActivityCreate: {+ agentActivity: { id: `activity-${activity}` },+ success: true,+ },+ },+ },+ };+ }++ return {+ name: "linear",+ capabilities: ["text-replies"],+ createChannel: (record) =>+ linearChannel({+ api: { fetch: recordingFetch(record, decode) },+ credentials: { accessToken: "linear-token", webhookSecret: SECRET },+ }),+ message: (text) => {+ if (delivery === 0) {+ return webhook({+ action: "created",+ agentSession: {+ creator: { id: "user_1" },+ id: sessionId,+ issue: { id: issueId, identifier: issueIdentifier, title: "Conformance test" },+ issueId,+ },+ promptContext: text,+ });+ }+ return webhook({+ action: "prompted",+ agentSession: { id: sessionId },+ agentActivity: {+ content: { body: text, type: "prompt" },+ id: `prompt-${delivery}`,+ user: { id: "user_1" },+ userId: "user_1",+ },+ });+ },+ findOptions(call, prompt) {+ if (call.method !== "AgentActivityCreate") return undefined;+ const input = (call.body as { readonly variables?: { readonly input?: LinearActivityInput } })+ .variables?.input;+ if (input?.content?.type !== "elicitation" || input.content.body.includes(prompt) === false) {+ return undefined;+ }+ return parseVisibleOptions(input.content.body);+ },+ press: () => {+ throw new Error("Linear native select pressing is not supported by this conformance driver.");+ },+ postedText(call) {+ if (call.method !== "AgentActivityCreate") return undefined;+ const input = (call.body as { readonly variables?: { readonly input?: LinearActivityInput } })+ .variables?.input;+ return input?.content?.type === "response" ? input.content.body : undefined;+ },+ };+}++interface LinearActivityInput {+ readonly content?: { readonly body: string; readonly type: string };+}++function parseVisibleOptions(body: string): RenderedOption[] {+ return [...body.matchAll(/^\s*(\d+)\.\s+(.+)\s*$/gmu)].map((match) => ({+ handle: match[1],+ label: match[2]!.trim().split(" - ", 1)[0]!.trim(),+ }));+}diff --git a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linq-driver.tsnew file mode 100644index 000000000..2d1c0a19b--- /dev/null+++ b/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts@@ -0,0 +1,118 @@+import { createHmac } from "node:crypto";++import { linqChannel } from "#public/channels/linq/index.js";+import {+ type ChannelDriver,+ type PlatformCall,+ recordingFetch,+} from "#internal/testing/channel-conformance/harness.js";++const API_KEY = "linq-conformance-api-key";+const SIGNING_KEY = Buffer.from("linq-conformance-signing-key");+const SIGNING_SECRET = `whsec_${SIGNING_KEY.toString("base64")}`;+const BASE_URL = "https://linq-conformance.invalid/api/partner";+const PERSON = "alice";+let nextChat = 0;++interface LinqPart {+ readonly type?: string;+ readonly value?: string;+}++interface LinqMessageBody {+ readonly message?: { readonly parts?: readonly LinqPart[] };+}++/** Drives the real Linq adapter through a signed direct-message webhook. */+export function linqDriver(): ChannelDriver {+ nextChat += 1;+ const chatId = `linq-conformance-chat-${nextChat}`;+ let messageId = 0;+ let restoreFetch: (() => void) | undefined;++ function signedMessage(text: string): Request {+ messageId += 1;+ const body = JSON.stringify({+ data: {+ chat: { id: chatId, is_group: false },+ direction: "inbound",+ id: `linq-inbound-${messageId}`,+ parts: [{ type: "text", value: text }],+ sender_handle: { handle: PERSON, id: PERSON, is_me: false },+ },+ event_type: "message.received",+ });+ const timestamp = Math.floor(Date.now() / 1000);+ const id = `linq-webhook-${messageId}`;+ const signature = createHmac("sha256", SIGNING_KEY)+ .update(`${id}.${timestamp}.${body}`)+ .digest("base64");+ return new Request("https://agent.example.com/eve/v1/linq", {+ body,+ headers: {+ "content-type": "application/json",+ "webhook-id": id,+ "webhook-signature": `v1,${signature}`,+ "webhook-timestamp": String(timestamp),+ },+ method: "POST",+ });+ }++ return {+ name: "linq",+ capabilities: ["text-replies"],+ createChannel(record) {+ const previousFetch = globalThis.fetch;+ const fakeFetch = recordingFetch(record, async (request) => {+ const url = new URL(request.url);+ const bodyText = await request.text();+ return {+ body: bodyText === "" ? {} : JSON.parse(bodyText),+ method: `${request.method} ${url.pathname}`,+ response: {},+ };+ });+ // The adapter accepts no fetch option, so route only its test host globally and preserve other fetches.+ globalThis.fetch = async (input, init) => {+ const url = new URL(input instanceof Request ? input.url : input.toString());+ if (url.hostname === new URL(BASE_URL).hostname) return fakeFetch(input, init);+ return previousFetch(input, init);+ };+ restoreFetch = () => {+ globalThis.fetch = previousFetch;+ restoreFetch = undefined;+ };+ return linqChannel({+ baseURL: BASE_URL,+ credentials: { apiKey: API_KEY, signingSecret: SIGNING_SECRET },+ });+ },+ dispose() {+ restoreFetch?.();+ },+ message: signedMessage,+ findOptions(call, prompt) {+ if (!call.method.endsWith("/messages") || !postedText(call)?.includes(prompt))+ return undefined;+ const text = postedText(call)!;+ const options = text.match(/Options:\s*(.+)$/u)?.[1];+ return options === undefined+ ? []+ : options.split(",").map((label) => ({ handle: label.trim(), label: label.trim() }));+ },+ press() {+ throw new Error("Linq has no pressable HITL controls.");+ },+ postedText,+ };+}++function postedText(call: PlatformCall): string | undefined {+ if (!call.method.endsWith("/messages")) return undefined;+ const parts = (call.body as LinqMessageBody).message?.parts;+ return parts+ ?.filter((part) => part.type === "text")+ .map((part) => part.value ?? "")+ .join("");+}diff --git a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts b/packages/eve/src/internal/testing/channel-conformance/teams-driver.tsnew file mode 100644index 000000000..e96f89c5f--- /dev/null+++ b/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts@@ -0,0 +1,124 @@+import { teamsChannel } from "#public/channels/teams/index.js";+import {+ type ChannelDriver,+ type PlatformCall,+ type RenderedOption,+ recordingFetch,+} from "#internal/testing/channel-conformance/harness.js";++let nextConversation = 0;+const TENANT = "TENANT";+const PERSON = { id: "USER", name: "Alice" } as const;+const BOT = { id: "BOT", name: "eve Bot" } as const;+const SERVICE_URL = "https://smba.example.test/teams";++interface Choice {+ readonly title: string;+ readonly value: string;+}++interface AdaptiveCard {+ readonly body?: readonly {+ readonly choices?: readonly Choice[];+ readonly type?: string;+ }[];+ readonly actions?: readonly {+ readonly data?: Record<string, unknown>;+ readonly type?: string;+ }[];+}++interface ActivityBody {+ readonly attachments?: readonly {+ readonly content?: AdaptiveCard;+ readonly contentType?: string;+ }[];+ readonly text?: string;+ readonly type?: string;+}++interface PressHandle {+ readonly data: Record<string, unknown>;+ readonly value: string;+}++/** Drives Teams' Bot Framework webhook and Connector API in a personal chat. */+export function teamsDriver(): ChannelDriver {+ // Conversation ids are part of Teams' continuation token, so isolate each driver instance.+ nextConversation += 1;+ const conversationId = `CONV-${nextConversation}`;+ let activityId = 0;++ function nextActivityId(): string {+ activityId += 1;+ return `ACT-${activityId}`;+ }++ function activity(payload: Record<string, unknown>): Request {+ return new Request("https://agent.example.com/eve/v1/teams", {+ body: JSON.stringify({+ channelData: { tenant: { id: TENANT } },+ conversation: { id: conversationId, conversationType: "personal" },+ from: PERSON,+ id: `MSG-${activityId + 1}`,+ recipient: BOT,+ serviceUrl: SERVICE_URL,+ ...payload,+ }),+ headers: { "content-type": "application/json" },+ method: "POST",+ });+ }++ async function decode(request: Request): Promise<PlatformCall> {+ const bodyText = await request.text();+ const body = bodyText === "" ? {} : JSON.parse(bodyText);+ return {+ body,+ method: new URL(request.url).pathname.split("/").at(-1) ?? "",+ response: { id: nextActivityId() },+ };+ }++ return {+ name: "teams",+ capabilities: ["buttons", "text-replies"],+ createChannel: (record) =>+ teamsChannel({+ api: { fetch: recordingFetch(record, decode) },+ credentials: { tokenProvider: () => "test-token", webhookVerifier: () => true },+ }),+ message: (text) => activity({ text, type: "message" }),+ findOptions(call, prompt) {+ const body = call.body as ActivityBody;+ if (body.type === "typing" || body.type !== "message" || !body.text?.includes(prompt)) {+ return undefined;+ }+ const card = body.attachments?.find(+ (attachment) => attachment.contentType === "application/vnd.microsoft.card.adaptive",+ )?.content;+ const choiceSet = card?.body?.find((element) => element.type === "Input.ChoiceSet");+ const submit = card?.actions?.find((action) => action.type === "Action.Submit");+ const data = submit?.data;+ if (choiceSet?.choices === undefined || data === undefined) return [];+ return choiceSet.choices.map((choice) => ({+ handle: { data, value: choice.value } satisfies PressHandle,+ label: choice.title,+ }));+ },+ press: (option: RenderedOption) => {+ const { data, value } = option.handle as PressHandle;+ return activity({+ id: `INVOKE-${activityId + 1}`,+ name: "adaptiveCard/action",+ type: "invoke",+ value: { action: { data: { ...data, eve_option: value } } },+ });+ },+ postedText(call) {+ const body = call.body as ActivityBody;+ if (body.type === "typing") return undefined;+ return body.type === "message" ? body.text : undefined;+ },+ };+}diff --git a/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts b/packages/eve/src/internal/testing/channel-conformance/twilio-driver.tsnew file mode 100644index 000000000..1a46ad828--- /dev/null+++ b/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts@@ -0,0 +1,76 @@+import { twilioChannel } from "#public/channels/twilio/index.js";+import { signTwilioRequest } from "#public/channels/twilio/verify.js";+import {+ type ChannelDriver,+ type PlatformCall,+ recordingFetch,+} from "#internal/testing/channel-conformance/harness.js";++const AUTH_TOKEN = "twilio-conformance-secret";+const TO = "+15550000001";+let nextPerson = 0;++/** Drives SMS webhooks and Twilio's form-encoded Messages API. */+export function twilioDriver(): ChannelDriver {+ nextPerson += 1;+ const PERSON = `+1555000${String(nextPerson).padStart(4, "0")}`;+ let messageSid = 0;+ const webhookUrl = "https://agent.example.com/eve/v1/twilio/messages";++ function message(text: string): Request {+ messageSid += 1;+ const params = new URLSearchParams({+ Body: text,+ From: PERSON,+ MessageSid: `SM-conformance-${messageSid}`,+ To: TO,+ });+ return new Request(webhookUrl, {+ body: params,+ headers: {+ "content-type": "application/x-www-form-urlencoded",+ "x-twilio-signature": signTwilioRequest({ authToken: AUTH_TOKEN, params, url: webhookUrl }),+ },+ method: "POST",+ });+ }++ async function decode(request: Request): Promise<PlatformCall> {+ const url = new URL(request.url);+ return {+ body: Object.fromEntries(new URLSearchParams(await request.text())),+ method: url.pathname.split("/").at(-1) ?? "",+ response: { sid: "SM-conformance-response" },+ };+ }++ return {+ name: "twilio",+ capabilities: ["text-replies"],+ createChannel: (record) =>+ twilioChannel({+ allowFrom: "*",+ credentials: { accountSid: "AC123", authToken: AUTH_TOKEN },+ api: { fetch: recordingFetch(record, decode) },+ messaging: { from: TO },+ }),+ message,+ findOptions(call, prompt) {+ if (call.method !== "Messages.json") return undefined;+ const body = call.body as { readonly Body?: unknown };+ if (typeof body.Body !== "string" || !body.Body.includes(prompt)) return undefined;+ return body.Body.split("\n")+ .map((line) => /^\s*\d+[.)]\s*(.+?)\s*$/u.exec(line)?.[1])+ .filter((label): label is string => label !== undefined)+ .map((label) => ({ handle: label, label }));+ },+ press: () => {+ throw new Error("Twilio SMS does not support pressing options.");+ },+ postedText(call) {+ if (call.method !== "Messages.json") return undefined;+ const body = call.body as { readonly Body?: unknown };+ return typeof body.Body === "string" ? body.Body : undefined;+ },+ };+}diff --git a/packages/eve/src/public/channels/discord/discordChannel.test.ts b/packages/eve/src/public/channels/discord/discordChannel.test.tsindex e27c2fdb8..7ac485658 100644--- a/packages/eve/src/public/channels/discord/discordChannel.test.ts+++ b/packages/eve/src/public/channels/discord/discordChannel.test.ts@@ -289,7 +289,7 @@ describe("discordChannel() inbound route", () => {);});- it("delivers HITL button clicks as inputResponses", async () => {+ it("acknowledges HITL button clicks with a deferred update", async () => {const { privateKey, publicKeyHex } = testKeys();const components = renderInputRequestComponents({action: { callId: "call_1", input: {}, kind: "tool-call", toolName: "ask_question" },@@ -313,17 +313,10 @@ describe("discordChannel() inbound route", () => {});const channel = discordChannel({ credentials: { publicKey: publicKeyHex } });- const { response, send } = await firePost(channel, signedRequest({ body, privateKey }));+ const { response } = await firePost(channel, signedRequest({ body, privateKey }));expect(response.status).toBe(200);await expect(response.json()).resolves.toEqual({ type: 6 });- expect(send).toHaveBeenCalledWith(- "C01:M01",- expect.objectContaining({- auth: null,- inputResponses: [{ optionId: "approve", requestId: "call_1" }],- }),- );});it("opens and resolves freeform HITL modals", async () => {diff --git a/packages/eve/src/public/channels/hitl-conformance.integration.test.ts b/packages/eve/src/public/channels/hitl-conformance.integration.test.tsindex ce0b90078..51162122f 100644--- a/packages/eve/src/public/channels/hitl-conformance.integration.test.ts+++ b/packages/eve/src/public/channels/hitl-conformance.integration.test.ts@@ -5,9 +5,18 @@ import {type ChannelDriver,withChannelConversation,} from "#internal/testing/channel-conformance/harness.js";-import { chatSdkDriver } from "#internal/testing/channel-conformance/chat-sdk-driver.js";+import {+ chatSdkDriver,+ chatSdkTextDriver,+} from "#internal/testing/channel-conformance/chat-sdk-driver.js";+import { discordDriver } from "#internal/testing/channel-conformance/discord-driver.js";+import { githubDriver } from "#internal/testing/channel-conformance/github-driver.js";+import { linearDriver } from "#internal/testing/channel-conformance/linear-driver.js";+import { linqDriver } from "#internal/testing/channel-conformance/linq-driver.js";import { slackDriver } from "#internal/testing/channel-conformance/slack-driver.js";+import { teamsDriver } from "#internal/testing/channel-conformance/teams-driver.js";import { telegramDriver } from "#internal/testing/channel-conformance/telegram-driver.js";+import { twilioDriver } from "#internal/testing/channel-conformance/twilio-driver.js";interface BrokenCell {readonly reason: string;@@ -27,7 +36,38 @@ interface BrokenCell {const channels: readonly {readonly driver: () => ChannelDriver;readonly broken?: Partial<Record<HitlRule, BrokenCell>>;-}[] = [{ driver: chatSdkDriver }, { driver: slackDriver }, { driver: telegramDriver }];+}[] = [+ { driver: chatSdkDriver },+ {+ driver: chatSdkTextDriver,+ broken: {+ "a rendered question shows every option a person can choose": {+ reason: "#3712: the card's fallback text is only the prompt",+ symptom: /the question showed \[\]/,+ },+ },+ },+ { driver: discordDriver },+ { driver: githubDriver },+ { driver: linearDriver },+ { driver: linqDriver },+ { driver: slackDriver },+ { driver: teamsDriver },+ { driver: telegramDriver },+ {+ driver: twilioDriver,+ broken: {+ "a rendered question shows every option a person can choose": {+ reason: "the channel never sends the question (no input.requested handler)",+ symptom: /Timed out waiting for the question "Which day works for the review\?" on twilio/,+ },+ "a text reply matching an option answers the only pending question": {+ reason: "the channel never sends the question (no input.requested handler)",+ symptom: /Timed out waiting for the question "Which day works for the review\?" on twilio/,+ },+ },+ },+];describe.each(channels.map((entry) => ({ ...entry, name: entry.driver().name })))("$name HITL contract",diff --git a/packages/eve/src/public/channels/teams/hitl.test.ts b/packages/eve/src/public/channels/teams/hitl.test.tsindex 7f4254e97..5b3cf9bb7 100644--- a/packages/eve/src/public/channels/teams/hitl.test.ts+++ b/packages/eve/src/public/channels/teams/hitl.test.ts@@ -5,7 +5,6 @@ import {isTeamsToolApprovalResponseActivity,readTeamsInputReplyToActivityId,renderInputRequestMessage,- TEAMS_HITL_CHOICE_INPUT_ID,TEAMS_HITL_DATA_KEY,TEAMS_HITL_FREEFORM_INPUT_ID,TEAMS_HITL_PROMPT_KEY,@@ -48,12 +47,6 @@ describe("Teams HITL helpers", () => {});});- it("renders select requests with a ChoiceSet", () => {- const body = renderInputRequestMessage({ ...request(), display: "select" });- const card = body.attachments?.[0]?.content as { body?: Array<Record<string, unknown>> };- expect(card.body?.some((entry) => entry.id === TEAMS_HITL_CHOICE_INPUT_ID)).toBe(true);- });-it("decodes message and invoke submission values", () => {const message = parseTeamsActivity(activityWithValue({