Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

[4/6] test(eve): add conformance drivers for the major channels (#4163)

Ben Pankowbd5afcf706e3
diff --git a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts
index d88ece98f..80a48a68b 100644
--- a/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/chat-sdk-driver.ts
@@ -2,11 +2,13 @@ import { chatSdkChannel } from "#public/channels/chat-sdk/index.js";
import {
type Adapter,
type AdapterPostableMessage,
+ BaseFormatConverter,
type ChatInstance,
Message,
type StateAdapter,
type WebhookOptions,
parseMarkdown,
+ toPlainText,
} from "#compiled/chat/index.js";
import { createMemoryState } from "#compiled/@chat-adapter/state-memory/index.js";
import type { ChannelDriver, PlatformCall } from "#internal/testing/channel-conformance/harness.js";
@@ -32,10 +34,76 @@ type Inbound =
* Drives `chatSdkChannel` with a card-capable direct-message adapter: one
* thread, no streaming, and every message handed to eve with an empty
* `context`. The fake adapter is the platform: it reads inbound JSON and
- * records every post and edit. Text-only adapters such as Linq and Photon
- * flatten cards, so they need their own drivers.
+ * records every post and edit.
*/
export function chatSdkDriver(): ChannelDriver {
+ const driver = chatSdkDriverWith({ name: "chat-sdk", render: (posted) => posted });
+ return {
+ ...driver,
+ capabilities: ["buttons", "text-replies"],
+ findOptions(call, prompt) {
+ if (!isPost(call)) return undefined;
+ const card = cardOf(call.body as AdapterPostableMessage);
+ if (card === undefined || !texts(card).includes(prompt)) return undefined;
+ return nodes(card)
+ .filter((node) => node.type === "button" && node.id !== undefined)
+ .map((button) => ({ handle: button, label: button.label ?? "" }));
+ },
+ press(option) {
+ const button = option.handle as CardNode;
+ return driver.inbound({ actionId: button.id!, kind: "action", value: button.value });
+ },
+ };
+}
+
+/**
+ * The same bridge behind a text-only adapter, as Photon iMessage's behaves:
+ * every post reaches the person as the `chat` package's default plain-text
+ * rendering, so a card shows only its fallback text and has nothing to press.
+ * Photon's real adapter sends over gRPC, so this is the closest in-process
+ * stand-in; it covers eve's bridge and the SDK fallback, not Photon itself.
+ */
+export function chatSdkTextDriver(): ChannelDriver {
+ const converter = new PlainTextConverter();
+ const driver = chatSdkDriverWith({
+ name: "chat-sdk-text",
+ render: (posted) => converter.renderPostable(posted),
+ });
+ return {
+ ...driver,
+ capabilities: ["text-replies"],
+ findOptions(call, prompt) {
+ if (!isPost(call) || typeof call.body !== "string" || !call.body.includes(prompt)) {
+ return undefined;
+ }
+ // #3715's fallback lists each choice as `"<id>" (<label>)`.
+ return [...call.body.matchAll(/"[^"]+" \(([^)]+)\)/gu)].map(([, label]) => ({
+ handle: label,
+ label: label!,
+ }));
+ },
+ press() {
+ throw new Error("A text-only Chat SDK adapter has nothing to press.");
+ },
+ };
+}
+
+class PlainTextConverter extends BaseFormatConverter {
+ fromAst(ast: Parameters<BaseFormatConverter["fromAst"]>[0]): string {
+ return toPlainText(ast);
+ }
+
+ toAst(text: string) {
+ return parseMarkdown(text);
+ }
+}
+
+function chatSdkDriverWith(input: {
+ readonly name: string;
+ readonly render: (posted: AdapterPostableMessage) => unknown;
+}): Omit<ChannelDriver, "capabilities" | "findOptions" | "press"> & {
+ inbound(body: Inbound): Request;
+} {
nextThread += 1;
const threadId = `${ADAPTER}:D${nextThread}`;
let sequence = 0;
@@ -49,11 +117,12 @@ export function chatSdkDriver(): ChannelDriver {
}
return {
- name: "chat-sdk",
- capabilities: ["buttons", "text-replies"],
+ name: input.name,
+ inbound,
createChannel(record) {
+ const adapter = fakeAdapter(threadId, record, () => (sequence += 1), input.render);
const bridge = chatSdkChannel({
- adapters: { [ADAPTER]: fakeAdapter(threadId, record, () => (sequence += 1)) },
+ adapters: { [ADAPTER]: adapter },
concurrency: "concurrent",
routes: { [ADAPTER]: `/eve/v1/${ADAPTER}` },
state: createMemoryState() as StateAdapter,
@@ -66,21 +135,8 @@ export function chatSdkDriver(): ChannelDriver {
return bridge.channel;
},
message: (text) => inbound({ kind: "message", text }),
- findOptions(call, prompt) {
- if (call.method !== "postMessage" && call.method !== "editMessage") return undefined;
- const card = cardOf(call.body as AdapterPostableMessage);
- if (card === undefined || !texts(card).includes(prompt)) return undefined;
- const buttons = nodes(card).filter((node) => node.type === "button" && node.id !== undefined);
- return buttons.length === 0
- ? undefined
- : buttons.map((button) => ({ handle: button, label: button.label ?? "" }));
- },
- press(option) {
- const button = option.handle as CardNode;
- return inbound({ actionId: button.id!, kind: "action", value: button.value });
- },
postedText(call: PlatformCall) {
- if (call.method !== "postMessage" && call.method !== "editMessage") return undefined;
+ if (!isPost(call)) return undefined;
const posted = call.body as AdapterPostableMessage;
if (typeof posted === "string") return posted;
if ("markdown" in posted) return posted.markdown;
@@ -90,10 +146,15 @@ export function chatSdkDriver(): ChannelDriver {
};
}
+function isPost(call: PlatformCall): boolean {
+ return call.method === "postMessage" || call.method === "editMessage";
+}
+
function fakeAdapter(
threadId: string,
record: (call: PlatformCall) => void,
nextId: () => number,
+ render: (posted: AdapterPostableMessage) => unknown,
): Adapter {
let chat: ChatInstance | null = null;
const self = {
@@ -145,11 +206,11 @@ function fakeAdapter(
}),
async postMessage(id: string, posted: AdapterPostableMessage) {
const messageId = `posted-${nextId()}`;
- record({ body: posted, method: "postMessage", response: { id: messageId } });
+ record({ body: render(posted), method: "postMessage", response: { id: messageId } });
return { id: messageId, raw: posted, threadId: id };
},
async editMessage(id: string, messageId: string, posted: AdapterPostableMessage) {
- record({ body: posted, method: "editMessage", response: { id: messageId } });
+ record({ body: render(posted), method: "editMessage", response: { id: messageId } });
return { id: messageId, raw: posted, threadId: id };
},
async addReaction() {},
diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.ts
index a1e8c9ce0..ae2c79eed 100644
--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts
@@ -24,9 +24,7 @@ const PROMPT = "Which day works for the review?";
async function askWhichDay(conversation: ChannelConversation) {
await conversation.say(ASK);
- const options = await conversation.waitForQuestion(PROMPT);
- expect(options.map((option) => option.label)).toEqual(["Saturday", "Sunday"]);
- return options;
+ return await conversation.waitForQuestion(PROMPT);
}
function expectAnsweredSaturday(output: unknown) {
@@ -38,12 +36,26 @@ function expectAnsweredSaturday(output: unknown) {
}
export const hitlContract = [
+ {
+ rule: "a rendered question shows every option a person can choose",
+ source: "docs/tools/human-in-the-loop.md#questions",
+ requires: [],
+ async run(conversation) {
+ const options = await askWhichDay(conversation);
+ expect(
+ options.map((option) => option.label),
+ `the question showed ${JSON.stringify(options.map((option) => option.label))}`,
+ ).toEqual(["Saturday", "Sunday"]);
+ },
+ },
{
rule: "pressing a rendered option answers the pending question with that option",
source: "docs/tools/human-in-the-loop.md#answering-from-a-client-or-channel",
requires: ["buttons"],
async run(conversation) {
- const [saturday] = await askWhichDay(conversation);
+ const options = await askWhichDay(conversation);
+ const saturday = options.find((option) => option.label === "Saturday");
+ expect(saturday, "a Saturday option to press").toBeDefined();
await conversation.press(saturday!);
expectAnsweredSaturday(await conversation.waitForToolResult("ask_question"));
},
diff --git a/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts b/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts
new file mode 100644
index 000000000..f772043f9
--- /dev/null
+++ b/packages/eve/src/internal/testing/channel-conformance/discord-driver.ts
@@ -0,0 +1,155 @@
+import { generateKeyPairSync, sign, type KeyObject } from "node:crypto";
+
+import { discordChannel } from "#public/channels/discord/index.js";
+import {
+ type ChannelDriver,
+ type PlatformCall,
+ type RenderedOption,
+ recordingFetch,
+} from "#internal/testing/channel-conformance/harness.js";
+
+let nextChannel = 0;
+
+/** A message component eve renders for a choice: a button (type 2) or a select menu (type 3). */
+interface DiscordComponent {
+ readonly custom_id?: string;
+ readonly label?: string;
+ readonly options?: readonly { readonly label: string; readonly value: string }[];
+ readonly type?: number;
+}
+
+interface PressHandle {
+ readonly customId: string;
+ readonly messageId: string;
+ /** Set for a select menu option; a button press carries no value. */
+ readonly value?: string;
+}
+
+function testKeys(): { privateKey: KeyObject; publicKeyHex: string } {
+ const { privateKey, publicKey } = generateKeyPairSync("ed25519");
+ const der = publicKey.export({ format: "der", type: "spki" });
+ return {
+ privateKey,
+ publicKeyHex: Buffer.from(der).subarray(-32).toString("hex"),
+ };
+}
+
+/** Drives Discord's signed application-command and component interaction webhooks. */
+export function discordDriver(): ChannelDriver {
+ nextChannel += 1;
+ const channelId = `C_CONFORMANCE_${nextChannel}`;
+ const { privateKey, publicKeyHex } = testKeys();
+ let interactionId = 0;
+ let messageId = 0;
+
+ function signed(body: string): Request {
+ const timestamp = String(Math.floor(Date.now() / 1000));
+ const signature = sign(null, Buffer.from(`${timestamp}${body}`), privateKey).toString("hex");
+ return new Request("https://agent.example.com/eve/v1/discord", {
+ body,
+ headers: {
+ "content-type": "application/json",
+ "x-signature-ed25519": signature,
+ "x-signature-timestamp": timestamp,
+ },
+ method: "POST",
+ });
+ }
+
+ async function decode(request: Request): Promise<PlatformCall> {
+ const text = await request.text();
+ const body = text === "" ? {} : JSON.parse(text);
+ const path = new URL(request.url).pathname;
+ if (path.endsWith("/typing")) {
+ return { body, method: "POST", response: {} };
+ }
+ messageId += 1;
+ return {
+ body,
+ method: request.method,
+ response: { channel_id: channelId, id: `M_CONFORMANCE_${messageId}` },
+ };
+ }
+
+ function nextInteraction(): string {
+ interactionId += 1;
+ return `I_CONFORMANCE_${interactionId}`;
+ }
+
+ return {
+ name: "discord",
+ capabilities: ["buttons"],
+ createChannel: (record) =>
+ discordChannel({
+ api: { fetch: recordingFetch(record, decode) },
+ credentials: { applicationId: "APP1", botToken: "bot-token", publicKey: publicKeyHex },
+ }),
+ message: (text) => {
+ const id = nextInteraction();
+ return signed(
+ JSON.stringify({
+ application_id: "APP1",
+ channel: { type: 1 },
+ channel_id: channelId,
+ data: { name: "ask", options: [{ name: "message", type: 3, value: text }] },
+ id,
+ token: `tok-${id}`,
+ type: 2,
+ user: { id: "U_CONFORMANCE", username: "alice" },
+ version: 1,
+ }),
+ );
+ },
+ findOptions(call, prompt) {
+ if (call.method !== "PATCH") return undefined;
+ const body = call.body as {
+ readonly components?: readonly { readonly components?: readonly DiscordComponent[] }[];
+ readonly content?: string;
+ };
+ if (body.content?.includes(prompt) !== true) return undefined;
+ const messageId = (call.response as { readonly id?: string }).id ?? "";
+ return (body.components ?? []).flatMap((row) =>
+ (row.components ?? []).flatMap((component) => visibleChoices(component, messageId)),
+ );
+ },
+ press(option) {
+ const handle = option.handle as PressHandle;
+ const id = nextInteraction();
+ return signed(
+ JSON.stringify({
+ application_id: "APP1",
+ channel_id: channelId,
+ data:
+ handle.value === undefined
+ ? { component_type: 2, custom_id: handle.customId }
+ : { component_type: 3, custom_id: handle.customId, values: [handle.value] },
+ id,
+ message: { id: handle.messageId },
+ token: `tok-${id}`,
+ type: 3,
+ user: { id: "U_CONFORMANCE", username: "alice" },
+ version: 1,
+ }),
+ );
+ },
+ postedText(call) {
+ if (call.method !== "POST" && call.method !== "PATCH") return undefined;
+ return (call.body as { readonly content?: string }).content;
+ },
+ };
+}
+
+function visibleChoices(component: DiscordComponent, messageId: string): RenderedOption[] {
+ if (component.custom_id === undefined) return [];
+ const customId = component.custom_id;
+ if (component.type === 2 && component.label !== undefined) {
+ return [{ handle: { customId, messageId } satisfies PressHandle, label: component.label }];
+ }
+ if (component.type === 3) {
+ return (component.options ?? []).map((option) => ({
+ handle: { customId, messageId, value: option.value } satisfies PressHandle,
+ label: option.label,
+ }));
+ }
+ return [];
+}
diff --git a/packages/eve/src/internal/testing/channel-conformance/github-driver.ts b/packages/eve/src/internal/testing/channel-conformance/github-driver.ts
new file mode 100644
index 000000000..96aad056c
--- /dev/null
+++ b/packages/eve/src/internal/testing/channel-conformance/github-driver.ts
@@ -0,0 +1,92 @@
+import { githubChannel } from "#public/channels/github/index.js";
+import { signGitHubWebhookBody } from "#public/channels/github/verify.js";
+import {
+ type ChannelDriver,
+ type PlatformCall,
+ type RenderedOption,
+ recordingFetch,
+} from "#internal/testing/channel-conformance/harness.js";
+
+const SECRET = "github-conformance-secret";
+let nextConversation = 0;
+
+/** Drives GitHub issue comments through the signed App webhook route. */
+export function githubDriver(): ChannelDriver {
+ // Repository and issue identities are the continuation key, so isolate each driver instance.
+ nextConversation += 1;
+ const repositoryId = 10_000 + nextConversation;
+ const issueNumber = 100 + nextConversation;
+ let delivery = 0;
+ let commentId = 0;
+
+ function message(text: string): Request {
+ delivery += 1;
+ commentId += 1;
+ const payload = {
+ action: "created",
+ comment: {
+ body: `@testbot ${text}`,
+ id: commentId,
+ user: { id: 1, login: "octocat", type: "User" },
+ },
+ installation: { id: 55 },
+ issue: { number: issueNumber },
+ repository: {
+ full_name: "vercel/eve",
+ id: repositoryId,
+ name: "eve",
+ owner: { login: "vercel" },
+ },
+ sender: { id: 1, login: "octocat", type: "User" },
+ };
+ const body = JSON.stringify(payload);
+ return new Request("https://agent.example.com/eve/v1/github", {
+ body,
+ headers: {
+ "content-type": "application/json",
+ "x-github-delivery": `github-conformance-${delivery}`,
+ "x-github-event": "issue_comment",
+ "x-hub-signature-256": signGitHubWebhookBody(body, SECRET),
+ },
+ method: "POST",
+ });
+ }
+
+ async function decode(request: Request): Promise<PlatformCall> {
+ const bodyText = await request.text();
+ return {
+ body: bodyText === "" ? {} : JSON.parse(bodyText),
+ method: new URL(request.url).pathname,
+ response: { id: 50_000 + commentId },
+ };
+ }
+
+ return {
+ name: "github",
+ capabilities: ["text-replies"],
+ createChannel: (record) =>
+ githubChannel({
+ api: { fetch: recordingFetch(record, decode) },
+ botName: "testbot",
+ credentials: { installationToken: "ghs_test", webhookSecret: SECRET },
+ progress: { reactions: false },
+ }),
+ message,
+ findOptions(call, prompt) {
+ if (!call.method.endsWith(`/issues/${issueNumber}/comments`)) return undefined;
+ const body = call.body as { readonly body?: string };
+ if (body.body?.includes(prompt) !== true) return undefined;
+ return [...(body.body.matchAll(/^\s*(\d+)\.\s*(.+?)\s*$/gmu) ?? [])].map((match) => ({
+ handle: match[1],
+ label: match[2]!.split(" - ", 1)[0]!,
+ })) satisfies RenderedOption[];
+ },
+ press: () => {
+ throw new Error("GitHub does not support pressing rendered options.");
+ },
+ postedText: (call) => {
+ if (!call.method.endsWith(`/issues/${issueNumber}/comments`)) return undefined;
+ return (call.body as { readonly body?: string }).body;
+ },
+ };
+}
diff --git a/packages/eve/src/internal/testing/channel-conformance/harness.ts b/packages/eve/src/internal/testing/channel-conformance/harness.ts
index 57d75d2fd..2059aa703 100644
--- a/packages/eve/src/internal/testing/channel-conformance/harness.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/harness.ts
@@ -47,9 +47,16 @@ export interface ChannelDriver {
* to `record`. HTTP platforms use {@link recordingFetch}.
*/
createChannel(record: (call: PlatformCall) => void): unknown;
+ /** Undoes anything `createChannel` installed outside the channel, such as a global `fetch`. */
+ dispose?(): void;
/** A webhook request carrying a person's message. */
message(text: string): Request;
- /** Options rendered in one outbound call for a question, or `undefined` when it isn't one. */
+ /**
+ * The options a person can see in one outbound call that posts the question:
+ * `undefined` when the call isn't the question, `[]` when it shows no options.
+ * Read what the platform displays (buttons, or labels in the text), not eve's
+ * request metadata.
+ */
findOptions(call: PlatformCall, prompt: string): readonly RenderedOption[] | undefined;
/** A webhook request pressing a rendered option. */
press(option: RenderedOption): Request;
@@ -69,7 +76,7 @@ export interface ChannelConversation {
waitForToolResult(tool: string): Promise<unknown>;
}
-const WAIT_TIMEOUT_MS = 15_000;
+const WAIT_TIMEOUT_MS = 10_000;
/** A `fetch` for an HTTP platform API: `decode` turns each request into a call and its answer. */
export function recordingFetch(
@@ -86,13 +93,25 @@ export function recordingFetch(
/**
* Runs `body` against an agent with `ask_question` and `driver`'s channel. Every
* interaction goes through the channel's real webhook routes; the only fake is
- * the platform API behind the channel's injected `fetch`.
+ * the platform behind the channel, usually its injected `fetch`.
*/
export async function withChannelConversation(
driver: ChannelDriver,
body: (conversation: ChannelConversation) => Promise<void>,
): Promise<void> {
const calls: PlatformCall[] = [];
+ try {
+ await converse(driver, calls, body);
+ } finally {
+ driver.dispose?.();
+ }
+}
+
+async function converse(
+ driver: ChannelDriver,
+ calls: PlatformCall[],
+ body: (conversation: ChannelConversation) => Promise<void>,
+): Promise<void> {
const created = driver.createChannel((call) => void calls.push(call));
if (!isCompiledChannel(created)) throw new Error(`${driver.name} is not a compiled channel.`);
const channel: CompiledChannel = created;
diff --git a/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts
new file mode 100644
index 000000000..937cc73c1
--- /dev/null
+++ b/packages/eve/src/internal/testing/channel-conformance/linear-driver.ts
@@ -0,0 +1,125 @@
+import {
+ type ChannelDriver,
+ type PlatformCall,
+ type RenderedOption,
+ recordingFetch,
+} from "#internal/testing/channel-conformance/harness.js";
+import { linearChannel } from "#public/channels/linear/index.js";
+import { signLinearWebhookBody } from "#public/channels/linear/verify.js";
+
+const SECRET = "linear-conformance-secret";
+let nextConversation = 0;
+
+/** Drives Linear Agent Session webhooks and Agent Activity GraphQL mutations. */
+export function linearDriver(): ChannelDriver {
+ nextConversation += 1;
+ const sessionId = `agent-session-conformance-${nextConversation}`;
+ const issueId = `issue-conformance-${nextConversation}`;
+ const issueIdentifier = `EVE-${nextConversation}`;
+ let delivery = 0;
+ let activity = 0;
+
+ function webhook(payload: Record<string, unknown>): Request {
+ const body = JSON.stringify({
+ type: "AgentSessionEvent",
+ webhookTimestamp: Date.now(),
+ organizationId: "org_1",
+ appUserId: "app_user_1",
+ ...payload,
+ });
+ delivery += 1;
+ return new Request("https://agent.example.com/eve/v1/linear", {
+ body,
+ headers: {
+ "content-type": "application/json",
+ "linear-delivery": `delivery-${delivery}`,
+ "linear-event": "AgentSessionEvent",
+ "linear-signature": signLinearWebhookBody(body, SECRET),
+ },
+ method: "POST",
+ });
+ }
+
+ async function decode(request: Request): Promise<PlatformCall> {
+ const body = JSON.parse(await request.text()) as {
+ readonly query?: string;
+ readonly variables?: unknown;
+ };
+ activity += 1;
+ return {
+ body,
+ method: body.query?.includes("AgentActivityCreate") ? "AgentActivityCreate" : "GraphQL",
+ response: {
+ data: {
+ agentActivityCreate: {
+ agentActivity: { id: `activity-${activity}` },
+ success: true,
+ },
+ },
+ },
+ };
+ }
+
+ return {
+ name: "linear",
+ capabilities: ["text-replies"],
+ createChannel: (record) =>
+ linearChannel({
+ api: { fetch: recordingFetch(record, decode) },
+ credentials: { accessToken: "linear-token", webhookSecret: SECRET },
+ }),
+ message: (text) => {
+ if (delivery === 0) {
+ return webhook({
+ action: "created",
+ agentSession: {
+ creator: { id: "user_1" },
+ id: sessionId,
+ issue: { id: issueId, identifier: issueIdentifier, title: "Conformance test" },
+ issueId,
+ },
+ promptContext: text,
+ });
+ }
+ return webhook({
+ action: "prompted",
+ agentSession: { id: sessionId },
+ agentActivity: {
+ content: { body: text, type: "prompt" },
+ id: `prompt-${delivery}`,
+ user: { id: "user_1" },
+ userId: "user_1",
+ },
+ });
+ },
+ findOptions(call, prompt) {
+ if (call.method !== "AgentActivityCreate") return undefined;
+ const input = (call.body as { readonly variables?: { readonly input?: LinearActivityInput } })
+ .variables?.input;
+ if (input?.content?.type !== "elicitation" || input.content.body.includes(prompt) === false) {
+ return undefined;
+ }
+ return parseVisibleOptions(input.content.body);
+ },
+ press: () => {
+ throw new Error("Linear native select pressing is not supported by this conformance driver.");
+ },
+ postedText(call) {
+ if (call.method !== "AgentActivityCreate") return undefined;
+ const input = (call.body as { readonly variables?: { readonly input?: LinearActivityInput } })
+ .variables?.input;
+ return input?.content?.type === "response" ? input.content.body : undefined;
+ },
+ };
+}
+
+interface LinearActivityInput {
+ readonly content?: { readonly body: string; readonly type: string };
+}
+
+function parseVisibleOptions(body: string): RenderedOption[] {
+ return [...body.matchAll(/^\s*(\d+)\.\s+(.+)\s*$/gmu)].map((match) => ({
+ handle: match[1],
+ label: match[2]!.trim().split(" - ", 1)[0]!.trim(),
+ }));
+}
diff --git a/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts b/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts
new file mode 100644
index 000000000..2d1c0a19b
--- /dev/null
+++ b/packages/eve/src/internal/testing/channel-conformance/linq-driver.ts
@@ -0,0 +1,118 @@
+import { createHmac } from "node:crypto";
+
+import { linqChannel } from "#public/channels/linq/index.js";
+import {
+ type ChannelDriver,
+ type PlatformCall,
+ recordingFetch,
+} from "#internal/testing/channel-conformance/harness.js";
+
+const API_KEY = "linq-conformance-api-key";
+const SIGNING_KEY = Buffer.from("linq-conformance-signing-key");
+const SIGNING_SECRET = `whsec_${SIGNING_KEY.toString("base64")}`;
+const BASE_URL = "https://linq-conformance.invalid/api/partner";
+const PERSON = "alice";
+let nextChat = 0;
+
+interface LinqPart {
+ readonly type?: string;
+ readonly value?: string;
+}
+
+interface LinqMessageBody {
+ readonly message?: { readonly parts?: readonly LinqPart[] };
+}
+
+/** Drives the real Linq adapter through a signed direct-message webhook. */
+export function linqDriver(): ChannelDriver {
+ nextChat += 1;
+ const chatId = `linq-conformance-chat-${nextChat}`;
+ let messageId = 0;
+ let restoreFetch: (() => void) | undefined;
+
+ function signedMessage(text: string): Request {
+ messageId += 1;
+ const body = JSON.stringify({
+ data: {
+ chat: { id: chatId, is_group: false },
+ direction: "inbound",
+ id: `linq-inbound-${messageId}`,
+ parts: [{ type: "text", value: text }],
+ sender_handle: { handle: PERSON, id: PERSON, is_me: false },
+ },
+ event_type: "message.received",
+ });
+ const timestamp = Math.floor(Date.now() / 1000);
+ const id = `linq-webhook-${messageId}`;
+ const signature = createHmac("sha256", SIGNING_KEY)
+ .update(`${id}.${timestamp}.${body}`)
+ .digest("base64");
+ return new Request("https://agent.example.com/eve/v1/linq", {
+ body,
+ headers: {
+ "content-type": "application/json",
+ "webhook-id": id,
+ "webhook-signature": `v1,${signature}`,
+ "webhook-timestamp": String(timestamp),
+ },
+ method: "POST",
+ });
+ }
+
+ return {
+ name: "linq",
+ capabilities: ["text-replies"],
+ createChannel(record) {
+ const previousFetch = globalThis.fetch;
+ const fakeFetch = recordingFetch(record, async (request) => {
+ const url = new URL(request.url);
+ const bodyText = await request.text();
+ return {
+ body: bodyText === "" ? {} : JSON.parse(bodyText),
+ method: `${request.method} ${url.pathname}`,
+ response: {},
+ };
+ });
+ // The adapter accepts no fetch option, so route only its test host globally and preserve other fetches.
+ globalThis.fetch = async (input, init) => {
+ const url = new URL(input instanceof Request ? input.url : input.toString());
+ if (url.hostname === new URL(BASE_URL).hostname) return fakeFetch(input, init);
+ return previousFetch(input, init);
+ };
+ restoreFetch = () => {
+ globalThis.fetch = previousFetch;
+ restoreFetch = undefined;
+ };
+ return linqChannel({
+ baseURL: BASE_URL,
+ credentials: { apiKey: API_KEY, signingSecret: SIGNING_SECRET },
+ });
+ },
+ dispose() {
+ restoreFetch?.();
+ },
+ message: signedMessage,
+ findOptions(call, prompt) {
+ if (!call.method.endsWith("/messages") || !postedText(call)?.includes(prompt))
+ return undefined;
+ const text = postedText(call)!;
+ const options = text.match(/Options:\s*(.+)$/u)?.[1];
+ return options === undefined
+ ? []
+ : options.split(",").map((label) => ({ handle: label.trim(), label: label.trim() }));
+ },
+ press() {
+ throw new Error("Linq has no pressable HITL controls.");
+ },
+ postedText,
+ };
+}
+
+function postedText(call: PlatformCall): string | undefined {
+ if (!call.method.endsWith("/messages")) return undefined;
+ const parts = (call.body as LinqMessageBody).message?.parts;
+ return parts
+ ?.filter((part) => part.type === "text")
+ .map((part) => part.value ?? "")
+ .join("");
+}
diff --git a/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts b/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts
new file mode 100644
index 000000000..e96f89c5f
--- /dev/null
+++ b/packages/eve/src/internal/testing/channel-conformance/teams-driver.ts
@@ -0,0 +1,124 @@
+import { teamsChannel } from "#public/channels/teams/index.js";
+import {
+ type ChannelDriver,
+ type PlatformCall,
+ type RenderedOption,
+ recordingFetch,
+} from "#internal/testing/channel-conformance/harness.js";
+
+let nextConversation = 0;
+const TENANT = "TENANT";
+const PERSON = { id: "USER", name: "Alice" } as const;
+const BOT = { id: "BOT", name: "eve Bot" } as const;
+const SERVICE_URL = "https://smba.example.test/teams";
+
+interface Choice {
+ readonly title: string;
+ readonly value: string;
+}
+
+interface AdaptiveCard {
+ readonly body?: readonly {
+ readonly choices?: readonly Choice[];
+ readonly type?: string;
+ }[];
+ readonly actions?: readonly {
+ readonly data?: Record<string, unknown>;
+ readonly type?: string;
+ }[];
+}
+
+interface ActivityBody {
+ readonly attachments?: readonly {
+ readonly content?: AdaptiveCard;
+ readonly contentType?: string;
+ }[];
+ readonly text?: string;
+ readonly type?: string;
+}
+
+interface PressHandle {
+ readonly data: Record<string, unknown>;
+ readonly value: string;
+}
+
+/** Drives Teams' Bot Framework webhook and Connector API in a personal chat. */
+export function teamsDriver(): ChannelDriver {
+ // Conversation ids are part of Teams' continuation token, so isolate each driver instance.
+ nextConversation += 1;
+ const conversationId = `CONV-${nextConversation}`;
+ let activityId = 0;
+
+ function nextActivityId(): string {
+ activityId += 1;
+ return `ACT-${activityId}`;
+ }
+
+ function activity(payload: Record<string, unknown>): Request {
+ return new Request("https://agent.example.com/eve/v1/teams", {
+ body: JSON.stringify({
+ channelData: { tenant: { id: TENANT } },
+ conversation: { id: conversationId, conversationType: "personal" },
+ from: PERSON,
+ id: `MSG-${activityId + 1}`,
+ recipient: BOT,
+ serviceUrl: SERVICE_URL,
+ ...payload,
+ }),
+ headers: { "content-type": "application/json" },
+ method: "POST",
+ });
+ }
+
+ async function decode(request: Request): Promise<PlatformCall> {
+ const bodyText = await request.text();
+ const body = bodyText === "" ? {} : JSON.parse(bodyText);
+ return {
+ body,
+ method: new URL(request.url).pathname.split("/").at(-1) ?? "",
+ response: { id: nextActivityId() },
+ };
+ }
+
+ return {
+ name: "teams",
+ capabilities: ["buttons", "text-replies"],
+ createChannel: (record) =>
+ teamsChannel({
+ api: { fetch: recordingFetch(record, decode) },
+ credentials: { tokenProvider: () => "test-token", webhookVerifier: () => true },
+ }),
+ message: (text) => activity({ text, type: "message" }),
+ findOptions(call, prompt) {
+ const body = call.body as ActivityBody;
+ if (body.type === "typing" || body.type !== "message" || !body.text?.includes(prompt)) {
+ return undefined;
+ }
+ const card = body.attachments?.find(
+ (attachment) => attachment.contentType === "application/vnd.microsoft.card.adaptive",
+ )?.content;
+ const choiceSet = card?.body?.find((element) => element.type === "Input.ChoiceSet");
+ const submit = card?.actions?.find((action) => action.type === "Action.Submit");
+ const data = submit?.data;
+ if (choiceSet?.choices === undefined || data === undefined) return [];
+ return choiceSet.choices.map((choice) => ({
+ handle: { data, value: choice.value } satisfies PressHandle,
+ label: choice.title,
+ }));
+ },
+ press: (option: RenderedOption) => {
+ const { data, value } = option.handle as PressHandle;
+ return activity({
+ id: `INVOKE-${activityId + 1}`,
+ name: "adaptiveCard/action",
+ type: "invoke",
+ value: { action: { data: { ...data, eve_option: value } } },
+ });
+ },
+ postedText(call) {
+ const body = call.body as ActivityBody;
+ if (body.type === "typing") return undefined;
+ return body.type === "message" ? body.text : undefined;
+ },
+ };
+}
diff --git a/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts b/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts
new file mode 100644
index 000000000..1a46ad828
--- /dev/null
+++ b/packages/eve/src/internal/testing/channel-conformance/twilio-driver.ts
@@ -0,0 +1,76 @@
+import { twilioChannel } from "#public/channels/twilio/index.js";
+import { signTwilioRequest } from "#public/channels/twilio/verify.js";
+import {
+ type ChannelDriver,
+ type PlatformCall,
+ recordingFetch,
+} from "#internal/testing/channel-conformance/harness.js";
+
+const AUTH_TOKEN = "twilio-conformance-secret";
+const TO = "+15550000001";
+let nextPerson = 0;
+
+/** Drives SMS webhooks and Twilio's form-encoded Messages API. */
+export function twilioDriver(): ChannelDriver {
+ nextPerson += 1;
+ const PERSON = `+1555000${String(nextPerson).padStart(4, "0")}`;
+ let messageSid = 0;
+ const webhookUrl = "https://agent.example.com/eve/v1/twilio/messages";
+
+ function message(text: string): Request {
+ messageSid += 1;
+ const params = new URLSearchParams({
+ Body: text,
+ From: PERSON,
+ MessageSid: `SM-conformance-${messageSid}`,
+ To: TO,
+ });
+ return new Request(webhookUrl, {
+ body: params,
+ headers: {
+ "content-type": "application/x-www-form-urlencoded",
+ "x-twilio-signature": signTwilioRequest({ authToken: AUTH_TOKEN, params, url: webhookUrl }),
+ },
+ method: "POST",
+ });
+ }
+
+ async function decode(request: Request): Promise<PlatformCall> {
+ const url = new URL(request.url);
+ return {
+ body: Object.fromEntries(new URLSearchParams(await request.text())),
+ method: url.pathname.split("/").at(-1) ?? "",
+ response: { sid: "SM-conformance-response" },
+ };
+ }
+
+ return {
+ name: "twilio",
+ capabilities: ["text-replies"],
+ createChannel: (record) =>
+ twilioChannel({
+ allowFrom: "*",
+ credentials: { accountSid: "AC123", authToken: AUTH_TOKEN },
+ api: { fetch: recordingFetch(record, decode) },
+ messaging: { from: TO },
+ }),
+ message,
+ findOptions(call, prompt) {
+ if (call.method !== "Messages.json") return undefined;
+ const body = call.body as { readonly Body?: unknown };
+ if (typeof body.Body !== "string" || !body.Body.includes(prompt)) return undefined;
+ return body.Body.split("\n")
+ .map((line) => /^\s*\d+[.)]\s*(.+?)\s*$/u.exec(line)?.[1])
+ .filter((label): label is string => label !== undefined)
+ .map((label) => ({ handle: label, label }));
+ },
+ press: () => {
+ throw new Error("Twilio SMS does not support pressing options.");
+ },
+ postedText(call) {
+ if (call.method !== "Messages.json") return undefined;
+ const body = call.body as { readonly Body?: unknown };
+ return typeof body.Body === "string" ? body.Body : undefined;
+ },
+ };
+}
diff --git a/packages/eve/src/public/channels/discord/discordChannel.test.ts b/packages/eve/src/public/channels/discord/discordChannel.test.ts
index e27c2fdb8..7ac485658 100644
--- a/packages/eve/src/public/channels/discord/discordChannel.test.ts
+++ b/packages/eve/src/public/channels/discord/discordChannel.test.ts
@@ -289,7 +289,7 @@ describe("discordChannel() inbound route", () => {
);
});
- it("delivers HITL button clicks as inputResponses", async () => {
+ it("acknowledges HITL button clicks with a deferred update", async () => {
const { privateKey, publicKeyHex } = testKeys();
const components = renderInputRequestComponents({
action: { callId: "call_1", input: {}, kind: "tool-call", toolName: "ask_question" },
@@ -313,17 +313,10 @@ describe("discordChannel() inbound route", () => {
});
const channel = discordChannel({ credentials: { publicKey: publicKeyHex } });
- const { response, send } = await firePost(channel, signedRequest({ body, privateKey }));
+ const { response } = await firePost(channel, signedRequest({ body, privateKey }));
expect(response.status).toBe(200);
await expect(response.json()).resolves.toEqual({ type: 6 });
- expect(send).toHaveBeenCalledWith(
- "C01:M01",
- expect.objectContaining({
- auth: null,
- inputResponses: [{ optionId: "approve", requestId: "call_1" }],
- }),
- );
});
it("opens and resolves freeform HITL modals", async () => {
diff --git a/packages/eve/src/public/channels/hitl-conformance.integration.test.ts b/packages/eve/src/public/channels/hitl-conformance.integration.test.ts
index ce0b90078..51162122f 100644
--- a/packages/eve/src/public/channels/hitl-conformance.integration.test.ts
+++ b/packages/eve/src/public/channels/hitl-conformance.integration.test.ts
@@ -5,9 +5,18 @@ import {
type ChannelDriver,
withChannelConversation,
} from "#internal/testing/channel-conformance/harness.js";
-import { chatSdkDriver } from "#internal/testing/channel-conformance/chat-sdk-driver.js";
+import {
+ chatSdkDriver,
+ chatSdkTextDriver,
+} from "#internal/testing/channel-conformance/chat-sdk-driver.js";
+import { discordDriver } from "#internal/testing/channel-conformance/discord-driver.js";
+import { githubDriver } from "#internal/testing/channel-conformance/github-driver.js";
+import { linearDriver } from "#internal/testing/channel-conformance/linear-driver.js";
+import { linqDriver } from "#internal/testing/channel-conformance/linq-driver.js";
import { slackDriver } from "#internal/testing/channel-conformance/slack-driver.js";
+import { teamsDriver } from "#internal/testing/channel-conformance/teams-driver.js";
import { telegramDriver } from "#internal/testing/channel-conformance/telegram-driver.js";
+import { twilioDriver } from "#internal/testing/channel-conformance/twilio-driver.js";
interface BrokenCell {
readonly reason: string;
@@ -27,7 +36,38 @@ interface BrokenCell {
const channels: readonly {
readonly driver: () => ChannelDriver;
readonly broken?: Partial<Record<HitlRule, BrokenCell>>;
-}[] = [{ driver: chatSdkDriver }, { driver: slackDriver }, { driver: telegramDriver }];
+}[] = [
+ { driver: chatSdkDriver },
+ {
+ driver: chatSdkTextDriver,
+ broken: {
+ "a rendered question shows every option a person can choose": {
+ reason: "#3712: the card's fallback text is only the prompt",
+ symptom: /the question showed \[\]/,
+ },
+ },
+ },
+ { driver: discordDriver },
+ { driver: githubDriver },
+ { driver: linearDriver },
+ { driver: linqDriver },
+ { driver: slackDriver },
+ { driver: teamsDriver },
+ { driver: telegramDriver },
+ {
+ driver: twilioDriver,
+ broken: {
+ "a rendered question shows every option a person can choose": {
+ reason: "the channel never sends the question (no input.requested handler)",
+ symptom: /Timed out waiting for the question "Which day works for the review\?" on twilio/,
+ },
+ "a text reply matching an option answers the only pending question": {
+ reason: "the channel never sends the question (no input.requested handler)",
+ symptom: /Timed out waiting for the question "Which day works for the review\?" on twilio/,
+ },
+ },
+ },
+];
describe.each(channels.map((entry) => ({ ...entry, name: entry.driver().name })))(
"$name HITL contract",
diff --git a/packages/eve/src/public/channels/teams/hitl.test.ts b/packages/eve/src/public/channels/teams/hitl.test.ts
index 7f4254e97..5b3cf9bb7 100644
--- a/packages/eve/src/public/channels/teams/hitl.test.ts
+++ b/packages/eve/src/public/channels/teams/hitl.test.ts
@@ -5,7 +5,6 @@ import {
isTeamsToolApprovalResponseActivity,
readTeamsInputReplyToActivityId,
renderInputRequestMessage,
- TEAMS_HITL_CHOICE_INPUT_ID,
TEAMS_HITL_DATA_KEY,
TEAMS_HITL_FREEFORM_INPUT_ID,
TEAMS_HITL_PROMPT_KEY,
@@ -48,12 +47,6 @@ describe("Teams HITL helpers", () => {
});
});
- it("renders select requests with a ChoiceSet", () => {
- const body = renderInputRequestMessage({ ...request(), display: "select" });
- const card = body.attachments?.[0]?.content as { body?: Array<Record<string, unknown>> };
- expect(card.body?.some((entry) => entry.id === TEAMS_HITL_CHOICE_INPUT_ID)).toBe(true);
- });
-
it("decodes message and invoke submission values", () => {
const message = parseTeamsActivity(
activityWithValue({