The Open Framework for Building Agents
[1/3] test(eve): add a TUI driver to the HITL conformance suite (#4218)
Ben Pankowbfc122e50873
diff --git a/packages/eve/src/cli/dev/tui/hitl-conformance.integration.test.ts b/packages/eve/src/cli/dev/tui/hitl-conformance.integration.test.tsnew file mode 100644index 000000000..2851a1e9e--- /dev/null+++ b/packages/eve/src/cli/dev/tui/hitl-conformance.integration.test.ts@@ -0,0 +1,3 @@+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";++describeHitlConformance("tui");diff --git a/packages/eve/src/cli/dev/tui/test/mock-terminal.ts b/packages/eve/src/cli/dev/tui/test/mock-terminal.tsindex c9b959a0f..198235bc4 100644--- a/packages/eve/src/cli/dev/tui/test/mock-terminal.ts+++ b/packages/eve/src/cli/dev/tui/test/mock-terminal.ts@@ -78,9 +78,17 @@ export class MockScreen extends EventEmitter implements TerminalOutput {rows: number;#rawOutput = "";#lines: string[] = [];+ /** Parallel to `#lines`: `d` marks a cell drawn dim (SGR 2), a space any other cell. */+ #dimCells: string[] = [];+ #dim = false;#cursorLine = 0;#cursorColumn = 0;- #mainScreen?: { lines: string[]; cursorLine: number; cursorColumn: number };+ #mainScreen?: {+ lines: string[];+ dimCells: string[];+ cursorLine: number;+ cursorColumn: number;+ };#waiters: Array<{text: string;resolve: () => void;@@ -118,8 +126,21 @@ export class MockScreen extends EventEmitter implements TerminalOutput {this.emit("resize");}- snapshot() {- return this.#lines.join("\n");+ /**+ * The visible grid as text. `hideDim` blanks dim cells, leaving what the+ * screen draws at full weight, such as option labels without descriptions.+ */+ snapshot({ hideDim = false }: { readonly hideDim?: boolean } = {}) {+ if (!hideDim) return this.#lines.join("\n");+ return this.#lines+ .map((line, index) => {+ const dim = this.#dimCells[index] ?? "";+ // Index by UTF-16 unit: a cell's marks repeat for each unit it occupies.+ return line.replace(/./gsu, (cell, column: number) =>+ dim[column] === "d" ? " ".repeat(cell.length) : cell,+ );+ })+ .join("\n");}rawOutput() {@@ -249,14 +270,17 @@ export class MockScreen extends EventEmitter implements TerminalOutput {if (command === "h") {this.#mainScreen = {lines: this.#lines,+ dimCells: this.#dimCells,cursorLine: this.#cursorLine,cursorColumn: this.#cursorColumn,};this.#lines = [];+ this.#dimCells = [];this.#cursorLine = 0;this.#cursorColumn = 0;} else if (this.#mainScreen !== undefined) {this.#lines = this.#mainScreen.lines;+ this.#dimCells = this.#mainScreen.dimCells;this.#cursorLine = this.#mainScreen.cursorLine;this.#cursorColumn = this.#mainScreen.cursorColumn;this.#mainScreen = undefined;@@ -300,6 +324,9 @@ export class MockScreen extends EventEmitter implements TerminalOutput {case "K":this.#eraseInLine(first(0));break;+ case "m":+ this.#applySgr(parameters);+ break;default:break;}@@ -307,9 +334,32 @@ export class MockScreen extends EventEmitter implements TerminalOutput {return startIndex + sequence.length;}+ /**+ * Tracks only intensity: 2 starts dim; 0 and 22 end it, as in a real terminal+ * where bold (1) leaves faint on. Extended colors (38, 48, 58) carry a+ * `5;n` or `2;r;g;b` payload whose numbers are not SGR codes.+ */+ #applySgr(parameters: readonly string[]) {+ const codes = (parameters.length === 0 ? [""] : parameters).map((parameter) =>+ parameter === "" ? 0 : Number(parameter),+ );+ for (let index = 0; index < codes.length; index += 1) {+ const code = codes[index];+ if (code === 38 || code === 48 || code === 58) {+ const mode = codes[index + 1];+ index += mode === 5 ? 2 : mode === 2 ? 4 : 1;+ } else if (code === 2) {+ this.#dim = true;+ } else if (code === 0 || code === 22) {+ this.#dim = false;+ }+ }+ }+#eraseInDisplay(mode: number) {if (mode === 2 || mode === 3) {this.#lines = [];+ this.#dimCells = [];this.#cursorLine = 0;this.#cursorColumn = 0;return;@@ -319,6 +369,7 @@ export class MockScreen extends EventEmitter implements TerminalOutput {// Cursor to start of screen.for (let line = 0; line < this.#cursorLine; line += 1) {this.#lines[line] = "";+ this.#dimCells[line] = "";}this.#eraseInLine(1);return;@@ -328,33 +379,31 @@ export class MockScreen extends EventEmitter implements TerminalOutput {// and drop every line below it.this.#eraseInLine(0);this.#lines.length = Math.min(this.#lines.length, this.#cursorLine + 1);+ this.#dimCells.length = Math.min(this.#dimCells.length, this.#cursorLine + 1);}#eraseInLine(mode: number) {- const line = this.#lines[this.#cursorLine] ?? "";-- if (mode === 2) {- this.#lines[this.#cursorLine] = "";- return;- }-- if (mode === 1) {- this.#lines[this.#cursorLine] =- " ".repeat(this.#cursorColumn) + line.slice(this.#cursorColumn);- return;+ for (const grid of [this.#lines, this.#dimCells]) {+ const line = grid[this.#cursorLine] ?? "";+ if (mode === 2) {+ grid[this.#cursorLine] = "";+ } else if (mode === 1) {+ grid[this.#cursorLine] = " ".repeat(this.#cursorColumn) + line.slice(this.#cursorColumn);+ } else {+ // mode 0: clear from cursor to end of line.+ grid[this.#cursorLine] = line.slice(0, this.#cursorColumn);+ }}-- // mode 0: clear from cursor to end of line.- this.#lines[this.#cursorLine] = line.slice(0, this.#cursorColumn);}#writeCharacter(character: string) {- const line = (this.#lines[this.#cursorLine] ?? "").padEnd(this.#cursorColumn, " ");- const nextLine =- line.slice(0, this.#cursorColumn) +- character +- line.slice(this.#cursorColumn + character.length);- this.#lines[this.#cursorLine] = nextLine;+ const write = (grid: string[], cell: string) => {+ const line = (grid[this.#cursorLine] ?? "").padEnd(this.#cursorColumn, " ");+ grid[this.#cursorLine] =+ line.slice(0, this.#cursorColumn) + cell + line.slice(this.#cursorColumn + cell.length);+ };+ write(this.#lines, character);+ write(this.#dimCells, (this.#dim ? "d" : " ").repeat(character.length));this.#cursorColumn += character.length;}}diff --git a/packages/eve/src/internal/testing/channel-conformance/conformance.ts b/packages/eve/src/internal/testing/channel-conformance/conformance.tsindex 5faedef81..c588d06a9 100644--- a/packages/eve/src/internal/testing/channel-conformance/conformance.ts+++ b/packages/eve/src/internal/testing/channel-conformance/conformance.ts@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";import { type HitlRule, hitlContract } from "#internal/testing/channel-conformance/contract.js";import {type ChannelDriver,+ type ClientDriver,withChannelConversation,} from "#internal/testing/channel-conformance/harness.js";import {@@ -16,6 +17,7 @@ import { linqDriver } from "#internal/testing/channel-conformance/linq-driver.jsimport { slackDriver } from "#internal/testing/channel-conformance/slack-driver.js";import { teamsDriver } from "#internal/testing/channel-conformance/teams-driver.js";import { telegramDriver } from "#internal/testing/channel-conformance/telegram-driver.js";+import { tuiDriver } from "#internal/testing/channel-conformance/tui-driver.js";import { twilioDriver } from "#internal/testing/channel-conformance/twilio-driver.js";export interface BrokenCell {@@ -25,17 +27,28 @@ export interface BrokenCell {}interface ConformanceChannel {- readonly driver: () => ChannelDriver;+ readonly driver: () => ChannelDriver | ClientDriver;readonly broken?: Partial<Record<HitlRule, BrokenCell>>;+ /** Rules this client deliberately doesn't offer, with why, beyond what its capabilities rule out. */+ readonly unsupported?: Partial<Record<HitlRule, string>>;}+const TUI_TYPED_APPROVAL =+ "the approval drawer holds the keyboard; a person answers it with y or n";++const TUI_APPROVAL_PROMPT: BrokenCell = {+ reason: "the approval drawer titles the raw tool name instead of the request's prompt",+ symptom: /Timed out waiting for the question "Approve Deploy release\?" on tui/,+};+/**- * Every first-party channel's place in the HITL contract, keyed by the channel- * directory whose `hitl-conformance.integration.test.ts` runs it. Each cell is+ * Every first-party channel's and client's place in the HITL contract, keyed by+ * the directory whose `hitl-conformance.integration.test.ts` runs it. Each cell is* one of:** - must pass;- * - not supported: the platform lacks a capability the rule requires (skipped);+ * - not supported: the platform lacks a capability the rule requires, or the+ * entry declines the rule as `unsupported` (skipped);* - broken: the channel should pass but doesn't yet. The cell passes only when* the rule fails with the recorded `symptom`, so a fix or an unrelated* failure (such as harness breakage) both turn it red.@@ -49,6 +62,20 @@ const hitlConformance = {slack: [{ driver: slackDriver }],teams: [{ driver: teamsDriver }],telegram: [{ driver: telegramDriver }],+ tui: [+ {+ driver: tuiDriver,+ broken: {+ "a tool approval shows a choice to approve and one to cancel": TUI_APPROVAL_PROMPT,+ "pressing Approve runs the gated tool": TUI_APPROVAL_PROMPT,+ "pressing Cancel stops the gated tool without running it": TUI_APPROVAL_PROMPT,+ },+ unsupported: {+ "a text reply of approve runs the gated tool": TUI_TYPED_APPROVAL,+ "a text reply of cancel stops the gated tool without running it": TUI_TYPED_APPROVAL,+ },+ },+ ],twilio: [{ driver: twilioDriver }],} satisfies Record<string, readonly ConformanceChannel[]>;@@ -67,13 +94,16 @@ export function describeHitlConformance(channel: keyof typeof hitlConformance):const entries: readonly ConformanceChannel[] = hitlConformance[channel];describe.each(entries.map((entry) => ({ ...entry, name: entry.driver().name })))("$name HITL contract",- ({ driver, broken }) => {+ ({ driver, broken, unsupported }) => {const { capabilities } = driver();for (const rule of hitlContract) {const supported = rule.requires.every((capability) => capabilities.includes(capability));const known = broken?.[rule.rule];+ const declined = unsupported?.[rule.rule];if (!supported) {it.skip(`${rule.rule} (not supported)`, () => {});+ } else if (declined !== undefined) {+ it.skip(`${rule.rule} (not supported: ${declined})`, () => {});} else if (known === undefined) {it(rule.rule, () => withChannelConversation(driver(), (c) => rule.run(c)), 60_000);} else {diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.tsindex 0833b01b5..6ee2ba8d3 100644--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts@@ -29,6 +29,9 @@ async function askWhichDay(conversation: ChannelConversation) {}const DEPLOY = `Use ${GATED_TOOL} to ship the release.`;+// Channels word the two approval choices differently; the TUI asks Yes or No.+const APPROVE_LABELS = ["Approve", "Yes"];+const CANCEL_LABELS = ["Cancel", "No"];const APPROVAL_PROMPT = "Approve Deploy release?";async function askToDeploy(conversation: ChannelConversation) {@@ -97,16 +100,22 @@ export const hitlContract = [},},{- rule: "a tool approval shows Approve and Cancel",+ rule: "a tool approval shows a choice to approve and one to cancel",source: "docs/tools/human-in-the-loop.md#approvals",requires: [],async run(conversation) {const options = await askToDeploy(conversation);- const labels = options.map((option) => option.label).sort();- expect(labels, `the approval showed ${JSON.stringify(labels)}`).toEqual([- "Approve",- "Cancel",- ]);+ const labels = options.map((option) => option.label);+ const message = `the approval showed ${JSON.stringify(labels)}`;+ expect(labels, message).toHaveLength(2);+ expect(+ labels.filter((label) => APPROVE_LABELS.includes(label)),+ message,+ ).toHaveLength(1);+ expect(+ labels.filter((label) => CANCEL_LABELS.includes(label)),+ message,+ ).toHaveLength(1);},},{@@ -115,7 +124,7 @@ export const hitlContract = [requires: ["buttons"],async run(conversation) {const options = await askToDeploy(conversation);- const approve = options.find((option) => option.label === "Approve");+ const approve = options.find((option) => APPROVE_LABELS.includes(option.label));expect(approve, "an Approve option to press").toBeDefined();await conversation.press(approve!);await expectDeployed(conversation);@@ -127,7 +136,7 @@ export const hitlContract = [requires: ["buttons"],async run(conversation) {const options = await askToDeploy(conversation);- const cancel = options.find((option) => option.label === "Cancel");+ const cancel = options.find((option) => CANCEL_LABELS.includes(option.label));expect(cancel, "a Cancel option to press").toBeDefined();await conversation.press(cancel!);await expectNotDeployed(conversation);diff --git a/packages/eve/src/internal/testing/channel-conformance/harness.ts b/packages/eve/src/internal/testing/channel-conformance/harness.tsindex 9f278958e..23412d83b 100644--- a/packages/eve/src/internal/testing/channel-conformance/harness.ts+++ b/packages/eve/src/internal/testing/channel-conformance/harness.ts@@ -1,11 +1,15 @@import { createChannelOperations } from "#channel/channel-operations.js";import { type CompiledChannel, isCompiledChannel } from "#channel/compiled-channel.js";import { type RouteHandlerArgs, isHttpRouteDefinition } from "#channel/routes.js";+import { sessionInboxHookToken } from "#execution/session-inbox/address.js";import { createWorkflowRuntime } from "#execution/workflow-runtime.js";import { createTestRuntime } from "#internal/testing/app-harness.js";import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent-cache.js";-import type { Session } from "#channel/session.js";+import { createAttachSessionFn, type Session } from "#channel/session.js";+import { attachRouteSessionCreator } from "#internal/nitro/routes/channel-route-context.js";+import { none } from "#public/channels/auth.js";+import { eveChannel } from "#public/channels/eve.js";import { z } from "#compiled/zod/index.js";import { always } from "#tools/approval/policies.js";import { defineTool } from "#tools/definition.js";@@ -94,12 +98,56 @@ export interface ChannelConversation {readonly gatedToolRuns: number;}+/**+ * Teaches the suite to drive a client UI that talks to eve's own HTTP channel+ * (`eveChannel`) through `eve/client`, such as the dev TUI. Like a webhook+ * driver, it reads only what the client shows a person.+ */+export interface ClientDriver {+ readonly name: string;+ readonly capabilities: readonly ChannelCapability[];+ /**+ * Starts the client against the agent at `host`. The global `fetch` serves+ * that origin from the eve channel's routes until the client closes.+ */+ open(host: string, wait: Wait): Promise<ClientView>;+}++/** A running client, as a person sees and uses it. */+export interface ClientView {+ /** The person sends a plain-text message. */+ say(text: string): Promise<void>;+ /** Waits for the client to show `prompt` with choices, returning them. */+ waitForQuestion(prompt: string): Promise<readonly RenderedOption[]>;+ /** The person presses one shown choice. */+ press(option: RenderedOption): Promise<void>;+ /** The bot replies the client shows now. */+ replies(): readonly string[];+ /** What the client shows now, for timeout errors. */+ describe(): string;+ /** Stops the client and releases anything it holds, such as its event stream. */+ close(): Promise<void>;+}++/**+ * Polls `select` until it returns a value. On timeout, the error names+ * `label`, the driver, and `describe()`'s account of what a person sees.+ */+export type Wait = <T>(+ label: string,+ select: () => T | undefined | Promise<T | undefined>,+ describe: () => string,+) => Promise<T>;+/*** How long to wait for a platform call. Steps finish in well under a second, so* this mostly absorbs the first conversation's cold start on a busy machine.*/const WAIT_TIMEOUT_MS = 30_000;+/** Stand-in origin for the agent a {@link ClientDriver} talks to. */+const CLIENT_HOST = "https://agent.example.com";+/** The test agent's tool that always needs a person's approval before it runs. */export const GATED_TOOL = "deploy_release";@@ -121,39 +169,141 @@ export function recordingFetch(}/**- * Runs `body` against an agent with `ask_question` and `driver`'s channel. Every- * interaction goes through the channel's real webhook routes; the only fake is- * the platform behind the channel, usually its injected `fetch`.+ * Runs `body` against an agent with `ask_question`, an approval-gated tool, and+ * `driver`'s channel. Every interaction goes through the channel's real routes.+ * A webhook driver fakes only the platform behind its channel, usually its+ * injected `fetch`; a client driver runs the real client against `eveChannel`.** Conversations must not overlap: each compiles its own agent, and concurrent* ones can resolve each other's compiled artifacts in the shared workflow world.*/export async function withChannelConversation(- driver: ChannelDriver,+ driver: ChannelDriver | ClientDriver,body: (conversation: ChannelConversation) => Promise<void>,options: { readonly waitTimeoutMs?: number } = {},): Promise<void> {+ const waitTimeoutMs = options.waitTimeoutMs ?? WAIT_TIMEOUT_MS;+ const wait: Wait = (label, select, describe) =>+ poll(`${label} on ${driver.name}`, select, describe, waitTimeoutMs);+ if (isClientDriver(driver)) {+ await converse(driver.name, "eve", eveChannel({ auth: none() }), body, wait, (dispatch) =>+ openClient(driver, dispatch, wait),+ );+ return;+ }const calls: PlatformCall[] = [];try {- await converse(driver, calls, body, options.waitTimeoutMs ?? WAIT_TIMEOUT_MS);+ const channel = driver.createChannel((call) => void calls.push(call));+ await converse(driver.name, driver.name, channel, body, wait, async (dispatch) =>+ webhookView(driver, calls, dispatch, wait),+ );} finally {driver.dispose?.();}}-async function converse(+function isClientDriver(driver: ChannelDriver | ClientDriver): driver is ClientDriver {+ return "open" in driver;+}++async function poll<T>(+ label: string,+ select: () => T | undefined | Promise<T | undefined>,+ describe: () => string,+ timeoutMs: number,+): Promise<T> {+ const deadline = Date.now() + timeoutMs;+ while (Date.now() < deadline) {+ const selected = await select();+ if (selected !== undefined) return selected;+ await new Promise((resolve) => setTimeout(resolve, 25));+ }+ throw new Error(`Timed out waiting for ${label}. ${describe()}`);+}++/** Serves one request from the channel's routes; `settled` resolves once its `waitUntil` work does. */+type Dispatch = (request: Request) => Promise<{ response: Response; settled: Promise<unknown> }>;++/** A webhook channel seen through its platform calls: what the bot posted. */+function webhookView(driver: ChannelDriver,- calls: PlatformCall[],+ calls: readonly PlatformCall[],+ dispatch: Dispatch,+ wait: Wait,+): ClientView {+ async function post(request: Request): Promise<void> {+ const { response, settled } = await dispatch(request);+ await settled;+ if (!response.ok) throw new Error(`${driver.name} webhook answered ${response.status}.`);+ }+ const describe = () => `Platform calls:\n${JSON.stringify(calls, null, 2)}`;++ return {+ say: (text) => post(driver.message(text)),+ press: (option) => post(driver.press(option)),+ waitForQuestion: (prompt) =>+ wait(+ `the question "${prompt}"`,+ () => {+ for (const call of calls) {+ const options = driver.findOptions(call, prompt);+ if (options !== undefined) return options;+ }+ return undefined;+ },+ describe,+ ),+ replies: () => calls.flatMap((call) => driver.postedText(call) ?? []),+ describe,+ close: async () => {},+ };+}++/** Starts `driver`'s client with the client host's `fetch` routed to the channel until it closes. */+async function openClient(+ driver: ClientDriver,+ dispatch: Dispatch,+ wait: Wait,+): Promise<ClientView> {+ const original = globalThis.fetch;+ globalThis.fetch = async (input, init) => {+ const request = new Request(input, init);+ if (new URL(request.url).origin !== CLIENT_HOST) return await original(input, init);+ return (await dispatch(request)).response;+ };+ const restore = () => void (globalThis.fetch = original);+ try {+ const view = await driver.open(CLIENT_HOST, wait);+ return {+ ...view,+ async close() {+ try {+ await view.close();+ } finally {+ restore();+ }+ },+ };+ } catch (error) {+ restore();+ throw error;+ }+}++async function converse(+ label: string,+ channelName: string,+ created: unknown,body: (conversation: ChannelConversation) => Promise<void>,- waitTimeoutMs: number,+ wait: Wait,+ open: (dispatch: Dispatch) => Promise<ClientView>,): Promise<void> {- const created = driver.createChannel((call) => void calls.push(call));- if (!isCompiledChannel(created)) throw new Error(`${driver.name} is not a compiled channel.`);+ if (!isCompiledChannel(created)) throw new Error(`${label} is not a compiled channel.`);const channel: CompiledChannel = created;let gatedToolRuns = 0;const runtime = await createTestRuntime({- agent: { name: `${driver.name}-hitl-conformance` },+ agent: { name: `${label}-hitl-conformance` },modules: [{logicalPath: "tools/ask_question.ts",@@ -174,7 +324,7 @@ async function converse(}),},{- logicalPath: `channels/${driver.name}.ts`,+ logicalPath: `channels/${channelName}.ts`,loadNamespace: async () => ({ default: channel }),},],@@ -183,119 +333,166 @@ async function converse(await runtime.run(async () => {const compiledArtifactsSource = createBundledRuntimeCompiledArtifactsSource();const bundle = await getCompiledRuntimeAgentBundle({ compiledArtifactsSource });- const entry = bundle.graph.root.channels.find((candidate) => candidate.name === driver.name);- if (entry?.adapter === undefined) throw new Error(`Expected the ${driver.name} adapter.`);- // Mirrors the operations production route dispatch builds for each request.+ const entry = bundle.graph.root.channels.find((candidate) => candidate.name === channelName);+ if (entry?.adapter === undefined) throw new Error(`Expected the ${channelName} adapter.`);+ const { adapter, turnPolicy } = entry;+ const workflowRuntime = createWorkflowRuntime({ compiledArtifactsSource });+ // Mirrors the route arguments production channel dispatch builds for each request.const operations = createChannelOperations({- adapter: entry.adapter,- channelName: driver.name,- runtime: createWorkflowRuntime({ compiledArtifactsSource }),- turnPolicy: entry.turnPolicy,+ adapter,+ channelName,+ runtime: workflowRuntime,+ turnPolicy,+ });+ const attachSession = createAttachSessionFn(workflowRuntime, {+ channelKind: adapter.kind,+ channelName,+ turnPolicy,});const sessions = new Map<string, Session>();+ const background: Promise<unknown>[] = [];++ function track(session: Session): Session {+ sessions.set(session.id, session);+ return session;+ }- async function post(request: Request): Promise<void> {+ const dispatch: Dispatch = async (request) => {+ const { params, route } = findRoute(channel, request);const pending: Promise<unknown>[] = [];- const args: RouteHandlerArgs = {- ...operations,- from: (address) => {- const source = operations.from(address);- return {- ...source,- send: async (...sendArgs) => track(await source.send(...sendArgs)),- respond: async (...respondArgs) => track(await source.respond(...respondArgs)),- };+ const args = attachRouteSessionCreator<RouteHandlerArgs>(+ {+ ...operations,+ from: (address) => {+ const source = operations.from(address);+ return {+ ...source,+ send: async (...sendArgs) => track(await source.send(...sendArgs)),+ respond: async (...respondArgs) => {+ await waitForAddress(address);+ return track(await source.respond(...respondArgs));+ },+ };+ },+ attachSession,+ params,+ requestIp: null,+ to: unsupported("to"),+ waitUntil: (task) => void pending.push(task),},- attachSession: unsupported("attachSession"),- params: {},- requestIp: null,- to: unsupported("to"),- waitUntil: (task) => void pending.push(task),- };- const response = await findRoute(channel, request).handler(request, args);- await Promise.all(pending);- if (!response.ok) throw new Error(`${driver.name} webhook answered ${response.status}.`);- }+ async (input) => {+ const handle = await workflowRuntime.createSession({+ ...input,+ adapter,+ channelName,+ continuationToken:+ input.continuationToken === undefined+ ? undefined+ : `${channelName}:${input.continuationToken}`,+ });+ track(attachSession(handle.sessionId));+ return handle;+ },+ );+ const response = await route.handler(request, args);+ const settled = Promise.all(pending);+ background.push(settled);+ return { response, settled };+ };- async function waitFor<T>(label: string, select: (call: PlatformCall) => T | undefined) {- const deadline = Date.now() + waitTimeoutMs;- while (Date.now() < deadline) {- for (const call of calls) {- const selected = select(call);- if (selected !== undefined) return selected;- }- await new Promise((resolve) => setTimeout(resolve, 25));- }- throw new Error(- `Timed out waiting for ${label} on ${driver.name}. Platform calls:\n${JSON.stringify(calls, null, 2)}`,+ /** How many times each session's turn has held for input, as of the last wait. */+ const inputHolds = new Map<string, number>();+ const view = await open(dispatch);+ const replyWait = <T>(label: string, select: (reply: string) => T | undefined) =>+ wait(+ label,+ () => {+ for (const reply of view.replies()) {+ const selected = select(reply);+ if (selected !== undefined) return selected;+ }+ return undefined;+ },+ view.describe,);- }const conversation: ChannelConversation = {- say: (text) => post(driver.message(text)),- press: (option) => post(driver.press(option)),+ say: (text) => view.say(text),+ press: (option) => view.press(option),async waitForQuestion(prompt) {- const options = await waitFor(`the question "${prompt}"`, (call) =>- driver.findOptions(call, prompt),- );- await waitForTurnToHoldForInput();+ const options = await view.waitForQuestion(prompt);+ await holdForInput();return options;},waitForToolResult: (tool) =>- waitFor(`${tool} to return`, (call) => {- const text = driver.postedText(call);- return text === undefined ? undefined : readMockToolReply(text, tool);- }),+ replyWait(`${tool} to return`, (reply) => readMockToolReply(reply, tool)),waitForToolOutcome: (tool) =>- waitFor(`${tool} to run or be denied`, (call): ToolOutcome | undefined => {- const text = driver.postedText(call);- if (text === undefined) return undefined;- const output = readMockToolReply(text, tool);+ replyWait(`${tool} to run or be denied`, (reply): ToolOutcome | undefined => {+ const output = readMockToolReply(reply, tool);if (output !== undefined) return { kind: "ran", output };- return isMockDenialReply(text) ? { kind: "denied" } : undefined;+ return isMockDenialReply(reply) ? { kind: "denied" } : undefined;}),get gatedToolRuns() {return gatedToolRuns;},};- /** How many times each session's turn has held for input, as of the last wait. */- const inputHolds = new Map<string, number>();+ /**+ * The turn emits `turn.waiting` inside the step that raised the request, but+ * an address the channel aliased in that step (such as Discord's message id)+ * is only claimed once the step commits. Answering through it before then+ * finds no session.+ */+ async function waitForAddress(address: string): Promise<void> {+ const world = await getWorld();+ const token = sessionInboxHookToken(`${channelName}:${address}`);+ await wait(+ `the channel to claim the address "${address}"`,+ () =>+ world.hooks.getByToken(token).then(+ () => true,+ () => undefined,+ ),+ () => "",+ );+ }/*** A person answers once the bot has finished asking. Answering the moment- * the question appears races the channel's own bookkeeping for it, such as- * Discord aliasing the session to the message it just posted.+ * the question appears races the turn's own bookkeeping for it; see also+ * {@link waitForAddress}.*/- async function waitForTurnToHoldForInput(): Promise<void> {- const deadline = Date.now() + waitTimeoutMs;- while (Date.now() < deadline) {- for (const session of sessions.values()) {- const holds = await countInputHolds(session);- if (holds > (inputHolds.get(session.id) ?? 0)) {- inputHolds.set(session.id, holds);- return;+ function holdForInput(): Promise<true> {+ return wait(+ "the turn to hold for input",+ async () => {+ for (const session of sessions.values()) {+ const holds = await countInputHolds(session);+ if (holds > (inputHolds.get(session.id) ?? 0)) {+ inputHolds.set(session.id, holds);+ return true;+ }}- }- await new Promise((resolve) => setTimeout(resolve, 25));- }- throw new Error(`Timed out waiting for the turn to hold for input on ${driver.name}.`);- }-- function track(session: Session): Session {- sessions.set(session.id, session);- return session;+ return undefined;+ },+ () => "",+ );}// The test file's workflow world closes after its last test, so a session// still writing then fails with an unhandled rejection.- const settle = () => Promise.all([...sessions.values()].map(cancelUntilResting));+ const settle = async () => {+ await Promise.all([...sessions.values()].map(cancelUntilResting));+ await Promise.all(background);+ };try {await body(conversation);} catch (error) {+ await view.close().catch(() => {});await settle().catch(() => {});throw error;}+ await view.close();await settle();});}@@ -342,16 +539,26 @@ async function countInputHolds(session: Session): Promise<number> {function findRoute(channel: CompiledChannel, request: Request) {const { pathname } = new URL(request.url);- const route = channel.routes.find(- (candidate) =>- isHttpRouteDefinition(candidate) &&- candidate.method === request.method &&- candidate.path === pathname,- );- if (route === undefined || !isHttpRouteDefinition(route)) {- throw new Error(`No ${request.method} route for ${pathname}.`);+ for (const route of channel.routes) {+ if (!isHttpRouteDefinition(route) || route.method !== request.method) continue;+ const params = matchPath(route.path, pathname);+ if (params !== undefined) return { params, route };+ }+ throw new Error(`No ${request.method} route for ${pathname}.`);+}++/** Matches a route path whose `:name` segments each capture one path segment. */+function matchPath(pattern: string, pathname: string): Record<string, string> | undefined {+ const expected = pattern.split("/");+ const actual = pathname.split("/");+ if (expected.length !== actual.length) return undefined;+ const params: Record<string, string> = {};+ for (const [index, segment] of expected.entries()) {+ const value = actual[index]!;+ if (segment.startsWith(":")) params[segment.slice(1)] = decodeURIComponent(value);+ else if (segment !== value) return undefined;}- return route;+ return params;}function unsupported(name: string): () => never {@@ -365,7 +572,7 @@ function unsupported(name: string): () => never {* `Used <tool> for "<message>": <JSON output>`. Reading the output back from* the posted reply keeps rules on what a person sees.*/-function readMockToolReply(text: string, tool: string): unknown {+export function readMockToolReply(text: string, tool: string): unknown {if (!text.startsWith(`Used ${tool} for "`)) return undefined;const output = /": (\{.*\})\s*$/su.exec(text)?.[1];if (output === undefined) return undefined;diff --git a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts b/packages/eve/src/internal/testing/channel-conformance/tui-driver.tsnew file mode 100644index 000000000..1fd0601e9--- /dev/null+++ b/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts@@ -0,0 +1,150 @@+import { Client } from "#client/client.js";+import { EveTUIRunner } from "#cli/dev/tui/runner.js";+import { TerminalRenderer } from "#cli/dev/tui/terminal-renderer.js";+import { MockScreen, MockUserInput } from "#cli/dev/tui/test/mock-terminal.js";+import type {+ ClientDriver,+ RenderedOption,+} from "#internal/testing/channel-conformance/harness.js";++/** Each drawer's footer: a question's, then a tool approval's. */+const DRAWER_FOOTERS = [+ "↑/↓ move · enter to select · esc to dismiss",+ "y yes · n no · Ctrl-C cancel",+] as const;+const FREEFORM_ROW = "Type your own answer…";++/**+ * Drives the real `eve dev` TUI on a mock terminal. It reads only the screen+ * and acts only through keys, as a person at the terminal would.+ */+export function tuiDriver(): ClientDriver {+ return {+ name: "tui",+ capabilities: ["buttons", "text-replies"],+ async open(host, wait) {+ // Wide enough that no reply wraps, so each one stays on one screen row.+ const screen = new MockScreen({ columns: 1000, rows: 60 });+ const keyboard = new MockUserInput();+ const renderer = new TerminalRenderer({+ captureForeignOutput: false,+ input: keyboard,+ output: screen,+ unicode: true,+ });+ const run = new EveTUIRunner({ client: new Client({ host }), name: "tui", renderer }).run();+ const describe = () => `Screen:\n${screen.snapshot()}`;+ try {+ // Surfaces a runner that fails during startup instead of waiting out the timeout.+ await Promise.race([+ wait("the composer", () => (composerOpen(screen) ? true : undefined), describe),+ run.then(() => {+ throw new Error(`The TUI exited before its composer opened. ${describe()}`);+ }),+ ]);+ } catch (error) {+ // The caller only gets `close()` once `open()` resolves, so stop the runner here.+ renderer.requestInterrupt();+ await run.catch(() => {});+ throw error;+ }++ return {+ async say(text) {+ // A question drawer holds the keyboard until Esc hands it back to the composer.+ if (drawerOpen(screen)) keyboard.send("\x1b");+ await wait("the composer", () => (composerOpen(screen) ? true : undefined), describe);+ keyboard.type(text);+ keyboard.enter();+ },+ async waitForQuestion(prompt) {+ await wait(`the question "${prompt}"`, () => focusedRow(screen, prompt), describe);+ return readOptions(screen, keyboard, prompt);+ },+ async press(option) {+ // `waitForQuestion` leaves the cursor on the first option.+ for (let row = 0; row < (option.handle as number); row += 1) keyboard.down();+ keyboard.enter();+ },+ replies: () => readReplies(screen.snapshot()),+ describe,+ async close() {+ renderer.requestInterrupt();+ await run;+ },+ };+ },+ };+}++/** Each assistant reply: a `▲ ` row and the indented rows its text continues on. */+function readReplies(snapshot: string): string[] {+ const replies: string[] = [];+ let reply: string[] | undefined;+ for (const line of snapshot.split("\n")) {+ if (line.startsWith("▲ ")) {+ reply = [line.slice(2)];+ replies.push("");+ } else if (reply !== undefined && line.startsWith(" ")) {+ reply.push(line.slice(2));+ } else {+ reply = undefined;+ }+ if (reply !== undefined) replies[replies.length - 1] = reply.join("\n");+ }+ return replies;+}++function drawerOpen(screen: MockScreen): boolean {+ const snapshot = screen.snapshot();+ return DRAWER_FOOTERS.some((footer) => snapshot.includes(footer));+}++function composerOpen(screen: MockScreen): boolean {+ return !drawerOpen(screen) && /^❯/mu.test(screen.snapshot());+}++/**+ * A drawer draws only the row under its cursor at full weight; every other+ * option, each description, and the freeform row are dim. A person reads the+ * options by moving the cursor through them, and so does the driver, until the+ * cursor wraps to the first option or stops at the last. The TUI repaints+ * synchronously on each key, so the screen is current right after one.+ */+function readOptions(screen: MockScreen, keyboard: MockUserInput, prompt: string) {+ const options: RenderedOption[] = [];+ const first = focusedRow(screen, prompt);+ let row = first;+ while (row !== undefined) {+ if (!row.freeform) options.push({ handle: options.length, label: row.text });+ keyboard.down();+ const next = focusedRow(screen, prompt);+ if (next === undefined || next.line === row.line || next.line === first?.line) break;+ row = next;+ }+ // Leave the cursor on the first option, as a fresh drawer has it.+ for (let moves = 0; moves < 10 && focusedRow(screen, prompt)?.line !== first?.line; moves++) {+ keyboard.up();+ }+ return options;+}++/** The full-weight row between `prompt` and its drawer's footer: the one under the cursor. */+function focusedRow(+ screen: MockScreen,+ prompt: string,+): { readonly freeform: boolean; readonly line: number; readonly text: string } | undefined {+ const lines = screen.snapshot().split("\n");+ const bright = screen.snapshot({ hideDim: true }).split("\n");+ const footer = lines.findLastIndex((line) =>+ DRAWER_FOOTERS.some((candidate) => line.includes(candidate)),+ );+ const start = bright.slice(0, footer).findLastIndex((line) => line.trim() === prompt);+ if (footer === -1 || start === -1) return undefined;+ for (let line = start + 1; line < footer; line += 1) {+ const text = bright[line]!.trim();+ if (text.length === 0) continue;+ return { freeform: lines[line]!.includes(FREEFORM_ROW), line, text };+ }+ return undefined;+}diff --git a/packages/eve/test/tui-client/tui-questions.ts b/packages/eve/test/tui-client/tui-questions.tsdeleted file mode 100644index 0815cef0e..000000000--- a/packages/eve/test/tui-client/tui-questions.ts+++ /dev/null@@ -1,127 +0,0 @@-import { setTimeout as sleep } from "node:timers/promises";--import { Client } from "eve/client";-import { EveTUIRunner, MockScreen, MockUserInput } from "./lib/tui.ts";--import { run } from "./lib/run.ts";-import { theme } from "./lib/theme.ts";--/**- * End-to-end proof that the TUI handles `input.requested` events with- * `display: "select"` correctly. The legacy translator collapsed every- * input request into a `tool-approval-request` chunk, so multi-option- * `ask_question` calls were unanswerable through the TUI. This smoke- * test drives the new `readInputQuestion` path:- *- * 1. Start the apps/fixtures/agent-tui-client server.- * 2. Boot an `EveTUIRunner` with a mock terminal.- * 3. Type a prompt that asks the model to call `ask_question` with- * two options (Red/Blue).- * 4. Wait for the question drawer's control row, which proves it is ready- * for input.- * 5. Send Down arrow + Enter to pick the second option (blue).- * 6. Wait for the answered marker (`⎿ <label>`) in the transcript.- * 7. Wait for the post-answer assistant turn to render. The runner- * only returns to its prompt-reading state if the chosen option- * flowed through to the agent and resolved the pending input.- */--process.env.EVE_TUI_UNICODE = "1";--run({ app: "agent-tui-client", kind: "local-build" }, async (target) => {- const client = new Client({ host: target.baseUrl });- const screen = new MockScreen({ columns: 100, rows: 40 });- const input = new MockUserInput();- const runner = new EveTUIRunner({- client,- screen,- userInput: input,- name: "TUI smoke",- });-- const runPromise = runner.run().catch((error: unknown) => {- if (error instanceof Error && error.message === "Interrupted") {- return;- }- throw error;- });-- await screen.waitForIdlePrompt(5_000);-- const promptLines = [- "Use the ask_question tool exactly once to ask me which color I prefer.",- "Set question to: 'Pick a color.'",- 'Provide exactly two options: label "Red" and label "Blue".',- "Wait for my response.",- ];- input.type(promptLines.join(" · "));- input.enter();-- await screen.waitForText("↑/↓ move · enter to select · esc to dismiss", 60_000);- console.log(theme.muted("[tui-questions] select drawer is live"));-- // Bridges a server-side race where the park hook isn't yet- // registered when `session.waiting` reaches the client. The TUI- // is human-paced in practice; scripted clients need a short- // handshake delay before delivering the response.- await sleep(1_000);-- input.down();- input.enter();- console.log(theme.muted("[tui-questions] selected Blue"));-- await screen.waitForText("⎿ Blue", 5_000);- console.log(theme.muted("[tui-questions] answer recorded in body"));-- // The post-answer turn must produce a `▲`-prefixed assistant- // section whose body contains "blue", proof that the chosen- // optionId flowed through to the agent and the model produced a- // reply reflecting the choice. We only search the region after the- // "⎿ Blue" answer marker to avoid a false positive from the marker- // itself (transcript blocks commit in order, so the follow-up reply- // always renders below the answered question).- await waitForCondition(() => assistantReplyAfter(screen.snapshot(), "⎿ Blue", "blue"), {- timeoutMs: 60_000,- label: "follow-up assistant section mentioning blue",- });- console.log(theme.muted("[tui-questions] follow-up assistant turn rendered"));-- const finalSnapshot = screen.snapshot();- if (finalSnapshot.includes("Error")) {- throw new Error(`Final screen contains an Error section:\n${finalSnapshot}`);- }-- // The turn is complete; wait until the runner is back at the prompt so- // Two idle Ctrl+C presses exit; mid-stream Ctrl+C cancels cooperatively.- await screen.waitForIdlePrompt(30_000);- input.ctrlC();- input.ctrlC();- await runPromise;-});--/**- * True when an assistant section (a `▲`-prefixed line and its wrapped- * continuations) below `afterMarker` contains `needle`. Used to assert a- * follow-up reply without matching transcript content above the marker.- */-function assistantReplyAfter(snapshot: string, afterMarker: string, needle: string): boolean {- const markerIndex = snapshot.indexOf(afterMarker);- if (markerIndex === -1) return false;- const region = snapshot.slice(markerIndex + afterMarker.length);- const brandIndex = region.indexOf("▲ ");- if (brandIndex === -1) return false;- return region.slice(brandIndex).toLowerCase().includes(needle.toLowerCase());-}--async function waitForCondition(- predicate: () => boolean,- options: { timeoutMs: number; label: string; intervalMs?: number },-): Promise<void> {- const intervalMs = options.intervalMs ?? 100;- const deadline = Date.now() + options.timeoutMs;- while (Date.now() < deadline) {- if (predicate()) return;- await sleep(intervalMs);- }- throw new Error(`Timed out waiting for: ${options.label}`);-}