Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

[1/3] test(eve): add a TUI driver to the HITL conformance suite (#4218)

Ben Pankowbfc122e50873
diff --git a/packages/eve/src/cli/dev/tui/hitl-conformance.integration.test.ts b/packages/eve/src/cli/dev/tui/hitl-conformance.integration.test.ts
new file mode 100644
index 000000000..2851a1e9e
--- /dev/null
+++ b/packages/eve/src/cli/dev/tui/hitl-conformance.integration.test.ts
@@ -0,0 +1,3 @@
+import { describeHitlConformance } from "#internal/testing/channel-conformance/conformance.js";
+
+describeHitlConformance("tui");
diff --git a/packages/eve/src/cli/dev/tui/test/mock-terminal.ts b/packages/eve/src/cli/dev/tui/test/mock-terminal.ts
index c9b959a0f..198235bc4 100644
--- a/packages/eve/src/cli/dev/tui/test/mock-terminal.ts
+++ b/packages/eve/src/cli/dev/tui/test/mock-terminal.ts
@@ -78,9 +78,17 @@ export class MockScreen extends EventEmitter implements TerminalOutput {
rows: number;
#rawOutput = "";
#lines: string[] = [];
+ /** Parallel to `#lines`: `d` marks a cell drawn dim (SGR 2), a space any other cell. */
+ #dimCells: string[] = [];
+ #dim = false;
#cursorLine = 0;
#cursorColumn = 0;
- #mainScreen?: { lines: string[]; cursorLine: number; cursorColumn: number };
+ #mainScreen?: {
+ lines: string[];
+ dimCells: string[];
+ cursorLine: number;
+ cursorColumn: number;
+ };
#waiters: Array<{
text: string;
resolve: () => void;
@@ -118,8 +126,21 @@ export class MockScreen extends EventEmitter implements TerminalOutput {
this.emit("resize");
}
- snapshot() {
- return this.#lines.join("\n");
+ /**
+ * The visible grid as text. `hideDim` blanks dim cells, leaving what the
+ * screen draws at full weight, such as option labels without descriptions.
+ */
+ snapshot({ hideDim = false }: { readonly hideDim?: boolean } = {}) {
+ if (!hideDim) return this.#lines.join("\n");
+ return this.#lines
+ .map((line, index) => {
+ const dim = this.#dimCells[index] ?? "";
+ // Index by UTF-16 unit: a cell's marks repeat for each unit it occupies.
+ return line.replace(/./gsu, (cell, column: number) =>
+ dim[column] === "d" ? " ".repeat(cell.length) : cell,
+ );
+ })
+ .join("\n");
}
rawOutput() {
@@ -249,14 +270,17 @@ export class MockScreen extends EventEmitter implements TerminalOutput {
if (command === "h") {
this.#mainScreen = {
lines: this.#lines,
+ dimCells: this.#dimCells,
cursorLine: this.#cursorLine,
cursorColumn: this.#cursorColumn,
};
this.#lines = [];
+ this.#dimCells = [];
this.#cursorLine = 0;
this.#cursorColumn = 0;
} else if (this.#mainScreen !== undefined) {
this.#lines = this.#mainScreen.lines;
+ this.#dimCells = this.#mainScreen.dimCells;
this.#cursorLine = this.#mainScreen.cursorLine;
this.#cursorColumn = this.#mainScreen.cursorColumn;
this.#mainScreen = undefined;
@@ -300,6 +324,9 @@ export class MockScreen extends EventEmitter implements TerminalOutput {
case "K":
this.#eraseInLine(first(0));
break;
+ case "m":
+ this.#applySgr(parameters);
+ break;
default:
break;
}
@@ -307,9 +334,32 @@ export class MockScreen extends EventEmitter implements TerminalOutput {
return startIndex + sequence.length;
}
+ /**
+ * Tracks only intensity: 2 starts dim; 0 and 22 end it, as in a real terminal
+ * where bold (1) leaves faint on. Extended colors (38, 48, 58) carry a
+ * `5;n` or `2;r;g;b` payload whose numbers are not SGR codes.
+ */
+ #applySgr(parameters: readonly string[]) {
+ const codes = (parameters.length === 0 ? [""] : parameters).map((parameter) =>
+ parameter === "" ? 0 : Number(parameter),
+ );
+ for (let index = 0; index < codes.length; index += 1) {
+ const code = codes[index];
+ if (code === 38 || code === 48 || code === 58) {
+ const mode = codes[index + 1];
+ index += mode === 5 ? 2 : mode === 2 ? 4 : 1;
+ } else if (code === 2) {
+ this.#dim = true;
+ } else if (code === 0 || code === 22) {
+ this.#dim = false;
+ }
+ }
+ }
+
#eraseInDisplay(mode: number) {
if (mode === 2 || mode === 3) {
this.#lines = [];
+ this.#dimCells = [];
this.#cursorLine = 0;
this.#cursorColumn = 0;
return;
@@ -319,6 +369,7 @@ export class MockScreen extends EventEmitter implements TerminalOutput {
// Cursor to start of screen.
for (let line = 0; line < this.#cursorLine; line += 1) {
this.#lines[line] = "";
+ this.#dimCells[line] = "";
}
this.#eraseInLine(1);
return;
@@ -328,33 +379,31 @@ export class MockScreen extends EventEmitter implements TerminalOutput {
// and drop every line below it.
this.#eraseInLine(0);
this.#lines.length = Math.min(this.#lines.length, this.#cursorLine + 1);
+ this.#dimCells.length = Math.min(this.#dimCells.length, this.#cursorLine + 1);
}
#eraseInLine(mode: number) {
- const line = this.#lines[this.#cursorLine] ?? "";
-
- if (mode === 2) {
- this.#lines[this.#cursorLine] = "";
- return;
- }
-
- if (mode === 1) {
- this.#lines[this.#cursorLine] =
- " ".repeat(this.#cursorColumn) + line.slice(this.#cursorColumn);
- return;
+ for (const grid of [this.#lines, this.#dimCells]) {
+ const line = grid[this.#cursorLine] ?? "";
+ if (mode === 2) {
+ grid[this.#cursorLine] = "";
+ } else if (mode === 1) {
+ grid[this.#cursorLine] = " ".repeat(this.#cursorColumn) + line.slice(this.#cursorColumn);
+ } else {
+ // mode 0: clear from cursor to end of line.
+ grid[this.#cursorLine] = line.slice(0, this.#cursorColumn);
+ }
}
-
- // mode 0: clear from cursor to end of line.
- this.#lines[this.#cursorLine] = line.slice(0, this.#cursorColumn);
}
#writeCharacter(character: string) {
- const line = (this.#lines[this.#cursorLine] ?? "").padEnd(this.#cursorColumn, " ");
- const nextLine =
- line.slice(0, this.#cursorColumn) +
- character +
- line.slice(this.#cursorColumn + character.length);
- this.#lines[this.#cursorLine] = nextLine;
+ const write = (grid: string[], cell: string) => {
+ const line = (grid[this.#cursorLine] ?? "").padEnd(this.#cursorColumn, " ");
+ grid[this.#cursorLine] =
+ line.slice(0, this.#cursorColumn) + cell + line.slice(this.#cursorColumn + cell.length);
+ };
+ write(this.#lines, character);
+ write(this.#dimCells, (this.#dim ? "d" : " ").repeat(character.length));
this.#cursorColumn += character.length;
}
}
diff --git a/packages/eve/src/internal/testing/channel-conformance/conformance.ts b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
index 5faedef81..c588d06a9 100644
--- a/packages/eve/src/internal/testing/channel-conformance/conformance.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/conformance.ts
@@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest";
import { type HitlRule, hitlContract } from "#internal/testing/channel-conformance/contract.js";
import {
type ChannelDriver,
+ type ClientDriver,
withChannelConversation,
} from "#internal/testing/channel-conformance/harness.js";
import {
@@ -16,6 +17,7 @@ import { linqDriver } from "#internal/testing/channel-conformance/linq-driver.js
import { slackDriver } from "#internal/testing/channel-conformance/slack-driver.js";
import { teamsDriver } from "#internal/testing/channel-conformance/teams-driver.js";
import { telegramDriver } from "#internal/testing/channel-conformance/telegram-driver.js";
+import { tuiDriver } from "#internal/testing/channel-conformance/tui-driver.js";
import { twilioDriver } from "#internal/testing/channel-conformance/twilio-driver.js";
export interface BrokenCell {
@@ -25,17 +27,28 @@ export interface BrokenCell {
}
interface ConformanceChannel {
- readonly driver: () => ChannelDriver;
+ readonly driver: () => ChannelDriver | ClientDriver;
readonly broken?: Partial<Record<HitlRule, BrokenCell>>;
+ /** Rules this client deliberately doesn't offer, with why, beyond what its capabilities rule out. */
+ readonly unsupported?: Partial<Record<HitlRule, string>>;
}
+const TUI_TYPED_APPROVAL =
+ "the approval drawer holds the keyboard; a person answers it with y or n";
+
+const TUI_APPROVAL_PROMPT: BrokenCell = {
+ reason: "the approval drawer titles the raw tool name instead of the request's prompt",
+ symptom: /Timed out waiting for the question "Approve Deploy release\?" on tui/,
+};
+
/**
- * Every first-party channel's place in the HITL contract, keyed by the channel
- * directory whose `hitl-conformance.integration.test.ts` runs it. Each cell is
+ * Every first-party channel's and client's place in the HITL contract, keyed by
+ * the directory whose `hitl-conformance.integration.test.ts` runs it. Each cell is
* one of:
*
* - must pass;
- * - not supported: the platform lacks a capability the rule requires (skipped);
+ * - not supported: the platform lacks a capability the rule requires, or the
+ * entry declines the rule as `unsupported` (skipped);
* - broken: the channel should pass but doesn't yet. The cell passes only when
* the rule fails with the recorded `symptom`, so a fix or an unrelated
* failure (such as harness breakage) both turn it red.
@@ -49,6 +62,20 @@ const hitlConformance = {
slack: [{ driver: slackDriver }],
teams: [{ driver: teamsDriver }],
telegram: [{ driver: telegramDriver }],
+ tui: [
+ {
+ driver: tuiDriver,
+ broken: {
+ "a tool approval shows a choice to approve and one to cancel": TUI_APPROVAL_PROMPT,
+ "pressing Approve runs the gated tool": TUI_APPROVAL_PROMPT,
+ "pressing Cancel stops the gated tool without running it": TUI_APPROVAL_PROMPT,
+ },
+ unsupported: {
+ "a text reply of approve runs the gated tool": TUI_TYPED_APPROVAL,
+ "a text reply of cancel stops the gated tool without running it": TUI_TYPED_APPROVAL,
+ },
+ },
+ ],
twilio: [{ driver: twilioDriver }],
} satisfies Record<string, readonly ConformanceChannel[]>;
@@ -67,13 +94,16 @@ export function describeHitlConformance(channel: keyof typeof hitlConformance):
const entries: readonly ConformanceChannel[] = hitlConformance[channel];
describe.each(entries.map((entry) => ({ ...entry, name: entry.driver().name })))(
"$name HITL contract",
- ({ driver, broken }) => {
+ ({ driver, broken, unsupported }) => {
const { capabilities } = driver();
for (const rule of hitlContract) {
const supported = rule.requires.every((capability) => capabilities.includes(capability));
const known = broken?.[rule.rule];
+ const declined = unsupported?.[rule.rule];
if (!supported) {
it.skip(`${rule.rule} (not supported)`, () => {});
+ } else if (declined !== undefined) {
+ it.skip(`${rule.rule} (not supported: ${declined})`, () => {});
} else if (known === undefined) {
it(rule.rule, () => withChannelConversation(driver(), (c) => rule.run(c)), 60_000);
} else {
diff --git a/packages/eve/src/internal/testing/channel-conformance/contract.ts b/packages/eve/src/internal/testing/channel-conformance/contract.ts
index 0833b01b5..6ee2ba8d3 100644
--- a/packages/eve/src/internal/testing/channel-conformance/contract.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/contract.ts
@@ -29,6 +29,9 @@ async function askWhichDay(conversation: ChannelConversation) {
}
const DEPLOY = `Use ${GATED_TOOL} to ship the release.`;
+// Channels word the two approval choices differently; the TUI asks Yes or No.
+const APPROVE_LABELS = ["Approve", "Yes"];
+const CANCEL_LABELS = ["Cancel", "No"];
const APPROVAL_PROMPT = "Approve Deploy release?";
async function askToDeploy(conversation: ChannelConversation) {
@@ -97,16 +100,22 @@ export const hitlContract = [
},
},
{
- rule: "a tool approval shows Approve and Cancel",
+ rule: "a tool approval shows a choice to approve and one to cancel",
source: "docs/tools/human-in-the-loop.md#approvals",
requires: [],
async run(conversation) {
const options = await askToDeploy(conversation);
- const labels = options.map((option) => option.label).sort();
- expect(labels, `the approval showed ${JSON.stringify(labels)}`).toEqual([
- "Approve",
- "Cancel",
- ]);
+ const labels = options.map((option) => option.label);
+ const message = `the approval showed ${JSON.stringify(labels)}`;
+ expect(labels, message).toHaveLength(2);
+ expect(
+ labels.filter((label) => APPROVE_LABELS.includes(label)),
+ message,
+ ).toHaveLength(1);
+ expect(
+ labels.filter((label) => CANCEL_LABELS.includes(label)),
+ message,
+ ).toHaveLength(1);
},
},
{
@@ -115,7 +124,7 @@ export const hitlContract = [
requires: ["buttons"],
async run(conversation) {
const options = await askToDeploy(conversation);
- const approve = options.find((option) => option.label === "Approve");
+ const approve = options.find((option) => APPROVE_LABELS.includes(option.label));
expect(approve, "an Approve option to press").toBeDefined();
await conversation.press(approve!);
await expectDeployed(conversation);
@@ -127,7 +136,7 @@ export const hitlContract = [
requires: ["buttons"],
async run(conversation) {
const options = await askToDeploy(conversation);
- const cancel = options.find((option) => option.label === "Cancel");
+ const cancel = options.find((option) => CANCEL_LABELS.includes(option.label));
expect(cancel, "a Cancel option to press").toBeDefined();
await conversation.press(cancel!);
await expectNotDeployed(conversation);
diff --git a/packages/eve/src/internal/testing/channel-conformance/harness.ts b/packages/eve/src/internal/testing/channel-conformance/harness.ts
index 9f278958e..23412d83b 100644
--- a/packages/eve/src/internal/testing/channel-conformance/harness.ts
+++ b/packages/eve/src/internal/testing/channel-conformance/harness.ts
@@ -1,11 +1,15 @@
import { createChannelOperations } from "#channel/channel-operations.js";
import { type CompiledChannel, isCompiledChannel } from "#channel/compiled-channel.js";
import { type RouteHandlerArgs, isHttpRouteDefinition } from "#channel/routes.js";
+import { sessionInboxHookToken } from "#execution/session-inbox/address.js";
import { createWorkflowRuntime } from "#execution/workflow-runtime.js";
import { createTestRuntime } from "#internal/testing/app-harness.js";
import { createBundledRuntimeCompiledArtifactsSource } from "#runtime/compiled-artifacts-source.js";
import { getCompiledRuntimeAgentBundle } from "#runtime/sessions/compiled-agent-cache.js";
-import type { Session } from "#channel/session.js";
+import { createAttachSessionFn, type Session } from "#channel/session.js";
+import { attachRouteSessionCreator } from "#internal/nitro/routes/channel-route-context.js";
+import { none } from "#public/channels/auth.js";
+import { eveChannel } from "#public/channels/eve.js";
import { z } from "#compiled/zod/index.js";
import { always } from "#tools/approval/policies.js";
import { defineTool } from "#tools/definition.js";
@@ -94,12 +98,56 @@ export interface ChannelConversation {
readonly gatedToolRuns: number;
}
+/**
+ * Teaches the suite to drive a client UI that talks to eve's own HTTP channel
+ * (`eveChannel`) through `eve/client`, such as the dev TUI. Like a webhook
+ * driver, it reads only what the client shows a person.
+ */
+export interface ClientDriver {
+ readonly name: string;
+ readonly capabilities: readonly ChannelCapability[];
+ /**
+ * Starts the client against the agent at `host`. The global `fetch` serves
+ * that origin from the eve channel's routes until the client closes.
+ */
+ open(host: string, wait: Wait): Promise<ClientView>;
+}
+
+/** A running client, as a person sees and uses it. */
+export interface ClientView {
+ /** The person sends a plain-text message. */
+ say(text: string): Promise<void>;
+ /** Waits for the client to show `prompt` with choices, returning them. */
+ waitForQuestion(prompt: string): Promise<readonly RenderedOption[]>;
+ /** The person presses one shown choice. */
+ press(option: RenderedOption): Promise<void>;
+ /** The bot replies the client shows now. */
+ replies(): readonly string[];
+ /** What the client shows now, for timeout errors. */
+ describe(): string;
+ /** Stops the client and releases anything it holds, such as its event stream. */
+ close(): Promise<void>;
+}
+
+/**
+ * Polls `select` until it returns a value. On timeout, the error names
+ * `label`, the driver, and `describe()`'s account of what a person sees.
+ */
+export type Wait = <T>(
+ label: string,
+ select: () => T | undefined | Promise<T | undefined>,
+ describe: () => string,
+) => Promise<T>;
+
/**
* How long to wait for a platform call. Steps finish in well under a second, so
* this mostly absorbs the first conversation's cold start on a busy machine.
*/
const WAIT_TIMEOUT_MS = 30_000;
+/** Stand-in origin for the agent a {@link ClientDriver} talks to. */
+const CLIENT_HOST = "https://agent.example.com";
+
/** The test agent's tool that always needs a person's approval before it runs. */
export const GATED_TOOL = "deploy_release";
@@ -121,39 +169,141 @@ export function recordingFetch(
}
/**
- * Runs `body` against an agent with `ask_question` and `driver`'s channel. Every
- * interaction goes through the channel's real webhook routes; the only fake is
- * the platform behind the channel, usually its injected `fetch`.
+ * Runs `body` against an agent with `ask_question`, an approval-gated tool, and
+ * `driver`'s channel. Every interaction goes through the channel's real routes.
+ * A webhook driver fakes only the platform behind its channel, usually its
+ * injected `fetch`; a client driver runs the real client against `eveChannel`.
*
* Conversations must not overlap: each compiles its own agent, and concurrent
* ones can resolve each other's compiled artifacts in the shared workflow world.
*/
export async function withChannelConversation(
- driver: ChannelDriver,
+ driver: ChannelDriver | ClientDriver,
body: (conversation: ChannelConversation) => Promise<void>,
options: { readonly waitTimeoutMs?: number } = {},
): Promise<void> {
+ const waitTimeoutMs = options.waitTimeoutMs ?? WAIT_TIMEOUT_MS;
+ const wait: Wait = (label, select, describe) =>
+ poll(`${label} on ${driver.name}`, select, describe, waitTimeoutMs);
+ if (isClientDriver(driver)) {
+ await converse(driver.name, "eve", eveChannel({ auth: none() }), body, wait, (dispatch) =>
+ openClient(driver, dispatch, wait),
+ );
+ return;
+ }
const calls: PlatformCall[] = [];
try {
- await converse(driver, calls, body, options.waitTimeoutMs ?? WAIT_TIMEOUT_MS);
+ const channel = driver.createChannel((call) => void calls.push(call));
+ await converse(driver.name, driver.name, channel, body, wait, async (dispatch) =>
+ webhookView(driver, calls, dispatch, wait),
+ );
} finally {
driver.dispose?.();
}
}
-async function converse(
+function isClientDriver(driver: ChannelDriver | ClientDriver): driver is ClientDriver {
+ return "open" in driver;
+}
+
+async function poll<T>(
+ label: string,
+ select: () => T | undefined | Promise<T | undefined>,
+ describe: () => string,
+ timeoutMs: number,
+): Promise<T> {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ const selected = await select();
+ if (selected !== undefined) return selected;
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ }
+ throw new Error(`Timed out waiting for ${label}. ${describe()}`);
+}
+
+/** Serves one request from the channel's routes; `settled` resolves once its `waitUntil` work does. */
+type Dispatch = (request: Request) => Promise<{ response: Response; settled: Promise<unknown> }>;
+
+/** A webhook channel seen through its platform calls: what the bot posted. */
+function webhookView(
driver: ChannelDriver,
- calls: PlatformCall[],
+ calls: readonly PlatformCall[],
+ dispatch: Dispatch,
+ wait: Wait,
+): ClientView {
+ async function post(request: Request): Promise<void> {
+ const { response, settled } = await dispatch(request);
+ await settled;
+ if (!response.ok) throw new Error(`${driver.name} webhook answered ${response.status}.`);
+ }
+ const describe = () => `Platform calls:\n${JSON.stringify(calls, null, 2)}`;
+
+ return {
+ say: (text) => post(driver.message(text)),
+ press: (option) => post(driver.press(option)),
+ waitForQuestion: (prompt) =>
+ wait(
+ `the question "${prompt}"`,
+ () => {
+ for (const call of calls) {
+ const options = driver.findOptions(call, prompt);
+ if (options !== undefined) return options;
+ }
+ return undefined;
+ },
+ describe,
+ ),
+ replies: () => calls.flatMap((call) => driver.postedText(call) ?? []),
+ describe,
+ close: async () => {},
+ };
+}
+
+/** Starts `driver`'s client with the client host's `fetch` routed to the channel until it closes. */
+async function openClient(
+ driver: ClientDriver,
+ dispatch: Dispatch,
+ wait: Wait,
+): Promise<ClientView> {
+ const original = globalThis.fetch;
+ globalThis.fetch = async (input, init) => {
+ const request = new Request(input, init);
+ if (new URL(request.url).origin !== CLIENT_HOST) return await original(input, init);
+ return (await dispatch(request)).response;
+ };
+ const restore = () => void (globalThis.fetch = original);
+ try {
+ const view = await driver.open(CLIENT_HOST, wait);
+ return {
+ ...view,
+ async close() {
+ try {
+ await view.close();
+ } finally {
+ restore();
+ }
+ },
+ };
+ } catch (error) {
+ restore();
+ throw error;
+ }
+}
+
+async function converse(
+ label: string,
+ channelName: string,
+ created: unknown,
body: (conversation: ChannelConversation) => Promise<void>,
- waitTimeoutMs: number,
+ wait: Wait,
+ open: (dispatch: Dispatch) => Promise<ClientView>,
): Promise<void> {
- const created = driver.createChannel((call) => void calls.push(call));
- if (!isCompiledChannel(created)) throw new Error(`${driver.name} is not a compiled channel.`);
+ if (!isCompiledChannel(created)) throw new Error(`${label} is not a compiled channel.`);
const channel: CompiledChannel = created;
let gatedToolRuns = 0;
const runtime = await createTestRuntime({
- agent: { name: `${driver.name}-hitl-conformance` },
+ agent: { name: `${label}-hitl-conformance` },
modules: [
{
logicalPath: "tools/ask_question.ts",
@@ -174,7 +324,7 @@ async function converse(
}),
},
{
- logicalPath: `channels/${driver.name}.ts`,
+ logicalPath: `channels/${channelName}.ts`,
loadNamespace: async () => ({ default: channel }),
},
],
@@ -183,119 +333,166 @@ async function converse(
await runtime.run(async () => {
const compiledArtifactsSource = createBundledRuntimeCompiledArtifactsSource();
const bundle = await getCompiledRuntimeAgentBundle({ compiledArtifactsSource });
- const entry = bundle.graph.root.channels.find((candidate) => candidate.name === driver.name);
- if (entry?.adapter === undefined) throw new Error(`Expected the ${driver.name} adapter.`);
- // Mirrors the operations production route dispatch builds for each request.
+ const entry = bundle.graph.root.channels.find((candidate) => candidate.name === channelName);
+ if (entry?.adapter === undefined) throw new Error(`Expected the ${channelName} adapter.`);
+ const { adapter, turnPolicy } = entry;
+ const workflowRuntime = createWorkflowRuntime({ compiledArtifactsSource });
+ // Mirrors the route arguments production channel dispatch builds for each request.
const operations = createChannelOperations({
- adapter: entry.adapter,
- channelName: driver.name,
- runtime: createWorkflowRuntime({ compiledArtifactsSource }),
- turnPolicy: entry.turnPolicy,
+ adapter,
+ channelName,
+ runtime: workflowRuntime,
+ turnPolicy,
+ });
+ const attachSession = createAttachSessionFn(workflowRuntime, {
+ channelKind: adapter.kind,
+ channelName,
+ turnPolicy,
});
const sessions = new Map<string, Session>();
+ const background: Promise<unknown>[] = [];
+
+ function track(session: Session): Session {
+ sessions.set(session.id, session);
+ return session;
+ }
- async function post(request: Request): Promise<void> {
+ const dispatch: Dispatch = async (request) => {
+ const { params, route } = findRoute(channel, request);
const pending: Promise<unknown>[] = [];
- const args: RouteHandlerArgs = {
- ...operations,
- from: (address) => {
- const source = operations.from(address);
- return {
- ...source,
- send: async (...sendArgs) => track(await source.send(...sendArgs)),
- respond: async (...respondArgs) => track(await source.respond(...respondArgs)),
- };
+ const args = attachRouteSessionCreator<RouteHandlerArgs>(
+ {
+ ...operations,
+ from: (address) => {
+ const source = operations.from(address);
+ return {
+ ...source,
+ send: async (...sendArgs) => track(await source.send(...sendArgs)),
+ respond: async (...respondArgs) => {
+ await waitForAddress(address);
+ return track(await source.respond(...respondArgs));
+ },
+ };
+ },
+ attachSession,
+ params,
+ requestIp: null,
+ to: unsupported("to"),
+ waitUntil: (task) => void pending.push(task),
},
- attachSession: unsupported("attachSession"),
- params: {},
- requestIp: null,
- to: unsupported("to"),
- waitUntil: (task) => void pending.push(task),
- };
- const response = await findRoute(channel, request).handler(request, args);
- await Promise.all(pending);
- if (!response.ok) throw new Error(`${driver.name} webhook answered ${response.status}.`);
- }
+ async (input) => {
+ const handle = await workflowRuntime.createSession({
+ ...input,
+ adapter,
+ channelName,
+ continuationToken:
+ input.continuationToken === undefined
+ ? undefined
+ : `${channelName}:${input.continuationToken}`,
+ });
+ track(attachSession(handle.sessionId));
+ return handle;
+ },
+ );
+ const response = await route.handler(request, args);
+ const settled = Promise.all(pending);
+ background.push(settled);
+ return { response, settled };
+ };
- async function waitFor<T>(label: string, select: (call: PlatformCall) => T | undefined) {
- const deadline = Date.now() + waitTimeoutMs;
- while (Date.now() < deadline) {
- for (const call of calls) {
- const selected = select(call);
- if (selected !== undefined) return selected;
- }
- await new Promise((resolve) => setTimeout(resolve, 25));
- }
- throw new Error(
- `Timed out waiting for ${label} on ${driver.name}. Platform calls:\n${JSON.stringify(calls, null, 2)}`,
+ /** How many times each session's turn has held for input, as of the last wait. */
+ const inputHolds = new Map<string, number>();
+ const view = await open(dispatch);
+ const replyWait = <T>(label: string, select: (reply: string) => T | undefined) =>
+ wait(
+ label,
+ () => {
+ for (const reply of view.replies()) {
+ const selected = select(reply);
+ if (selected !== undefined) return selected;
+ }
+ return undefined;
+ },
+ view.describe,
);
- }
const conversation: ChannelConversation = {
- say: (text) => post(driver.message(text)),
- press: (option) => post(driver.press(option)),
+ say: (text) => view.say(text),
+ press: (option) => view.press(option),
async waitForQuestion(prompt) {
- const options = await waitFor(`the question "${prompt}"`, (call) =>
- driver.findOptions(call, prompt),
- );
- await waitForTurnToHoldForInput();
+ const options = await view.waitForQuestion(prompt);
+ await holdForInput();
return options;
},
waitForToolResult: (tool) =>
- waitFor(`${tool} to return`, (call) => {
- const text = driver.postedText(call);
- return text === undefined ? undefined : readMockToolReply(text, tool);
- }),
+ replyWait(`${tool} to return`, (reply) => readMockToolReply(reply, tool)),
waitForToolOutcome: (tool) =>
- waitFor(`${tool} to run or be denied`, (call): ToolOutcome | undefined => {
- const text = driver.postedText(call);
- if (text === undefined) return undefined;
- const output = readMockToolReply(text, tool);
+ replyWait(`${tool} to run or be denied`, (reply): ToolOutcome | undefined => {
+ const output = readMockToolReply(reply, tool);
if (output !== undefined) return { kind: "ran", output };
- return isMockDenialReply(text) ? { kind: "denied" } : undefined;
+ return isMockDenialReply(reply) ? { kind: "denied" } : undefined;
}),
get gatedToolRuns() {
return gatedToolRuns;
},
};
- /** How many times each session's turn has held for input, as of the last wait. */
- const inputHolds = new Map<string, number>();
+ /**
+ * The turn emits `turn.waiting` inside the step that raised the request, but
+ * an address the channel aliased in that step (such as Discord's message id)
+ * is only claimed once the step commits. Answering through it before then
+ * finds no session.
+ */
+ async function waitForAddress(address: string): Promise<void> {
+ const world = await getWorld();
+ const token = sessionInboxHookToken(`${channelName}:${address}`);
+ await wait(
+ `the channel to claim the address "${address}"`,
+ () =>
+ world.hooks.getByToken(token).then(
+ () => true,
+ () => undefined,
+ ),
+ () => "",
+ );
+ }
/**
* A person answers once the bot has finished asking. Answering the moment
- * the question appears races the channel's own bookkeeping for it, such as
- * Discord aliasing the session to the message it just posted.
+ * the question appears races the turn's own bookkeeping for it; see also
+ * {@link waitForAddress}.
*/
- async function waitForTurnToHoldForInput(): Promise<void> {
- const deadline = Date.now() + waitTimeoutMs;
- while (Date.now() < deadline) {
- for (const session of sessions.values()) {
- const holds = await countInputHolds(session);
- if (holds > (inputHolds.get(session.id) ?? 0)) {
- inputHolds.set(session.id, holds);
- return;
+ function holdForInput(): Promise<true> {
+ return wait(
+ "the turn to hold for input",
+ async () => {
+ for (const session of sessions.values()) {
+ const holds = await countInputHolds(session);
+ if (holds > (inputHolds.get(session.id) ?? 0)) {
+ inputHolds.set(session.id, holds);
+ return true;
+ }
}
- }
- await new Promise((resolve) => setTimeout(resolve, 25));
- }
- throw new Error(`Timed out waiting for the turn to hold for input on ${driver.name}.`);
- }
-
- function track(session: Session): Session {
- sessions.set(session.id, session);
- return session;
+ return undefined;
+ },
+ () => "",
+ );
}
// The test file's workflow world closes after its last test, so a session
// still writing then fails with an unhandled rejection.
- const settle = () => Promise.all([...sessions.values()].map(cancelUntilResting));
+ const settle = async () => {
+ await Promise.all([...sessions.values()].map(cancelUntilResting));
+ await Promise.all(background);
+ };
try {
await body(conversation);
} catch (error) {
+ await view.close().catch(() => {});
await settle().catch(() => {});
throw error;
}
+ await view.close();
await settle();
});
}
@@ -342,16 +539,26 @@ async function countInputHolds(session: Session): Promise<number> {
function findRoute(channel: CompiledChannel, request: Request) {
const { pathname } = new URL(request.url);
- const route = channel.routes.find(
- (candidate) =>
- isHttpRouteDefinition(candidate) &&
- candidate.method === request.method &&
- candidate.path === pathname,
- );
- if (route === undefined || !isHttpRouteDefinition(route)) {
- throw new Error(`No ${request.method} route for ${pathname}.`);
+ for (const route of channel.routes) {
+ if (!isHttpRouteDefinition(route) || route.method !== request.method) continue;
+ const params = matchPath(route.path, pathname);
+ if (params !== undefined) return { params, route };
+ }
+ throw new Error(`No ${request.method} route for ${pathname}.`);
+}
+
+/** Matches a route path whose `:name` segments each capture one path segment. */
+function matchPath(pattern: string, pathname: string): Record<string, string> | undefined {
+ const expected = pattern.split("/");
+ const actual = pathname.split("/");
+ if (expected.length !== actual.length) return undefined;
+ const params: Record<string, string> = {};
+ for (const [index, segment] of expected.entries()) {
+ const value = actual[index]!;
+ if (segment.startsWith(":")) params[segment.slice(1)] = decodeURIComponent(value);
+ else if (segment !== value) return undefined;
}
- return route;
+ return params;
}
function unsupported(name: string): () => never {
@@ -365,7 +572,7 @@ function unsupported(name: string): () => never {
* `Used <tool> for "<message>": <JSON output>`. Reading the output back from
* the posted reply keeps rules on what a person sees.
*/
-function readMockToolReply(text: string, tool: string): unknown {
+export function readMockToolReply(text: string, tool: string): unknown {
if (!text.startsWith(`Used ${tool} for "`)) return undefined;
const output = /": (\{.*\})\s*$/su.exec(text)?.[1];
if (output === undefined) return undefined;
diff --git a/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts b/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts
new file mode 100644
index 000000000..1fd0601e9
--- /dev/null
+++ b/packages/eve/src/internal/testing/channel-conformance/tui-driver.ts
@@ -0,0 +1,150 @@
+import { Client } from "#client/client.js";
+import { EveTUIRunner } from "#cli/dev/tui/runner.js";
+import { TerminalRenderer } from "#cli/dev/tui/terminal-renderer.js";
+import { MockScreen, MockUserInput } from "#cli/dev/tui/test/mock-terminal.js";
+import type {
+ ClientDriver,
+ RenderedOption,
+} from "#internal/testing/channel-conformance/harness.js";
+
+/** Each drawer's footer: a question's, then a tool approval's. */
+const DRAWER_FOOTERS = [
+ "↑/↓ move · enter to select · esc to dismiss",
+ "y yes · n no · Ctrl-C cancel",
+] as const;
+const FREEFORM_ROW = "Type your own answer…";
+
+/**
+ * Drives the real `eve dev` TUI on a mock terminal. It reads only the screen
+ * and acts only through keys, as a person at the terminal would.
+ */
+export function tuiDriver(): ClientDriver {
+ return {
+ name: "tui",
+ capabilities: ["buttons", "text-replies"],
+ async open(host, wait) {
+ // Wide enough that no reply wraps, so each one stays on one screen row.
+ const screen = new MockScreen({ columns: 1000, rows: 60 });
+ const keyboard = new MockUserInput();
+ const renderer = new TerminalRenderer({
+ captureForeignOutput: false,
+ input: keyboard,
+ output: screen,
+ unicode: true,
+ });
+ const run = new EveTUIRunner({ client: new Client({ host }), name: "tui", renderer }).run();
+ const describe = () => `Screen:\n${screen.snapshot()}`;
+ try {
+ // Surfaces a runner that fails during startup instead of waiting out the timeout.
+ await Promise.race([
+ wait("the composer", () => (composerOpen(screen) ? true : undefined), describe),
+ run.then(() => {
+ throw new Error(`The TUI exited before its composer opened. ${describe()}`);
+ }),
+ ]);
+ } catch (error) {
+ // The caller only gets `close()` once `open()` resolves, so stop the runner here.
+ renderer.requestInterrupt();
+ await run.catch(() => {});
+ throw error;
+ }
+
+ return {
+ async say(text) {
+ // A question drawer holds the keyboard until Esc hands it back to the composer.
+ if (drawerOpen(screen)) keyboard.send("\x1b");
+ await wait("the composer", () => (composerOpen(screen) ? true : undefined), describe);
+ keyboard.type(text);
+ keyboard.enter();
+ },
+ async waitForQuestion(prompt) {
+ await wait(`the question "${prompt}"`, () => focusedRow(screen, prompt), describe);
+ return readOptions(screen, keyboard, prompt);
+ },
+ async press(option) {
+ // `waitForQuestion` leaves the cursor on the first option.
+ for (let row = 0; row < (option.handle as number); row += 1) keyboard.down();
+ keyboard.enter();
+ },
+ replies: () => readReplies(screen.snapshot()),
+ describe,
+ async close() {
+ renderer.requestInterrupt();
+ await run;
+ },
+ };
+ },
+ };
+}
+
+/** Each assistant reply: a `▲ ` row and the indented rows its text continues on. */
+function readReplies(snapshot: string): string[] {
+ const replies: string[] = [];
+ let reply: string[] | undefined;
+ for (const line of snapshot.split("\n")) {
+ if (line.startsWith("▲ ")) {
+ reply = [line.slice(2)];
+ replies.push("");
+ } else if (reply !== undefined && line.startsWith(" ")) {
+ reply.push(line.slice(2));
+ } else {
+ reply = undefined;
+ }
+ if (reply !== undefined) replies[replies.length - 1] = reply.join("\n");
+ }
+ return replies;
+}
+
+function drawerOpen(screen: MockScreen): boolean {
+ const snapshot = screen.snapshot();
+ return DRAWER_FOOTERS.some((footer) => snapshot.includes(footer));
+}
+
+function composerOpen(screen: MockScreen): boolean {
+ return !drawerOpen(screen) && /^❯/mu.test(screen.snapshot());
+}
+
+/**
+ * A drawer draws only the row under its cursor at full weight; every other
+ * option, each description, and the freeform row are dim. A person reads the
+ * options by moving the cursor through them, and so does the driver, until the
+ * cursor wraps to the first option or stops at the last. The TUI repaints
+ * synchronously on each key, so the screen is current right after one.
+ */
+function readOptions(screen: MockScreen, keyboard: MockUserInput, prompt: string) {
+ const options: RenderedOption[] = [];
+ const first = focusedRow(screen, prompt);
+ let row = first;
+ while (row !== undefined) {
+ if (!row.freeform) options.push({ handle: options.length, label: row.text });
+ keyboard.down();
+ const next = focusedRow(screen, prompt);
+ if (next === undefined || next.line === row.line || next.line === first?.line) break;
+ row = next;
+ }
+ // Leave the cursor on the first option, as a fresh drawer has it.
+ for (let moves = 0; moves < 10 && focusedRow(screen, prompt)?.line !== first?.line; moves++) {
+ keyboard.up();
+ }
+ return options;
+}
+
+/** The full-weight row between `prompt` and its drawer's footer: the one under the cursor. */
+function focusedRow(
+ screen: MockScreen,
+ prompt: string,
+): { readonly freeform: boolean; readonly line: number; readonly text: string } | undefined {
+ const lines = screen.snapshot().split("\n");
+ const bright = screen.snapshot({ hideDim: true }).split("\n");
+ const footer = lines.findLastIndex((line) =>
+ DRAWER_FOOTERS.some((candidate) => line.includes(candidate)),
+ );
+ const start = bright.slice(0, footer).findLastIndex((line) => line.trim() === prompt);
+ if (footer === -1 || start === -1) return undefined;
+ for (let line = start + 1; line < footer; line += 1) {
+ const text = bright[line]!.trim();
+ if (text.length === 0) continue;
+ return { freeform: lines[line]!.includes(FREEFORM_ROW), line, text };
+ }
+ return undefined;
+}
diff --git a/packages/eve/test/tui-client/tui-questions.ts b/packages/eve/test/tui-client/tui-questions.ts
deleted file mode 100644
index 0815cef0e..000000000
--- a/packages/eve/test/tui-client/tui-questions.ts
+++ /dev/null
@@ -1,127 +0,0 @@
-import { setTimeout as sleep } from "node:timers/promises";
-
-import { Client } from "eve/client";
-import { EveTUIRunner, MockScreen, MockUserInput } from "./lib/tui.ts";
-
-import { run } from "./lib/run.ts";
-import { theme } from "./lib/theme.ts";
-
-/**
- * End-to-end proof that the TUI handles `input.requested` events with
- * `display: "select"` correctly. The legacy translator collapsed every
- * input request into a `tool-approval-request` chunk, so multi-option
- * `ask_question` calls were unanswerable through the TUI. This smoke
- * test drives the new `readInputQuestion` path:
- *
- * 1. Start the apps/fixtures/agent-tui-client server.
- * 2. Boot an `EveTUIRunner` with a mock terminal.
- * 3. Type a prompt that asks the model to call `ask_question` with
- * two options (Red/Blue).
- * 4. Wait for the question drawer's control row, which proves it is ready
- * for input.
- * 5. Send Down arrow + Enter to pick the second option (blue).
- * 6. Wait for the answered marker (`⎿ <label>`) in the transcript.
- * 7. Wait for the post-answer assistant turn to render. The runner
- * only returns to its prompt-reading state if the chosen option
- * flowed through to the agent and resolved the pending input.
- */
-
-process.env.EVE_TUI_UNICODE = "1";
-
-run({ app: "agent-tui-client", kind: "local-build" }, async (target) => {
- const client = new Client({ host: target.baseUrl });
- const screen = new MockScreen({ columns: 100, rows: 40 });
- const input = new MockUserInput();
- const runner = new EveTUIRunner({
- client,
- screen,
- userInput: input,
- name: "TUI smoke",
- });
-
- const runPromise = runner.run().catch((error: unknown) => {
- if (error instanceof Error && error.message === "Interrupted") {
- return;
- }
- throw error;
- });
-
- await screen.waitForIdlePrompt(5_000);
-
- const promptLines = [
- "Use the ask_question tool exactly once to ask me which color I prefer.",
- "Set question to: 'Pick a color.'",
- 'Provide exactly two options: label "Red" and label "Blue".',
- "Wait for my response.",
- ];
- input.type(promptLines.join(" · "));
- input.enter();
-
- await screen.waitForText("↑/↓ move · enter to select · esc to dismiss", 60_000);
- console.log(theme.muted("[tui-questions] select drawer is live"));
-
- // Bridges a server-side race where the park hook isn't yet
- // registered when `session.waiting` reaches the client. The TUI
- // is human-paced in practice; scripted clients need a short
- // handshake delay before delivering the response.
- await sleep(1_000);
-
- input.down();
- input.enter();
- console.log(theme.muted("[tui-questions] selected Blue"));
-
- await screen.waitForText("⎿ Blue", 5_000);
- console.log(theme.muted("[tui-questions] answer recorded in body"));
-
- // The post-answer turn must produce a `▲`-prefixed assistant
- // section whose body contains "blue", proof that the chosen
- // optionId flowed through to the agent and the model produced a
- // reply reflecting the choice. We only search the region after the
- // "⎿ Blue" answer marker to avoid a false positive from the marker
- // itself (transcript blocks commit in order, so the follow-up reply
- // always renders below the answered question).
- await waitForCondition(() => assistantReplyAfter(screen.snapshot(), "⎿ Blue", "blue"), {
- timeoutMs: 60_000,
- label: "follow-up assistant section mentioning blue",
- });
- console.log(theme.muted("[tui-questions] follow-up assistant turn rendered"));
-
- const finalSnapshot = screen.snapshot();
- if (finalSnapshot.includes("Error")) {
- throw new Error(`Final screen contains an Error section:\n${finalSnapshot}`);
- }
-
- // The turn is complete; wait until the runner is back at the prompt so
- // Two idle Ctrl+C presses exit; mid-stream Ctrl+C cancels cooperatively.
- await screen.waitForIdlePrompt(30_000);
- input.ctrlC();
- input.ctrlC();
- await runPromise;
-});
-
-/**
- * True when an assistant section (a `▲`-prefixed line and its wrapped
- * continuations) below `afterMarker` contains `needle`. Used to assert a
- * follow-up reply without matching transcript content above the marker.
- */
-function assistantReplyAfter(snapshot: string, afterMarker: string, needle: string): boolean {
- const markerIndex = snapshot.indexOf(afterMarker);
- if (markerIndex === -1) return false;
- const region = snapshot.slice(markerIndex + afterMarker.length);
- const brandIndex = region.indexOf("▲ ");
- if (brandIndex === -1) return false;
- return region.slice(brandIndex).toLowerCase().includes(needle.toLowerCase());
-}
-
-async function waitForCondition(
- predicate: () => boolean,
- options: { timeoutMs: number; label: string; intervalMs?: number },
-): Promise<void> {
- const intervalMs = options.intervalMs ?? 100;
- const deadline = Date.now() + options.timeoutMs;
- while (Date.now() < deadline) {
- if (predicate()) return;
- await sleep(intervalMs);
- }
- throw new Error(`Timed out waiting for: ${options.label}`);
-}