Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World
import { type CliCall, listOf, ndjsonOf, type Target, workflow, jsonOf } from './cli.ts';
/**
* What a user should be able to rely on when they point the stock Workflow
* CLI at a harness session. `collect` runs the CLI; `evaluate` turns what it
* printed into findings, without throwing, so one report holds every gap.
*/
export type InvariantId =
| 'listed'
| 'status'
| 'steps'
| 'payloads'
| 'events'
| 'hooks'
| 'streams-listed'
| 'streams-read'
| 'cli-errors'
| 'agree';
export const INVARIANTS: readonly { id: InvariantId; title: string }[] = [
{ id: 'listed', title: 'listed under its full workflow name' },
{ id: 'status', title: 'run status, timestamps and input' },
{ id: 'steps', title: 'step names, order and statuses' },
{ id: 'payloads', title: 'step and hook payloads decode to plain values' },
{ id: 'events', title: 'events in order, one lifecycle per step, one hook_received per submission' },
{ id: 'hooks', title: 'the inbox hook while open, none once closed' },
{ id: 'streams-listed', title: 'streams listed, matching the World' },
{ id: 'streams-read', title: 'every stream decodes, and ends when it should' },
{ id: 'cli-errors', title: 'no CLI errors' },
{ id: 'agree', title: 'the CLI agrees with the World and the kernel' },
];
export type Phase = 'open' | 'closed';
export interface Finding {
invariant: InvariantId;
/** A short machine-stable code for the kind of gap (`stream-decode`, `string-payload`, ...). */
code: string;
message: string;
}
export interface SessionExpect {
/** The run's workflow name: `workflow//durable-harness@<version>//<name>`. */
workflowName: string;
/** What the run input's `harness` field says. */
inputHarness: string;
phase: Phase;
/** Inbox submissions (`hook_received` events). Undefined: at least one, count not checked. */
submissions?: number;
/** The inbox hook's token while open. */
hookToken: string;
/** Model and tool calls, in order, by effect name (`model`, `tool.write_file`). */
calls: string[];
/** `exact` for kernel harnesses; `subsequence` where bookkeeping steps (journal, place, fs) interleave. */
callsMatch: 'exact' | 'subsequence';
/** Which effect names count as calls. Default: `model` and `tool.*`. */
callPattern?: RegExp;
/** Every effect name the harness writes. */
vocabulary: RegExp;
/** Error messages a failed step may carry (the recovered session's interrupted tool). */
failedSteps?: RegExp;
/** `submission`: kernel inbox payloads `{kind, text?, nonce}`. `any-object`: any object. */
hookPayload: 'submission' | 'any-object';
/** Values that are opaque by design (fx's checkpoint), by step name and path. */
opaqueOk?: (stepName: string, path: string) => boolean;
}
/** What the World API and the kernel's fold say, read in-process by the scenario (the CLI's ground truth). */
export interface Truth {
/** The World's steps. Undefined where only the kernel's view is at hand (world-vercel, read through a deployment). */
steps?: { stepId: string; stepName: string; status: string }[];
eventCount: number;
/** The run's streams. Undefined where they cannot be listed from here (world-vercel, read through a deployment). */
streams?: string[];
/** How many chunks each stream holds (`getInfo`), where known. */
streamChunks?: Record<string, number>;
/** Streams that hold chunks but that the World does not list. */
unlistedStreams?: string[];
/** The kernel's effects (`SessionState.effectOrder`), with the steps it ignores. Undefined for pi-world (no kernel fold). */
kernel?: { effects: { stepId: string; name: string; status: string }[]; ignored: string[] };
}
export interface StreamRead {
streamId: string;
/** It must end on its own: the session is closed, or it is the stream of a step that settled. */
mustEnd: boolean;
call: CliCall;
chunks: unknown[];
decodeError?: string;
}
export interface Collected {
runId: string;
phase: Phase;
runs?: CliCall;
run?: CliCall;
steps?: CliCall;
stepDetails: CliCall[];
events?: CliCall;
hooks?: CliCall;
streams?: CliCall;
streamReads: StreamRead[];
/** Calls that failed to produce JSON, by command. */
broken: { command: string; error: string }[];
}
const STEP_PREFIX = 'step//harness-world//';
const shortName = (stepName: string) => (stepName.startsWith(STEP_PREFIX) ? stepName.slice(STEP_PREFIX.length) : stepName);
/** Run with at most `n` at a time. */
async function pool<T, R>(items: T[], n: number, fn: (item: T) => Promise<R>): Promise<R[]> {
const out: R[] = new Array(items.length);
let next = 0;
await Promise.all(
Array.from({ length: Math.min(n, items.length) }, async () => {
while (next < items.length) {
const i = next++;
out[i] = await fn(items[i]!);
}
})
);
return out;
}
/**
* How long a stream read may take to end (or to print its first value). Only
* a stream that does not end waits this long; a closed one ends as soon as the
* CLI has read it, which on a busy CI runner can take well over 15s.
*/
export const STREAM_TIMEOUT_MS = 45_000;
export interface CollectOptions {
workflowName: string;
/** How long a stream read may take: to end, for one that must; to print its first value, for one that stays open. */
streamTimeoutMs?: number;
/** Chunks per stream, from the World: an open stream with none is not read (nothing to decode, and it would not end). */
streamChunks?: Record<string, number>;
/** Stream reads, at most (the steps' streams are dropped first). */
maxStreams?: number;
}
/** Run every `workflow inspect` view a user would open for one session. */
export async function collect(target: Target, runId: string, phase: Phase, options: CollectOptions): Promise<Collected> {
const out: Collected = { runId, phase, stepDetails: [], streamReads: [], broken: [] };
const [runs, run, steps, events, hooks, streams] = await Promise.all([
workflow(target, ['inspect', 'runs', '-n', options.workflowName, '--limit', '100', '--sort', 'desc']),
workflow(target, ['inspect', 'run', runId]),
// --all (vercel/workflow#4760) follows the cursors: every step and event, however long the session.
workflow(target, ['inspect', 'steps', '-r', runId, '-d', '--sort', 'asc', '--limit', '100', '--all']),
workflow(target, ['inspect', 'events', '-r', runId, '-d', '--sort', 'asc', '--limit', '100', '--all']),
workflow(target, ['inspect', 'hooks', '-r', runId]),
workflow(target, ['inspect', 'streams', '-r', runId]),
]);
Object.assign(out, { runs, run, steps, events, hooks, streams });
// One single-step view per step name: the view a user opens from the list.
let stepList: Record<string, any>[] = [];
try {
stepList = listOf(steps);
} catch (err) {
out.broken.push({ command: 'steps', error: String(err) });
}
const firstOfName = new Map<string, string>();
for (const s of stepList) if (!firstOfName.has(s.stepName)) firstOfName.set(s.stepName, s.stepId);
// `-r` is required: a step id names a step within its run (cli-quirks.test.ts, Q6).
out.stepDetails = await pool([...firstOfName.values()], 6, (id) => workflow(target, ['inspect', 'step', id, '-r', runId]));
let streamIds: string[] = [];
try {
streamIds = listOf<{ streamId: string }>(streams).map((s) => s.streamId);
} catch (err) {
out.broken.push({ command: 'streams', error: String(err) });
}
const max = options.maxStreams ?? 16;
const chosen = streamIds.length > max ? [...streamIds.filter((id) => !id.includes('_step_')), ...streamIds.filter((id) => id.includes('_step_'))].slice(0, max) : streamIds;
const settledSteps = new Set(stepList.filter((s) => s.status === 'completed' || s.status === 'failed').map((s) => `${runId}_${s.stepId}`));
const timeoutMs = options.streamTimeoutMs ?? STREAM_TIMEOUT_MS;
const plans = chosen
.map((streamId) => ({ streamId, mustEnd: phase === 'closed' || settledSteps.has(streamId) }))
.filter((p) => p.mustEnd || (options.streamChunks?.[p.streamId] ?? 1) > 0);
const failedRead = (stderr: string) => /Failed to read stream/.test(stderr);
out.streamReads = await pool(plans, 6, async ({ streamId, mustEnd }) => {
const call = await workflow(target, ['inspect', 'stream', streamId, '-r', runId], {
timeoutMs,
// A stream that stays open never ends: its first decoded value (or an error) is all there is to see.
// One that must end is read to its end: a stream the CLI cannot decode exits non-zero on its own (#4760).
enough: mustEnd ? undefined : (stdout, err) => stdout.includes('\n') || failedRead(err),
});
const read: StreamRead = { streamId, mustEnd, call, chunks: [] };
try {
read.chunks = ndjsonOf(call);
} catch (err) {
read.decodeError = `stdout is not JSON lines: ${String(err)}`;
}
const failed = call.stderr.match(/\{"error":"Failed to read stream[^\n]*/);
if (failed) {
try {
read.decodeError = (JSON.parse(failed[0]) as { details?: string }).details ?? failed[0];
} catch {
read.decodeError = failed[0];
}
} else if (failedRead(call.stderr)) read.decodeError = (call.stderr.match(/Failed to read stream[^\n]*/) ?? ['Failed to read stream'])[0];
return read;
});
return out;
}
/** An out-of-band document edit: a `hook_received` on the docs hook (its token, or its `{ $docs: 1, id, docs }` payload). */
function isDocsEdit(e: { eventData?: { token?: unknown; payload?: unknown } }): boolean {
const token = e.eventData?.token;
if (typeof token === 'string') return token.endsWith(':docs');
const payload = e.eventData?.payload as { $docs?: unknown } | null | undefined;
return !!payload && typeof payload === 'object' && payload.$docs === 1;
}
/** A value a user cannot read: bytes, a raw `devl` payload, a long base64 blob, or JSON text where an object belongs. */
export function opaqueKind(value: unknown, topLevel: boolean): string | undefined {
if (value === null || value === undefined) return undefined;
if (typeof value === 'string') {
if (value.startsWith('devl')) return 'devalue-text';
if (value.length >= 256 && /^[A-Za-z0-9+/=_-]+$/.test(value)) return 'base64';
if (topLevel && /^\s*[[{]/.test(value)) {
try {
const parsed = JSON.parse(value) as unknown;
if (parsed && typeof parsed === 'object') return 'json-text';
} catch {}
}
return undefined;
}
if (typeof value === 'object') {
const v = value as Record<string, unknown>;
if (v.type === 'Buffer' && Array.isArray(v.data)) return 'bytes';
const keys = Object.keys(v);
if (keys.length >= 8 && keys.every((k) => /^\d+$/.test(k)) && Object.values(v).every((x) => typeof x === 'number')) return 'bytes';
}
return undefined;
}
/** Every opaque value under `value`, with its path. JSON text is only flagged at the top level (`input`, `output`, `payload`). */
function opaqueValues(value: unknown, path: string, depth = 0, out: { path: string; kind: string }[] = []): { path: string; kind: string }[] {
const kind = opaqueKind(value, depth === 0);
if (kind) {
out.push({ path, kind });
return out;
}
if (depth > 8 || !value || typeof value !== 'object') return out;
for (const [k, v] of Object.entries(value as Record<string, unknown>)) opaqueValues(v, `${path}.${k}`, depth + 1, out);
return out;
}
const time = (v: unknown) => (typeof v === 'string' ? Date.parse(v) : Number.NaN);
const SKEW_MS = 1000;
/** Longer than any scripted session waits between creating a run or step and starting it. */
const GAP_MS = 10 * 60_000;
function inOrder(label: string, pairs: [string, unknown][], findings: Finding[]) {
for (let i = 1; i < pairs.length; i++) {
const [aName, a] = pairs[i - 1]!;
const [bName, b] = pairs[i]!;
const ta = time(a);
const tb = time(b);
if (Number.isNaN(ta) || Number.isNaN(tb)) continue;
if (ta > tb + SKEW_MS) {
findings.push({
invariant: 'events',
code: 'timestamps',
message: `${label}: ${aName} ${a} is ${Math.round((ta - tb) / 1000)}s after ${bName} ${b}`,
});
} else if (aName === 'createdAt' && tb - ta > GAP_MS) {
// A scripted session starts what it creates at once: hours between the two is a clock, not a queue.
findings.push({
invariant: 'events',
code: 'timestamps',
message: `${label}: ${aName} ${a} is ${((tb - ta) / 3_600_000).toFixed(1)}h before ${bName} ${b}`,
});
}
}
}
function isSubsequence(needle: string[], hay: string[]): boolean {
let i = 0;
for (const h of hay) if (i < needle.length && h === needle[i]) i++;
return i === needle.length;
}
export interface Observed {
workflowName?: string;
status?: string;
steps: string[];
stepStatuses: Record<string, number>;
superseded: number;
events?: number;
hookTokens: string[];
streams: number;
streamsDecoded: number;
streamsFailed: number;
streamsOpen: number;
cliMs: number;
}
export interface Evaluation {
findings: Finding[];
observed: Observed;
}
/** Judge what the CLI printed against what the scenario expects and what the World and the kernel say. */
export function evaluate(c: Collected, expect: SessionExpect, truth: Truth): Evaluation {
const findings: Finding[] = [];
const add = (invariant: InvariantId, code: string, message: string) => findings.push({ invariant, code, message });
const callPattern = expect.callPattern ?? /^(model|tool\..+)$/;
const ignored = new Set(truth.kernel?.ignored ?? []);
const observed: Observed = {
steps: [],
stepStatuses: {},
superseded: 0,
hookTokens: [],
streams: 0,
streamsDecoded: 0,
streamsFailed: 0,
streamsOpen: 0,
cliMs: 0,
};
const all = [c.runs, c.run, c.steps, c.events, c.hooks, c.streams, ...c.stepDetails, ...c.streamReads.map((r) => r.call)].filter(Boolean) as CliCall[];
observed.cliMs = all.reduce((n, call) => n + call.ms, 0);
// 9. No CLI errors (stream reads are judged in 8).
for (const call of [c.runs, c.run, c.steps, c.events, c.hooks, c.streams, ...c.stepDetails]) {
if (!call) continue;
const cmd = `workflow ${call.argv.slice(0, 2).join(' ')}`;
if (call.timedOut) add('cli-errors', 'timeout', `${cmd} timed out after ${call.ms}ms`);
else if (call.code !== 0) add('cli-errors', 'exit-code', `${cmd} exited ${call.code}: ${call.errors.slice(0, 2).join(' | ')}`);
else if (call.errors.length) add('cli-errors', 'stderr', `${cmd} printed: ${call.errors.slice(0, 2).join(' | ')}`);
}
for (const b of c.broken) add('cli-errors', 'no-json', `${b.command}: ${b.error.slice(0, 300)}`);
const safeList = (call: CliCall | undefined): Record<string, any>[] | undefined => {
if (!call) return undefined;
try {
return listOf(call);
} catch {
return undefined;
}
};
const safeJson = (call: CliCall | undefined): any => {
if (!call) return undefined;
try {
return jsonOf(call);
} catch {
return undefined;
}
};
// 1. Listed under its workflow name.
const runs = safeList(c.runs);
if (!runs) add('listed', 'no-list', 'inspect runs -n printed no list');
else if (!runs.some((r) => r.runId === c.runId)) add('listed', 'missing', `inspect runs -n ${expect.workflowName} does not list ${c.runId} (${runs.length} runs listed)`);
// 2. Run status, timestamps, input.
const run = safeJson(c.run);
if (!run) add('status', 'no-run', 'inspect run printed no run');
else {
observed.workflowName = run.workflowName;
observed.status = run.status;
if (run.workflowName !== expect.workflowName) add('status', 'workflow-name', `workflowName is ${run.workflowName}, expected ${expect.workflowName}`);
const want = expect.phase === 'open' ? 'running' : 'completed';
if (run.status !== want) add('status', 'status', `status is ${run.status} while ${expect.phase}, expected ${want}${run.error ? ` (error: ${JSON.stringify(run.error).slice(0, 200)})` : ''}`);
if (expect.phase === 'open' && run.completedAt) add('status', 'completed-at', `an open session has completedAt ${run.completedAt}`);
if (expect.phase === 'closed' && !run.completedAt) add('status', 'completed-at', 'a closed session has no completedAt');
if (!run.input || typeof run.input !== 'object' || run.input.harness !== expect.inputHarness) {
add('status', 'input', `input does not decode to { harness: '${expect.inputHarness}' }: ${JSON.stringify(run.input)?.slice(0, 200)}`);
}
if (run.status === 'failed' && typeof run.error?.message !== 'string') add('status', 'error', `a failed run's error has no message: ${JSON.stringify(run.error)}`);
const created = time(run.createdAt);
const started = time(run.startedAt);
const completed = time(run.completedAt);
inOrder('run', [['createdAt', run.createdAt], ['startedAt', run.startedAt]], findings);
if (completed && started > completed + SKEW_MS) add('events', 'timestamps', `run: startedAt ${run.startedAt} is after completedAt ${run.completedAt}`);
}
// 3. Steps.
const steps = safeList(c.steps);
if (!steps) add('steps', 'no-list', 'inspect steps printed no list');
else {
// `--sort asc` asked for the oldest first. Judge the sequence in creation order either way.
const byCreation = [...steps].sort((a, b) => time(a.createdAt) - time(b.createdAt) || String(a.stepId).localeCompare(String(b.stepId)));
if (byCreation.some((s, i) => s.stepId !== steps[i]!.stepId)) {
add('steps', 'sort-order', `inspect steps --sort asc lists steps out of creation order: ${steps.map((s) => shortName(String(s.stepName))).join(', ')}`);
}
for (const s of byCreation) {
const name = shortName(String(s.stepName));
observed.stepStatuses[s.status] = (observed.stepStatuses[s.status] ?? 0) + 1;
if (!String(s.stepName).startsWith(STEP_PREFIX)) add('steps', 'step-name', `${s.stepId}: stepName ${s.stepName} is not step//harness-world//<effect>`);
else if (!expect.vocabulary.test(name)) add('steps', 'vocabulary', `${s.stepId}: effect ${name} is not one this harness writes`);
if (ignored.has(s.stepId)) {
observed.superseded++;
continue;
}
if (s.status === 'pending' || s.status === 'running') add('steps', 'unsettled', `${name} (${s.stepId}) is ${s.status} in a settled session`);
if (s.status === 'failed') {
const message = String(s.error?.message ?? JSON.stringify(s.error));
if (!expect.failedSteps?.test(message)) add('steps', 'failed', `${name} (${s.stepId}) failed: ${message.slice(0, 200)}`);
}
observed.steps.push(name);
}
const calls = observed.steps.filter((n) => callPattern.test(n));
const ok = expect.callsMatch === 'exact' ? JSON.stringify(calls) === JSON.stringify(expect.calls) : isSubsequence(expect.calls, calls);
if (!ok) add('steps', 'calls', `calls are [${calls.join(', ')}], expected ${expect.callsMatch === 'exact' ? '' : 'to contain '}[${expect.calls.join(', ')}]`);
}
// 4. Payloads decode (the list view with -d, and each single-step view).
const checkStep = (s: Record<string, any>, view: string) => {
const name = shortName(String(s.stepName));
if (!s.input || typeof s.input !== 'object' || typeof s.input.key !== 'string') {
add('payloads', 'envelope', `${view} ${name}: input is not the kernel's step envelope { key, replay, input }: ${JSON.stringify(s.input)?.slice(0, 160)}`);
}
for (const [field, value] of [['input.input', s.input?.input], ['output', s.output], ['error', s.error]] as const) {
for (const o of opaqueValues(value, field)) {
if (expect.opaqueOk?.(name, o.path)) continue;
add('payloads', o.kind, `${view} ${name}: ${o.path} is ${o.kind}`);
}
}
};
for (const s of steps ?? []) checkStep(s, 'steps');
for (const call of c.stepDetails) {
const s = safeJson(call);
if (s) checkStep(s, 'step');
}
// 5. Events.
const events = safeList(c.events);
if (!events) add('events', 'no-list', 'inspect events printed no list');
else {
observed.events = events.length;
for (let i = 1; i < events.length; i++) {
if (!(String(events[i]!.eventId) > String(events[i - 1]!.eventId))) add('events', 'order', `event ${events[i]!.eventId} follows ${events[i - 1]!.eventId}`);
}
const lifecycle = new Map<string, string[]>();
for (const e of events) {
if (String(e.eventType).startsWith('step_')) {
const list = lifecycle.get(e.correlationId) ?? [];
list.push(String(e.eventType));
lifecycle.set(e.correlationId, list);
}
}
for (const [stepId, types] of lifecycle) {
if (ignored.has(stepId)) continue;
const terminal = types.filter((t) => t === 'step_completed' || t === 'step_failed');
const shape = types.filter((t) => t !== 'step_retrying');
if (shape[0] !== 'step_created' || shape[1] !== 'step_started' || terminal.length !== 1 || !/step_(completed|failed)/.test(shape.at(-1) ?? '')) {
add('events', 'step-lifecycle', `${stepId}: ${types.join(' → ')}`);
}
}
// Submissions only: an out-of-band document edit (a `hook_received` on the
// session's docs hook, harness-world docs.ts) is not one.
const received = events.filter((e) => e.eventType === 'hook_received' && !isDocsEdit(e));
if (expect.submissions !== undefined && received.length !== expect.submissions) add('events', 'submissions', `${received.length} hook_received events, expected ${expect.submissions}`);
if (expect.submissions === undefined && received.length === 0) add('events', 'submissions', 'no hook_received events');
for (const e of received) {
const payload = e.eventData?.payload;
const kind = opaqueKind(payload, true);
if (kind) add('payloads', kind === 'json-text' ? 'string-payload' : kind, `hook_received ${e.eventId}: payload is ${kind}${typeof payload === 'string' ? ` (${payload.slice(0, 80)})` : ''}`);
else if (!payload || typeof payload !== 'object') add('payloads', 'hook-payload', `hook_received ${e.eventId}: payload is ${typeof payload}`);
else if (expect.hookPayload === 'submission' && (typeof payload.kind !== 'string' || typeof payload.nonce !== 'string')) {
add('payloads', 'hook-payload', `hook_received ${e.eventId}: payload is not a submission { kind, text, nonce }: ${JSON.stringify(payload).slice(0, 160)}`);
}
}
if (expect.phase === 'closed') {
const runEvents = events.filter((e) => String(e.eventType).startsWith('run_'));
if (runEvents.at(-1)?.eventType !== 'run_completed') add('events', 'run-completed', `a closed session's last run event is ${runEvents.at(-1)?.eventType}`);
}
for (const s of steps ?? []) {
if (ignored.has(s.stepId)) continue;
inOrder(`step ${shortName(String(s.stepName))} ${s.stepId}`, [['createdAt', s.createdAt], ['startedAt', s.startedAt], ['completedAt', s.completedAt]], findings);
}
}
// 6. Hooks.
const hooks = safeList(c.hooks);
if (!hooks) add('hooks', 'no-list', 'inspect hooks printed no list');
else {
observed.hookTokens = hooks.map((h) => String(h.token));
// A session that had an out-of-band document edit also has its docs hook.
const inboxes = hooks.filter((h) => !String(h.token).endsWith(':docs'));
if (expect.phase === 'open' && (inboxes.length !== 1 || inboxes[0]!.token !== expect.hookToken)) {
add('hooks', 'inbox', `open: hooks are [${observed.hookTokens.join(', ')}], expected [${expect.hookToken}]`);
}
if (expect.phase === 'closed' && hooks.length) add('hooks', 'disposed', `closed: hooks are still [${observed.hookTokens.join(', ')}]`);
}
// 7. Streams listed.
const streams = safeList(c.streams);
if (!streams) add('streams-listed', 'no-list', 'inspect streams printed no list');
else {
const ids = streams.map((s) => String(s.streamId));
observed.streams = ids.length;
if (truth.streams) {
const world = new Set(truth.streams);
const missing = truth.streams.filter((s) => !ids.includes(s));
const extra = ids.filter((s) => !world.has(s));
if (missing.length || extra.length) add('streams-listed', 'mismatch', `CLI and World disagree: missing [${missing.join(', ')}], extra [${extra.join(', ')}]`);
}
for (const name of truth.unlistedStreams ?? []) {
add('streams-listed', 'unlisted', `${name} holds ${truth.streamChunks?.[name] ?? 'some'} chunks, but neither the World nor the CLI lists it`);
}
const stepIds = new Set((steps ?? []).map((s) => String(s.stepId)));
for (const id of ids) {
const step = id.slice(c.runId.length + 1);
if (step.startsWith('step_') && !stepIds.has(step)) add('streams-listed', 'orphan', `${id} names no listed step`);
}
}
// 8. Streams read.
const kinds = new Map<string, number>();
for (const r of c.streamReads) {
const kind = r.streamId.slice(c.runId.length + 1).replace(/^step_.*/, 'step');
if (r.decodeError) {
observed.streamsFailed++;
kinds.set(`${kind}: ${r.decodeError}`, (kinds.get(`${kind}: ${r.decodeError}`) ?? 0) + 1);
continue;
}
observed.streamsDecoded++;
// A chunk may be any value the CLI can print legibly (fx's model steps stream the raw SSE text); bytes or encoded payloads are not.
const opaque = r.chunks.filter((x) => opaqueKind(x, false));
if (opaque.length) add('streams-read', 'chunk', `${r.streamId}: ${opaque.length} chunks are ${opaqueKind(opaque[0], false)} (${JSON.stringify(opaque[0]).slice(0, 80)})`);
if (r.call.timedOut) {
observed.streamsOpen++;
if (r.mustEnd) add('streams-read', 'not-closed', `${kind} stream ${r.streamId} did not end within ${r.call.ms}ms (${expect.phase === 'closed' ? 'the session is closed' : 'its step settled'})`);
else add('streams-read', 'no-output', `${kind} stream ${r.streamId} printed nothing within ${r.call.ms}ms, though the World holds ${truth.streamChunks?.[r.streamId] ?? 'some'} chunks`);
} else if (r.call.stoppedEarly) observed.streamsOpen++;
else if (r.call.code !== 0) add('streams-read', 'exit-code', `${r.streamId}: exited ${r.call.code}`);
}
if (kinds.size) {
const list = [...kinds].map(([what, n]) => (n > 1 ? `${what} (x${n})` : what)).join('; ');
add('streams-read', 'stream-decode', `${observed.streamsFailed} of ${c.streamReads.length} streams fail to decode: ${list}`);
}
// 10. The CLI agrees with the World and the kernel.
if (steps) {
const cli = new Map(steps.map((s) => [String(s.stepId), String(s.status)]));
if (truth.steps) {
const world = new Map(truth.steps.map((s) => [s.stepId, s.status]));
for (const [id, status] of world) {
if (!cli.has(id)) add('agree', 'step-missing', `the World has step ${id} (${status}); the CLI does not list it`);
else if (cli.get(id) !== status) add('agree', 'step-status', `step ${id}: CLI says ${cli.get(id)}, the World ${status}`);
}
for (const id of cli.keys()) if (!world.has(id)) add('agree', 'step-extra', `the CLI lists step ${id}; the World does not have it`);
}
for (const e of truth.kernel?.effects ?? []) {
if (e.status !== 'completed' && e.status !== 'failed') continue;
if (!cli.has(e.stepId)) add('agree', 'kernel-step', `the kernel's ${e.name} (${e.stepId}) is not in the CLI's steps`);
else if (cli.get(e.stepId) !== e.status) add('agree', 'kernel-status', `${e.name} (${e.stepId}): kernel ${e.status}, CLI ${cli.get(e.stepId)}`);
}
}
if (events && events.length !== truth.eventCount) add('agree', 'event-count', `the CLI lists ${events.length} events, the World has ${truth.eventCount}`);
return { findings: aggregate(findings), observed };
}
/** One finding per invariant and code: the first message, and how many more there were. */
function aggregate(findings: Finding[]): Finding[] {
const groups = new Map<string, Finding[]>();
for (const f of findings) {
const key = `${f.invariant}/${f.code}`;
groups.set(key, [...(groups.get(key) ?? []), f]);
}
return [...groups.values()].map((group) =>
group.length === 1 ? group[0]! : { ...group[0]!, message: `${group[0]!.message} (and ${group.length - 1} more like it)` }
);
}