Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World
import { appendFileSync } from 'node:fs';
import { InMemorySpanExporter, type ReadableSpan, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base';
import { type AgentRuns, agentRuns } from 'durable-harness/instrumentation/otel';
import { durableSessions, type Harness, QUEUE_PREFIX, type SessionState } from 'harness-world';
import type { World } from '@workflow/world';
import { expect } from 'vitest';
import { settled, until } from './scenario.ts';
import { type OpenWorld, storageFor } from './worlds.ts';
/**
* Vercel Agent Runs checks: durable-harness's `agentRuns()` (on
* `@vercel/agent-runs`) with an in-memory span processor, and what one tool
* turn of a harness must give it.
*
* Sessions use an unknown audience. Each scripted harness runs once with
* content enabled and once with the default metadata-only policy.
*/
export interface Tracing {
readonly content: boolean;
integration: AgentRuns;
exporter: InMemorySpanExporter;
/** The finished spans of one session (every span carries `agent.run.id`). */
spansOf(sessionId: string): ReadableSpan[];
/** Wait until the session has a finished root for `harness` (the drive that ran its turn returned). */
finishedTurn(sessionId: string, harness: string, timeoutMs?: number): Promise<ReadableSpan[]>;
/** Unregister (the integration installs itself on `globalThis`). */
close(): Promise<void>;
}
export function tracing(content = true): Tracing {
const exporter = new InMemorySpanExporter();
const integration = agentRuns({ ...(content ? { capture: () => ({ emit: true, recordInputs: true, recordOutputs: true }) } : {}), spanProcessors: [new SimpleSpanProcessor(exporter)] });
integration.register();
const spansOf = (sessionId: string) => exporter.getFinishedSpans().filter((s) => s.attributes['agent.run.id'] === sessionId);
return {
content,
integration,
exporter,
spansOf,
async finishedTurn(sessionId, harness, timeoutMs = 60_000) {
await until(() => Promise.resolve(spansOf(sessionId).some((s) => s.name === `invoke_agent ${harness}`)), timeoutMs, `the ${harness} turn's root span`);
await integration.forceFlush();
const spans = spansOf(sessionId);
for (const root of spans.filter(span => span.name.startsWith('invoke_agent'))) {
expect(root.attributes['agent.trace.content.input']).toBe(content);
expect(root.attributes['agent.trace.content.output']).toBe(content);
}
return spans;
},
async close() {
await integration.forceFlush();
delete (globalThis as Record<symbol, unknown>)[Symbol.for('durable-harness.agent-runs')];
},
};
}
export async function expectNoCheckpointContent(world: World, runId: string, marker: string): Promise<void> {
const names = (await world.streams.list(runId)).filter(name => name.includes('_agent-runs_'));
expect(names.length, 'durable trace checkpoints exist').toBeGreaterThan(0);
for (const name of names) {
let cursor: string | undefined;
let text = '';
const decoder = new TextDecoder();
for (;;) {
const page = await world.streams.getChunks(runId, name, { ...(cursor ? { cursor } : {}) });
for (const chunk of page.data) text += decoder.decode(chunk.data, { stream: true });
if (!page.hasMore || !page.cursor) break;
cursor = page.cursor;
}
expect(text).not.toContain(marker);
}
}
export interface TurnExpect {
sessionId: string;
/** The root's agent name: the harness's name (`invoke_agent <harness>`). */
harness: string;
/** A string in the prompt and in the tool call's arguments. */
marker: string;
/** The tool the turn calls. Undefined: any tool, as long as its arguments carry the marker. */
tool?: string;
/** The audience the session recorded. Default `unknown`. */
audience?: 'public' | 'private' | 'unknown';
/** The model's prompt reaches its span (a model that speaks a provider's format). */
modelContent?: boolean;
/** The model id every model span names (`chat <model>`). */
model?: string | RegExp;
/** Model spans carry token usage on their step. */
usage?: boolean;
}
/** What one tool turn must give Agent Runs: the span tree and its content. */
export function expectToolTurn(spans: ReadableSpan[], o: TurnExpect): void {
// `AGENT_RUNS_DUMP=<file>`: append the span tree (names and ids, no content) for a look by eye.
if (process.env.AGENT_RUNS_DUMP) appendFileSync(process.env.AGENT_RUNS_DUMP, `${o.harness} ${o.sessionId}\n${outline(spans)}\n`);
const byId = new Map(spans.map((s) => [s.spanContext().spanId, s]));
const parentOf = (s: ReadableSpan) => (s.parentSpanContext ? byId.get(s.parentSpanContext.spanId) : undefined);
const roots = spans.filter((s) => s.name === `invoke_agent ${o.harness}`);
expect(roots.length, 'an invoke_agent root per turn').toBeGreaterThanOrEqual(1);
const root = roots[0]!;
expect(root.parentSpanContext, 'the root has no parent').toBeUndefined();
expect(root.instrumentationScope.name).toBe('eve.agent');
expect(root.attributes).toMatchObject({
'agent.framework.name': 'durable-harness',
'agent.name': o.harness,
'agent.run.id': o.sessionId,
'gen_ai.operation.name': 'invoke_agent',
'agent.trace.schema.version': 4,
'agent.channel.audience': o.audience ?? 'unknown',
'agent.turn.outcome': 'completed',
});
const content = root.attributes['agent.trace.content.input'] === true;
if (content) expect(String(root.attributes['agent.session.title'])).toContain(o.marker);
else {
expect(root.attributes['agent.session.title']).toBeUndefined();
expect(JSON.stringify(spans.map(span => ({ attributes: span.attributes, events: span.events, status: span.status })))).not.toContain(o.marker);
}
expect(new Set(spans.map((s) => s.spanContext().traceId)), 'one trace per turn').toEqual(new Set([root.spanContext().traceId]));
// Steps hang off a root; model and tool spans off a step.
const steps = spans.filter((s) => s.name === 'agent.step');
expect(steps.length).toBeGreaterThanOrEqual(2);
for (const step of steps) expect(parentOf(step)?.name).toBe(`invoke_agent ${o.harness}`);
const ofRoot = (s: ReadableSpan) => parentOf(parentOf(s)!)?.spanContext().spanId === root.spanContext().spanId;
const models = spans.filter((s) => s.attributes['gen_ai.operation.name'] === 'chat');
expect(models.length, 'a model span').toBeGreaterThanOrEqual(1);
for (const m of models) expect(parentOf(m)?.name).toBe('agent.step');
expect(models.some(ofRoot), "a model span in the tool turn's trace").toBe(true);
if (content) expect(models[0]!.attributes['gen_ai.output.messages']).toBeDefined();
if (o.modelContent && content) expect(models.map((m) => String(m.attributes['gen_ai.input.messages'])).join(''), "the prompt in a model span's input").toContain(o.marker);
if (o.model) for (const m of models) expect(m.name).toMatch(typeof o.model === 'string' ? new RegExp(`^chat ${o.model.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`) : o.model);
if (o.usage) {
const used = models.map((m) => parentOf(m)!).filter((step) => typeof step.attributes['agent.usage.input_tokens'] === 'number');
expect(used.length, 'token usage on a model step').toBeGreaterThanOrEqual(1);
}
const tools = spans.filter((s) => s.attributes['gen_ai.operation.name'] === 'execute_tool' && (o.tool === undefined || s.name === `execute_tool ${o.tool}`));
const calls = content ? tools.filter((s) => String(s.attributes['gen_ai.tool.call.arguments']).includes(o.marker)) : tools;
expect(calls.length, `an execute_tool ${o.tool ?? '*'} span with the marker in its arguments (tools: ${tools.map((s) => s.name).join(', ') || 'none'})`).toBeGreaterThanOrEqual(1);
if (o.tool) expect(tools).toHaveLength(1);
const call = calls[0]!;
expect(parentOf(call)?.name).toBe('agent.step');
expect(ofRoot(call)).toBe(true);
expect(call.attributes['gen_ai.tool.name']).toBe(call.name.slice('execute_tool '.length));
if (content) expect(call.attributes['gen_ai.tool.call.result']).toBeDefined();
// Where the model's reply names its tool calls, the tool span carries that
// call's id: Agent Runs joins the two into one tool call by it.
const asked = models.flatMap((m) => {
try {
return (JSON.parse(String(m.attributes['gen_ai.output.messages'])) as { parts?: { type?: string; id?: unknown }[] }[])
.flatMap((msg) => msg.parts ?? []).filter((p) => p.type === 'tool_call').map((p) => p.id);
} catch {
return [];
}
});
if (process.env.AGENT_RUNS_DUMP) appendFileSync(process.env.AGENT_RUNS_DUMP, `${o.harness} asked ${JSON.stringify(asked)} ran ${String(call.attributes['gen_ai.tool.call.id'])}\n`);
if (asked.length) expect(asked, "the tool span's call id is the model's").toContain(call.attributes['gen_ai.tool.call.id']);
expect(
spans.filter((s) => s.status.code === 2).map((s) => `${s.name}: ${s.status.message}`),
'no span failed'
).toEqual([]);
}
/**
* A kernel session on a fresh world-local data directory: its first prompt
* in (with `traceAudience: 'unknown'`), the queue served in this process,
* settled once `done`. Returns its spans once its turn's root has ended.
*/
export async function kernelTurn(
t: Tracing,
disposers: (() => Promise<void> | void)[],
name: string,
harness: Harness,
prompt: string,
done: (s: SessionState) => boolean,
timeoutMs = 90_000
): Promise<{ sessionId: string; state: SessionState; spans: ReadableSpan[]; world: World }> {
const open: OpenWorld = await storageFor('local', `agent-runs-${name}`).open();
disposers.push(() => open.close());
const sessions = durableSessions({ world: open.world, harness });
await open.serve(QUEUE_PREFIX, sessions.handler);
const { sessionId } = await sessions.start({ prompt, config: { traceAudience: 'unknown' } });
const state = await settled(open.world, sessions, sessionId, done, { what: `the ${name} tool turn`, timeoutMs });
const spans = await t.finishedTurn(sessionId, harness.name);
return { sessionId, state, spans, world: open.world };
}
/** The span tree, one line per span, indented under its parent: names and step ids, no content. */
export function outline(spans: readonly ReadableSpan[]): string {
const children = new Map<string | undefined, ReadableSpan[]>();
for (const s of spans) children.set(s.parentSpanContext?.spanId, [...(children.get(s.parentSpanContext?.spanId) ?? []), s]);
const ids = new Set(spans.map((s) => s.spanContext().spanId));
const lines: string[] = [];
const walk = (s: ReadableSpan, depth: number) => {
const usage = s.attributes['agent.usage.input_tokens'] !== undefined ? ` usage ${s.attributes['agent.usage.input_tokens']}/${s.attributes['agent.usage.output_tokens']}` : '';
lines.push(`${' '.repeat(depth)}${s.name}${usage}${s.status.code === 2 ? ' ERROR' : ''}`);
for (const c of (children.get(s.spanContext().spanId) ?? []).sort((a, b) => a.startTime[0] - b.startTime[0] || a.startTime[1] - b.startTime[1])) walk(c, depth + 1);
};
for (const s of spans.filter((x) => !x.parentSpanContext || !ids.has(x.parentSpanContext.spanId))) walk(s, 1);
return lines.join('\n');
}