Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World
import { mkdtemp } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { Event, World } from '@workflow/world';
import { createWorld as createLocalWorld } from '@workflow/world-local';
import { monotonicFactory } from 'ulid';
import { afterEach, describe, expect, it } from 'vitest';
import {
chatRoute,
defineDoc,
deliver,
type DriveContext,
type DriverOptions,
DocConflict,
DocTooLarge,
DocVersionError,
DOC_MAX_BYTES,
durableSessions,
EffectFailure,
type EffectIO,
type Harness,
hostToolIO,
liveStreamName,
rawSessions,
readDoc,
readJsonLines,
type Submission,
SessionLog,
vercelAnchors,
} from './index.ts';
import { encode } from './codec.ts';
import { hostProcess, leasedWorld } from './testing/index.ts';
/**
* Documents in the kernel (docs.ts), on world-local always and on
* world-postgres when `WORKFLOW_POSTGRES_URL` names a database with the World
* migrations applied: an edit commits with its effect's outcome (a crash, a
* throw, a refused stale writer leave none), replays fold to the same state,
* out-of-band edits race under optimistic concurrency, the size limit and
* schema versions hold, and the live `doc` event reaches a chat route.
*/
const ulid = monotonicFactory();
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
async function until<T>(fn: () => Promise<T | undefined> | T | undefined, timeoutMs = 10_000): Promise<T> {
const start = Date.now();
for (;;) {
const value = await fn();
if (value !== undefined) return value;
if (Date.now() - start > timeoutMs) throw new Error('timed out');
await sleep(20);
}
}
type Closable = World & { close?(): Promise<void> };
interface WorldCase {
name: string;
/** A World on a fresh store, and a way to open another instance on the same store (another process's). */
make(): Promise<Closable & { reopen(): Promise<Closable> }>;
}
const WORLDS: WorldCase[] = [
{
name: 'world-local',
make: async () => {
const dataDir = await mkdtemp(join(tmpdir(), 'hw-docs-'));
const open = () => createLocalWorld({ dataDir, recoverActiveRuns: false });
return Object.assign(open(), { reopen: async () => open() });
},
},
];
if (process.env.WORKFLOW_POSTGRES_URL) {
WORLDS.push({
name: 'world-postgres',
make: async () => {
const { createWorld } = await import('@workflow/world-postgres');
const open = () => createWorld({ connectionString: process.env.WORKFLOW_POSTGRES_URL!, jobPrefix: `hw_docs_${ulid().toLowerCase()}_` }) as unknown as Closable;
return Object.assign(open(), { reopen: async () => open() });
},
});
}
let opened: Array<{ close?(): Promise<void> }> = [];
afterEach(async () => {
for (const w of opened.splice(0)) await w.close?.().catch(() => {});
});
/** `base` with its queue kept in memory (no queue process in these tests: deliveries are explicit) and its event writes counted. */
function memQueue(base: World) {
const created: Array<{ runId: string; eventType: string; inBand?: boolean }> = [];
const events = new Proxy(base.events, {
get(target, key) {
const value = (target as unknown as Record<string | symbol, unknown>)[key];
if (key === 'create') {
return (runId: string, data: { eventType: string }, ...rest: unknown[]) => {
created.push({ runId, eventType: data.eventType, ...((rest[0] as { inBand?: boolean } | undefined)?.inBand ? { inBand: true } : {}) });
return (value as (...a: unknown[]) => unknown).call(target, runId, data, ...rest);
};
}
return typeof value === 'function' ? (value as (...a: unknown[]) => unknown).bind(target) : value;
},
});
const world = new Proxy(base, {
get(target, key) {
if (key === 'events') return events;
if (key === 'queue') return async () => ({ messageId: `msg_${ulid()}` });
const value = (target as unknown as Record<string | symbol, unknown>)[key];
return typeof value === 'function' ? (value as (...a: unknown[]) => unknown).bind(target) : value;
},
}) as World;
return { world, created };
}
async function allEvents(world: World, runId: string): Promise<Event[]> {
const out: Event[] = [];
let cursor: string | undefined;
for (;;) {
const page = await world.events.list({ runId, pagination: { sortOrder: 'asc', limit: 1000, ...(cursor ? { cursor } : {}) }, resolveData: 'none' });
out.push(...page.data);
if (!page.hasMore || !page.cursor) return out;
cursor = page.cursor;
}
}
const liveDocs = async (world: World, runId: string) =>
((await readJsonLines(world, runId, liveStreamName(runId))) as Array<{ type?: string }>).filter((e) => e?.type === 'doc') as Array<{ type: 'doc'; kind: string; revision: number; value: unknown; by: unknown }>;
/** One turn per prompt: a `model` effect that places it, then `turn` (tool calls). Answered turns replay from the log. */
function scripted(turn: (ctx: DriveContext, sub: Submission) => Promise<void>, name = 'docs'): Harness {
return {
name,
async drive(ctx) {
const pending = new Set(ctx.pending().map((p) => p.id));
for (const sub of ctx.state().submissions) {
if (sub.kind !== 'prompt') continue;
if (!pending.has(sub.id) && !ctx.state().placed.has(sub.id)) continue;
await ctx.effect(`model:${sub.id}`, { name: 'model', replay: 'safe', input: sub.text, placed: [sub.id] }, async () => 'ok');
await turn(ctx, sub);
}
ctx.live({ type: 'idle' });
},
};
}
/** A tool call: its result, or the committed failure. */
async function call<T>(ctx: DriveContext, key: string, fn: (io: EffectIO) => Promise<T>, replay: 'safe' | 'never' = 'safe', name = 'tool.edit'): Promise<T | EffectFailure> {
try {
return await ctx.effect(key, { name, replay, input: key }, fn);
} catch (err) {
if (err instanceof EffectFailure) return err;
throw err;
}
}
const Todos = defineDoc<{ items: string[] }>({ kind: 'app.todos', version: 1, initial: () => ({ items: [] }) });
const Board = defineDoc<{ cards: Record<string, string> }>({ kind: 'app.board', version: 1, initial: () => ({ cards: {} }) });
for (const wc of WORLDS) {
describe(`documents on ${wc.name}`, () => {
async function open() {
const base = await wc.make();
opened.push(base);
return { base, ...memQueue(base) };
}
const msg = (m: string) => ({ anchor: { k: 'msg' as const, m } });
it('a tool edit commits with its outcome: the fold, inspect, sessions.doc and the live stream agree', async () => {
const { world } = await open();
const harness = scripted(async (ctx, sub) => {
await call(ctx, `add:${sub.id}`, async (io) => {
const todos = io.docs!.update(Todos, (d) => void d.items.push(sub.text!));
// The call sees its own update; nothing is committed yet.
expect(io.docs!.read(Todos)).toEqual({ value: todos, revision: ctx.state().docs.get('app.todos')?.revision ?? 0 });
return `added; ${todos.items.length}`;
});
});
const sessions = durableSessions({ world, harness, marks: false });
const { sessionId } = await sessions.start();
await sessions.send(sessionId, 'one');
await deliver({ world, harness, marks: false }, sessionId, msg('m1'));
await sessions.send(sessionId, 'two');
await deliver({ world, harness, marks: false }, sessionId, msg('m2'));
const state = await sessions.inspect(sessionId);
const [one, two] = state.submissions;
expect(state.effects.get(`add:${one!.id}`)).toMatchObject({ status: 'completed', output: 'added; 1', docs: [{ kind: 'app.todos', revision: 1 }] });
expect(state.effects.get(`add:${two!.id}`)).toMatchObject({ status: 'completed', output: 'added; 2', docs: [{ kind: 'app.todos', revision: 2 }] });
expect(await sessions.doc(sessionId, Todos)).toEqual({ value: { items: ['one', 'two'] }, revision: 2 });
expect(readDoc(state, Todos).value.items).toEqual(['one', 'two']);
// Never written: initial at revision 0.
expect(await sessions.doc(sessionId, Board)).toEqual({ value: { cards: {} }, revision: 0 });
const live = await liveDocs(world, sessionId);
expect(live.map((e) => [e.kind, e.revision, e.value])).toEqual([
['app.todos', 1, { items: ['one'] }],
['app.todos', 2, { items: ['one', 'two'] }],
]);
expect(live[0]!.by).toEqual({ effect: `add:${one!.id}`, step: state.effects.get(`add:${one!.id}`)!.stepId });
});
for (const replay of ['never', 'safe'] as const) {
it(`a crash mid-tool leaves no edit (replay: '${replay}'): ${replay === 'never' ? 'the call is interrupted and the document unchanged' : 'the retry reads the document unchanged and its edit lands once'}`, async () => {
const { world } = await open();
const reads: number[] = [];
let attempts = 0;
const harness = (slowMs: number) =>
scripted(async (ctx, sub) => {
await call(
ctx,
`add:${sub.id}`,
async (io) => {
attempts++;
reads.push(io.docs!.read(Todos).revision);
io.docs!.update(Todos, (d) => void d.items.push(`attempt ${attempts}`));
await sleep(slowMs);
return 'added';
},
replay
);
});
const sessions = durableSessions({ world, harness: harness(10), marks: false });
const { sessionId } = await sessions.start();
await sessions.chaos(sessionId, { at: 'tool', afterMs: 150 });
await sessions.send(sessionId, 'go');
// Process A drives, and is killed mid-tool, after its update.
const a = hostProcess(world);
let crashed!: () => void;
const died = new Promise<void>((r) => (crashed = r));
void deliver({ world: a.world, harness: harness(60_000), marks: false, anchors: vercelAnchors(), chaos: true, crash: () => (a.kill(), crashed()) }, sessionId, msg('m_a'));
await died;
expect(attempts).toBe(1);
expect((await sessions.doc(sessionId, Todos)).revision).toBe(0);
// Process B: the crash record proves A's drive dead, so it recovers at once.
await deliver({ world, harness: harness(10), marks: false, anchors: vercelAnchors() }, sessionId, msg('m_b'));
const state = await sessions.inspect(sessionId);
const key = `add:${state.submissions.find((s) => s.kind === 'prompt')!.id}`;
if (replay === 'never') {
expect(state.effects.get(key)).toMatchObject({ status: 'failed', error: { interrupted: true } });
expect(attempts).toBe(1);
expect(await sessions.doc(sessionId, Todos)).toEqual({ value: { items: [] }, revision: 0 });
expect(state.docs.size).toBe(0);
} else {
expect(state.effects.get(key)).toMatchObject({ status: 'completed', output: 'added', attempt: 2 });
expect(reads).toEqual([0, 0]);
expect(await sessions.doc(sessionId, Todos)).toEqual({ value: { items: ['attempt 2'] }, revision: 1 });
}
});
}
it('a tool that throws after its update changes nothing', async () => {
const { world } = await open();
const harness = scripted(async (ctx, sub) => {
const failed = await call(ctx, `bad:${sub.id}`, async (io) => {
io.docs!.update(Todos, (d) => void d.items.push('never'));
throw new Error('the tool failed after updating');
});
expect(failed).toBeInstanceOf(EffectFailure);
await call(ctx, `good:${sub.id}`, async (io) => (io.docs!.update(Board, (d) => void (d.cards.a = 'kept')), 'ok'));
});
const sessions = durableSessions({ world, harness, marks: false });
const { sessionId } = await sessions.start();
await sessions.send(sessionId, 'go');
await deliver({ world, harness, marks: false }, sessionId, msg('m1'));
const state = await sessions.inspect(sessionId);
expect(state.effectOrder.find((e) => e.key.startsWith('bad:'))).toMatchObject({ status: 'failed', error: { message: 'the tool failed after updating' } });
expect(state.docs.has('app.todos')).toBe(false);
expect((await sessions.doc(sessionId, Board)).value).toEqual({ cards: { a: 'kept' } });
});
it('replay gives the same state: a fresh process folds the same documents, and memoized calls do not run their mutators again', async () => {
const { base, world } = await open();
let mutations = 0;
const harness = scripted(async (ctx, sub) => {
await call(ctx, `add:${sub.id}`, async (io) => {
io.docs!.update(Todos, (d) => {
mutations++;
d.items.push(sub.text!);
});
io.docs!.update(Board, (d) => void (d.cards[sub.text!] = `card ${sub.text}`));
return 'ok';
});
});
const sessions = durableSessions({ world, harness, marks: false });
const { sessionId } = await sessions.start();
for (const text of ['a', 'b', 'c']) {
await sessions.send(sessionId, text);
await deliver({ world, harness, marks: false }, sessionId, msg(`m_${text}`));
// An app's edit between turns.
await sessions.updateDoc(sessionId, Board, (d) => void (d.cards[`app-${text}`] = 'from the app'));
}
expect(mutations).toBe(3);
const before = await sessions.inspect(sessionId);
// Another process, another World instance on the same store.
const other = await base.reopen();
opened.push(other);
const log = new SessionLog(other, sessionId);
await log.refresh();
expect([...log.state.docs]).toEqual([...before.docs]);
expect([...log.state.docEdits]).toEqual([...before.docEdits]);
expect(readDoc(log.state, Todos)).toEqual({ value: { items: ['a', 'b', 'c'] }, revision: 3 });
expect(readDoc(log.state, Board).revision).toBe(6);
// A later drive replays every earlier turn from the log: no mutator runs again.
await sessions.send(sessionId, 'd');
await deliver({ world: memQueue(other).world, harness, marks: false }, sessionId, msg('m_d'));
expect(mutations).toBe(4);
expect((await sessions.doc(sessionId, Todos)).value.items).toEqual(['a', 'b', 'c', 'd']);
});
it('concurrent out-of-band updates: without expectRevision every one lands (re-reading after a lost race); with it one wins each revision and the rest throw DocConflict; one docs hook', async () => {
const { world, created } = await open();
const harness = scripted(async () => {});
const sessions = durableSessions({ world, harness, marks: false });
const { sessionId } = await sessions.start();
// Separate sessions objects: separate logs and caches, as separate instances would have.
const writers = Array.from({ length: 5 }, () => durableSessions({ world, harness, marks: false }));
let runs = 0;
const results = await Promise.all(
writers.map((w, i) =>
w.updateDoc(sessionId, Board, (d) => {
runs++;
d.cards[`w${i}`] = `writer ${i}`;
})
)
);
const after = await sessions.doc(sessionId, Board);
expect(after.revision).toBe(5);
expect(Object.keys(after.value.cards).sort()).toEqual(['w0', 'w1', 'w2', 'w3', 'w4']);
expect(results.map((r) => r.revision).sort()).toEqual([1, 2, 3, 4, 5]);
expect(runs).toBeGreaterThan(5); // some lost a race and ran their mutator again
let state = await sessions.inspect(sessionId);
expect(state.docsHook?.token).toBe(`harness:${sessionId}:docs`);
expect([...state.docEdits.values()].filter((e) => e.applied)).toHaveLength(5);
// Every first edit raced to create the hook. The first `:docs` hook in log order is the docs hook, and
// every edit went to it. The losers were answered with a conflict (world-local logs a hook_conflict;
// world-postgres may refuse the write, or, racing, accept a second hook with the same token, which
// the fold ignores) and used the winner's.
const log = await allEvents(world, sessionId);
const docsHooks = log.filter((e) => e.eventType === 'hook_created' && e.correlationId !== state.inbox?.hookId);
expect(docsHooks[0]!.correlationId).toBe(state.docsHook!.hookId);
expect(log.filter((e) => e.eventType === 'hook_received').every((e) => e.correlationId === state.docsHook!.hookId)).toBe(true);
if (wc.name === 'world-local') {
expect(docsHooks).toHaveLength(1);
expect(log.filter((e) => e.eventType === 'hook_conflict').length).toBeGreaterThan(0);
}
// With expectRevision: all five expect revision 5; exactly one lands.
const settled = await Promise.allSettled(writers.map((w, i) => w.updateDoc(sessionId, Board, (d) => void (d.cards.winner = `writer ${i}`), { expectRevision: 5 })));
expect(settled.filter((s) => s.status === 'fulfilled')).toHaveLength(1);
for (const s of settled.filter((x) => x.status === 'rejected')) {
expect((s as PromiseRejectedResult).reason).toBeInstanceOf(DocConflict);
expect((s as PromiseRejectedResult).reason).toMatchObject({ kind: 'app.board', expected: 5 });
}
expect((await sessions.doc(sessionId, Board)).revision).toBe(6);
// A stale expectRevision is refused before anything is written.
const writes = created.filter((c) => c.runId === sessionId && c.eventType === 'hook_received').length;
await expect(sessions.updateDoc(sessionId, Board, (d) => void (d.cards.late = 'x'), { expectRevision: 2 })).rejects.toMatchObject({ name: 'DocConflict', revision: 6, expected: 2 });
expect(created.filter((c) => c.runId === sessionId && c.eventType === 'hook_received').length).toBe(writes);
// No change, no write.
await sessions.updateDoc(sessionId, Board, () => {});
expect(created.filter((c) => c.runId === sessionId && c.eventType === 'hook_received').length).toBe(writes);
state = await sessions.inspect(sessionId);
expect(state.submissions).toHaveLength(0); // an edit is never a submission
// Every applied edit published its live event.
expect((await liveDocs(world, sessionId)).map((e) => e.revision).sort((x, y) => x - y)).toEqual([1, 2, 3, 4, 5, 6]);
});
it("out-of-band edits are inputs: never fenced, even through a log a drive armed, and a drive's own writes still are", async () => {
const { world, created } = await open();
const harness = scripted(async (ctx, sub) => {
await call(ctx, `t:${sub.id}`, async (io) => (io.docs!.update(Todos, (d) => void d.items.push('tool')), 'ok'));
});
const sessions = durableSessions({ world, harness, marks: false });
const { sessionId } = await sessions.start();
// A log armed by a drive (the drive route hands the submit's log to its drive; a route's updateDoc may share it).
const log = new SessionLog(world, sessionId);
await log.refresh();
expect(log.arm('drv_test')).toBe(true);
await log.write({ eventType: 'hook_created', correlationId: `hook_${ulid()}`, eventData: { token: `harness:${sessionId}:docs` } });
await log.write({
eventType: 'hook_received',
correlationId: log.state.docsHook!.hookId,
eventData: { token: log.state.docsHook!.token, payload: encode({ $docs: 1, id: `dedit_${ulid()}`, docs: [{ kind: 'app.todos', base: 0, schema: 1, ops: [{ op: 'set', path: [], value: { items: ['app'] } }] }] }) },
});
expect(log.superseded).toBeUndefined();
await sessions.send(sessionId, 'go');
await deliver({ world, harness, marks: false }, sessionId, msg('m1'));
const mine = created.filter((c) => c.runId === sessionId);
expect(mine.filter((c) => c.eventType.startsWith('hook_')).every((c) => !c.inBand)).toBe(true);
expect(mine.filter((c) => c.eventType.startsWith('step_')).every((c) => c.inBand)).toBe(true);
expect(await sessions.doc(sessionId, Todos)).toEqual({ value: { items: ['app', 'tool'] }, revision: 2 });
});
it('an out-of-band edit whose answer was lost is looked up before it is sent again: it lands once', async () => {
const { world } = await open();
let dropped = 0;
// The World commits the edit, then the answer is lost (a transport failure).
const lossy = new Proxy(world, {
get(target, key) {
if (key !== 'events') return (target as unknown as Record<string | symbol, unknown>)[key];
return new Proxy(target.events, {
get(events, k) {
const value = (events as unknown as Record<string | symbol, unknown>)[k];
if (k !== 'create') return typeof value === 'function' ? (value as (...a: unknown[]) => unknown).bind(events) : value;
return async (runId: string, data: { eventType: string }, ...rest: unknown[]) => {
const result = await (value as (...a: unknown[]) => Promise<unknown>).call(events, runId, data, ...rest);
if (data.eventType === 'hook_received' && dropped++ === 0) throw Object.assign(new Error('the relay dropped'), { ambiguous: true });
return result;
};
},
});
},
}) as World;
const sessions = durableSessions({ world, harness: scripted(async () => {}), marks: false });
const { sessionId } = await sessions.start();
const logs: string[] = [];
const writer = durableSessions({ world: lossy, harness: scripted(async () => {}), marks: false, log: (m) => logs.push(m) });
expect(await writer.updateDoc(sessionId, Todos, (d) => void d.items.push('once'))).toEqual({ value: { items: ['once'] }, revision: 1 });
expect(logs).toContain('updateDoc: ambiguous write, looking it up');
const received = (await allEvents(world, sessionId)).filter((e) => e.eventType === 'hook_received');
expect(received).toHaveLength(1);
expect((await sessions.inspect(sessionId)).docEdits.size).toBe(1);
});
it('an out-of-band edit racing an in-band one: the app edit lands, the tool edit rebases onto it (both keys survive), and the drive does not yield contended for it', async () => {
const { world } = await open();
let release!: () => void;
const gate = new Promise<void>((r) => (release = r));
const app = durableSessions({ world, harness: scripted(async () => {}), marks: false });
const logs: string[] = [];
let sessionId = '';
const harness = scripted(async (ctx, sub) => {
await call(ctx, `tool:${sub.id}`, async (io) => {
io.docs!.update(Board, (d) => void (d.cards.tool = 'from the tool'));
await gate;
return 'ok';
});
// Between this drive's last read and its next intent: an app's edit (and the docs hook's creation).
await app.updateDoc(sessionId, Todos, (d) => void d.items.push('between'));
await call(ctx, `next:${sub.id}`, async () => 'next ran');
});
const sessions = durableSessions({ world, harness, marks: false });
({ sessionId } = await sessions.start());
await sessions.send(sessionId, 'go');
const driving = deliver({ world, harness, marks: false, log: (m) => logs.push(m) }, sessionId, msg('m1'));
await until(async () => ((await sessions.inspect(sessionId)).effectOrder.some((e) => e.key.startsWith('tool:') && e.status === 'running') ? true : undefined));
// The tool pinned revision 0; the app writes revision 1 meanwhile.
await app.updateDoc(sessionId, Board, (d) => void (d.cards.app = 'from the app'));
release();
await driving;
const state = await sessions.inspect(sessionId);
const tool = state.effectOrder.find((e) => e.key.startsWith('tool:'))!;
expect(tool.docs).toEqual([{ kind: 'app.board', revision: 2, rebased: true }]);
expect(await sessions.doc(sessionId, Board)).toEqual({ value: { cards: { app: 'from the app', tool: 'from the tool' } }, revision: 2 });
expect(state.effectOrder.find((e) => e.key.startsWith('next:'))).toMatchObject({ status: 'completed', output: 'next ran' });
expect(logs.some((m) => m.includes('contended'))).toBe(false);
expect((await sessions.doc(sessionId, Todos)).value.items).toEqual(['between']);
});
it('parallel sibling calls (an AI SDK-style step) edit different keys of one document: both survive', async () => {
const { world } = await open();
let arrived = 0;
let release!: () => void;
const both = new Promise<void>((r) => (release = r));
const harness = scripted(async (ctx, sub) => {
await Promise.all(
['left', 'right'].map((side) =>
call(ctx, `${side}:${sub.id}`, async (io) => {
io.docs!.update(Board, (d) => void (d.cards[side] = `${side} card`));
if (++arrived === 2) release();
await both;
return side;
})
)
);
});
const sessions = durableSessions({ world, harness, marks: false });
const { sessionId } = await sessions.start();
await sessions.send(sessionId, 'go');
await deliver({ world, harness, marks: false }, sessionId, msg('m1'));
const doc = await sessions.doc(sessionId, Board);
expect(doc).toEqual({ value: { cards: { left: 'left card', right: 'right card' } }, revision: 2 });
const records = (await sessions.inspect(sessionId)).effectOrder.filter((e) => e.name === 'tool.edit');
expect(records.flatMap((r) => r.docs ?? []).filter((d) => d.rebased)).toHaveLength(1);
});
it('run-ahead: the next call sees an edit at once, before it is durable', async () => {
const { world } = await open();
const seen: number[] = [];
const harness = scripted(async (ctx, sub) => {
for (const n of [1, 2, 3]) {
await call(ctx, `t${n}:${sub.id}`, async (io) => {
seen.push(io.docs!.read(Todos).revision);
io.docs!.update(Todos, (d) => void d.items.push(`t${n}`));
return n;
});
}
});
const sessions = durableSessions({ world, harness, marks: false, runAhead: true });
const { sessionId } = await sessions.start();
await sessions.send(sessionId, 'go');
await deliver({ world, harness, marks: false, runAhead: true }, sessionId, msg('m1'));
expect(seen).toEqual([0, 1, 2]);
expect(await sessions.doc(sessionId, Todos)).toEqual({ value: { items: ['t1', 't2', 't3'] }, revision: 3 });
});
for (const fence of [true, false]) {
it(
fence
? "a fenced stale writer's edit is refused: a frozen holder's late outcome is refused by the World after its redelivery took over, and only the takeover's edit is in the document"
: "without the fence the frozen holder's late outcome lands, inert, and its edit is not applied",
async () => {
const { world } = await open();
const leased = leasedWorld(world, { visibilityMs: 200 });
let releaseFirst!: () => void;
const firstGate = new Promise<void>((r) => (releaseFirst = r));
let attempts = 0;
const harness = scripted(async (ctx, sub) => {
await call(ctx, `edit:${sub.id}`, async (io) => {
const mine = ++attempts;
io.docs!.update(Todos, (d) => void d.items.push(mine === 1 ? 'from the frozen holder' : 'from the takeover'));
if (mine === 1) await firstGate;
return `attempt ${mine}`;
});
});
const logs: string[] = [];
const sessions = durableSessions({ world: leased, harness, anchors: vercelAnchors(), inBandFence: fence, hints: false, marks: false, log: (m) => logs.push(m) });
const { sessionId } = await sessions.start();
await sessions.send(sessionId, 'go');
const first = leased.broker.deliver()!;
const toolRunning = async () => ((await sessions.inspect(sessionId)).effectOrder.find((e) => e.name === 'tool.edit')?.status === 'running' ? true : undefined);
await until(toolRunning);
const staleStep = (await sessions.inspect(sessionId)).effectOrder.find((e) => e.name === 'tool.edit')!.stepId;
// The holder freezes (its World calls wait), its tool returns meanwhile, its lease lapses.
first.freeze();
releaseFirst();
const again = await until(() => leased.broker.deliver());
await again.done;
expect(await sessions.doc(sessionId, Todos)).toEqual({ value: { items: ['from the takeover'] }, revision: 1 });
// The holder thaws: its outcome, edit and all, goes out now.
first.thaw();
await until(async () => (logs.includes(fence ? 'in-band fence: a write was refused' : 'preempted') || (await allEvents(world, sessionId)).some((e) => e.correlationId === staleStep && e.eventType === 'step_completed') ? true : undefined), 5000);
await sleep(300);
const late = (await allEvents(world, sessionId)).filter((e) => e.correlationId === staleStep && e.eventType === 'step_completed');
expect(late).toHaveLength(fence ? 0 : 1);
const state = await sessions.inspect(sessionId);
expect(await sessions.doc(sessionId, Todos)).toEqual({ value: { items: ['from the takeover'] }, revision: 1 });
expect(state.effects.get(`edit:${state.submissions[0]!.id}`)).toMatchObject({ status: 'completed', output: 'attempt 2' });
expect(attempts).toBe(2);
}
);
}
it('the size limit: a tool update past 64 KB fails the call and commits nothing; updateDoc refuses before writing; exactly 64 KB is allowed', async () => {
const { world, created } = await open();
const Notes = defineDoc<{ text: string }>({ kind: 'app.notes', version: 1, initial: () => ({ text: '' }) });
const fill = (bytes: number) => 'n'.repeat(bytes - JSON.stringify({ text: '' }).length);
const harness = scripted(async (ctx, sub) => {
await call(ctx, `big:${sub.id}`, async (io) => {
io.docs!.update(Notes, (d) => void (d.text = fill(DOC_MAX_BYTES + Number(sub.text))));
return 'wrote';
});
});
const sessions = durableSessions({ world, harness, marks: false });
const { sessionId } = await sessions.start();
await sessions.send(sessionId, '1');
await deliver({ world, harness, marks: false }, sessionId, msg('m1'));
let state = await sessions.inspect(sessionId);
expect(state.effectOrder.find((e) => e.key.startsWith('big:'))).toMatchObject({ status: 'failed', error: { message: 'harness-world: document app.notes would be 65,537 bytes; the limit is 65,536' } });
expect(state.docs.has('app.notes')).toBe(false);
await sessions.send(sessionId, '0');
await deliver({ world, harness, marks: false }, sessionId, msg('m2'));
expect((await sessions.doc(sessionId, Notes)).revision).toBe(1);
const writes = created.filter((c) => c.runId === sessionId && c.eventType.startsWith('hook_')).length;
await expect(sessions.updateDoc(sessionId, Notes, (d) => void (d.text += '!'))).rejects.toBeInstanceOf(DocTooLarge);
expect(created.filter((c) => c.runId === sessionId && c.eventType.startsWith('hook_')).length).toBe(writes);
// Shrinking it is fine.
state = await sessions.inspect(sessionId);
await sessions.updateDoc(sessionId, Notes, (d) => void (d.text = d.text.slice(1000)));
expect((await sessions.doc(sessionId, Notes)).revision).toBe(2);
});
it('schema versions: an older stored document migrates on read, a write after migrating stores the new schema, a newer one fails loudly', async () => {
const { world } = await open();
const V2 = defineDoc<{ items: { text: string; done: boolean }[] }>({
kind: 'app.todos',
version: 2,
initial: () => ({ items: [] }),
migrate: (old, from) => {
if (from !== 1) throw new Error(`no migration from ${from}`);
return { items: (old.items as string[]).map((text) => ({ text, done: false })) };
},
});
const NoMigrate = defineDoc<{ items: { text: string; done: boolean }[] }>({ kind: 'app.todos', version: 2, initial: () => ({ items: [] }) });
let def: typeof Todos | typeof V2 = Todos;
const harness = scripted(async (ctx, sub) => {
await call(ctx, `t:${sub.id}`, async (io) => {
if (def === Todos) io.docs!.update(Todos, (d) => void d.items.push(sub.text!));
else io.docs!.update(V2, (d) => void d.items.push({ text: sub.text!, done: true }));
return 'ok';
});
});
const sessions = durableSessions({ world, harness, marks: false });
const { sessionId } = await sessions.start();
await sessions.send(sessionId, 'old');
await deliver({ world, harness, marks: false }, sessionId, msg('m1'));
// New code reads the v1 document through migrate; the log is unchanged.
expect(await sessions.doc(sessionId, V2)).toEqual({ value: { items: [{ text: 'old', done: false }] }, revision: 1 });
expect((await sessions.inspect(sessionId)).docs.get('app.todos')).toMatchObject({ schema: 1 });
await expect(sessions.doc(sessionId, NoMigrate)).rejects.toBeInstanceOf(DocVersionError);
// New code writes: a root set at schema 2.
def = V2;
await sessions.send(sessionId, 'new');
await deliver({ world, harness, marks: false }, sessionId, msg('m2'));
const state = await sessions.inspect(sessionId);
expect(state.docs.get('app.todos')).toMatchObject({ schema: 2, revision: 2, value: { items: [{ text: 'old', done: false }, { text: 'new', done: true }] } });
// Old code (a deploy behind) refuses to read it rather than silently downgrade.
await expect(sessions.doc(sessionId, Todos)).rejects.toThrow(/newer than this code's 1/);
await expect(sessions.updateDoc(sessionId, Todos, (d) => void d.items.push('v1'))).rejects.toBeInstanceOf(DocVersionError);
expect(await sessions.doc(sessionId, V2)).toMatchObject({ revision: 2 });
});
it('chatRoute({ docs }) starts each response with the current value of each listed document (no id) and forwards their live events; others are dropped, and by default none are sent', async () => {
const { world } = await open();
const Secret = defineDoc<{ token: string }>({ kind: 'app.secret', version: 1, initial: () => ({ token: '' }) });
const harness = scripted(async (ctx, sub) => {
await call(ctx, `t:${sub.id}`, async (io) => {
io.docs!.update(Todos, (d) => void d.items.push(sub.text!));
io.docs!.update(Secret, (d) => void (d.token = `secret ${sub.text}`));
return 'ok';
});
});
const options: DriverOptions = { world, harness, marks: false };
const sessions = durableSessions(options);
const send = async (id: string, text: string, kind: 'prompt' | 'steer' | 'followUp') => {
const submission = await sessions.send(id, text, kind);
void deliver(options, id, msg(`m_${ulid()}`));
return submission;
};
const read = async (response: Response) => {
const text = await response.text();
return text
.split('\n\n')
.filter((b) => b.includes('data:'))
.map((b) => ({ id: /^id: (.*)$/m.exec(b)?.[1], event: /^event: (.*)$/m.exec(b)?.[1], data: JSON.parse(/^data: (.*)$/m.exec(b)![1]!) }));
};
const listed = chatRoute(sessions, { send, docs: [Todos] });
const post = (route: typeof listed, body: unknown) => route.POST(new Request('http://x/api/chat', { method: 'POST', body: JSON.stringify(body) }));
const first = await post(listed, { text: 'one' });
const sessionId = first.headers.get('x-session-id')!;
const one = await read(first);
const docs = one.filter((m) => m.data.type === 'doc');
expect(docs.map((m) => [m.data.kind, m.data.revision, m.data.value])).toEqual([['app.todos', 1, { items: ['one'] }]]);
expect(docs[0]!.id).toBeDefined(); // a live event keeps its cursor
// The second response opens with the current value, then streams the turn.
const two = await read(await post(listed, { sessionId, text: 'two' }));
expect(two[0]).toEqual({ id: undefined, event: undefined, data: { type: 'doc', kind: 'app.todos', revision: 1, schema: 1, value: { items: ['one'] }, by: { effect: expect.stringMatching(/^t:/), step: expect.stringMatching(/^step_/) }, t: expect.any(Number) } });
expect(two.filter((m) => m.data.type === 'doc').map((m) => m.data.revision)).toEqual([1, 2]);
expect(two.some((m) => m.data.kind === 'app.secret')).toBe(false);
// Default: no documents at all.
const plain = chatRoute(sessions, { send });
const three = await read(await post(plain, { sessionId, text: 'three' }));
expect(three.some((m) => m.data.type === 'doc')).toBe(false);
expect(three.some((m) => m.data.type === 'idle')).toBe(true);
// The live stream had them all; the routes chose.
expect((await liveDocs(world, sessionId)).filter((e) => e.kind === 'app.secret')).toHaveLength(3);
});
it("a document defined with live: 'revision' is published without its value (a tool's edit, an app's, the chat route's opening values); the log has the value", async () => {
const { world } = await open();
const Manifest = defineDoc<{ files: Record<string, string> }>({ kind: 'app.manifest', version: 1, live: 'revision', initial: () => ({ files: {} }) });
const harness = scripted(async (ctx, sub) => {
await call(ctx, `t:${sub.id}`, async (io) => {
io.docs!.update(Manifest, (d) => void (d.files[sub.text!] = 'x'.repeat(1000)));
io.docs!.update(Todos, (d) => void d.items.push(sub.text!));
return 'ok';
});
});
const options: DriverOptions = { world, harness, marks: false };
const sessions = durableSessions(options);
const { sessionId } = await sessions.start();
await sessions.send(sessionId, 'a');
await deliver(options, sessionId, msg('m1'));
await sessions.updateDoc(sessionId, Manifest, (d) => void (d.files.app = 'y'));
const live = await liveDocs(world, sessionId);
expect(live.filter((e) => e.kind === 'app.manifest').map((e) => [e.revision, 'value' in e])).toEqual([
[1, false],
[2, false],
]);
expect(live.find((e) => e.kind === 'app.manifest')).toMatchObject({ type: 'doc', kind: 'app.manifest', revision: 1, schema: 1, by: { effect: expect.stringMatching(/^t:/) } });
// Other documents keep the default: the whole value.
expect(live.filter((e) => e.kind === 'app.todos').map((e) => e.value)).toEqual([{ items: ['a'] }]);
expect(await sessions.doc(sessionId, Manifest)).toEqual({ value: { files: { a: 'x'.repeat(1000), app: 'y' } }, revision: 2 });
// The chat route: the opening value and the turn's events say the revision only.
const send = async (id: string, text: string, kind: 'prompt' | 'steer' | 'followUp') => {
const submission = await sessions.send(id, text, kind);
void deliver(options, id, msg(`m_${ulid()}`));
return submission;
};
const route = chatRoute(sessions, { send, docs: [Manifest, Todos] });
const response = await route.POST(new Request('http://x/api/chat', { method: 'POST', body: JSON.stringify({ sessionId, text: 'b' }) }));
const events = (await response.text())
.split('\n\n')
.filter((b) => b.includes('data:'))
.map((b) => JSON.parse(/^data: (.*)$/m.exec(b)![1]!) as { type?: string; kind?: string; revision?: number; value?: unknown });
const manifest = events.filter((e) => e.type === 'doc' && e.kind === 'app.manifest');
expect(manifest.map((e) => [e.revision, 'value' in e])).toEqual([
[2, false],
[3, false],
]);
expect(events.filter((e) => e.type === 'doc' && e.kind === 'app.todos').map((e) => e.value)).toEqual([{ items: ['a'] }, { items: ['a', 'b'] }]);
});
});
}
describe('documents without durability (rawSessions)', () => {
it('tools edit documents in memory, updateDoc applies, live events go out, and a snapshot round-trips them', async () => {
const harness = scripted(async (ctx, sub) => {
await call(ctx, `t:${sub.id}`, async (io) => {
io.docs!.update(Todos, (d) => void d.items.push(sub.text!));
return 'ok';
});
});
const raw = rawSessions({ harness, marks: false });
const { sessionId } = await raw.start();
const seen: unknown[] = [];
raw.subscribe(sessionId, (e) => seen.push(e));
await raw.send(sessionId, 'one');
await raw.idle(sessionId);
await raw.updateDoc(sessionId, Todos, (d) => void d.items.push('app'));
await expect(raw.updateDoc(sessionId, Todos, (d) => void d.items.push('stale'), { expectRevision: 1 })).rejects.toBeInstanceOf(DocConflict);
expect(await raw.doc(sessionId, Todos)).toEqual({ value: { items: ['one', 'app'] }, revision: 2 });
expect(seen.filter((e) => (e as { type?: string }).type === 'doc')).toHaveLength(2);
// A document defined with live: 'revision': its events carry no value, in a tool and from updateDoc.
const Quiet = defineDoc<{ n: number }>({ kind: 'app.quiet', version: 1, live: 'revision', initial: () => ({ n: 0 }) });
await raw.updateDoc(sessionId, Quiet, (d) => void (d.n = 1));
expect(seen.at(-1)).toMatchObject({ type: 'doc', kind: 'app.quiet', revision: 1 });
expect(seen.at(-1)).not.toHaveProperty('value');
const snapshot = JSON.parse(JSON.stringify(raw.snapshot(sessionId)));
raw.close(sessionId);
const again = raw.restore(snapshot);
expect(await raw.doc(again, Todos)).toEqual({ value: { items: ['one', 'app'] }, revision: 2 });
await raw.send(again, 'two');
await raw.idle(again);
expect(await raw.doc(again, Todos)).toEqual({ value: { items: ['one', 'app', 'two'] }, revision: 3 });
expect((await raw.inspect(again)).effectOrder.at(-1)?.docs).toEqual([{ kind: 'app.todos', revision: 3 }]);
});
});
describe('hostToolIO', () => {
it("passes every field of the effect's io a tool may use, documents included", () => {
const docs = { read: () => ({ value: {}, revision: 0 }), update: () => ({}) } as unknown as EffectIO['docs'];
const signal = new AbortController().signal;
const io: EffectIO = { signal, progress: () => {}, stepId: 'step_1', park: async () => ({ stepId: 'p', settle: async () => true }), chaosArmedMs: 50, docs };
const ctx = {} as DriveContext;
const mine = new AbortController().signal;
const out = hostToolIO(io, ctx, { signal: mine });
expect(out).toMatchObject({ signal: mine, ctx, stepId: 'step_1', park: io.park, chaosArmedMs: 50, docs });
expect(hostToolIO({ ...io, docs: undefined, chaosArmedMs: undefined }, ctx)).not.toHaveProperty('docs');
});
});