Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World
import { AsyncLocalStorage } from 'node:async_hooks';
import { dirname } from 'node:path';
import { type DriveContext, type EffectRecord, type Harness, type HostTool, type ReplayPolicy, type SessionState, type Submission, Unavailable, UNAVAILABLE_STEP } from 'harness-world';
import {
type AcquireMachine,
BuiltinCalls,
type BuiltinStart,
CODEX_ROUTES,
failureText,
type GatewayLedger,
gatewayCredential,
gatewayHandler,
gatewayUrl,
isBrandedMachine,
type MachineTail,
ModelSteps,
Mutex,
NativeJournal,
placeSubmissions,
recoveryNote,
restartEvent,
type SessionMachine,
type SessionProcess,
settleAbandonedModelSteps,
type ToolResult,
toolBridge,
toolKey,
transcriptOf,
type TranscriptEntry,
} from 'harness-world-process';
import { acknowledgeInDrive, attachInDrive, cliPath, CODEX_CLI, DriveFiles, SANDBOX_DOC } from 'harness-world-sandbox/cli';
import { AppServer, type RpcNotification, type RpcServerRequest } from './app-server.ts';
import { BUILTINS_ON, checkConfig, codexConfig, type TomlValue, toToml } from './config.ts';
import {
callIdOf,
endsTurn,
isRolloutPath,
isSessionMeta,
type RolloutCall,
rolloutPathIn,
rolloutStream,
rolloutText,
summarizeRollout,
THREAD_STREAM,
type ThreadRecord,
} from './rollout.ts';
/**
* Codex on harness-world: a kernel `Harness` that drives `codex app-server
* --stdio` (Codex's JSON-RPC protocol for IDEs) **in the session's Vercel
* Sandbox VM**, never on the machine that drives the session
* (docs/workspaces.md §1 decision 2, §5.6). Every drive acquires the
* session's VM through the sandbox tier (`attachInDrive` with the session
* factory's acquire, `machine.acquire`), and refuses a machine the factory
* didn't brand: `Unavailable('sandbox_required')`.
*
* - **Where it runs.** app-server runs as the VM's unprivileged user, its
* stdio over the relay (`RemoteProcess`, a `LineChannel`), its working
* directory `/workspace` (the session's Drive, the same on every VM), its
* `HOME` and `CODEX_HOME` under `/var/lib/dh/s`, wiped before every spawn.
* `CODEX_HOME` is outside every writable root of Codex's own sandbox, so a
* command can't rewrite the rollout or `config.toml`.
* - **Tools.** Tool-less by default: the app's `HostTool`s (`tools`) are
* app-server dynamic tools, run in the Function, each call the kernel's
* effect sandwich keyed by Codex's own `call_id` (`tool:<call_id>`), and
* they are its only tools (shell, `apply_patch` and images off, a
* `read-only` sandbox). With `builtins: true`, Codex's own shell and
* `apply_patch` are on, on `/workspace`, under Codex's own sandbox
* (`sandboxMode`: by default `workspace-write` when bubblewrap's self-test
* passes in the VM, else `danger-full-access`, the VM being the boundary;
* the first verdict is recorded in the log, `codex.sandbox`, and every
* later drive keeps it).
* Every command and patch asks the drive first (`approvalPolicy:
* 'untrusted'`): its intent is committed once the rollout line that made
* the call is in the log, and only then approved, once. Anything else is
* declined: a second approval for the same call (Codex's "retry without
* sandbox?"), input to a running command, a sub-command, more
* permissions, network access. Its outcome is committed when Codex
* reports the item completed, after the Drive is flushed. Nothing reruns
* one after a crash: the Drive is durable, not transactional.
* - **The rollout is mirrored into the log.** Codex keeps its thread as an
* append-only JSONL file; the relay tails it (`TAIL`, inode and size
* checked: a replaced or shrunk file stops the drive, and the third time
* in a turn ends the turn) and every line
* becomes a record of the session's native journal (`rollout:<threadId>`).
* Before every `replay: 'never'` call the drive drains the tail (`SYNC`)
* and checks the call's own line is there (a bounded number of times):
* without it, the call is declined, never run, and the model is told to
* try again.
* - **Every drive restores from the log.** `CODEX_HOME` is wiped and gets a
* fresh `config.toml` and the mirrored rollout, byte for byte (`FILES`);
* app-server resumes the thread by id.
* - **Recovery.** A turn a crash cut off is resumed natively: calls the
* rollout left without an output get one injected from the log
* (`thread/inject_items`), then a recovery note built from the log starts
* a turn. A link to the VM lost mid-drive is re-attached in the same
* invocation (a bounded number of times, and only while there is work
* left), once the app's tools still running have settled, and the drive
* resumes from the log.
* - **The inbox.** Each submission is placed, then sent with its id as
* Codex's `clientUserMessageId`: `turn/start`, or `turn/steer` for a
* steer that arrives while a turn runs.
*
* Model traffic goes from the VM's loopback model port through the relay to
* this drive's `gatewayHandler` (`CODEX_ROUTES`: the fixed path table, the
* body policy, the model, the budget), which adds the credential: the VM
* never holds it. Codex's provider key is the attach's model key
* (`SessionMachine.modelKey`, which the relay checks with the caller's pid),
* hidden from the model's commands
* (`shell_environment_policy`). Each response's cost is a record in the log
* (`codex.spend`), so the dollar caps hold over the turn and the session,
* across invocations. app-server runs no git of the Drive's: `GIT_DIR`
* points nowhere and no git config is read.
*/
export interface CodexHarnessOptions {
/**
* The app's tools, as app-server dynamic tools: each call a kernel effect
* keyed by Codex's `call_id`, with the tool's replay policy, run where the
* drive runs (the Function). Its only tools, or (with `builtins: true`)
* next to the built-ins. Default none.
*/
tools?: HostTool[];
/**
* Codex's own shell and `apply_patch` (and `view_image`), on `/workspace`
* in the session's VM. Default false (tool-less: `tools` are its only
* tools). Image generation, web search, sub-agents and plugins stay off
* either way.
*/
builtins?: boolean;
/**
* Codex's sandbox for its commands, in the VM (the Codex SDK's word; this
* option was `sandbox`). Needs `builtins: true`. Default: `workspace-write`
* (writes to `/workspace` and its home, network on) when bubblewrap's
* self-test passes in the VM at attach, else `danger-full-access` (the VM
* is the boundary, and holds no credential; the provider key's
* placeholder is then readable by the model's commands, docs/workspaces.md
* §5.5).
*/
sandboxMode?: 'read-only' | 'workspace-write' | 'danger-full-access';
/** An AI Gateway model id. Default `openai/gpt-5.4-mini`. */
model?: string;
/** Developer instructions for the thread (`developerInstructions`). */
instructions?: string;
/** Replace Codex's own base instructions (they describe its built-in tools). */
baseInstructions?: string;
/** Reasoning effort per turn. Default Codex's. */
effort?: 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
/** The Gateway credential, read per request in the Function. Default `gatewayCredential` (AI_GATEWAY_API_KEY, else the Vercel OIDC token). */
credential?: () => Promise<string>;
/** The Gateway's base URL. Default `gatewayUrl()`. */
gatewayUrl?: string;
/** The kernel name (its sessions' runs are `workflow//durable-harness@<version>//<name>`). Default `codex`. */
name?: string;
/** The model's context window (Gateway ids are unknown to Codex). Default 272,000. */
contextWindow?: number;
/** Extra `config.toml` settings, merged over the generated ones. The keys the harness sets itself (`RESERVED_CONFIG_KEYS`) throw. */
config?: Record<string, TomlValue>;
/** How often to read the inbox for steers while a turn runs, without hints. Default 500 ms (5 s with hints). */
inboxPollMs?: number;
/** Diagnostics (stderr noise, declined approvals, the sandbox's lifecycle). */
log?: (message: string, fields?: Record<string, unknown>) => void;
/**
* The session's VM (internal: durable-harness's `codex()` resolves
* `workspace: sandbox()` into it). Without it, or with an acquire that
* gives a machine the sandbox tier's session factory didn't brand, a
* drive fails its turn with `Unavailable('sandbox_required')`.
*/
machine?: CodexMachine;
/**
* The stand-in's seam (docs/workspaces.md §5.8): the command that speaks
* app-server's protocol instead of the image's `codex`. Refused outside a
* test run (`DH_TEST_LOCAL_SANDBOX=1` under a test runner, not on Vercel,
* not in production).
*/
test?: { command: string; args?: string[] };
}
/** The session's VM, as a `sandbox()` descriptor resolves it (durable-harness's `codex()` builds it). */
export interface CodexMachine {
/** The sandbox tier's acquire: its session factory (`sandboxSession`), which brands the machines it gives. */
acquire: AcquireMachine;
/** `sandbox({ env })`: non-secret variables for app-server and for the model's commands (`shell_environment_policy.set`). */
env?: Readonly<Record<string, string>>;
/** `sandbox({ network })`: the gateway handler refuses what would reach past it through the provider. Default `allow-all`. */
network?: 'allow-all' | 'deny-all' | { readonly allow: readonly string[] };
/** `sandbox({ budget })`'s dollar caps, over the turn and the session (each response's cost is in the log). Default none (`Infinity`). */
budget?: { readonly turnUsd?: number; readonly sessionUsd?: number };
/** `sandbox({ commandTimeoutMs })`: a built-in command's longest run, within the time left in the invocation. */
commandTimeoutMs?: number;
}
export const DEFAULT_MODEL = 'openai/gpt-5.4-mini';
/** Options this harness no longer takes, with what replaced them (docs/workspaces.md §3.4). */
const REMOVED: Readonly<Record<string, string>> = {
sandbox: "`sandbox` is now `sandboxMode` (Codex's own sandbox for its commands; `sandbox()` is the session's environment, `workspace: sandbox()`)",
host: "Codex runs only in the session's Vercel Sandbox VM: pass `workspace: sandbox()` (durable-harness/codex), not a host",
cwd: "the working directory is always /workspace, the session's Drive",
bin: 'the codex binary is the one the sandbox image pins',
env: 'pass non-secret variables with sandbox({ env })',
keepSessionDir: "CODEX_HOME is in the session's VM and is rebuilt from the log every drive",
workspaceNote: "/workspace is the session's Drive, the same on every VM",
};
const SANDBOX_MODES = new Set(['read-only', 'workspace-write', 'danger-full-access']);
/** Codex's command actions that only read: a command made of these alone is `safe` (the note says "safe to repeat"). */
const READ_ACTIONS = new Set(['read', 'listFiles', 'search']);
/** A Codex item's replay policy: a command whose every action reads is `safe`; any other command, and every patch, `never`. */
export function codexItemReplay(item: { type?: string; commandActions?: { type?: string }[] }): ReplayPolicy {
if (item.type !== 'commandExecution') return 'never';
const actions = item.commandActions ?? [];
return actions.length > 0 && actions.every((a) => READ_ACTIONS.has(String(a.type))) ? 'safe' : 'never';
}
/**
* The env var Codex reads its provider key from (`env_key`; it sends it as
* `authorization: Bearer`). It holds the attach's model key
* (`SessionMachine.modelKey`), never a credential: the gateway handler adds
* that in the Function. Hidden from the
* model's commands by `shell_environment_policy` (`HARNESS_*`), and exempt
* from the spawn check's credential-name rule (harness-world-sandbox's
* `PLACEHOLDER_NAMES`), never from its value rules.
*/
export const KEY_ENV = 'HARNESS_CODEX_GATEWAY_KEY';
/** How long the drive waits for a call's `function_call` line before it declines the call: `SYNC` and look again, this many times, with a growing pause. */
const BARRIER_TRIES = 5;
/** What the model is told when a call was declined because its line wasn't in the log yet. */
export const LAGGING_NOTE = "[harness] The previous tool call was not run: the session's log had not caught up with it yet. Nothing happened; make the same call again.";
/** A dynamic tool's result when its call was declined for the same reason. */
export const LAGGING_RESULT = "Not run: the session's log had not caught up with this call yet. Nothing happened; make the same call again.";
/** A command still running this close to the invocation's end is stopped, with a definite outcome (the kernel's 15 s margin, and time to record it). */
const COMMAND_MARGIN_MS = 25_000;
/** The kernel's own bound on an invocation when the World reports no deadline (`DriverOptions.maxInvocationMs`'s default). */
const DEFAULT_INVOCATION_MS = 10 * 60_000;
/** Re-attaches after a lost link within one invocation (the session factory has its own bound too). */
const MAX_RELINKS = 3;
/** The outcome of a command the invocation's deadline stopped. */
export const TIMEOUT_TEXT = 'The command ran out of time in this invocation and was stopped: it may have partly taken effect. Check the state it left before running it again.';
/** The outcome of a command approved too close to the invocation's end: declined, never run. */
export const NO_TIME_TEXT = 'Not run: too little time was left in this invocation to run it and record its outcome. Nothing happened; make the same call again.';
/** Rollout tail failures in one turn before the turn ends with an `Unavailable` instead of another redelivery. */
const MAX_TAIL_FAILURES = 3;
/** The journal stream of response costs: `{ usd, turn, estimated? }` (`turn`: the session turn it was spent in). */
export const SPEND_STREAM = 'codex.spend';
/** The journal stream of the bubblewrap self-test's verdict: `{ sandboxMode, reason? }`, the first one kept. */
export const SANDBOX_STREAM = 'codex.sandbox';
/** The journal stream of rollout tail failures: `{ turn, error }`. */
export const TAIL_STREAM = 'codex.tail';
/** Where app-server's git looks for a repository: nowhere (it would otherwise read `/workspace/.git`'s config, which the model writes, outside bwrap). */
const NO_GIT_DIR = '/nonexistent/dh-no-git';
/** How a Codex turn ended, as the transcript's `turn` entry says it. */
interface TurnOutcome {
status: 'completed' | 'failed' | 'interrupted';
error?: string;
}
interface ActiveTurn {
id?: string;
/** Completed agent messages since the last transcript entry. */
said: string[];
/** Agent messages still streaming (item id to text so far). */
partial: Map<string, string>;
error?: string;
done: (turn: { status?: string; error?: { message?: string } | null }) => void;
completed: Promise<{ status?: string; error?: { message?: string } | null }>;
}
/** A built-in call whose `function_call` line never reached the log: declined, never run. */
const LAGGING = Object.freeze({ run: false, lagging: true } as const);
type Begun = BuiltinStart | typeof LAGGING;
/** The rollout tail failed (the file shrank, was replaced, or is not the session user's): what the drive halts with. */
export class RolloutTailError extends Error {
constructor(cause: unknown) {
super(`harness-world-codex: the rollout can no longer be mirrored (${String((cause as Error)?.message ?? cause).slice(0, 200)}): the drive stops, and the next one rebuilds CODEX_HOME from the log`, { cause });
this.name = 'RolloutTailError';
}
}
/** The stand-in's guard (`test`), the same as harness-world-sandbox's. */
function testGuard(): void {
if (process.env.DH_TEST_LOCAL_SANDBOX !== '1') throw new TypeError('harness-world-codex: codexHarness({ test }) needs DH_TEST_LOCAL_SANDBOX=1 (the stand-in, docs/workspaces.md §5.8)');
if (!process.env.VITEST && process.env.NODE_ENV !== 'test') throw new TypeError('harness-world-codex: codexHarness({ test }) runs only under a test runner');
if (process.env.VERCEL) throw new TypeError('harness-world-codex: codexHarness({ test }) refuses to run with VERCEL set');
if (process.env.NODE_ENV === 'production') throw new TypeError('harness-world-codex: codexHarness({ test }) refuses to run in production');
}
/** When this invocation ends: the World's runtime deadline if it reports one, at most the kernel's default bound away. */
async function invocationDeadline(ctx: DriveContext): Promise<number> {
const reported = await (ctx.world as { getRuntimeDeadline?: () => Promise<Date | undefined> }).getRuntimeDeadline?.().catch(() => undefined);
const cap = Date.now() + DEFAULT_INVOCATION_MS;
const at = reported instanceof Date ? reported.getTime() : NaN;
return Number.isFinite(at) ? Math.min(at, cap) : cap;
}
function sandboxRequired(why: string): Unavailable {
return new Unavailable(`Codex runs only in a Vercel Sandbox: ${why}`, {
code: 'sandbox_required',
hint: "Use durable-harness/codex's codex() or codexHarness() with workspace: sandbox() (the default).",
});
}
/** The session turn the log is in: turns ended (a `turn` entry, or an `unavailable` fact), so the open turn's index. */
function turnIndex(state: SessionState): number {
return transcriptOf(state).filter((r) => r.data?.type === 'turn').length + state.effectOrder.filter((e) => e.name === UNAVAILABLE_STEP).length;
}
/** A `codex.spend` record. */
interface SpendRecord {
usd: number;
turn: number;
estimated?: true;
}
/**
* The dollar ledger, from the log: every response's cost is a `codex.spend`
* record tagged with its session turn, so the session's total and the open
* turn's hold across invocations and redeliveries (docs/workspaces.md §5.5).
* A crash can lose the costs of the last group commit's responses (a
* couple of seconds), never more.
*/
function logLedger(journal: NativeJournal, turn: number): GatewayLedger & { turnEnded(): void; close(): void } {
let session = 0;
let thisTurn = 0;
let open = true;
for (const r of journal.records<SpendRecord>(SPEND_STREAM)) {
const usd = Number(r.data?.usd);
if (!Number.isFinite(usd) || usd < 0) continue;
session += usd;
if (r.data?.turn === turn) thisTurn += usd;
}
return {
spent: () => ({ turnUsd: thisTurn, sessionUsd: session }),
add: (usd, info) => {
thisTurn += usd;
session += usd;
// A late response after the drive let go of the log: counted in memory only.
if (open) journal.append(SPEND_STREAM, { usd, turn, ...(info.estimated ? { estimated: true } : {}) } satisfies SpendRecord);
},
turnEnded: () => {
turn++;
thisTurn = 0;
},
close: () => void (open = false),
};
}
/** What the drives of one invocation share across a re-attach. */
interface Invocation {
/** One app tool at a time, across re-attaches too. */
fg: Mutex;
/** The app's tool calls still running (in the Function): a re-attach waits for them. */
inflight: Set<Promise<unknown>>;
/** The sandbox's notes went to the model in a turn of this invocation (they are not sent again). */
notesShown: boolean;
/** What the notes the model saw described (`SessionMachine.notesKey` of the attach that carried them): only that loss is acknowledged. */
notesKey?: string;
/** And the record was told. */
notesAcknowledged: boolean;
}
export function codexHarness(options: CodexHarnessOptions = {}): Harness {
for (const [key, why] of Object.entries(REMOVED)) {
if ((options as Record<string, unknown>)[key] !== undefined) throw new TypeError(`harness-world-codex: the ${key} option is gone: ${why}`);
}
const builtinsOn = options.builtins ?? false;
if (options.sandboxMode !== undefined) {
if (!SANDBOX_MODES.has(options.sandboxMode)) throw new TypeError(`harness-world-codex: sandboxMode must be 'read-only', 'workspace-write' or 'danger-full-access'`);
if (!builtinsOn) throw new TypeError('harness-world-codex: sandboxMode needs builtins: true (with the built-ins off, Codex runs no command)');
}
checkConfig(options.config);
if (options.test) testGuard();
const model = options.model ?? DEFAULT_MODEL;
const log = options.log ?? (() => {});
const tools = options.tools ?? [];
const spec = options.machine;
const codex = options.test ? { command: options.test.command, args: options.test.args ?? [] } : { command: cliPath(CODEX_CLI), args: [] as string[] };
const network = spec?.network === undefined || spec.network === 'allow-all' ? 'allow-all' : spec.network === 'deny-all' ? 'deny-all' : 'custom';
const budget = { turnUsd: spec?.budget?.turnUsd ?? Infinity, sessionUsd: spec?.budget?.sessionUsd ?? Infinity };
/** Nothing left to do: no turn a crash cut off, nothing in the inbox. */
const idle = (ctx: DriveContext) => !recoveryNote(ctx.state()) && ctx.pending().length === 0;
return {
name: options.name ?? 'codex',
// Each model request reaches Agent Runs from the Gateway proxy (`ModelSteps.request`).
modelCalls: 'self',
async drive(ctx: DriveContext) {
// Nothing to do (an idle wake, a chaos arming): no VM. A model step a
// crash left open after its turn ended is closed (no request will retry it).
if (idle(ctx)) {
await settleAbandonedModelSteps(ctx);
return;
}
if (!spec) throw sandboxRequired('this harness was given no sandbox machine.');
const deadline = await invocationDeadline(ctx);
const invocation: Invocation = { fg: new Mutex(), inflight: new Set(), notesShown: false, notesAcknowledged: false };
// A link to the VM lost mid-drive: the drive is rebuilt from the log on a re-attached machine, in this invocation.
for (let relinks = 0; ; relinks++) {
const held: { lost?: boolean } = {};
try {
return await driveOn(ctx, spec, deadline, invocation, held);
} catch (err) {
const lost = held.lost === true && !ctx.signal.aborted;
if (!lost || relinks >= MAX_RELINKS) throw err;
// The link dropped after the work was done (the last turn's end is in the log): nothing to re-attach for.
await ctx.refresh();
if (idle(ctx)) return;
log('codex: the link to the sandbox was lost; re-attaching', { relinks: relinks + 1, error: String((err as Error)?.message ?? err).slice(0, 200) });
// Notes shown in a turn that never committed may not be in the rollout the thread resumes from: shown again, until a turn that carried them is in the log.
if (!invocation.notesAcknowledged) invocation.notesShown = false;
}
}
},
};
/** One drive on one attach of the session's VM. */
async function driveOn(ctx: DriveContext, spec: CodexMachine, deadline: number, invocation: Invocation, held: { lost?: boolean }): Promise<void> {
const journal = new NativeJournal(ctx);
// A rollout that keeps being tampered with (only possible under
// danger-full-access): the turn ends, instead of a redelivery that boots
// the VM and spends model calls again.
const turnAt = turnIndex(ctx.state());
const tailFailures = journal.records<{ turn?: number }>(TAIL_STREAM).filter((r) => r.data?.turn === turnAt).length;
if (tailFailures >= MAX_TAIL_FAILURES) {
throw new Unavailable(`The turn stopped: Codex's rollout in the session's sandbox was replaced or shrunk ${tailFailures} times, so the log can no longer mirror it.`, {
code: 'unavailable',
hint: 'Something in the VM rewrites $CODEX_HOME (possible only with sandboxMode danger-full-access): run the session with workspace-write.',
});
}
// The handler asks for the credential per request, in this drive's async
// context: on Vercel the OIDC token is read from the invocation's.
const credential = options.credential ?? gatewayCredential;
const inDrive = AsyncLocalStorage.snapshot();
// Every model request, through the handler: the drive's model steps, written around a "Crash during: model" drill.
const steps = new ModelSteps(ctx, { log });
/** The spend cap a model call of this drive was refused for, if one was. */
let capped: 'turnUsd' | 'sessionUsd' | undefined;
const ledger = logLedger(journal, turnAt);
const handler = gatewayHandler({
routes: CODEX_ROUTES,
models: { model },
budget,
ledger,
network,
credential: () => inDrive(credential),
target: options.gatewayUrl ?? gatewayUrl(),
track: steps.request,
onCap: (cap) => {
if (cap !== 'concurrency') capped = cap;
},
// Without the app's own log, the handler's default: a refused model call is a warning in the Function's logs, not silence.
...(options.log ? { log: (message: string, fields?: Record<string, unknown>) => log(`codex gateway: ${message}`, fields) } : {}),
});
let machine: SessionMachine;
try {
machine = await attachInDrive(ctx, spec.acquire, { model: handler, deadline });
} catch (err) {
ledger.close();
await steps.close();
throw err;
}
if (!isBrandedMachine(machine)) {
ledger.close();
await Promise.resolve((machine as { close?: () => Promise<void> } | undefined)?.close?.()).catch(() => {});
await steps.close();
throw sandboxRequired("the machine it was given is not one the sandbox tier's session factory made.");
}
const { dirs, root: cwd } = machine;
const home = dirs.home;
const codexHome = dirs['codex-home'];
// The Drive's manifest (`hw.sandbox-files`): what the Files panel shows, kept in the log so it reads with no VM.
const driveId = ctx.docs?.read(SANDBOX_DOC).value.driveId;
const driveFiles = new DriveFiles(ctx, machine, { ...(typeof driveId === 'string' ? { driveId } : {}), log });
// The session's thread and its rollout, as the log has them. A thread
// whose first line (session_meta) never reached the log cannot be
// resumed: the drive starts a new one. A path the log holds is checked
// again: only a canonical one is ever uploaded or tailed.
const thread = journal.records<ThreadRecord>(THREAD_STREAM).at(-1)?.data;
const restored = thread ? (await journal.read<string>(rolloutStream(thread.threadId))).map((r) => r.data) : [];
const resume = thread && isSessionMeta(restored[0]) && isRolloutPath(thread.path, thread.threadId) ? thread : undefined;
const lines = resume ? restored : [];
const rolloutBytes = new TextEncoder().encode(rolloutText(lines));
// Codex's key: the attach's model key, which the relay checks (its hash armed the relay; only the attach's processes may use it). Never a credential: the handler adds that.
const key = machine.modelKey;
// A `never` tool waits for the journal: its call is in the log before its intent (the barrier ran first, below).
const bridge = toolBridge(ctx, tools, { beforeNever: () => journal.flush() });
/** What the model must hear about this attach (a lost Drive), once per invocation, with its first message; acknowledged after that turn. */
let notes = machine.notes.length && !invocation.notesShown ? machine.notes.map((n) => `[Sandbox] ${n}`).join('\n') : undefined;
let proc: SessionProcess | undefined;
let tail: MachineTail | undefined;
let tailBroken = false;
let poll: ReturnType<typeof setInterval> | undefined;
let builtins: BuiltinCalls | undefined;
const clamps = new Map<string, ReturnType<typeof setTimeout>>();
try {
// The log wins: the VM's cache dirs are wiped, and CODEX_HOME rebuilt from the log.
await machine.files.wipe();
let sandboxMode: NonNullable<CodexHarnessOptions['sandboxMode']> = 'read-only';
if (builtinsOn) sandboxMode = options.sandboxMode ?? (await sandboxVerdict(ctx, spec, machine, journal));
const config = codexConfig({
model,
baseUrl: `${machine.modelUrl}/v1`,
envKey: KEY_ENV,
contextWindow: options.contextWindow ?? 272_000,
cwd,
...(spec.env ? { commandEnv: spec.env } : {}),
extra: {
sandbox_mode: sandboxMode,
...(sandboxMode === 'workspace-write' ? { sandbox_workspace_write: { network_access: true, writable_roots: [cwd, home] } } : {}),
...(builtinsOn ? { features: { ...BUILTINS_ON } } : {}),
...(options.config ?? {}),
},
});
await machine.files.upload('codex-home', [
{ path: 'config.toml', bytes: new TextEncoder().encode(toToml(config)) },
...(resume ? [{ path: resume.path, bytes: rolloutBytes }] : []),
]);
proc = await machine.spawn({
command: codex.command,
args: [...codex.args, 'app-server', '--stdio'],
cwd,
signal: ctx.signal,
env: {
...(spec.env ?? {}),
HOME: home,
CODEX_HOME: codexHome,
TMPDIR: dirs.tmp,
XDG_CONFIG_HOME: `${home}/.config`,
XDG_DATA_HOME: `${home}/.local/share`,
XDG_STATE_HOME: `${home}/.local/state`,
XDG_CACHE_HOME: dirs.cache,
NO_COLOR: '1',
// app-server runs git itself, outside bwrap, with the key in its env:
// no repository (the Drive's `.git/config` is the model's to write,
// `core.fsmonitor` among it) and no global or system config (the
// global one is under HOME, a writable root of the model's commands).
// The commands' own env is the policy's, without these.
GIT_DIR: NO_GIT_DIR,
GIT_CONFIG_GLOBAL: '/dev/null',
GIT_CONFIG_NOSYSTEM: '1',
// Last: the provider key.
[KEY_ENV]: key,
},
// With the built-ins off it runs no command. With them on, the model port's peer check rests on app-server not being
// dumpable (its binary is execute-only in the base), never on a claim.
...(builtinsOn ? {} : { isolatedCommands: true }),
});
const child = proc;
// Everything the drive waits on also ends when the child dies, the link is lost or the drive stops.
const stopped = new Promise<never>((_, reject) => {
const fail = () => reject(ctx.signal.reason ?? new Error('harness-world-codex: the drive stopped'));
if (ctx.signal.aborted) fail();
else ctx.signal.addEventListener('abort', fail, { once: true });
});
const lost = new Promise<never>((_, reject) => {
const fail = () => reject(new Error("harness-world-codex: the link to the session's sandbox was lost"));
if (machine.signal.aborted) fail();
else machine.signal.addEventListener('abort', fail, { once: true });
});
let rpc!: AppServer;
let active: ActiveTurn | undefined;
let turnEnds = 0;
let stream = '';
let threadId = '';
/** The calls whose `function_call` (or `custom_tool_call`) line is in the log: what the barrier looks for. */
const logged = new Set<string>();
for (const line of lines) {
const id = callIdOf(line);
if (id) logged.add(id);
}
/**
* The tail failed: the file was replaced or shrank (tampered with), or
* is not the agent's. Nothing more is mirrored; the failure is in the
* log (a turn ends after `MAX_TAIL_FAILURES`), then the drive stops.
*/
const broken = (err: unknown) => {
if (tailBroken) return;
tailBroken = true;
const error = String((err as Error)?.message ?? err).slice(0, 200);
log('codex: the rollout tail failed: the drive stops', { error });
journal.append(TAIL_STREAM, { turn: turnAt, error });
void journal
.flush()
.catch(() => {})
.finally(() => ctx.halt?.(new RolloutTailError(err)));
};
// The rollout, mirrored line by line: every 2 s while lines arrive
// (group commit keeps that to a batch or so), and at every barrier
// (a tool call, a turn's end), where it is drained (`SYNC`) and flushed.
const mirror = () => {
if (!tail || tailBroken || journal.poisoned || ctx.signal.aborted) return;
let fresh: string[];
try {
fresh = tail.read();
} catch (err) {
if (!machine.signal.aborted) broken(err);
return;
}
for (const line of fresh) {
if (!line) continue;
journal.append(stream, line);
const id = callIdOf(line);
if (id) logged.add(id);
if (endsTurn(line)) turnEnds++;
}
};
/** Drain the tail to the end of the file (`SYNC`), then mirror what came. */
const synced = async () => {
if (!tail) return;
try {
await tail.sync();
} catch (err) {
if (!machine.signal.aborted) broken(err);
throw err;
}
mirror();
};
/**
* The hard barrier before a `never` call (§5.6): its `function_call`
* line in the log, flushed, before its intent. Codex's rollout writer is
* asynchronous, so the line is looked for again after each drain, a
* bounded number of times. False: it never came, and the call must be
* declined (it never ran; the model is told to make it again).
*/
const barrier = async (callId: string): Promise<boolean> => {
for (let i = 0; ; i++) {
await synced();
if (logged.has(callId)) {
await journal.flush();
return true;
}
if (i + 1 >= BARRIER_TRIES) break;
await sleep(50 * 2 ** i);
}
log('codex: declined a call whose function_call line is not in the log', { callId });
return false;
};
/** After a declined call: the model hears it was not run and may try again (a steer into the running turn; in the transcript once Codex took it). */
const retryNote = (callId: string) => {
const t = active;
if (!t?.id || !threadId) return;
void rpc
.request('turn/steer', { threadId, expectedTurnId: t.id, clientUserMessageId: `harness:retry:${callId}`, input: [{ type: 'text', text: LAGGING_NOTE, text_elements: [] }] })
.then(
() => void journal.message({ type: 'message', role: 'user', text: LAGGING_NOTE, synthetic: 'retry' }),
() => {}
);
};
const takeSaid = (t: ActiveTurn | undefined) => {
if (!t) return '';
// The model sometimes repeats a message verbatim (its commentary, then the same as its answer): say it once.
const text = [...t.said, ...t.partial.values()].filter((m, i, all) => !!m && m !== all[i - 1]).join('\n\n');
t.said = [];
t.partial.clear();
return text;
};
// The built-ins: each command and patch an intent before Codex runs it, its outcome when the item completes.
const calls = (builtins = new BuiltinCalls(ctx, {
live: true,
// The barrier ran before `begin` (and declined the call without its line): flush what it mirrored.
beforeNever: async () => {
await journal.flush();
},
}));
const starting = new Map<string, Promise<Begun>>();
/** Items an approval was asked for: at most one approval per item. */
const asked = new Set<string>();
/** And those it was accepted for. */
const accepted = new Set<string>();
/** Commands the deadline stopped: their outcome says so. */
const timedOut = new Set<string>();
/** A running command past the time this invocation has left: its turn is interrupted, and its outcome (when the item completes) says it was stopped. */
const clamp = (id: string, turnId: string | undefined) => {
if (clamps.has(id)) return;
const limit = Math.min(spec.commandTimeoutMs ?? Infinity, deadline - Date.now() - COMMAND_MARGIN_MS);
clamps.set(
id,
setTimeout(
() => {
clamps.delete(id);
// Not running: its call ended, or its approval is pending or was declined (the approval re-arms it when accepted).
if (!calls.has(id) || (asked.has(id) && !accepted.has(id))) return;
const t = active;
// Only its own turn is interrupted: a command that outlived its turn (unified exec) is stopped with the drive.
if (!t?.id || (turnId && t.id !== turnId)) {
log('codex: a command ran out of time after its turn ended: it is stopped with the drive', { callId: id });
return;
}
log('codex: a command ran out of time in this invocation: stopped', { callId: id });
timedOut.add(id);
void rpc.request('turn/interrupt', { threadId, turnId: t.id }).catch((err) => {
timedOut.delete(id);
log('codex: the turn could not be interrupted', { callId: id, error: String((err as Error)?.message ?? err).slice(0, 200) });
});
},
Math.max(0, Math.min(limit, 2 ** 31 - 1))
)
);
};
/** An item of a built-in (a command or a patch): its intent, once, after its `function_call` line is in the log. */
const beginItem = (item: Record<string, any>, turnId?: string): Promise<Begun> | undefined => {
if (!builtinsOn || (item.type !== 'commandExecution' && item.type !== 'fileChange') || typeof item.id !== 'string') return undefined;
let started = starting.get(item.id);
if (!started) {
const callId = item.id;
started = (async (): Promise<Begun> => {
const replay = codexItemReplay(item);
if (replay === 'never' && !ctx.isMemoized(toolKey(callId)) && !(await barrier(callId))) return LAGGING;
const name = item.type === 'commandExecution' ? 'shell' : 'apply_patch';
const text = takeSaid(active);
journal.message({ type: 'message', role: 'assistant', text, toolCalls: [{ id: callId, name }] });
const input = item.type === 'commandExecution' ? { command: item.command, cwd: item.cwd } : { changes: item.changes };
const start = await calls.begin({ callId, name, input, replay });
// Every command that runs is bounded by the invocation, asked about or not (Codex runs a known-safe one without asking).
if (start.run && item.type === 'commandExecution') clamp(callId, turnId);
return start;
})();
started.catch(() => {});
starting.set(item.id, started);
}
return started;
};
const endItem = async (item: Record<string, any>) => {
const started = beginItem(item);
if (!started) return;
const start = await started.catch(() => undefined);
if (!start?.run) return;
clearTimeout(clamps.get(item.id));
clamps.delete(item.id);
// What the call wrote is on the Drive before its outcome is in the log (§5.6 FSYNC); a flush that can't run leaves the call open ("may have half-run").
try {
await machine.fsync();
} catch (err) {
log('codex: the Drive flush failed; the call is left open', { callId: item.id, error: String((err as Error)?.message ?? err).slice(0, 200) });
return;
}
const failed = item.status === 'failed' || item.status === 'declined' || (typeof item.exitCode === 'number' && item.exitCode !== 0);
let text =
item.type === 'commandExecution'
? `${typeof item.exitCode === 'number' ? `exit code ${item.exitCode}\n` : ''}${item.aggregatedOutput ?? ''}`.trim() || `(${item.status ?? 'no output'})`
: (item.changes ?? []).map((c: { path?: string; kind?: { type?: string } }) => `${c.kind?.type ?? 'change'} ${c.path ?? ''}`).join('\n') || `(${item.status ?? 'no changes'})`;
// The deadline stopped it: one writer of its outcome, here, once the item reports it ended.
if (timedOut.has(item.id)) text = `${TIMEOUT_TEXT}\n\n${text}`;
await calls.end(item.id, { text, isError: failed || timedOut.has(item.id) }).catch(() => {});
// The Drive's manifest hears of it: a live event, and the paths a patch named (a command may have changed anything).
const paths = item.type === 'fileChange' ? (item.changes ?? []).map((c: { path?: unknown }) => c.path).filter((p: unknown): p is string => typeof p === 'string') : [];
driveFiles.changed({ tool: item.type === 'commandExecution' ? 'shell' : 'apply_patch', callId: item.id, ...(paths.length ? { paths } : {}) });
};
/** Built-in outcomes being recorded (after their Drive flush): a turn ends only once they are in the log. */
const ending = new Set<Promise<void>>();
const onNotification = (n: RpcNotification) => {
const p = n.params ?? {};
if (n.method === 'item/started' && p.item) void beginItem(p.item, typeof p.turnId === 'string' ? p.turnId : undefined);
else if (n.method === 'item/completed' && p.item) {
const done = endItem(p.item).catch(() => {});
ending.add(done);
void done.finally(() => ending.delete(done));
}
const t = active;
if (n.method === 'item/agentMessage/delta') {
if (t && typeof p.delta === 'string') {
t.partial.set(p.itemId, (t.partial.get(p.itemId) ?? '') + p.delta);
ctx.live({ type: 'text', delta: p.delta });
}
return;
}
if (!t) return;
if (n.method === 'turn/started' && typeof p.turn?.id === 'string') t.id ??= p.turn.id;
else if (n.method === 'item/completed' && p.item?.type === 'agentMessage') {
t.partial.delete(p.item.id);
if (p.item.text) t.said.push(String(p.item.text));
} else if (n.method === 'error' && !p.willRetry) t.error = p.error?.message ?? 'Codex reported an error';
else if (n.method === 'turn/completed' && (!t.id || p.turn?.id === t.id)) t.done(p.turn ?? {});
};
/**
* Why an approval is declined before its intent is looked at, if it is:
* the drive approves one call once, as its intent names it, and nothing
* that would run something else or run it outside bwrap.
*/
const refusal = (p: Record<string, any>): string | undefined => {
// Input to a running command (`write_stdin`), or a shell's sub-command: they share the parent's item id, but are not the call its intent names.
if (p.kind === 'writeStdin') return 'writeStdin';
if (p.approvalId != null) return 'approvalId';
if (typeof p.reason === 'string' && /without (the )?sandbox/i.test(p.reason)) return 'unsandboxed';
// Codex 0.160 asks again for the same item to rerun a command bwrap denied without the sandbox ("command failed; retry without sandbox?").
if (asked.has(String(p.itemId))) return 'repeat';
if (p.networkApprovalContext != null || (Array.isArray(p.proposedNetworkPolicyAmendments) && p.proposedNetworkPolicyAmendments.length > 0)) return 'network';
// More than the sandbox gives (extra permissions, a write root): bwrap is the boundary, not advice.
if (p.additionalPermissions != null) return 'additionalPermissions';
if (p.grantRoot != null) return 'grantRoot';
return undefined;
};
const onRequest = async (req: RpcServerRequest): Promise<unknown> => {
const p = req.params ?? {};
switch (req.method) {
case 'item/tool/call': {
const callId = String(p.callId ?? '');
const tool = bridge.get(String(p.tool));
// A `never` tool's call: its own line in the log first (the barrier), else it is declined, never run.
if (tool?.replay === 'never' && callId && !ctx.isMemoized(toolKey(callId)) && !(await barrier(callId))) {
return { contentItems: [{ type: 'inputText', text: LAGGING_RESULT }], success: false };
}
mirror();
const text = takeSaid(active);
journal.message({ type: 'message', role: 'assistant', text, toolCalls: [{ id: callId, name: String(p.tool) }] });
// Tracked for the invocation: a re-attach waits for it, so the next drive reads its real outcome.
const run = invocation.fg.run(() => bridge.call(String(p.tool), p.arguments ?? {}, callId));
invocation.inflight.add(run);
void run.then(
() => invocation.inflight.delete(run),
() => invocation.inflight.delete(run)
);
const result: ToolResult = await run;
return { contentItems: [{ type: 'inputText', text: result.text }], success: !result.isError };
}
case 'item/tool/requestUserInput':
// In Default mode Codex answers the model's request_user_input
// itself, without asking. Should it ask, no one is here to answer.
return { answers: {} };
case 'item/commandExecution/requestApproval':
case 'item/fileChange/requestApproval': {
// Built-ins off: one slipped past the config; refuse it.
if (!builtinsOn) {
log('codex: declined a built-in tool', { method: req.method, itemId: p.itemId });
return { decision: 'decline' };
}
const itemId = String(p.itemId ?? '');
const why = refusal(p);
asked.add(itemId);
if (why) {
log('codex: declined an approval asking for more than its call', { method: req.method, itemId, escalation: why });
return { decision: 'decline' };
}
const item = { ...p, id: itemId, type: req.method.includes('fileChange') ? 'fileChange' : 'commandExecution' };
const turnId = typeof p.turnId === 'string' ? p.turnId : undefined;
try {
// Too close to the invocation's end to run a command and record its outcome: declined, never run (an intent already written gets that outcome).
if (item.type === 'commandExecution' && deadline - Date.now() < COMMAND_MARGIN_MS) {
const begun = await starting.get(itemId)?.catch(() => undefined);
if (begun && !('lagging' in begun) && begun.run) await calls.end(itemId, { text: NO_TIME_TEXT, isError: true });
log('codex: declined a command too close to the end of the invocation', { itemId });
return { decision: 'decline' };
}
// Approved once its intent is in the log (its item/started came first; else from here).
const start = await (starting.get(itemId) ?? beginItem(item, turnId))!;
if ('lagging' in start) {
retryNote(itemId);
return { decision: 'decline' };
}
if (start.run) {
accepted.add(itemId);
if (item.type === 'commandExecution') clamp(itemId, turnId);
return { decision: 'accept' };
}
log('codex: declined a call the log already has', { itemId });
return { decision: 'decline' };
} catch (err) {
log('codex: declined a call whose intent could not be committed', { itemId, error: String(err) });
return { decision: 'decline' };
}
}
case 'item/permissions/requestApproval':
// More permissions for the rest of the turn: none granted.
log('codex: granted no extra permissions', { itemId: p.itemId });
return { permissions: {}, scope: 'turn' };
case 'execCommandApproval':
case 'applyPatchApproval':
// The same, in the protocol's older names.
log('codex: declined a built-in tool', { method: req.method });
return { decision: { denied: { rejection: 'Built-in tools are off in this session: use the tools you are given.' } } };
case 'mcpServer/elicitation/request':
return { action: 'decline', content: null, _meta: null };
default:
throw Object.assign(new Error(`harness-world-codex does not support ${req.method}`), { code: -32601 });
}
};
rpc = new AppServer(child, {
onNotification,
onRequest,
onNoise: (line) => log('codex: stdout noise', { line: line.slice(0, 200) }),
});
const died = rpc.closed.then((err) => {
throw err;
});
stopped.catch(() => {});
lost.catch(() => {});
died.catch(() => {});
const guard = <T>(p: Promise<T>): Promise<T> => Promise.race([p, stopped, lost, died]);
const init = await guard(rpc.initialize());
const homes = [codexHome, ...(typeof init?.codexHome === 'string' ? [init.codexHome] : [])];
const threadParams = {
model,
cwd,
approvalPolicy: 'untrusted',
approvalsReviewer: 'user',
sandbox: sandboxMode,
...(options.instructions ? { developerInstructions: options.instructions } : {}),
...(options.baseInstructions ? { baseInstructions: options.baseInstructions } : {}),
};
if (resume) {
await guard(rpc.request('thread/resume', { threadId: resume.threadId, ...threadParams, excludeTurns: true }));
threadId = resume.threadId;
tail = await machine.tail('codex-home', resume.path, { offset: rolloutBytes.length });
} else {
const started = await guard(
rpc.request('thread/start', {
...threadParams,
dynamicTools: bridge.tools.map((t) => ({ type: 'function', name: t.name, description: t.description, inputSchema: t.inputSchema })),
})
);
const id = String(started?.thread?.id ?? '');
// The path is the VM's word: the driver tails only the one it would derive itself (no file is no thread to mirror: every later drive would start over).
const path = rolloutPathIn(homes, String(started?.thread?.path ?? ''), id);
threadId = id;
journal.append(THREAD_STREAM, { threadId, path } satisfies ThreadRecord, { id: `thread:${threadId}` });
tail = await machine.tail('codex-home', path, { offset: 0 });
}
stream = rolloutStream(threadId);
poll = setInterval(mirror, 2000);
mirror();
// Steers Codex refused (their turn ended first): each gets the next Codex turn of the same session turn.
const carry: Submission[] = [];
/** One Codex turn, to its end line. The session's turn ends in `turn`. */
const codexTurn = async (text: string, clientId: string): Promise<TurnOutcome> => {
// The invocation's first message carries, once, what the model must hear about the sandbox (its files were lost).
if (notes) {
if (!clientId.startsWith(RECOVER_PREFIX)) journal.message({ type: 'message', role: 'user', text: notes, synthetic: 'environment' });
text = `${notes}\n\n${text}`;
notes = undefined;
invocation.notesShown = true;
invocation.notesKey = machine.notesKey;
}
let done!: ActiveTurn['done'];
const completed = new Promise<Parameters<ActiveTurn['done']>[0]>((r) => (done = r));
const t: ActiveTurn = { said: [], partial: new Map(), done, completed };
active = t;
const endsBefore = turnEnds;
const started = await guard(
rpc.request('turn/start', {
threadId,
clientUserMessageId: clientId,
input: [{ type: 'text', text, text_elements: [] }],
// No environment is no shell and no apply_patch: the built-ins off.
...(builtinsOn ? {} : { environments: [] }),
approvalPolicy: 'untrusted',
...(options.effort ? { effort: options.effort } : {}),
})
);
t.id ??= started?.turn?.id;
const steering = steer(t);
let result: Awaited<ActiveTurn['completed']>;
try {
result = await guard(completed);
} finally {
await steering.stop();
}
// The outcomes of the turn's built-ins first: a turn never ends before them.
await guard(Promise.all([...ending]));
// Its end line (task_complete) follows the notification closely: drained, then looked for.
for (let i = 0; i < 20 && turnEnds <= endsBefore; i++) {
await guard(synced());
if (turnEnds <= endsBefore) await sleep(100);
}
const said = takeSaid(t);
if (said) journal.message({ type: 'message', role: 'assistant', text: said });
active = undefined;
// A model call the budget refused: the turn ends here, saying which cap (no redelivery spends more).
if (capped) {
throw new Unavailable(`The turn stopped: the ${capped === 'turnUsd' ? "turn's" : "session's"} model spend reached its cap (sandbox({ budget: { ${capped} } })).`, {
hint: `Raise sandbox({ budget: { ${capped} } }) to let the session spend more.`,
});
}
const status = result.status === 'completed' ? 'completed' : result.status === 'interrupted' ? 'interrupted' : 'failed';
const error = status === 'completed' ? undefined : (t.error ?? result.error?.message ?? undefined);
return { status, ...(error ? { error } : {}) };
};
const endTurn = async (outcome: TurnOutcome) => {
journal.message({ type: 'turn', ...outcome });
await guard(journal.flush());
ledger.turnEnded();
// The turn that showed the model its notes is in the log: the record stops repeating them. Notes that never reached the model stay owed.
if (invocation.notesShown && !invocation.notesAcknowledged && machine.notes.length) {
await guard(acknowledgeInDrive(ctx, invocation.notesKey));
invocation.notesAcknowledged = true;
}
ctx.live({ type: 'idle' });
// The Drive as the turn left it, for the Files panel (best effort: bounded by the invocation's time left).
await driveFiles.snapshot();
};
/**
* A turn of the session: Codex's turn, then one more Codex turn per
* steer it refused because its turn had just ended. Those steers were
* placed in this turn, so the turn ends only after them: a crash in
* one of them is recovered as part of it.
*/
const turn = async (text: string, clientId: string) => {
let outcome = await codexTurn(text, clientId);
for (let s = carry.shift(); s; s = carry.shift()) outcome = await codexTurn(s.text ?? '', s.id);
await endTurn(outcome);
};
// Steers that arrive while a turn runs go into it (`turn/steer`).
const steer = (t: ActiveTurn) => {
let stopped = false;
let running: Promise<void> | undefined;
const check = () => {
if (stopped || running || !t.id) return;
running = (async () => {
try {
await ctx.refresh();
for (const s of ctx.pending().filter((x) => x.kind === 'steer')) {
if (stopped) return;
await invocation.fg.run(() => placeSubmissions(ctx, [s]));
try {
await rpc.request('turn/steer', { threadId, expectedTurnId: t.id, clientUserMessageId: s.id, input: [{ type: 'text', text: s.text ?? '', text_elements: [] }] });
} catch {
carry.push(s);
}
}
} catch (err) {
// A placement the log refused (a withdraw won it, the drive lost the session): stop.
if (!ctx.signal.aborted) ctx.halt?.(err);
} finally {
running = undefined;
}
})();
};
const off = ctx.onHint?.((h) => {
if (h.eventType === 'hook_received' || h.eventType === 'reconnect') check();
});
const timer = setInterval(check, options.inboxPollMs ?? (off ? 5000 : 500));
return {
async stop() {
stopped = true;
off?.();
clearInterval(timer);
await running;
},
};
};
// An earlier invocation that ended before its snapshot: the manifest catches up first.
await driveFiles.catchUp();
// A turn a crash cut off.
const note = recoveryNote(ctx.state());
if (note) {
const summary = summarizeRollout(lines);
const owed = owedSubmissions(ctx.state(), note.turn.placed, summary.clientIds);
if (resume && summary.dangling.length > 0) {
// Calls the rollout has no output for get theirs from the log.
const items = summary.dangling.map((c) => ({ type: 'function_call_output', call_id: c.callId, output: repairOutput(ctx.state(), c) }));
await guard(rpc.request('thread/inject_items', { threadId, items }));
await guard(synced());
}
// Codex finished the turn (its last turn was this one's, and completed): only the end was not in the log yet.
const finished =
resume &&
summary.idle &&
summary.lastTurn === 'completed' &&
summary.dangling.length === 0 &&
owed.length === 0 &&
[...summary.lastTurnClientIds].some((id) => note.turn.placed.includes(id) || id.startsWith(RECOVER_PREFIX));
if (finished) {
const last = summary.lastAgentMessage;
const said = note.turn.entries.some((e) => e.data.type === 'message' && e.data.role === 'assistant' && e.data.text === last);
if (last && !said) journal.message({ type: 'message', role: 'assistant', text: last });
await endTurn({ status: 'completed' });
} else {
let text = note.text;
if (owed.length) {
text += `\n\nMessages that may not have reached you before the interruption:\n${owed.map((s) => quote(s.text ?? '')).join('\n')}`;
}
const n = transcriptOf(ctx.state()).filter((r) => r.data.type === 'message' && r.data.role === 'user').length;
journal.message({ type: 'message', role: 'user', text: notes ? `${notes}\n\n${text}` : text, synthetic: 'recovery' } satisfies TranscriptEntry);
await placeSubmissions(ctx, note.pending);
// Codex answers again from the interrupted step: what a reader saw of its reply is replaced by what the log kept.
ctx.live(restartEvent(note.turn));
// Its client id names the submissions the note carries: once Codex has recorded it, they are not owed again.
await turn(text, recoveryClientId(n, [...owed, ...note.pending]));
}
}
// The inbox: one submission per turn, steers mid-turn.
for (;;) {
await guard(ctx.refresh());
const [s] = ctx.pending();
if (!s) break;
await placeSubmissions(ctx, [s]);
await turn(s.text ?? '', s.id);
}
// A model step a crash left open that no request of this drive made again: closed.
await guard(steps.settleAbandoned());
// Done: let app-server exit, keep whatever it wrote last (its exit is no failure now).
child.end();
await Promise.race([child.exited, sleep(2000)]);
if (!ctx.signal.aborted && !machine.signal.aborted) {
const live = <T>(p: Promise<T>): Promise<T> => Promise.race([p, stopped, lost]);
await live(synced());
await live(journal.flush());
}
} finally {
if (poll) clearInterval(poll);
for (const t of clamps.values()) clearTimeout(t);
tail?.close();
// Whether the link was lost before this drive let go of it (a re-attach), and what the mirror read meanwhile, in the log.
held.lost = machine.signal.aborted;
// The app's tools still running in the Function (an email confirming) settle first, within the invocation: the next drive reads their real outcome, and no retry runs beside them.
if (invocation.inflight.size && !ctx.signal.aborted) {
let timer: ReturnType<typeof setTimeout> | undefined;
await Promise.race([Promise.allSettled([...invocation.inflight]), new Promise<void>((r) => (timer = setTimeout(r, Math.max(0, deadline - Date.now()))))]);
clearTimeout(timer);
}
if (held.lost && !ctx.signal.aborted) await journal.flush().catch(() => {});
ledger.close();
// No model step outlives the drive.
await steps.close();
// Nothing the model started outlives the drive (§5.4 "Ending an invocation"): the session's processes are killed, the VM and the Drive stay.
if (proc?.alive()) proc.kill('SIGKILL');
await machine.close().catch(() => {});
// A command or patch Codex never reported (it died mid-call, the link
// was lost, or the turn ended with a command still running):
// interrupted, "may or may not have taken effect", after the kill. A
// stopped drive's are left to the kernel.
if (builtins?.size) await builtins.interruptAll('the command was still running when its drive ended: it may or may not have taken effect');
}
}
/**
* Codex's sandbox for its commands, decided once per session: the first
* drive with the built-ins runs bubblewrap's self-test in the VM and
* records its verdict in the log (`codex.sandbox`); every later drive keeps
* it, so nothing the model leaves on the Drive (or a slow VM) can move the
* session to `danger-full-access`. On a later VM where bwrap fails, Codex's
* commands fail closed: its offer to rerun one without the sandbox is
* declined.
*/
async function sandboxVerdict(ctx: DriveContext, spec: CodexMachine, machine: SessionMachine, journal: NativeJournal): Promise<'workspace-write' | 'danger-full-access'> {
const recorded = journal.records<{ sandboxMode?: string }>(SANDBOX_STREAM).find((r) => r.data?.sandboxMode === 'workspace-write' || r.data?.sandboxMode === 'danger-full-access')?.data?.sandboxMode;
if (recorded === 'workspace-write' || recorded === 'danger-full-access') return recorded;
const verdict = await selfTest(ctx, spec, machine);
journal.append(SANDBOX_STREAM, verdict);
return verdict.sandboxMode;
}
/**
* Codex's bubblewrap self-test in the VM (P0.10, §12 question 6): `codex
* sandbox` with the workspace-write profile, as the session's user, in an
* empty directory of its own (the Drive has no say in it). Exit 0:
* `workspace-write`. A definite failure (a non-zero exit): `danger-full-
* access` (the VM is the boundary, and holds no credential). One that
* didn't finish or couldn't start is tried once more, then fails the turn:
* it is no verdict.
*/
async function selfTest(ctx: DriveContext, spec: CodexMachine, machine: SessionMachine): Promise<{ sandboxMode: 'workspace-write' | 'danger-full-access'; reason?: string }> {
// A CODEX_HOME and a working directory of its own, under the cache dir: nothing it writes is near the session's.
const selfHome = 'codex-selftest';
await machine.files.upload('cache', [
{ path: `${selfHome}/home/.keep`, bytes: new Uint8Array() },
{ path: `${selfHome}/cwd/.keep`, bytes: new Uint8Array() },
]);
const dir = `${machine.dirs.cache}/${selfHome}`;
let why = '';
for (let attempt = 1; attempt <= 2; attempt++) {
let test: SessionProcess;
try {
test = await machine.spawn({
command: codex.command,
args: [...codex.args, 'sandbox', '-P', ':workspace', '-C', `${dir}/cwd`, '--', 'true'],
cwd: `${dir}/cwd`,
signal: ctx.signal,
env: { ...(spec.env ?? {}), HOME: machine.dirs.home, CODEX_HOME: `${dir}/home`, TMPDIR: machine.dirs.tmp, NO_COLOR: '1' },
});
} catch (err) {
if (ctx.signal.aborted || machine.signal.aborted) throw err;
why = `it did not start (${String((err as Error)?.message ?? err).slice(0, 120)})`;
continue;
}
test.end();
void (async () => {
for await (const _ of test.lines());
})().catch(() => {});
let timer: ReturnType<typeof setTimeout> | undefined;
const exit = await Promise.race([test.exited, new Promise<undefined>((r) => (timer = setTimeout(r, 20_000, undefined)))]);
clearTimeout(timer);
if (!exit) {
test.kill('SIGKILL');
await test.exited;
}
if (exit?.code === 0) return { sandboxMode: 'workspace-write' };
if (typeof exit?.code === 'number') {
const reason = `bubblewrap's self-test failed (exit code ${exit.code})`;
log("codex: bubblewrap's self-test failed in the VM: commands run with danger-full-access (the VM is the boundary)", { reason, stderr: test.stderrTail().slice(-500) });
return { sandboxMode: 'danger-full-access', reason };
}
why = exit ? `it was killed (${exit.signal})` : 'it did not finish in 20 s';
log("codex: bubblewrap's self-test gave no verdict", { attempt, why });
}
throw new Unavailable(`Codex's sandbox self-test gave no verdict in the session's sandbox: ${why}.`, {
code: 'unavailable',
hint: 'The VM may be slow to start: the next message retries it.',
});
}
}
/** The client id prefix of a recovery note's turn: `recover:<n>`, then `:<id>,<id>` for the submissions it carries. */
export const RECOVER_PREFIX = 'recover:';
/** The client id a recovery note's turn is sent with. */
export function recoveryClientId(n: number, carried: readonly Submission[]): string {
return `${RECOVER_PREFIX}${n}${carried.length ? `:${carried.map((s) => s.id).join(',')}` : ''}`;
}
/**
* Placed submissions of the open turn that Codex never recorded: no user
* message in the rollout has their id as its `client_id`, alone or among
* the ids a recovery note's turn carried. A crash between placing and
* sending, or a steer it accepted but had not written yet.
*/
export function owedSubmissions(state: SessionState, placed: string[], clientIds: Set<string>): Submission[] {
const recorded = new Set<string>();
for (const id of clientIds) {
recorded.add(id);
if (!id.startsWith(RECOVER_PREFIX)) continue;
const at = id.indexOf(':', RECOVER_PREFIX.length);
if (at >= 0) for (const carried of id.slice(at + 1).split(',')) recorded.add(carried);
}
return placed
.filter((id) => !recorded.has(id))
.map((id) => state.submissions.find((s) => s.id === id))
.filter((s): s is Submission => !!s && (s.kind === 'prompt' || s.kind === 'steer' || s.kind === 'followUp'));
}
/**
* The output injected for a call the rollout left dangling, from what the
* log knows: the committed result (exactly once), the recorded failure, or
* that it never started.
*/
export function repairOutput(state: SessionState, call: RolloutCall): string {
const e: EffectRecord | undefined = state.effects.get(toolKey(call.callId));
// Input to a running command (unified exec): that process went with the drive that ran it.
if (!e && call.name === 'write_stdin') return 'Interrupted: the session stopped, and the process this input was for was stopped with it. Check the state its command left before running it again.';
if (!e) return 'Not executed: the session stopped before this call started. Call it again if it is still needed.';
if (e.status === 'completed') return typeof e.output === 'string' ? e.output : JSON.stringify(e.output);
if (e.status === 'failed' && e.error) {
if (e.error.interrupted && e.replay === 'safe') return 'Interrupted before it reported a result. It is safe to repeat.';
return failureText(e.error);
}
return 'Interrupted: the session stopped while this call ran. It may have taken effect: check before repeating it.';
}
const quote = (t: string) =>
t
.split('\n')
.map((l) => `> ${l}`)
.join('\n');
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));