Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

34.0 KB
Raw Blame
import type {
  CreateEventParams,
  CreateEventRequest,
  Event,
  EventListResponse,
  EventResult,
  World,
} from '@workflow/world';
import { eventIdToSlot, slotToEventId } from '@workflow/world';
import { emptyState, fold } from './fold.ts';
import { covers, inBandFenceSupported, isDefinitiveRefusal, isInBandSuperseded, probeInBandCounters } from './fence.ts';
import { isEpochStep } from './lanes.ts';
import type { DriveFence, SessionState } from './types.ts';

/** A write refused because this log was fenced (a crash is under way in this process). */
export class LogFenced extends Error {
  constructor(reason: string) {
    super(`harness-world: the log is fenced (${reason}); nothing more is written from here`);
    this.name = 'LogFenced';
  }
}

/**
 * An in-band write of this log was refused by the World's in-band writer
 * fence (`InBandSupersededError`): another drive wrote in-band since this
 * log's count was taken, so the drive that armed it no longer owns the
 * session. Nothing of the refused write landed, and every later in-band write
 * of the drive is refused here, before it is sent. Out-of-band writes (inputs)
 * still go. See `SessionLog.arm`.
 */
export class LogSuperseded extends Error {
  constructor(
    /** The count this log wrote at. */
    readonly expected: number,
    /** The World's count when it refused (diagnostic only: never adopted). */
    readonly seqInBand?: number
  ) {
    super(
      `harness-world: an in-band write was refused at count ${expected}${seqInBand !== undefined ? ` (the run is at ${seqInBand})` : ''}: another drive owns the session`
    );
    this.name = 'LogSuperseded';
  }
}

/** What the in-band fence did on a log (logs and marks only). */
export type FenceNote =
  | { kind: 'armed'; count: number }
  | { kind: 'refused'; expected: number; seqInBand?: number }
  | { kind: 'recount'; expected: number; seqInBand: number }
  | { kind: 'legacy'; reason: string };

const now = () => globalThis.performance?.now() ?? Date.now();

/** The docs hook's token (`harness:<runId>:docs`, docs.ts). */
const isDocsToken = (token: unknown) => typeof token === 'string' && token.endsWith(':docs');

/** Steps that record facts about other steps (or drives, or machines), not a driver's progress. */
const FACT_STEPS = new Set(['park', 'settle', 'detach', 'close', 'chaos', 'host']);

/** See `SessionLog.slowWriteProbeMs`. */
export const SLOW_WRITE_PROBE_MS = 2000;
/** Writes a probe can recognise in the log: one per type and step. */
const PROBED_WRITES = new Set(['step_started', 'step_completed', 'step_failed', 'step_retrying']);

export interface WriteInfo {
  eventType: string;
  slot: number | null;
  ms: number;
  bumped: boolean;
}

export interface WriteResult {
  event?: Event;
  foreign: Event[];
  preempted: boolean;
}

/**
 * One reader/writer's view of a run's log: the events it has read, folded.
 *
 * Every write states how much of the log the writer has seen
 * (`eventCount`). The World never rejects a write for being stale; it commits
 * at the next free slot and reports what the writer missed (bump and report).
 * This class folds the missed events back in, so after any write the view is
 * again a prefix of the log, never a prefix with a hole.
 *
 * In-band writer fence (see `arm`). On a World that declares
 * `capabilities.inBandFence`, a drive arms the log it drives: from then on
 * its writes are *in-band* (`inBand: true, expectedSeqInBand`), and the World
 * refuses one whose count is stale, writing nothing. Submissions
 * (`hook_received`) are always *out-of-band* and never fenced, and so is every
 * write of a log no drive has armed (collectors, `sessions.start` and
 * `close`, a delivery deciding whether to drive at all). The count comes only
 * from the World: a refresh's snapshot while unarmed (once the refresh covers
 * every position it counts), or on a World that fences but lists no snapshot
 * the counters of a refused probe followed by such a read (`probe`); then it
 * is advanced by the positions each accepted in-band write allocated.
 */
export class SessionLog {
  readonly state: SessionState;
  private cursor: string | null = null;
  /** Reads and writes fold into one state, so they take turns. */
  private lock: Promise<unknown> = Promise.resolve();
  private readonly observers = new Set<() => void>();
  private readonly foldListeners = new Set<(events: Event[]) => void>();
  private fenced?: { token: string; reason: string };
  /** Whether the World fences in-band writes (`inBandFenceSupported`). */
  private readonly fenceable: boolean;
  /**
   * The World's in-band count as of this fold, when known: the first page's
   * snapshot of a refresh made while unarmed and covering it (or a probe's
   * counters, `probe`), advanced by our own accepted in-band writes.
   * Undefined when the World gave none, or after a write whose outcome is
   * unknown.
   */
  private fenceCount?: number;
  /** Where `fenceCount` came from: a list snapshot, or a probe's counters (a World that lists no snapshot). */
  private countFrom?: 'snapshot' | 'probe';
  /**
   * The last in-band advance that was a guess (`positionsOf`: an accepted
   * born-running intent the World reported no `allocated` for): the count
   * before it and after. A refusal of the very next in-band write that proves
   * nothing else was written (see `inBandFailed`) shows the guess was wrong.
   */
  private guess?: { from: number; to: number };
  /** How many in-band positions this World counts for a born-running intent, once a refusal showed it (see `guess`). */
  private bornRunningPositions?: number;
  /** The drive whose writes are in-band (`arm`). */
  private armedBy?: string;
  /** The count the armed drive's first accepted in-band write was checked against (`fenceOf`). */
  private accepted?: { drv: string; count: number };
  /** Set once the World refused an in-band write of the armed drive. */
  private refused?: LogSuperseded;
  /** What the fence did (logs and marks only; never state). */
  onFence?: (note: FenceNote) => void;
  /**
   * Called after every write the World answered: what committed, at which
   * slot, how long the write took, and whether another writer landed in
   * between. Timing marks only (see marks.ts); never state.
   */
  onWrite?: (info: WriteInfo) => void;

  constructor(
    private readonly world: World,
    readonly runId: string
  ) {
    this.state = emptyState(runId);
    this.fenceable = inBandFenceSupported(world);
  }

  /**
   * Make `drv`'s writes through this log in-band: each carries the World's
   * in-band count, and a write the World refuses as stale (another drive wrote
   * in-band since this count) throws `LogSuperseded`, as does every later
   * in-band write of this log, unsent. Called by a drive once it has decided
   * to drive (deliver step 2), from the count of the load it decided from.
   *
   * Returns false, and leaves every write out-of-band (unfenced, today's
   * behavior), when the World does not declare the fence or gave no count
   * with this log's last refresh (an older World, a run without slots).
   */
  arm(drv: string, count: number | undefined = this.fenceCount): boolean {
    if (!this.fenceable || count === undefined) {
      this.armedBy = undefined;
      if (this.fenceable) this.note({ kind: 'legacy', reason: 'no in-band count from the World on the last read' });
      return false;
    }
    // The count of the view the drive decided from (`count`), not whatever a
    // read made since then adopted: a read that folded another drive's
    // in-band write after the decision would otherwise let the decision
    // through the fence.
    this.fenceCount = count;
    this.armedBy = drv;
    this.refused = undefined;
    this.accepted = undefined;
    this.note({ kind: 'armed', count });
    return true;
  }

  /**
   * Whether this log has no in-band count although its World fences (a World
   * that lists no snapshot, world-vercel today): `probe` can get one.
   */
  get needsProbe(): boolean {
    return this.fenceable && this.fenceCount === undefined && !this.armedBy;
  }

  /**
   * Get an in-band count on a World that fences but lists no snapshot
   * (world-vercel: its backend fences, its list reports no counters): read
   * the World's counters with a write its fence refuses
   * (`probeInBandCounters`), then read the log, and adopt the count once that
   * read covers every slot the counters may count. That is what a snapshot
   * is (counters read under the fence before the listing), so the count is
   * only ever one the World gave for a view this log has folded. Returns
   * whether the log now has a count; without one, drives write unfenced.
   */
  probe(): Promise<boolean> {
    return this.exclusive(async () => {
      if (!this.fenceable || this.armedBy) return this.fenceCount !== undefined;
      const counters = await probeInBandCounters(this.world, this.runId);
      if (!counters) {
        this.note({ kind: 'legacy', reason: 'the World gave no in-band count (no snapshot, no counters on a refusal)' });
        return false;
      }
      const kept = { count: this.fenceCount, from: this.countFrom };
      await this.refreshUnlocked();
      if (this.armedBy) return this.fenceCount !== undefined;
      if (!covers(this.state.eventCount, counters)) {
        this.note({ kind: 'legacy', reason: `the log does not show every position its count includes yet (${this.state.eventCount} of ${counters.seq})` });
        // The count this log had, if the read dropped it: at worst behind, so the World refuses rather than lets a stale write through.
        if (this.fenceCount === undefined && kept.count !== undefined) ({ count: this.fenceCount, from: this.countFrom } = kept);
        return this.fenceCount !== undefined;
      }
      this.fenceCount = counters.seqInBand;
      this.countFrom = 'probe';
      return true;
    });
  }

  /** `drv` writes nothing more in-band through this log (it let go of the session, or handed it to another lane). */
  disarm(drv: string): void {
    if (this.armedBy !== drv) return;
    this.armedBy = undefined;
    this.refused = undefined;
  }

  /** The World's in-band count as this log knows it (undefined when the World gave none, or after an unknown outcome). */
  get inBandCount(): number | undefined {
    return this.fenceCount;
  }

  /**
   * `drv`'s fencing token (`DriveContext.fence`): the World's in-band count
   * its first accepted in-band write since it armed was checked against, so
   * it stays the same for the drive. Exact even where this log guessed how
   * far a write moved the count (the World accepts a write only at its own
   * count). Kept after a refusal or a disarm, until the log is armed again:
   * lower than any write the World accepted since. `fenced` while `drv` is
   * armed and its writes still in-band: not after a write whose outcome is
   * unknown (the rest of the drive writes unfenced), a refusal, or a disarm.
   * Undefined before `drv`'s first accepted in-band write, and on a World
   * without the fence.
   */
  fenceOf(drv: string): DriveFence | undefined {
    if (this.accepted?.drv !== drv) return undefined;
    return { count: this.accepted.count, fenced: this.armedBy === drv && this.fenceCount !== undefined && !this.refused };
  }

  /** The refusal that stopped the armed drive, if the World refused one of its writes. */
  get superseded(): LogSuperseded | undefined {
    return this.refused;
  }

  private note(note: FenceNote): void {
    try {
      this.onFence?.(note);
    } catch {}
  }

  /** Read every event past the ones already folded. */
  refresh(): Promise<Event[]> {
    return this.exclusive(() => this.refreshUnlocked());
  }

  /**
   * Append one event. Returns the committed event, the events *another live
   * driver* landed between our last read and it (`foreign`: effective
   * events that are neither ours nor inbox submissions; a fenced or inert
   * write by a driver that lost the session does not count), and whether one
   * of them was a handoff (`preempted`: the session moved away).
   */
  write(
    request: CreateEventRequest,
    options: { fenceToken?: string; onFolded?: (result: WriteResult) => void; resume?: Required<Pick<CreateEventParams, 'resumeId' | 'resumePayloadDigest'>> } = {}
  ): Promise<WriteResult> {
    return this.exclusive(async () => {
      // Checked when the write's turn comes, not when it was queued: a write
      // waiting behind the lock when the fence went up is refused too.
      if (this.fenced && options.fenceToken !== this.fenced.token) {
        throw new LogFenced(this.fenced.reason);
      }
      const result = await this.writeUnlocked(request, options.resume);
      // Still inside the lock: nothing else folds before the caller has
      // seen this write land (a pipeline drops it from its pending tail here).
      options.onFolded?.(result);
      return result;
    });
  }

  /** Whether the log is fenced (a crash is under way in this process). */
  get isFenced(): boolean {
    return !!this.fenced;
  }

  /**
   * Call `fn` with the events of every fold (a read or a write), in log
   * order, before the fold's observers run. Returns an unsubscribe.
   */
  onFold(fn: (events: Event[]) => void): () => void {
    this.foldListeners.add(fn);
    return () => this.foldListeners.delete(fn);
  }

  /**
   * Refuse every later write that does not carry the returned token. A
   * process that has decided to crash fences its log first, so nothing it
   * was about to write (the next checkpoint, an effect's result) commits
   * between the decision and the kill. Reads still work.
   */
  fence(reason: string): string {
    const token = `fence_${Math.random().toString(36).slice(2)}`;
    this.fenced = { token, reason };
    return token;
  }

  /** Lift a fence (the crash it guarded turned out not to apply). */
  unfence(token: string): void {
    if (this.fenced?.token === token) this.fenced = undefined;
  }

  /**
   * Call `fn` after every fold (a read or a write), e.g. to notice the epoch
   * moving. Returns an unsubscribe.
   */
  observe(fn: () => void): () => void {
    this.observers.add(fn);
    return () => this.observers.delete(fn);
  }

  private folded(events: Event[]): void {
    for (const fn of this.foldListeners) {
      try {
        fn(events);
      } catch {}
    }
    for (const fn of this.observers) {
      try {
        fn();
      } catch {}
    }
  }

  private exclusive<T>(fn: () => Promise<T>): Promise<T> {
    this.queued++;
    const run = this.lock.then(fn, fn).finally(() => {
      this.queued--;
    });
    this.lock = run.catch(() => {});
    return run;
  }

  private queued = 0;

  /** A read or write is in flight or waiting on this log (a refresh now would queue behind it). */
  get busy(): boolean {
    return this.queued > 0;
  }

  private async refreshUnlocked(): Promise<Event[]> {
    const fresh: Event[] = [];
    let hasMore = true;
    let first = true;
    let snapshot: EventListResponse['snapshot'];
    while (hasMore) {
      const page: EventListResponse = await this.world.events.list({
        runId: this.runId,
        pagination: {
          sortOrder: 'asc',
          limit: 1000,
          ...(this.cursor ? { cursor: this.cursor } : {}),
        },
        resolveData: 'all',
      });
      // The first page's snapshot was read before the listing began, so this
      // whole read (on top of the prefix already folded) can cover every
      // position it counts. Adopted below, once the read is folded.
      if (first) snapshot = page.snapshot;
      first = false;
      for (const event of page.data) {
        if (this.isNew(event)) fresh.push(event);
      }
      if (page.cursor) this.cursor = page.cursor;
      hasMore = page.hasMore;
    }
    const effective = new Set<Event>();
    fold(this.state, fresh, (e, ok) => ok && effective.add(e));
    this.lastEffective = effective;
    // Adopted only from a read that completed and covers every position the
    // snapshot counts (a listing can stop short of it: world-vercel's sealed
    // log ends a read at a slot still in flight), so the count never includes
    // an in-band write this log has not folded. A read that gives no count
    // keeps the one this log has: it was the World's for a view this fold
    // contains, and is at worst behind (then the World refuses, never the
    // other way round). And only while no drive writes in-band here: an armed
    // drive's count is its own, and moves only by its own writes (a sibling's
    // in-band write is what the World then refuses ours for).
    if (this.fenceable && !this.armedBy && snapshot && covers(this.state.eventCount, snapshot)) {
      this.fenceCount = snapshot.seqInBand;
      this.countFrom = 'snapshot';
    } else if (this.countFrom === 'probe' && !this.armedBy && fresh.some((e) => e.eventType !== 'hook_received')) {
      // A probed count (a World that lists no snapshot) was the World's for
      // the view it was read with. A read that folds another writer's
      // progress (anything but an input: inputs are never in-band) may show
      // in-band writes it does not count, so a log kept between turns (a
      // route's, whose last turn another process drove) probes again before a
      // drive decides from it. Kept, the World would refuse that drive's
      // first write and the drive would stand down as superseded, its turn
      // left to the queue's backstop.
      this.fenceCount = undefined;
      this.countFrom = undefined;
    }
    this.folded(fresh);
    return fresh;
  }

  /** Which events of the last refresh changed the fold. */
  private lastEffective = new Set<Event>();

  private async writeUnlocked(
    request: CreateEventRequest,
    resume: Required<Pick<CreateEventParams, 'resumeId' | 'resumePayloadDigest'>> | undefined
  ): Promise<WriteResult> {
    // In-band: the armed drive's own progress. Inputs are never fenced: a
    // submission must land whoever is driving, and so must an app's document
    // edit and the docs hook it is written on (a route that shares a log with
    // a drive in this process writes them through it too).
    const input =
      request.eventType === 'hook_received' ||
      (request.eventType === 'hook_created' && isDocsToken((request as { eventData?: { token?: unknown } }).eventData?.token));
    const armed = !input && this.armedBy !== undefined;
    const armedBy = this.armedBy;
    // A stale writer writes nothing more in-band.
    if (armed && this.refused) throw this.refused;
    // Unfenced when the count is unknown (see `inBandFailed`).
    let expected = armed ? this.fenceCount : undefined;
    const expectedSlot = this.state.eventCount + 1;
    const began = now();
    this.writingSince = began;
    let result: EventResult;
    try {
      for (;;) {
        try {
          result = await this.probeWhileSlow(
            this.world.events.create(this.runId, request, {
              eventCount: this.state.eventCount,
              ...resume,
              ...(expected !== undefined ? { inBand: true, expectedSeqInBand: expected } : {}),
            }),
            request
          );
          break;
        } catch (err) {
          if (expected === undefined) throw err;
          // A refusal that only shows this log guessed its own last write's
          // positions wrong: the World's count, and the write again (once).
          const recounted = await this.inBandFailed(err, expected);
          if (recounted === undefined) throw err;
          expected = recounted;
        }
      }
    } finally {
      this.writingSince = undefined;
    }
    if (expected !== undefined) {
      // The drive's first write accepted since it armed, at `expected`: the World's own count, whatever this log guessed before.
      if (armedBy !== undefined && this.accepted?.drv !== armedBy) this.accepted = { drv: armedBy, count: expected };
      const allocated = (result as { allocated?: unknown }).allocated;
      const guessed = typeof allocated !== 'number' && this.isBornRunning(request, result);
      this.fenceCount = expected + (typeof allocated === 'number' ? allocated : this.positionsOf(request, result));
      this.guess = guessed ? { from: expected, to: this.fenceCount } : undefined;
    }
    const event = result.event;
    const slot = event ? eventIdToSlot(event.eventId) : null;
    // A born-running intent (`step_started` carrying the input) on a step the
    // World had not seen commits a synthetic `step_created` first, so it
    // always lands one slot past the one we asked for. When the World says it
    // was this write that created the step (`stepCreated`) and the gap is
    // exactly that one slot, the slot in between is our own `step_created`
    // (it was drawn before the start and after everything we had folded), so
    // nothing foreign landed and there is nothing to re-read. The server does
    // not report skipped slots on `step_started`, so without this every
    // intent of a durable turn paid a full `events.list`.
    const ownCreate = event && slot !== null && slot === expectedSlot + 1 ? bornRunningCreate(request, result, event, expectedSlot) : undefined;
    try {
      this.onWrite?.({ eventType: request.eventType, slot, ms: now() - began, bumped: slot !== expectedSlot && !ownCreate });
    } catch {}
    if (event && ownCreate) {
      const sent = [ownCreate, { ...event, eventData: (request as { eventData: object }).eventData } as Event];
      fold(this.state, sent);
      this.folded(sent);
      return { event, foreign: [], preempted: false };
    }
    if (event && slot === expectedSlot) {
      // Fold what we wrote, not what came back: a World may return the event
      // without its payload (world-vercel stores payloads as refs), and a
      // step_failed folded without its error loses the interrupted result.
      const sent = 'eventData' in request && event.eventType === request.eventType
        ? ({ ...event, eventData: request.eventData } as Event)
        : event;
      fold(this.state, [sent]);
      this.folded([sent]);
      return { event, foreign: [], preempted: false };
    }
    // Someone wrote in between (or the World returned no event). Our view
    // must become a prefix of the log again, including our own write. A
    // World that reports what we skipped (bump and report: `events` holds
    // every slot after the one we claimed to have seen, ending with ours)
    // saves the re-read; otherwise list from our cursor.
    const fresh = this.foldReport(result as { events?: Event[] }, event, request) ?? (await this.refreshUnlocked());
    const ours = (e: Event) =>
      e.eventId === event?.eventId ||
      // A born-running step_started commits a synthetic step_created first.
      (e.eventType === 'step_created' &&
        'correlationId' in request &&
        e.correlationId === request.correlationId);
    // Inputs: submissions, and out-of-band document edits (the docs hook,
    // its creation, and a second writer's conflict on its token). None is
    // another driver's progress.
    const inbox = (e: Event) =>
      (e.eventType === 'hook_received' &&
        (e.correlationId === this.state.inbox?.hookId || (!!this.state.docsHook && e.correlationId === this.state.docsHook.hookId))) ||
      ((e.eventType === 'hook_created' || e.eventType === 'hook_conflict') && isDocsToken((e as { eventData?: { token?: unknown } }).eventData?.token));
    const effective = this.lastEffective;
    // Facts about other steps (a park, a settlement, a crash decision) are
    // written by collectors, not by a competing driver.
    const facts = new Set(this.state.effectOrder.filter((r) => FACT_STEPS.has(r.name)).map((r) => r.stepId));
    const fact = (e: Event) => 'correlationId' in e && !!e.correlationId && facts.has(e.correlationId);
    const foreign = fresh.filter((e) => effective.has(e) && !ours(e) && !inbox(e) && !fact(e));
    const handoffs = new Set(this.state.effectOrder.filter((r) => isEpochStep(r.name)).map((r) => r.stepId));
    const preempted = foreign.some(
      (e) => e.eventType === 'step_created' && handoffs.has(e.correlationId)
    );
    return { event, foreign, preempted };
  }

  /**
   * The in-band positions an accepted write took, on a World that does not
   * report them (`allocated`). world-local reports them. world-postgres
   * counts one per create that appended its event, a born-running intent
   * (two events) included, and lists a snapshot. world-vercel's backend
   * counts one per event it appended, so a born-running intent the World
   * created the step for (`stepCreated`) takes two; it lists no snapshot
   * (this log's count then came from a probe), which is how it is told
   * apart. A World that lists a snapshot and counts per event too (world-vercel
   * once its backend lists one, without `allocated`) makes the guess wrong:
   * the World refuses the drive's next in-band write, never accepts a stale
   * one, and that refusal (with nothing else written) corrects the count
   * (`inBandFailed`), and this log counts a born-running intent as the World
   * does from then on.
   */
  private positionsOf(request: CreateEventRequest, result: EventResult): number {
    if (!result.event) return 0;
    if (!this.isBornRunning(request, result)) return 1;
    return this.bornRunningPositions ?? (this.countFrom === 'probe' ? 2 : 1);
  }

  /** A born-running intent the World created the step for (two events, one create). */
  private isBornRunning(request: CreateEventRequest, result: EventResult): boolean {
    return !!result.event && request.eventType === 'step_started' && (result as { stepCreated?: unknown }).stepCreated === true;
  }

  /**
   * An in-band write threw. A refusal by the fence means another drive wrote
   * in-band since `expected`: fold what it wrote (the caller decides from the
   * log whether the session moved), latch the refusal, tell the observers (the
   * drive's ownership check) and throw `LogSuperseded`. Never retried: the
   * write was decided from a log that is no longer the log.
   *
   * One refusal is not another drive's: the write right after a guessed
   * advance (`guess`: a born-running intent the World reported no
   * `allocated` for), refused with counters whose `seq` is the slot this log
   * has folded up to. Nothing was written since this log's own last write
   * then, by anyone, so the World's `seqInBand` is this drive's own count and
   * only the guess was off: adopt it, learn how this World counts a
   * born-running intent, and return it for the write to be sent again.
   *
   * Any other failure that may have been counted (not a definitive 4xx
   * refusal) leaves the count unknown, so the rest of this drive writes
   * unfenced, as on a World without the fence; the fold's own rules still
   * hold for every write.
   */
  private async inBandFailed(err: unknown, expected: number): Promise<number | undefined> {
    if (isInBandSuperseded(err)) {
      const { seq, seqInBand } = err as { seq?: unknown; seqInBand?: unknown };
      const guess = this.guess;
      this.guess = undefined;
      if (
        guess &&
        guess.to === expected &&
        typeof seq === 'number' &&
        typeof seqInBand === 'number' &&
        seq === this.state.eventCount &&
        seqInBand > guess.from &&
        seqInBand !== expected
      ) {
        this.bornRunningPositions = seqInBand - guess.from;
        this.fenceCount = seqInBand;
        this.note({ kind: 'recount', expected, seqInBand });
        return seqInBand;
      }
      this.refused = new LogSuperseded(expected, typeof seqInBand === 'number' ? seqInBand : undefined);
      // Never ours again: only a read made after this drive lets go may give this log a count.
      this.fenceCount = undefined;
      this.note({ kind: 'refused', expected, ...(this.refused.seqInBand !== undefined ? { seqInBand: this.refused.seqInBand } : {}) });
      await this.refreshUnlocked().catch(() => {});
      for (const fn of this.observers) {
        try {
          fn();
        } catch {}
      }
      throw this.refused;
    }
    this.guess = undefined;
    if (!isDefinitiveRefusal(err)) {
      this.fenceCount = undefined;
      this.note({ kind: 'legacy', reason: `an in-band write's outcome is unknown (${err instanceof Error ? err.message : String(err)})` });
    }
    return undefined;
  }

  /**
   * How long a step write may go unanswered before the log is read to see
   * whether it already committed (0: never). world-vercel's event writes
   * carry no deadline of their own (the WebSocket one is 60s), and a write
   * whose reply is lost holds the whole log, and with it the drive's queue
   * delivery, until it comes back: on production one sat for 126s. A write
   * that stays unanswered past the driver's write deadline makes the drive
   * give its delivery up (driver.ts, `WRITE_STALL_MS`).
   */
  slowWriteProbeMs = SLOW_WRITE_PROBE_MS;

  /** When the write in flight was sent (`performance.now()`), if one is in flight. */
  private writingSince?: number;

  /** How long the write in flight has been waiting for its answer, in ms (0 when none is). */
  get stalledMs(): number {
    return this.writingSince === undefined ? 0 : now() - this.writingSince;
  }

  /**
   * Wait for `pending`; while it is slow, look for the write in the log every
   * `slowWriteProbeMs`. A step event is unique by type and step id, so
   * finding it means it committed: answer as the World would have (the
   * event, with every slot from ours-to-be up to it as a bump report) and
   * leave the hung request behind. Inbox writes (`hook_received`) are not
   * unique, so they only ever wait.
   */
  private async probeWhileSlow(pending: Promise<EventResult>, request: CreateEventRequest): Promise<EventResult> {
    const every = this.slowWriteProbeMs;
    const correlationId = (request as { correlationId?: string }).correlationId;
    if (!(every > 0) || !correlationId || !PROBED_WRITES.has(request.eventType)) return pending;
    let done = false;
    const settled = pending.finally(() => {
      done = true;
    });
    settled.catch(() => {});
    for (;;) {
      let timer: ReturnType<typeof setTimeout> | undefined;
      const tick = new Promise<'tick'>((r) => {
        timer = setTimeout(() => r('tick'), every);
      });
      const first = await Promise.race([settled, tick]).finally(() => clearTimeout(timer));
      if (first !== 'tick') return first as EventResult;
      if (done) return settled;
      const found = await this.findCommitted(request.eventType, correlationId).catch(() => undefined);
      if (done) return settled;
      if (found) {
        try {
          this.onSlowWrite?.({ eventType: request.eventType, slot: eventIdToSlot(found.event.eventId) });
        } catch {}
        return { event: found.event, events: found.events, cursor: null, hasMore: false } as unknown as EventResult;
      }
    }
  }

  /** Called when a slow write was found committed by a probe (marks only). */
  onSlowWrite?: (info: { eventType: string; slot: number | null }) => void;

  /** The events past our fold up to and including the first `type` event of `correlationId`, if it is there. */
  private async findCommitted(type: string, correlationId: string): Promise<{ event: Event; events: Event[] } | undefined> {
    const seen: Event[] = [];
    let cursor = this.cursor;
    for (;;) {
      const page = await this.world.events.list({
        runId: this.runId,
        pagination: { sortOrder: 'asc', limit: 1000, ...(cursor ? { cursor } : {}) },
        resolveData: 'all',
      });
      for (const e of page.data) {
        if (!this.isNew(e)) continue;
        seen.push(e);
        if (e.eventType === type && (e as { correlationId?: string }).correlationId === correlationId) {
          return { event: e, events: seen };
        }
      }
      if (!page.hasMore || !page.cursor) return undefined;
      cursor = page.cursor;
    }
  }

  /**
   * Fold a bump report if it is complete: consecutive slots starting right
   * after what we had seen, ending at our own event. Anything short of that
   * (a World that reports nothing for this event type, a gap) returns
   * undefined and the caller re-lists.
   */
  private foldReport(result: { events?: Event[] }, event: Event | undefined, request: CreateEventRequest): Event[] | undefined {
    const report = result.events;
    if (!event || !Array.isArray(report) || report.length === 0) return undefined;
    const ownSlot = eventIdToSlot(event.eventId);
    if (ownSlot === null) return undefined;
    const withOwn = report.some((e) => e.eventId === event.eventId) ? report : [...report, event];
    const fresh = withOwn.filter((e) => this.isNew(e));
    let expected = this.state.eventCount + 1;
    for (const e of fresh) {
      if (eventIdToSlot(e.eventId) !== expected) return undefined;
      expected++;
    }
    if (expected - 1 !== ownSlot) return undefined;
    // As on the fast path: fold what we wrote for our own event.
    const sent = fresh.map((e) =>
      e.eventId === event.eventId && 'eventData' in request && e.eventType === request.eventType
        ? ({ ...e, eventData: request.eventData } as Event)
        : e
    );
    const effective = new Set<Event>();
    fold(this.state, sent, (e, ok) => ok && effective.add(e));
    this.lastEffective = effective;
    this.folded(sent);
    return sent;
  }

  private isNew(event: Event): boolean {
    const slot = eventIdToSlot(event.eventId);
    return slot === null || slot > this.state.eventCount;
  }
}

/**
 * The `step_created` a born-running `step_started` committed ahead of itself
 * at `slot`, rebuilt from the request, when the World says this write created
 * the step and returned no report. Undefined otherwise.
 */
function bornRunningCreate(request: CreateEventRequest, result: unknown, event: Event, slot: number): Event | undefined {
  if (request.eventType !== 'step_started' || event.eventType !== 'step_started') return undefined;
  const r = result as { stepCreated?: boolean; events?: unknown };
  if (r.stepCreated !== true || Array.isArray(r.events)) return undefined;
  const data = (request as { eventData?: { stepName?: string; input?: unknown } }).eventData;
  const correlationId = (request as { correlationId?: string }).correlationId;
  if (!data || data.input === undefined || typeof data.stepName !== 'string' || correlationId !== event.correlationId) return undefined;
  return {
    ...event,
    eventId: slotToEventId(slot),
    eventType: 'step_created',
    correlationId,
    eventData: { stepName: data.stepName, input: data.input },
  } as Event;
}