Skip to content

Durable coding-agent harnesses (Claude Code, Codex, OpenCode, Pi, fx) on a Workflow SDK World

15.2 KB
Raw Blame
import { markTime, PROCESS_ID } from 'harness-world/marks';
import type { BenchClient, SendReceipt } from './recorder.ts';

/**
 * eve (`apps/demo/agent`, mounted at `/eve/v1/*` by `withEve`) as a client of
 * the bench and of `/compare`: eve's HTTP session API, with its NDJSON
 * stream translated into the events and marks every other lane emits.
 *
 * eve is its own harness on Vercel Workflow, so it writes none of
 * harness-world's marks. Every eve event carries `meta.at`, the server time it
 * was stamped just before eve wrote it to the session's stream, and the marks
 * here are made from those:
 *
 * | mark                | from                                                          |
 * | ------------------- | ------------------------------------------------------------- |
 * | `model.request`     | `step.started` (AI SDK's `onStepStart`, just before the call) |
 * | `model.first_token` | the step's first text, reasoning or tool-input delta          |
 * | `model.done`        | `actions.requested` (the calls streamed in full), else the    |
 * |                     | step's last output when `step.completed` arrives              |
 * | `tool.start/end`    | `actions.requested` / `action.result`, per call               |
 * | `idle`              | `turn.completed` / `turn.failed` / `turn.cancelled`           |
 * | `submit` (steer)    | the steer POST accepted (this process's clock)                |
 *
 * eve emits `step.started` from AI SDK's `onStepStart` callback and awaits
 * its stream write before the provider request goes out, so the model's
 * "own" time in an eve column includes that one write (stamped before it):
 * eve's overhead per model call reads low by one stream write, not high. All
 * of eve's marks share one pid (`eve`): its steps can run on different
 * instances, whose clocks are NTP-synced but not identical.
 *
 * A prewarmed session (`POST /session` with no message) is answered before
 * its workflow has started its inbox; a message sent before then gets `409
 * session_not_ready` and is retried. That wait is the session starting, which
 * the other lanes pay inside `start()` (their create returns once the run and
 * its hook exist), so `send` reports it (`SendReceipt.notReadyMs`) and the
 * bench and `/compare` start the turn's clock at the attempt eve accepted.
 * A reopened stream connection is reported as `{ type: 'reconnect' }`, as
 * the server lanes' client reports theirs, so it flags the turn the same way.
 */

export interface EveEvent {
  type: string;
  data?: any;
  meta?: { at?: string; id?: string; deliveryIds?: string[] };
  /** Transport control records (`{ $eve: 'stream.lease-ended' }`). */
  $eve?: string;
}

export const EVE_PID = 'eve';
export const EVE_PREFIX = '/eve/v1';

type Out = Record<string, unknown> & { type: string };

/**
 * One subscription's translation of an eve stream (pure: feed it eve events
 * in stream order, get the lane-neutral events back).
 */
export class EveTranslator {
  private step?: { key: string; first: boolean; done: boolean; lastOutputT?: number };
  private placed = new Set<string>();

  /** Steer delivery ids this session sent: the step that first carries one reports it as `placed`. */
  constructor(private readonly steers: ReadonlySet<string> = new Set()) {}

  feed(e: EveEvent): Out[] {
    if (!e || typeof e !== 'object' || e.$eve || typeof e.type !== 'string') return [];
    const d = e.data ?? {};
    const t = e.meta?.at ? Date.parse(e.meta.at) : markTime();
    const mark = (name: string, fields: Record<string, unknown> = {}): Out => ({ type: 'mark', name, t, pid: EVE_PID, runId: 'eve', ...fields });
    const out: Out[] = [];
    const first = (kind: string) => {
      if (this.step && !this.step.first) {
        this.step.first = true;
        out.push(mark('model.first_token', { key: this.step.key, kind }));
      }
      if (this.step) this.step.lastOutputT = t;
    };
    const done = (at = t) => {
      if (this.step && !this.step.done) {
        this.step.done = true;
        // Back-dated to the step's last output when it ends at step.completed:
        // `commitMs` says how much later it was written (as harness-world's
        // end marks do), so a reader does not take the wait for a slow stream.
        out.push({ ...mark('model.done', { key: this.step.key }), t: at, ...(t > at ? { commitMs: Math.round(t - at) } : {}) });
      }
    };
    switch (e.type) {
      case 'step.started': {
        this.step = { key: `${d.turnId ?? 'turn'}:${d.stepIndex ?? 0}`, first: false, done: false };
        const carried = (e.meta?.deliveryIds ?? []).filter((id) => this.steers.has(id) && !this.placed.has(id));
        for (const id of carried) this.placed.add(id);
        out.push(mark('model.request', { key: this.step.key, ...(carried.length ? { placed: carried } : {}) }));
        break;
      }
      case 'message.appended':
        first('text_delta');
        if (typeof d.messageDelta === 'string' && d.messageDelta) out.push({ type: 'text', delta: d.messageDelta });
        break;
      case 'reasoning.appended':
        first('thinking_delta');
        break;
      case 'action.input.appended':
        first('toolcall_delta');
        break;
      case 'message.completed':
        if (this.step) this.step.lastOutputT = t;
        break;
      case 'actions.requested':
        first('toolcall');
        done();
        for (const a of (d.actions ?? []) as { callId?: string; toolName?: string; input?: unknown }[]) {
          out.push(mark('tool.start', { key: a.callId, tool: a.toolName }));
          out.push({ type: 'tool_start', name: a.toolName ?? 'tool', args: a.input ?? {} });
        }
        break;
      case 'action.result': {
        const r = d.result ?? {};
        const isError = d.status !== 'completed' || /error/.test(String(r.kind ?? ''));
        const output = typeof r.output === 'string' ? r.output : r.output === undefined ? undefined : JSON.stringify(r.output);
        out.push(mark('tool.end', { key: r.callId, tool: r.toolName }));
        out.push({ type: 'tool_end', name: r.toolName ?? 'tool', ...(isError ? { isError: true } : {}), ...(output !== undefined ? { output: output.length > 4000 ? `${output.slice(0, 4000)}…` : output } : {}) });
        break;
      }
      case 'step.completed':
        done(this.step?.lastOutputT ?? t);
        this.step = undefined;
        break;
      case 'turn.failed':
      case 'session.failed':
        // `failed`, as the durable lanes report one: the bench fails the sample
        // at once (instead of timing out waiting for output) and /compare shows it.
        out.push({ type: 'failed', message: String(d.message ?? d.code ?? e.type) });
        // The session goes on after a failed turn: the turn has ended.
        if (e.type === 'turn.failed') out.push({ type: 'idle' }, mark('idle'));
        break;
      case 'turn.completed':
      case 'turn.cancelled':
        out.push({ type: 'idle' }, mark('idle'));
        break;
    }
    return out;
  }
}

export interface EveClientOptions {
  /** The app's origin. Empty for same-origin. */
  origin?: string;
  headers?: Record<string, string>;
  fetch?: typeof fetch;
  /** Where eve is mounted. Default `/eve/v1`. */
  prefix?: string;
  /** Retry `409 session_not_ready` for up to this long. Default 20 s. */
  notReadyMs?: number;
  /**
   * Reopen a stream connection that delivers nothing for this long, from the
   * last event it delivered. Default 15 s, eve's own client's read idle
   * timeout: a Workflow stream read can stall with events behind it.
   */
  idleReconnectMs?: number;
}

export interface EveClient extends BenchClient {
  /** Send one message, with eve's default `turnPolicy: 'steer'`. Resolves to its delivery id. */
  message(id: string, text: string): Promise<string | undefined>;
  send(id: string, text: string, kind: 'prompt' | 'steer'): Promise<SendReceipt>;
}

const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

/**
 * eve's session API as a bench client: plain fetch (the /bench and /compare
 * pages and Node alike). Every message, prompt or steer, goes with eve's
 * default `turnPolicy: 'steer'`: on an idle session it starts a turn, on a
 * busy one it lands at the running turn's next model call. Never `queue`,
 * which holds a message back until the running turn ends.
 */
export function eveClient(o: EveClientOptions = {}): EveClient {
  const base = `${(o.origin ?? '').replace(/\/+$/, '')}${o.prefix ?? EVE_PREFIX}`;
  const doFetch: typeof fetch = o.fetch ?? ((input, init) => globalThis.fetch(input, init));
  const idleMs = o.idleReconnectMs ?? 15_000;
  const sessions = new Map<string, { steers: Set<string>; listeners: Set<(e: any) => void> }>();
  const session = (id: string) => {
    let s = sessions.get(id);
    if (!s) sessions.set(id, (s = { steers: new Set(), listeners: new Set() }));
    return s;
  };

  /** POST, retrying `409 session_not_ready`; `notReadyMs`: first attempt → the accepted attempt's start (0 without a retry). */
  async function post(path: string, body: unknown): Promise<{ body: any; notReadyMs: number }> {
    const deadline = Date.now() + (o.notReadyMs ?? 20_000);
    const first = markTime();
    for (let wait = 100; ; wait = Math.min(wait * 2, 1000)) {
      const attempt = markTime();
      const r = await doFetch(base + path, { method: 'POST', headers: { 'content-type': 'application/json', ...o.headers }, body: JSON.stringify(body) });
      if (r.ok) return { body: await r.json().catch(() => ({})), notReadyMs: Math.round(attempt - first) };
      const text = await r.text().catch(() => '');
      // The session's inbox is still starting: eve asks callers to retry.
      if (r.status === 409 && /session_not_ready/.test(text) && Date.now() < deadline) {
        await sleep(wait);
        continue;
      }
      throw new Error(`eve ${path} ${r.status} ${text}`);
    }
  }

  // No `turnPolicy`: eve's default, `steer`.
  const deliver = async (id: string, text: string) => {
    const { body: res, notReadyMs } = await post(`/session/${encodeURIComponent(id)}`, { message: text });
    return { deliveryId: typeof res?.deliveryId === 'string' ? (res.deliveryId as string) : undefined, notReadyMs };
  };
  const message: EveClient['message'] = async (id, text) => (await deliver(id, text)).deliveryId;
  const receipt = (notReadyMs: number): SendReceipt => (notReadyMs > 0 ? { notReadyMs } : {});

  return {
    signalsReady: true,
    message,
    /**
     * Without a prompt this is eve's prewarm, what `useEveAgent({ prewarm: true })`
     * does on mount: `POST /eve/v1/session` with no message creates the
     * durable session (its workflow and inbox) without starting a turn. `/compare`
     * and the bench always start sessions this way and attach the stream before
     * the first send; a send that finds the inbox still starting reports the
     * wait (`notReadyMs`), and the turn's clock starts after it.
     */
    async start(_harness, prompt) {
      const { body: res } = await post('/session', prompt ? { message: prompt } : {});
      if (typeof res?.sessionId !== 'string') throw new Error('eve did not return a sessionId');
      return res.sessionId;
    },
    async send(id, text, kind) {
      const s = session(id);
      // Prompt or steer, the same request (eve's default steer policy); a steer is also tracked to its placement.
      const { deliveryId, notReadyMs } = await deliver(id, text);
      if (kind !== 'steer' || !deliveryId) return receipt(notReadyMs);
      s.steers.add(deliveryId);
      // When the steer was accepted, in this process's clock (the durable lanes' `submit` mark).
      const submit = { type: 'mark', name: 'submit', kind: 'steer', id: deliveryId, t: markTime(), pid: PROCESS_ID, runId: id };
      for (const fn of s.listeners) fn(submit);
      return receipt(notReadyMs);
    },
    live(id, onEvent, onReady) {
      const s = session(id);
      const translator = new EveTranslator(s.steers);
      const abort = new AbortController();
      let index = 0;
      let ready = false;
      let opened = 0;
      s.listeners.add(onEvent);
      (async () => {
        let wait = 250;
        while (!abort.signal.aborted) {
          const before = index;
          // One connection: aborted with the subscription, or when it goes quiet.
          const connection = new AbortController();
          const cut = () => connection.abort();
          abort.signal.addEventListener('abort', cut, { once: true });
          let idle: ReturnType<typeof setTimeout> | undefined;
          const touch = () => {
            clearTimeout(idle);
            idle = setTimeout(cut, idleMs);
          };
          let body: ReadableStream<Uint8Array> | undefined;
          let leaseEnded = false;
          try {
            touch();
            const q = new URLSearchParams({ streamControlVersion: '1', ...(index ? { startIndex: String(index) } : {}) });
            const r = await doFetch(`${base}/session/${encodeURIComponent(id)}/stream?${q}`, {
              headers: { accept: 'application/x-ndjson', ...o.headers },
              cache: 'no-store',
              signal: connection.signal,
            });
            if (!r.ok || !r.body) throw new Error(`eve stream ${r.status}`);
            body = r.body;
            // A re-open (the response ended, went quiet or dropped): said, as the
            // server lanes' client says theirs, so a timing view can tell a
            // reconnect from harness latency and flag the turn alike.
            if (opened++ > 0) onEvent({ type: 'reconnect', t: Date.now(), n: opened - 1 });
            if (!ready) {
              ready = true;
              onReady?.();
            }
            const reader = r.body.pipeThrough(new TextDecoderStream()).getReader();
            let buffer = '';
            for (;;) {
              const { value, done } = await reader.read();
              if (done) break;
              touch();
              buffer += value;
              for (let nl = buffer.indexOf('\n'); nl >= 0; nl = buffer.indexOf('\n')) {
                const line = buffer.slice(0, nl).trim();
                buffer = buffer.slice(nl + 1);
                if (!line) continue;
                let e: EveEvent;
                try {
                  e = JSON.parse(line);
                } catch {
                  continue;
                }
                // Control records are not events (they do not count toward startIndex).
                if (e.$eve) {
                  if (e.$eve === 'stream.lease-ended') leaseEnded = true;
                  continue;
                }
                index++;
                for (const out of translator.feed(e)) onEvent(out);
              }
            }
          } catch {
            // A dropped or silent connection: reopened below from `index`.
          } finally {
            clearTimeout(idle);
            abort.signal.removeEventListener('abort', cut);
            // Release the server's Workflow stream before opening another.
            body?.cancel().catch(() => {});
            connection.abort();
          }
          if (abort.signal.aborted) break;
          // The response ended or went quiet: reopen where it left off (at once
          // when eve said its lease ended or events were flowing).
          wait = leaseEnded ? 0 : index > before ? 250 : Math.min(wait * 2, 2000);
          await sleep(wait);
        }
      })();
      return () => {
        abort.abort();
        s.listeners.delete(onEvent);
      };
    },
  };
}