Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

25.2 KB
Raw Blame
import { isTerminalEngine, type AgentEngine } from '../engines/types.js'
import type { RegisteredSession } from './registry.js'
import {
  probeTerminalAgents,
  targetForRuntime,
  type DiscoveredTerminalAgent,
  type TerminalAgentProbe,
} from './terminalAgentDiscovery.js'
import type { TerminalBackend } from './terminalBackend.js'
import { mergeTerminalRuntimes, processIdentityKey, sameProcessIdentity, terminalInstanceId, terminalPlacementKey, terminalRouteKey } from './terminalRuntime.js'
import type { TerminalRuntimeRef } from './terminalTypes.js'

const MISS_LIMIT = 2

/**
 * How long one reconcile pass may take before the core stops waiting on it.
 *
 * A pass asks tmux and ps (the probe), then applies what they said inside a registry transaction.
 * Nothing bounded the whole: a probe that never answered, or an apply stuck on an engine's files,
 * held the pass forever, and with it every hook waiting for a pass to bind its agent, every later
 * pass (they are serial), and every registry save (a transaction holds them back). Past this, a probe
 * that has not answered is given up — nothing is applied, every agent kept as it is, and the next pass
 * probes again — whoever waits for the pass goes on without it, and the registry stops holding saves
 * back for it. A pass that is merely slow still finishes, and passes stay one at a time.
 */
export const RECONCILE_PASS_DEADLINE_MS = 30_000

const GIVEN_UP = Symbol('given up')
/** [work]'s value, or GIVEN_UP once [ms] have passed without it. A rejection after that is dropped. */
function withinDeadline<T>(work: Promise<T>, ms: number): Promise<T | typeof GIVEN_UP> {
  let timer: NodeJS.Timeout | undefined
  const deadline = new Promise<typeof GIVEN_UP>((resolve) => {
    timer = setTimeout(() => resolve(GIVEN_UP), ms)
    timer.unref?.()
  })
  return Promise.race([work, deadline]).then((value) => {
    if (value === GIVEN_UP) work.catch(() => {})
    return value
  }).finally(() => clearTimeout(timer))
}

export interface TerminalAgentReconcilerDeps {
  current: () => RegisteredSession[]
  backends: readonly TerminalBackend[]
  backendOrder: readonly string[]
  onDiscovered: (agent: DiscoveredTerminalAgent) => void | Promise<void>
  onObserved: (agent: DiscoveredTerminalAgent, current: RegisteredSession) => void | Promise<void>
  onDormant: (current: RegisteredSession, reason: string) => void | Promise<void>
  onRemoved: (current: RegisteredSession, reason: string) => void | Promise<void>
  onTerminalAvailability?: (current: RegisteredSession, available: boolean) => void | Promise<void>
  onReconciled?: (agents: readonly DiscoveredTerminalAgent[]) => void
  onProbeStatus?: (status: { ready: true; error: string | null }) => void
  transaction?: <T>(apply: () => T | Promise<T>) => Promise<T>
  probe?: (hints: ReadonlyMap<string, AgentEngine>) => Promise<TerminalAgentProbe>
  daemonPid?: number
  /** Hooks may arrive while reboot restoration is still allocating panes. Keep their hints,
   * but do not scan or retire any saved owners until start() opens discovery. */
  deferUntilStart?: boolean
  /** [RECONCILE_PASS_DEADLINE_MS], shorter in tests. */
  passDeadlineMs?: number
}

function currentProcessKey(session: RegisteredSession): string | null {
  return session.processIdentity ? processIdentityKey(session.engine, session.processIdentity) : null
}

function sharesPlacement(
  current: Pick<RegisteredSession, 'runtimes'>,
  observed: Pick<DiscoveredTerminalAgent, 'runtimes'>,
): boolean {
  const placements = new Set(current.runtimes.map(terminalPlacementKey))
  return observed.runtimes.some((runtime) => placements.has(terminalPlacementKey(runtime)))
}

/**
 * A process-backed agent exists before an engine session does. During that interval the terminal route
 * is its stable identity: launchers are allowed to exec/fork into the real native binary while painting
 * a first-run prompt (Claude folder trust is the common case). Once a session binds, process identity is
 * authoritative again so a different process in the same pane cannot inherit an existing transcript.
 */
/**
 * Whether a row may own an observed engine process at its own route. The same engine, or a
 * terminal — a shell somebody typed `claude` into: the process is what the terminal is running now,
 * and the row adopts the engine (cli.ts `onObserved` → `registry.adoptEngine`) rather than a second
 * agent being minted for the same pane.
 */
function routeEngineMatches(current: Pick<RegisteredSession, 'engine'>, observed: Pick<DiscoveredTerminalAgent, 'engine'>): boolean {
  return current.engine === observed.engine || isTerminalEngine(current.engine)
}

/**
 * The observation for a row that cannot be matched by process identity, because it does not have one
 * yet. Its terminal route is the only identity it has.
 *
 * ⚠️ `!sessionId` is NOT the test, and that was a real bug. A RESUMED row keeps the archived session
 * id while `resumePendingAgent` clears its `processIdentity` — so it has an id and no process, and
 * the old test made exactly the row that is waiting to be confirmed invisible to every scan. Nothing
 * else looks at it either: `bindObservedAgent` keys on `byProcess`, and the dormancy branch below
 * skips it because a failed launch already cleared `active`. A resume whose engine never sent a
 * startup hook therefore sat at "Starting" — measured at 19 hours over a pane its owner could type
 * in (openharness#189) — with the heal in cli.ts's `onObserved` (`if (wasLaunching) setLaunch(ready)`)
 * never reached.
 *
 * A row with BOTH an id and a process keeps the stricter rule: process identity is authoritative
 * again, so a different process in the same pane cannot inherit its transcript.
 */
function unboundRouteObservation(
  current: RegisteredSession,
  observed: readonly DiscoveredTerminalAgent[],
): DiscoveredTerminalAgent | undefined {
  if (current.sessionId && current.processIdentity) return undefined
  const matches = observed.filter((candidate) => (
    routeEngineMatches(current, candidate) && sharesPlacement(current, candidate)
  ))
  return matches.length === 1 ? matches[0] : undefined
}

/** Serialized, failure-isolated reconciliation across every enabled backend instance. */
export class TerminalAgentReconciler {
  private readonly misses = new Map<string, number>()
  /** Consecutive scans that found no engine for an agent, counted against the process identity they
   *  missed: an engine identified since is a different engine, and its count starts again. */
  private readonly engineMisses = new Map<string, { processKey: string | null; count: number }>()
  private readonly suppressed = new Set<string>()
  private readonly hints = new Map<string, AgentEngine>()
  /** Terminal routes currently mid an in-place process swap (restart). Held by ROUTE, not by process
   *  identity like `suppressed` — the replacement process's identity is not known until the swap
   *  finishes, so there is nothing to key a process-identity suppression on yet. */
  private readonly heldRoutes = new Set<string>()
  private readonly heldRouteOwners = new Map<string, { timer?: NodeJS.Timeout }>()
  /** When each route was last held or released, as a sequence number, so a probe can tell a route
   *  that changed hands while it ran (see `reconcileOnce`). */
  private routeSeq = 0
  private readonly routeTouched = new Map<string, number>()
  private pending = false
  private inFlight: Promise<void> | null = null
  private timer: NodeJS.Timeout | null = null
  private waitingForStart: boolean

  constructor(private readonly deps: TerminalAgentReconcilerDeps) {
    this.waitingForStart = deps.deferUntilStart === true
  }

  /**
   * Arm the interval FIRST, then run the opening pass.
   *
   * The other way round — await, then schedule — meant a first pass that threw left discovery
   * unscheduled for the life of the daemon: no new agents, no liveness, `discoveryReady` never true,
   * and the caller's own start-up rejected on top of it. Neither is worth one bad probe. The opening
   * pass is reported and dropped; the interval retries it a few seconds later.
   */
  async start(intervalMs: number): Promise<void> {
    this.waitingForStart = false
    this.timer = setInterval(() => { void this.trigger() }, intervalMs)
    this.timer.unref?.()
    await this.trigger().catch((error) => {
      console.warn(`[discovery] first pass failed, retrying on the interval · ${error instanceof Error ? error.message : error}`)
    })
  }

  stop(): void {
    if (this.timer) clearInterval(this.timer)
    this.timer = null
  }

  /** Asks for a pass that knows `engine` is starting in `runtime`'s pane. As [trigger]. */
  triggerHint(runtime: TerminalRuntimeRef, engine: AgentEngine): Promise<boolean> {
    this.hints.set(terminalRouteKey(runtime), engine)
    return this.trigger()
  }

  /** Hide an explicitly deleted process until an authoritative scan proves that process exited. */
  suppress(session: Pick<RegisteredSession, 'engine' | 'processIdentity'>): void {
    if (session.processIdentity) this.suppressed.add(processIdentityKey(session.engine, session.processIdentity))
  }

  /**
   * Hide one terminal route from BOTH halves of reconciliation — the existing-agent liveness/dormant
   * loop and new-process discovery — for the duration of an in-place process swap (restart). Without
   * this, the old process going away can flicker the agent dormant mid-kill, and the replacement
   * process appearing in the same pane before the restart handler rebinds it would otherwise be opened
   * as a brand-new agent (`onDiscovered` mints a fresh `agentId`).
   *
   * A timed hold is useful when there is no operation to await. Pass null for a bounded operation
   * that owns the route through publication and releases it in finally: a timer expiring while its
   * replacement is still being confirmed would let discovery create a second agent for that process.
   */
  holdRoute(routeKey: string, autoReleaseMs: number | null = 30_000): () => void {
    this.heldRoutes.add(routeKey)
    this.routeTouched.set(routeKey, ++this.routeSeq)
    const existing = this.heldRouteOwners.get(routeKey)
    if (existing?.timer) clearTimeout(existing.timer)
    const owner: { timer?: NodeJS.Timeout } = {}
    this.heldRouteOwners.set(routeKey, owner)
    // A cancelled restore cannot release the newer Stop/restart that took this route over.
    const release = () => { if (this.heldRouteOwners.get(routeKey) === owner) this.releaseRoute(routeKey) }
    if (autoReleaseMs !== null) {
      owner.timer = setTimeout(release, autoReleaseMs)
      owner.timer.unref?.()
    }
    return release
  }

  /** Resume normal reconciliation for a route held by `holdRoute`. Idempotent. */
  releaseRoute(routeKey: string): void {
    if (this.heldRoutes.delete(routeKey)) this.routeTouched.set(routeKey, ++this.routeSeq)
    const owner = this.heldRouteOwners.get(routeKey)
    if (owner?.timer) clearTimeout(owner.timer)
    this.heldRouteOwners.delete(routeKey)
  }

  /** Held now, or held or released since the probe numbered `probeSeq` began: that probe cannot speak
   *  for the route. A stop holds its agent's route while it retires the pane, and a probe taken before
   *  the stop, landing after it, otherwise opened a second agent for the engine it saw starting there
   *  (e2e/races.e2e.ts). */
  private routeHeld(runtimes: readonly TerminalRuntimeRef[], probeSeq = this.routeSeq): boolean {
    return runtimes.some((runtime) => {
      const key = terminalRouteKey(runtime)
      return this.heldRoutes.has(key) || (this.routeTouched.get(key) ?? 0) > probeSeq
    })
  }

  /** Asks for a pass, and resolves once it is done: true, or false when the pass outran its deadline
   *  ([RECONCILE_PASS_DEADLINE_MS]) and the caller goes on without what it would have found. */
  trigger(): Promise<boolean> {
    this.pending = true
    // Return promptly to startup hooks: waiting for start() here can hold up the very engines
    // restore is trying to launch. The opening pass consumes every retained hint after restore.
    if (this.waitingForStart) return Promise.resolve(true)
    if (!this.inFlight) this.inFlight = this.drain().finally(() => { this.inFlight = null })
    return this.waitFor(this.inFlight)
  }

  /** A pass, waited for until the pass deadline and no longer: see [RECONCILE_PASS_DEADLINE_MS]. */
  private async waitFor(pass: Promise<void>): Promise<boolean> {
    // Done in time, but on a probe it gave up: it found nothing either.
    if (await withinDeadline(pass, this.passDeadlineMs) !== GIVEN_UP) return !this.probeGivenUp
    if (this.overdue !== pass) {
      this.overdue = pass
      console.warn(`[discovery] a pass has run for ${this.passDeadlineMs} ms; whoever waits for it goes on without it`)
    }
    return false
  }
  private overdue: Promise<void> | null = null
  /** Whether the last pass gave up on its probe. */
  private probeGivenUp = false
  private get passDeadlineMs(): number { return this.deps.passDeadlineMs ?? RECONCILE_PASS_DEADLINE_MS }

  private async drain(): Promise<void> {
    while (this.pending) {
      this.pending = false
      await this.reconcileOnce()
    }
  }

  private async reconcileOnce(): Promise<void> {
    const hints = new Map(this.hints)
    // What every agent was when the probe began. A probe is evidence only about what it could have
    // seen: an agent created, a pane given to it, or an engine identified while the probe ran (the
    // new-pane watcher binds one between two scans) is judged by the next probe, not this one. Four
    // agents created at once proved it: a probe that began before one engine started counted that
    // engine's second "miss", and the agent was retired 22ms after its engine was found
    // (e2e/soak.e2e.ts, 2026-10-04).
    const probedAs = new Map(this.deps.current().map((agent) => [agent.agentId, {
      processKey: currentProcessKey(agent),
      starting: agent.launch?.state === 'starting',
      placements: new Set(agent.runtimes.map(terminalPlacementKey)),
    }]))
    // Routes that change hands from here on are this probe's blind spot. Reconciliation is serial, so
    // anything older matters to no probe still to come.
    const probeSeq = this.routeSeq
    for (const [key, seq] of this.routeTouched) if (seq <= probeSeq) this.routeTouched.delete(key)
    const probe = await withinDeadline(this.deps.probe
      ? this.deps.probe(hints)
      : probeTerminalAgents(
        this.deps.backends,
        this.deps.backendOrder,
        this.deps.daemonPid ?? process.pid,
        hints,
      ), this.passDeadlineMs)
    this.probeGivenUp = probe === GIVEN_UP
    if (probe === GIVEN_UP) {
      console.warn(`[discovery] the terminal probe has not answered in ${this.passDeadlineMs} ms; this pass is given up, every agent kept as it is`)
      return
    }
    const availableTargets = probe.targets.filter((target) => target.result.state === 'available')
    const livePlacements = new Set(availableTargets.flatMap((target) =>
      target.result.state === 'available'
        ? target.result.roots.map((root) => terminalPlacementKey(root.runtime))
        : []))
    const probeError = !probe.processTableAvailable
      ? 'process table unavailable'
      : probe.targets.length > 0 && availableTargets.length === 0
        ? probe.targets.map((target) => `${target.instanceId}: ${target.result.state === 'available' ? 'available' : target.result.reason}`).join('; ')
        : null
    // Terminal placement liveness does not depend on finding an engine process. This is what keeps a
    // retained trust/setup/shell pane visible after restart, including when `ps` itself is unavailable.
    const markVerifiedPlacements = async (): Promise<void> => {
      for (const current of this.deps.current()) {
        if (current.runtimes.some((runtime) => livePlacements.has(terminalPlacementKey(runtime)))) {
          await this.deps.onTerminalAvailability?.(current, true)
        }
      }
    }
    if (this.deps.transaction) await this.deps.transaction(markVerifiedPlacements)
    else await markVerifiedPlacements()
    if (!probe.processTableAvailable) {
      console.warn('[discovery] process table unavailable; keeping existing terminal agents')
      this.deps.onProbeStatus?.({ ready: true, error: probeError })
      return
    }
    this.hints.clear()

    const observedKeys = new Set(probe.agents.map((agent) => processIdentityKey(agent.engine, agent.processIdentity)))
    for (const key of [...this.suppressed]) if (!observedKeys.has(key)) this.suppressed.delete(key)
    probe.agents = probe.agents.filter((agent) => !this.suppressed.has(processIdentityKey(agent.engine, agent.processIdentity)))

    const apply = async (): Promise<void> => {
      const before = this.deps.current()
      const observedByProcess = new Map(probe.agents.map((agent) => [
        processIdentityKey(agent.engine, agent.processIdentity),
        agent,
      ]))
      const matchedProcesses = new Set<string>()

      // Refresh existing process identities first. New route owners are opened afterwards so split/merge
      // conflict handling never depends on backend probe completion order.
      for (const current of before) {
        // A restart in progress on this route: leave it untouched. The old process going dormant here
        // and the new one being adopted by the discovery loop below are both races restart's `holdRoute`
        // exists to prevent — see the class-level comment on `heldRoutes`.
        if (this.routeHeld(current.runtimes, probeSeq)) continue
        const processKey = currentProcessKey(current)
        const observed = (processKey ? observedByProcess.get(processKey) : undefined)
          // A row saved before start ticks is keyed by its marker, the same process observed with them
          // by its ticks: match it here, or the pass takes it for a new process (ProcessIdentity.startTicks).
          ?? probe.agents.find((agent) => agent.engine === current.engine
            && sameProcessIdentity(agent.processIdentity, current.processIdentity))
          ?? unboundRouteObservation(current, probe.agents)
        if (observed) matchedProcesses.add(processIdentityKey(observed.engine, observed.processIdentity))

        let nextRuntimes = current.runtimes
        let terminalVerified = current.runtimes.some((runtime) => livePlacements.has(terminalPlacementKey(runtime)))
        for (const runtime of current.runtimes) {
          const target = targetForRuntime(probe, runtime)
          if (!target || target.result.state !== 'available') continue
          const placement = terminalPlacementKey(runtime)
          const replacement = observed?.runtimes.find((candidate) => terminalPlacementKey(candidate) === placement)
          const missKey = `${current.agentId}\u0000${placement}`
          // Inventory is the authority for terminal existence. A live pane without a recognized engine
          // is a dormant but still viewable agent, not a missing runtime.
          if (livePlacements.has(placement)) {
            this.misses.delete(missKey)
            terminalVerified = true
            if (replacement) nextRuntimes = mergeTerminalRuntimes(nextRuntimes, [replacement])
            continue
          }
          if (probe.ambiguousPlacements.has(placement)) continue
          if (replacement) {
            this.misses.delete(missKey)
            nextRuntimes = mergeTerminalRuntimes(nextRuntimes, [replacement])
            terminalVerified = true
            continue
          }
          // The aggregate inventory/process snapshot is deliberately cheap, but it is not stronger
          // than a backend-specific check of this exact runtime and saved process identity. New-agent
          // creation already proved the pane this way; without the same fallback here, one snapshot
          // miss hid that freshly adopted agent on the very next reconciliation cycle even while its
          // Claude trust prompt remained alive in tmux.
          const backend = this.deps.backends.find((candidate) => candidate.instanceId === target.instanceId)
          if (backend && current.processIdentity) {
            const validation = await backend.validate(runtime, {
              engine: current.engine,
              // This check answers only whether the terminal placement still exists. Engine/process
              // identity is reconciled independently from the aggregate process snapshot below.
              processIdentity: undefined,
            }).catch((error: unknown) => ({
              state: 'unknown' as const,
              reason: error instanceof Error ? error.message : 'terminal validation failed',
            }))
            if (validation.state === 'alive') {
              this.misses.delete(missKey)
              terminalVerified = true
              continue
            }
            // A timeout/unreadable backend is not evidence that the process exited. Preserve both the
            // route and its active UI entry, just as a failed whole-process-table read does above.
            if (validation.state === 'unknown') continue
          }
          if (!probedAs.get(current.agentId)?.placements.has(placement)) continue
          const misses = (this.misses.get(missKey) ?? 0) + 1
          if (misses < MISS_LIMIT) {
            this.misses.set(missKey, misses)
            continue
          }
          this.misses.delete(missKey)
          nextRuntimes = nextRuntimes.filter((candidate) => terminalPlacementKey(candidate) !== placement)
        }

        if (terminalVerified) await this.deps.onTerminalAvailability?.(current, true)
        const probed = probedAs.get(current.agentId)
        if (observed || (processKey === null && (current.launch?.state === 'starting' || probed?.starting))) {
          // Before the first engine exists, two quick scans prove no exit. The
          // startup watcher owns that decision; discovery still checks the pane.
          // A pre-start snapshot cannot count after readiness changes mid-probe.
          this.engineMisses.delete(current.agentId)
        } else if (probed && probed.processKey === processKey
          && current.active && terminalVerified && !isTerminalEngine(current.engine) && !current.runtimes.some((runtime) =>
          probe.ambiguousPlacements.has(terminalPlacementKey(runtime)))) {
          // A live pane with no engine process in it. For an agent that is a dormant engine; for a
          // terminal (`engine === 'terminal'`) it is simply a shell at its prompt, which is why the
          // branch is skipped for one. A terminal that ADOPTED an engine (`terminalHost`, engine no
          // longer `terminal`) does come through here when that engine exits — the handler turns it
          // back into a terminal rather than marking it dormant (cli.ts `onDormant`).
          const prior = this.engineMisses.get(current.agentId)
          const misses = (prior?.processKey === processKey ? prior.count : 0) + 1
          if (misses >= MISS_LIMIT) {
            this.engineMisses.delete(current.agentId)
            await this.deps.onDormant(current, `engine process absent after ${MISS_LIMIT} confirmed scans`)
          } else {
            this.engineMisses.set(current.agentId, { processKey, count: misses })
          }
        }

        if (observed) {
          const availableInstances = new Set(probe.targets
            .filter((target) => target.result.state === 'available')
            .map((target) => target.instanceId))
          const unknownRuntimes = current.runtimes.filter((runtime) => !availableInstances.has(
            terminalInstanceId(runtime),
          ))
          const merged: DiscoveredTerminalAgent = {
            ...observed,
            runtimes: mergeTerminalRuntimes(unknownRuntimes, [...nextRuntimes, ...observed.runtimes]),
          }
          await this.deps.onObserved(merged, current)
        } else if (nextRuntimes.length < current.runtimes.length) {
          if (nextRuntimes.length === 0) {
            await this.deps.onTerminalAvailability?.(current, false)
            await this.deps.onRemoved(current, `terminal runtime absent after ${MISS_LIMIT} confirmed scans`)
          } else {
            await this.deps.onObserved({
              engine: current.engine,
              cwd: current.cwd ?? '',
              processIdentity: current.processIdentity!,
              args: current.processIdentity?.executable ?? '',
              resumeSessionId: null,
              runtimes: nextRuntimes,
              primaryRuntimeKey: nextRuntimes.some((runtime) => terminalRouteKey(runtime) === current.primaryRuntimeKey)
                ? current.primaryRuntimeKey
                : terminalRouteKey(nextRuntimes[0]),
            }, current)
          }
        }
      }

      for (const observed of probe.agents) {
        const key = processIdentityKey(observed.engine, observed.processIdentity)
        if (matchedProcesses.has(key)) continue
        // The replacement process for a restart in progress: the restart handler will bind it via
        // `updateProcessIdentity` itself once confirmed, not through ordinary discovery.
        if (this.routeHeld(observed.runtimes, probeSeq)) continue
        await this.deps.onDiscovered(observed)
      }
    }

    if (this.deps.transaction) await this.deps.transaction(apply)
    else await apply()
    // Readiness is published last: clients must never observe ready=true between the inventory read and
    // the authoritative availability/registry update.
    this.deps.onProbeStatus?.({ ready: true, error: probeError })
    this.deps.onReconciled?.(probe.agents.filter(agent => !this.routeHeld(agent.runtimes, probeSeq)))
  }
}