Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

18.6 KB
Raw Blame
import type { AgentEngine } from '../engines/types.js'
import type { RegisteredSession } from './registry.js'
import { TERMINAL_UNAVAILABLE_CODE, type TerminalBackend, type TerminalBackendPrepared } from './terminalBackend.js'
import { processIdentityKey, terminalInstanceId, terminalPlacementKey, terminalRouteKey } from './terminalRuntime.js'
import {
  terminalActionNotStarted,
  type RuntimeValidation,
  type SubmitOptions,
  type TerminalActionResult,
  type TerminalCaptureOptions,
  type TerminalLogicalKey,
  type TerminalReadResult,
  type TerminalRuntimeRef,
  type TerminalStreamHandle,
  type TerminalStreamSink,
  type TerminalStreamSize,
} from './terminalTypes.js'

/**
 * Ready every configured backend at once (`TerminalBackend.prepare`), each handed its own panes. One whose prepare
 * throws is unavailable, as one that says so is: a backend that cannot start never stops the daemon or the others.
 */
export function prepareTerminalBackends<B extends TerminalBackend>(
  backends: readonly B[],
  ownedPanes: (backend: B) => ReadonlyMap<string, AgentEngine>,
): Promise<Array<{ backend: B; result: TerminalBackendPrepared }>> {
  return Promise.all(backends.map(async (backend) => {
    try {
      return { backend, result: await backend.prepare({ ownedPanes: ownedPanes(backend) }) }
    } catch (error) {
      const reason = `could not prepare ${backend.name}: ${error instanceof Error ? error.message : String(error)}`
      console.error(`[terminal] ${backend.name}: ${reason}`)
      return { backend, result: { state: 'unavailable' as const, code: TERMINAL_UNAVAILABLE_CODE[backend.name], reason } }
    }
  }))
}

export interface TerminalControlLease {
  agentId: string
  runtime: TerminalRuntimeRef
  placementKey: string
  generation: string
}

function primaryPlacement(session: Pick<RegisteredSession, 'runtimes' | 'primaryRuntimeKey'>): string {
  const primary = session.runtimes.find((runtime) => terminalRouteKey(runtime) === session.primaryRuntimeKey)
  return primary ? terminalPlacementKey(primary) : ''
}

function runtimeGeneration(session: Pick<RegisteredSession, 'engine' | 'processIdentity' | 'runtimes' | 'primaryRuntimeKey'>): string {
  const process = session.processIdentity ? processIdentityKey(session.engine, session.processIdentity) : ''
  return `${process}\u0000${primaryPlacement(session)}\u0000${session.runtimes.map(terminalPlacementKey).sort().join('\u0001')}`
}

const LEGACY_KEYS: Record<string, TerminalLogicalKey> = {
  Enter: 'enter', Escape: 'escape', Tab: 'tab', BTab: 'backtab', Up: 'up', Down: 'down', Left: 'left', Right: 'right',
  Home: 'home', End: 'end', BSpace: 'backspace', DC: 'delete', PPage: 'pageup', NPage: 'pagedown',
  'C-c': 'ctrl-c', 'C-d': 'ctrl-d', 'C-u': 'ctrl-u', 'C-w': 'ctrl-w', Space: 'space',
  '0': '0', '1': '1', '2': '2', '3': '3', '4': '4', '5': '5', '6': '6', '7': '7', '8': '8', '9': '9',
}

export function terminalLogicalKey(key: string): TerminalLogicalKey | null {
  return LEGACY_KEYS[key] ?? null
}

/** How long to wait before asking a runtime again when its validation could not answer: about 8 s in
 *  all, the longest a held event loop is expected to keep a probe's answer unread (e2e/stall.e2e.ts). */
export const UNKNOWN_RETRY_MS: readonly number[] = [250, 500, 1_000, 2_000, 4_000]

export interface TerminalBackendCoordinatorOptions {
  unknownRetryMs?: readonly number[]
  sleep?: (ms: number) => Promise<void>
}

const defaultSleep = (ms: number): Promise<void> => new Promise((resolve) => {
  const timer = setTimeout(resolve, ms)
  timer.unref?.()
})

/** Resolves all terminal I/O through validated backend-scoped locators. */
export class TerminalBackendCoordinator {
  private readonly byInstance = new Map<string, TerminalBackend>()
  /** Backends added after construction (a guest's host, reached on demand): kept across `replaceBackends`, which
   *  re-lists the configured hosts and must not drop them. */
  private readonly added = new Set<TerminalBackend>()
  private readonly unknownRetryMs: readonly number[]
  private readonly sleep: (ms: number) => Promise<void>
  private changing: (agentId: string) => boolean = () => false

  constructor(
    backends: readonly TerminalBackend[],
    private readonly backendOrder: readonly string[],
    options: TerminalBackendCoordinatorOptions = {},
  ) {
    this.unknownRetryMs = options.unknownRetryMs ?? UNKNOWN_RETRY_MS
    this.sleep = options.sleep ?? defaultSleep
    this.replaceBackends(backends)
  }

  /** Whether a lifecycle operation (restart, retarget, resume) is replacing an agent's engine right now.
   *  Set by cli.ts once those exist: they are made long after this coordinator. */
  whileChanging(test: (agentId: string) => boolean): void {
    this.changing = test
  }

  /**
   * One runtime's validation, asked again while it cannot answer yet. Every caller gives up on anything
   * but `alive`, and three answers are only "not yet":
   *
   * - `unknown`, a probe that failed: tmux or `ps` timed out, most often because the core's own event
   *   loop was held and woke to the deadline before it read the answer (e2e/stall.e2e.ts). A held loop
   *   used to drop the message it was about to type into an engine that was fine ("This agent process
   *   is no longer running.") and fail the lease a question's answer was typed through.
   * - `replaced`, the engine in the pane is not the process the row recorded: what a restart looks like
   *   between the new engine starting and the restart recording it. A message sent then, by a client
   *   that saw the agent still active, was dropped the same way (e2e/races.e2e.ts, a restart nobody
   *   waited for).
   * - any `gone` while a lifecycle operation is replacing the engine: between the old engine's exit and
   *   the new one's start there is no engine in the pane at all. Found by QA on a quiet machine:
   *   restore has the same gap while its launch is `starting`, before the engine process exists.
   *
   * The row is read again on every ask: `session` is the registry's own, and a restart records the new
   * process on it.
   */
  private async validateRuntime(backend: TerminalBackend, runtime: TerminalRuntimeRef, session: RegisteredSession): Promise<RuntimeValidation> {
    const ask = () => backend.validate(runtime, { engine: session.engine, processIdentity: session.processIdentity ?? undefined })
    let result = await ask()
    for (const ms of this.unknownRetryMs) {
      if (result.state === 'alive' || (result.state === 'gone' && !result.replaced
        && session.launch?.state !== 'starting' && !this.changing(session.agentId))) return result
      await this.sleep(ms)
      result = await ask()
    }
    return result
  }

  replaceBackends(backends: readonly TerminalBackend[]): void {
    this.byInstance.clear()
    for (const backend of [...backends, ...this.added]) this.byInstance.set(backend.instanceId, backend)
  }

  /**
   * A backend reached after this daemon started (a guest's own host on Windows): its panes are routed and validated
   * as the configured hosts' are, and it stays listed through any `replaceBackends`.
   */
  addBackend(backend: TerminalBackend): void {
    this.added.add(backend)
    this.byInstance.set(backend.instanceId, backend)
  }

  /** The backend is gone for good (its guest was removed): its panes are no longer this daemon's to route. */
  removeBackend(backend: TerminalBackend): void {
    this.added.delete(backend)
    if (this.byInstance.get(backend.instanceId) === backend) this.byInstance.delete(backend.instanceId)
  }

  instances(): TerminalBackend[] {
    return [...this.byInstance.values()]
  }

  async titles(): Promise<Map<string, string>> {
    const snapshots = await Promise.all(this.instances().map((backend) =>
      backend.titles().catch(() => ({ state: 'failed' as const, reason: 'terminal title snapshot failed' }))))
    const titles = new Map<string, string>()
    for (const snapshot of snapshots) {
      if (snapshot.state !== 'succeeded') continue
      for (const [route, title] of snapshot.value) titles.set(route, title)
    }
    return titles
  }

  titleFor(session: RegisteredSession, titles: ReadonlyMap<string, string>): string | undefined {
    for (const runtime of this.orderedRuntimes(session)) {
      const title = titles.get(terminalRouteKey(runtime))
      if (title) return title
    }
    return undefined
  }

  backendFor(runtime: TerminalRuntimeRef): TerminalBackend | undefined {
    return this.byInstance.get(terminalInstanceId(runtime))
  }

  private orderedRuntimes(session: RegisteredSession): TerminalRuntimeRef[] {
    const configured = session.runtimes.filter((runtime) => this.backendFor(runtime))
    return configured.toSorted((a, b) => {
      const aPrimary = terminalRouteKey(a) === session.primaryRuntimeKey
      const bPrimary = terminalRouteKey(b) === session.primaryRuntimeKey
      if (aPrimary !== bPrimary) return aPrimary ? -1 : 1
      const ab = this.backendOrder.indexOf(a.backend)
      const bb = this.backendOrder.indexOf(b.backend)
      if (ab !== bb) return (ab < 0 ? Number.MAX_SAFE_INTEGER : ab) - (bb < 0 ? Number.MAX_SAFE_INTEGER : bb)
      return terminalRouteKey(a).localeCompare(terminalRouteKey(b))
    })
  }

  async validate(session: RegisteredSession): Promise<RuntimeValidation> {
    if (!session.active) return { state: 'gone', reason: 'terminal agent is dormant' }
    let unknown: RuntimeValidation | null = null
    for (const runtime of this.orderedRuntimes(session)) {
      const result = await this.validateRuntime(this.backendFor(runtime)!, runtime, session)
      if (result.state === 'alive') return result
      if (result.state === 'unknown') unknown = result
    }
    return unknown ?? { state: 'gone', reason: 'no configured terminal runtime is alive' }
  }

  async acquireLease(session: RegisteredSession): Promise<TerminalReadResult<TerminalControlLease>> {
    if (!session.active) return { state: 'failed', reason: 'terminal agent is dormant' }
    for (const runtime of this.orderedRuntimes(session)) {
      const validation = await this.validateRuntime(this.backendFor(runtime)!, runtime, session)
      if (validation.state === 'alive') {
        return {
          state: 'succeeded',
          value: {
            agentId: session.agentId,
            runtime,
            placementKey: terminalPlacementKey(runtime),
            generation: runtimeGeneration(session),
          },
        }
      }
    }
    return { state: 'failed', reason: 'no terminal runtime could be leased' }
  }

  leaseIsCurrent(lease: TerminalControlLease, session: RegisteredSession): boolean {
    if (lease.agentId !== session.agentId || lease.generation !== runtimeGeneration(session)) return false
    const current = session.runtimes.find((runtime) => terminalPlacementKey(runtime) === lease.placementKey)
    if (!current) return false
    lease.runtime = current
    return true
  }

  async validateLease(lease: TerminalControlLease, session: RegisteredSession): Promise<boolean> {
    if (!this.leaseIsCurrent(lease, session)) return false
    const backend = this.backendFor(lease.runtime)
    if (!backend) return false
    // Asked again after the check: it waits while a restart records its new engine, and then answers
    // `alive` for THAT engine, which this lease was not taken for. Kept, a question's answer went on
    // being keyed through it into the new engine's composer.
    return (await this.validateRuntime(backend, lease.runtime, session)).state === 'alive' && this.leaseIsCurrent(lease, session)
  }

  async capture(session: RegisteredSession, options?: TerminalCaptureOptions): Promise<TerminalReadResult<string>> {
    if (!session.active) return { state: 'failed', reason: 'terminal agent is dormant' }
    return this.captureRetained(session, options)
  }

  /** Save a retained pane even when no engine is active (for example, a shell
   * after a daemon restart). This is read-only: control leases and activity
   * reads keep their active-process guards. Close checks identity again before
   * stopping anything and still requires a successful durable checkpoint. */
  async captureRetained(session: RegisteredSession, options?: TerminalCaptureOptions): Promise<TerminalReadResult<string>> {
    let reason = 'no configured terminal runtime is available'
    for (const runtime of this.orderedRuntimes(session)) {
      const result = await this.backendFor(runtime)!.capture(runtime, options)
      if (result.state === 'succeeded') return result
      reason = result.reason
    }
    return { state: 'failed', reason }
  }

  async captureLease(lease: TerminalControlLease, options?: TerminalCaptureOptions): Promise<TerminalReadResult<string>> {
    const backend = this.backendFor(lease.runtime)
    return backend ? backend.capture(lease.runtime, options) : { state: 'failed', reason: 'leased terminal backend is disabled' }
  }

  /** Open a byte stream on the first validated streaming-capable runtime. */
  async openStream(
    session: RegisteredSession,
    size: TerminalStreamSize,
    sink: TerminalStreamSink,
    readOnly = false,
  ): Promise<TerminalReadResult<TerminalStreamHandle>> {
    // `active` describes whether the engine process was discovered, not whether the retained
    // terminal runtime is still viewable. A dormant agent can legitimately be waiting at setup,
    // trust, or a shell prompt. Let each backend authoritatively check whether its pane still
    // exists. Engine-targeted operations (capture, leases, and submit helpers) keep their active
    // guards elsewhere in this coordinator; the opened stream may still carry raw terminal input.
    let reason = 'TERMINAL_RUNTIME_UNAVAILABLE'
    for (const runtime of this.orderedRuntimes(session)) {
      const backend = this.backendFor(runtime)
      if (!backend?.openStream) continue
      const result = await backend.openStream(runtime, {
        engine: session.engine,
        processIdentity: session.processIdentity ?? undefined,
      }, size, sink, readOnly)
      if (result.state === 'succeeded') return result as TerminalReadResult<TerminalStreamHandle>
      reason = result.reason
    }
    return { state: 'failed', reason }
  }

  typeLiteralLease(lease: TerminalControlLease, text: string): Promise<TerminalActionResult> {
    const backend = this.backendFor(lease.runtime)
    return backend
      ? backend.typeLiteral(lease.runtime, text)
      : Promise.resolve(terminalActionNotStarted('leased terminal backend is disabled'))
  }

  submitTextLease(lease: TerminalControlLease, text: string, options?: SubmitOptions): Promise<TerminalActionResult> {
    const backend = this.backendFor(lease.runtime)
    return backend
      ? backend.submitText(lease.runtime, text, options)
      : Promise.resolve(terminalActionNotStarted('leased terminal backend is disabled'))
  }

  /** Single-operation lease dispatch with exact-once-safe fallback to another validated locator. */
  async submitTextForLease(
    session: RegisteredSession,
    lease: TerminalControlLease,
    text: string,
    options?: SubmitOptions,
  ): Promise<TerminalActionResult> {
    if (!this.leaseIsCurrent(lease, session)) return terminalActionNotStarted('terminal control lease changed')
    const ordered = this.orderedRuntimes(session)
    const candidates = [
      lease.runtime,
      ...ordered.filter((runtime) => terminalPlacementKey(runtime) !== lease.placementKey),
    ]
    let last: TerminalActionResult = terminalActionNotStarted('no configured terminal runtime is available')
    for (const runtime of candidates) {
      const backend = this.backendFor(runtime)
      if (!backend) continue
      const validation = await this.validateRuntime(backend, runtime, session)
      if (validation.state !== 'alive') continue
      if (options?.allowed && !options.allowed()) return terminalActionNotStarted('terminal control revoked')
      const result = await backend.submitText(runtime, text, options)
      if (result.state === 'succeeded') {
        lease.runtime = runtime
        lease.placementKey = terminalPlacementKey(runtime)
        return result
      }
      if (result.dispatch === 'possibly_executed') return result
      last = result
      if (result.dispatch !== 'not_started' && result.dispatch !== 'rejected') return result
    }
    return last
  }

  sendLegacyKeyLease(lease: TerminalControlLease, key: string): Promise<TerminalActionResult> {
    const backend = this.backendFor(lease.runtime)
    const logical = terminalLogicalKey(key)
    return backend && logical
      ? backend.sendKey(lease.runtime, logical)
      : Promise.resolve(terminalActionNotStarted(backend
        ? 'unsupported logical terminal key'
        : 'leased terminal backend is disabled'))
  }

  private async sideEffect(
    session: RegisteredSession,
    action: (backend: TerminalBackend, runtime: TerminalRuntimeRef) => Promise<TerminalActionResult>,
  ): Promise<TerminalActionResult> {
    if (!session.active) return terminalActionNotStarted('terminal agent is dormant')
    let last: TerminalActionResult = terminalActionNotStarted('no configured terminal runtime is available')
    for (const runtime of this.orderedRuntimes(session)) {
      const backend = this.backendFor(runtime)!
      const validation = await this.validateRuntime(backend, runtime, session)
      if (validation.state !== 'alive') continue
      const result = await action(backend, runtime)
      if (result.state === 'succeeded' || result.dispatch === 'possibly_executed') return result
      last = result
      // Only proven pre-dispatch or explicit server rejection may use another validated locator.
      if (result.dispatch !== 'not_started' && result.dispatch !== 'rejected') return result
    }
    return last
  }

  submitText(session: RegisteredSession, text: string, options?: SubmitOptions): Promise<TerminalActionResult> {
    return this.sideEffect(session, (backend, runtime) => backend.submitText(runtime, text, options))
  }

  typeLiteral(session: RegisteredSession, text: string): Promise<TerminalActionResult> {
    return this.sideEffect(session, (backend, runtime) => backend.typeLiteral(runtime, text))
  }

  sendKey(session: RegisteredSession, key: TerminalLogicalKey): Promise<TerminalActionResult> {
    return this.sideEffect(session, (backend, runtime) => backend.sendKey(runtime, key))
  }

  sendLegacyKey(session: RegisteredSession, key: string): Promise<TerminalActionResult> {
    const logical = terminalLogicalKey(key)
    return logical ? this.sendKey(session, logical) : Promise.resolve(terminalActionNotStarted('unsupported logical terminal key'))
  }

  setTitle(session: RegisteredSession, title: string): Promise<TerminalActionResult> {
    return this.sideEffect(session, (backend, runtime) => backend.setTitle(runtime, title))
  }

  notify(session: RegisteredSession, title: string, body: string): Promise<TerminalActionResult> {
    return this.sideEffect(session, (backend, runtime) => backend.notify(runtime, title, body))
  }
}