Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
19.1 KB
Raw Blame
/**
 * `PersistQueue` — main-thread façade in front of the SQLite worker thread.
 *
 * - `enqueueSync(op)` resolves when the worker ACKs (or rejects on error /
 *   timeout). Used for IronClaw-critical writes: suspendForApproval, approve,
 *   deny, completeTurn with billed usage.
 * - `enqueueAsync(sessionKey, op)` does not wait for the ACK but preserves the
 *   `sessionKey`-scoped ordering chain (per `SessionProcessor`'s FIFO
 *   invariant). An async write stays pending until the worker answers or
 *   exits; a slow ACK is only logged, so its outcome is never guessed. A
 *   failed async write marks the queue failed: every later write rejects
 *   with that failure and `onFailure` fires once, so the Host stops instead
 *   of accepting writes it can no longer persist. The same holds after an
 *   unexpected worker exit.
 * - Progress bound: once the database is open, or once `close()` has started,
 *   while the worker has operations in flight it must complete one within
 *   `stallTimeoutMs` of the previous completion (or of the moment work became
 *   in flight, or of the start of `close()` before open). A slow
 *   worker keeps completing operations, however long its backlog; a wedged
 *   one stops, and the queue fails the same way as after a worker exit. The
 *   bound measures worker progress, never how long a write has waited.
 *
 * The implementation is transport-agnostic: callers inject a `WorkerLike`
 * (typically a `node:worker_threads.Worker`, but tests inject an in-process
 * dispatcher). This keeps the unit-test surface tractable without spinning
 * up real worker threads.
 */
import { randomUUID } from 'node:crypto'
import type {
  HeartbeatPayload,
  WorkerMessage,
  WorkerOp,
  WorkerReply,
} from '../../../db/worker/protocol'
import { HEARTBEAT_ID } from '../../../db/worker/protocol'
import { logger } from '../../../logger'

export interface WorkerLike {
  postMessage(msg: WorkerMessage): void
  on(event: 'message', listener: (reply: WorkerReply) => void): void
  on(event: 'error', listener: (err: Error) => void): void
  on(event: 'exit', listener: (code: number) => void): void
  terminate(): Promise<number> | void
}

export interface PersistQueueOptions {
  syncTimeoutMs: number
  /** An async write not ACKed within this time is logged and kept pending. */
  asyncTimeoutMs: number
  /**
   * Longest time the worker may go without completing an operation while it
   * has operations in flight, in service and during `close()`. In service it
   * applies only once the `awaitOpen()` ping is answered; once `close()` has
   * started it applies even if the database never opened, counted from the
   * start of `close()`.
   */
  stallTimeoutMs: number
  /** Called whenever a heartbeat reply is observed (id === HEARTBEAT_ID). */
  onHeartbeat?: (payload: HeartbeatPayload) => void
  /** Called on transport-level errors (worker crashed, message channel broke). */
  onTransportError?: (err: Error) => void
  /**
   * Called with the exit code when the worker exits before ACKing `shutdown`,
   * for logging. The same exit is reported through `onFailure`, which ends
   * the process. The exit that follows a `shutdown` ACK is not reported.
   */
  onExit?: (code: number) => void
  /**
   * Called once, with the first failure that makes the queue unusable: an
   * async write failed, the worker made no progress within `stallTimeoutMs`,
   * or the worker exited before ACKing `shutdown`.
   */
  onFailure?: (err: Error) => void
}

interface PendingEntry {
  resolve: (v: unknown) => void
  reject: (e: Error) => void
  /** Absent for `awaitOpen`, which has no timeout. */
  timeoutHandle?: ReturnType<typeof setTimeout>
  op: WorkerOp
}

export class PersistQueue {
  private readonly pending = new Map<string, PendingEntry>()
  private readonly writeChain = new Map<string, Promise<void>>()
  /** Set once admission closes (graceful `close()` or worker exit). */
  private closed = false
  private exited = false
  /** The worker ACKed `shutdown`; its exit that follows is expected. */
  private shutdownAcked = false
  /** First failure that makes further writes unsafe; never cleared. */
  private failure: Error | null = null
  private closePromise: Promise<void> | null = null
  /**
   * Ids posted to the worker and not yet answered. Unlike `pending`, an id
   * stays here after its caller's timeout: the worker is still busy with it.
   */
  private readonly inFlight = new Set<string>()
  private inFlightWaiters: Array<() => void> = []
  /** The `awaitOpen()` ping was answered; the progress bound applies. */
  private opened = false
  /** `performance.now()` of the last completion, or of in-flight going 0 → 1. */
  private lastProgressAt = 0
  private stallTimer: ReturnType<typeof setTimeout> | undefined
  /** The progress bound expired and the worker was terminated. */
  private stalled = false

  constructor(
    private readonly worker: WorkerLike,
    private readonly opts: PersistQueueOptions
  ) {
    this.worker.on('message', reply => this.handleReply(reply))
    this.worker.on('error', err => {
      if (this.opts.onTransportError) this.opts.onTransportError(err)
    })
    this.worker.on('exit', code => {
      this.closed = true
      this.exited = true
      // Flush pending with an explicit error so callers don't hang.
      for (const entry of this.pending.values()) {
        clearTimeout(entry.timeoutHandle)
        entry.reject(
          new Error(`db worker exited (code=${code}) while op=${entry.op.kind} was pending`)
        )
      }
      this.pending.clear()
      this.settleInFlight()
      if (this.shutdownAcked) return
      if (this.opts.onExit) this.opts.onExit(code)
      this.recordFailure(new Error(`db worker exited (code=${code})`))
    })
  }

  /**
   * Write without waiting for the ACK. Ordering is preserved per `sessionKey`
   * so a `startTurn` precedes its `completeTurn` on disk regardless of
   * scheduling jitter. Throws when the queue is closed or an earlier write
   * failed; a failure of this write is recorded and surfaces on the next
   * write, through `onFailure` and from `close()`.
   */
  enqueueAsync(sessionKey: string, op: WorkerOp): void {
    this.assertAdmissible()
    const prev = this.writeChain.get(sessionKey) ?? Promise.resolve()
    const next = prev
      .then(() => this.sendUnlessFailed(op, this.opts.asyncTimeoutMs, 'warn'))
      .then(() => {
        /* discard the result — async path */
      })
      .catch((err: Error) => {
        // An earlier failure already recorded and logged.
        if (err === this.failure) return
        const failure = new Error(
          `async write failed (sessionKey=${sessionKey}): ${err.message ?? String(err)}`
        )
        logger.error({ sessionKey, err }, '[PersistQueue] async write failed')
        this.recordFailure(failure)
      })
      .finally(() => {
        // Drop the chain reference once it settles so we don't keep arbitrarily
        // long histories per session.
        if (this.writeChain.get(sessionKey) === next) {
          this.writeChain.delete(sessionKey)
        }
      })
    this.writeChain.set(sessionKey, next)
  }

  /**
   * Sync write. Resolves with the worker's `result` field. Rejects on:
   * - worker-reported error (`reply.ok === false`)
   * - timeout (`syncTimeoutMs` elapsed)
   * - transport disconnect (worker exit)
   *
   * When `sessionKey` is provided, the sync write happens AFTER all
   * preceding async writes for that key. This preserves the durable
   * write order — e.g. `persistTurnStart` (async) lands in SQLite before
   * `persistSuspend` (sync). Without the chain the sync write could
   * race the async one and the snapshot reads inconsistent state.
   */
  enqueueSync<T = unknown>(op: WorkerOp, sessionKey?: string): Promise<T> {
    try {
      this.assertAdmissible()
    } catch (err) {
      return Promise.reject(err as Error)
    }
    if (sessionKey) {
      const prev = this.writeChain.get(sessionKey) ?? Promise.resolve()
      const next = prev.then(() => this.sendUnlessFailed<T>(op, this.opts.syncTimeoutMs))
      // The chain derivative must never surface the rejection itself — the
      // caller gets it via the returned `next` (loud there). Without the
      // rejection handler a failed sync write would ALSO raise a process-level
      // unhandledRejection from this internal bookkeeping copy (D3). The
      // cleanup compares against the derivative, because that is what the map
      // holds.
      const chained: Promise<void> = next
        .then(
          () => undefined,
          () => undefined
        )
        .finally(() => {
          if (this.writeChain.get(sessionKey) === chained) {
            this.writeChain.delete(sessionKey)
          }
        })
      this.writeChain.set(sessionKey, chained)
      return next
    }
    return this.send<T>(op, this.opts.syncTimeoutMs)
  }

  /**
   * Resolves when the worker answers a `ping`. The worker registers its
   * message handler only after it has opened and migrated the database, so
   * this resolves once the database is open. No timeout: opening (spawn,
   * migrations, recovery of a large WAL) can outlast any write budget, so
   * the progress bound applies only once this ping is answered, or once
   * `close()` has started.
   * Rejects on a worker error reply, a worker exit, or when the worker makes
   * no progress within `stallTimeoutMs` after `close()` started; a `close()`
   * that finds the ping answered in time lets it resolve.
   */
  awaitOpen(): Promise<void> {
    try {
      this.assertAdmissible()
    } catch (err) {
      return Promise.reject(err as Error)
    }
    return this.send<unknown>({ kind: 'ping' }, null).then(() => {
      this.opened = true
      this.lastProgressAt = performance.now()
      this.armStallTimer()
    })
  }

  /**
   * Wait until every pending async write for `sessionKey` settles. Used by
   * `SqliteConversationStore` before sync write-through operations that
   * read other rows (e.g. load_active_session after persistTurnStart).
   */
  async drainSessionKey(sessionKey: string): Promise<void> {
    const chain = this.writeChain.get(sessionKey)
    if (!chain) return
    try {
      await chain
    } catch {
      /* errors are reported by the chained .catch in enqueueAsync */
    }
  }

  /**
   * Wait until every pending async write whose `sessionKey` STARTS WITH
   * `prefix` settles. The prefix analogue of `drainSessionKey`: used by
   * `SqliteConversationStore.prefetchUserSessions` before its bulk
   * `list_sessions_by_prefix` read, so a just-completed session still in the
   * per-key write chain isn't reconstructed from stale rows (sqlite-stores-3).
   */
  async drainPrefix(prefix: string): Promise<void> {
    const chains: Array<Promise<void>> = []
    for (const [key, chain] of this.writeChain) {
      if (key.startsWith(prefix)) chains.push(chain)
    }
    if (chains.length === 0) return
    await Promise.allSettled(chains)
  }

  /**
   * Wait for the keyed async chains, not for in-flight sync operations. Used
   * by the stateless suspend checkpoint (`main.ts`, `flushFinalCheckpoint`).
   * `close()` does its own stricter wait.
   */
  async drain(): Promise<void> {
    const chains = Array.from(this.writeChain.values())
    if (chains.length === 0) return
    await Promise.allSettled(chains)
  }

  /**
   * Graceful shutdown: close admission, wait until every accepted write has
   * settled (the keyed chains and every operation in flight, sync writes
   * without a `sessionKey` included), wait for the worker to ACK `shutdown`
   * (it closes the database before replying), then terminate it. There is no
   * total deadline: a slow worker drains its whole backlog, and only the
   * progress bound (`stallTimeoutMs`) ends the wait for a wedged one. If the
   * database has not opened yet, the bound starts with this call, so a worker
   * that never answers the open ping fails within `stallTimeoutMs` while a
   * slow but healthy open can still finish.
   * Rejects with the first failure, so the caller sees that the durable
   * trail is incomplete.
   */
  close(): Promise<void> {
    // A second caller waits for the same shutdown instead of returning early.
    this.closePromise ??= this.shutdown()
    return this.closePromise
  }

  private async shutdown(): Promise<void> {
    this.closed = true
    if (!this.opened) {
      // The open ping may still be in flight; the bound starts now, not when
      // the ping was posted, so a slow but healthy open can still finish.
      this.lastProgressAt = performance.now()
      this.armStallTimer()
    }
    // Admission is closed, so no chain starts after this snapshot.
    await Promise.allSettled(Array.from(this.writeChain.values()))
    await this.inFlightSettled()
    if (!this.exited && !this.stalled) {
      try {
        // Checkpointing on close can outlast `syncTimeoutMs`; the progress
        // bound covers this operation like any other.
        await this.send({ kind: 'shutdown' }, null)
      } catch (err) {
        // Recorded before terminate(), so the exit it causes is not reported
        // as a second failure.
        this.recordFailure(err as Error)
      }
    }
    if (!this.stalled) {
      // A stalled worker was already terminated by checkStall().
      const terminated = this.worker.terminate()
      if (terminated && typeof (terminated as Promise<number>).then === 'function') {
        await terminated
      }
    }
    for (const entry of this.pending.values()) {
      clearTimeout(entry.timeoutHandle)
      entry.reject(new Error('PersistQueue closed'))
    }
    this.pending.clear()
    if (this.failure) throw this.failure
  }

  private inFlightSettled(): Promise<void> {
    if (this.inFlight.size === 0) return Promise.resolve()
    return new Promise(resolve => this.inFlightWaiters.push(resolve))
  }

  /** Forget every in-flight id (worker gone or stalled) and wake waiters. */
  private settleInFlight(): void {
    this.inFlight.clear()
    clearTimeout(this.stallTimer)
    this.stallTimer = undefined
    this.notifyInFlightSettled()
  }

  private notifyInFlightSettled(): void {
    if (this.inFlight.size !== 0) return
    const waiters = this.inFlightWaiters
    this.inFlightWaiters = []
    for (const resolve of waiters) resolve()
  }

  private armStallTimer(): void {
    // Still armed after another failure: close() waits for in-flight work.
    // Before open the bound applies only once close() has started.
    if (
      (!this.opened && !this.closed) ||
      this.stalled ||
      this.stallTimer ||
      this.inFlight.size === 0
    ) {
      return
    }
    const remaining = this.opts.stallTimeoutMs - (performance.now() - this.lastProgressAt)
    this.stallTimer = setTimeout(
      () => {
        this.stallTimer = undefined
        // Timers run before the poll phase that delivers worker replies. After
        // a main-thread stall an answer can be waiting; read it first.
        setImmediate(() => this.checkStall())
      },
      Math.max(0, remaining)
    )
  }

  private checkStall(): void {
    if (this.stalled || this.inFlight.size === 0) return
    if (performance.now() - this.lastProgressAt < this.opts.stallTimeoutMs) {
      this.armStallTimer()
      return
    }
    const count = this.inFlight.size
    this.stalled = true
    const failure = new Error(
      `db worker made no progress for ${this.opts.stallTimeoutMs}ms with ${count} operation(s) in flight`
    )
    // The logger keeps only an error's name, so the cause is logged as fields.
    logger.error(
      { stallTimeoutMs: this.opts.stallTimeoutMs, inFlight: count },
      '[PersistQueue] db worker made no progress; terminating it'
    )
    this.recordFailure(failure)
    for (const entry of this.pending.values()) {
      clearTimeout(entry.timeoutHandle)
      entry.reject(failure)
    }
    this.pending.clear()
    this.settleInFlight()
    // The failure is recorded first, so the exit that follows is not reported
    // as a second one. Not awaited: a wedged thread may take a while to stop.
    const terminated = this.worker.terminate()
    if (terminated && typeof (terminated as Promise<number>).then === 'function') {
      ;(terminated as Promise<number>).catch((err: unknown) => {
        logger.warn({ err }, '[PersistQueue] terminating the stalled db worker failed')
      })
    }
  }

  private recordFailure(err: Error): void {
    if (this.failure) return
    this.failure = err
    if (this.opts.onFailure) this.opts.onFailure(err)
  }

  private assertAdmissible(): void {
    if (this.failure) throw this.failure
    if (this.closed) throw new Error('PersistQueue is closed')
  }

  /**
   * Post a write that waited on its `sessionKey` chain. The queue may have
   * failed while it waited, and admission was checked before that.
   */
  private sendUnlessFailed<T>(
    op: WorkerOp,
    timeoutMs: number,
    onTimeout: 'reject' | 'warn' = 'reject'
  ): Promise<T> {
    if (this.failure) return Promise.reject(this.failure)
    return this.send<T>(op, timeoutMs, onTimeout)
  }

  /**
   * `onTimeout: 'warn'` logs once when `timeoutMs` elapses and keeps waiting
   * for the ACK or the worker exit.
   */
  private send<T>(
    op: WorkerOp,
    timeoutMs: number | null,
    onTimeout: 'reject' | 'warn' = 'reject'
  ): Promise<T> {
    const id = randomUUID()
    return new Promise<T>((resolve, reject) => {
      const timeoutHandle =
        timeoutMs === null
          ? undefined
          : setTimeout(() => {
              if (onTimeout === 'warn') {
                if (this.pending.has(id)) {
                  logger.warn(
                    { op: op.kind, timeoutMs },
                    '[PersistQueue] async write not acknowledged yet; still waiting'
                  )
                }
              } else if (this.pending.delete(id)) {
                reject(new Error(`db worker timeout (${timeoutMs}ms) for op=${op.kind}`))
              }
            }, timeoutMs)
      const entry: PendingEntry = {
        resolve: resolve as PendingEntry['resolve'],
        reject,
        timeoutHandle,
        op,
      }
      this.pending.set(id, entry)
      try {
        this.worker.postMessage({ id, op })
      } catch (err) {
        this.pending.delete(id)
        clearTimeout(timeoutHandle)
        reject(err as Error)
        return
      }
      if (this.inFlight.size === 0) this.lastProgressAt = performance.now()
      this.inFlight.add(id)
      this.armStallTimer()
    })
  }

  private handleReply(reply: WorkerReply): void {
    if (reply.id === HEARTBEAT_ID) {
      if (this.opts.onHeartbeat) {
        this.opts.onHeartbeat(reply.result as HeartbeatPayload)
      }
      return
    }
    if (this.inFlight.delete(reply.id)) {
      this.lastProgressAt = performance.now()
      clearTimeout(this.stallTimer)
      this.stallTimer = undefined
      this.armStallTimer()
      this.notifyInFlightSettled()
    }
    const entry = this.pending.get(reply.id)
    if (!entry) return
    clearTimeout(entry.timeoutHandle)
    this.pending.delete(reply.id)
    if (reply.ok) {
      // Set synchronously: the worker's exit can be delivered before the
      // awaiting `close()` resumes.
      if (entry.op.kind === 'shutdown') this.shutdownAcked = true
      entry.resolve(reply.result)
    } else {
      const err = new Error(reply.error?.message ?? 'unknown worker error')
      ;(err as Error & { code?: string }).code = reply.error?.code
      entry.reject(err)
    }
  }
}