Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

6.6 KB
Raw Blame
import type { RelayDatabase } from './database.js'

export type RelayReadinessFailure =
  | 'jwks_fetch_failed'
  | 'jwks_http_failed'
  | 'jwks_timed_out'
  | 'sql_failed'

export type RelayReadinessDependency = 'jwks' | 'sql'

export type RelayReadinessObservation = {
  ready: boolean
  failure?: RelayReadinessFailure
  // Both dependencies are probed every poll, so both can fail in the same one.
  failures?: RelayReadinessFailure[]
  degraded?: true
  degradedDependencies?: RelayReadinessDependency[]
  jwksLatencyMs: number
  sqlLatencyMs: number
  totalLatencyMs: number
}

export type RelayReadinessGraceEvent = {
  dependency: RelayReadinessDependency
  grace: 'entered' | 'recovered' | 'expired'
  failure?: RelayReadinessFailure
  lastSuccessAgeMs?: number
  graceMs: number
}

export type RelayReadinessProbe = {
  check: () => Promise<boolean>
  degradedDependencies: () => RelayReadinessDependency[]
  // Failing on the last probe, in grace or past it.
  failingDependencies: () => RelayReadinessDependency[]
}

// Cached verifier keys survive JWKS outages; failed fetches prevent key rotation, not verification.
export const RELAY_READINESS_JWKS_GRACE_MS = 900_000
// Failing a cell’s readiness loses its hostname rather than rerouting hosts; 3min rides failover without masking longer cell faults.
// SQL-dead hellos can get WRONG_CELL and director replacement; generic DB errors preserve existing controls.
export const RELAY_READINESS_SQL_GRACE_MS = 180_000
export const RELAY_MAX_READINESS_GRACE_MS = 3_600_000

type RelayReadinessOptions = {
  fetch?: typeof fetch
  timeoutMs?: number
  cacheMs?: number
  jwksGraceMs?: number
  sqlGraceMs?: number
  now?: () => number
  observe?: (observation: RelayReadinessObservation) => void
  observeGrace?: (event: RelayReadinessGraceEvent) => void
}

type DependencySettlement = {
  satisfied: boolean
  degraded: boolean
  event?: RelayReadinessGraceEvent
}

function fetchFailure(error: unknown): RelayReadinessFailure {
  return error instanceof Error && error.name === 'TimeoutError'
    ? 'jwks_timed_out'
    : 'jwks_fetch_failed'
}

function graceTransition(
  degraded: boolean,
  failure: RelayReadinessFailure | undefined
): RelayReadinessGraceEvent['grace'] {
  if (degraded) return 'entered'
  return failure === undefined ? 'recovered' : 'expired'
}

// Separate dependency clocks or healthy JWKS polls keep dead PostgreSQL in grace forever.
function createDependencyGrace(dependency: RelayReadinessDependency, graceMs: number) {
  let lastSuccessAt: number | undefined
  let inGrace = false

  return (at: number, failure: RelayReadinessFailure | undefined): DependencySettlement => {
    if (failure === undefined) lastSuccessAt = at
    const lastSuccessAgeMs =
      lastSuccessAt === undefined ? undefined : Math.max(0, at - lastSuccessAt)
    const degraded =
      failure !== undefined && lastSuccessAgeMs !== undefined && lastSuccessAgeMs < graceMs
    const crossed = degraded !== inGrace
    inGrace = degraded
    return {
      satisfied: failure === undefined || degraded,
      degraded,
      ...(crossed
        ? {
            event: {
              dependency,
              grace: graceTransition(degraded, failure),
              ...(failure ? { failure } : {}),
              ...(lastSuccessAgeMs === undefined ? {} : { lastSuccessAgeMs }),
              graceMs
            }
          }
        : {})
    }
  }
}

async function timed<T>(
  now: () => number,
  run: () => Promise<T>
): Promise<{ value: T; latencyMs: number }> {
  const startedAt = now()
  const value = await run()
  return { value, latencyMs: Math.max(0, now() - startedAt) }
}

export function createRelayReadiness(
  database: RelayDatabase,
  jwksUrl: string,
  options: RelayReadinessOptions = {}
): RelayReadinessProbe {
  const fetchImpl = options.fetch ?? fetch
  const timeoutMs = options.timeoutMs ?? 2_000
  const cacheMs = options.cacheMs ?? 10_000
  const now = options.now ?? Date.now
  const settleJwks = createDependencyGrace(
    'jwks',
    options.jwksGraceMs ?? RELAY_READINESS_JWKS_GRACE_MS
  )
  const settleSql = createDependencyGrace('sql', options.sqlGraceMs ?? RELAY_READINESS_SQL_GRACE_MS)
  let cachedAt = Number.NEGATIVE_INFINITY
  let cached = false
  let pending: Promise<boolean> | null = null
  let lastObservedReady: boolean | undefined
  let degraded: RelayReadinessDependency[] = []
  let failing: RelayReadinessDependency[] = []

  const probeJwks = async (): Promise<RelayReadinessFailure | undefined> => {
    try {
      const response = await fetchImpl(jwksUrl, { signal: AbortSignal.timeout(timeoutMs) })
      return response.ok ? undefined : 'jwks_http_failed'
    } catch (error) {
      return fetchFailure(error)
    }
  }

  const probeSql = async (): Promise<RelayReadinessFailure | undefined> => {
    try {
      await database.query('SELECT 1 AS ready')
      return undefined
    } catch {
      // The load balancer only needs the boolean; the safe reason is all that is emitted.
      return 'sql_failed'
    }
  }

  const probe = async (): Promise<boolean> => {
    const startedAt = now()
    const [jwks, sql] = await Promise.all([timed(now, probeJwks), timed(now, probeSql)])
    const completedAt = now()
    const jwksSettlement = settleJwks(completedAt, jwks.value)
    const sqlSettlement = settleSql(completedAt, sql.value)
    const failures = [jwks.value, sql.value].filter((value) => value !== undefined)
    const failure = failures[0]
    degraded = []
    failing = []
    if (jwks.value !== undefined) failing.push('jwks')
    if (sql.value !== undefined) failing.push('sql')
    if (jwksSettlement.degraded) degraded.push('jwks')
    if (sqlSettlement.degraded) degraded.push('sql')
    cached = jwksSettlement.satisfied && sqlSettlement.satisfied
    cachedAt = completedAt
    if (failure !== undefined || cached !== lastObservedReady) {
      options.observe?.({
        ready: cached,
        ...(failure ? { failure } : {}),
        ...(failures.length > 1 ? { failures } : {}),
        ...(degraded.length > 0 ? { degraded: true, degradedDependencies: [...degraded] } : {}),
        jwksLatencyMs: jwks.latencyMs,
        sqlLatencyMs: sql.latencyMs,
        totalLatencyMs: Math.max(0, completedAt - startedAt)
      })
    }
    for (const event of [jwksSettlement.event, sqlSettlement.event]) {
      if (event) options.observeGrace?.(event)
    }
    lastObservedReady = cached
    return cached
  }

  const check = async (): Promise<boolean> => {
    if (now() - cachedAt < cacheMs) return cached
    pending ??= probe().finally(() => {
      pending = null
    })
    return pending
  }

  return {
    check,
    degradedDependencies: () => [...degraded],
    failingDependencies: () => [...failing]
  }
}