Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import type { RelayDatabase } from './database.js'
export type RelayReadinessFailure =
| 'jwks_fetch_failed'
| 'jwks_http_failed'
| 'jwks_timed_out'
| 'sql_failed'
export type RelayReadinessDependency = 'jwks' | 'sql'
export type RelayReadinessObservation = {
ready: boolean
failure?: RelayReadinessFailure
// Both dependencies are probed every poll, so both can fail in the same one.
failures?: RelayReadinessFailure[]
degraded?: true
degradedDependencies?: RelayReadinessDependency[]
jwksLatencyMs: number
sqlLatencyMs: number
totalLatencyMs: number
}
export type RelayReadinessGraceEvent = {
dependency: RelayReadinessDependency
grace: 'entered' | 'recovered' | 'expired'
failure?: RelayReadinessFailure
lastSuccessAgeMs?: number
graceMs: number
}
export type RelayReadinessProbe = {
check: () => Promise<boolean>
degradedDependencies: () => RelayReadinessDependency[]
// Failing on the last probe, in grace or past it.
failingDependencies: () => RelayReadinessDependency[]
}
// Cached verifier keys survive JWKS outages; failed fetches prevent key rotation, not verification.
export const RELAY_READINESS_JWKS_GRACE_MS = 900_000
// Failing a cell’s readiness loses its hostname rather than rerouting hosts; 3min rides failover without masking longer cell faults.
// SQL-dead hellos can get WRONG_CELL and director replacement; generic DB errors preserve existing controls.
export const RELAY_READINESS_SQL_GRACE_MS = 180_000
export const RELAY_MAX_READINESS_GRACE_MS = 3_600_000
type RelayReadinessOptions = {
fetch?: typeof fetch
timeoutMs?: number
cacheMs?: number
jwksGraceMs?: number
sqlGraceMs?: number
now?: () => number
observe?: (observation: RelayReadinessObservation) => void
observeGrace?: (event: RelayReadinessGraceEvent) => void
}
type DependencySettlement = {
satisfied: boolean
degraded: boolean
event?: RelayReadinessGraceEvent
}
function fetchFailure(error: unknown): RelayReadinessFailure {
return error instanceof Error && error.name === 'TimeoutError'
? 'jwks_timed_out'
: 'jwks_fetch_failed'
}
function graceTransition(
degraded: boolean,
failure: RelayReadinessFailure | undefined
): RelayReadinessGraceEvent['grace'] {
if (degraded) return 'entered'
return failure === undefined ? 'recovered' : 'expired'
}
// Separate dependency clocks or healthy JWKS polls keep dead PostgreSQL in grace forever.
function createDependencyGrace(dependency: RelayReadinessDependency, graceMs: number) {
let lastSuccessAt: number | undefined
let inGrace = false
return (at: number, failure: RelayReadinessFailure | undefined): DependencySettlement => {
if (failure === undefined) lastSuccessAt = at
const lastSuccessAgeMs =
lastSuccessAt === undefined ? undefined : Math.max(0, at - lastSuccessAt)
const degraded =
failure !== undefined && lastSuccessAgeMs !== undefined && lastSuccessAgeMs < graceMs
const crossed = degraded !== inGrace
inGrace = degraded
return {
satisfied: failure === undefined || degraded,
degraded,
...(crossed
? {
event: {
dependency,
grace: graceTransition(degraded, failure),
...(failure ? { failure } : {}),
...(lastSuccessAgeMs === undefined ? {} : { lastSuccessAgeMs }),
graceMs
}
}
: {})
}
}
}
async function timed<T>(
now: () => number,
run: () => Promise<T>
): Promise<{ value: T; latencyMs: number }> {
const startedAt = now()
const value = await run()
return { value, latencyMs: Math.max(0, now() - startedAt) }
}
export function createRelayReadiness(
database: RelayDatabase,
jwksUrl: string,
options: RelayReadinessOptions = {}
): RelayReadinessProbe {
const fetchImpl = options.fetch ?? fetch
const timeoutMs = options.timeoutMs ?? 2_000
const cacheMs = options.cacheMs ?? 10_000
const now = options.now ?? Date.now
const settleJwks = createDependencyGrace(
'jwks',
options.jwksGraceMs ?? RELAY_READINESS_JWKS_GRACE_MS
)
const settleSql = createDependencyGrace('sql', options.sqlGraceMs ?? RELAY_READINESS_SQL_GRACE_MS)
let cachedAt = Number.NEGATIVE_INFINITY
let cached = false
let pending: Promise<boolean> | null = null
let lastObservedReady: boolean | undefined
let degraded: RelayReadinessDependency[] = []
let failing: RelayReadinessDependency[] = []
const probeJwks = async (): Promise<RelayReadinessFailure | undefined> => {
try {
const response = await fetchImpl(jwksUrl, { signal: AbortSignal.timeout(timeoutMs) })
return response.ok ? undefined : 'jwks_http_failed'
} catch (error) {
return fetchFailure(error)
}
}
const probeSql = async (): Promise<RelayReadinessFailure | undefined> => {
try {
await database.query('SELECT 1 AS ready')
return undefined
} catch {
// The load balancer only needs the boolean; the safe reason is all that is emitted.
return 'sql_failed'
}
}
const probe = async (): Promise<boolean> => {
const startedAt = now()
const [jwks, sql] = await Promise.all([timed(now, probeJwks), timed(now, probeSql)])
const completedAt = now()
const jwksSettlement = settleJwks(completedAt, jwks.value)
const sqlSettlement = settleSql(completedAt, sql.value)
const failures = [jwks.value, sql.value].filter((value) => value !== undefined)
const failure = failures[0]
degraded = []
failing = []
if (jwks.value !== undefined) failing.push('jwks')
if (sql.value !== undefined) failing.push('sql')
if (jwksSettlement.degraded) degraded.push('jwks')
if (sqlSettlement.degraded) degraded.push('sql')
cached = jwksSettlement.satisfied && sqlSettlement.satisfied
cachedAt = completedAt
if (failure !== undefined || cached !== lastObservedReady) {
options.observe?.({
ready: cached,
...(failure ? { failure } : {}),
...(failures.length > 1 ? { failures } : {}),
...(degraded.length > 0 ? { degraded: true, degradedDependencies: [...degraded] } : {}),
jwksLatencyMs: jwks.latencyMs,
sqlLatencyMs: sql.latencyMs,
totalLatencyMs: Math.max(0, completedAt - startedAt)
})
}
for (const event of [jwksSettlement.event, sqlSettlement.event]) {
if (event) options.observeGrace?.(event)
}
lastObservedReady = cached
return cached
}
const check = async (): Promise<boolean> => {
if (now() - cachedAt < cacheMs) return cached
pending ??= probe().finally(() => {
pending = null
})
return pending
}
return {
check,
degradedDependencies: () => [...degraded],
failingDependencies: () => [...failing]
}
}