Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import { describe, expect, it, vi } from 'vitest'
import type { RelayDatabase } from './database.js'
import {
createRelayReadiness,
RELAY_READINESS_JWKS_GRACE_MS,
RELAY_READINESS_SQL_GRACE_MS,
type RelayReadinessGraceEvent,
type RelayReadinessObservation
} from './relay-readiness.js'
function database(query: () => Promise<Record<string, unknown>[]>): RelayDatabase {
return {
query,
queryLocked: query,
transaction: async (operation) => await operation(database(query)),
close: async () => {}
}
}
describe('relay readiness', () => {
it.each([
{
name: 'JWKS HTTP failure',
fetch: vi.fn(async () => new Response('', { status: 503 })) as typeof fetch,
query: vi.fn(async () => [{ ready: 1 }]),
failure: 'jwks_http_failed'
},
{
name: 'JWKS timeout',
fetch: vi.fn(async () => {
throw new DOMException('redacted', 'TimeoutError')
}) as typeof fetch,
query: vi.fn(async () => [{ ready: 1 }]),
failure: 'jwks_timed_out'
},
{
name: 'JWKS fetch failure',
fetch: vi.fn(async () => {
throw new Error('redacted')
}) as typeof fetch,
query: vi.fn(async () => [{ ready: 1 }]),
failure: 'jwks_fetch_failed'
},
{
name: 'SQL failure',
fetch: vi.fn(async () => new Response('{}', { status: 200 })) as typeof fetch,
query: vi.fn(async () => {
throw new Error('redacted')
}),
failure: 'sql_failed'
}
])('reports a safe reason for $name', async ({ fetch, query, failure }) => {
const observations: RelayReadinessObservation[] = []
const readiness = createRelayReadiness(database(query), 'https://jwks', {
fetch,
cacheMs: 0,
observe: (observation) => observations.push(observation)
})
expect(await readiness.check()).toBe(false)
expect(observations).toEqual([expect.objectContaining({ ready: false, failure })])
expect(JSON.stringify(observations)).not.toContain('redacted')
expect(query).toHaveBeenCalledTimes(1)
})
it('reports the initial success but not healthy repeats or cached reads', async () => {
const observations: RelayReadinessObservation[] = []
let now = 100
const readiness = createRelayReadiness(database(async () => [{ ready: 1 }]), 'https://jwks', {
fetch: vi.fn(async () => new Response('{}', { status: 200 })) as typeof fetch,
cacheMs: 10_000,
now: () => now,
observe: (observation) => observations.push(observation)
})
expect(await readiness.check()).toBe(true)
now += 1_000
expect(await readiness.check()).toBe(true)
now += 10_000
expect(await readiness.check()).toBe(true)
expect(observations).toEqual([
{
ready: true,
jwksLatencyMs: 0,
sqlLatencyMs: 0,
totalLatencyMs: 0
}
])
})
})
describe('relay readiness last-known-good grace', () => {
function graceProbe(input: {
jwksGraceMs?: number
sqlGraceMs?: number
cacheMs?: number
jwksOk: () => boolean
sqlOk: () => boolean
now: () => number
}) {
const observations: RelayReadinessObservation[] = []
const graceEvents: RelayReadinessGraceEvent[] = []
const query = vi.fn(async () => {
if (!input.sqlOk()) throw new Error('redacted')
return [{ ready: 1 }]
})
const fetchImpl = vi.fn(
async () => new Response('{}', { status: input.jwksOk() ? 200 : 503 })
) as typeof fetch
const readiness = createRelayReadiness(database(query), 'https://jwks', {
fetch: fetchImpl,
cacheMs: input.cacheMs ?? 0,
...(input.jwksGraceMs === undefined ? {} : { jwksGraceMs: input.jwksGraceMs }),
...(input.sqlGraceMs === undefined ? {} : { sqlGraceMs: input.sqlGraceMs }),
now: input.now,
observe: (observation) => observations.push(observation),
observeGrace: (event) => graceEvents.push(event)
})
return { readiness, observations, graceEvents, query, fetchImpl }
}
it('stays ready while a JWKS failure sits inside the default fifteen minute window', async () => {
let now = 1_000
let jwksOk = true
const { readiness, observations } = graceProbe({
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += RELAY_READINESS_JWKS_GRACE_MS - 1
expect(await readiness.check()).toBe(true)
expect(readiness.degradedDependencies()).toEqual(['jwks'])
expect(observations.at(-1)).toEqual(
expect.objectContaining({
ready: true,
degraded: true,
degradedDependencies: ['jwks'],
failure: 'jwks_http_failed'
})
)
})
it('drops readiness once the JWKS window expires', async () => {
let now = 1_000
let jwksOk = true
const { readiness, observations } = graceProbe({
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += RELAY_READINESS_JWKS_GRACE_MS
expect(await readiness.check()).toBe(false)
expect(readiness.degradedDependencies()).toEqual([])
expect(observations.at(-1)).toEqual(
expect.objectContaining({ ready: false, failure: 'jwks_http_failed' })
)
expect(observations.at(-1)).not.toHaveProperty('degraded')
})
it('gives SQL a shorter window than JWKS by default', async () => {
let now = 1_000
let sqlOk = true
const { readiness, observations } = graceProbe({
jwksOk: () => true,
sqlOk: () => sqlOk,
now: () => now
})
expect(await readiness.check()).toBe(true)
sqlOk = false
now += RELAY_READINESS_SQL_GRACE_MS - 1
expect(await readiness.check()).toBe(true)
expect(observations.at(-1)).toEqual(
expect.objectContaining({
ready: true,
degraded: true,
degradedDependencies: ['sql'],
failure: 'sql_failed'
})
)
now += 1
expect(await readiness.check()).toBe(false)
expect(RELAY_READINESS_SQL_GRACE_MS).toBeLessThan(RELAY_READINESS_JWKS_GRACE_MS)
})
it('keeps a process that never succeeded out of the grace window', async () => {
let now = 1_000
const { readiness, observations, graceEvents } = graceProbe({
jwksOk: () => false,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(false)
now += 1_000
expect(await readiness.check()).toBe(false)
expect(graceEvents).toEqual([])
expect(observations.every((observation) => observation.degraded === undefined)).toBe(true)
})
it('logs once on entering grace and once on leaving it', async () => {
let now = 1_000
let jwksOk = true
const { readiness, graceEvents } = graceProbe({
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
now += 1_000
expect(await readiness.check()).toBe(true)
jwksOk = true
now += 1_000
expect(await readiness.check()).toBe(true)
expect(graceEvents).toEqual([
{
dependency: 'jwks',
grace: 'entered',
failure: 'jwks_http_failed',
lastSuccessAgeMs: 1_000,
graceMs: RELAY_READINESS_JWKS_GRACE_MS
},
{
dependency: 'jwks',
grace: 'recovered',
lastSuccessAgeMs: 0,
graceMs: RELAY_READINESS_JWKS_GRACE_MS
}
])
})
it('reports an expired window once when the dependency never comes back', async () => {
let now = 1_000
let jwksOk = true
const { readiness, graceEvents } = graceProbe({
jwksGraceMs: 10_000,
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
now += 20_000
expect(await readiness.check()).toBe(false)
expect(await readiness.check()).toBe(false)
expect(graceEvents.map((event) => event.grace)).toEqual(['entered', 'expired'])
})
it('restarts the grace window from the most recent success', async () => {
let now = 1_000
let jwksOk = true
const { readiness } = graceProbe({
jwksGraceMs: 10_000,
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 9_000
expect(await readiness.check()).toBe(true)
jwksOk = true
now += 1_000
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 9_000
expect(await readiness.check()).toBe(true)
})
it('never serves grace when the window is disabled', async () => {
let now = 1_000
let jwksOk = true
const { readiness, graceEvents } = graceProbe({
jwksGraceMs: 0,
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
expect(await readiness.check()).toBe(false)
expect(graceEvents).toEqual([])
})
it('keeps one clock per dependency so a healthy JWKS cannot hold SQL open', async () => {
let now = 1_000
let sqlOk = true
const { readiness, graceEvents } = graceProbe({
sqlGraceMs: 10_000,
jwksOk: () => true,
sqlOk: () => sqlOk,
now: () => now
})
expect(await readiness.check()).toBe(true)
sqlOk = false
for (let elapsed = 1_000; elapsed <= 11_000; elapsed += 1_000) {
now = 1_000 + elapsed
await readiness.check()
}
expect(await readiness.check()).toBe(false)
expect(graceEvents.map((event) => [event.dependency, event.grace])).toEqual([
['sql', 'entered'],
['sql', 'expired']
])
})
it('logs both sides of an overlap when one dependency recovers as the other fails', async () => {
let now = 1_000
let jwksOk = true
let sqlOk = true
const { readiness, graceEvents } = graceProbe({
jwksOk: () => jwksOk,
sqlOk: () => sqlOk,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
jwksOk = true
sqlOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
expect(readiness.degradedDependencies()).toEqual(['sql'])
expect(graceEvents.map((event) => [event.dependency, event.grace])).toEqual([
['jwks', 'entered'],
['jwks', 'recovered'],
['sql', 'entered']
])
})
it('probes SQL on every poll even while JWKS is failing', async () => {
let now = 1_000
let jwksOk = true
let sqlOk = true
const { readiness, observations, query } = graceProbe({
sqlGraceMs: 10_000,
jwksOk: () => jwksOk,
sqlOk: () => sqlOk,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
sqlOk = false
now += 1_000
expect(await readiness.check()).toBe(true)
expect(observations.at(-1)).toEqual(
expect.objectContaining({
ready: true,
failure: 'jwks_http_failed',
failures: ['jwks_http_failed', 'sql_failed'],
degradedDependencies: ['jwks', 'sql']
})
)
// The SQL clock runs on evidence, so its window opens at the poll that saw SQL fail.
now += 10_000
expect(await readiness.check()).toBe(false)
expect(query).toHaveBeenCalledTimes(4)
})
it('bounds a cached ready answer after dependency success has expired', async () => {
let now = 1_000
let jwksOk = true
const { readiness, fetchImpl } = graceProbe({
cacheMs: 10_000,
jwksGraceMs: 5_000,
jwksOk: () => jwksOk,
sqlOk: () => true,
now: () => now
})
expect(await readiness.check()).toBe(true)
jwksOk = false
now += 6_000
expect(await readiness.check()).toBe(true)
now += 3_999
expect(await readiness.check()).toBe(true)
expect(fetchImpl).toHaveBeenCalledTimes(1)
now += 1
expect(await readiness.check()).toBe(false)
expect(fetchImpl).toHaveBeenCalledTimes(2)
})
})