Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import { describe, expect, it, vi } from 'vitest'
import {
CELL_POLL_DEAD_MAX_GAP_MS,
CELL_POLL_DEAD_STREAK_MS,
CELL_POLL_DEAD_MIN_POLLED,
CellPollReachability,
type PolledCell
} from './cell-poll-reachability.js'
import { startShadowSeatPoller, type SeatFeedCell } from './shadow-seat-directory.js'
// Fails `cellId` once a second from `from` for `ms`, as the 1 s poller does.
function failFor(reach: CellPollReachability, cellId: string, from: number, ms: number): number {
let at = from
for (; at <= from + ms; at += 1_000) reach.unanswered(cellId, at)
return at - 1_000
}
// Production's shape: 28 US cells and 6 in Asia.
const FLEET: PolledCell[] = [
...Array.from({ length: 28 }, (_, index) => ({ cellId: `us${index}`, region: 'us-central1' })),
...Array.from({ length: 6 }, (_, index) => ({ cellId: `asia${index}`, region: 'asia-east2' }))
]
describe('CellPollReachability', () => {
it('calls a cell dead only after a continuous failing streak of 45 s', () => {
const reach = new CellPollReachability()
const last = failFor(reach, 'us1', 0, CELL_POLL_DEAD_STREAK_MS - 1_000)
expect(reach.verdict('us1', FLEET, last)).toBeNull()
reach.unanswered('us1', last + 1_000)
expect(reach.verdict('us1', FLEET, last + 1_000)).toEqual({
dead: true,
fleetFailing: 1,
fleetPolled: 34,
regionFailing: 1,
regionPolled: 28
})
})
it('needs a failure within the last 5 s: a frozen director has no verdict until it polls again', () => {
const reach = new CellPollReachability()
const last = failFor(reach, 'us1', 0, CELL_POLL_DEAD_STREAK_MS)
expect(reach.verdict('us1', FLEET, last)?.dead).toBe(true)
// The director froze: nothing was asked, so nothing is known.
expect(reach.verdict('us1', FLEET, last + CELL_POLL_DEAD_MAX_GAP_MS + 1)).toBeNull()
// After the freeze a single failure starts a new streak rather than resuming the old one.
reach.unanswered('us1', last + 60_000)
expect(reach.verdict('us1', FLEET, last + 60_000)).toBeNull()
})
it('resets on any answer from the cell', () => {
const reach = new CellPollReachability()
const last = failFor(reach, 'us1', 0, CELL_POLL_DEAD_STREAK_MS)
reach.answered('us1')
reach.unanswered('us1', last + 1_000)
expect(reach.verdict('us1', FLEET, last + 1_000)).toBeNull()
})
it(`never decides from fewer than ${CELL_POLL_DEAD_MIN_POLLED} polled cells`, () => {
const reach = new CellPollReachability()
const last = failFor(reach, 'us1', 0, CELL_POLL_DEAD_STREAK_MS)
const five = FLEET.slice(0, 5)
expect(reach.verdict('us1', five, last)).toMatchObject({ dead: false, withheld: 'sample' })
expect(reach.verdict('us1', FLEET.slice(0, 6), last)?.dead).toBe(true)
})
it('withholds when more than a third of the fleet fails at once: this director, not the cells', () => {
const reach = new CellPollReachability()
let last = 0
// 12 US cells: 12 of 34 is more than a third; 12 of 28 in the region is under half.
for (let index = 0; index < 12; index += 1) last = failFor(reach, `us${index}`, 0, CELL_POLL_DEAD_STREAK_MS)
expect(reach.verdict('us1', FLEET, last)).toMatchObject({ dead: false, withheld: 'fleet', fleetFailing: 12 })
reach.answered('us11')
expect(reach.verdict('us1', FLEET, last)?.dead).toBe(true)
})
it('withholds when half or more of the cell\'s region fails: a cross-region partition', () => {
const reach = new CellPollReachability()
let last = 0
// Every Asia cell: 6 of 34 passes the fleet guard; the region guard catches it.
for (let index = 0; index < 6; index += 1) last = failFor(reach, `asia${index}`, 0, CELL_POLL_DEAD_STREAK_MS)
expect(reach.verdict('asia0', FLEET, last)).toEqual({
dead: false,
withheld: 'region',
fleetFailing: 6,
fleetPolled: 34,
regionFailing: 6,
regionPolled: 6
})
// Half the region is still a partition.
for (const cellId of ['asia3', 'asia4', 'asia5']) reach.answered(cellId)
expect(reach.verdict('asia0', FLEET, last)?.withheld).toBe('region')
// A zone loss (2 of 6) is cells.
reach.answered('asia2')
expect(reach.verdict('asia0', FLEET, last)).toMatchObject({ dead: true, regionFailing: 2, regionPolled: 6 })
})
it('a staggered partition never judges its first cell dead: young streaks count as failing', () => {
const reach = new CellPollReachability()
// Six Asia cells cut off at 0-2 s offsets, each polled once a second for two minutes.
const offsets = [0, 400, 800, 1_200, 1_600, 2_000]
for (let at = 0; at <= 120_000; at += 100) {
offsets.forEach((offset, index) => {
if (at >= offset && (at - offset) % 1_000 === 0) reach.unanswered(`asia${index}`, at)
})
for (let index = 0; index < 6; index += 1) {
expect(reach.verdict(`asia${index}`, FLEET, at)?.dead ?? false).toBe(false)
}
}
})
it('forgets cells no longer listed', () => {
const reach = new CellPollReachability()
const last = failFor(reach, 'us1', 0, CELL_POLL_DEAD_STREAK_MS)
reach.forget(new Set(['us2']))
expect(reach.verdict('us1', FLEET, last)).toBeNull()
})
})
const INCARNATION = '11111111-1111-4111-8111-111111111111'
const feedCell = (cellId: string): SeatFeedCell => ({
cellId,
cellUrl: `https://${cellId}.example.test`,
region: 'us-central1',
heartbeatExpiresAt: 2_000_000_000_000,
requiredForComplete: true,
capacityRequests: 10
})
const CELLS = ['c1', 'c2', 'c3', 'c4', 'c5', 'c6'].map(feedCell)
// Legacy rows: they answer 404 (no feed), so they are neither the sample nor the denominator.
const LEGACY = Array.from({ length: 16 }, (_, index) => feedCell(`legacy${index}`))
const page = (cellId: string) =>
new Response(JSON.stringify({ v: 1, cellId, incarnation: INCARNATION, seq: 0, at: 1, full: [] }), {
status: 200,
headers: { 'content-type': 'application/json' }
})
const answer = (status: number) => new Response('{}', { status })
// Every cell serves a feed page on the first round; `respond` answers every round after it.
function poller(
respond: (cellId: string) => Response | Promise<Response>,
options: { identityFails?: boolean; cells?: SeatFeedCell[]; neverServed?: ReadonlySet<string> } = {}
) {
let now = 1_000
let round = 0
const logs: string[] = []
const started = startShadowSeatPoller(
{ role: 'director', rehomeAudience: 'https://relay.example.test/rehome', shadowSeatFeedCells: 'all' },
{
listCells: async () => options.cells ?? CELLS,
fetch: vi.fn(async (input: string | URL | Request) => {
const cellId = new URL(String(input)).hostname.split('.')[0] ?? ''
if (cellId.startsWith('legacy')) return answer(404)
if (options.neverServed?.has(cellId)) return answer(502)
return round === 0 ? page(cellId) : await respond(cellId)
}),
identityToken: vi.fn(async () => {
if (options.identityFails && round > 0) throw new Error('metadata_unavailable')
return 'token-1'
}),
now: () => now,
pollMs: 3_600_000,
log: (line) => logs.push(line)
}
)
if (!started) throw new Error('poller not started')
return {
...started,
logs,
// One poll a second for `ms`, after the first round of feed pages.
run: async (ms: number) => {
await started.tick()
round = 1
for (let elapsed = 0; elapsed <= ms; elapsed += 1_000) {
now += 1_000
await started.tick()
}
return now
}
}
}
describe('seat-feed poller reachability', () => {
it('counts timeouts, connection failures and load-balancer 502/503/504 as unanswered', async () => {
for (const fail of [
() => Promise.reject(new Error('connect ECONNREFUSED')),
() => answer(502),
() => answer(503),
() => answer(504)
]) {
const cells = poller((cellId) => (cellId === 'c2' ? fail() : page(cellId)))
const at = await cells.run(CELL_POLL_DEAD_STREAK_MS)
cells.stop()
expect(cells.deadCellVerdict('c2', at)?.dead).toBe(true)
expect(cells.deadCellVerdict('c1', at)).toBeNull()
}
})
it('treats any status the cell itself sends as alive: 401, 404, 500', async () => {
for (const status of [401, 404, 500]) {
const cells = poller((cellId) => (cellId === 'c2' ? answer(status) : page(cellId)))
const at = await cells.run(CELL_POLL_DEAD_STREAK_MS)
cells.stop()
expect(cells.deadCellVerdict('c2', at)).toBeNull()
}
})
it('does not blame a cell for a token this director cannot mint', async () => {
const cells = poller((cellId) => page(cellId), { identityFails: true })
const at = await cells.run(CELL_POLL_DEAD_STREAK_MS)
cells.stop()
expect(cells.deadCellVerdict('c2', at)).toBeNull()
})
it('counts only cells that served a feed: legacy 404 rows neither fill the sample nor dilute it', async () => {
// Staging's shape: a few real cells and 16 legacy rows. Every real cell fails.
const cells = poller(() => answer(502), { cells: [...CELLS.slice(0, 3), ...LEGACY] })
const at = await cells.run(CELL_POLL_DEAD_STREAK_MS)
cells.stop()
expect(cells.deadCellVerdict('c2', at)).toMatchObject({
dead: false,
withheld: 'sample',
fleetFailing: 3,
fleetPolled: 3
})
})
it('a director restarted during a lone outage still judges the dead cell: the candidate is always sampled', async () => {
// c2 died before this director started, so it never served it a feed page.
const cells = poller((cellId) => (cellId === 'c2' ? answer(502) : page(cellId)), {
neverServed: new Set(['c2'])
})
const at = await cells.run(CELL_POLL_DEAD_STREAK_MS)
cells.stop()
expect(cells.deadCellVerdict('c2', at)).toMatchObject({
dead: true,
fleetFailing: 1,
fleetPolled: 6,
regionFailing: 1,
regionPolled: 6
})
})
it('a partition that began before the restart withholds: the candidate is its region\'s only sample', async () => {
const asia = ['a1', 'a2', 'a3'].map((cellId) => ({ ...feedCell(cellId), region: 'asia-east2' as const }))
const cells = poller((cellId) => (cellId.startsWith('a') ? answer(502) : page(cellId)), {
cells: [...CELLS, ...asia],
neverServed: new Set(['a1', 'a2', 'a3'])
})
const at = await cells.run(CELL_POLL_DEAD_STREAK_MS)
cells.stop()
expect(cells.deadCellVerdict('a1', at)).toMatchObject({
dead: false,
withheld: 'region',
fleetPolled: 7,
regionFailing: 1,
regionPolled: 1
})
})
it('counts no cell without capacity, the candidate included', async () => {
const empty = (cell: SeatFeedCell) => ({ ...cell, capacityRequests: 0 })
const peersEmpty = poller((cellId) => (cellId === 'c2' ? answer(502) : page(cellId)), {
cells: CELLS.map((cell) => (['c4', 'c5', 'c6'].includes(cell.cellId) ? empty(cell) : cell))
})
const at = await peersEmpty.run(CELL_POLL_DEAD_STREAK_MS)
peersEmpty.stop()
expect(peersEmpty.deadCellVerdict('c2', at)).toMatchObject({ withheld: 'sample', fleetPolled: 3 })
const candidateEmpty = poller((cellId) => (cellId === 'c2' ? answer(502) : page(cellId)), {
cells: CELLS.map((cell) => (cell.cellId === 'c2' ? empty(cell) : cell))
})
const candidateAt = await candidateEmpty.run(CELL_POLL_DEAD_STREAK_MS)
candidateEmpty.stop()
expect(candidateEmpty.deadCellVerdict('c2', candidateAt)).toMatchObject({ withheld: 'sample', fleetPolled: 5 })
})
it('logs every withheld verdict, sample included, once a minute per cell, for the alert', async () => {
const down = new Set(['c1', 'c2', 'c3'])
const cells = poller((cellId) => (down.has(cellId) ? answer(502) : page(cellId)))
const at = await cells.run(CELL_POLL_DEAD_STREAK_MS)
cells.stop()
expect(cells.deadCellVerdict('c2', at)).toMatchObject({ dead: false, withheld: 'fleet' })
cells.deadCellVerdict('c2', at + 1_000)
const withheld = cells.logs.filter((line) => line.includes('orca_relay_dead_cell_verdict_withheld'))
expect(withheld).toHaveLength(1)
expect(JSON.parse(withheld[0]!)).toMatchObject({ cellId: 'c2', withheld: 'fleet', fleetFailing: 3, fleetPolled: 6 })
const small = poller((cellId) => (cellId === 'c2' ? answer(502) : page(cellId)), { cells: CELLS.slice(0, 3) })
const smallAt = await small.run(CELL_POLL_DEAD_STREAK_MS)
small.stop()
expect(small.deadCellVerdict('c2', smallAt)).toMatchObject({ withheld: 'sample' })
expect(small.logs.some((line) => line.includes('"withheld":"sample"'))).toBe(true)
})
})