Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import { createHash } from 'node:crypto'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore, RelayHomeCellUnavailableError } from './assignment-store.js'
import { ReservePaceError } from './reserve-assignment.js'
import {
encodeMembership,
type CellAdmissionMembership,
type CellAdmissionState
} from './cell-admission-selector.js'
import type { RelayCellConfig } from './config.js'
import {
consumeRelayCellInventoryHold,
openInMemoryRelayDatabase,
type RelayDatabase,
type RelayLockOptions,
type RelayTransactionOptions,
type SqlRow
} from './database.js'
// migration-only also holds evacuation targets, Asia rollback and failed-wave/new cells; only same-cap isolate writes rollIsolatedCells, and restore clears its stamp atomically (orca-relay-operations.md).
const HEARTBEAT_TTL_MS = 45_000
const START_MS = 100
const IDENTITY = { userId: 'user-a', relayHostId: 'host000000000001' }
// A connection-limited cell is what makes deadCellFenceRequirement report capped,
// which is the branch that answers 503 relay_home_cell_unavailable.
const CAPPED = {
capacityRequests: 1_000,
connectionHardCap: 600,
connectionUnobservedBound: 50
} as const
const CELLS: RelayCellConfig[] = [
{ id: 'us-c1', url: 'https://us-c1.example.com', region: 'us-central1', ...CAPPED },
{ id: 'us-c2', url: 'https://us-c2.example.com', region: 'us-central1', ...CAPPED },
{ id: 'asia-c1', url: 'https://asia-c1.example.com', region: 'asia-east2', ...CAPPED }
]
const databases: RelayDatabase[] = []
afterEach(async () => {
vi.restoreAllMocks()
for (const database of databases.splice(0)) await database.close()
})
// Counts the reads this change adds, so the general hot path can be held to a
// single admission lookup and no migration lookup at all.
class QueryCountingDatabase implements RelayDatabase {
readonly sql: string[] = []
// Fails the first statement containing this fragment, so a test can roll a
// transaction back at a chosen point.
failOnce: string | undefined
// Locked statements that report the row busy, in order: each fragment fires
// once, on the first locked statement containing it after the one before.
lockUnavailableSequence: string[] = []
constructor(private readonly delegate: RelayDatabase) {}
get dialect(): 'sqlite' | 'postgres' | undefined {
return this.delegate.dialect
}
count(fragment: string): number {
return this.sql.filter((statement) => statement.includes(fragment)).length
}
record(sql: string): void {
this.sql.push(sql)
if (this.failOnce !== undefined && sql.includes(this.failOnce)) {
this.failOnce = undefined
throw new Error('injected_placement_write_failure')
}
}
async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
this.record(sql)
return await this.delegate.query(sql, params)
}
async queryLocked(
sql: string,
params?: unknown[],
options?: RelayLockOptions
): Promise<SqlRow[]> {
this.recordLocked(sql)
return await this.delegate.queryLocked(sql, params, options)
}
recordLocked(sql: string): void {
this.record(sql)
const next = this.lockUnavailableSequence[0]
if (next !== undefined && sql.includes(next)) {
this.lockUnavailableSequence.shift()
throw new Error('database_lock_unavailable')
}
}
async transaction<T>(
operation: (transaction: RelayDatabase) => Promise<T>,
options?: RelayTransactionOptions
): Promise<T> {
return await this.delegate.transaction(
async (inner) => await operation(this.recording(inner)),
options
)
}
async close(): Promise<void> {
await this.delegate.close()
}
// A nested transaction shares this one's tape, so a whole assign() is
// measured as a unit.
private recording(inner: RelayDatabase): RelayDatabase {
return {
dialect: inner.dialect,
query: async (sql, params) => {
this.record(sql)
return await inner.query(sql, params)
},
queryLocked: async (sql, params, options) => {
this.recordLocked(sql)
return await inner.queryLocked(sql, params, options)
},
transaction: async (operation, options) => await inner.transaction(operation, options),
close: async () => await inner.close()
}
}
}
interface Harness {
store: RelayAssignmentStore
database: RelayDatabase
counter: QueryCountingDatabase
heartbeat: (cell: RelayCellConfig, at?: number) => Promise<void>
setNow: (value: number) => void
/** The real isolate path: one selector apply that names the cell it stamps. */
isolateForRoll: (cellId: string) => Promise<void>
/** The real restore path: back to 'general', which clears the stamp. */
restore: (cellId: string) => Promise<void>
/** An admission move with no stamp — every flow that is not a same-cap roll. */
park: (cellId: string, state: CellAdmissionState) => Promise<void>
rollIsolatedAt: (cellId: string) => Promise<number | null>
}
async function setup(cells: RelayCellConfig[] = CELLS): Promise<Harness> {
const inner = await openInMemoryRelayDatabase()
databases.push(inner)
const counter = new QueryCountingDatabase(inner)
let now = START_MS
const store = new RelayAssignmentStore(counter, () => now, {
requireLiveCells: true,
heartbeatTtlMs: HEARTBEAT_TTL_MS
})
await store.reconcileCells(cells, true)
const heartbeat = async (cell: RelayCellConfig, at?: number): Promise<void> => {
const previous = now
if (at !== undefined) now = at
try {
await store.recordCellHeartbeat({
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: `1111111${cells.indexOf(cell)}-1111-4111-8111-111111111111`,
startedAt: 50,
ready: true,
observedRequests: 0,
region: cell.region,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionHardCap: cell.connectionHardCap ?? 600,
connectionUnobservedBound: cell.connectionUnobservedBound ?? 50
})
} finally {
if (at !== undefined) now = previous
}
}
for (const cell of cells) await heartbeat(cell)
const applySelector = async (
states: Record<string, CellAdmissionState>,
rollIsolatedCells?: string[]
): Promise<void> => {
const current = await store.inspectCellAdmissionSelector()
const membership: CellAdmissionMembership = {
existingOnly: [],
migrationOnly: [],
general: []
}
for (const cell of cells) {
const state =
states[cell.id] ??
(current.selector.membership.migrationOnly.includes(cell.id)
? 'migration-only'
: current.selector.membership.existingOnly.includes(cell.id)
? 'existing-only'
: 'general')
if (state === 'migration-only') membership.migrationOnly.push(cell.id)
else if (state === 'existing-only') membership.existingOnly.push(cell.id)
else membership.general.push(cell.id)
}
await store.applyCellAdmissionSelector({
attemptId: `attempt-${current.selector.generation}-${Object.keys(states).join('-')}`,
expectedGeneration: current.selector.generation,
...(current.selector.generation === 0
? {
expectedMembershipSha256: createHash('sha256')
.update(encodeMembership(current.selector.membership))
.digest('hex')
}
: {}),
membership,
...(rollIsolatedCells ? { rollIsolatedCells } : {})
})
}
return {
store,
database: inner,
counter,
heartbeat,
setNow: (value: number) => (now = value),
isolateForRoll: async (cellId) =>
await applySelector({ [cellId]: 'migration-only' }, [cellId]),
restore: async (cellId) => await applySelector({ [cellId]: 'general' }),
park: async (cellId, state) => await applySelector({ [cellId]: state }),
rollIsolatedAt: async (cellId) => {
const rows = await inner.query(
`SELECT roll_isolated_at FROM relay_cell_admission WHERE cell_id = ?`,
[cellId]
)
const value = rows[0]?.['roll_isolated_at']
return value === undefined || value === null ? null : Number(value)
}
}
}
async function insertMigration(
database: RelayDatabase,
input: {
sourceCellId: string
targetCellId: string
assignmentEpoch: number
leases: number
settled?: 'completed' | 'aborted'
}
): Promise<void> {
await database.query(
`INSERT INTO relay_assignment_migrations
(user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
assignment_epoch, source_request_units, target_reserved_units, expires_at,
target_registered_at, completed_at, aborted_at, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, 1, 1, ?, NULL, ?, ?, ?, ?)`,
[
IDENTITY.userId,
IDENTITY.relayHostId,
input.sourceCellId,
input.targetCellId,
input.assignmentEpoch - 1,
input.assignmentEpoch,
START_MS + 900_000,
input.settled === 'completed' ? START_MS : null,
input.settled === 'aborted' ? START_MS : null,
START_MS,
START_MS
]
)
await database.query(
`UPDATE relay_assignments SET migration_leases = ?
WHERE user_id = ? AND relay_host_id = ?`,
[input.leases, IDENTITY.userId, IDENTITY.relayHostId]
)
}
type ConsoleWarnSpy = { mock: { calls: unknown[][] } }
function jsonEvents(warn: ConsoleWarnSpy, event: string): unknown[] {
return warn.mock.calls
.map((call) => (typeof call[0] === 'string' ? call[0] : ''))
.filter((line) => line.includes(`"${event}"`))
.map((line) => JSON.parse(line) as unknown)
}
describe('re-placing a host off a cell isolated for a roll', () => {
it('leaves a host on a live general cell untouched, at one added admission read', async () => {
const { store, counter } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
counter.sql.length = 0
const second = await store.assign(IDENTITY, 'us-central1')
// Identical, not merely same-celled: the clock has not moved, so every
// field including the lease deadline must match; only the first was placed.
expect(second).toEqual({ ...first, placed: undefined })
// One row read answers both the stranded rule and the roll-stamp check.
expect(counter.count('relay_cell_admission')).toBe(1)
expect(counter.count('relay_assignment_migrations')).toBe(0)
// No placement: the sticky lane never reaches the fleet-wide inventory.
expect(counter.count('ORDER BY cell_id ASC')).toBe(0)
})
it('keeps a host pinned to a migration-only cell with no roll stamp', async () => {
// Unstamped non-roll parking must retain hosts or undo evacuation/Asia rollback/operator intent.
const { store, park, rollIsolatedAt } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await park(first.cellId, 'migration-only')
expect(await rollIsolatedAt(first.cellId)).toBeNull()
const pinned = await store.assign(IDENTITY, 'us-central1')
expect(pinned).toMatchObject({ cellId: first.cellId, assignmentEpoch: first.assignmentEpoch })
expect(pinned.placed).toBeUndefined()
})
it('re-places a host once the roll stamp is set', async () => {
const { store, counter, isolateForRoll, rollIsolatedAt } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
expect(await rollIsolatedAt(first.cellId)).toBe(START_MS)
counter.sql.length = 0
const moved = await store.assign(IDENTITY, 'us-central1')
expect(moved.cellId).not.toBe(first.cellId)
expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1)
expect(moved.placed).toBe(true)
expect(counter.count('relay_assignment_migrations')).toBeGreaterThan(0)
// Durable: the next dial does not bounce back to the isolated cell.
const kept = await store.assign(IDENTITY, 'us-central1')
expect(kept).toMatchObject({ cellId: moved.cellId, assignmentEpoch: moved.assignmentEpoch })
expect(kept.placed).toBeUndefined()
})
it.each([
{ label: 'just inside the bound', age: 2 * 60 * 60_000 - 1, moves: true },
{ label: 'just outside the bound', age: 2 * 60 * 60_000 + 1, moves: false }
])('treats a stamp $label as $moves', async ({ age, moves }) => {
// Past a normal ~15-minute roll, stamps may be failed-wave or old-restore orphans; keep possibly healthy hosts pinned for operator recovery.
const { store, heartbeat, isolateForRoll, setNow } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
const later = START_MS + age
setNow(later)
for (const cell of CELLS) await heartbeat(cell, later)
const grant = await store.assign(IDENTITY, 'us-central1')
if (moves) {
expect(grant.cellId).not.toBe(first.cellId)
expect(grant.assignmentEpoch).toBe(first.assignmentEpoch + 1)
} else {
expect(grant.cellId).toBe(first.cellId)
expect(grant.assignmentEpoch).toBe(first.assignmentEpoch)
}
})
// Why its own site: the drain-return lane's service time is either this lock or
// the inventory, and only a separate p99 can say which.
it('samples the regional target rows it locks under their own hold site', async () => {
const { store, database, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
consumeRelayCellInventoryHold(database)
await store.assign(IDENTITY, 'us-central1')
expect(consumeRelayCellInventoryHold(database)).toMatchObject({
cellInventoryHolds: 1,
cellInventoryHoldMaxSite: 'isolated-replacement',
isolatedReplacementHolds: 1
})
})
it('stops re-placing once restore clears the stamp', async () => {
const { store, isolateForRoll, restore, rollIsolatedAt } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
await restore(first.cellId)
expect(await rollIsolatedAt(first.cellId)).toBeNull()
expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
cellId: first.cellId,
assignmentEpoch: first.assignmentEpoch
})
})
it('keeps a host pinned to an existing-only cell that still serves it', async () => {
// Why: existing-only cells serve the hosts they already hold (PR #194). Only
// assignmentStrandedOnUnservedCell may release that pin, and only on proof
// the cell stopped serving this host.
const { store, database, heartbeat, park, setNow } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await park(first.cellId, 'existing-only')
await database.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, 'control:live-1', 'control', ?, 1, ?, ?)`,
[IDENTITY.userId, IDENTITY.relayHostId, first.cellId, START_MS + 600_000, START_MS]
)
// Past the stranded rule's minimum grant age, which outlives the heartbeat TTL.
setNow(START_MS + 61_000)
for (const cell of CELLS) await heartbeat(cell, START_MS + 61_000)
expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
cellId: first.cellId,
assignmentEpoch: first.assignmentEpoch
})
})
it('keeps a host pinned on an unstamped cell whose migration has completed', async () => {
// Completed evacuations await separate promotion in migration-only; re-placement would undo them.
const { store, database, park } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await insertMigration(database, {
sourceCellId: 'us-c2',
targetCellId: first.cellId,
assignmentEpoch: first.assignmentEpoch,
leases: 0,
settled: 'completed'
})
await park(first.cellId, 'migration-only')
expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
cellId: first.cellId,
assignmentEpoch: first.assignmentEpoch
})
})
it('re-places a stamped cell whose migration has completed', async () => {
const { store, database, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await insertMigration(database, {
sourceCellId: 'us-c2',
targetCellId: first.cellId,
assignmentEpoch: first.assignmentEpoch,
leases: 0,
settled: 'completed'
})
await isolateForRoll(first.cellId)
expect((await store.assign(IDENTITY, 'us-central1')).cellId).not.toBe(first.cellId)
})
it('keeps the pin while a migration lease is outstanding', async () => {
const { store, database, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await insertMigration(database, {
sourceCellId: first.cellId,
targetCellId: 'us-c2',
assignmentEpoch: first.assignmentEpoch,
leases: 1
})
await isolateForRoll(first.cellId)
expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
cellId: first.cellId,
assignmentEpoch: first.assignmentEpoch
})
})
it('keeps the pin while a migration row is open but its lease has lapsed', async () => {
// Migration rows outlive their 15-minute counters and cannot roll back from non-general sources; re-placement would strand them.
const { store, database, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await insertMigration(database, {
sourceCellId: first.cellId,
targetCellId: 'us-c2',
assignmentEpoch: first.assignmentEpoch,
leases: 0
})
await isolateForRoll(first.cellId)
expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
cellId: first.cellId,
assignmentEpoch: first.assignmentEpoch
})
})
it('crosses a region boundary only when its own region has no room', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { store, isolateForRoll, park } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
const other = CELLS.find((cell) => cell.region === 'us-central1' && cell.id !== first.cellId)!
await park(other.id, 'migration-only')
await isolateForRoll(first.cellId)
warn.mockClear()
expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
cellId: 'asia-c1',
assignmentEpoch: first.assignmentEpoch + 1
})
expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([
{
event: 'orca_relay_sticky_replaced_off_isolated_cell',
fromCellId: first.cellId,
fromRegion: 'us-central1',
admissionState: 'migration-only',
toCellId: 'asia-c1',
region: 'asia-east2'
}
])
})
it('keeps the pin, and says so, when no region has a general cell with room', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { store, isolateForRoll, park } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
for (const cell of CELLS) {
if (cell.id !== first.cellId) await park(cell.id, 'migration-only')
}
await isolateForRoll(first.cellId)
warn.mockClear()
expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
cellId: first.cellId,
assignmentEpoch: first.assignmentEpoch
})
expect(jsonEvents(warn, 'orca_relay_sticky_replacement_deferred')).toEqual([
{
event: 'orca_relay_sticky_replacement_deferred',
reason: 'no_same_region_headroom',
cellId: first.cellId,
region: 'us-central1'
}
])
expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([])
})
it('prefers its own region over a less loaded cell elsewhere', async () => {
const { store, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
const moved = await store.assign(IDENTITY, 'us-central1')
expect(moved.region).toBe('us-central1')
expect(moved.cellId).not.toBe(first.cellId)
})
it('moves the only cell of a region to another region', async () => {
const { store, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'asia-east2')
expect(first.region).toBe('asia-east2')
await isolateForRoll(first.cellId)
// asia-c1 is the only Asia cell, so the only in-region candidate is gone.
const moved = await store.assign(IDENTITY, 'asia-east2')
expect(moved.region).toBe('us-central1')
expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1)
})
describe('with a reserve cell in its own region', () => {
const onReserve = {
...IDENTITY,
cellId: 'asia-reserve',
cellUrl: 'https://asia-reserve.example.com',
assignmentEpoch: 9,
leaseExpiresAt: START_MS + 1_000_000,
region: 'asia-east2' as const
}
it('books the reserve cell before leaving the region, and the canary still counts the move', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { store, database, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'asia-east2')
await isolateForRoll(first.cellId)
warn.mockClear()
const regionReserve = vi.fn(async () => onReserve)
await expect(
store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
).resolves.toEqual({ ...onReserve, placed: true })
expect(regionReserve).toHaveBeenCalledOnce()
expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([
{
event: 'orca_relay_sticky_replaced_off_isolated_cell',
fromCellId: first.cellId,
fromRegion: 'asia-east2',
admissionState: 'migration-only',
toCellId: 'asia-reserve',
region: 'asia-east2',
via: 'reserve'
}
])
// The refused in-region attempt wrote nothing.
expect(
await database.query(`SELECT cell_id, assignment_epoch FROM relay_assignments`)
).toEqual([{ cell_id: first.cellId, assignment_epoch: first.assignmentEpoch }])
})
it('leaves the region as before when the reserve cell has no room either', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { store, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'asia-east2')
await isolateForRoll(first.cellId)
warn.mockClear()
const regionReserve = vi.fn(async () => null)
const moved = await store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
expect(moved).toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 })
expect(regionReserve).toHaveBeenCalledOnce()
expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([
{ fromCellId: first.cellId, region: 'us-central1' }
])
})
it('treats a throttled reserve cell as no room, so the drain does not wait on its intake', async () => {
const { store, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'asia-east2')
await isolateForRoll(first.cellId)
const regionReserve = vi.fn(async () => {
throw new ReservePaceError(4)
})
await expect(
store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
).resolves.toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 })
expect(regionReserve).toHaveBeenCalledOnce()
})
it('counts a share-draw booking off the isolated cell for the canary', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { store, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'asia-east2')
await isolateForRoll(first.cellId)
warn.mockClear()
await expect(
store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { placeFresh: async () => onReserve })
).resolves.toEqual({ ...onReserve, placed: true })
expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([
{ fromCellId: first.cellId, toCellId: 'asia-reserve', via: 'reserve' }
])
})
describe('when no database cell has room anywhere', () => {
const onUsReserve = { ...onReserve, cellId: 'us-reserve', region: 'us-central1' as const }
const isolateWithNoDatabaseRoom = async () => {
const harness = await setup()
const first = await harness.store.assign(IDENTITY, 'asia-east2')
await harness.isolateForRoll(first.cellId)
for (const cell of CELLS.filter((entry) => entry.region === 'us-central1')) {
await harness.park(cell.id, 'migration-only')
}
return { ...harness, first }
}
// Today's rule: a live isolated cell keeps a host no other cell can take.
it('keeps the pin and never asks the spill', async () => {
const { store, first } = await isolateWithNoDatabaseRoom()
const reserveSpill = vi.fn(async () => onUsReserve)
const kept = await store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { reserveSpill })
expect(kept).toMatchObject({ cellId: first.cellId, assignmentEpoch: first.assignmentEpoch })
expect(reserveSpill).not.toHaveBeenCalled()
})
})
it('does not hold a host whose preferred region is elsewhere', async () => {
const { store, isolateForRoll, park } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
const other = CELLS.find((cell) => cell.region === 'us-central1' && cell.id !== first.cellId)!
await park(other.id, 'migration-only')
await isolateForRoll(first.cellId)
const regionReserve = vi.fn(async () => onReserve)
await expect(
store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
).resolves.toMatchObject({ cellId: 'asia-c1' })
expect(regionReserve).not.toHaveBeenCalled()
})
})
it('takes the dead-cell path when a stamped cell stops heartbeating', async () => {
// Why: the fence is the only proof that a cell we cannot reach has stopped
// serving the sockets it still holds. A stamp does not make it reachable.
const { store, heartbeat, isolateForRoll, setNow } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
const stale = START_MS + HEARTBEAT_TTL_MS + 1_000
setNow(stale)
for (const cell of CELLS) {
if (cell.id !== first.cellId) await heartbeat(cell, stale)
}
await expect(store.assign(IDENTITY, 'us-central1')).rejects.toBeInstanceOf(
RelayHomeCellUnavailableError
)
})
it('leaves the source cell activity leases in place', async () => {
// Contrast with the fence and stranded paths, which delete them: an isolated
// cell is alive and still owns drainable work behind those leases.
const { store, database, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await database.query(
`INSERT INTO relay_assignment_activity_leases
(user_id, relay_host_id, activity_id, activity_kind, cell_id,
request_units, expires_at, updated_at)
VALUES (?, ?, 'splice:keep-1', 'splice', ?, 1, ?, ?)`,
[IDENTITY.userId, IDENTITY.relayHostId, first.cellId, START_MS + 600_000, START_MS]
)
await isolateForRoll(first.cellId)
await store.assign(IDENTITY, 'us-central1')
expect(
await database.query(
`SELECT activity_id FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_id = 'splice:keep-1'`,
[IDENTITY.userId, IDENTITY.relayHostId]
)
).toHaveLength(1)
})
it('emits nothing when the placement rolls back after the decision', async () => {
// Log only committed moves; stdout cannot roll back with a failed lease/reservation transaction.
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { store, counter, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
warn.mockClear()
// The first write after the event is decided.
counter.failOnce = 'INSERT INTO relay_assignments'
await expect(store.assign(IDENTITY, 'us-central1')).rejects.toThrow(
'injected_placement_write_failure'
)
expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([])
// Prove the move rolled back so the missing event is required, not merely delayed.
expect(await store.resolve(IDENTITY)).toMatchObject({
cellId: first.cellId,
assignmentEpoch: first.assignmentEpoch
})
// Successful control proves event absence above comes from rollback, not a broken harness.
warn.mockClear()
const moved = await store.assign(IDENTITY, 'us-central1')
expect(moved.cellId).not.toBe(first.cellId)
expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toHaveLength(1)
})
it('logs one event naming both cells, the admission state and the region', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { store, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
warn.mockClear()
const moved = await store.assign(IDENTITY, 'us-central1')
expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([
{
event: 'orca_relay_sticky_replaced_off_isolated_cell',
fromCellId: first.cellId,
fromRegion: 'us-central1',
admissionState: 'migration-only',
toCellId: moved.cellId,
region: moved.region
}
])
})
describe('a host whose leases span two cells after the move', () => {
const NEIGHBOUR = { userId: 'user-b', relayHostId: 'host000000000002' }
async function reserved(database: RelayDatabase, cellId: string): Promise<number> {
const rows = await database.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cellId]
)
return Number(rows[0]!['reserved_requests'])
}
async function expectAccounting(database: RelayDatabase): Promise<void> {
for (const cell of CELLS) {
const leased = await database.query(
`SELECT COALESCE(SUM(request_units), 0) AS units
FROM relay_assignment_activity_leases WHERE cell_id = ?`,
[cell.id]
)
expect({ cell: cell.id, reserved: await reserved(database, cell.id) }).toEqual({
cell: cell.id,
reserved: Number(leased[0]!['units'])
})
}
}
async function counters(database: RelayDatabase): Promise<number[]> {
const rows = await database.query(
`SELECT reserved_controls, reserved_splices FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?`,
[IDENTITY.userId, IDENTITY.relayHostId]
)
return [Number(rows[0]!['reserved_controls']), Number(rows[0]!['reserved_splices'])]
}
// us-c1 is isolated with the host's control and splice on it; the host
// moves to us-c2, where a neighbour's splice outlives every other lease.
async function movedHost(): Promise<Harness> {
const harness = await setup()
const { store, isolateForRoll } = harness
const first = await store.assign(IDENTITY, 'us-central1')
expect(first.cellId).toBe('us-c1')
await store.activateControl(IDENTITY, {
cellId: 'us-c1',
assignmentEpoch: first.assignmentEpoch,
generation: 1
})
await store.acquireActivity(IDENTITY, {
activityId: 'splice:keep-1',
kind: 'splice',
cellId: 'us-c1'
})
await isolateForRoll('us-c1')
expect((await store.assign(NEIGHBOUR, 'us-central1')).cellId).toBe('us-c2')
await store.acquireActivity(NEIGHBOUR, {
activityId: 'splice:neighbour',
kind: 'splice',
cellId: 'us-c2',
expiresAt: START_MS + 10 * 60 * 60_000
})
expect((await store.assign(IDENTITY, 'us-central1')).cellId).toBe('us-c2')
expect(await counters(harness.database)).toEqual([2, 1])
await expectAccounting(harness.database)
return harness
}
it('is left to the lease sweep by aggregate expiry', async () => {
const { store, database, setNow } = await movedHost()
setNow(START_MS + 60 * 60_000)
await store.releaseExpiredActivity()
expect(await counters(database)).toEqual([2, 1])
await expectAccounting(database)
await store.releaseExpiredActivityLeases()
await store.releaseExpiredActivity()
expect(await counters(database)).toEqual([0, 0])
expect(await reserved(database, 'us-c1')).toBe(0)
expect(await reserved(database, 'us-c2')).toBe(2)
await expectAccounting(database)
})
it('frees the source leases on their own cell when the target dies', async () => {
const { store, database, setNow, heartbeat, restore } = await movedHost()
await restore('us-c1')
// Uncapped, so the dead-cell path needs no fence to move off it.
await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = 'us-c2'`)
setNow(START_MS + HEARTBEAT_TTL_MS + 5_000)
await heartbeat(CELLS[0]!)
await heartbeat(CELLS[2]!)
expect((await store.assign(IDENTITY, 'us-central1')).cellId).toBe('us-c1')
expect(await counters(database)).toEqual([1, 0])
expect(await store.releaseActivity(IDENTITY, 'control:us-c1:1')).toBe(false)
expect(await reserved(database, 'us-c1')).toBe(1)
expect(await reserved(database, 'us-c2')).toBe(3)
await expectAccounting(database)
})
})
it('retries a busy isolated attempt in its own tier, never under the all-rows lock', async () => {
const { store, counter, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
counter.sql.length = 0
// The first attempt finds a target row busy, then its retry finds the host's
// lease rows busy after it has already taken the tier's rows.
counter.lockUnavailableSequence = [
'SELECT * FROM relay_cells WHERE cell_id IN (',
'FROM relay_assignment_activity_leases'
]
const moved = await store.assign(IDENTITY, 'us-central1')
expect(counter.lockUnavailableSequence).toEqual([])
expect(moved).toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 })
expect(counter.count('SELECT * FROM relay_cells ORDER BY cell_id ASC')).toBe(0)
})
})
describe('classifying a reconnect whose home is isolated for a roll', () => {
const classify = { classifyHomeRollIsolation: true }
it('marks the host only while the roll stamp is current', async () => {
const { store, heartbeat, isolateForRoll, restore, setNow } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
await isolateForRoll(first.cellId)
expect(await store.resolve(IDENTITY, classify)).toMatchObject({
cellId: first.cellId,
homeCellRollIsolated: true
})
// The cell's own callers do not ask, and their read is unchanged.
expect(await store.resolve(IDENTITY)).not.toHaveProperty('homeCellRollIsolated')
const stale = START_MS + 2 * 60 * 60_000 + 1
setNow(stale)
for (const cell of CELLS) await heartbeat(cell, stale)
expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
setNow(START_MS)
await restore(first.cellId)
expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
})
it('does not mark a host parked without a stamp or held by a migration', async () => {
const { store, database, isolateForRoll, park } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await park(first.cellId, 'migration-only')
expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
await isolateForRoll(first.cellId)
await insertMigration(database, {
sourceCellId: first.cellId,
targetCellId: 'us-c2',
assignmentEpoch: first.assignmentEpoch,
leases: 1
})
expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
// A stalled migration whose lease counter lapsed still owns the epoch.
await database.query(`UPDATE relay_assignments SET migration_leases = 0`)
expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
})
it('reads the classification in the verification query itself', async () => {
const { store, counter, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await isolateForRoll(first.cellId)
counter.sql.length = 0
await store.resolve(IDENTITY, classify)
expect(counter.sql.filter((sql) => sql.includes('FROM relay_assignments'))).toHaveLength(1)
expect(counter.count('relay_cell_admission')).toBe(1)
})
it('still treats a host its draining cell already released as a drain return', async () => {
// A drain releases the host's control lease before it tells the host, so the redial
// finds no activity: the stamp, not a live lease, has to carry the classification.
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const { store, database, isolateForRoll } = await setup()
const first = await store.assign(IDENTITY, 'us-central1')
await store.activateControl(IDENTITY, {
cellId: first.cellId,
assignmentEpoch: first.assignmentEpoch,
generation: 1
})
await isolateForRoll(first.cellId)
expect(await store.releaseActivity(IDENTITY, `control:${first.cellId}:1`)).toBe(true)
expect(
await database.query(
`SELECT activity_id FROM relay_assignment_activity_leases WHERE user_id = ?`,
[IDENTITY.userId]
)
).toEqual([])
expect(await store.resolve(IDENTITY, classify)).toMatchObject({
cellId: first.cellId,
homeCellRollIsolated: true
})
warn.mockClear()
const moved = await store.assign(IDENTITY, 'us-central1')
expect(moved.cellId).not.toBe(first.cellId)
expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1)
expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([
{ fromCellId: first.cellId, toCellId: moved.cellId }
])
})
})