Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import { createHash } from 'node:crypto'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentRowBusyError, RelayAssignmentStore } from './assignment-store.js'
import {
encodeMembership,
type CellAdmissionMembership,
type CellAdmissionState
} from './cell-admission-selector.js'
import type { RelayCellConfig } from './config.js'
import {
consumeRelayCellInventoryHold,
consumeRelayDatabasePoolPressure,
openRelayDatabase,
type RelayDatabase
} from './database.js'
import { RelayPublicAssignmentAdmission } from './public-assignment-admission.js'
import {
openDelayedPostgresDatabase,
type StatementDelay
} from './test-fixtures/delayed-postgres-database.js'
// Reproduce Asia source-row release locks during reconnect placement: fleet locking fell to ~9/s at 6 releases/s and ~5/s at 18/s. Assert/report target-only locking avoids that contention.
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
// asia-east2 cell to the us-central1 database, measured in production.
const ASIA_ROUND_TRIP_MS = 171
const LOCAL_ROUND_TRIP_MS = 1
// Keep the measured contention rates; repeating them for longer only repeats the same locks.
const WINDOW_MS = 5_000
// Reconnecting hosts dialling the directors during the window. Above the
// ~6-15/s the production tail sustained, so the director side is never idle.
const DIAL_RATE_PER_SECOND = 20
// Production shapes: five director instances (production.tfvars), each with
// the default pool and sticky lane (variables.tf, config.ts), and an Asia
// cell's pool (validate-relay-asia-topology-plan.mjs).
const DIRECTOR_INSTANCES = 5
const DIRECTOR_POOL_MAX = 3
const STICKY_LANE = { maxConcurrent: 1, maxQueued: 64, waitMs: 2_000, minIntervalMs: 2_000 }
const CELL_POOL_MAX = 16
const DIRECTOR_APPLICATION = 'drain-release-contention-director'
const USER_PREFIX = 'drain-release-contention'
const NOW = 1_000_000
const CAPPED = {
capacityRequests: 6_000,
connectionHardCap: 3_000,
connectionUnobservedBound: 60
} as const
const SOURCE: RelayCellConfig = {
id: 'drain-release-contention-a-source',
url: 'https://drain-release-contention-source.example.test',
region: 'asia-east2',
...CAPPED
}
const TARGETS: RelayCellConfig[] = ['b', 'c', 'd'].map((suffix) => ({
id: `drain-release-contention-${suffix}-target`,
url: `https://drain-release-contention-${suffix}-target.example.test`,
region: 'asia-east2',
...CAPPED
}))
// Next to the database. Taken only once every Asia neighbour is at its cap.
const US_CELL: RelayCellConfig = {
id: 'drain-release-contention-e-us',
url: 'https://drain-release-contention-us.example.test',
region: 'us-central1',
...CAPPED
}
const CELLS = [SOURCE, ...TARGETS, US_CELL]
const RELEASED_ACTIVITY = `control:${SOURCE.id}:1`
// The desktop client books each host's next /v1/assign 5-5.5s after its last
// one, whatever Retry-After says (src/main/runtime/mobile-cloud-relay/relay-assign-rate-gate.ts).
const HOST_ASSIGN_MIN_INTERVAL_MS = 5_000
const HOST_ASSIGN_INTERVAL_JITTER_MS = 500
// A host still unplaced this long after its release counts as never placed.
const REDIAL_GIVE_UP_MS = 30_000
type Identity = { userId: string; relayHostId: string }
type Tally = Record<string, number>
type Instance = { database: RelayDatabase; store: RelayAssignmentStore }
type RunReport = {
roundTripMs: number
releaseRate: number
neighboursCapped: boolean
releases: { attempted: number; ok: number; failed: Tally; p50Ms: number; p95Ms: number }
dials: {
attempted: number
placed: number
placedInWindow: number
placedCrossRegion: number
admissionRejected: Tally
failed: Tally
}
activations: { ok: number; failed: Tally }
placementsPerSecond: number
director: {
lockUnavailable: number
lockTimeouts: number
holds: number
holdMsMax: number
// Mean director backends blocked on a lock per 20ms sample, of those active.
lockWaitingMean: number
activeMean: number
}
sourcePoolWaitersMax: number
}
type DepartingReport = {
dialDelayMs: number
neighboursCapped: boolean
hosts: number
placed: number
firstAttempt: { placed: number; rejected: Tally; failed: Tally }
// Row-busy refusals on any attempt, and errors no client would retry.
busyRefusals: number
// Hosts refused more than once, for any reason, before being placed.
hostsRefusedTwice: number
unexpected: Tally
redials: number
// Release start to the grant that re-placed the host, redials included.
timeToPlacedMs: { p50: number; p95: number; max: number }
placementsPerSecond: number
releases: { ok: number; failed: Tally }
lockWaitingMean: number
activeMean: number
}
function hostIdentity(index: number): Identity {
return {
userId: `${USER_PREFIX}-${index}`,
// Relay host ids are fixed-width opaque ids.
relayHostId: `drainrel${String(index).padStart(8, '0')}`
}
}
function failureCode(error: unknown): string {
if (error instanceof Error) {
const code = 'code' in error ? error.code : undefined
return typeof code === 'string' ? code : error.message
}
return String(error)
}
function count(tally: Tally, key: string): void {
tally[key] = (tally[key] ?? 0) + 1
}
function total(tally: Tally): number {
return Object.values(tally).reduce((sum, value) => sum + value, 0)
}
function percentile(values: number[], fraction: number): number {
if (values.length === 0) return 0
const sorted = [...values].sort((left, right) => left - right)
return Math.round(sorted[Math.min(sorted.length - 1, Math.floor(fraction * sorted.length))]!)
}
function mean(sum: number, samples: number): number {
return samples === 0 ? 0 : Number((sum / samples).toFixed(2))
}
// Fires `operation` at a fixed rate without awaiting it, like independent
// socket closes or dials would.
async function paced<T>(
ratePerSecond: number,
total: number,
operation: (index: number) => Promise<T>
): Promise<Promise<T>[]> {
const startedAt = performance.now()
const started: Promise<T>[] = []
for (let index = 0; index < total; index += 1) {
const dueMs = (index * 1_000) / ratePerSecond - (performance.now() - startedAt)
if (dueMs > 0) await new Promise((resolve) => setTimeout(resolve, dueMs))
started.push(operation(index))
}
return started
}
describePostgres('PostgreSQL drain releases against director placement', () => {
const storeOptions = { requireLiveCells: true, heartbeatTtlMs: 45_000 }
// One switch for every Asia pool: the source cell and each target cell.
const delay: StatementDelay = { enabled: false, delayMs: LOCAL_ROUND_TRIP_MS }
const directors: Instance[] = []
const servingCells = new Map<string, Instance>()
let observer: RelayDatabase
const reports: RunReport[] = []
const departingReports: DepartingReport[] = []
function admin(): Instance {
return directors[0]!
}
function servingCell(cellId: string): Instance {
const instance = servingCells.get(cellId)
if (!instance) throw new Error(`no cell pool for ${cellId}`)
return instance
}
async function applySelector(
states: Record<string, CellAdmissionState>,
rollIsolatedCells?: string[]
): Promise<void> {
const current = await admin().store.inspectCellAdmissionSelector()
// Built from relay_cells: the apply requires exact coverage of a shared fleet.
const fleet = await admin().database.query(
`SELECT cell_id FROM relay_cells ORDER BY cell_id ASC`
)
const membership: CellAdmissionMembership = {
existingOnly: [],
migrationOnly: [],
general: []
}
for (const row of fleet) {
const cellId = String(row['cell_id'])
const state =
states[cellId] ??
(current.selector.membership.existingOnly.includes(cellId)
? 'existing-only'
: current.selector.membership.migrationOnly.includes(cellId)
? 'migration-only'
: 'general')
if (state === 'existing-only') membership.existingOnly.push(cellId)
else if (state === 'migration-only') membership.migrationOnly.push(cellId)
else membership.general.push(cellId)
}
await admin().store.applyCellAdmissionSelector({
attemptId: `drain-release-${current.selector.generation}`,
expectedGeneration: current.selector.generation,
...(current.selector.generation === 0
? {
expectedMembershipSha256: createHash('sha256')
.update(encodeMembership(current.selector.membership))
.digest('hex')
}
: {}),
membership,
...(rollIsolatedCells ? { rollIsolatedCells } : {})
})
}
// Other Postgres files write admission through the generation-0 helpers, so
// the selector goes back to generation 0 exactly as found.
async function resetSelectorBoundary(): Promise<void> {
await admin().database.query(
`UPDATE relay_admission_selectors SET generation = 0, attempt_id = NULL
WHERE selector_id = 'general'`
)
await admin().database.query(
`DELETE FROM relay_admission_selector_intents WHERE attempt_id LIKE 'drain-release-%'`
)
await admin().store.reconcileCells([], false)
}
async function deleteHostRows(): Promise<void> {
for (const table of [
'relay_control_capabilities',
'relay_control_connection_reservations',
'relay_assignment_activity_leases',
'relay_assignment_migrations',
'relay_assignment_region_preferences',
'relay_assignments'
]) {
await admin().database.query(`DELETE FROM ${table} WHERE user_id LIKE '${USER_PREFIX}-%'`)
}
}
// `atCap` reports those cells at their connection cap: no headroom left.
async function heartbeatAll(atCap: RelayCellConfig[] = []): Promise<void> {
for (const [index, config] of CELLS.entries()) {
await admin().store.recordCellHeartbeat({
cellId: config.id,
cellUrl: config.url,
cellIncarnation: `2222222${index}-2222-4222-8222-222222222222`,
startedAt: NOW - 1_000,
ready: true,
observedRequests: 0,
region: config.region,
// The cell checks enforced = total + in-flight + reserved.
totalConnections: atCap.includes(config) ? CAPPED.connectionHardCap : 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: atCap.includes(config) ? CAPPED.connectionHardCap : 0,
connectionHardCap: CAPPED.connectionHardCap,
connectionUnobservedBound: CAPPED.connectionUnobservedBound
})
}
}
// Holds `cellIds`' relay_cells rows in an open transaction until the returned release runs.
async function holdCellRows(cellIds: string[]): Promise<() => Promise<void>> {
let release!: () => void
const released = new Promise<void>((resolve) => (release = resolve))
let held!: () => void
const acquired = new Promise<void>((resolve) => (held = resolve))
const holder = observer.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT cell_id FROM relay_cells WHERE cell_id IN (${cellIds.map(() => '?').join(', ')})
ORDER BY cell_id ASC`,
cellIds
)
held()
await released
})
await acquired
return async () => {
release()
await holder
}
}
// Every host holds a live control lease on the source, then the source is
// isolated for a roll: the state an Asia drain starts from.
async function seed(hosts: Identity[]): Promise<void> {
await deleteHostRows()
await admin().database.query(
`UPDATE relay_cells SET reserved_requests = 0 WHERE cell_id LIKE '${USER_PREFIX}-%'`
)
// Restoring the source to general also clears the previous run's roll stamp.
await applySelector({
[SOURCE.id]: 'general',
...Object.fromEntries(TARGETS.map((target) => [target.id, 'migration-only' as const]))
})
for (const identity of hosts) {
const grant = await admin().store.assign(identity, 'asia-east2', 'asia-east2')
expect(grant.cellId).toBe(SOURCE.id)
await admin().store.activateControl(identity, {
cellId: SOURCE.id,
assignmentEpoch: grant.assignmentEpoch,
generation: 1
})
}
await applySelector(
Object.fromEntries(TARGETS.map((target) => [target.id, 'general' as const]))
)
await applySelector({ [SOURCE.id]: 'migration-only' }, [SOURCE.id])
}
// Samples director backends so the report shows where director time goes: a
// failed or slow placement leaves no hold sample behind.
function sampleDirectorWaits(): { stop: () => Promise<{ waiting: number; active: number }> } {
let running = true
let samples = 0
let waiting = 0
let active = 0
let samplingError: unknown
const loop = (async () => {
while (running) {
const rows = await observer.query(
`SELECT wait_event_type FROM pg_stat_activity
WHERE application_name = ? AND state = 'active'`,
[DIRECTOR_APPLICATION]
)
samples += 1
active += rows.length
waiting += rows.filter((row) => row['wait_event_type'] === 'Lock').length
await new Promise((resolve) => setTimeout(resolve, 20))
}
})().catch((error: unknown) => {
samplingError = error
})
return {
stop: async () => {
running = false
await loop
if (samplingError) {
throw samplingError
}
return { waiting: mean(waiting, samples), active: mean(active, samples) }
}
}
}
async function run(
roundTripMs: number,
releaseRate: number,
neighboursCapped = false
): Promise<RunReport> {
const releaseCount = Math.round((releaseRate * WINDOW_MS) / 1_000)
const dialCount = Math.round((DIAL_RATE_PER_SECOND * WINDOW_MS) / 1_000)
const releasing = Array.from({ length: releaseCount }, (_, index) => hostIdentity(index))
const dialling = Array.from({ length: dialCount }, (_, index) =>
hostIdentity(releaseCount + index)
)
await seed([...releasing, ...dialling])
if (neighboursCapped) await heartbeatAll(TARGETS)
const releaseLatencies: number[] = []
const releaseFailed: Tally = {}
const releases = { attempted: 0, ok: 0, failed: releaseFailed }
const admissionRejected: Tally = {}
const dialFailed: Tally = {}
const dials = {
attempted: 0,
placed: 0,
placedInWindow: 0,
placedCrossRegion: 0,
admissionRejected,
failed: dialFailed
}
const activationFailed: Tally = {}
const activations = { ok: 0, failed: activationFailed }
const activationWork: Promise<void>[] = []
const stickyLanes = directors.map(() => new RelayPublicAssignmentAdmission(STICKY_LANE))
for (const instance of directors) consumeRelayCellInventoryHold(instance.database)
consumeRelayDatabasePoolPressure(servingCell(SOURCE.id).database)
delay.delayMs = roundTripMs
delay.enabled = true
const sampler = sampleDirectorWaits()
const startedAt = performance.now()
try {
const releaseWork = paced(releaseRate, releaseCount, async (index) => {
releases.attempted += 1
const began = performance.now()
try {
const released = await servingCell(SOURCE.id).store.releaseActivity(
releasing[index]!,
RELEASED_ACTIVITY
)
if (released) {
releases.ok += 1
} else {
count(releases.failed, 'lease_missing')
}
} catch (error) {
count(releases.failed, failureCode(error))
}
releaseLatencies.push(performance.now() - began)
})
// The reconnect path, round-robin over directors: sticky-lane admission,
// the durable-pin check, then the assign the route calls, which re-places
// off the isolated cell. The host then attaches to its new Asia cell.
const dialWork = paced(DIAL_RATE_PER_SECOND, dialCount, async (index) => {
const identity = dialling[index]!
const director = directors[index % directors.length]!
dials.attempted += 1
let rejection = 'unknown'
const lease = await stickyLanes[index % directors.length]!.acquire(
identity.relayHostId,
(reason) => {
rejection = reason
}
)
if (!lease) {
count(dials.admissionRejected, rejection)
return
}
try {
if (!(await director.store.resolve(identity))) {
count(dials.failed, 'unverified')
return
}
const grant = await director.store.assign(identity, 'asia-east2', 'asia-east2')
if (grant.cellId === SOURCE.id) {
count(dials.failed, 'kept_on_source')
return
}
dials.placed += 1
if (grant.region !== SOURCE.region) {
dials.placedCrossRegion += 1
}
if (performance.now() - startedAt <= WINDOW_MS) {
dials.placedInWindow += 1
}
activationWork.push(
servingCell(grant.cellId)
.store.activateControl(identity, {
cellId: grant.cellId,
assignmentEpoch: grant.assignmentEpoch,
generation: 1
})
.then(
() => {
activations.ok += 1
},
(error: unknown) => count(activations.failed, failureCode(error))
)
)
} catch (error) {
count(dials.failed, failureCode(error))
} finally {
lease.release()
}
})
const outcomes = await Promise.allSettled([...(await releaseWork), ...(await dialWork)])
for (const outcome of outcomes) {
if (outcome.status === 'rejected') {
throw outcome.reason
}
}
expect(releases.attempted).toBe(releaseCount)
expect(dials.attempted).toBe(dialCount)
expect(dials.placed + total(dials.failed) + total(dials.admissionRejected)).toBe(dialCount)
expect(dials.placed).toBeGreaterThan(0)
const activationOutcomes = await Promise.allSettled(activationWork)
for (const outcome of activationOutcomes) {
if (outcome.status === 'rejected') {
throw outcome.reason
}
}
const waits = await sampler.stop()
delay.enabled = false
if (neighboursCapped) {
await heartbeatAll()
}
const holds = directors.map((instance) => consumeRelayCellInventoryHold(instance.database))
const report: RunReport = {
roundTripMs,
releaseRate,
neighboursCapped,
releases: {
...releases,
p50Ms: percentile(releaseLatencies, 0.5),
p95Ms: percentile(releaseLatencies, 0.95)
},
dials,
activations,
placementsPerSecond: Number(((dials.placedInWindow * 1_000) / WINDOW_MS).toFixed(1)),
director: {
lockUnavailable: holds.reduce((sum, hold) => sum + hold.cellInventoryLockUnavailable, 0),
lockTimeouts: holds.reduce((sum, hold) => sum + hold.cellInventoryLockTimeouts, 0),
holds: holds.reduce((sum, hold) => sum + hold.cellInventoryHolds, 0),
holdMsMax: Math.round(Math.max(...holds.map((hold) => hold.cellInventoryHoldMsMax))),
lockWaitingMean: waits.waiting,
activeMean: waits.active
},
sourcePoolWaitersMax: consumeRelayDatabasePoolPressure(servingCell(SOURCE.id).database)
.databasePoolWaitersMax
}
reports.push(report)
console.info(JSON.stringify({ event: 'drain_release_cell_row_contention', ...report }))
return report
} finally {
delay.enabled = false
await sampler.stop()
}
}
// Production's shape: the host whose socket the draining cell closes is the
// one that redials. Its release locks its own assignment row first and holds
// it while queued on the source row, so its own placement can meet it.
async function runDepartingRedial(
dialDelayMs: number,
neighboursCapped: boolean
): Promise<DepartingReport> {
const releaseRate = 18
const hosts = Array.from({ length: (releaseRate * WINDOW_MS) / 1_000 }, (_, index) =>
hostIdentity(index)
)
await seed(hosts)
if (neighboursCapped) await heartbeatAll(TARGETS)
const stickyLanes = directors.map(() => new RelayPublicAssignmentAdmission(STICKY_LANE))
const firstRejected: Tally = {}
const firstFailed: Tally = {}
const releaseFailed: Tally = {}
const firstAttempt = { placed: 0, rejected: firstRejected, failed: firstFailed }
const releases = { ok: 0, failed: releaseFailed }
const timeToPlaced: number[] = []
let redials = 0
let busyRefusals = 0
let hostsRefusedTwice = 0
const unexpected: Tally = {}
let placedInWindow = 0
const activationWork: Promise<unknown>[] = []
delay.delayMs = ASIA_ROUND_TRIP_MS
delay.enabled = true
const sampler = sampleDirectorWaits()
const startedAt = performance.now()
try {
// One dial through the sticky lane; 'placed', or the reason it was not.
async function dial(identity: Identity, director: number): Promise<string> {
let rejection = 'unknown'
const lease = await stickyLanes[director]!.acquire(identity.relayHostId, (reason) => {
rejection = reason
})
if (!lease) {
return `rejected:${rejection}`
}
try {
if (!(await directors[director]!.store.resolve(identity))) {
return 'failed:unverified'
}
const grant = await directors[director]!.store.assign(
identity,
'asia-east2',
'asia-east2'
)
if (grant.cellId === SOURCE.id) {
return 'failed:kept_on_source'
}
activationWork.push(
servingCell(grant.cellId).store.activateControl(identity, {
cellId: grant.cellId,
assignmentEpoch: grant.assignmentEpoch,
generation: 1
})
)
return 'placed'
} catch (error) {
if (error instanceof RelayAssignmentRowBusyError) {
return `rejected:${error.message}`
}
return `failed:${failureCode(error)}`
} finally {
lease.release()
}
}
const work = paced(releaseRate, hosts.length, async (index) => {
const identity = hosts[index]!
const releasedAt = performance.now()
const release = servingCell(SOURCE.id)
.store.releaseActivity(identity, RELEASED_ACTIVITY)
.then(
(released) => {
if (released) {
releases.ok += 1
} else {
count(releases.failed, 'lease_missing')
}
},
(error: unknown) => count(releases.failed, failureCode(error))
)
await new Promise((resolve) => setTimeout(resolve, dialDelayMs))
const director = index % directors.length
for (let attempt = 0; ; attempt += 1) {
const sentAt = performance.now()
const outcome = await dial(identity, director)
if (outcome === `rejected:${new RelayAssignmentRowBusyError().message}`) {
busyRefusals += 1
}
if (outcome.startsWith('failed:')) {
count(unexpected, outcome.slice(7))
}
if (attempt === 0) {
if (outcome === 'placed') {
firstAttempt.placed += 1
} else if (outcome.startsWith('rejected:')) {
count(firstRejected, outcome.slice(9))
} else {
count(firstFailed, outcome.slice(7))
}
}
if (outcome === 'placed') {
const placedAt = performance.now()
timeToPlaced.push(placedAt - releasedAt)
if (placedAt - startedAt <= WINDOW_MS + dialDelayMs) {
placedInWindow += 1
}
break
}
if (!outcome.startsWith('rejected:')) {
break
}
if (attempt === 1) {
hostsRefusedTwice += 1
}
const nextAt =
sentAt + HOST_ASSIGN_MIN_INTERVAL_MS + Math.random() * HOST_ASSIGN_INTERVAL_JITTER_MS
if (nextAt - releasedAt > REDIAL_GIVE_UP_MS) {
break
}
redials += 1
await new Promise((resolve) => setTimeout(resolve, nextAt - performance.now()))
}
await release
})
const outcomes = await Promise.allSettled(await work)
for (const outcome of outcomes) {
if (outcome.status === 'rejected') {
throw outcome.reason
}
}
expect(firstAttempt.placed + total(firstRejected) + total(firstFailed)).toBe(hosts.length)
expect(timeToPlaced.length).toBeGreaterThan(0)
const activationOutcomes = await Promise.allSettled(activationWork)
for (const outcome of activationOutcomes) {
if (outcome.status === 'rejected') {
throw outcome.reason
}
}
const waits = await sampler.stop()
delay.enabled = false
if (neighboursCapped) {
await heartbeatAll()
}
const report: DepartingReport = {
dialDelayMs,
neighboursCapped,
hosts: hosts.length,
placed: timeToPlaced.length,
firstAttempt,
busyRefusals,
hostsRefusedTwice,
unexpected,
redials,
timeToPlacedMs: {
p50: percentile(timeToPlaced, 0.5),
p95: percentile(timeToPlaced, 0.95),
max: percentile(timeToPlaced, 1)
},
placementsPerSecond: Number(((placedInWindow * 1_000) / WINDOW_MS).toFixed(1)),
releases,
lockWaitingMean: waits.waiting,
activeMean: waits.active
}
departingReports.push(report)
console.info(JSON.stringify({ event: 'drain_departing_host_redial', ...report }))
return report
} finally {
delay.enabled = false
await sampler.stop()
}
}
beforeAll(async () => {
for (let index = 0; index < DIRECTOR_INSTANCES; index += 1) {
const database = await openRelayDatabase({
databaseUrl,
dataDir: '',
poolMax: DIRECTOR_POOL_MAX,
applicationName: DIRECTOR_APPLICATION
})
directors.push({
database,
store: new RelayAssignmentStore(database, () => NOW, storeOptions)
})
}
for (const config of CELLS) {
const database = openDelayedPostgresDatabase(
databaseUrl!,
config === US_CELL ? { enabled: false, delayMs: 0 } : delay,
CELL_POOL_MAX
)
servingCells.set(config.id, {
database,
store: new RelayAssignmentStore(database, () => NOW, storeOptions)
})
}
observer = await openRelayDatabase({ databaseUrl, dataDir: '', poolMax: 1 })
await deleteHostRows()
await admin().store.reconcileCells(CELLS, false)
await resetSelectorBoundary()
await heartbeatAll()
})
afterAll(async () => {
delay.enabled = false
if (reports.length > 0) console.table(reports.map(flattenReport))
if (departingReports.length > 0) console.table(departingReports.map(flattenDeparting))
if (directors.length > 0) {
await deleteHostRows()
// Cells before the selector rebuild, or its membership names missing rows.
for (const config of CELLS) {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cell_admission',
'relay_cell_regions',
'relay_cells'
]) {
await admin().database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [config.id])
}
}
await resetSelectorBoundary()
}
for (const instance of [...directors, ...servingCells.values()]) await instance.database.close()
await observer?.close()
})
it('keeps placing during a paced drain when the cells are next to the database', async () => {
for (const rate of [6, 18]) {
const report = await run(LOCAL_ROUND_TRIP_MS, rate)
expect(report.dials.failed).toEqual({})
// Failures, rejections and lock waits, not placements/s: pacing on a slow runner
// moves the rate without any lock being involved.
expect(total(report.dials.admissionRejected)).toBeLessThan(0.1 * report.dials.attempted)
expect(report.director.lockWaitingMean).toBeLessThan(0.5)
}
}, 120_000)
it('places a reconnecting host off the isolated source while that source row is held', async () => {
delay.enabled = false
await heartbeatAll()
const host = hostIdentity(9_000)
await seed([host])
// A drain release holds the source row for its whole transaction; target-only placement never asks for it.
const sourceHeld = await holdCellRows([SOURCE.id])
try {
const grant = await Promise.race([
directors[1]!.store.assign(host, 'asia-east2', 'asia-east2'),
new Promise<'blocked'>((resolve) => setTimeout(() => resolve('blocked'), 3_000))
])
expect(grant).not.toBe('blocked')
expect(TARGETS.map(({ id }) => id)).toContain(grant === 'blocked' ? null : grant.cellId)
} finally {
await sourceHeld()
}
// Control: the same hold on the target rows does block a placement, so the hold above was real.
const other = hostIdentity(9_001)
await seed([other])
const targetsHeld = await holdCellRows(TARGETS.map(({ id }) => id))
const placement = directors[2]!.store.assign(other, 'asia-east2', 'asia-east2')
try {
const raced = await Promise.race([
placement.then(() => 'placed' as const),
new Promise<'blocked'>((resolve) => setTimeout(() => resolve('blocked'), 1_000))
])
expect(raced).toBe('blocked')
} finally {
await targetsHeld()
}
expect((await placement).cellId).not.toBe(SOURCE.id)
}, 30_000)
it('keeps placing while an Asia drain holds the source row', async () => {
// Baseline sticky rejects were 90–140/200; assert outcomes, since runner pacing alone changes placements/s and the sampled lock-wait mean (reported, not asserted).
for (const [rate, neighboursCapped] of [
[6, false],
[18, false],
[18, true]
] as const) {
const report = await run(ASIA_ROUND_TRIP_MS, rate, neighboursCapped)
expect(report.dials.failed).toEqual({})
expect(total(report.dials.admissionRejected)).toBeLessThan(0.1 * report.dials.attempted)
expect(report.dials.placedCrossRegion).toBe(neighboursCapped ? report.dials.placed : 0)
if (rate === 18) {
// Fused counter/COMMIT removes the lock-held reply round trip; separate COMMIT once completed only 79/180 releases before pool timeouts.
expect(report.releases.ok).toBe(report.releases.attempted)
}
}
}, 240_000)
it('re-places departing hosts that redial after their own release', async () => {
for (const neighboursCapped of [false, true]) {
for (const dialDelayMs of [150, 400, 1_000]) {
const report = await runDepartingRedial(dialDelayMs, neighboursCapped)
expect(report.unexpected).toEqual({})
expect(report.placed).toBe(report.hosts)
// A busy own row is refused at once by design; what must not happen is
// the sticky slot backing up behind it. Before: 72-143 wait-timeouts at
// 400ms and 1s (the sampled lock-wait mean is reported, not asserted).
const { relay_assignment_row_busy: _refused, ...slotRejections } =
report.firstAttempt.rejected
expect(total(slotRejections)).toBeLessThan(0.1 * report.hosts)
// A redial that beats its own release meets its own row (16-80 of 180, by how many
// releases are still in flight at the dial) and is answered at once. That release
// has finished by the next dial, so no host is refused twice. Measured p95 5.6-6.5s;
// before the row-busy answer, p95 was 11-16s.
expect(report.hostsRefusedTwice).toBe(0)
expect(report.timeToPlacedMs.p95).toBeLessThanOrEqual(8_000)
}
}
}, 600_000)
})
function flattenReport(report: RunReport): Record<string, string | number> {
const failed = (tally: Tally): string =>
Object.entries(tally)
.map(([key, value]) => `${key}:${value}`)
.join(' ') || '-'
return {
rttMs: report.roundTripMs,
capped: report.neighboursCapped ? 'yes' : 'no',
crossRegion: report.dials.placedCrossRegion,
releasesPerSec: report.releaseRate,
releasesOk: report.releases.ok,
releasesFailed: failed(report.releases.failed),
releaseP95Ms: report.releases.p95Ms,
dialsPlaced: report.dials.placed,
placementsPerSec: report.placementsPerSecond,
dialsFailed: failed(report.dials.failed),
stickyRejected: failed(report.dials.admissionRejected),
lockTimeouts: report.director.lockTimeouts,
lockUnavailable: report.director.lockUnavailable,
lockWaiting: `${report.director.lockWaitingMean}/${report.director.activeMean}`,
activationsFailed: failed(report.activations.failed)
}
}
function flattenDeparting(report: DepartingReport): Record<string, string | number> {
const failed = (tally: Tally): string =>
Object.entries(tally)
.map(([key, value]) => `${key}:${value}`)
.join(' ') || '-'
return {
delayMs: report.dialDelayMs,
capped: report.neighboursCapped ? 'yes' : 'no',
placed: `${report.placed}/${report.hosts}`,
firstPlaced: report.firstAttempt.placed,
firstRejected: failed(report.firstAttempt.rejected),
busy: report.busyRefusals,
unexpected: failed(report.unexpected),
redials: report.redials,
placedP50Ms: report.timeToPlacedMs.p50,
placedP95Ms: report.timeToPlacedMs.p95,
placedMaxMs: report.timeToPlacedMs.max,
placementsPerSec: report.placementsPerSecond,
lockWaiting: `${report.lockWaitingMean}/${report.activeMean}`
}
}