Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
import {
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT,
REGIONAL_REHOME_SQL_FAILURES_LIMIT,
REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT
} from './regional-rehome-safety.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
describePostgres('PostgreSQL regional rehoming', () => {
let primary: RelayDatabase
let secondary: RelayDatabase
let sequence = 0
beforeAll(async () => {
primary = await openRelayDatabase({ databaseUrl, dataDir: '' })
secondary = await openRelayDatabase({ databaseUrl, dataDir: '' })
})
beforeEach(async () => await cleanup())
afterAll(async () => {
await cleanup()
await secondary.close()
await primary.close()
})
async function cleanup(): Promise<void> {
for (const table of ['relay_region_decisions', 'relay_control_capabilities']) {
await primary.query(`DELETE FROM ${table} WHERE user_id LIKE 'pg-rehome-user-%'`)
}
await primary.query(
`DELETE FROM relay_region_rehome_attempts WHERE user_id LIKE 'pg-rehome-user-%'`
)
await primary.query(`DELETE FROM relay_region_rehome_worker_state`)
await primary.query(`DELETE FROM relay_region_rehome_control`)
await primary.query(
`DELETE FROM relay_control_connection_reservations
WHERE user_id LIKE 'pg-rehome-user-%'`
)
await primary.query(
`DELETE FROM relay_assignment_migration_incarnations
WHERE user_id LIKE 'pg-rehome-user-%'`
)
await primary.query(
`DELETE FROM relay_assignment_activity_leases
WHERE user_id LIKE 'pg-rehome-user-%'`
)
await primary.query(
`DELETE FROM relay_assignment_migrations WHERE user_id LIKE 'pg-rehome-user-%'`
)
await primary.query(
`DELETE FROM relay_assignment_region_preferences
WHERE user_id LIKE 'pg-rehome-user-%'`
)
await primary.query(`DELETE FROM relay_assignments WHERE user_id LIKE 'pg-rehome-user-%'`)
for (const table of [
'relay_cell_rehome_safety',
'relay_cell_capabilities',
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_runtime',
'relay_cell_connection_limits',
'relay_cell_admission',
'relay_cell_regions',
'relay_cells'
]) {
await primary.query(`DELETE FROM ${table} WHERE cell_id LIKE 'pg-rehome-cell-%'`)
}
}
it('defaults to a closed correction cohort even with enabled durable control', async () => {
const context = await fixture()
const closed = new RelayAssignmentStore(primary, context.now, {
requireLiveCells: true,
heartbeatTtlMs: 45_000
})
expect(await cutover(closed, context.now())).toBeNull()
const preview = await closed.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.cohortPercent).toBe(0)
expect(preview.counts['outside-cohort']).toBe(1)
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('counts existing generic migrations against the optimization cap and preview', async () => {
const context = await fixture()
const safety = {
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
const before = await context.store.previewRegionalRehomeEligibility(safety)
expect(before.counts['eligible:us-central1-to-asia-east2']).toBe(1)
for (let index = 0; index < 8; index++) {
const identity = {
userId: `pg-rehome-user-budget-${sequence}-${index}`,
relayHostId: `budgethost${String(index).padStart(6, '0')}`
}
await context.store.assign(identity, undefined, 'us-central1')
await context.store.startEvacuation(identity, context.target.id)
}
const preview = await context.store.previewRegionalRehomeEligibility(safety)
expect(preview.openMigrations).toBe(8)
expect(preview.availableMigrationSlots).toBe(0)
expect(preview.counts['concurrent-migration-cap']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('preview excludes request capacity exhaustion before a claim', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cells SET capacity_requests = reserved_requests + 1 WHERE cell_id = ?`,
[context.target.id]
)
const preview = await context.store.previewRegionalRehomeEligibility({
observedAt: context.now(),
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
})
expect(preview.counts['no-target-headroom']).toBe(1)
expect(await cutover(context.store, context.now())).toBeNull()
})
it('claims through ambient per-cell sql retry noise', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cell_rehome_safety
SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT}
WHERE cell_id IN (?, ?)`,
[context.source.id, context.target.id]
)
expect(await cutover(context.store, context.now())).not.toBeNull()
})
it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
const context = await fixture()
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'asia-east2',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'asia-east2',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
})
it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
const context = await fixture({
sourceRegion: 'asia-east2',
targetRegion: 'us-central1'
})
const attempt = await cutover(context.store, context.now())
expect(attempt).toMatchObject({
preferredRegion: 'us-central1',
sourceCellId: context.source.id,
targetCellId: context.target.id
})
// The durable attempt row must accept the reverse direction too.
expect(
await primary.query(
`SELECT preferred_region, source_cell_id, target_cell_id
FROM relay_region_rehome_attempts WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([
{
preferred_region: 'us-central1',
source_cell_id: context.source.id,
target_cell_id: context.target.id
}
])
expect(
await primary.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
context.identity.userId
])
).toEqual([{ cell_id: context.target.id }])
})
it('leaves a host whose preference already matches its own region', async () => {
const context = await fixture({ preferredRegion: 'us-central1' })
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('leaves a host whose preference is older than the configured max age', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_region_decisions SET observed_at = ?
WHERE user_id = ? AND relay_host_id = ?`,
[context.now() - 24 * 60 * 60_000 - 1, context.identity.userId, context.identity.relayHostId]
)
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('leaves a host inside its per-host rehome cooldown, in either direction', async () => {
const context = await fixture({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
// A move this host already made, whichever way it went.
await primary.query(
`INSERT INTO relay_region_rehome_attempts
(attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
source_cell_incarnation, target_cell_id, target_cell_incarnation,
previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
completed_at, created_at, updated_at)
VALUES (?, ?, ?, 'us-central1', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?, ?)`,
[
`pg-rehome-cooldown-${context.identity.relayHostId}`,
context.identity.userId,
context.identity.relayHostId,
context.target.id,
'22222222-2222-4222-8222-222222222222',
context.source.id,
'11111111-1111-4111-8111-111111111111',
context.now(),
context.now() - 3 * 24 * 60 * 60_000 + 1,
context.now()
]
)
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true,
hostCooldownMs: 3 * 24 * 60 * 60_000
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 1,
migrations: 0
})
// One millisecond past the window the same host is a candidate again.
await primary.query(
`UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
[context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
)
await expect(cutover(context.store, context.now())).resolves.toMatchObject({
sourceCellId: context.source.id,
targetCellId: context.target.id
})
})
it('leaves a host whose preferred region holds no drainable cell', async () => {
// A cell that cannot be drained cannot be a target: the host would land
// where no later rehome could move it out again.
const context = await fixture({ targetProtocol: 0 })
await expect(cutover(context.store, context.now())).resolves.toBeNull()
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('skips an unclean cell without latching the control off', async () => {
const context = await fixture()
await primary.query(
`UPDATE relay_cell_rehome_safety
SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1}
WHERE cell_id = ?`,
[context.target.id]
)
expect(await cutover(context.store, context.now())).toBeNull()
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 1,
enabled: true
})
expect(await attemptAndMigrationCounts(context.identity)).toEqual({
attempts: 0,
migrations: 0
})
})
it('lets only one director claim a host', async () => {
const context = await fixture()
const claims = await Promise.all([
cutover(context.store, context.now()),
cutover(context.competingStore, context.now())
])
expect(claims.filter(Boolean).length).toBeGreaterThanOrEqual(1)
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '1' }])
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE user_id = ? AND completed_at IS NULL AND aborted_at IS NULL`,
[context.identity.userId]
)
).toEqual([{ count: '1' }])
})
it('increments the disable generation once across competing directors', async () => {
const context = await fixture()
const disabled = await Promise.all([
context.store.disableRegionalRehomeControl(),
context.competingStore.disableRegionalRehomeControl()
])
expect(disabled.sort()).toEqual([false, true])
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 2,
enabled: false
})
})
it('rechecks a preference changed while the assignment row is locked', async () => {
const context = await fixture()
const [candidate] = await context.store.selectIdleRegionalRehomeCandidates(
safety(context.now())
)
expect(candidate).toBeDefined()
let unlock!: () => void
let locked!: () => void
const lockedPromise = new Promise<void>((resolve) => (locked = resolve))
const unlockPromise = new Promise<void>((resolve) => (unlock = resolve))
const held = secondary.transaction(async (transaction) => {
await transaction.queryLocked(
`SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
[context.identity.userId, context.identity.relayHostId]
)
locked()
await unlockPromise
})
await lockedPromise
const claim = context.store.commitIdleRegionalRehome(candidate!, safety(context.now()))
try {
await vi.waitFor(async () => {
const [row] = await primary.query(
`SELECT COUNT(*) AS waiting FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'
AND query LIKE 'SELECT %relay_assignments%FOR UPDATE%'`
)
expect(Number(row?.waiting)).toBeGreaterThan(0)
})
await primary.query(
`UPDATE relay_region_decisions SET preferred_region = 'us-central1',
observed_at = ? WHERE user_id = ? AND relay_host_id = ?`,
[context.now(), context.identity.userId, context.identity.relayHostId]
)
} finally {
unlock()
await held
}
await expect(claim).resolves.toEqual({ outcome: 'deferred', reason: 'candidate-ineligible' })
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
// The commit reads fleet safety as a snapshot; it no longer locks the safety
// table, so the recheck covers a change committed after selection.
it('rechecks fleet safety inside the commit before mutating a candidate', async () => {
const context = await fixture()
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
await secondary.query(
`UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`,
[context.target.id]
)
const claim = context.store.commitIdleRegionalRehome(request!, safety(context.now()))
await expect(claim).resolves.toEqual({ outcome: 'deferred', reason: 'fleet-safety' })
// A sql spike alone pauses the claim; only the director's sustained check latches.
expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
generation: 1,
enabled: true
})
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('pauses when one required cell exceeds the reconnect limit', async () => {
const context = await fixture()
const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
expect(request).toBeDefined()
await primary.query(`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, [
REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1,
context.source.id
])
await expect(
context.store.commitIdleRegionalRehome(request!, safety(context.now()))
).resolves.toEqual({ outcome: 'deferred', reason: 'fleet-safety' })
await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
generation: 2,
enabled: false
})
expect(
await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ count: '0' }])
})
it('makes concurrent completion and expiry cleanup idempotent', async () => {
const context = await fixture()
const attempt = await cutover(context.store, context.now())
const targetControl = await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch
})
await context.store.releaseActivity(context.identity, context.sourceControl)
context.advance(24 * 60 * 60_000)
await heartbeat(
context.store,
context.source,
'11111111-1111-4111-8111-111111111111',
1,
900_000,
2
)
await heartbeat(
context.store,
context.target,
'22222222-2222-4222-8222-222222222222',
1,
900_000,
2
)
await context.store.renewControlActivity(context.identity, {
activityId: targetControl,
cellId: context.target.id,
expiresAt: context.now() + 90_000
})
const outcomes = await Promise.all([
context.store.completeReadyRegionalRehomes(),
context.competingStore.abortExpiredRegionalRehomes()
])
expect(outcomes).toEqual(expect.arrayContaining([0, 1]))
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ completed: true, aborted: false }])
})
it('will not complete against a replacement target incarnation', async () => {
const context = await fixture()
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch
})
await context.store.releaseActivity(context.identity, context.sourceControl)
context.advance(1)
await heartbeat(
context.store,
context.target,
'44444444-4444-4444-8444-444444444444',
1,
context.now()
)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(0)
expect(
await primary.query(
`SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ completed_at: null, aborted_at: null }])
})
it('does not roll an unregistered target back to a stale regional source', async () => {
const context = await fixture()
await cutover(context.store, context.now())
context.advance(6 * 60_000)
await heartbeat(
context.store,
context.target,
'22222222-2222-4222-8222-222222222222',
1,
900_000,
2
)
await expect(context.store.refreshRegionalRehomeLeases()).resolves.toBe(0)
await expect(context.store.abortExpiredEvacuations()).resolves.toBe(0)
expect(
await primary.query(
`SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
})
it('completes after the drained host re-resolves through the director', async () => {
const context = await fixture()
const attempt = await cutover(context.store, context.now())
// The drain recovery lands while both controls are still live.
await context.store.assign(context.identity, 'asia-east2')
expect(await controlAccounting(context.identity)).toEqual({
reservedControls: 2,
controlLeases: 2
})
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch
})
await context.store.releaseActivity(context.identity, context.sourceControl)
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(
await primary.query(
`SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
WHERE user_id = ?`,
[context.identity.userId]
)
).toEqual([{ completed: true }])
expect(await controlAccounting(context.identity)).toEqual({
reservedControls: 1,
controlLeases: 1
})
})
it('repairs a skewed control counter before completing the rehome', async () => {
const context = await fixture()
const attempt = await cutover(context.store, context.now())
await context.store.activateControl(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch,
generation: 1
})
await context.store.markMigrationTargetRegistered(context.identity, {
cellId: context.target.id,
assignmentEpoch: attempt!.assignmentEpoch
})
await context.store.releaseActivity(context.identity, context.sourceControl)
// Damage already written by a pre-fix sticky grant.
await primary.query(`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, [
context.identity.userId
])
await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
expect(await controlAccounting(context.identity)).toEqual({
reservedControls: 1,
controlLeases: 1
})
})
async function cutover(store: RelayAssignmentStore, now: number) {
const [request] = await store.selectIdleRegionalRehomeCandidates(safety(now))
if (!request) return null
const result = await store.commitIdleRegionalRehome(request, safety(now))
if (result.outcome !== 'committed') return null
const [attempt] = await primary.query(
`SELECT preferred_region, assignment_epoch FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
[request.attemptId]
)
return {
...request,
preferredRegion: String(attempt!.preferred_region),
assignmentEpoch: Number(attempt!.assignment_epoch)
}
}
async function attemptAndMigrationCounts(identity: {
userId: string
relayHostId: string
}): Promise<{ attempts: number; migrations: number }> {
const attempts = await primary.query(
`SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
const migrations = await primary.query(
`SELECT COUNT(*) AS count FROM relay_assignment_migrations
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
return {
attempts: Number(attempts[0]!.count),
migrations: Number(migrations[0]!.count)
}
}
async function controlAccounting(identity: {
userId: string
relayHostId: string
}): Promise<{ reservedControls: number; controlLeases: number }> {
const assignment = (
await primary.query(
`SELECT reserved_controls FROM relay_assignments
WHERE user_id = ? AND relay_host_id = ?`,
[identity.userId, identity.relayHostId]
)
)[0]!
const leases = await primary.query(
`SELECT COUNT(*) AS controls FROM relay_assignment_activity_leases
WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'`,
[identity.userId, identity.relayHostId]
)
return {
reservedControls: Number(assignment.reserved_controls),
controlLeases: Number(leases[0]!.controls)
}
}
async function fixture(options: FixtureOptions = {}) {
sequence++
let now = 1_000_000
const suffix = String(sequence)
const sourceRegion = options.sourceRegion ?? 'us-central1'
const targetRegion = options.targetRegion ?? 'asia-east2'
const preferredRegion = options.preferredRegion ?? targetRegion
const source = cell(suffix, 'source', sourceRegion)
const target = cell(suffix, 'target', targetRegion)
const store = new RelayAssignmentStore(primary, () => now, storeOptions)
const competingStore = new RelayAssignmentStore(secondary, () => now, storeOptions)
await store.inspectRegionalRehomeControl()
now += 24 * 60 * 60_000
await store.applyRegionalRehomeControl({
expectedGeneration: 0,
enabled: true,
notBefore: now,
ratePerMinute: 10,
preferenceMaxAgeMs: 24 * 60 * 60_000,
hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
drainGraceMs: 60_000
})
await store.reconcileCells([source, target])
await heartbeat(store, source, '11111111-1111-4111-8111-111111111111', 3, 900_000)
await heartbeat(
store,
target,
'22222222-2222-4222-8222-222222222222',
options.targetProtocol ?? 3,
900_000
)
const identity = {
userId: `pg-rehome-user-${suffix}`,
relayHostId: `rehomehost${suffix.padStart(6, '0')}`
}
const assignment = await store.assign(identity, undefined, sourceRegion)
const sourceControl = await store.activateControl(identity, {
cellId: source.id,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1,
idleRegionalRehome: true,
cellIncarnation: '11111111-1111-4111-8111-111111111111'
})
await store.assign(identity, preferredRegion)
const issued = await store.exchangeRegionCorrection(
identity,
{ v: 1, action: 'issue-window' },
assignment.assignmentEpoch
)
await store.exchangeRegionCorrection(
identity,
{
v: 1,
action: 'report',
generation: issued.window!.generation,
assignmentEpoch: assignment.assignmentEpoch,
policyVersion: 1,
outcome: 'conclusive',
measurements: {
'us-central1': preferredRegion === 'us-central1' ? 50 : 150,
'asia-east2': preferredRegion === 'asia-east2' ? 50 : 150
}
},
assignment.assignmentEpoch
)
return {
preferredRegion,
store,
competingStore,
identity,
source,
target,
sourceControl,
now: () => now,
advance: (milliseconds: number) => {
now += milliseconds
}
}
}
})
const storeOptions = {
regionalRehomeCohortPercent: 100,
requireLiveCells: true,
heartbeatTtlMs: 45_000
}
type Region = 'us-central1' | 'asia-east2'
type FixtureOptions = {
sourceRegion?: Region
targetRegion?: Region
preferredRegion?: Region
targetProtocol?: number
hostCooldownMs?: number
}
function cell(suffix: string, role: string, region: Region) {
return {
id: `pg-rehome-cell-${suffix}-${role}`,
url: `https://pg-rehome-${suffix}-${role}.example.test`,
region,
capacityRequests: 100,
connectionHardCap: 1_000 as const,
connectionUnobservedBound: 60
}
}
async function heartbeat(
store: RelayAssignmentStore,
cellConfig: ReturnType<typeof cell>,
cellIncarnation: string,
regionalRehomeProtocol: number,
startedAt: number,
connectionInclusionWatermark = 1
): Promise<void> {
await store.recordCellHeartbeat({
cellId: cellConfig.id,
cellUrl: cellConfig.url,
region: cellConfig.region,
cellIncarnation,
startedAt,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark,
connectionHardCap: 1_000,
connectionUnobservedBound: 60
})
await store.recordCellRegionalRehomeStatus({
cellId: cellConfig.id,
cellIncarnation,
regionalRehomeProtocol,
safety: {
observedAt: 1_000_000 + 24 * 60 * 60_000,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
})
}
function safety(now: number) {
return {
observedAt: now,
sqlFailures: 0,
reconnects: 0,
controlActivityRecoveryFailures: 0,
databasePoolWaiting: 0,
databasePoolWaitersMax: 0,
databasePoolWaitMsMax: 0
}
}