Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

28.9 KB
Raw Blame
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
import {
  REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT,
  REGIONAL_REHOME_SQL_FAILURES_LIMIT,
  REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT
} from './regional-rehome-safety.js'

const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip

describePostgres('PostgreSQL regional rehoming', () => {
  let primary: RelayDatabase
  let secondary: RelayDatabase
  let sequence = 0

  beforeAll(async () => {
    primary = await openRelayDatabase({ databaseUrl, dataDir: '' })
    secondary = await openRelayDatabase({ databaseUrl, dataDir: '' })
  })

  beforeEach(async () => await cleanup())

  afterAll(async () => {
    await cleanup()
    await secondary.close()
    await primary.close()
  })

  async function cleanup(): Promise<void> {
    for (const table of ['relay_region_decisions', 'relay_control_capabilities']) {
      await primary.query(`DELETE FROM ${table} WHERE user_id LIKE 'pg-rehome-user-%'`)
    }
    await primary.query(
      `DELETE FROM relay_region_rehome_attempts WHERE user_id LIKE 'pg-rehome-user-%'`
    )
    await primary.query(`DELETE FROM relay_region_rehome_worker_state`)
    await primary.query(`DELETE FROM relay_region_rehome_control`)
    await primary.query(
      `DELETE FROM relay_control_connection_reservations
       WHERE user_id LIKE 'pg-rehome-user-%'`
    )
    await primary.query(
      `DELETE FROM relay_assignment_migration_incarnations
       WHERE user_id LIKE 'pg-rehome-user-%'`
    )
    await primary.query(
      `DELETE FROM relay_assignment_activity_leases
       WHERE user_id LIKE 'pg-rehome-user-%'`
    )
    await primary.query(
      `DELETE FROM relay_assignment_migrations WHERE user_id LIKE 'pg-rehome-user-%'`
    )
    await primary.query(
      `DELETE FROM relay_assignment_region_preferences
       WHERE user_id LIKE 'pg-rehome-user-%'`
    )
    await primary.query(`DELETE FROM relay_assignments WHERE user_id LIKE 'pg-rehome-user-%'`)
    for (const table of [
      'relay_cell_rehome_safety',
      'relay_cell_capabilities',
      'relay_cell_connection_snapshots',
      'relay_cell_connection_runtime',
      'relay_cell_runtime',
      'relay_cell_connection_limits',
      'relay_cell_admission',
      'relay_cell_regions',
      'relay_cells'
    ]) {
      await primary.query(`DELETE FROM ${table} WHERE cell_id LIKE 'pg-rehome-cell-%'`)
    }
  }

  it('defaults to a closed correction cohort even with enabled durable control', async () => {
    const context = await fixture()
    const closed = new RelayAssignmentStore(primary, context.now, {
      requireLiveCells: true,
      heartbeatTtlMs: 45_000
    })
    expect(await cutover(closed, context.now())).toBeNull()
    const preview = await closed.previewRegionalRehomeEligibility({
      observedAt: context.now(),
      sqlFailures: 0,
      reconnects: 0,
      controlActivityRecoveryFailures: 0,
      databasePoolWaiting: 0,
      databasePoolWaitersMax: 0,
      databasePoolWaitMsMax: 0
    })
    expect(preview.cohortPercent).toBe(0)
    expect(preview.counts['outside-cohort']).toBe(1)
    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
      attempts: 0,
      migrations: 0
    })
  })

  it('counts existing generic migrations against the optimization cap and preview', async () => {
    const context = await fixture()
    const safety = {
      observedAt: context.now(),
      sqlFailures: 0,
      reconnects: 0,
      controlActivityRecoveryFailures: 0,
      databasePoolWaiting: 0,
      databasePoolWaitersMax: 0,
      databasePoolWaitMsMax: 0
    }
    const before = await context.store.previewRegionalRehomeEligibility(safety)
    expect(before.counts['eligible:us-central1-to-asia-east2']).toBe(1)
    for (let index = 0; index < 8; index++) {
      const identity = {
        userId: `pg-rehome-user-budget-${sequence}-${index}`,
        relayHostId: `budgethost${String(index).padStart(6, '0')}`
      }
      await context.store.assign(identity, undefined, 'us-central1')
      await context.store.startEvacuation(identity, context.target.id)
    }
    const preview = await context.store.previewRegionalRehomeEligibility(safety)
    expect(preview.openMigrations).toBe(8)
    expect(preview.availableMigrationSlots).toBe(0)
    expect(preview.counts['concurrent-migration-cap']).toBe(1)
    expect(await cutover(context.store, context.now())).toBeNull()
    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
      attempts: 0,
      migrations: 0
    })
  })

  it('preview excludes request capacity exhaustion before a claim', async () => {
    const context = await fixture()
    await primary.query(
      `UPDATE relay_cells SET capacity_requests = reserved_requests + 1 WHERE cell_id = ?`,
      [context.target.id]
    )
    const preview = await context.store.previewRegionalRehomeEligibility({
      observedAt: context.now(),
      sqlFailures: 0,
      reconnects: 0,
      controlActivityRecoveryFailures: 0,
      databasePoolWaiting: 0,
      databasePoolWaitersMax: 0,
      databasePoolWaitMsMax: 0
    })
    expect(preview.counts['no-target-headroom']).toBe(1)
    expect(await cutover(context.store, context.now())).toBeNull()
  })

  it('claims through ambient per-cell sql retry noise', async () => {
    const context = await fixture()
    await primary.query(
      `UPDATE relay_cell_rehome_safety
       SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT}
       WHERE cell_id IN (?, ?)`,
      [context.source.id, context.target.id]
    )

    expect(await cutover(context.store, context.now())).not.toBeNull()
  })

  it('moves a us-central1 host onto a cell in its preferred asia-east2 region', async () => {
    const context = await fixture()

    const attempt = await cutover(context.store, context.now())
    expect(attempt).toMatchObject({
      preferredRegion: 'asia-east2',
      sourceCellId: context.source.id,
      targetCellId: context.target.id
    })
    expect(
      await primary.query(
        `SELECT preferred_region, source_cell_id, target_cell_id
       FROM relay_region_rehome_attempts WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([
      {
        preferred_region: 'asia-east2',
        source_cell_id: context.source.id,
        target_cell_id: context.target.id
      }
    ])
  })

  it('moves an asia-east2 host back onto a cell in its preferred us-central1 region', async () => {
    const context = await fixture({
      sourceRegion: 'asia-east2',
      targetRegion: 'us-central1'
    })

    const attempt = await cutover(context.store, context.now())
    expect(attempt).toMatchObject({
      preferredRegion: 'us-central1',
      sourceCellId: context.source.id,
      targetCellId: context.target.id
    })
    // The durable attempt row must accept the reverse direction too.
    expect(
      await primary.query(
        `SELECT preferred_region, source_cell_id, target_cell_id
       FROM relay_region_rehome_attempts WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([
      {
        preferred_region: 'us-central1',
        source_cell_id: context.source.id,
        target_cell_id: context.target.id
      }
    ])
    expect(
      await primary.query(`SELECT cell_id FROM relay_assignments WHERE user_id = ?`, [
        context.identity.userId
      ])
    ).toEqual([{ cell_id: context.target.id }])
  })

  it('leaves a host whose preference already matches its own region', async () => {
    const context = await fixture({ preferredRegion: 'us-central1' })

    await expect(cutover(context.store, context.now())).resolves.toBeNull()
    await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
      generation: 1,
      enabled: true
    })
    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
      attempts: 0,
      migrations: 0
    })
  })

  it('leaves a host whose preference is older than the configured max age', async () => {
    const context = await fixture()
    await primary.query(
      `UPDATE relay_region_decisions SET observed_at = ?
       WHERE user_id = ? AND relay_host_id = ?`,
      [context.now() - 24 * 60 * 60_000 - 1, context.identity.userId, context.identity.relayHostId]
    )

    await expect(cutover(context.store, context.now())).resolves.toBeNull()
    await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
      generation: 1,
      enabled: true
    })
    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
      attempts: 0,
      migrations: 0
    })
  })

  it('leaves a host inside its per-host rehome cooldown, in either direction', async () => {
    const context = await fixture({ hostCooldownMs: 3 * 24 * 60 * 60_000 })
    // A move this host already made, whichever way it went.
    await primary.query(
      `INSERT INTO relay_region_rehome_attempts
       (attempt_id, user_id, relay_host_id, preferred_region, source_cell_id,
        source_cell_incarnation, target_cell_id, target_cell_incarnation,
        previous_epoch, assignment_epoch, drain_grace_ms, send_attempts,
        completed_at, created_at, updated_at)
       VALUES (?, ?, ?, 'us-central1', ?, ?, ?, ?, 0, 1, 0, 0, ?, ?, ?)`,
      [
        `pg-rehome-cooldown-${context.identity.relayHostId}`,
        context.identity.userId,
        context.identity.relayHostId,
        context.target.id,
        '22222222-2222-4222-8222-222222222222',
        context.source.id,
        '11111111-1111-4111-8111-111111111111',
        context.now(),
        context.now() - 3 * 24 * 60 * 60_000 + 1,
        context.now()
      ]
    )

    await expect(cutover(context.store, context.now())).resolves.toBeNull()
    await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
      generation: 1,
      enabled: true,
      hostCooldownMs: 3 * 24 * 60 * 60_000
    })
    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
      attempts: 1,
      migrations: 0
    })

    // One millisecond past the window the same host is a candidate again.
    await primary.query(
      `UPDATE relay_region_rehome_attempts SET created_at = ? WHERE user_id = ?`,
      [context.now() - 3 * 24 * 60 * 60_000, context.identity.userId]
    )
    await expect(cutover(context.store, context.now())).resolves.toMatchObject({
      sourceCellId: context.source.id,
      targetCellId: context.target.id
    })
  })

  it('leaves a host whose preferred region holds no drainable cell', async () => {
    // A cell that cannot be drained cannot be a target: the host would land
    // where no later rehome could move it out again.
    const context = await fixture({ targetProtocol: 0 })

    await expect(cutover(context.store, context.now())).resolves.toBeNull()
    await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
      generation: 1,
      enabled: true
    })
    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
      attempts: 0,
      migrations: 0
    })
  })

  it('skips an unclean cell without latching the control off', async () => {
    const context = await fixture()
    await primary.query(
      `UPDATE relay_cell_rehome_safety
       SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_PER_CELL_LIMIT + 1}
       WHERE cell_id = ?`,
      [context.target.id]
    )

    expect(await cutover(context.store, context.now())).toBeNull()
    expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
      generation: 1,
      enabled: true
    })
    expect(await attemptAndMigrationCounts(context.identity)).toEqual({
      attempts: 0,
      migrations: 0
    })
  })

  it('lets only one director claim a host', async () => {
    const context = await fixture()
    const claims = await Promise.all([
      cutover(context.store, context.now()),
      cutover(context.competingStore, context.now())
    ])

    expect(claims.filter(Boolean).length).toBeGreaterThanOrEqual(1)
    expect(
      await primary.query(
        `SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
       WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([{ count: '1' }])
    expect(
      await primary.query(
        `SELECT COUNT(*) AS count FROM relay_assignment_migrations
       WHERE user_id = ? AND completed_at IS NULL AND aborted_at IS NULL`,
        [context.identity.userId]
      )
    ).toEqual([{ count: '1' }])
  })

  it('increments the disable generation once across competing directors', async () => {
    const context = await fixture()
    const disabled = await Promise.all([
      context.store.disableRegionalRehomeControl(),
      context.competingStore.disableRegionalRehomeControl()
    ])

    expect(disabled.sort()).toEqual([false, true])
    expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
      generation: 2,
      enabled: false
    })
  })

  it('rechecks a preference changed while the assignment row is locked', async () => {
    const context = await fixture()
    const [candidate] = await context.store.selectIdleRegionalRehomeCandidates(
      safety(context.now())
    )
    expect(candidate).toBeDefined()
    let unlock!: () => void
    let locked!: () => void
    const lockedPromise = new Promise<void>((resolve) => (locked = resolve))
    const unlockPromise = new Promise<void>((resolve) => (unlock = resolve))
    const held = secondary.transaction(async (transaction) => {
      await transaction.queryLocked(
        `SELECT * FROM relay_assignments WHERE user_id = ? AND relay_host_id = ?`,
        [context.identity.userId, context.identity.relayHostId]
      )
      locked()
      await unlockPromise
    })
    await lockedPromise
    const claim = context.store.commitIdleRegionalRehome(candidate!, safety(context.now()))
    try {
      await vi.waitFor(async () => {
        const [row] = await primary.query(
          `SELECT COUNT(*) AS waiting FROM pg_stat_activity
           WHERE datname = current_database() AND wait_event_type = 'Lock'
             AND query LIKE 'SELECT %relay_assignments%FOR UPDATE%'`
        )
        expect(Number(row?.waiting)).toBeGreaterThan(0)
      })
      await primary.query(
        `UPDATE relay_region_decisions SET preferred_region = 'us-central1',
           observed_at = ? WHERE user_id = ? AND relay_host_id = ?`,
        [context.now(), context.identity.userId, context.identity.relayHostId]
      )
    } finally {
      unlock()
      await held
    }

    await expect(claim).resolves.toEqual({ outcome: 'deferred', reason: 'candidate-ineligible' })
    expect(
      await primary.query(
        `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([{ count: '0' }])
  })

  // The commit reads fleet safety as a snapshot; it no longer locks the safety
  // table, so the recheck covers a change committed after selection.
  it('rechecks fleet safety inside the commit before mutating a candidate', async () => {
    const context = await fixture()
    const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
    expect(request).toBeDefined()
    await secondary.query(
      `UPDATE relay_cell_rehome_safety SET sql_failures = ${REGIONAL_REHOME_SQL_FAILURES_LIMIT + 1} WHERE cell_id = ?`,
      [context.target.id]
    )
    const claim = context.store.commitIdleRegionalRehome(request!, safety(context.now()))

    await expect(claim).resolves.toEqual({ outcome: 'deferred', reason: 'fleet-safety' })
    // A sql spike alone pauses the claim; only the director's sustained check latches.
    expect(await context.store.inspectRegionalRehomeControl()).toMatchObject({
      generation: 1,
      enabled: true
    })
    expect(
      await primary.query(
        `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([{ count: '0' }])
  })

  it('pauses when one required cell exceeds the reconnect limit', async () => {
    const context = await fixture()
    const [request] = await context.store.selectIdleRegionalRehomeCandidates(safety(context.now()))
    expect(request).toBeDefined()
    await primary.query(`UPDATE relay_cell_rehome_safety SET reconnects = ? WHERE cell_id = ?`, [
      REGIONAL_REHOME_RECONNECTS_PER_CELL_LIMIT + 1,
      context.source.id
    ])

    await expect(
      context.store.commitIdleRegionalRehome(request!, safety(context.now()))
    ).resolves.toEqual({ outcome: 'deferred', reason: 'fleet-safety' })
    await expect(context.store.inspectRegionalRehomeControl()).resolves.toMatchObject({
      generation: 2,
      enabled: false
    })
    expect(
      await primary.query(
        `SELECT COUNT(*) AS count FROM relay_assignment_migrations WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([{ count: '0' }])
  })

  it('makes concurrent completion and expiry cleanup idempotent', async () => {
    const context = await fixture()
    const attempt = await cutover(context.store, context.now())
    const targetControl = await context.store.activateControl(context.identity, {
      cellId: context.target.id,
      assignmentEpoch: attempt!.assignmentEpoch,
      generation: 1
    })
    await context.store.markMigrationTargetRegistered(context.identity, {
      cellId: context.target.id,
      assignmentEpoch: attempt!.assignmentEpoch
    })
    await context.store.releaseActivity(context.identity, context.sourceControl)
    context.advance(24 * 60 * 60_000)
    await heartbeat(
      context.store,
      context.source,
      '11111111-1111-4111-8111-111111111111',
      1,
      900_000,
      2
    )
    await heartbeat(
      context.store,
      context.target,
      '22222222-2222-4222-8222-222222222222',
      1,
      900_000,
      2
    )
    await context.store.renewControlActivity(context.identity, {
      activityId: targetControl,
      cellId: context.target.id,
      expiresAt: context.now() + 90_000
    })

    const outcomes = await Promise.all([
      context.store.completeReadyRegionalRehomes(),
      context.competingStore.abortExpiredRegionalRehomes()
    ])
    expect(outcomes).toEqual(expect.arrayContaining([0, 1]))
    expect(
      await primary.query(
        `SELECT completed_at IS NOT NULL AS completed, aborted_at IS NOT NULL AS aborted
       FROM relay_assignment_migrations WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([{ completed: true, aborted: false }])
  })

  it('will not complete against a replacement target incarnation', async () => {
    const context = await fixture()
    const attempt = await cutover(context.store, context.now())
    await context.store.activateControl(context.identity, {
      cellId: context.target.id,
      assignmentEpoch: attempt!.assignmentEpoch,
      generation: 1
    })
    await context.store.markMigrationTargetRegistered(context.identity, {
      cellId: context.target.id,
      assignmentEpoch: attempt!.assignmentEpoch
    })
    await context.store.releaseActivity(context.identity, context.sourceControl)
    context.advance(1)
    await heartbeat(
      context.store,
      context.target,
      '44444444-4444-4444-8444-444444444444',
      1,
      context.now()
    )

    await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(0)
    expect(
      await primary.query(
        `SELECT completed_at, aborted_at FROM relay_assignment_migrations WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([{ completed_at: null, aborted_at: null }])
  })

  it('does not roll an unregistered target back to a stale regional source', async () => {
    const context = await fixture()
    await cutover(context.store, context.now())
    context.advance(6 * 60_000)
    await heartbeat(
      context.store,
      context.target,
      '22222222-2222-4222-8222-222222222222',
      1,
      900_000,
      2
    )

    await expect(context.store.refreshRegionalRehomeLeases()).resolves.toBe(0)
    await expect(context.store.abortExpiredEvacuations()).resolves.toBe(0)
    expect(
      await primary.query(
        `SELECT cell_id, assignment_epoch FROM relay_assignments WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([{ cell_id: context.target.id, assignment_epoch: '2' }])
  })

  it('completes after the drained host re-resolves through the director', async () => {
    const context = await fixture()
    const attempt = await cutover(context.store, context.now())
    // The drain recovery lands while both controls are still live.
    await context.store.assign(context.identity, 'asia-east2')
    expect(await controlAccounting(context.identity)).toEqual({
      reservedControls: 2,
      controlLeases: 2
    })

    await context.store.activateControl(context.identity, {
      cellId: context.target.id,
      assignmentEpoch: attempt!.assignmentEpoch,
      generation: 1
    })
    await context.store.markMigrationTargetRegistered(context.identity, {
      cellId: context.target.id,
      assignmentEpoch: attempt!.assignmentEpoch
    })
    await context.store.releaseActivity(context.identity, context.sourceControl)

    await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
    expect(
      await primary.query(
        `SELECT completed_at IS NOT NULL AS completed FROM relay_assignment_migrations
       WHERE user_id = ?`,
        [context.identity.userId]
      )
    ).toEqual([{ completed: true }])
    expect(await controlAccounting(context.identity)).toEqual({
      reservedControls: 1,
      controlLeases: 1
    })
  })

  it('repairs a skewed control counter before completing the rehome', async () => {
    const context = await fixture()
    const attempt = await cutover(context.store, context.now())
    await context.store.activateControl(context.identity, {
      cellId: context.target.id,
      assignmentEpoch: attempt!.assignmentEpoch,
      generation: 1
    })
    await context.store.markMigrationTargetRegistered(context.identity, {
      cellId: context.target.id,
      assignmentEpoch: attempt!.assignmentEpoch
    })
    await context.store.releaseActivity(context.identity, context.sourceControl)
    // Damage already written by a pre-fix sticky grant.
    await primary.query(`UPDATE relay_assignments SET reserved_controls = 0 WHERE user_id = ?`, [
      context.identity.userId
    ])

    await expect(context.store.completeReadyRegionalRehomes()).resolves.toBe(1)
    expect(await controlAccounting(context.identity)).toEqual({
      reservedControls: 1,
      controlLeases: 1
    })
  })

  async function cutover(store: RelayAssignmentStore, now: number) {
    const [request] = await store.selectIdleRegionalRehomeCandidates(safety(now))
    if (!request) return null
    const result = await store.commitIdleRegionalRehome(request, safety(now))
    if (result.outcome !== 'committed') return null
    const [attempt] = await primary.query(
      `SELECT preferred_region, assignment_epoch FROM relay_region_rehome_attempts WHERE attempt_id = ?`,
      [request.attemptId]
    )
    return {
      ...request,
      preferredRegion: String(attempt!.preferred_region),
      assignmentEpoch: Number(attempt!.assignment_epoch)
    }
  }

  async function attemptAndMigrationCounts(identity: {
    userId: string
    relayHostId: string
  }): Promise<{ attempts: number; migrations: number }> {
    const attempts = await primary.query(
      `SELECT COUNT(*) AS count FROM relay_region_rehome_attempts
       WHERE user_id = ? AND relay_host_id = ?`,
      [identity.userId, identity.relayHostId]
    )
    const migrations = await primary.query(
      `SELECT COUNT(*) AS count FROM relay_assignment_migrations
       WHERE user_id = ? AND relay_host_id = ?`,
      [identity.userId, identity.relayHostId]
    )
    return {
      attempts: Number(attempts[0]!.count),
      migrations: Number(migrations[0]!.count)
    }
  }

  async function controlAccounting(identity: {
    userId: string
    relayHostId: string
  }): Promise<{ reservedControls: number; controlLeases: number }> {
    const assignment = (
      await primary.query(
        `SELECT reserved_controls FROM relay_assignments
         WHERE user_id = ? AND relay_host_id = ?`,
        [identity.userId, identity.relayHostId]
      )
    )[0]!
    const leases = await primary.query(
      `SELECT COUNT(*) AS controls FROM relay_assignment_activity_leases
       WHERE user_id = ? AND relay_host_id = ? AND activity_kind = 'control'`,
      [identity.userId, identity.relayHostId]
    )
    return {
      reservedControls: Number(assignment.reserved_controls),
      controlLeases: Number(leases[0]!.controls)
    }
  }

  async function fixture(options: FixtureOptions = {}) {
    sequence++
    let now = 1_000_000
    const suffix = String(sequence)
    const sourceRegion = options.sourceRegion ?? 'us-central1'
    const targetRegion = options.targetRegion ?? 'asia-east2'
    const preferredRegion = options.preferredRegion ?? targetRegion
    const source = cell(suffix, 'source', sourceRegion)
    const target = cell(suffix, 'target', targetRegion)
    const store = new RelayAssignmentStore(primary, () => now, storeOptions)
    const competingStore = new RelayAssignmentStore(secondary, () => now, storeOptions)
    await store.inspectRegionalRehomeControl()
    now += 24 * 60 * 60_000
    await store.applyRegionalRehomeControl({
      expectedGeneration: 0,
      enabled: true,
      notBefore: now,
      ratePerMinute: 10,
      preferenceMaxAgeMs: 24 * 60 * 60_000,
      hostCooldownMs: options.hostCooldownMs ?? 7 * 24 * 60 * 60_000,
      drainGraceMs: 60_000
    })
    await store.reconcileCells([source, target])
    await heartbeat(store, source, '11111111-1111-4111-8111-111111111111', 3, 900_000)
    await heartbeat(
      store,
      target,
      '22222222-2222-4222-8222-222222222222',
      options.targetProtocol ?? 3,
      900_000
    )
    const identity = {
      userId: `pg-rehome-user-${suffix}`,
      relayHostId: `rehomehost${suffix.padStart(6, '0')}`
    }
    const assignment = await store.assign(identity, undefined, sourceRegion)
    const sourceControl = await store.activateControl(identity, {
      cellId: source.id,
      assignmentEpoch: assignment.assignmentEpoch,
      generation: 1,
      idleRegionalRehome: true,
      cellIncarnation: '11111111-1111-4111-8111-111111111111'
    })
    await store.assign(identity, preferredRegion)
    const issued = await store.exchangeRegionCorrection(
      identity,
      { v: 1, action: 'issue-window' },
      assignment.assignmentEpoch
    )
    await store.exchangeRegionCorrection(
      identity,
      {
        v: 1,
        action: 'report',
        generation: issued.window!.generation,
        assignmentEpoch: assignment.assignmentEpoch,
        policyVersion: 1,
        outcome: 'conclusive',
        measurements: {
          'us-central1': preferredRegion === 'us-central1' ? 50 : 150,
          'asia-east2': preferredRegion === 'asia-east2' ? 50 : 150
        }
      },
      assignment.assignmentEpoch
    )
    return {
      preferredRegion,
      store,
      competingStore,
      identity,
      source,
      target,
      sourceControl,
      now: () => now,
      advance: (milliseconds: number) => {
        now += milliseconds
      }
    }
  }
})

const storeOptions = {
  regionalRehomeCohortPercent: 100,
  requireLiveCells: true,
  heartbeatTtlMs: 45_000
}

type Region = 'us-central1' | 'asia-east2'
type FixtureOptions = {
  sourceRegion?: Region
  targetRegion?: Region
  preferredRegion?: Region
  targetProtocol?: number
  hostCooldownMs?: number
}

function cell(suffix: string, role: string, region: Region) {
  return {
    id: `pg-rehome-cell-${suffix}-${role}`,
    url: `https://pg-rehome-${suffix}-${role}.example.test`,
    region,
    capacityRequests: 100,
    connectionHardCap: 1_000 as const,
    connectionUnobservedBound: 60
  }
}

async function heartbeat(
  store: RelayAssignmentStore,
  cellConfig: ReturnType<typeof cell>,
  cellIncarnation: string,
  regionalRehomeProtocol: number,
  startedAt: number,
  connectionInclusionWatermark = 1
): Promise<void> {
  await store.recordCellHeartbeat({
    cellId: cellConfig.id,
    cellUrl: cellConfig.url,
    region: cellConfig.region,
    cellIncarnation,
    startedAt,
    ready: true,
    observedRequests: 0,
    totalConnections: 0,
    inFlightConnections: 0,
    reservedConnectionUnits: 0,
    enforcedConnectionUnits: 0,
    connectionInclusionWatermark,
    connectionHardCap: 1_000,
    connectionUnobservedBound: 60
  })
  await store.recordCellRegionalRehomeStatus({
    cellId: cellConfig.id,
    cellIncarnation,
    regionalRehomeProtocol,
    safety: {
      observedAt: 1_000_000 + 24 * 60 * 60_000,
      sqlFailures: 0,
      reconnects: 0,
      controlActivityRecoveryFailures: 0,
      databasePoolWaiting: 0,
      databasePoolWaitersMax: 0,
      databasePoolWaitMsMax: 0
    }
  })
}

function safety(now: number) {
  return {
    observedAt: now,
    sqlFailures: 0,
    reconnects: 0,
    controlActivityRecoveryFailures: 0,
    databasePoolWaiting: 0,
    databasePoolWaitersMax: 0,
    databasePoolWaitMsMax: 0
  }
}