Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

38.3 KB
Raw Blame
import { createHash } from 'node:crypto'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore, RelayHomeCellUnavailableError } from './assignment-store.js'
import { ReservePaceError } from './reserve-assignment.js'
import {
  encodeMembership,
  type CellAdmissionMembership,
  type CellAdmissionState
} from './cell-admission-selector.js'
import type { RelayCellConfig } from './config.js'
import {
  consumeRelayCellInventoryHold,
  openInMemoryRelayDatabase,
  type RelayDatabase,
  type RelayLockOptions,
  type RelayTransactionOptions,
  type SqlRow
} from './database.js'

// migration-only also holds evacuation targets, Asia rollback and failed-wave/new cells; only same-cap isolate writes rollIsolatedCells, and restore clears its stamp atomically (orca-relay-operations.md).
const HEARTBEAT_TTL_MS = 45_000
const START_MS = 100
const IDENTITY = { userId: 'user-a', relayHostId: 'host000000000001' }

// A connection-limited cell is what makes deadCellFenceRequirement report capped,
// which is the branch that answers 503 relay_home_cell_unavailable.
const CAPPED = {
  capacityRequests: 1_000,
  connectionHardCap: 600,
  connectionUnobservedBound: 50
} as const
const CELLS: RelayCellConfig[] = [
  { id: 'us-c1', url: 'https://us-c1.example.com', region: 'us-central1', ...CAPPED },
  { id: 'us-c2', url: 'https://us-c2.example.com', region: 'us-central1', ...CAPPED },
  { id: 'asia-c1', url: 'https://asia-c1.example.com', region: 'asia-east2', ...CAPPED }
]

const databases: RelayDatabase[] = []

afterEach(async () => {
  vi.restoreAllMocks()
  for (const database of databases.splice(0)) await database.close()
})

// Counts the reads this change adds, so the general hot path can be held to a
// single admission lookup and no migration lookup at all.
class QueryCountingDatabase implements RelayDatabase {
  readonly sql: string[] = []
  // Fails the first statement containing this fragment, so a test can roll a
  // transaction back at a chosen point.
  failOnce: string | undefined
  // Locked statements that report the row busy, in order: each fragment fires
  // once, on the first locked statement containing it after the one before.
  lockUnavailableSequence: string[] = []

  constructor(private readonly delegate: RelayDatabase) {}

  get dialect(): 'sqlite' | 'postgres' | undefined {
    return this.delegate.dialect
  }

  count(fragment: string): number {
    return this.sql.filter((statement) => statement.includes(fragment)).length
  }

  record(sql: string): void {
    this.sql.push(sql)
    if (this.failOnce !== undefined && sql.includes(this.failOnce)) {
      this.failOnce = undefined
      throw new Error('injected_placement_write_failure')
    }
  }

  async query(sql: string, params?: unknown[]): Promise<SqlRow[]> {
    this.record(sql)
    return await this.delegate.query(sql, params)
  }

  async queryLocked(
    sql: string,
    params?: unknown[],
    options?: RelayLockOptions
  ): Promise<SqlRow[]> {
    this.recordLocked(sql)
    return await this.delegate.queryLocked(sql, params, options)
  }

  recordLocked(sql: string): void {
    this.record(sql)
    const next = this.lockUnavailableSequence[0]
    if (next !== undefined && sql.includes(next)) {
      this.lockUnavailableSequence.shift()
      throw new Error('database_lock_unavailable')
    }
  }

  async transaction<T>(
    operation: (transaction: RelayDatabase) => Promise<T>,
    options?: RelayTransactionOptions
  ): Promise<T> {
    return await this.delegate.transaction(
      async (inner) => await operation(this.recording(inner)),
      options
    )
  }

  async close(): Promise<void> {
    await this.delegate.close()
  }

  // A nested transaction shares this one's tape, so a whole assign() is
  // measured as a unit.
  private recording(inner: RelayDatabase): RelayDatabase {
    return {
      dialect: inner.dialect,
      query: async (sql, params) => {
        this.record(sql)
        return await inner.query(sql, params)
      },
      queryLocked: async (sql, params, options) => {
        this.recordLocked(sql)
        return await inner.queryLocked(sql, params, options)
      },
      transaction: async (operation, options) => await inner.transaction(operation, options),
      close: async () => await inner.close()
    }
  }
}

interface Harness {
  store: RelayAssignmentStore
  database: RelayDatabase
  counter: QueryCountingDatabase
  heartbeat: (cell: RelayCellConfig, at?: number) => Promise<void>
  setNow: (value: number) => void
  /** The real isolate path: one selector apply that names the cell it stamps. */
  isolateForRoll: (cellId: string) => Promise<void>
  /** The real restore path: back to 'general', which clears the stamp. */
  restore: (cellId: string) => Promise<void>
  /** An admission move with no stamp — every flow that is not a same-cap roll. */
  park: (cellId: string, state: CellAdmissionState) => Promise<void>
  rollIsolatedAt: (cellId: string) => Promise<number | null>
}

async function setup(cells: RelayCellConfig[] = CELLS): Promise<Harness> {
  const inner = await openInMemoryRelayDatabase()
  databases.push(inner)
  const counter = new QueryCountingDatabase(inner)
  let now = START_MS
  const store = new RelayAssignmentStore(counter, () => now, {
    requireLiveCells: true,
    heartbeatTtlMs: HEARTBEAT_TTL_MS
  })
  await store.reconcileCells(cells, true)
  const heartbeat = async (cell: RelayCellConfig, at?: number): Promise<void> => {
    const previous = now
    if (at !== undefined) now = at
    try {
      await store.recordCellHeartbeat({
        cellId: cell.id,
        cellUrl: cell.url,
        cellIncarnation: `1111111${cells.indexOf(cell)}-1111-4111-8111-111111111111`,
        startedAt: 50,
        ready: true,
        observedRequests: 0,
        region: cell.region,
        totalConnections: 0,
        inFlightConnections: 0,
        reservedConnectionUnits: 0,
        enforcedConnectionUnits: 0,
        connectionHardCap: cell.connectionHardCap ?? 600,
        connectionUnobservedBound: cell.connectionUnobservedBound ?? 50
      })
    } finally {
      if (at !== undefined) now = previous
    }
  }
  for (const cell of cells) await heartbeat(cell)

  const applySelector = async (
    states: Record<string, CellAdmissionState>,
    rollIsolatedCells?: string[]
  ): Promise<void> => {
    const current = await store.inspectCellAdmissionSelector()
    const membership: CellAdmissionMembership = {
      existingOnly: [],
      migrationOnly: [],
      general: []
    }
    for (const cell of cells) {
      const state =
        states[cell.id] ??
        (current.selector.membership.migrationOnly.includes(cell.id)
          ? 'migration-only'
          : current.selector.membership.existingOnly.includes(cell.id)
            ? 'existing-only'
            : 'general')
      if (state === 'migration-only') membership.migrationOnly.push(cell.id)
      else if (state === 'existing-only') membership.existingOnly.push(cell.id)
      else membership.general.push(cell.id)
    }
    await store.applyCellAdmissionSelector({
      attemptId: `attempt-${current.selector.generation}-${Object.keys(states).join('-')}`,
      expectedGeneration: current.selector.generation,
      ...(current.selector.generation === 0
        ? {
            expectedMembershipSha256: createHash('sha256')
              .update(encodeMembership(current.selector.membership))
              .digest('hex')
          }
        : {}),
      membership,
      ...(rollIsolatedCells ? { rollIsolatedCells } : {})
    })
  }

  return {
    store,
    database: inner,
    counter,
    heartbeat,
    setNow: (value: number) => (now = value),
    isolateForRoll: async (cellId) =>
      await applySelector({ [cellId]: 'migration-only' }, [cellId]),
    restore: async (cellId) => await applySelector({ [cellId]: 'general' }),
    park: async (cellId, state) => await applySelector({ [cellId]: state }),
    rollIsolatedAt: async (cellId) => {
      const rows = await inner.query(
        `SELECT roll_isolated_at FROM relay_cell_admission WHERE cell_id = ?`,
        [cellId]
      )
      const value = rows[0]?.['roll_isolated_at']
      return value === undefined || value === null ? null : Number(value)
    }
  }
}

async function insertMigration(
  database: RelayDatabase,
  input: {
    sourceCellId: string
    targetCellId: string
    assignmentEpoch: number
    leases: number
    settled?: 'completed' | 'aborted'
  }
): Promise<void> {
  await database.query(
    `INSERT INTO relay_assignment_migrations
     (user_id, relay_host_id, source_cell_id, target_cell_id, previous_epoch,
      assignment_epoch, source_request_units, target_reserved_units, expires_at,
      target_registered_at, completed_at, aborted_at, created_at, updated_at)
     VALUES (?, ?, ?, ?, ?, ?, 1, 1, ?, NULL, ?, ?, ?, ?)`,
    [
      IDENTITY.userId,
      IDENTITY.relayHostId,
      input.sourceCellId,
      input.targetCellId,
      input.assignmentEpoch - 1,
      input.assignmentEpoch,
      START_MS + 900_000,
      input.settled === 'completed' ? START_MS : null,
      input.settled === 'aborted' ? START_MS : null,
      START_MS,
      START_MS
    ]
  )
  await database.query(
    `UPDATE relay_assignments SET migration_leases = ?
     WHERE user_id = ? AND relay_host_id = ?`,
    [input.leases, IDENTITY.userId, IDENTITY.relayHostId]
  )
}

type ConsoleWarnSpy = { mock: { calls: unknown[][] } }

function jsonEvents(warn: ConsoleWarnSpy, event: string): unknown[] {
  return warn.mock.calls
    .map((call) => (typeof call[0] === 'string' ? call[0] : ''))
    .filter((line) => line.includes(`"${event}"`))
    .map((line) => JSON.parse(line) as unknown)
}

describe('re-placing a host off a cell isolated for a roll', () => {
  it('leaves a host on a live general cell untouched, at one added admission read', async () => {
    const { store, counter } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')

    counter.sql.length = 0
    const second = await store.assign(IDENTITY, 'us-central1')

    // Identical, not merely same-celled: the clock has not moved, so every
    // field including the lease deadline must match; only the first was placed.
    expect(second).toEqual({ ...first, placed: undefined })
    // One row read answers both the stranded rule and the roll-stamp check.
    expect(counter.count('relay_cell_admission')).toBe(1)
    expect(counter.count('relay_assignment_migrations')).toBe(0)
    // No placement: the sticky lane never reaches the fleet-wide inventory.
    expect(counter.count('ORDER BY cell_id ASC')).toBe(0)
  })

  it('keeps a host pinned to a migration-only cell with no roll stamp', async () => {
    // Unstamped non-roll parking must retain hosts or undo evacuation/Asia rollback/operator intent.
    const { store, park, rollIsolatedAt } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await park(first.cellId, 'migration-only')

    expect(await rollIsolatedAt(first.cellId)).toBeNull()
    const pinned = await store.assign(IDENTITY, 'us-central1')
    expect(pinned).toMatchObject({ cellId: first.cellId, assignmentEpoch: first.assignmentEpoch })
    expect(pinned.placed).toBeUndefined()
  })

  it('re-places a host once the roll stamp is set', async () => {
    const { store, counter, isolateForRoll, rollIsolatedAt } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)

    expect(await rollIsolatedAt(first.cellId)).toBe(START_MS)
    counter.sql.length = 0
    const moved = await store.assign(IDENTITY, 'us-central1')

    expect(moved.cellId).not.toBe(first.cellId)
    expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1)
    expect(moved.placed).toBe(true)
    expect(counter.count('relay_assignment_migrations')).toBeGreaterThan(0)

    // Durable: the next dial does not bounce back to the isolated cell.
    const kept = await store.assign(IDENTITY, 'us-central1')
    expect(kept).toMatchObject({ cellId: moved.cellId, assignmentEpoch: moved.assignmentEpoch })
    expect(kept.placed).toBeUndefined()
  })

  it.each([
    { label: 'just inside the bound', age: 2 * 60 * 60_000 - 1, moves: true },
    { label: 'just outside the bound', age: 2 * 60 * 60_000 + 1, moves: false }
  ])('treats a stamp $label as $moves', async ({ age, moves }) => {
    // Past a normal ~15-minute roll, stamps may be failed-wave or old-restore orphans; keep possibly healthy hosts pinned for operator recovery.
    const { store, heartbeat, isolateForRoll, setNow } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)

    const later = START_MS + age
    setNow(later)
    for (const cell of CELLS) await heartbeat(cell, later)
    const grant = await store.assign(IDENTITY, 'us-central1')

    if (moves) {
      expect(grant.cellId).not.toBe(first.cellId)
      expect(grant.assignmentEpoch).toBe(first.assignmentEpoch + 1)
    } else {
      expect(grant.cellId).toBe(first.cellId)
      expect(grant.assignmentEpoch).toBe(first.assignmentEpoch)
    }
  })

  // Why its own site: the drain-return lane's service time is either this lock or
  // the inventory, and only a separate p99 can say which.
  it('samples the regional target rows it locks under their own hold site', async () => {
    const { store, database, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)
    consumeRelayCellInventoryHold(database)

    await store.assign(IDENTITY, 'us-central1')

    expect(consumeRelayCellInventoryHold(database)).toMatchObject({
      cellInventoryHolds: 1,
      cellInventoryHoldMaxSite: 'isolated-replacement',
      isolatedReplacementHolds: 1
    })
  })

  it('stops re-placing once restore clears the stamp', async () => {
    const { store, isolateForRoll, restore, rollIsolatedAt } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)
    await restore(first.cellId)

    expect(await rollIsolatedAt(first.cellId)).toBeNull()
    expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
      cellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch
    })
  })

  it('keeps a host pinned to an existing-only cell that still serves it', async () => {
    // Why: existing-only cells serve the hosts they already hold (PR #194). Only
    // assignmentStrandedOnUnservedCell may release that pin, and only on proof
    // the cell stopped serving this host.
    const { store, database, heartbeat, park, setNow } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await park(first.cellId, 'existing-only')
    await database.query(
      `INSERT INTO relay_assignment_activity_leases
       (user_id, relay_host_id, activity_id, activity_kind, cell_id,
        request_units, expires_at, updated_at)
       VALUES (?, ?, 'control:live-1', 'control', ?, 1, ?, ?)`,
      [IDENTITY.userId, IDENTITY.relayHostId, first.cellId, START_MS + 600_000, START_MS]
    )

    // Past the stranded rule's minimum grant age, which outlives the heartbeat TTL.
    setNow(START_MS + 61_000)
    for (const cell of CELLS) await heartbeat(cell, START_MS + 61_000)
    expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
      cellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch
    })
  })

  it('keeps a host pinned on an unstamped cell whose migration has completed', async () => {
    // Completed evacuations await separate promotion in migration-only; re-placement would undo them.
    const { store, database, park } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await insertMigration(database, {
      sourceCellId: 'us-c2',
      targetCellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch,
      leases: 0,
      settled: 'completed'
    })
    await park(first.cellId, 'migration-only')

    expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
      cellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch
    })
  })

  it('re-places a stamped cell whose migration has completed', async () => {
    const { store, database, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await insertMigration(database, {
      sourceCellId: 'us-c2',
      targetCellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch,
      leases: 0,
      settled: 'completed'
    })
    await isolateForRoll(first.cellId)

    expect((await store.assign(IDENTITY, 'us-central1')).cellId).not.toBe(first.cellId)
  })

  it('keeps the pin while a migration lease is outstanding', async () => {
    const { store, database, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await insertMigration(database, {
      sourceCellId: first.cellId,
      targetCellId: 'us-c2',
      assignmentEpoch: first.assignmentEpoch,
      leases: 1
    })
    await isolateForRoll(first.cellId)

    expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
      cellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch
    })
  })

  it('keeps the pin while a migration row is open but its lease has lapsed', async () => {
    // Migration rows outlive their 15-minute counters and cannot roll back from non-general sources; re-placement would strand them.
    const { store, database, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await insertMigration(database, {
      sourceCellId: first.cellId,
      targetCellId: 'us-c2',
      assignmentEpoch: first.assignmentEpoch,
      leases: 0
    })
    await isolateForRoll(first.cellId)

    expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
      cellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch
    })
  })

  it('crosses a region boundary only when its own region has no room', async () => {
    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
    const { store, isolateForRoll, park } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    const other = CELLS.find((cell) => cell.region === 'us-central1' && cell.id !== first.cellId)!
    await park(other.id, 'migration-only')
    await isolateForRoll(first.cellId)

    warn.mockClear()
    expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
      cellId: 'asia-c1',
      assignmentEpoch: first.assignmentEpoch + 1
    })
    expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([
      {
        event: 'orca_relay_sticky_replaced_off_isolated_cell',
        fromCellId: first.cellId,
        fromRegion: 'us-central1',
        admissionState: 'migration-only',
        toCellId: 'asia-c1',
        region: 'asia-east2'
      }
    ])
  })

  it('keeps the pin, and says so, when no region has a general cell with room', async () => {
    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
    const { store, isolateForRoll, park } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    for (const cell of CELLS) {
      if (cell.id !== first.cellId) await park(cell.id, 'migration-only')
    }
    await isolateForRoll(first.cellId)

    warn.mockClear()
    expect(await store.assign(IDENTITY, 'us-central1')).toMatchObject({
      cellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch
    })
    expect(jsonEvents(warn, 'orca_relay_sticky_replacement_deferred')).toEqual([
      {
        event: 'orca_relay_sticky_replacement_deferred',
        reason: 'no_same_region_headroom',
        cellId: first.cellId,
        region: 'us-central1'
      }
    ])
    expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([])
  })

  it('prefers its own region over a less loaded cell elsewhere', async () => {
    const { store, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)
    const moved = await store.assign(IDENTITY, 'us-central1')
    expect(moved.region).toBe('us-central1')
    expect(moved.cellId).not.toBe(first.cellId)
  })

  it('moves the only cell of a region to another region', async () => {
    const { store, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'asia-east2')
    expect(first.region).toBe('asia-east2')
    await isolateForRoll(first.cellId)

    // asia-c1 is the only Asia cell, so the only in-region candidate is gone.
    const moved = await store.assign(IDENTITY, 'asia-east2')
    expect(moved.region).toBe('us-central1')
    expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1)
  })

  describe('with a reserve cell in its own region', () => {
    const onReserve = {
      ...IDENTITY,
      cellId: 'asia-reserve',
      cellUrl: 'https://asia-reserve.example.com',
      assignmentEpoch: 9,
      leaseExpiresAt: START_MS + 1_000_000,
      region: 'asia-east2' as const
    }

    it('books the reserve cell before leaving the region, and the canary still counts the move', async () => {
      const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
      const { store, database, isolateForRoll } = await setup()
      const first = await store.assign(IDENTITY, 'asia-east2')
      await isolateForRoll(first.cellId)
      warn.mockClear()
      const regionReserve = vi.fn(async () => onReserve)
      await expect(
        store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
      ).resolves.toEqual({ ...onReserve, placed: true })
      expect(regionReserve).toHaveBeenCalledOnce()
      expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([
        {
          event: 'orca_relay_sticky_replaced_off_isolated_cell',
          fromCellId: first.cellId,
          fromRegion: 'asia-east2',
          admissionState: 'migration-only',
          toCellId: 'asia-reserve',
          region: 'asia-east2',
          via: 'reserve'
        }
      ])
      // The refused in-region attempt wrote nothing.
      expect(
        await database.query(`SELECT cell_id, assignment_epoch FROM relay_assignments`)
      ).toEqual([{ cell_id: first.cellId, assignment_epoch: first.assignmentEpoch }])
    })

    it('leaves the region as before when the reserve cell has no room either', async () => {
      const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
      const { store, isolateForRoll } = await setup()
      const first = await store.assign(IDENTITY, 'asia-east2')
      await isolateForRoll(first.cellId)
      warn.mockClear()
      const regionReserve = vi.fn(async () => null)
      const moved = await store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
      expect(moved).toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 })
      expect(regionReserve).toHaveBeenCalledOnce()
      expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([
        { fromCellId: first.cellId, region: 'us-central1' }
      ])
    })

    it('treats a throttled reserve cell as no room, so the drain does not wait on its intake', async () => {
      const { store, isolateForRoll } = await setup()
      const first = await store.assign(IDENTITY, 'asia-east2')
      await isolateForRoll(first.cellId)
      const regionReserve = vi.fn(async () => {
        throw new ReservePaceError(4)
      })
      await expect(
        store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
      ).resolves.toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 })
      expect(regionReserve).toHaveBeenCalledOnce()
    })

    it('counts a share-draw booking off the isolated cell for the canary', async () => {
      const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
      const { store, isolateForRoll } = await setup()
      const first = await store.assign(IDENTITY, 'asia-east2')
      await isolateForRoll(first.cellId)
      warn.mockClear()
      await expect(
        store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { placeFresh: async () => onReserve })
      ).resolves.toEqual({ ...onReserve, placed: true })
      expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([
        { fromCellId: first.cellId, toCellId: 'asia-reserve', via: 'reserve' }
      ])
    })

    describe('when no database cell has room anywhere', () => {
      const onUsReserve = { ...onReserve, cellId: 'us-reserve', region: 'us-central1' as const }
      const isolateWithNoDatabaseRoom = async () => {
        const harness = await setup()
        const first = await harness.store.assign(IDENTITY, 'asia-east2')
        await harness.isolateForRoll(first.cellId)
        for (const cell of CELLS.filter((entry) => entry.region === 'us-central1')) {
          await harness.park(cell.id, 'migration-only')
        }
        return { ...harness, first }
      }

      // Today's rule: a live isolated cell keeps a host no other cell can take.
      it('keeps the pin and never asks the spill', async () => {
        const { store, first } = await isolateWithNoDatabaseRoom()
        const reserveSpill = vi.fn(async () => onUsReserve)
        const kept = await store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { reserveSpill })
        expect(kept).toMatchObject({ cellId: first.cellId, assignmentEpoch: first.assignmentEpoch })
        expect(reserveSpill).not.toHaveBeenCalled()
      })
    })

    it('does not hold a host whose preferred region is elsewhere', async () => {
      const { store, isolateForRoll, park } = await setup()
      const first = await store.assign(IDENTITY, 'us-central1')
      const other = CELLS.find((cell) => cell.region === 'us-central1' && cell.id !== first.cellId)!
      await park(other.id, 'migration-only')
      await isolateForRoll(first.cellId)
      const regionReserve = vi.fn(async () => onReserve)
      await expect(
        store.assign(IDENTITY, 'asia-east2', 'asia-east2', undefined, { regionReserve })
      ).resolves.toMatchObject({ cellId: 'asia-c1' })
      expect(regionReserve).not.toHaveBeenCalled()
    })
  })

  it('takes the dead-cell path when a stamped cell stops heartbeating', async () => {
    // Why: the fence is the only proof that a cell we cannot reach has stopped
    // serving the sockets it still holds. A stamp does not make it reachable.
    const { store, heartbeat, isolateForRoll, setNow } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)

    const stale = START_MS + HEARTBEAT_TTL_MS + 1_000
    setNow(stale)
    for (const cell of CELLS) {
      if (cell.id !== first.cellId) await heartbeat(cell, stale)
    }

    await expect(store.assign(IDENTITY, 'us-central1')).rejects.toBeInstanceOf(
      RelayHomeCellUnavailableError
    )
  })

  it('leaves the source cell activity leases in place', async () => {
    // Contrast with the fence and stranded paths, which delete them: an isolated
    // cell is alive and still owns drainable work behind those leases.
    const { store, database, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await database.query(
      `INSERT INTO relay_assignment_activity_leases
       (user_id, relay_host_id, activity_id, activity_kind, cell_id,
        request_units, expires_at, updated_at)
       VALUES (?, ?, 'splice:keep-1', 'splice', ?, 1, ?, ?)`,
      [IDENTITY.userId, IDENTITY.relayHostId, first.cellId, START_MS + 600_000, START_MS]
    )
    await isolateForRoll(first.cellId)

    await store.assign(IDENTITY, 'us-central1')
    expect(
      await database.query(
        `SELECT activity_id FROM relay_assignment_activity_leases
         WHERE user_id = ? AND relay_host_id = ? AND activity_id = 'splice:keep-1'`,
        [IDENTITY.userId, IDENTITY.relayHostId]
      )
    ).toHaveLength(1)
  })

  it('emits nothing when the placement rolls back after the decision', async () => {
    // Log only committed moves; stdout cannot roll back with a failed lease/reservation transaction.
    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
    const { store, counter, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)

    warn.mockClear()
    // The first write after the event is decided.
    counter.failOnce = 'INSERT INTO relay_assignments'
    await expect(store.assign(IDENTITY, 'us-central1')).rejects.toThrow(
      'injected_placement_write_failure'
    )

    expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([])
    // Prove the move rolled back so the missing event is required, not merely delayed.
    expect(await store.resolve(IDENTITY)).toMatchObject({
      cellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch
    })

    // Successful control proves event absence above comes from rollback, not a broken harness.
    warn.mockClear()
    const moved = await store.assign(IDENTITY, 'us-central1')
    expect(moved.cellId).not.toBe(first.cellId)
    expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toHaveLength(1)
  })

  it('logs one event naming both cells, the admission state and the region', async () => {
    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
    const { store, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)

    warn.mockClear()
    const moved = await store.assign(IDENTITY, 'us-central1')

    expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toEqual([
      {
        event: 'orca_relay_sticky_replaced_off_isolated_cell',
        fromCellId: first.cellId,
        fromRegion: 'us-central1',
        admissionState: 'migration-only',
        toCellId: moved.cellId,
        region: moved.region
      }
    ])
  })
  describe('a host whose leases span two cells after the move', () => {
    const NEIGHBOUR = { userId: 'user-b', relayHostId: 'host000000000002' }

    async function reserved(database: RelayDatabase, cellId: string): Promise<number> {
      const rows = await database.query(
        `SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
        [cellId]
      )
      return Number(rows[0]!['reserved_requests'])
    }

    async function expectAccounting(database: RelayDatabase): Promise<void> {
      for (const cell of CELLS) {
        const leased = await database.query(
          `SELECT COALESCE(SUM(request_units), 0) AS units
           FROM relay_assignment_activity_leases WHERE cell_id = ?`,
          [cell.id]
        )
        expect({ cell: cell.id, reserved: await reserved(database, cell.id) }).toEqual({
          cell: cell.id,
          reserved: Number(leased[0]!['units'])
        })
      }
    }

    async function counters(database: RelayDatabase): Promise<number[]> {
      const rows = await database.query(
        `SELECT reserved_controls, reserved_splices FROM relay_assignments
         WHERE user_id = ? AND relay_host_id = ?`,
        [IDENTITY.userId, IDENTITY.relayHostId]
      )
      return [Number(rows[0]!['reserved_controls']), Number(rows[0]!['reserved_splices'])]
    }

    // us-c1 is isolated with the host's control and splice on it; the host
    // moves to us-c2, where a neighbour's splice outlives every other lease.
    async function movedHost(): Promise<Harness> {
      const harness = await setup()
      const { store, isolateForRoll } = harness
      const first = await store.assign(IDENTITY, 'us-central1')
      expect(first.cellId).toBe('us-c1')
      await store.activateControl(IDENTITY, {
        cellId: 'us-c1',
        assignmentEpoch: first.assignmentEpoch,
        generation: 1
      })
      await store.acquireActivity(IDENTITY, {
        activityId: 'splice:keep-1',
        kind: 'splice',
        cellId: 'us-c1'
      })
      await isolateForRoll('us-c1')
      expect((await store.assign(NEIGHBOUR, 'us-central1')).cellId).toBe('us-c2')
      await store.acquireActivity(NEIGHBOUR, {
        activityId: 'splice:neighbour',
        kind: 'splice',
        cellId: 'us-c2',
        expiresAt: START_MS + 10 * 60 * 60_000
      })
      expect((await store.assign(IDENTITY, 'us-central1')).cellId).toBe('us-c2')
      expect(await counters(harness.database)).toEqual([2, 1])
      await expectAccounting(harness.database)
      return harness
    }

    it('is left to the lease sweep by aggregate expiry', async () => {
      const { store, database, setNow } = await movedHost()
      setNow(START_MS + 60 * 60_000)
      await store.releaseExpiredActivity()
      expect(await counters(database)).toEqual([2, 1])
      await expectAccounting(database)
      await store.releaseExpiredActivityLeases()
      await store.releaseExpiredActivity()
      expect(await counters(database)).toEqual([0, 0])
      expect(await reserved(database, 'us-c1')).toBe(0)
      expect(await reserved(database, 'us-c2')).toBe(2)
      await expectAccounting(database)
    })

    it('frees the source leases on their own cell when the target dies', async () => {
      const { store, database, setNow, heartbeat, restore } = await movedHost()
      await restore('us-c1')
      // Uncapped, so the dead-cell path needs no fence to move off it.
      await database.query(`DELETE FROM relay_cell_connection_limits WHERE cell_id = 'us-c2'`)
      setNow(START_MS + HEARTBEAT_TTL_MS + 5_000)
      await heartbeat(CELLS[0]!)
      await heartbeat(CELLS[2]!)

      expect((await store.assign(IDENTITY, 'us-central1')).cellId).toBe('us-c1')
      expect(await counters(database)).toEqual([1, 0])
      expect(await store.releaseActivity(IDENTITY, 'control:us-c1:1')).toBe(false)
      expect(await reserved(database, 'us-c1')).toBe(1)
      expect(await reserved(database, 'us-c2')).toBe(3)
      await expectAccounting(database)
    })
  })
  it('retries a busy isolated attempt in its own tier, never under the all-rows lock', async () => {
    const { store, counter, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)
    counter.sql.length = 0
    // The first attempt finds a target row busy, then its retry finds the host's
    // lease rows busy after it has already taken the tier's rows.
    counter.lockUnavailableSequence = [
      'SELECT * FROM relay_cells WHERE cell_id IN (',
      'FROM relay_assignment_activity_leases'
    ]

    const moved = await store.assign(IDENTITY, 'us-central1')
    expect(counter.lockUnavailableSequence).toEqual([])
    expect(moved).toMatchObject({ region: 'us-central1', assignmentEpoch: first.assignmentEpoch + 1 })
    expect(counter.count('SELECT * FROM relay_cells ORDER BY cell_id ASC')).toBe(0)
  })
})

describe('classifying a reconnect whose home is isolated for a roll', () => {
  const classify = { classifyHomeRollIsolation: true }

  it('marks the host only while the roll stamp is current', async () => {
    const { store, heartbeat, isolateForRoll, restore, setNow } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')

    expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
    await isolateForRoll(first.cellId)
    expect(await store.resolve(IDENTITY, classify)).toMatchObject({
      cellId: first.cellId,
      homeCellRollIsolated: true
    })
    // The cell's own callers do not ask, and their read is unchanged.
    expect(await store.resolve(IDENTITY)).not.toHaveProperty('homeCellRollIsolated')

    const stale = START_MS + 2 * 60 * 60_000 + 1
    setNow(stale)
    for (const cell of CELLS) await heartbeat(cell, stale)
    expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')

    setNow(START_MS)
    await restore(first.cellId)
    expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
  })

  it('does not mark a host parked without a stamp or held by a migration', async () => {
    const { store, database, isolateForRoll, park } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await park(first.cellId, 'migration-only')
    expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')

    await isolateForRoll(first.cellId)
    await insertMigration(database, {
      sourceCellId: first.cellId,
      targetCellId: 'us-c2',
      assignmentEpoch: first.assignmentEpoch,
      leases: 1
    })
    expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
    // A stalled migration whose lease counter lapsed still owns the epoch.
    await database.query(`UPDATE relay_assignments SET migration_leases = 0`)
    expect(await store.resolve(IDENTITY, classify)).not.toHaveProperty('homeCellRollIsolated')
  })

  it('reads the classification in the verification query itself', async () => {
    const { store, counter, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await isolateForRoll(first.cellId)

    counter.sql.length = 0
    await store.resolve(IDENTITY, classify)

    expect(counter.sql.filter((sql) => sql.includes('FROM relay_assignments'))).toHaveLength(1)
    expect(counter.count('relay_cell_admission')).toBe(1)
  })

  it('still treats a host its draining cell already released as a drain return', async () => {
    // A drain releases the host's control lease before it tells the host, so the redial
    // finds no activity: the stamp, not a live lease, has to carry the classification.
    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
    const { store, database, isolateForRoll } = await setup()
    const first = await store.assign(IDENTITY, 'us-central1')
    await store.activateControl(IDENTITY, {
      cellId: first.cellId,
      assignmentEpoch: first.assignmentEpoch,
      generation: 1
    })
    await isolateForRoll(first.cellId)
    expect(await store.releaseActivity(IDENTITY, `control:${first.cellId}:1`)).toBe(true)
    expect(
      await database.query(
        `SELECT activity_id FROM relay_assignment_activity_leases WHERE user_id = ?`,
        [IDENTITY.userId]
      )
    ).toEqual([])

    expect(await store.resolve(IDENTITY, classify)).toMatchObject({
      cellId: first.cellId,
      homeCellRollIsolated: true
    })
    warn.mockClear()
    const moved = await store.assign(IDENTITY, 'us-central1')
    expect(moved.cellId).not.toBe(first.cellId)
    expect(moved.assignmentEpoch).toBe(first.assignmentEpoch + 1)
    expect(jsonEvents(warn, 'orca_relay_sticky_replaced_off_isolated_cell')).toMatchObject([
      { fromCellId: first.cellId, toCellId: moved.cellId }
    ])
  })
})