Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import { openRelayDatabase, type RelayDatabase } from './database.js'
const databaseUrl = process.env.ORCA_RELAY_TEST_POSTGRES_URL
const describePostgres = databaseUrl ? describe : describe.skip
// Pin to last ascending cell: a fleet scan holds earlier rows while waiting, whereas per-cell locking leaves them free; first-cell pins cannot distinguish these.
const cells = ['a', 'b', 'c'].map((suffix) => ({
id: `percell-postgres-${suffix}`,
url: `https://percell-postgres-${suffix}.example.com`,
capacityRequests: 1_000,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}))
const [cellA, cellB, cellC] = cells as [(typeof cells)[0], (typeof cells)[0], (typeof cells)[0]]
const identity = { userId: 'percell-postgres-user', relayHostId: 'percellhost00001' }
function heartbeat(cell: (typeof cells)[number]) {
return {
cellId: cell.id,
cellUrl: cell.url,
cellIncarnation: '11111111-1111-4111-8111-111111111111',
startedAt: 50,
ready: true,
observedRequests: 0,
totalConnections: 0,
inFlightConnections: 0,
reservedConnectionUnits: 0,
enforcedConnectionUnits: 0,
connectionInclusionWatermark: 1,
connectionHardCap: 600 as const,
connectionUnobservedBound: 50
}
}
describePostgres('PostgreSQL per-cell inventory locking', () => {
const databases: RelayDatabase[] = []
beforeAll(async () => {
for (let index = 0; index < 3; index++) {
databases.push(await openRelayDatabase({ databaseUrl, dataDir: '' }))
}
})
async function removeTestRows(database: RelayDatabase): Promise<void> {
await database.query(
`DELETE FROM relay_control_connection_reservations WHERE user_id LIKE 'percell-postgres-%'`
)
for (const table of [
'relay_assignment_activity_leases',
'relay_post_drain_migration_pins',
'relay_assignment_migration_incarnations',
'relay_assignment_migrations',
'relay_assignment_region_preferences',
'relay_assignments'
]) {
await database.query(`DELETE FROM ${table} WHERE user_id LIKE 'percell-postgres-%'`)
}
for (const cell of cells) {
for (const table of [
'relay_cell_connection_snapshots',
'relay_cell_connection_runtime',
'relay_cell_connection_limits',
'relay_cell_runtime',
'relay_cells'
]) {
await database.query(`DELETE FROM ${table} WHERE cell_id = ?`, [cell.id])
}
}
}
afterAll(async () => {
if (databases[0]) await removeTestRows(databases[0])
for (const connection of databases) await connection.close()
})
async function pinHostToLastCell(store: RelayAssignmentStore): Promise<void> {
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
await store.setCellEnabled(cellA.id, false)
await store.setCellEnabled(cellB.id, false)
const assignment = await store.assign(identity)
expect(assignment.cellId).toBe(cellC.id)
await store.setCellEnabled(cellA.id, true)
await store.setCellEnabled(cellB.id, true)
}
async function lockWaiterAppeared(database: RelayDatabase): Promise<boolean> {
const deadline = Date.now() + 4_000
while (Date.now() < deadline) {
const rows = await database.query(
`SELECT count(*) AS waiting FROM pg_stat_activity
WHERE datname = current_database() AND wait_event_type = 'Lock'`
)
if (Number(rows[0]!.waiting) > 0) return true
await new Promise((resolve) => setTimeout(resolve, 10))
}
return false
}
// Sticky NOWAIT retry must lock only the pinned cell first; fleet-wide retry previously stalled every cell’s reconnects.
it('waits only on the pinned cell row while refreshing a sticky assignment', async () => {
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await pinHostToLastCell(store)
// A host whose control lease was already reaped still holds its pin; that
// is the shape that reaches the cell-row probe instead of touchAssignment.
await databases[0]!.query(
`DELETE FROM relay_assignment_activity_leases WHERE user_id = ?`,
[identity.userId]
)
let releaseRow!: () => void
const rowReleased = new Promise<void>((resolve) => {
releaseRow = resolve
})
let rowHeld!: () => void
const rowHeldPromise = new Promise<void>((resolve) => {
rowHeld = resolve
})
const holder = databases[1]!.transaction(async (transaction) => {
await transaction.queryLocked(`SELECT * FROM relay_cells WHERE cell_id = ?`, [cellC.id])
rowHeld()
await rowReleased
})
await rowHeldPromise
const refresh = store.assign(identity)
const heldWhileRefreshWaits: string[] = []
try {
expect(await lockWaiterAppeared(databases[2]!)).toBe(true)
// The refresh is blocked on cell C. Every earlier row must still be free:
// the ordered fleet-wide scan would be holding both of them by now.
await databases[2]!.transaction(async (transaction) => {
for (const cell of [cellA, cellB]) {
try {
await transaction.queryLocked(
`SELECT * FROM relay_cells WHERE cell_id = ?`,
[cell.id],
{ failIfUnavailable: true }
)
} catch {
heldWhileRefreshWaits.push(cell.id)
}
}
})
} finally {
releaseRow()
await holder
await refresh.catch(() => undefined)
}
expect(heldWhileRefreshWaits).toEqual([])
expect((await refresh).cellId).toBe(cellC.id)
}, 15_000)
// Why: the counter moves by a delta now instead of an absolute value read
// from a snapshot, so concurrent movement on the same cell must still sum.
it('keeps a cell reservation exact under concurrent same-cell activity', async () => {
await removeTestRows(databases[0]!)
const store = new RelayAssignmentStore(databases[0]!, () => 100)
await store.reconcileCells(cells)
for (const cell of cells) await store.recordCellHeartbeat(heartbeat(cell))
await store.setCellEnabled(cellA.id, false)
await store.setCellEnabled(cellB.id, false)
const hosts = Array.from({ length: 6 }, (_, index) => ({
userId: `percell-postgres-user-${index}`,
relayHostId: `percellhost0000${index}`
}))
const stores = databases.map((database) => new RelayAssignmentStore(database, () => 100))
await Promise.all(hosts.map((host, index) => stores[index % stores.length]!.assign(host)))
// One splice each (2 units) on the same cell, from three connections at once.
await Promise.all(
hosts.map((host, index) =>
stores[index % stores.length]!.acquireActivity(host, {
activityId: `splice:percell-${index}`,
kind: 'splice',
cellId: cellC.id
})
)
)
const afterAcquire = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cellC.id]
)
// 6 pending control grants + 6 splices at 2 units each.
expect(Number(afterAcquire[0]!.reserved_requests)).toBe(6 + 12)
await Promise.all(
hosts.map((host, index) =>
stores[index % stores.length]!.releaseActivity(host, `splice:percell-${index}`)
)
)
const afterRelease = await databases[0]!.query(
`SELECT reserved_requests FROM relay_cells WHERE cell_id = ?`,
[cellC.id]
)
expect(Number(afterRelease[0]!.reserved_requests)).toBe(6)
await store.setCellEnabled(cellA.id, true)
await store.setCellEnabled(cellB.id, true)
}, 15_000)
})