Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import { ASSIGNMENT_LIMITS } from '@orca-cloud/relay-contract'
import { describe, expect, it, vi } from 'vitest'
import { RelayAssignmentStore } from './assignment-store.js'
import type { ControlRenewalOutcome } from './control-renewal-statement.js'
import {
openInMemoryRelayDatabase,
type RelayDatabase,
type SqlRow
} from './database.js'
const now = 1_900_000_000_000
const expiresAt = now + 105_000
function renewal(userId: string, relayHostId: string, expiry = expiresAt) {
return {
identity: { userId, relayHostId },
activityId: 'control:cell-a:1',
cellId: 'cell-a',
expiresAt: expiry
}
}
// A PostgreSQL-dialect database that answers the renewal statement without a
// server, so the statement count and its parameter arrays are observable.
class RenewalStatementProbe implements RelayDatabase {
readonly dialect = 'postgres' as const
readonly statements: Array<{ sql: string; params: unknown[] }> = []
failuresRemaining = 0
failureMessage = 'canceling statement due to statement timeout'
constructor(private readonly outcomeFor: (userId: string) => ControlRenewalOutcome) {}
async query(sql: string, params: unknown[] = []): Promise<SqlRow[]> {
this.statements.push({ sql, params })
if (this.failuresRemaining > 0) {
this.failuresRemaining -= 1
throw new Error(this.failureMessage)
}
const userIds = params[0] as string[]
return userIds.map((userId, index) => ({
row_index: String(index + 1),
outcome: this.outcomeFor(userId)
}))
}
async queryLocked(): Promise<SqlRow[]> {
throw new Error('unexpected_locked_query')
}
async transaction<T>(): Promise<T> {
// Renewals must never open one: that is the write transaction per host this
// batch exists to remove.
throw new Error('unexpected_transaction')
}
async close(): Promise<void> {}
}
describe('batched control renewals on PostgreSQL', () => {
it('spends one statement on every host that came due', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000003')
])
expect(outcomes).toEqual(['renewed', 'renewed', 'renewed'])
expect(probe.statements).toHaveLength(1)
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b'])
expect(probe.statements[0]!.params[4]).toEqual([expiresAt, expiresAt, expiresAt])
})
it('locks assignment rows in primary-key order and still answers in input order', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'control_activity_moved' : 'renewed'
)
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-c', 'host000000000003'),
renewal('user-a', 'host000000000002'),
renewal('user-b', 'host000000000001'),
renewal('user-a', 'host000000000001')
])
// (user_id, relay_host_id) is the primary key of relay_assignments, and the
// statement's ORDER BY repeats it: no batch can queue against another in a
// different sequence.
expect(probe.statements[0]!.params[0]).toEqual(['user-a', 'user-a', 'user-b', 'user-c'])
expect(probe.statements[0]!.params[1]).toEqual([
'host000000000001',
'host000000000002',
'host000000000001',
'host000000000003'
])
expect(outcomes).toEqual([
'renewed',
'renewed',
'control_activity_moved',
'renewed'
])
})
it('keeps a malformed request out of the statement and fails only that row', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const store = new RelayAssignmentStore(probe, () => now)
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002', now + ASSIGNMENT_LIMITS.activityLeaseMs * 10),
{ ...renewal('user-a', 'host000000000003'), activityId: '' },
renewal('user-a', 'host000000000004')
])
expect(outcomes).toEqual([
'renewed',
'invalid_activity_expiry',
'invalid_activity_id',
'renewed'
])
expect(probe.statements[0]!.params[1]).toEqual(['host000000000001', 'host000000000004'])
})
it('degrades to one statement per host when the batch statement fails', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const store = new RelayAssignmentStore(probe, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
expect(outcomes).toEqual(['renewed', 'renewed'])
expect(probe.statements).toHaveLength(3)
expect(probe.statements[1]!.params[1]).toEqual(['host000000000001'])
expect(probe.statements[2]!.params[1]).toEqual(['host000000000002'])
expect(JSON.parse(String(warn.mock.calls[0]![0]))).toMatchObject({
event: 'orca_relay_control_renewal_batch_failed',
rows: 2
})
} finally {
warn.mockRestore()
}
})
it('runs the batch on the priority lane and the per-host fallback on the general one', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const lanes: string[] = []
const database: RelayDatabase = {
dialect: 'postgres',
query: async (sql, params) => {
lanes.push('general')
return await probe.query(sql, params)
},
queryPriority: async (sql, params) => {
lanes.push('priority')
return await probe.query(sql, params)
},
queryLocked: async () => await probe.queryLocked(),
transaction: async () => await probe.transaction(),
close: async () => undefined
}
const store = new RelayAssignmentStore(database, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
// A 200-row fallback would otherwise jump the whole general queue.
expect(lanes).toEqual(['priority', 'general', 'general'])
} finally {
warn.mockRestore()
}
})
it('does not fan out per host when the batch never got a connection', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
probe.failureMessage = 'timeout exceeded when trying to connect'
const store = new RelayAssignmentStore(probe, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
expect(outcomes).toEqual(['database_error', 'database_error'])
expect(probe.statements).toHaveLength(1)
} finally {
warn.mockRestore()
}
})
it('reports a host that fails its own fallback statement without touching the rest', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 2
const store = new RelayAssignmentStore(probe, () => now)
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
try {
const outcomes = await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-a', 'host000000000002')
])
expect(outcomes).toEqual(['database_error', 'renewed'])
} finally {
warn.mockRestore()
}
})
it('counts a lone renewal that threw before rethrowing it', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
probe.failuresRemaining = 1
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
// A one-row flush keeps the pre-batch contract and rethrows, but the metric
// still owes an outcome for the attempt.
await expect(
store.renewControlActivities([renewal('user-a', 'host000000000001')])
).rejects.toThrow('statement timeout')
expect(recordControlRenewal).toHaveBeenCalledTimes(1)
expect(recordControlRenewal.mock.calls[0]![1]).toBe('database_error')
})
it('counts a batch in which every row was rejected before the statement', async () => {
const probe = new RenewalStatementProbe(() => 'renewed')
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
const outcomes = await store.renewControlActivities([
{ ...renewal('user-a', 'host000000000001'), activityId: '' },
renewal('user-a', 'host000000000002', now - 1)
])
expect(outcomes).toEqual(['invalid_activity_id', 'invalid_activity_expiry'])
expect(probe.statements).toHaveLength(0)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'invalid_activity_id',
'invalid_activity_expiry'
])
})
it('counts one renewal outcome metric per row', async () => {
const probe = new RenewalStatementProbe((userId) =>
userId === 'user-b' ? 'assignment_not_found' : 'renewed'
)
const recordControlRenewal = vi.fn()
const store = new RelayAssignmentStore(probe, () => now, { recordControlRenewal })
await store.renewControlActivities([
renewal('user-a', 'host000000000001'),
renewal('user-b', 'host000000000002')
])
expect(recordControlRenewal).toHaveBeenCalledTimes(2)
expect(recordControlRenewal.mock.calls.map((call) => call[1])).toEqual([
'renewed',
'assignment_not_found'
])
})
})
describe('batched control renewals on SQLite', () => {
it('renews every host through the transactional path', async () => {
let clock = now
const database = await openInMemoryRelayDatabase()
try {
const store = new RelayAssignmentStore(database, () => clock)
await store.reconcileCells([
{ id: 'cell-a', url: 'https://relay-a.example.com', capacityRequests: 10 }
])
const hosts = ['host000000000001', 'host000000000002']
const requests = []
for (const relayHostId of hosts) {
const identity = { userId: 'user-a', relayHostId }
const assignment = await store.assign(identity)
await store.activateControl(identity, {
cellId: assignment.cellId,
assignmentEpoch: assignment.assignmentEpoch,
generation: 1
})
requests.push({
identity,
activityId: `control:${assignment.cellId}:1`,
cellId: assignment.cellId,
expiresAt: clock + 105_000
})
}
// A host with no assignment at all must not cost the others their renewal.
requests.push({
identity: { userId: 'user-a', relayHostId: 'host000000000009' },
activityId: 'control:cell-a:1',
cellId: 'cell-a',
expiresAt: clock + 105_000
})
clock += 1_000
const outcomes = await store.renewControlActivities(requests)
expect(outcomes).toEqual(['renewed', 'renewed', 'assignment_not_found'])
const leases = await database.query(
`SELECT relay_host_id, expires_at FROM relay_assignment_activity_leases
WHERE user_id = ? ORDER BY relay_host_id ASC`,
['user-a']
)
expect(leases.map((lease) => Number(lease.expires_at))).toEqual([
now + 105_000,
now + 105_000
])
} finally {
await database.close()
}
})
})