Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import { CONNECTOR_LIMITS, ConnectorPollResponseSchema } from '@orca-cloud/connector-contract'
import { afterEach, describe, expect, it } from 'vitest'
import type { CloudDatabase } from '@orca-cloud/cloud-database'
import type { DeliveryBody } from './connector-inbox.js'
import {
cleanupConnectorHarnesses,
createConnectorHarness,
pauseBefore
} from './connector-server-harness.test-fixture.js'
afterEach(cleanupConnectorHarnesses)
const START: DeliveryBody = { kind: 'start', agent: 'claude', placeId: 'pl_1', prompt: 'fix it' }
const SEND: DeliveryBody = { kind: 'send', text: 'and the tests' }
// Fails the first statement starting with `prefix` once armed, as a transient database fault.
function failOnce(database: CloudDatabase, prefix: string) {
let armed = false
const wrap = (inner: CloudDatabase): CloudDatabase => ({
dialect: inner.dialect,
query: async (sql, params) => {
if (armed && sql.trimStart().startsWith(prefix)) {
armed = false
throw new Error('injected')
}
return await inner.query(sql, params)
},
transaction: (run) => inner.transaction((transaction) => run(wrap(transaction))),
lockQuotaScope: (key) => inner.lockQuotaScope(key),
tryLockScope: (key) => inner.tryLockScope(key),
tryLockSharedScope: (key) => inner.tryLockSharedScope(key),
close: () => inner.close()
})
return { database: wrap(database), arm: () => (armed = true) }
}
async function setup(
options: Parameters<typeof createConnectorHarness>[1] = {},
machineOptions: Parameters<Awaited<ReturnType<typeof createConnectorHarness>>['machine']>[1] = {}
) {
const harness = await createConnectorHarness({}, options)
const machine = await harness.machine('user-1', machineOptions)
expect(await machine.hello()).toEqual({ status: 200, helloRev: 1 })
const enqueue = (externalId: string, deliveryId: string, body: DeliveryBody, held = false) =>
harness.runtime.queueDelivery({
thread: {
conversation: { source: 'slack', externalId },
place: { workspaceId: 'T1', containerId: 'C1', threadId: externalId },
ownerUserId: 'U1',
hostId: machine.hostId,
placeId: 'pl_1'
},
deliveryId,
body,
held
})
const poll = async () => {
const { status, body } = await machine.poll()
expect(status).toBe(200)
return ConnectorPollResponseSchema.parse(body)
}
const ids = async () => (await poll()).deliveries.map((delivery) => delivery.deliveryId)
const attach = (deliveryId: string, prompt: string) =>
harness.database.transaction((transaction) =>
harness.runtime.inbox.attach(transaction, machine.hostId, deliveryId, prompt)
)
const states = async () =>
Object.fromEntries(
(await harness.database.query('SELECT delivery_id, state FROM connector_inbox')).map(
(row) => [String(row.delivery_id), String(row.state)]
)
)
return { harness, machine, enqueue, poll, ids, attach, states }
}
describe('connector inbox', () => {
it('refuses a request without a connector token', async () => {
const harness = await createConnectorHarness()
expect((await harness.json('/v1/connector/poll', { v: 1, helloRev: 1 })).status).toBe(401)
})
it('names a host by account and machine, whichever token it presents', async () => {
const { harness, machine, enqueue, ids } = await setup()
await enqueue('a', 'a1', START)
const sameMachine = await harness.machine('user-1', { machineId: machine.machineId })
expect(
ConnectorPollResponseSchema.parse((await sameMachine.poll()).body).deliveries
).toHaveLength(1)
const otherAccount = await harness.machine('user-2', { machineId: machine.machineId })
expect((await otherAccount.poll()).status).toBe(409)
expect(await ids()).toEqual([])
})
it('requires a hello for the revision a poll names', async () => {
const harness = await createConnectorHarness()
const machine = await harness.machine()
expect((await machine.poll()).status).toBe(409)
await machine.hello()
expect((await machine.poll(2)).status).toBe(409)
expect((await machine.poll(1)).status).toBe(200)
})
it('hands out a queued delivery with its envelope', async () => {
const { harness, enqueue, poll } = await setup()
await expect(enqueue('T1:C1:1.0:U1', 'd1', START)).resolves.toBe('queued')
const reply = await poll()
expect(reply.holdMs).toBe(0)
expect(reply.serverNow).toBe(harness.now())
expect(reply.deliveries).toEqual([
{
...START,
deliveryId: 'd1',
seq: 1,
expiresAt: harness.now() + CONNECTOR_LIMITS.deliveryTtlMs,
conversation: { source: 'slack', externalId: 'T1:C1:1.0:U1' }
}
])
await expect(enqueue('T1:C1:1.0:U1', 'd1', START)).resolves.toBe('duplicate')
})
it('wakes a held poll as soon as a delivery is queued', async () => {
const { enqueue, poll } = await setup({ pollHoldMs: 10_000 })
const held = poll()
await new Promise((resolve) => setTimeout(resolve, 100))
const startedAt = Date.now()
await enqueue('T1:C1:1.0:U1', 'd1', START)
expect((await held).deliveries.map((delivery) => delivery.deliveryId)).toEqual(['d1'])
expect(Date.now() - startedAt).toBeLessThan(5_000)
})
it('sends one delivery per conversation until it is received', async () => {
const { machine, enqueue, ids } = await setup()
await enqueue('a', 'a1', START)
await enqueue('a', 'a2', SEND)
await enqueue('b', 'b1', START)
expect(await ids()).toEqual(['a1', 'b1'])
expect(await ids()).toEqual([])
await machine.ack([{ deliveryId: 'a1', step: 'received' }])
expect(await ids()).toEqual(['a2'])
})
it('resends a delivery with the same id once its lease runs out unreceived', async () => {
const { harness, enqueue, ids } = await setup()
await enqueue('a', 'a1', START)
expect(await ids()).toEqual(['a1'])
harness.advance(CONNECTOR_LIMITS.leaseMs - 1)
expect(await ids()).toEqual([])
harness.advance(1)
expect(await ids()).toEqual(['a1'])
})
it('lists received-but-not-done deliveries on the first poll after each hello only', async () => {
const { machine, enqueue, poll } = await setup()
await enqueue('a', 'a1', START)
await enqueue('b', 'b1', START)
await poll()
await machine.ack([
{ deliveryId: 'a1', step: 'received' },
{ deliveryId: 'b1', step: 'received' },
{ deliveryId: 'b1', step: 'done', result: { outcome: 'accepted' } }
])
expect((await poll()).unsettled).toBeUndefined()
// The machine restarts: a new hello, then its first poll.
await machine.hello()
const first = ConnectorPollResponseSchema.parse((await machine.poll(2)).body)
expect(first.unsettled).toEqual([
{ deliveryId: 'a1', conversation: { source: 'slack', externalId: 'a' } }
])
const second = ConnectorPollResponseSchema.parse((await machine.poll(2)).body)
expect(second.unsettled).toBeUndefined()
})
// SQLite runs one transaction at a time, so only Postgres can interleave the sweep with a claim.
it.skipIf(!process.env.ORCA_CONNECTOR_TEST_DATABASE_URL)(
'keeps an expiry that lands while a resend is being claimed',
async () => {
let gate!: ReturnType<typeof pauseBefore>
const { harness, enqueue, poll } = await setup({
wrapDatabase: (database) => {
gate = pauseBefore(database, "UPDATE connector_inbox SET state = 'leased'", 1)
return gate.database
}
})
await enqueue('a', 'a1', START)
await poll()
harness.advance(CONNECTOR_LIMITS.leaseMs)
const resend = poll()
await gate.atStatement
// The sweep takes no host lock, so it can expire the row between the claim's read and write.
harness.advance(CONNECTOR_LIMITS.deliveryTtlMs)
expect(await harness.runtime.inbox.expireDue()).toHaveLength(1)
gate.release()
expect((await resend).deliveries).toEqual([])
const [row] = await harness.database.query(
"SELECT state FROM connector_inbox WHERE delivery_id = 'a1'"
)
expect(row?.state).toBe('expired')
}
)
it('keeps recovery for the next poll when the first one fails', async () => {
let fault!: ReturnType<typeof failOnce>
const { machine, enqueue, poll } = await setup({
wrapDatabase: (database) => {
fault = failOnce(database, 'SELECT delivery_id, source, external_id, seq')
return fault.database
}
})
await enqueue('a', 'a1', START)
await poll()
await machine.ack([{ deliveryId: 'a1', step: 'received' }])
await machine.hello()
fault.arm()
expect((await machine.poll(2)).status).toBe(500)
const retry = ConnectorPollResponseSchema.parse((await machine.poll(2)).body)
expect(retry.unsettled).toEqual([
{ deliveryId: 'a1', conversation: { source: 'slack', externalId: 'a' } }
])
})
it('sends a poll held across a newer hello back to hello', async () => {
const { machine } = await setup({ pollHoldMs: 10_000 })
const held = machine.poll(1)
await new Promise((resolve) => setTimeout(resolve, 100))
const startedAt = Date.now()
expect((await machine.hello()).status).toBe(200)
expect((await held).status).toBe(409)
expect(Date.now() - startedAt).toBeLessThan(5_000)
})
it('refuses acks under any hello but the current one, applying nothing', async () => {
const { harness, machine, enqueue, poll } = await setup()
await enqueue('a', 'a1', START)
await poll()
expect((await machine.ack([{ deliveryId: 'a1', step: 'received' }], 2)).status).toBe(409)
await machine.hello()
expect((await machine.ack([{ deliveryId: 'a1', step: 'received' }], 1)).status).toBe(409)
const [row] = await harness.database.query(
"SELECT state FROM connector_inbox WHERE delivery_id = 'a1'"
)
expect(row?.state).toBe('leased')
})
// SQLite runs one transaction at a time, so only Postgres can race an ack with a hello.
it.skipIf(!process.env.ORCA_CONNECTOR_TEST_DATABASE_URL)(
'refuses an ack from the old hello that waited on a newer hello; the lease resends it',
async () => {
let gate!: ReturnType<typeof pauseBefore>
const { harness, machine, enqueue, poll } = await setup({
wrapDatabase: (database) => {
gate = pauseBefore(database, 'INSERT INTO connector_hosts', 1)
return gate.database
}
})
await enqueue('a', 'a1', START)
await poll()
const hello = machine.hello()
await gate.atStatement
const late = machine.ack([{ deliveryId: 'a1', step: 'received' }], 1)
await new Promise((resolve) => setTimeout(resolve, 200))
gate.release()
expect((await hello).status).toBe(200)
expect((await late).status).toBe(409)
harness.advance(CONNECTOR_LIMITS.leaseMs)
const first = ConnectorPollResponseSchema.parse((await machine.poll(2)).body)
expect(first.deliveries.map((delivery) => delivery.deliveryId)).toEqual(['a1'])
}
)
it('gives each hello the next revision, and only the newest is current', async () => {
const { machine } = await setup()
expect(await machine.hello()).toEqual({ status: 200, helloRev: 2 })
expect((await machine.poll(1)).status).toBe(409)
expect((await machine.poll(2)).status).toBe(200)
})
// SQLite runs one transaction at a time, so only Postgres can race two hellos.
it.skipIf(!process.env.ORCA_CONNECTOR_TEST_DATABASE_URL)(
'gives racing hellos distinct revisions',
async () => {
let gate!: ReturnType<typeof pauseBefore>
const { machine } = await setup({
wrapDatabase: (database) => {
gate = pauseBefore(database, 'INSERT INTO connector_hosts', 1)
return gate.database
}
})
const first = machine.hello()
await gate.atStatement
const second = machine.hello()
await new Promise((resolve) => setTimeout(resolve, 200))
gate.release()
const revs = [(await first).helloRev, (await second).helloRev]
expect(revs).toEqual([2, 3])
expect((await machine.poll(3)).status).toBe(200)
}
)
it('keeps the first done', async () => {
const { harness, machine, enqueue, poll } = await setup()
await enqueue('a', 'a1', START)
await poll()
await machine.ack([{ deliveryId: 'a1', step: 'done', result: { outcome: 'accepted' } }])
await machine.ack([{ deliveryId: 'a1', step: 'done', result: { outcome: 'unsupported' } }])
const [row] = await harness.database.query(
"SELECT state, result, payload FROM connector_inbox WHERE delivery_id = 'a1'"
)
expect(row).toEqual({ state: 'done', result: '{"outcome":"accepted"}', payload: null })
})
it('drops a start the machine never received when it is cancelled', async () => {
const { enqueue, ids } = await setup()
await enqueue('a', 'a1', START)
await enqueue('a', 'a2', SEND)
await expect(enqueue('a', 'a3', { kind: 'cancel' })).resolves.toBe('dropped')
await expect(enqueue('a', 'a3', { kind: 'cancel' })).resolves.toBe('duplicate')
expect(await ids()).toEqual([])
})
it('lets a cancel skip the queue once its start is received', async () => {
const { machine, enqueue, ids } = await setup()
await enqueue('a', 'a1', START)
expect(await ids()).toEqual(['a1'])
await machine.ack([{ deliveryId: 'a1', step: 'received' }])
await enqueue('a', 'a2', SEND)
expect(await ids()).toEqual(['a2'])
await enqueue('a', 'a3', SEND)
await expect(enqueue('a', 'a4', { kind: 'cancel' })).resolves.toBe('queued')
expect(await ids()).toEqual(['a4'])
})
it('still lets a cancel skip the queue after its settled start is pruned', async () => {
const { harness, machine, enqueue, ids } = await setup()
await enqueue('a', 'a1', START)
await ids()
await machine.ack([{ deliveryId: 'a1', step: 'done', result: { outcome: 'accepted' } }])
// What retention leaves of a long conversation: its settled start is gone.
await harness.database.query("DELETE FROM connector_inbox WHERE delivery_id = 'a1'")
await enqueue('a', 'a2', SEND)
expect(await ids()).toEqual(['a2'])
await enqueue('a', 'a3', { kind: 'cancel' })
expect(await ids()).toEqual(['a3'])
})
it('queues nothing for a machine that never said hello', async () => {
const { harness, enqueue } = await setup()
const silent = await harness.machine('user-1')
await expect(
harness.runtime.queueDelivery({
thread: {
conversation: { source: 'slack', externalId: 'a' },
place: { workspaceId: 'T1', containerId: 'C1', threadId: 'a' },
ownerUserId: 'U1',
hostId: silent.hostId,
placeId: 'pl_1'
},
deliveryId: 'a1',
body: START
})
).resolves.toBe('unknown_host')
await expect(enqueue('b', 'b1', START)).resolves.toBe('queued')
})
it('expires pending work at its deadline and leased work 30 s after it', async () => {
const { harness, enqueue, ids } = await setup()
await enqueue('a', 'a1', START)
expect(await ids()).toEqual(['a1'])
await enqueue('b', 'b1', START)
harness.advance(CONNECTOR_LIMITS.deliveryTtlMs)
expect(await harness.runtime.inbox.expireDue()).toEqual([
{ hostId: expect.any(String), conversation: { source: 'slack', externalId: 'b' } }
])
expect(await ids()).toEqual([])
harness.advance(CONNECTOR_LIMITS.leasedExpiryGraceMs)
expect(await harness.runtime.inbox.expireDue()).toEqual([
{ hostId: expect.any(String), conversation: { source: 'slack', externalId: 'a' } }
])
})
it('holds a start, and what is behind it, until its prompt is attached', async () => {
const { enqueue, poll, ids, attach } = await setup()
await enqueue('a', 'a1', START, true)
await enqueue('a', 'a2', SEND)
await enqueue('b', 'b1', START)
expect(await ids()).toEqual(['b1'])
await expect(attach('a1', 'the thread\n\nfix it')).resolves.toBe(true)
await expect(attach('a1', 'again')).resolves.toBe(false)
expect((await poll()).deliveries).toEqual([
expect.objectContaining({ deliveryId: 'a1', prompt: 'the thread\n\nfix it' })
])
})
it('refuses to attach a held start past its deadline, swept yet or not', async () => {
const { harness, enqueue, ids, attach, states } = await setup()
await enqueue('a', 'a1', START, true)
harness.advance(CONNECTOR_LIMITS.deliveryTtlMs)
await expect(attach('a1', 'late')).resolves.toBe(false)
expect(await harness.runtime.inbox.expireDue()).toEqual([
{ hostId: expect.any(String), conversation: { source: 'slack', externalId: 'a' } }
])
await expect(attach('a1', 'late')).resolves.toBe(false)
expect(await states()).toEqual({ a1: 'expired' })
expect(await ids()).toEqual([])
})
it('expires what is queued behind a start that never left, and lets a new start through', async () => {
const { harness, enqueue, ids, states } = await setup()
await enqueue('a', 'a1', START, true)
harness.advance(1)
await enqueue('a', 'a2', SEND)
harness.advance(CONNECTOR_LIMITS.deliveryTtlMs - 1)
await enqueue('a', 'a3', START)
await enqueue('a', 'a4', SEND)
await harness.runtime.inbox.expireDue()
expect(await states()).toEqual({ a1: 'expired', a2: 'expired', a3: 'pending', a4: 'pending' })
expect(await ids()).toEqual(['a3'])
})
it('never hands out a send that committed behind a start an earlier sweep expired', async () => {
const { harness, enqueue, ids, states } = await setup()
await enqueue('a', 'a1', START)
harness.advance(CONNECTOR_LIMITS.deliveryTtlMs)
await harness.runtime.inbox.expireDue()
// Routed on a read from before the sweep.
await enqueue('a', 'a2', SEND)
expect(await ids()).toEqual([])
await harness.runtime.inbox.expireDue()
expect(await states()).toEqual({ a1: 'expired', a2: 'expired' })
})
it('keeps what is behind a start that was leased, since it may yet be received', async () => {
const { harness, machine, enqueue, ids, states } = await setup()
await enqueue('a', 'a1', START)
expect(await ids()).toEqual(['a1'])
harness.advance(CONNECTOR_LIMITS.deliveryTtlMs - 1)
await enqueue('a', 'a2', SEND)
harness.advance(CONNECTOR_LIMITS.leasedExpiryGraceMs + 1)
await harness.runtime.inbox.expireDue()
expect(await states()).toEqual({ a1: 'expired', a2: 'pending' })
await machine.ack([{ deliveryId: 'a1', step: 'received' }])
expect(await ids()).toEqual(['a2'])
})
it('accepts a late received for expired work', async () => {
const { harness, machine, enqueue, ids } = await setup()
await enqueue('a', 'a1', START)
await ids()
harness.advance(CONNECTOR_LIMITS.deliveryTtlMs + CONNECTOR_LIMITS.leasedExpiryGraceMs)
await harness.runtime.inbox.expireDue()
expect((await machine.ack([{ deliveryId: 'a1', step: 'received' }])).status).toBe(204)
const [row] = await harness.database.query(
"SELECT state FROM connector_inbox WHERE delivery_id = 'a1'"
)
expect(row?.state).toBe('received')
})
it('ends a held poll by its token expiry', async () => {
const { machine, poll } = await setup({ pollHoldMs: 10_000 }, { ttlSeconds: 2 })
const reply = await poll()
expect(reply.deliveries).toEqual([])
expect(Date.now()).toBeGreaterThanOrEqual(machine.expiresAt)
expect(Date.now() - machine.expiresAt).toBeLessThan(1_000)
})
it('rolls back a poll pass whose token expired while it ran', async () => {
let gate!: ReturnType<typeof pauseBefore>
const { harness, machine, enqueue } = await setup(
{
wrapDatabase: (database) => {
gate = pauseBefore(database, 'SELECT delivery_id, source, external_id, seq')
return gate.database
}
},
{ ttlSeconds: 2 }
)
await enqueue('a', 'a1', START)
const reply = machine.poll()
await gate.atStatement
await new Promise((resolve) => setTimeout(resolve, machine.expiresAt - Date.now() + 10))
gate.release()
expect(ConnectorPollResponseSchema.parse((await reply).body).deliveries).toEqual([])
const [row] = await harness.database.query(
"SELECT state FROM connector_inbox WHERE delivery_id = 'a1'"
)
expect(row?.state).toBe('pending')
const [host] = await harness.database.query('SELECT unsettled_due FROM connector_hosts')
expect(Number(host?.unsettled_due)).toBe(1)
})
it('wakes a held poll when a lease runs out', async () => {
const { harness, enqueue, ids, poll } = await setup({ pollHoldMs: 10_000 })
await enqueue('a', 'a1', START)
expect(await ids()).toEqual(['a1'])
harness.advance(CONNECTOR_LIMITS.leaseMs - 100)
const startedAt = Date.now()
const held = poll()
setTimeout(() => harness.advance(100), 50)
expect((await held).deliveries.map((delivery) => delivery.deliveryId)).toEqual(['a1'])
expect(Date.now() - startedAt).toBeLessThan(5_000)
})
it('answers held polls at once with a reconnect when the instance drains', async () => {
const { harness, machine } = await setup({ pollHoldMs: 10_000 })
const held = machine.poll()
await new Promise((resolve) => setTimeout(resolve, 100))
harness.runtime.waker.drain()
const reply = ConnectorPollResponseSchema.parse((await held).body)
expect(reply.control).toEqual({ reconnect: true })
})
})