Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript

fix(realtime): durable wakes go through the outbox (#321)

OKNOTOKa90c053fbea5
diff --git a/server/src/__tests__/realtime-outbox-order.test.ts b/server/src/__tests__/realtime-outbox-order.test.ts
new file mode 100644
index 0000000..432a01f
--- /dev/null
+++ b/server/src/__tests__/realtime-outbox-order.test.ts
@@ -0,0 +1,30 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { readFile } from 'node:fs/promises'
+
+process.env.CUMORA_RUNTIME_CLIENT = 'http'
+process.env.OPENAI_API_KEY ??= 'test-key'
+
+const { compareOutboxRows } = await import('../realtime-outbox.js')
+
+test('outbox delivery keeps created_at order inside a claimed batch', () => {
+ const rows = [
+ { id: 'b', createdAt: '2026-10-03T00:00:02.000Z' },
+ { id: 'a', createdAt: '2026-10-03T00:00:01.000Z' },
+ { id: 'c', createdAt: '2026-10-03T00:00:02.000Z' },
+ ]
+ assert.deepEqual([...rows].sort(compareOutboxRows).map((row) => row.id), ['a', 'b', 'c'])
+})
+
+test('durable message wakes are enqueued in the same transaction as the insert', async () => {
+ const mention = await readFile(new URL('../ws.ts', import.meta.url), 'utf8')
+ const wake = mention.slice(mention.indexOf('async function postDocMentionWake'))
+ const wakeCommit = wake.indexOf("await client.query('COMMIT')")
+ assert.ok(wake.indexOf('enqueueBroadcast(client, CH_MESSAGE_NEW') < wakeCommit)
+
+ const pull = await readFile(new URL('../agents/scanner_helper.ts', import.meta.url), 'utf8')
+ const pullCommit = pull.indexOf("await client.query('COMMIT')")
+ assert.ok(pull.indexOf('enqueueBroadcast(client, CH_MESSAGE_NEW') < pullCommit)
+ assert.ok(pull.indexOf('enqueueBroadcast(client, CH_GROUP_PULLED') < pullCommit)
+ assert.doesNotMatch(pull, /publish\(CH_MESSAGE_NEW/)
+})
diff --git a/server/src/agents/computer/registry.ts b/server/src/agents/computer/registry.ts
index e0492b5..cb535f4 100644
--- a/server/src/agents/computer/registry.ts
+++ b/server/src/agents/computer/registry.ts
@@ -19,7 +19,8 @@ import { InvalidModelIdError, parseModelId } from './model-id.js'
import { createHash, randomBytes, randomUUID } from 'node:crypto'
import { pool } from '../../db/pool.js'
import { audit } from '../../auth.js'
-import { CH_STATUS, publish } from '../../redis.js'
+import { CH_STATUS } from '../../redis.js'
+import { enqueueBroadcast, nudgeRealtimeOutbox } from '../../realtime-outbox.js'
import { normalizeTier, type Tier } from '../../tier.js'
import { signAgentToken } from '../runtime/jwt.js'
import type { EngineModelCatalog, EngineModelOption, FastModelScope, ModelCatalogSource } from './model-catalog.js'
@@ -38,12 +39,15 @@ export const COMPUTER_STALE_MS = 90_000
async function broadcastComputerStatus(
computerId: string, companyId: string, status: ComputerStatus,
): Promise<void> {
- await publish(CH_STATUS, { type: 'computers.status', computerId, status, companyId }).catch((error) => {
+ try {
+ await enqueueBroadcast(pool, CH_STATUS, { type: 'computers.status', computerId, status, companyId })
+ nudgeRealtimeOutbox()
+ } catch (error) {
console.warn(
- `[computer] durable ${computerId}=${status} update committed but publish failed`,
+ `[computer] durable ${computerId}=${status} update committed but outbox insert failed`,
error,
)
- })
+ }
}
/** Announce a just-paired computer as online. Split out from {@link pairComputer}
diff --git a/server/src/agents/runtime/probe-wake.ts b/server/src/agents/runtime/probe-wake.ts
index 06d5ea1..da5c720 100644
--- a/server/src/agents/runtime/probe-wake.ts
+++ b/server/src/agents/runtime/probe-wake.ts
@@ -9,6 +9,8 @@
* The conversation must already exist and have agent members.
*/
import { redis, CH_MESSAGE_NEW } from '../../redis.js'
+import { pool } from '../../db/pool.js'
+import { enqueueBroadcast, drainRealtimeOutbox } from '../../realtime-outbox.js'
async function main(): Promise<void> {
const conversationId = process.argv[2]
@@ -17,13 +19,23 @@ async function main(): Promise<void> {
console.error('usage: tsx probe-wake.ts <conversationId> <authorId>')
process.exit(2)
}
- await redis.publish(CH_MESSAGE_NEW, JSON.stringify({
+ await enqueueBroadcast(pool, CH_MESSAGE_NEW, {
type: 'message.new',
conversationId,
- message: { authorId },
- }))
- console.log(`[probe-wake] published msg.new for convo=${conversationId} author=${authorId}`)
+ message: {
+ id: `probe-${Date.now()}`,
+ conversationId,
+ authorId,
+ kind: 'text',
+ body: '',
+ sequence: 0,
+ at: new Date().toISOString(),
+ },
+ })
+ await drainRealtimeOutbox()
+ console.log(`[probe-wake] queued msg.new for convo=${conversationId} author=${authorId}`)
redis.disconnect()
+ await pool.end()
}
void main()
diff --git a/server/src/agents/scanner_helper.ts b/server/src/agents/scanner_helper.ts
index 25a42ad..950eecd 100644
--- a/server/src/agents/scanner_helper.ts
+++ b/server/src/agents/scanner_helper.ts
@@ -2,7 +2,8 @@
import { randomUUID } from 'node:crypto'
import { pool } from '../db/pool.js'
import { dispatchMessagePush } from '../push.js'
-import { CH_GROUP_PULLED, CH_MESSAGE_NEW, publish } from '../redis.js'
+import { CH_GROUP_PULLED, CH_MESSAGE_NEW } from '../redis.js'
+import { enqueueBroadcast, nudgeRealtimeOutbox } from '../realtime-outbox.js'
/** Hours an agent must wait between human-interrupting group pulls. */
const PULL_COOLDOWN_HOURS = 6
@@ -119,6 +120,18 @@ export async function startPulledGroup(args: {
VALUES ($1, $2, $3, 'text', $4, 1, $5)`,
[messageId, conversationId, instigatorId, opening, companyId],
)
+ await enqueueBroadcast(client, CH_GROUP_PULLED, {
+ type: 'group.pulled', conversationId, companyId, pulledById: instigatorId,
+ })
+ await enqueueBroadcast(client, CH_MESSAGE_NEW, {
+ type: 'message.new',
+ conversationId,
+ companyId,
+ message: {
+ id: messageId, conversationId, authorId: instigatorId,
+ kind: 'text', body: opening, sequence: 1, at: pulledAt,
+ },
+ })
await client.query('COMMIT')
} catch (error) {
await client.query('ROLLBACK').catch(() => {})
@@ -126,23 +139,7 @@ export async function startPulledGroup(args: {
} finally {
client.release()
}
-
- await publish(CH_GROUP_PULLED, {
- type: 'group.pulled', conversationId, companyId, pulledById: instigatorId,
- }).catch((error) => {
- console.warn(`[pull_group] durable conversation ${conversationId} committed but group publish failed`, error)
- })
- await publish(CH_MESSAGE_NEW, {
- type: 'message.new',
- conversationId,
- companyId,
- message: {
- id: messageId, conversationId, authorId: instigatorId,
- kind: 'text', body: opening, sequence: 1, at: pulledAt,
- },
- }).catch((error) => {
- console.warn(`[pull_group] durable message ${messageId} committed but publish failed`, error)
- })
+ nudgeRealtimeOutbox()
// The point of a pull is to interrupt someone — there is a six-hour cooldown
// on it for exactly that reason. Yet the interruption reached the websocket
diff --git a/server/src/onboardCompany.ts b/server/src/onboardCompany.ts
index b65f9be..7331e1b 100644
--- a/server/src/onboardCompany.ts
+++ b/server/src/onboardCompany.ts
@@ -379,7 +379,8 @@ export async function joinAllHands(args: {
// update — broadcast misses are tolerable (the 60s refresher backfills),
// a DB hiccup here should not roll back the actual membership change.
try {
- const { CH_STATUS, publish } = await import('./redis.js')
+ const { CH_STATUS } = await import('./redis.js')
+ const { enqueueBroadcast, nudgeRealtimeOutbox } = await import('./realtime-outbox.js')
const { rows: pRow } = await pool.query<{
id: string; kind: string; name: string; role: string | null;
initial: string; avatar_bg: string; avatar_url: string | null;
@@ -393,7 +394,7 @@ export async function joinAllHands(args: {
)
const p = pRow[0]
if (p && (p.kind === 'human' || p.kind === 'agent')) {
- await publish(CH_STATUS, {
+ await enqueueBroadcast(pool, CH_STATUS, {
type: 'participants.added',
companyId,
conversationId: convId,
@@ -405,6 +406,7 @@ export async function joinAllHands(args: {
statusUpdatedAt: p.status_updated_at,
},
})
+ nudgeRealtimeOutbox()
}
} catch (e) {
console.warn('[onboard] participants.added broadcast failed', e instanceof Error ? e.message : e)
diff --git a/server/src/realtime-outbox.ts b/server/src/realtime-outbox.ts
index e952d17..9276a8d 100644
--- a/server/src/realtime-outbox.ts
+++ b/server/src/realtime-outbox.ts
@@ -116,6 +116,7 @@ async function claimBatch(limit: number): Promise<PendingBroadcast[]> {
channel: string
payload: BroadcastEvent
attempts: number
+ created_at: Date | string
}>(
`WITH candidates AS (
SELECT id
@@ -135,10 +136,23 @@ async function claimBatch(limit: number): Promise<PendingBroadcast[]> {
locked_until = NOW() + ($5 * INTERVAL '1 millisecond')
FROM candidates c
WHERE o.id = c.id
- RETURNING o.id, o.channel, o.payload, o.attempts`,
+ RETURNING o.id, o.channel, o.payload, o.attempts, o.created_at`,
[limit, workerId, MAX_ATTEMPTS, MAX_AGE_HOURS, CLAIM_LEASE_MS],
)
return rows
+ .map((row) => ({
+ id: row.id,
+ channel: row.channel,
+ payload: row.payload,
+ attempts: row.attempts,
+ createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at),
+ }))
+ .sort(compareOutboxRows)
+}
+
+export function compareOutboxRows(a: { createdAt: string; id: string }, b: { createdAt: string; id: string }): number {
+ const byTime = a.createdAt < b.createdAt ? -1 : a.createdAt > b.createdAt ? 1 : 0
+ return byTime || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)
}
async function discardExpired(): Promise<number> {
@@ -218,9 +232,9 @@ export async function drainRealtimeOutbox(options: {
let published = 0
let failed = 0
- // Bound simultaneous Redis calls so a large recovered queue cannot create a
- // connection spike. Promise.all is safe at this batch size (32).
- await Promise.all(rows.map(async (row) => {
+ // In claim order. Promise.all would let a later event reach Redis before
+ // an earlier one from the same batch, and the scheduler wakes from this bus.
+ for (const row of rows) {
try {
await publishFn(row.channel, row.payload)
await markPublished(row.id)
@@ -229,7 +243,7 @@ export async function drainRealtimeOutbox(options: {
await markFailed(row, error)
failed += 1
}
- }))
+ }
return { claimed: rows.length, published, failed, discarded }
}
diff --git a/server/src/status.ts b/server/src/status.ts
index 31c85b7..5c5f09e 100644
--- a/server/src/status.ts
+++ b/server/src/status.ts
@@ -1,5 +1,6 @@
import { pool } from './db/pool.js'
-import { CH_STATUS, publish } from './redis.js'
+import { CH_STATUS } from './redis.js'
+import { enqueueBroadcast, nudgeRealtimeOutbox } from './realtime-outbox.js'
export const BUSY_STATUS_LEASE_MS = 90_000
export const BUSY_STATUS_HEARTBEAT_MS = 20_000
@@ -32,50 +33,69 @@ export async function setStatus(participantId: string, status: ParticipantStatus
// second (each fan-out re-resolving the tenant's WS recipients on every
// replica) to announce a status nobody changed. Busy statuses still always
// write: a repeat is a lease renewal.
- const { rows } = await pool.query<{ company_id: string; status_updated_at: Date }>(
- status === 'avail'
- ? `UPDATE participants
- SET status = $2,
- status_updated_at = NOW()
- WHERE id = $1 AND status IS DISTINCT FROM $2
- RETURNING company_id, status_updated_at`
- : `UPDATE participants
- SET status = $2,
- status_updated_at = NOW()
- WHERE id = $1
- RETURNING company_id, status_updated_at`,
- [participantId, status],
- )
- for (const r of rows) {
- await publish(CH_STATUS, {
- type: 'participants.status',
- participantId,
- status,
- statusUpdatedAt: toIso(r.status_updated_at),
- companyId: r.company_id,
- }).catch((error) => {
- console.warn(`[status] durable ${participantId}=${status} update committed but publish failed`, error)
- })
+ const client = await pool.connect()
+ try {
+ await client.query('BEGIN')
+ const { rows } = await client.query<{ company_id: string; status_updated_at: Date }>(
+ status === 'avail'
+ ? `UPDATE participants
+ SET status = $2,
+ status_updated_at = NOW()
+ WHERE id = $1 AND status IS DISTINCT FROM $2
+ RETURNING company_id, status_updated_at`
+ : `UPDATE participants
+ SET status = $2,
+ status_updated_at = NOW()
+ WHERE id = $1
+ RETURNING company_id, status_updated_at`,
+ [participantId, status],
+ )
+ for (const r of rows) {
+ await enqueueBroadcast(client, CH_STATUS, {
+ type: 'participants.status',
+ participantId,
+ status,
+ statusUpdatedAt: toIso(r.status_updated_at),
+ companyId: r.company_id,
+ })
+ }
+ await client.query('COMMIT')
+ if (rows.length > 0) nudgeRealtimeOutbox()
+ } catch (error) {
+ await client.query('ROLLBACK').catch(() => {})
+ throw error
+ } finally {
+ client.release()
}
}
/** Renew a busy status lease without changing the semantic status. */
export async function heartbeatStatus(participantId: string, status: Extract<ParticipantStatus, 'working' | 'thinking' | 'waiting'>): Promise<void> {
- const { rows } = await pool.query<{ company_id: string; status_updated_at: Date }>(
- `UPDATE participants
- SET status_updated_at = NOW()
- WHERE id = $1 AND status = $2
- RETURNING company_id, status_updated_at`,
- [participantId, status],
- )
- if (!rows[0]) return
- await publish(CH_STATUS, {
- type: 'participants.status',
- participantId,
- status,
- statusUpdatedAt: toIso(rows[0].status_updated_at),
- companyId: rows[0].company_id,
- }).catch((error) => {
- console.warn(`[status] durable ${participantId} heartbeat committed but publish failed`, error)
- })
+ const client = await pool.connect()
+ try {
+ await client.query('BEGIN')
+ const { rows } = await client.query<{ company_id: string; status_updated_at: Date }>(
+ `UPDATE participants
+ SET status_updated_at = NOW()
+ WHERE id = $1 AND status = $2
+ RETURNING company_id, status_updated_at`,
+ [participantId, status],
+ )
+ if (rows[0]) {
+ await enqueueBroadcast(client, CH_STATUS, {
+ type: 'participants.status',
+ participantId,
+ status,
+ statusUpdatedAt: toIso(rows[0].status_updated_at),
+ companyId: rows[0].company_id,
+ })
+ }
+ await client.query('COMMIT')
+ if (rows[0]) nudgeRealtimeOutbox()
+ } catch (error) {
+ await client.query('ROLLBACK').catch(() => {})
+ throw error
+ } finally {
+ client.release()
+ }
}
diff --git a/server/src/ws.ts b/server/src/ws.ts
index 1e77d83..c0a2375 100644
--- a/server/src/ws.ts
+++ b/server/src/ws.ts
@@ -7,10 +7,8 @@ import {
CH_GROUP_PULLED, CH_CONVO_UPDATED, CH_CONVENE,
CH_BOARDS, CH_DOCS, CH_CALENDAR_REMINDER, CH_CALENDAR_EVENTS, CH_DOC_MENTION,
CH_WORKSPACES,
- publish,
type DocMentionEvent,
} from './redis.js'
-import type { MessageNewEvent } from './redis.js'
import { env } from './env.js'
import { consumeWsTicket } from './auth.js'
import { pool } from './db/pool.js'
@@ -124,34 +122,19 @@ export async function resetHumanPresenceOnBoot(): Promise<void> {
if (rows.length > 0) {
console.log(`[ws] demoted ${rows.length} stale 'avail' human(s) to 'resting' on boot`)
}
- // Broadcast each transition so any already-connected clients in a
- // multi-instance setup see the reset. We fire publishes in
- // parallel and bound the whole batch with a hard timeout: the
- // old code was a sequential `for await publish` that hung
- // server.listen() at first deploy when the table was full of
- // pre-feature stale rows. Failures are swallowed (Promise.race
- // against a timeout) — at boot there are typically zero connected
- // clients anyway, so the publish is best-effort.
- if (rows.length === 0) return
- const PUBLISH_BATCH_TIMEOUT_MS = 10_000
- const broadcastAll = Promise.allSettled(rows.map((r) =>
- publish(CH_STATUS, {
+ // The status rows are already committed by the statement above. Record
+ // the broadcasts in the outbox so a Redis blip retries them, and do not
+ // wait for Redis here: this function runs before listen().
+ for (const r of rows) {
+ await enqueueBroadcast(pool, CH_STATUS, {
type: 'participants.status',
participantId: r.id,
status: 'resting',
statusUpdatedAt: r.status_updated_at.toISOString(),
companyId: r.company_id,
- }),
- ))
- await Promise.race([
- broadcastAll,
- new Promise<void>((resolve) =>
- setTimeout(() => {
- console.warn(`[ws] resetHumanPresenceOnBoot publishes still pending after ${PUBLISH_BATCH_TIMEOUT_MS}ms — continuing without them`)
- resolve()
- }, PUBLISH_BATCH_TIMEOUT_MS),
- ),
- ])
+ })
+ }
+ if (rows.length > 0) nudgeRealtimeOutbox()
} catch (e) {
console.warn('[ws] resetHumanPresenceOnBoot failed', e)
}
@@ -1177,6 +1160,22 @@ async function postDocMentionWake(args: {
[messageId, conversationId, mentionerId, body, sequence, companyId],
)
await client.query(`UPDATE conversations SET updated_at = NOW() WHERE id = $1`, [conversationId])
+ // Same transaction as the message row. A Redis blip used to leave this
+ // wake committed and unpublished, and nothing scanned for it later.
+ await enqueueBroadcast(client, CH_MESSAGE_NEW, {
+ type: 'message.new',
+ companyId,
+ conversationId,
+ message: {
+ id: messageId,
+ conversationId,
+ authorId: mentionerId,
+ kind: 'text',
+ body,
+ sequence,
+ at: new Date().toISOString(),
+ },
+ })
await client.query('COMMIT')
} catch (error) {
await client.query('ROLLBACK').catch(() => {})
@@ -1184,27 +1183,7 @@ async function postDocMentionWake(args: {
} finally {
client.release()
}
-
- // Publish on the same bus chat messages use → scheduler wakes the
- // agent's pod, the turn loop drains the inbox, the agent sees this
- // message with the doc id verbatim.
- const event: MessageNewEvent = {
- type: 'message.new',
- companyId,
- conversationId,
- message: {
- id: messageId,
- conversationId,
- authorId: mentionerId,
- kind: 'text',
- body,
- sequence,
- at: new Date().toISOString(),
- },
- }
- await publish(CH_MESSAGE_NEW, event).catch((error) => {
- console.warn(`[doc.mention] durable wake ${messageId} committed but publish failed`, error)
- })
+ nudgeRealtimeOutbox()
}
export function attachWebSocket(httpServer: Server) {