Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.
TypeScript
fix(realtime): durable wakes go through the outbox (#321)
OKNOTOKa90c053fbea5
diff --git a/server/src/__tests__/realtime-outbox-order.test.ts b/server/src/__tests__/realtime-outbox-order.test.tsnew file mode 100644index 0000000..432a01f--- /dev/null+++ b/server/src/__tests__/realtime-outbox-order.test.ts@@ -0,0 +1,30 @@+import assert from 'node:assert/strict'+import { test } from 'node:test'+import { readFile } from 'node:fs/promises'++process.env.CUMORA_RUNTIME_CLIENT = 'http'+process.env.OPENAI_API_KEY ??= 'test-key'++const { compareOutboxRows } = await import('../realtime-outbox.js')++test('outbox delivery keeps created_at order inside a claimed batch', () => {+ const rows = [+ { id: 'b', createdAt: '2026-10-03T00:00:02.000Z' },+ { id: 'a', createdAt: '2026-10-03T00:00:01.000Z' },+ { id: 'c', createdAt: '2026-10-03T00:00:02.000Z' },+ ]+ assert.deepEqual([...rows].sort(compareOutboxRows).map((row) => row.id), ['a', 'b', 'c'])+})++test('durable message wakes are enqueued in the same transaction as the insert', async () => {+ const mention = await readFile(new URL('../ws.ts', import.meta.url), 'utf8')+ const wake = mention.slice(mention.indexOf('async function postDocMentionWake'))+ const wakeCommit = wake.indexOf("await client.query('COMMIT')")+ assert.ok(wake.indexOf('enqueueBroadcast(client, CH_MESSAGE_NEW') < wakeCommit)++ const pull = await readFile(new URL('../agents/scanner_helper.ts', import.meta.url), 'utf8')+ const pullCommit = pull.indexOf("await client.query('COMMIT')")+ assert.ok(pull.indexOf('enqueueBroadcast(client, CH_MESSAGE_NEW') < pullCommit)+ assert.ok(pull.indexOf('enqueueBroadcast(client, CH_GROUP_PULLED') < pullCommit)+ assert.doesNotMatch(pull, /publish\(CH_MESSAGE_NEW/)+})diff --git a/server/src/agents/computer/registry.ts b/server/src/agents/computer/registry.tsindex e0492b5..cb535f4 100644--- a/server/src/agents/computer/registry.ts+++ b/server/src/agents/computer/registry.ts@@ -19,7 +19,8 @@ import { InvalidModelIdError, parseModelId } from './model-id.js'import { createHash, randomBytes, randomUUID } from 'node:crypto'import { pool } from '../../db/pool.js'import { audit } from '../../auth.js'-import { CH_STATUS, publish } from '../../redis.js'+import { CH_STATUS } from '../../redis.js'+import { enqueueBroadcast, nudgeRealtimeOutbox } from '../../realtime-outbox.js'import { normalizeTier, type Tier } from '../../tier.js'import { signAgentToken } from '../runtime/jwt.js'import type { EngineModelCatalog, EngineModelOption, FastModelScope, ModelCatalogSource } from './model-catalog.js'@@ -38,12 +39,15 @@ export const COMPUTER_STALE_MS = 90_000async function broadcastComputerStatus(computerId: string, companyId: string, status: ComputerStatus,): Promise<void> {- await publish(CH_STATUS, { type: 'computers.status', computerId, status, companyId }).catch((error) => {+ try {+ await enqueueBroadcast(pool, CH_STATUS, { type: 'computers.status', computerId, status, companyId })+ nudgeRealtimeOutbox()+ } catch (error) {console.warn(- `[computer] durable ${computerId}=${status} update committed but publish failed`,+ `[computer] durable ${computerId}=${status} update committed but outbox insert failed`,error,)- })+ }}/** Announce a just-paired computer as online. Split out from {@link pairComputer}diff --git a/server/src/agents/runtime/probe-wake.ts b/server/src/agents/runtime/probe-wake.tsindex 06d5ea1..da5c720 100644--- a/server/src/agents/runtime/probe-wake.ts+++ b/server/src/agents/runtime/probe-wake.ts@@ -9,6 +9,8 @@* The conversation must already exist and have agent members.*/import { redis, CH_MESSAGE_NEW } from '../../redis.js'+import { pool } from '../../db/pool.js'+import { enqueueBroadcast, drainRealtimeOutbox } from '../../realtime-outbox.js'async function main(): Promise<void> {const conversationId = process.argv[2]@@ -17,13 +19,23 @@ async function main(): Promise<void> {console.error('usage: tsx probe-wake.ts <conversationId> <authorId>')process.exit(2)}- await redis.publish(CH_MESSAGE_NEW, JSON.stringify({+ await enqueueBroadcast(pool, CH_MESSAGE_NEW, {type: 'message.new',conversationId,- message: { authorId },- }))- console.log(`[probe-wake] published msg.new for convo=${conversationId} author=${authorId}`)+ message: {+ id: `probe-${Date.now()}`,+ conversationId,+ authorId,+ kind: 'text',+ body: '',+ sequence: 0,+ at: new Date().toISOString(),+ },+ })+ await drainRealtimeOutbox()+ console.log(`[probe-wake] queued msg.new for convo=${conversationId} author=${authorId}`)redis.disconnect()+ await pool.end()}void main()diff --git a/server/src/agents/scanner_helper.ts b/server/src/agents/scanner_helper.tsindex 25a42ad..950eecd 100644--- a/server/src/agents/scanner_helper.ts+++ b/server/src/agents/scanner_helper.ts@@ -2,7 +2,8 @@import { randomUUID } from 'node:crypto'import { pool } from '../db/pool.js'import { dispatchMessagePush } from '../push.js'-import { CH_GROUP_PULLED, CH_MESSAGE_NEW, publish } from '../redis.js'+import { CH_GROUP_PULLED, CH_MESSAGE_NEW } from '../redis.js'+import { enqueueBroadcast, nudgeRealtimeOutbox } from '../realtime-outbox.js'/** Hours an agent must wait between human-interrupting group pulls. */const PULL_COOLDOWN_HOURS = 6@@ -119,6 +120,18 @@ export async function startPulledGroup(args: {VALUES ($1, $2, $3, 'text', $4, 1, $5)`,[messageId, conversationId, instigatorId, opening, companyId],)+ await enqueueBroadcast(client, CH_GROUP_PULLED, {+ type: 'group.pulled', conversationId, companyId, pulledById: instigatorId,+ })+ await enqueueBroadcast(client, CH_MESSAGE_NEW, {+ type: 'message.new',+ conversationId,+ companyId,+ message: {+ id: messageId, conversationId, authorId: instigatorId,+ kind: 'text', body: opening, sequence: 1, at: pulledAt,+ },+ })await client.query('COMMIT')} catch (error) {await client.query('ROLLBACK').catch(() => {})@@ -126,23 +139,7 @@ export async function startPulledGroup(args: {} finally {client.release()}-- await publish(CH_GROUP_PULLED, {- type: 'group.pulled', conversationId, companyId, pulledById: instigatorId,- }).catch((error) => {- console.warn(`[pull_group] durable conversation ${conversationId} committed but group publish failed`, error)- })- await publish(CH_MESSAGE_NEW, {- type: 'message.new',- conversationId,- companyId,- message: {- id: messageId, conversationId, authorId: instigatorId,- kind: 'text', body: opening, sequence: 1, at: pulledAt,- },- }).catch((error) => {- console.warn(`[pull_group] durable message ${messageId} committed but publish failed`, error)- })+ nudgeRealtimeOutbox()// The point of a pull is to interrupt someone — there is a six-hour cooldown// on it for exactly that reason. Yet the interruption reached the websocketdiff --git a/server/src/onboardCompany.ts b/server/src/onboardCompany.tsindex b65f9be..7331e1b 100644--- a/server/src/onboardCompany.ts+++ b/server/src/onboardCompany.ts@@ -379,7 +379,8 @@ export async function joinAllHands(args: {// update — broadcast misses are tolerable (the 60s refresher backfills),// a DB hiccup here should not roll back the actual membership change.try {- const { CH_STATUS, publish } = await import('./redis.js')+ const { CH_STATUS } = await import('./redis.js')+ const { enqueueBroadcast, nudgeRealtimeOutbox } = await import('./realtime-outbox.js')const { rows: pRow } = await pool.query<{id: string; kind: string; name: string; role: string | null;initial: string; avatar_bg: string; avatar_url: string | null;@@ -393,7 +394,7 @@ export async function joinAllHands(args: {)const p = pRow[0]if (p && (p.kind === 'human' || p.kind === 'agent')) {- await publish(CH_STATUS, {+ await enqueueBroadcast(pool, CH_STATUS, {type: 'participants.added',companyId,conversationId: convId,@@ -405,6 +406,7 @@ export async function joinAllHands(args: {statusUpdatedAt: p.status_updated_at,},})+ nudgeRealtimeOutbox()}} catch (e) {console.warn('[onboard] participants.added broadcast failed', e instanceof Error ? e.message : e)diff --git a/server/src/realtime-outbox.ts b/server/src/realtime-outbox.tsindex e952d17..9276a8d 100644--- a/server/src/realtime-outbox.ts+++ b/server/src/realtime-outbox.ts@@ -116,6 +116,7 @@ async function claimBatch(limit: number): Promise<PendingBroadcast[]> {channel: stringpayload: BroadcastEventattempts: number+ created_at: Date | string}>(`WITH candidates AS (SELECT id@@ -135,10 +136,23 @@ async function claimBatch(limit: number): Promise<PendingBroadcast[]> {locked_until = NOW() + ($5 * INTERVAL '1 millisecond')FROM candidates cWHERE o.id = c.id- RETURNING o.id, o.channel, o.payload, o.attempts`,+ RETURNING o.id, o.channel, o.payload, o.attempts, o.created_at`,[limit, workerId, MAX_ATTEMPTS, MAX_AGE_HOURS, CLAIM_LEASE_MS],)return rows+ .map((row) => ({+ id: row.id,+ channel: row.channel,+ payload: row.payload,+ attempts: row.attempts,+ createdAt: row.created_at instanceof Date ? row.created_at.toISOString() : String(row.created_at),+ }))+ .sort(compareOutboxRows)+}++export function compareOutboxRows(a: { createdAt: string; id: string }, b: { createdAt: string; id: string }): number {+ const byTime = a.createdAt < b.createdAt ? -1 : a.createdAt > b.createdAt ? 1 : 0+ return byTime || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)}async function discardExpired(): Promise<number> {@@ -218,9 +232,9 @@ export async function drainRealtimeOutbox(options: {let published = 0let failed = 0- // Bound simultaneous Redis calls so a large recovered queue cannot create a- // connection spike. Promise.all is safe at this batch size (32).- await Promise.all(rows.map(async (row) => {+ // In claim order. Promise.all would let a later event reach Redis before+ // an earlier one from the same batch, and the scheduler wakes from this bus.+ for (const row of rows) {try {await publishFn(row.channel, row.payload)await markPublished(row.id)@@ -229,7 +243,7 @@ export async function drainRealtimeOutbox(options: {await markFailed(row, error)failed += 1}- }))+ }return { claimed: rows.length, published, failed, discarded }}diff --git a/server/src/status.ts b/server/src/status.tsindex 31c85b7..5c5f09e 100644--- a/server/src/status.ts+++ b/server/src/status.ts@@ -1,5 +1,6 @@import { pool } from './db/pool.js'-import { CH_STATUS, publish } from './redis.js'+import { CH_STATUS } from './redis.js'+import { enqueueBroadcast, nudgeRealtimeOutbox } from './realtime-outbox.js'export const BUSY_STATUS_LEASE_MS = 90_000export const BUSY_STATUS_HEARTBEAT_MS = 20_000@@ -32,50 +33,69 @@ export async function setStatus(participantId: string, status: ParticipantStatus// second (each fan-out re-resolving the tenant's WS recipients on every// replica) to announce a status nobody changed. Busy statuses still always// write: a repeat is a lease renewal.- const { rows } = await pool.query<{ company_id: string; status_updated_at: Date }>(- status === 'avail'- ? `UPDATE participants- SET status = $2,- status_updated_at = NOW()- WHERE id = $1 AND status IS DISTINCT FROM $2- RETURNING company_id, status_updated_at`- : `UPDATE participants- SET status = $2,- status_updated_at = NOW()- WHERE id = $1- RETURNING company_id, status_updated_at`,- [participantId, status],- )- for (const r of rows) {- await publish(CH_STATUS, {- type: 'participants.status',- participantId,- status,- statusUpdatedAt: toIso(r.status_updated_at),- companyId: r.company_id,- }).catch((error) => {- console.warn(`[status] durable ${participantId}=${status} update committed but publish failed`, error)- })+ const client = await pool.connect()+ try {+ await client.query('BEGIN')+ const { rows } = await client.query<{ company_id: string; status_updated_at: Date }>(+ status === 'avail'+ ? `UPDATE participants+ SET status = $2,+ status_updated_at = NOW()+ WHERE id = $1 AND status IS DISTINCT FROM $2+ RETURNING company_id, status_updated_at`+ : `UPDATE participants+ SET status = $2,+ status_updated_at = NOW()+ WHERE id = $1+ RETURNING company_id, status_updated_at`,+ [participantId, status],+ )+ for (const r of rows) {+ await enqueueBroadcast(client, CH_STATUS, {+ type: 'participants.status',+ participantId,+ status,+ statusUpdatedAt: toIso(r.status_updated_at),+ companyId: r.company_id,+ })+ }+ await client.query('COMMIT')+ if (rows.length > 0) nudgeRealtimeOutbox()+ } catch (error) {+ await client.query('ROLLBACK').catch(() => {})+ throw error+ } finally {+ client.release()}}/** Renew a busy status lease without changing the semantic status. */export async function heartbeatStatus(participantId: string, status: Extract<ParticipantStatus, 'working' | 'thinking' | 'waiting'>): Promise<void> {- const { rows } = await pool.query<{ company_id: string; status_updated_at: Date }>(- `UPDATE participants- SET status_updated_at = NOW()- WHERE id = $1 AND status = $2- RETURNING company_id, status_updated_at`,- [participantId, status],- )- if (!rows[0]) return- await publish(CH_STATUS, {- type: 'participants.status',- participantId,- status,- statusUpdatedAt: toIso(rows[0].status_updated_at),- companyId: rows[0].company_id,- }).catch((error) => {- console.warn(`[status] durable ${participantId} heartbeat committed but publish failed`, error)- })+ const client = await pool.connect()+ try {+ await client.query('BEGIN')+ const { rows } = await client.query<{ company_id: string; status_updated_at: Date }>(+ `UPDATE participants+ SET status_updated_at = NOW()+ WHERE id = $1 AND status = $2+ RETURNING company_id, status_updated_at`,+ [participantId, status],+ )+ if (rows[0]) {+ await enqueueBroadcast(client, CH_STATUS, {+ type: 'participants.status',+ participantId,+ status,+ statusUpdatedAt: toIso(rows[0].status_updated_at),+ companyId: rows[0].company_id,+ })+ }+ await client.query('COMMIT')+ if (rows[0]) nudgeRealtimeOutbox()+ } catch (error) {+ await client.query('ROLLBACK').catch(() => {})+ throw error+ } finally {+ client.release()+ }}diff --git a/server/src/ws.ts b/server/src/ws.tsindex 1e77d83..c0a2375 100644--- a/server/src/ws.ts+++ b/server/src/ws.ts@@ -7,10 +7,8 @@ import {CH_GROUP_PULLED, CH_CONVO_UPDATED, CH_CONVENE,CH_BOARDS, CH_DOCS, CH_CALENDAR_REMINDER, CH_CALENDAR_EVENTS, CH_DOC_MENTION,CH_WORKSPACES,- publish,type DocMentionEvent,} from './redis.js'-import type { MessageNewEvent } from './redis.js'import { env } from './env.js'import { consumeWsTicket } from './auth.js'import { pool } from './db/pool.js'@@ -124,34 +122,19 @@ export async function resetHumanPresenceOnBoot(): Promise<void> {if (rows.length > 0) {console.log(`[ws] demoted ${rows.length} stale 'avail' human(s) to 'resting' on boot`)}- // Broadcast each transition so any already-connected clients in a- // multi-instance setup see the reset. We fire publishes in- // parallel and bound the whole batch with a hard timeout: the- // old code was a sequential `for await publish` that hung- // server.listen() at first deploy when the table was full of- // pre-feature stale rows. Failures are swallowed (Promise.race- // against a timeout) — at boot there are typically zero connected- // clients anyway, so the publish is best-effort.- if (rows.length === 0) return- const PUBLISH_BATCH_TIMEOUT_MS = 10_000- const broadcastAll = Promise.allSettled(rows.map((r) =>- publish(CH_STATUS, {+ // The status rows are already committed by the statement above. Record+ // the broadcasts in the outbox so a Redis blip retries them, and do not+ // wait for Redis here: this function runs before listen().+ for (const r of rows) {+ await enqueueBroadcast(pool, CH_STATUS, {type: 'participants.status',participantId: r.id,status: 'resting',statusUpdatedAt: r.status_updated_at.toISOString(),companyId: r.company_id,- }),- ))- await Promise.race([- broadcastAll,- new Promise<void>((resolve) =>- setTimeout(() => {- console.warn(`[ws] resetHumanPresenceOnBoot publishes still pending after ${PUBLISH_BATCH_TIMEOUT_MS}ms — continuing without them`)- resolve()- }, PUBLISH_BATCH_TIMEOUT_MS),- ),- ])+ })+ }+ if (rows.length > 0) nudgeRealtimeOutbox()} catch (e) {console.warn('[ws] resetHumanPresenceOnBoot failed', e)}@@ -1177,6 +1160,22 @@ async function postDocMentionWake(args: {[messageId, conversationId, mentionerId, body, sequence, companyId],)await client.query(`UPDATE conversations SET updated_at = NOW() WHERE id = $1`, [conversationId])+ // Same transaction as the message row. A Redis blip used to leave this+ // wake committed and unpublished, and nothing scanned for it later.+ await enqueueBroadcast(client, CH_MESSAGE_NEW, {+ type: 'message.new',+ companyId,+ conversationId,+ message: {+ id: messageId,+ conversationId,+ authorId: mentionerId,+ kind: 'text',+ body,+ sequence,+ at: new Date().toISOString(),+ },+ })await client.query('COMMIT')} catch (error) {await client.query('ROLLBACK').catch(() => {})@@ -1184,27 +1183,7 @@ async function postDocMentionWake(args: {} finally {client.release()}-- // Publish on the same bus chat messages use → scheduler wakes the- // agent's pod, the turn loop drains the inbox, the agent sees this- // message with the doc id verbatim.- const event: MessageNewEvent = {- type: 'message.new',- companyId,- conversationId,- message: {- id: messageId,- conversationId,- authorId: mentionerId,- kind: 'text',- body,- sequence,- at: new Date().toISOString(),- },- }- await publish(CH_MESSAGE_NEW, event).catch((error) => {- console.warn(`[doc.mention] durable wake ${messageId} committed but publish failed`, error)- })+ nudgeRealtimeOutbox()}export function attachWebSocket(httpServer: Server) {