Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.
TypeScript
import { WebSocketServer, type WebSocket } from 'ws'
import type { Server } from 'node:http'
import {
sub,
CH_MESSAGE_NEW, CH_MESSAGE_DELTA, CH_TYPING,
CH_STATUS, CH_REACTIONS, CH_POLLS,
CH_GROUP_PULLED, CH_CONVO_UPDATED, CH_CONVENE,
CH_BOARDS, CH_DOCS, CH_CALENDAR_REMINDER, CH_CALENDAR_EVENTS, CH_DOC_MENTION,
CH_WORKSPACES,
type DocMentionEvent,
} from './redis.js'
import { env } from './env.js'
import { consumeWsTicket } from './auth.js'
import { pool } from './db/pool.js'
import { enqueueBroadcast, nudgeRealtimeOutbox } from './realtime-outbox.js'
import { setStatus } from './status.js'
import {
subscribe as docSubscribe,
unsubscribe as docUnsubscribe,
applyLocalUpdate as docApplyLocalUpdate,
broadcastAwareness as docBroadcastAwareness,
type DocSubscriber,
} from './documents/rooms.js'
import { Semaphore } from './concurrency.js'
import { randomUUID } from 'node:crypto'
interface AuthedSocket {
ws: WebSocket
userId: string
/** Stable per-socket id used as the Yjs update origin. Lets the room
* manager echo-suppress on this client's own outbound updates. */
originId: string
/** Company ids loaded at connect. Used only to log membership count.
* Fan-out does not consult this set: delivery is by user id, and one
* socket receives frames for every workspace that user belongs to. */
companies: Set<string>
/** Active doc subscriptions on this socket. Released on close. */
docSubs: Map<string, DocSubscriber>
/** Document protocol frames are handled FIFO per socket. This keeps an
* async subscribe/hydration from racing a later unsubscribe or update. */
docFrameQueue: Promise<void>
pendingDocFrames: number
pendingDocBytes: number
closed: boolean
/** Heartbeat liveness flag. Set true on every received pong; the periodic
* ping loop flips it to false right before sending the next ping. If the
* next round still sees false, the socket is half-open and we terminate
* it — that triggers the 'close' handler, which decrements the
* presence counter and flips the user to 'resting'. Without this,
* laptop sleeps / network drops would leave 'avail' stuck until the
* OS finally noticed the dead TCP. */
isAlive: boolean
}
const clients = new Set<AuthedSocket>()
const redisFanoutQueues = new Map<string, Promise<void>>()
// Per-client WebSocket send backpressure caps (OOM fix). A socket that can't
// drain makes `ws` buffer unsent frames in process memory; without a cap, a high
// broadcast rate grows that buffer unbounded across clients until the pod OOMs.
// Above MAX we stop sending new frames to that client (let it drain); above
// TERMINATE it's hopelessly behind, so we kill it to reclaim the memory (it
// reconnects + re-syncs via REST).
const WS_MAX_BUFFERED_BYTES = 2 * 1024 * 1024 // 2 MB
const WS_TERMINATE_BUFFERED_BYTES = 8 * 1024 * 1024 // 8 MB
const DOC_SYNC_MAX_BYTES = 32 * 1024 * 1024 // bounded one-shot snapshot
const DOC_AUTH_CONCURRENCY = 4
const DOC_AUTH_MAX_PENDING_FRAMES = 512
const DOC_AUTH_MAX_PENDING_BYTES = 64 * 1024 * 1024
const DOC_AUTH_MAX_BATCH_FRAMES = 64
const DOC_AUTH_MAX_BATCH_BYTES = 512 * 1024
const DOC_AUTH_LOCK_RETRY_LIMIT = 2
// Keep legacy clients inside the same bounded window as pool.ts's 60s
// statement timeout. New clients use the short reconnect-safe retry window.
const DOC_AUTH_LEGACY_DEADLINE_MS = 60_000
const DOC_AUTH_RETRY_DELAY_MS = 50
/** Authorization is deliberately below the pg pool size. Admission is
* bounded separately because Semaphore itself has an unbounded waiter list. */
const docAuthSemaphore = new Semaphore(DOC_AUTH_CONCURRENCY)
const docReplaySupport = new WeakMap<DocSubscriber, boolean>()
let pendingDocAuthFrames = 0
let pendingDocAuthBytes = 0
function reserveDocAuth(bytes: number): boolean {
const bounded = Math.max(1, bytes)
if (
pendingDocAuthFrames >= DOC_AUTH_MAX_PENDING_FRAMES
|| pendingDocAuthBytes + bounded > DOC_AUTH_MAX_PENDING_BYTES
) return false
pendingDocAuthFrames++
pendingDocAuthBytes += bounded
return true
}
function releaseDocAuth(bytes: number): void {
pendingDocAuthFrames = Math.max(0, pendingDocAuthFrames - 1)
pendingDocAuthBytes = Math.max(0, pendingDocAuthBytes - Math.max(1, bytes))
}
/** Open WS connections per user. Drives real human presence:
* - 0 → 1 : user is now online, flip participant status to 'avail'.
* - 1 → 0 : user just went offline, flip to 'resting'.
* Multiple tabs / windows / devices coalesce on the same userId so
* presence stays "online" until the last one closes. */
const humanConnections = new Map<string, number>()
/** Called once at server boot. Demotes every human participant that's
* still flagged 'avail' from a previous run — those flags would otherwise
* persist as stale presence until that user reconnects/disconnects.
* Agents are handled by their own runtime lease + the GET /participants
* auto-expiry, so we leave their status alone here. */
export async function resetHumanPresenceOnBoot(): Promise<void> {
try {
const { rows } = await pool.query<{ id: string; company_id: string; status_updated_at: Date }>(
`UPDATE participants
SET status = 'resting',
status_updated_at = NOW()
WHERE kind = 'human' AND status = 'avail'
RETURNING id, company_id, status_updated_at`,
)
if (rows.length > 0) {
console.log(`[ws] demoted ${rows.length} stale 'avail' human(s) to 'resting' on boot`)
}
// The status rows are already committed by the statement above. Record
// the broadcasts in the outbox so a Redis blip retries them, and do not
// wait for Redis here: this function runs before listen().
for (const r of rows) {
await enqueueBroadcast(pool, CH_STATUS, {
type: 'participants.status',
participantId: r.id,
status: 'resting',
statusUpdatedAt: r.status_updated_at.toISOString(),
companyId: r.company_id,
})
}
if (rows.length > 0) nudgeRealtimeOutbox()
} catch (e) {
console.warn('[ws] resetHumanPresenceOnBoot failed', e)
}
}
async function onHumanConnect(userId: string): Promise<void> {
const cur = humanConnections.get(userId) ?? 0
humanConnections.set(userId, cur + 1)
if (cur === 0) {
try { await setStatus(userId, 'avail') }
catch (e) { console.warn(`[ws] setStatus(avail) failed for ${userId}`, e) }
}
}
async function onHumanDisconnect(userId: string): Promise<void> {
const cur = humanConnections.get(userId) ?? 0
if (cur <= 1) {
humanConnections.delete(userId)
try { await setStatus(userId, 'resting') }
catch (e) { console.warn(`[ws] setStatus(resting) failed for ${userId}`, e) }
} else {
humanConnections.set(userId, cur - 1)
}
}
async function loadMemberships(userId: string): Promise<Set<string>> {
const { rows } = await pool.query<{ company_id: string }>(
`SELECT company_id FROM company_members WHERE user_id = $1`,
[userId],
)
return new Set(rows.map((r) => r.company_id))
}
interface RoutedRedisEvent {
type?: string
companyId?: string
conversationId?: string
message?: { id?: string }
mentionedIds?: string[]
recipientUserIds?: string[]
}
/** Resolve every broadcast against live authorization, not the membership
* snapshot captured when a socket connected. Conversation events are routed
* only to current active human members. The sole exception is a persisted
* system message with delivery_recipient_id, which lets a just-removed human
* receive that one terminal leave/kick notice without reopening room access. */
export async function resolveWsEventRecipientUserIds(
event: RoutedRedisEvent,
): Promise<Set<string>> {
const companyId = typeof event.companyId === 'string' ? event.companyId : ''
if (!companyId) return new Set()
// Membership invalidations are terminal, user-targeted frames. Looking the
// recipients up through company_members would drop the exact event that tells
// a removed user to evict the workspace from their client. The publisher is a
// server-only mutation path and users are still checked for account liveness.
if (event.type === 'workspace.membership') {
const requested = [...new Set((event.recipientUserIds ?? []).filter((id) => typeof id === 'string'))]
if (requested.length === 0) return new Set()
const { rows } = await pool.query<{ id: string }>(
`SELECT id FROM users WHERE id = ANY($1::text[]) AND deleted_at IS NULL`,
[requested],
)
return new Set(rows.map((row) => row.id))
}
const targetedUserIds = event.type === 'doc.mention'
? event.mentionedIds
: event.type === 'calendar.reminder'
? event.recipientUserIds
: null
if (targetedUserIds) {
const requested = [...new Set(targetedUserIds.filter((id) => typeof id === 'string'))]
if (requested.length === 0) return new Set()
const { rows } = await pool.query<{ user_id: string }>(
`SELECT cm.user_id
FROM company_members cm
JOIN users u ON u.id = cm.user_id AND u.deleted_at IS NULL
JOIN participants p
ON p.id = cm.user_id
AND p.company_id = cm.company_id
AND p.kind = 'human'
AND p.departed_at IS NULL
WHERE cm.company_id = $1 AND cm.user_id = ANY($2::text[])`,
[companyId, requested],
)
return new Set(rows.map((row) => row.user_id))
}
const conversationId = typeof event.conversationId === 'string' ? event.conversationId : ''
if (!conversationId) {
const { rows } = await pool.query<{ user_id: string }>(
`SELECT cm.user_id
FROM company_members cm
JOIN users u ON u.id = cm.user_id AND u.deleted_at IS NULL
JOIN participants p
ON p.id = cm.user_id
AND p.company_id = cm.company_id
AND p.kind = 'human'
AND p.departed_at IS NULL
WHERE cm.company_id = $1`,
[companyId],
)
return new Set(rows.map((row) => row.user_id))
}
const durableMessageId = event.type === 'message.new' && typeof event.message?.id === 'string'
? event.message.id
: null
const { rows } = await pool.query<{ user_id: string }>(
`WITH scoped_conversation AS (
SELECT id, company_id
FROM conversations
WHERE id = $1 AND company_id = $2
), current_members AS (
SELECT company_member.user_id
FROM scoped_conversation c
JOIN conversation_members room_member
ON room_member.conversation_id = c.id
AND room_member.company_id = c.company_id
JOIN participants p
ON p.id = room_member.participant_id
AND p.company_id = c.company_id
AND p.kind = 'human'
AND p.departed_at IS NULL
JOIN company_members company_member
ON company_member.user_id = p.id
AND company_member.company_id = c.company_id
), durable_recipient AS (
SELECT cm.user_id
FROM scoped_conversation c
JOIN messages m
ON m.id = $3
AND m.conversation_id = c.id
AND m.company_id = c.company_id
AND m.kind = 'system'
AND m.delivery_recipient_id IS NOT NULL
JOIN participants p
ON p.id = m.delivery_recipient_id
AND p.company_id = c.company_id
AND p.kind = 'human'
AND p.departed_at IS NULL
JOIN company_members cm
ON cm.user_id = p.id
AND cm.company_id = c.company_id
)
SELECT user_id FROM current_members
UNION
SELECT user_id FROM durable_recipient`,
[conversationId, companyId, durableMessageId],
)
return new Set(rows.map((row) => row.user_id))
}
/** Look up a doc + verify the caller's tenant membership in one shot.
* Returns null when the doc doesn't exist OR the caller can't see it —
* same opaque posture the chat handlers use to avoid leaking existence. */
async function docCompanyFor(documentId: string, userId: string): Promise<string | null> {
return docAuthSemaphore.run(async () => {
const { rows } = await pool.query<{ company_id: string }>(
`SELECT d.company_id
FROM documents d
JOIN company_members m ON m.company_id = d.company_id AND m.user_id = $2
JOIN users u ON u.id = m.user_id AND u.deleted_at IS NULL
JOIN participants p
ON p.id = m.user_id
AND p.company_id = d.company_id
AND p.kind = 'human'
AND p.departed_at IS NULL
WHERE d.id = $1
LIMIT 1`,
[documentId, userId],
)
return rows[0]?.company_id ?? null
})
}
interface DocFanoutItem {
c: AuthedSocket
documentId: string
subRec: DocSubscriber
payload: string
bytes: number
replaySupported: boolean
released: boolean
onDone?: () => void
}
interface DocFanoutQueue {
items: DocFanoutItem[]
running: boolean
scheduled: boolean
}
const docFanoutQueues = new Map<string, DocFanoutQueue>()
function isLockNotAvailable(error: unknown): boolean {
const code = (error as { code?: unknown } | null)?.code
return code === '55P03' || code === '40P01'
}
function isTransientDocAuthError(error: unknown): boolean {
if (isLockNotAvailable(error)) return true
const message = error instanceof Error ? error.message : String(error)
return /timeout|ECONNREFUSED|ECONNRESET|connection|terminated|EOF/i.test(message)
}
function detachAndTerminate(item: DocFanoutItem): void {
const current = item.c.docSubs.get(item.documentId) === item.subRec
detachDocSubscription(item.c, item.documentId, item.subRec)
if (current && !item.c.closed) {
try { item.c.ws.terminate() } catch { /* ignore */ }
}
}
function itemIsCurrent(item: DocFanoutItem): boolean {
const { c, documentId, subRec } = item
return !c.closed
&& c.docSubs.get(documentId) === subRec
&& c.ws.readyState === c.ws.OPEN
}
function itemIsLive(item: DocFanoutItem): boolean {
return itemIsCurrent(item) && item.c.ws.bufferedAmount <= WS_MAX_BUFFERED_BYTES
}
function sendSerialized(ws: WebSocket, payload: string): boolean {
if (ws.readyState !== ws.OPEN) return false
try {
ws.send(payload)
return true
} catch {
return false
}
}
function finalizeDocFanoutItem(item: DocFanoutItem): void {
if (item.released) return
item.released = true
item.onDone?.()
releaseDocAuth(item.bytes)
}
/** Authorize one item after a contended batch has been split. The semaphore
* is acquired here only after the batch transaction has released its permit. */
type SingleDocAuthOutcome = 'sent' | 'revoked' | 'busy' | 'failed'
async function sendAuthorizedDocFrame(item: DocFanoutItem): Promise<SingleDocAuthOutcome> {
let sideEffectStarted = false
if (!itemIsCurrent(item)) return 'failed'
return docAuthSemaphore.run(async () => {
if (!itemIsCurrent(item)) return 'failed'
const client = await pool.connect()
try {
if (!itemIsCurrent(item)) return 'failed'
await client.query('BEGIN')
const authorized = await client.query(
`SELECT d.id
FROM documents d
JOIN company_members cm ON cm.company_id = d.company_id AND cm.user_id = $2
JOIN users u ON u.id = cm.user_id AND u.deleted_at IS NULL
JOIN participants p
ON p.id = cm.user_id
AND p.company_id = d.company_id
AND p.kind = 'human'
AND p.departed_at IS NULL
WHERE d.id = $1
FOR SHARE OF d, cm, u, p NOWAIT`,
[item.documentId, item.c.userId],
)
if (!authorized.rowCount) {
await client.query('ROLLBACK')
detachDocSubscription(item.c, item.documentId, item.subRec)
return 'revoked'
}
if (!itemIsLive(item)) {
await client.query('ROLLBACK')
detachAndTerminate(item)
return 'failed'
}
sideEffectStarted = true
if (!sendSerialized(item.c.ws, item.payload)) {
await client.query('ROLLBACK')
detachAndTerminate(item)
return 'failed'
}
await client.query('COMMIT')
return 'sent'
} catch (error) {
await client.query('ROLLBACK').catch(() => {})
if (!sideEffectStarted && isTransientDocAuthError(error)) return 'busy'
throw error
} finally {
client.release()
}
}).catch((error) => {
console.warn(`[ws] document authorization lookup failed for ${item.documentId}`, error)
if (sideEffectStarted) {
detachAndTerminate(item)
return 'failed' as const
}
if (isTransientDocAuthError(error)) return 'busy' as const
detachAndTerminate(item)
return 'failed' as const
})
}
interface DocBatchResult {
contended: boolean
fallback: DocFanoutItem[]
}
/** Authorize all distinct recipients for one document in one transaction.
* Invalid recipients are detached individually; they never discard valid
* peers in the same frame batch. NOWAIT keeps a writer from pinning every
* recipient behind one lock; the caller retries, then splits the batch. */
async function authorizeDocBatch(items: DocFanoutItem[]): Promise<DocBatchResult> {
const initial = items.filter((item) => itemIsCurrent(item))
if (initial.length === 0) return { contended: false, fallback: [] }
return docAuthSemaphore.run(async () => {
if (!initial.some((item) => itemIsCurrent(item))) return { contended: false, fallback: [] }
const client = await pool.connect()
let sideEffectStarted = false
let eligible: DocFanoutItem[] = []
try {
eligible = initial.filter((item) => itemIsCurrent(item))
if (eligible.length === 0) return { contended: false, fallback: [] }
await client.query('BEGIN')
const userIds = [...new Set(eligible.map((item) => item.c.userId))]
const { rows } = await client.query<{ user_id: string }>(
`SELECT cm.user_id
FROM documents d
JOIN company_members cm
ON cm.company_id = d.company_id
AND cm.user_id = ANY($2::text[])
JOIN users u ON u.id = cm.user_id AND u.deleted_at IS NULL
JOIN participants p
ON p.id = cm.user_id
AND p.company_id = d.company_id
AND p.kind = 'human'
AND p.departed_at IS NULL
WHERE d.id = $1
FOR SHARE OF d, cm, u, p NOWAIT`,
[eligible[0]?.documentId, userIds],
)
const authorized = new Set(rows.map((row) => row.user_id))
for (const item of eligible) {
if (!authorized.has(item.c.userId)) {
detachDocSubscription(item.c, item.documentId, item.subRec)
continue
}
if (!itemIsLive(item)) {
detachAndTerminate(item)
continue
}
// The lock is held until COMMIT, immediately after this synchronous
// send. A revocation cannot commit between auth and delivery.
sideEffectStarted = true
if (!sendSerialized(item.c.ws, item.payload)) detachAndTerminate(item)
}
await client.query('COMMIT')
return { contended: false, fallback: [] }
} catch (error) {
await client.query('ROLLBACK').catch(() => {})
if (!sideEffectStarted && isTransientDocAuthError(error)) return { contended: true, fallback: eligible }
for (const item of eligible) {
detachAndTerminate(item)
}
console.warn(`[ws] document authorization batch failed for ${items[0]?.documentId ?? 'unknown'}`, error)
return { contended: false, fallback: [] }
} finally {
client.release()
}
})
}
async function processDocBatch(items: DocFanoutItem[]): Promise<void> {
const active = items.filter((item) => {
if (!itemIsCurrent(item)) return false
if (item.c.ws.bufferedAmount > WS_MAX_BUFFERED_BYTES) {
detachAndTerminate(item)
return false
}
return true
})
try {
if (active.length === 0) return
let result: DocBatchResult = { contended: true, fallback: active }
for (let attempt = 0; attempt <= DOC_AUTH_LOCK_RETRY_LIMIT && result.contended; attempt++) {
result = await authorizeDocBatch(active)
if (result.contended && attempt < DOC_AUTH_LOCK_RETRY_LIMIT) {
await new Promise((resolve) => setTimeout(resolve, 5 * (attempt + 1)))
}
}
if (!result.contended) return
// The batch permit and client have been released before this split. Each
// item gets an independent NOWAIT authorization so a contended/revoked
// peer cannot discard healthy peers or deadlock through nested acquisition.
const groups = new Map<DocSubscriber, DocFanoutItem[]>()
for (const item of result.fallback) {
const group = groups.get(item.subRec) ?? []
group.push(item)
groups.set(item.subRec, group)
}
// Different subscribers can make progress in parallel (the semaphore
// still caps DB work), but each subscriber's frames remain FIFO.
await Promise.all([...groups.values()].map(async (group) => {
const item = group[0]!
const deadline = Date.now() + (item.replaySupported
? DOC_AUTH_RETRY_DELAY_MS * (DOC_AUTH_LOCK_RETRY_LIMIT + 1)
: DOC_AUTH_LEGACY_DEADLINE_MS)
for (const current of group) {
let outcome: SingleDocAuthOutcome = 'busy'
let attempt = 0
while (outcome === 'busy') {
outcome = await sendAuthorizedDocFrame(current)
if (outcome === 'busy') {
const remaining = deadline - Date.now()
if (remaining <= 0) break
const backoff = item.replaySupported
? 5 * (attempt + 1)
: Math.min(DOC_AUTH_RETRY_DELAY_MS * (attempt + 1), 1_000)
await new Promise((resolve) => setTimeout(resolve, Math.min(backoff, remaining)))
attempt++
}
}
if (outcome === 'busy') detachAndTerminate(current)
}
}))
} catch (error) {
console.warn(`[ws] document authorization batch failed for ${items[0]?.documentId ?? 'unknown'}`, error)
for (const item of items) detachAndTerminate(item)
} finally {
for (const item of items) finalizeDocFanoutItem(item)
}
}
function scheduleDocFanout(documentId: string): void {
const queue = docFanoutQueues.get(documentId)
if (!queue || queue.scheduled || queue.running) return
queue.scheduled = true
queueMicrotask(() => {
queue.scheduled = false
void drainDocFanout(documentId, queue)
})
}
async function drainDocFanout(documentId: string, queue: DocFanoutQueue): Promise<void> {
if (queue.running) return
queue.running = true
try {
while (queue.items.length > 0) {
const batch: DocFanoutItem[] = []
let bytes = 0
while (queue.items.length > 0 && batch.length < DOC_AUTH_MAX_BATCH_FRAMES) {
const next = queue.items[0]!
if (batch.length > 0 && bytes + next.bytes > DOC_AUTH_MAX_BATCH_BYTES) break
queue.items.shift()
batch.push(next)
bytes += next.bytes
}
await processDocBatch(batch)
}
} catch (error) {
console.warn(`[ws] document fan-out failed for ${documentId}`, error)
for (const item of queue.items.splice(0)) {
detachDocSubscription(item.c, item.documentId, item.subRec)
finalizeDocFanoutItem(item)
}
} finally {
queue.running = false
if (queue.items.length > 0) scheduleDocFanout(documentId)
else if (docFanoutQueues.get(documentId) === queue) docFanoutQueues.delete(documentId)
}
}
function removeQueuedDocItems(c: AuthedSocket): void {
for (const [documentId, queue] of docFanoutQueues) {
const retained: DocFanoutItem[] = []
for (const item of queue.items) {
if (item.c === c) finalizeDocFanoutItem(item)
else retained.push(item)
}
queue.items = retained
if (!queue.running && !queue.scheduled && queue.items.length === 0) {
docFanoutQueues.delete(documentId)
}
}
}
function enqueueAuthorizedDocFrame(
c: AuthedSocket,
documentId: string,
subRec: DocSubscriber,
payload: unknown,
byteLength: number,
replaySupported: boolean,
onDone?: () => void,
): void {
if (c.closed || c.docSubs.get(documentId) !== subRec) {
onDone?.()
return
}
const serialized = JSON.stringify(payload)
const bytes = Math.max(1, Buffer.byteLength(serialized))
if (!reserveDocAuth(bytes)) {
const rejected: DocFanoutItem = {
c, documentId, subRec, payload: serialized, bytes, replaySupported, released: true, onDone,
}
onDone?.()
detachAndTerminate(rejected)
return
}
// byteLength remains an explicit argument at call sites so a future caller
// cannot accidentally omit payload-size admission when its binary source is
// not yet serialized. The serialized size is the bound used for memory.
void byteLength
const item: DocFanoutItem = {
c, documentId, subRec, payload: serialized, bytes, replaySupported, released: false, onDone,
}
let queue = docFanoutQueues.get(documentId)
if (!queue) {
queue = { items: [], running: false, scheduled: false }
docFanoutQueues.set(documentId, queue)
}
queue.items.push(item)
scheduleDocFanout(documentId)
}
async function withAuthorizedDocOperation(
c: AuthedSocket,
documentId: string,
subRec: DocSubscriber,
task: (companyId: string, client: import('pg').PoolClient) => Promise<void>,
): Promise<boolean> {
const replaySupported = docReplaySupport.get(subRec) === true
const deadline = Date.now() + (replaySupported
? DOC_AUTH_RETRY_DELAY_MS * (DOC_AUTH_LOCK_RETRY_LIMIT + 1)
: DOC_AUTH_LEGACY_DEADLINE_MS)
let attempt = 0
while (true) {
if (c.closed || c.docSubs.get(documentId) !== subRec) return false
let taskStarted = false
try {
return await docAuthSemaphore.run(async () => {
if (c.closed || c.docSubs.get(documentId) !== subRec) return false
const client = await pool.connect()
try {
if (c.closed || c.docSubs.get(documentId) !== subRec) return false
await client.query('BEGIN')
const { rows } = await client.query<{ company_id: string }>(
`SELECT d.company_id
FROM documents d
JOIN company_members cm ON cm.company_id = d.company_id AND cm.user_id = $2
JOIN users u ON u.id = cm.user_id AND u.deleted_at IS NULL
JOIN participants p
ON p.id = cm.user_id
AND p.company_id = d.company_id
AND p.kind = 'human'
AND p.departed_at IS NULL
WHERE d.id = $1
FOR SHARE OF d, cm, u, p NOWAIT`,
[documentId, c.userId],
)
if (!rows[0] || c.closed || c.docSubs.get(documentId) !== subRec) {
await client.query('ROLLBACK')
detachDocSubscription(c, documentId, subRec)
return false
}
taskStarted = true
await task(rows[0].company_id, client)
await client.query('COMMIT')
return true
} catch (error) {
await client.query('ROLLBACK').catch(() => {})
throw error
} finally {
client.release()
}
})
} catch (error) {
if (!taskStarted && isTransientDocAuthError(error)) {
const remaining = deadline - Date.now()
if (remaining <= 0) throw error
const backoff = replaySupported
? 5 * (attempt + 1)
: Math.min(DOC_AUTH_RETRY_DELAY_MS * (attempt + 1), 1_000)
await new Promise((resolve) => setTimeout(resolve, Math.min(backoff, remaining)))
attempt++
continue
}
throw error
}
}
}
function sendJson(ws: WebSocket, payload: unknown): void {
if (ws.readyState !== ws.OPEN) return
try { ws.send(JSON.stringify(payload)) } catch { /* ignore */ }
}
function detachDocSubscription(
c: AuthedSocket,
documentId: string,
subRec: DocSubscriber,
): void {
if (c.docSubs.get(documentId) !== subRec) return
docUnsubscribe(documentId, subRec)
c.docSubs.delete(documentId)
docReplaySupport.delete(subRec)
}
async function handleDocFrame(c: AuthedSocket, msg: Record<string, unknown>): Promise<void> {
const type = msg.type as string | undefined
const documentId = typeof msg.documentId === 'string' ? msg.documentId : null
if (!documentId) return
if (type === 'doc.subscribe') {
if (c.closed || c.docSubs.has(documentId)) return // idempotent
const companyId = await docCompanyFor(documentId, c.userId)
if (c.closed) return
if (!companyId) {
sendJson(c.ws, { type: 'doc.error', documentId, error: 'not found' })
return
}
let subRec: DocSubscriber
const replaySupported = msg.replaySupported === true
let pendingOutboundBytes = 0
let pendingOutboundFrames = 0
const enqueueAuthorizedFrame = (
payload: unknown,
byteLength: number,
pendingByteLimit = WS_TERMINATE_BUFFERED_BYTES,
): void => {
const estimatedBytes = Math.max(1, Math.ceil(byteLength * 4 / 3) + 256)
if (
pendingOutboundFrames >= 64
|| pendingOutboundBytes + estimatedBytes > pendingByteLimit
) {
detachDocSubscription(c, documentId, subRec)
try { c.ws.terminate() } catch { /* ignore */ }
return
}
pendingOutboundFrames++
pendingOutboundBytes += estimatedBytes
enqueueAuthorizedDocFrame(c, documentId, subRec, payload, byteLength, replaySupported, () => {
pendingOutboundFrames = Math.max(0, pendingOutboundFrames - 1)
pendingOutboundBytes = Math.max(0, pendingOutboundBytes - estimatedBytes)
})
}
subRec = {
originId: c.originId,
onUpdate: (update, originId) => {
enqueueAuthorizedFrame({
type: 'doc.update',
documentId,
updateB64: Buffer.from(update).toString('base64'),
originId,
}, update.byteLength)
},
onAwareness: (update, originId) => {
enqueueAuthorizedFrame({
type: 'doc.awareness',
documentId,
updateB64: Buffer.from(update).toString('base64'),
originId,
}, update.byteLength)
},
}
c.docSubs.set(documentId, subRec)
docReplaySupport.set(subRec, replaySupported)
let initialState: Uint8Array
try {
({ initialState } = await docSubscribe(documentId, companyId, subRec))
} catch (error) {
detachDocSubscription(c, documentId, subRec)
throw error
}
if (c.closed || c.docSubs.get(documentId) !== subRec) {
docUnsubscribe(documentId, subRec)
if (c.docSubs.get(documentId) === subRec) c.docSubs.delete(documentId)
docReplaySupport.delete(subRec)
return
}
if (initialState.byteLength > DOC_SYNC_MAX_BYTES) {
detachDocSubscription(c, documentId, subRec)
sendJson(c.ws, {
type: 'doc.error',
documentId,
error: 'document snapshot exceeds the 32 MiB realtime sync limit',
})
return
}
enqueueAuthorizedFrame({
type: 'doc.sync',
documentId,
stateB64: Buffer.from(initialState).toString('base64'),
originId: c.originId,
}, initialState.byteLength, Math.ceil(DOC_SYNC_MAX_BYTES * 4 / 3) + 256)
return
}
if (type === 'doc.unsubscribe') {
const subRec = c.docSubs.get(documentId)
if (!subRec) return
detachDocSubscription(c, documentId, subRec)
return
}
if (type === 'doc.update') {
if (c.closed) return
const subRec = c.docSubs.get(documentId)
if (!subRec) return // must subscribe first
const updateB64 = typeof msg.updateB64 === 'string' ? msg.updateB64 : ''
if (!updateB64) return
const buf = Buffer.from(updateB64, 'base64')
const update = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
await withAuthorizedDocOperation(c, documentId, subRec, (companyId, client) =>
docApplyLocalUpdate(documentId, companyId, c.originId, c.userId, update, client))
return
}
if (type === 'doc.awareness') {
if (c.closed) return
const subRec = c.docSubs.get(documentId)
if (!subRec) return
const updateB64 = typeof msg.updateB64 === 'string' ? msg.updateB64 : ''
if (!updateB64) return
const buf = Buffer.from(updateB64, 'base64')
const update = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength)
await withAuthorizedDocOperation(c, documentId, subRec, (companyId) =>
docBroadcastAwareness(documentId, companyId, c.originId, update))
return
}
if (type === 'doc.mention.notify') {
if (c.closed) return
const rawIds = msg.mentionedIds
if (!Array.isArray(rawIds) || rawIds.length === 0) return
const requestedIds = rawIds.filter((x): x is string => typeof x === 'string')
if (requestedIds.length === 0) return
const companyId = await docCompanyFor(documentId, c.userId)
if (!companyId || c.closed) return
await processDocMention({
documentId, companyId, mentionerId: c.userId, requestedIds,
})
return
}
}
/** Persist + fan out one or more @-mentions inside a doc. Filters the
* caller-supplied list against tenant membership (so a stale client
* can't notify someone in a different company) and dedups against the
* most recent mention-row for the same (doc, mentioner, mentioned)
* tuple — we don't want a noisily editing user spamming the
* recipient. For mentioned AGENTS, also writes an `agent_log` row so
* the agent's history surfaces the mention.
*
* Exported for the integration suite: the only production caller is the
* `doc.mention.notify` WS frame above, and standing up a socket just to
* assert the durable/outbox contract would test the transport instead. */
export async function processDocMention(args: {
documentId: string
companyId: string
mentionerId: string
requestedIds: string[]
}): Promise<void> {
const { documentId, companyId, mentionerId, requestedIds } = args
const participantIds = [...new Set([mentionerId, ...requestedIds])].sort()
let mentionerName = mentionerId
let documentTitle = 'Untitled'
const freshRows: Array<{ id: string; kind: string; name: string }> = []
const client = await pool.connect()
try {
await client.query('BEGIN')
const { rows: participants } = await client.query<{ id: string; kind: string; name: string }>(
`SELECT p.id, p.kind, p.name FROM participants p
WHERE p.company_id = $1 AND p.id = ANY($2::text[])
AND p.kind IN ('agent', 'human') AND p.departed_at IS NULL
AND (
p.kind = 'agent'
OR EXISTS (
SELECT 1 FROM users u
JOIN company_members cm ON cm.user_id = u.id AND cm.company_id = p.company_id
WHERE u.id = p.id AND u.deleted_at IS NULL
)
)
ORDER BY p.id FOR SHARE`,
[companyId, participantIds],
)
const byId = new Map(participants.map((row) => [row.id, row]))
const mentioner = byId.get(mentionerId)
if (!mentioner || mentioner.kind !== 'human') {
throw new Error('document mentioner is no longer an active workspace user')
}
mentionerName = mentioner.name || mentionerId
const validRows = [...new Set(requestedIds)]
.map((id) => byId.get(id))
.filter((row): row is { id: string; kind: string; name: string } => Boolean(row))
if (validRows.length === 0) {
await client.query('COMMIT')
return
}
const { rows: document } = await client.query<{ title: string }>(
`SELECT title FROM documents
WHERE id = $1 AND company_id = $2
FOR SHARE`,
[documentId, companyId],
)
if (!document[0]) throw new Error('document is no longer available in this workspace')
documentTitle = document[0].title || 'Untitled'
await client.query(
`SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))`,
[documentId, mentionerId],
)
for (const row of validRows) {
const { rows: recent } = await client.query<{ id: string }>(
`SELECT id FROM document_mentions
WHERE document_id = $1 AND mentioner_id = $2 AND mentioned_id = $3
AND created_at > NOW() - INTERVAL '60 seconds'
LIMIT 1`,
[documentId, mentionerId, row.id],
)
if (recent[0]) continue
await client.query(
`INSERT INTO document_mentions
(id, document_id, company_id, mentioner_id, mentioned_id)
VALUES ($1, $2, $3, $4, $5)`,
[`dm_${randomUUID().replace(/-/g, '').slice(0, 16)}`,
documentId, companyId, mentionerId, row.id],
)
if (row.kind === 'agent') {
await client.query(
`INSERT INTO agent_log (id, agent_id, company_id, kind, body, ref)
VALUES ($1, $2, $3, 'doc_mention', $4, $5::jsonb)`,
[
`log_${randomUUID().replace(/-/g, '').slice(0, 16)}`,
row.id, companyId,
`${mentionerName} @-mentioned you in doc "${documentTitle}"`,
JSON.stringify({ documentId, mentionerId }),
],
)
}
freshRows.push(row)
}
// The toast rides the transactional outbox rather than a post-COMMIT
// publish. `document_mentions` is the dedup ledger: once these rows
// commit, the 60s window above swallows every retry, so a Redis outage
// that threw here used to lose the notice permanently while the caller
// saw a failure for work that had actually succeeded. Enqueued in-band,
// a degraded Redis only delays it. See realtime-outbox.ts.
if (freshRows.length > 0) {
const event: DocMentionEvent = {
type: 'doc.mention',
companyId,
documentId,
documentTitle,
mentionerId,
mentionerName,
mentionedIds: freshRows.map((row) => row.id),
}
await enqueueBroadcast(client, CH_DOC_MENTION, event)
}
await client.query('COMMIT')
nudgeRealtimeOutbox()
} catch (error) {
await client.query('ROLLBACK').catch(() => {})
throw error
} finally {
client.release()
}
if (freshRows.length === 0) return
for (const row of freshRows) {
if (row.kind !== 'agent') continue
try {
await postDocMentionWake({
companyId,
mentionerId,
agentId: row.id,
documentId,
})
} catch (e) {
console.warn(`[doc.mention] wake post for ${row.id} failed`, e)
}
}
}
/** Post a synthetic chat message that wakes the mentioned agent with
* enough context to act. The agent's mailbox scheduler subscribes to
* CH_MESSAGE_NEW and runs a turn for every recipient — so dropping a
* real `text` message into a conversation the agent is in is the
* cheapest, most-reliable wake-with-context primitive available
* today (same pattern boards / scanner use indirectly).
*
* Conversation selection — first match wins:
* 1. The doc's pinned `conversation_id`, IF the agent is a member.
* 2. An existing 1:1 DM between mentioner + agent.
* 3. A freshly-created DM (idempotent — same shape as the auto-DM
* created on agent onboarding).
*
* Body is plain prose — looks like a regular nudge to the agent's
* inbox parser, but carries the doc id verbatim so the agent's tool
* loop can call `cumora doc read <id>` without guessing. */
async function postDocMentionWake(args: {
companyId: string
mentionerId: string
agentId: string
documentId: string
}): Promise<void> {
const { companyId, mentionerId, agentId, documentId } = args
const participantIds = [mentionerId, agentId].sort()
let conversationId = ''
let documentTitle = 'Untitled'
let sequence = 0
const messageId = `m-${randomUUID()}`
let body = ''
const client = await pool.connect()
try {
await client.query('BEGIN')
const { rows: participants } = await client.query<{ id: string; kind: string; name: string }>(
`SELECT p.id, p.kind, p.name FROM participants p
WHERE p.company_id = $1 AND p.id = ANY($2::text[])
AND p.kind IN ('agent', 'human') AND p.departed_at IS NULL
AND (
p.kind = 'agent'
OR EXISTS (
SELECT 1 FROM users u
JOIN company_members cm ON cm.user_id = u.id AND cm.company_id = p.company_id
WHERE u.id = p.id AND u.deleted_at IS NULL
)
)
ORDER BY p.id FOR SHARE`,
[companyId, participantIds],
)
const byId = new Map(participants.map((row) => [row.id, row]))
const mentioner = byId.get(mentionerId)
const agent = byId.get(agentId)
if (mentioner?.kind !== 'human' || agent?.kind !== 'agent') {
throw new Error('document mention participants are no longer active in this workspace')
}
const { rows: document } = await client.query<{ title: string; conversation_id: string | null }>(
`SELECT title, conversation_id FROM documents
WHERE id = $1 AND company_id = $2
FOR SHARE`,
[documentId, companyId],
)
if (!document[0]) throw new Error('document is no longer available in this workspace')
documentTitle = document[0].title || 'Untitled'
// Serialize direct-conversation creation with every other DM entry point.
await client.query(
`SELECT pg_advisory_xact_lock(hashtext($1), hashtext($2))`,
[companyId, JSON.stringify(participantIds)],
)
// 1) Use the document's current pinned conversation only while both
// participants still belong to it. 2) Otherwise reuse/create one DM.
if (document[0].conversation_id) {
const { rows } = await client.query<{ id: string }>(
`SELECT c.id FROM conversations c
WHERE c.id = $1 AND c.company_id = $2
AND EXISTS (
SELECT 1 FROM conversation_members cm
WHERE cm.conversation_id = c.id AND cm.company_id = c.company_id
AND cm.participant_id = $3
)
AND EXISTS (
SELECT 1 FROM conversation_members cm
WHERE cm.conversation_id = c.id AND cm.company_id = c.company_id
AND cm.participant_id = $4
)
FOR UPDATE OF c`,
[document[0].conversation_id, companyId, mentionerId, agentId],
)
conversationId = rows[0]?.id ?? ''
}
if (!conversationId) {
const { rows } = await client.query<{ id: string }>(
`SELECT c.id FROM conversations c
WHERE c.kind = 'direct' AND c.company_id = $3
AND EXISTS (
SELECT 1 FROM conversation_members cm
WHERE cm.conversation_id = c.id AND cm.company_id = c.company_id
AND cm.participant_id = $1
)
AND EXISTS (
SELECT 1 FROM conversation_members cm
WHERE cm.conversation_id = c.id AND cm.company_id = c.company_id
AND cm.participant_id = $2
)
AND (SELECT COUNT(*) FROM conversation_members cm
WHERE cm.conversation_id = c.id AND cm.company_id = c.company_id) = 2
ORDER BY c.updated_at DESC LIMIT 1
FOR UPDATE OF c`,
[mentionerId, agentId, companyId],
)
conversationId = rows[0]?.id ?? ''
}
if (!conversationId) {
conversationId = `direct-${agentId}-${randomUUID().slice(0, 6)}`
await client.query(
`INSERT INTO conversations
(id, kind, title, subtitle, members, pinned, tag, company_id)
VALUES ($1, 'direct', $2, NULL, $3::jsonb, FALSE, NULL, $4)`,
[conversationId, agent.name || agentId, JSON.stringify([mentionerId, agentId]), companyId],
)
}
const seqRes = await client.query<{ seq: number }>(
`INSERT INTO conversation_counters (conversation_id, next_sequence)
VALUES ($1, 2)
ON CONFLICT (conversation_id) DO UPDATE
SET next_sequence = conversation_counters.next_sequence + 1
RETURNING next_sequence - 1 AS seq`,
[conversationId],
)
sequence = seqRes.rows[0]?.seq ?? 1
body = `@${agentId} heads-up — I @-mentioned you in the doc "${documentTitle}". Take a look with \`cumora doc read ${documentId}\`, then either reply here or edit the doc directly (\`cumora doc append/replace ${documentId} …\`).`
await client.query(
`INSERT INTO messages (id, conversation_id, author_id, kind, body, sequence, company_id)
VALUES ($1, $2, $3, 'text', $4, $5, $6)`,
[messageId, conversationId, mentionerId, body, sequence, companyId],
)
await client.query(`UPDATE conversations SET updated_at = NOW() WHERE id = $1`, [conversationId])
// Same transaction as the message row. A Redis blip used to leave this
// wake committed and unpublished, and nothing scanned for it later.
await enqueueBroadcast(client, CH_MESSAGE_NEW, {
type: 'message.new',
companyId,
conversationId,
message: {
id: messageId,
conversationId,
authorId: mentionerId,
kind: 'text',
body,
sequence,
at: new Date().toISOString(),
},
})
await client.query('COMMIT')
} catch (error) {
await client.query('ROLLBACK').catch(() => {})
throw error
} finally {
client.release()
}
nudgeRealtimeOutbox()
}
export function attachWebSocket(httpServer: Server) {
const wss = new WebSocketServer({ server: httpServer, path: '/ws', maxPayload: 4 * 1024 * 1024 })
wss.on('connection', async (ws, req) => {
const ip = req.socket.remoteAddress
let earlyClosed = false
const onEarlyClose = () => { earlyClosed = true }
const onEarlyError = () => { earlyClosed = true }
// Install lifecycle observers before ticket/member lookups. A client can
// close during either await; otherwise hydration could finish and attach a
// subscription to a socket whose close event was already missed.
ws.once('close', onEarlyClose)
ws.once('error', onEarlyError)
// The WS connect URL carries a SHORT-LIVED one-shot ticket
// (?t=<ws-ticket>), not the session token. Tickets are minted via
// POST /auth/ws-ticket and consumed atomically here. This keeps
// session tokens out of URLs / access logs / referrer headers.
let ticket: string | undefined
try {
const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`)
const t = url.searchParams.get('t')
if (t) ticket = t
} catch { /* ignore */ }
if (!ticket) {
console.log(`[ws] rejecting unauthenticated connection (${ip})`)
try { ws.close(4401, 'missing ws ticket') } catch { /* ignore */ }
return
}
let session: Awaited<ReturnType<typeof consumeWsTicket>>
try {
session = await consumeWsTicket(ticket)
} catch (error) {
console.warn(`[ws] ticket lookup failed (${ip})`, error)
try { ws.close(1011, 'ticket lookup failed') } catch { /* ignore */ }
return
}
if (!session) {
console.log(`[ws] rejecting bad/expired/used ticket (${ip})`)
try { ws.close(4401, 'invalid ws ticket') } catch { /* ignore */ }
return
}
if (earlyClosed) return
let companies: Set<string>
try {
companies = await loadMemberships(session.userId)
} catch (error) {
console.warn(`[ws] membership lookup failed (${ip})`, error)
try { ws.close(1011, 'membership lookup failed') } catch { /* ignore */ }
return
}
if (earlyClosed) return
ws.off('close', onEarlyClose)
ws.off('error', onEarlyError)
const c: AuthedSocket = {
ws,
userId: session.userId,
originId: randomUUID(),
companies,
docSubs: new Map(),
docFrameQueue: Promise.resolve(),
pendingDocFrames: 0,
pendingDocBytes: 0,
closed: false,
isAlive: true,
}
clients.add(c)
// Browsers auto-respond to ws.ping() with pong at the protocol level —
// no JS involvement on the client side. The pong handler here is the
// server's only signal that the socket is still alive end-to-end.
ws.on('pong', () => { c.isAlive = true })
console.log(`[ws] client connected (${ip}, user=${session.userId}, companies=${companies.size}) · total ${clients.size}`)
void onHumanConnect(session.userId)
// Single-fire disconnect handler — both 'close' and 'error' route
// through here so we never double-decrement the connection counter.
let released = false
const release = () => {
if (released) return
released = true
c.closed = true
removeQueuedDocItems(c)
for (const [docId, subRec] of c.docSubs) {
docUnsubscribe(docId, subRec)
docReplaySupport.delete(subRec)
}
c.docSubs.clear()
clients.delete(c)
void onHumanDisconnect(session.userId)
}
try {
ws.send(JSON.stringify({ type: 'hello', instanceId: env.INSTANCE_ID, ts: Date.now() }))
} catch { /* ignore */ }
const enqueueInboundDocFrame = (msg: Record<string, unknown>, rawBytes: number): void => {
if (c.closed) return
const bytes = Math.max(1, rawBytes)
if (
c.pendingDocFrames >= 128
|| c.pendingDocBytes + bytes > 4 * 1024 * 1024
|| !reserveDocAuth(bytes)
) {
try { c.ws.terminate() } catch { /* ignore */ }
return
}
c.pendingDocFrames++
c.pendingDocBytes += bytes
const current = c.docFrameQueue.catch(() => {}).then(async () => {
if (c.closed) return
await handleDocFrame(c, msg)
}).finally(() => {
c.pendingDocFrames = Math.max(0, c.pendingDocFrames - 1)
c.pendingDocBytes = Math.max(0, c.pendingDocBytes - bytes)
releaseDocAuth(bytes)
})
c.docFrameQueue = current
void current.catch((error) => {
if (c.closed) return
console.warn('[ws] doc frame error', error)
// A doc.update may be optimistic in the browser. Closing forces a
// fresh hello/doc.sync cycle, where yjsClient replays local state the
// server snapshot did not contain; a doc.error alone would strand it.
try { c.ws.terminate() } catch { /* ignore */ }
})
}
ws.on('message', (raw) => {
let msg: Record<string, unknown>
try { msg = JSON.parse(raw.toString()) as Record<string, unknown> } catch { return }
const type = typeof msg.type === 'string' ? msg.type : ''
if (type.startsWith('doc.')) {
enqueueInboundDocFrame(msg, Buffer.byteLength(raw.toString()))
}
// Other inbound types (ping etc.) would land here later; today the
// chat protocol is pure REST + broadcast so there's nothing else.
})
ws.on('close', () => {
release()
console.log(`[ws] client disconnected · total ${clients.size}`)
})
ws.on('error', (err) => {
console.warn('[ws] socket error', err)
release()
})
})
// Bridge Redis pubsub → local WS fan-out, scoped per company. The doc
// channels (CH_DOC_UPDATE / CH_DOC_AWARENESS) are intentionally NOT in
// this list — the room manager handles them, since recipients need to
// be filtered by doc-subscription, not just company.
sub.subscribe(
CH_MESSAGE_NEW, CH_MESSAGE_DELTA, CH_TYPING,
CH_STATUS, CH_REACTIONS, CH_POLLS,
CH_GROUP_PULLED, CH_CONVO_UPDATED, CH_CONVENE,
CH_BOARDS, CH_DOCS, CH_CALENDAR_REMINDER, CH_CALENDAR_EVENTS, CH_DOC_MENTION, CH_WORKSPACES,
).then((count) => {
console.log(`[ws] subscribed to ${count} redis channels`)
})
sub.on('message', (channel, payload) => {
// Doc channels are room-scoped, not company-scoped — skip them here.
if (channel === 'cumora:doc.update' || channel === 'cumora:doc.awareness') return
// Untagged events are dropped. Tagged events are then delivered to the
// user ids resolveWsEventRecipientUserIds returns, not by intersecting
// companyId with the socket's membership snapshot. A connected user
// receives frames for every workspace they belong to.
let event: RoutedRedisEvent
try {
event = JSON.parse(payload) as RoutedRedisEvent
} catch { /* malformed — drop */ return }
const companyId = typeof event.companyId === 'string' ? event.companyId : undefined
if (!companyId) {
// Conservative: untagged events have no tenant — refuse to route.
// (If an untagged event ever reaches here it's a publisher bug; logging
// helps catch the gap during the rollout.)
console.warn('[ws] dropping untagged event')
return
}
// Membership resolution is asynchronous. Serialize per conversation (or
// company for workspace-wide frames) so a message and its delta/reaction
// cannot be reordered while independent rooms still route in parallel.
const routeKey = event.conversationId
? `${companyId}:conversation:${event.conversationId}`
: `${companyId}:workspace`
const previous = redisFanoutQueues.get(routeKey) ?? Promise.resolve()
const current = previous.catch(() => {}).then(async () => {
const recipients = await resolveWsEventRecipientUserIds(event)
for (const c of clients) {
if (!recipients.has(c.userId)) continue
if (c.ws.readyState !== c.ws.OPEN) continue
// Backpressure guard (OOM fix): `ws.send()` buffers unsent frames in
// process memory when a socket can't drain (slow/stuck client). Under a
// high broadcast rate that buffer grows UNBOUNDED across clients → the pod
// OOMs. If a socket is backed up past the cap it isn't keeping up — drop
// this frame for it; if it's wildly backed up, terminate it to reclaim the
// memory (it reconnects and re-syncs via REST). Bounds WS memory to
// ~WS_MAX_BUFFERED_BYTES per client.
const buffered = c.ws.bufferedAmount
if (buffered > WS_TERMINATE_BUFFERED_BYTES) {
try { c.ws.terminate() } catch { /* ignore */ }
continue
}
if (buffered > WS_MAX_BUFFERED_BYTES) continue // skip frame; let it drain
try { c.ws.send(payload) } catch { /* ignore */ }
}
})
redisFanoutQueues.set(routeKey, current)
void current.catch((error) => {
// Fail closed: a routing lookup failure drops the frame instead of
// falling back to the stale socket membership snapshot.
console.warn(`[ws] live authorization lookup failed for ${routeKey}`, error)
}).finally(() => {
if (redisFanoutQueues.get(routeKey) === current) redisFanoutQueues.delete(routeKey)
})
})
// Heartbeat sweeper. Real-deal human presence used to drift because TCP
// can keep a half-open socket "alive" for many minutes after the
// laptop sleeps / network drops, so the close handler never fired and
// the user stayed 'avail' forever. Now we actively ping every 30s; a
// client that doesn't pong before the NEXT tick is terminated, which
// routes through the same close handler that decrements the presence
// counter. End-to-end effect: status flips to 'resting' within ~60s
// of a real disconnect.
const HEARTBEAT_MS = 30_000
const heartbeat = setInterval(() => {
for (const c of clients) {
if (!c.isAlive) {
// Missed two ticks in a row — kill it. terminate() bypasses the
// close handshake and fires our 'close' listener immediately.
try { c.ws.terminate() } catch { /* ignore */ }
continue
}
c.isAlive = false
try { c.ws.ping() } catch { /* ignore */ }
}
}, HEARTBEAT_MS)
heartbeat.unref()
wss.on('close', () => clearInterval(heartbeat))
return wss
}