Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.
TypeScript
import express from 'express'
import compression from 'compression'
import http from 'node:http'
import { mkdir } from 'node:fs/promises'
import { existsSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { env } from './env.js'
import { verifySchemaWithBootRetry } from './db/schema-version.js'
import { seedIfEmpty } from './seed.js'
import { api } from './api/router.js'
import { storage, UPLOAD_DIR } from './storage.js'
import { attachWebSocket, resetHumanPresenceOnBoot } from './ws.js'
import { bootDocumentBus } from './documents/rooms.js'
import { healthPool, pool } from './db/pool.js'
import { redis } from './redis.js'
import { startScanner } from './agents/scanner.js'
import { startScheduler } from './agents/scheduler.js'
import { startIdleScheduler } from './agents/idle.js'
import { backfillStarterAgents, backfillHumanGravatars, backfillStarterAvatars } from './onboardCompany.js'
import { backfillMemoryEmbeddings } from './agents/embeddings.js'
import { startStaleAgentRunSweeper } from './agents/observability.js'
import { sweepOfflineComputers } from './agents/computer/registry.js'
import { sweepStaleNamespaces } from './agents/runtime/fs-namespace.js'
import { runtimeRouter } from './agents/runtime/server.js'
import { startCompletedPodGc, startClusterFuseMonitor, startChromeProfilePvcGc } from './agents/runtime/orchestrator.js'
import { inboundEmailRouter } from './api/inbound-email.js'
import { startEmailRetryWorker } from './email-retry.js'
import { startEmailGcWorker, stopEmailGcWorker } from './email-gc.js'
import { startDbGcWorker, stopDbGcWorker } from './db-gc.js'
import { startCalendarScheduler, stopCalendarScheduler } from './calendar.js'
import { startPollExpirationSweeper, stopPollExpirationSweeper } from './polls.js'
import { startLlmRollupRefresher, stopLlmRollupRefresher } from './agents/llm-rollup.js'
import { startTrialSweepWorker, stopTrialSweepWorker } from './trial-sweep.js'
import { seedAdmins } from './admin.js'
import { notifyAlert } from './alerting.js'
import { startShippingMaintenance } from './shipping-maintenance.js'
import { startRealtimeOutboxWorker, stopRealtimeOutboxWorker } from './realtime-outbox.js'
import { startWorkspaceCleanupWorker, stopWorkspaceCleanupWorker } from './workspace-cleanup.js'
const BOOT_BACKFILL_LOCK_KEY = 7_643_178_926_532n
async function runBootBackfill(): Promise<void> {
const client = await pool.connect()
try {
const lock = await client.query<{ ok: boolean }>(
'SELECT pg_try_advisory_lock($1) AS ok', [BOOT_BACKFILL_LOCK_KEY],
)
if (lock.rows[0]?.ok !== true) return
try {
await seedIfEmpty()
await seedAdmins()
await backfillStarterAgents()
await backfillHumanGravatars()
await backfillStarterAvatars()
} finally {
await client.query('SELECT pg_advisory_unlock($1)', [BOOT_BACKFILL_LOCK_KEY])
.catch(() => { /* session teardown releases it anyway */ })
}
} finally {
client.release()
}
}
async function main() {
const schemaVersion = await verifySchemaWithBootRetry()
console.log(`[boot] schema version ${schemaVersion} is compatible`)
// In local-storage mode the uploads dir backs the /uploads/ static handler
// below. In R2 mode neither is needed — files live in object storage and
// public URLs are served directly by R2 / CDN.
if (storage.mode === 'local') {
await mkdir(UPLOAD_DIR, { recursive: true })
}
const app = express()
// gzip responses ≥1kb. Uncompressed JSON to the BYOA fleet was the #1 GCP
// egress cost (~160GiB/wk, /runtime/inbox alone ~90% of API bytes). SSE
// must stay uncompressed: compression buffers event-stream chunks, which
// would stall wake-streams until the buffer flushes.
app.use(compression({
filter: (req, res) => {
const type = res.getHeader('Content-Type')
if (typeof type === 'string' && type.includes('text/event-stream')) return false
return compression.filter(req, res)
},
}))
// CORS — only kicks in when CUMORA_CORS_ORIGINS is set. Browsers send
// `Origin` only on cross-origin requests, so same-origin (Vite proxy,
// colocated static deploy) is unaffected. `*` allows any origin but
// disables credentials per the CORS spec. We do NOT enable credentials
// because auth here is header-based (Bearer token), not cookies.
const corsAllow = new Set(env.CORS_ORIGINS)
const corsAny = corsAllow.has('*')
app.use((req, res, next) => {
const origin = req.headers.origin
if (origin && (corsAny || corsAllow.has(origin))) {
res.setHeader('Access-Control-Allow-Origin', corsAny ? '*' : origin)
res.setHeader('Vary', 'Origin')
res.setHeader('Access-Control-Allow-Methods', 'GET,POST,PUT,PATCH,DELETE,OPTIONS')
res.setHeader('Access-Control-Allow-Headers', 'content-type,authorization,x-company-id,x-cumora-dev-mode')
res.setHeader('Access-Control-Max-Age', '600')
if (req.method === 'OPTIONS') { res.status(204).end(); return }
}
next()
})
// Inbound-email webhook (Cloudflare Email Worker → here). It owns its
// endpoint-specific 25MB parser and raw-body HMAC capture; the API and
// runtime routers below own their smaller/authenticated JSON policies.
app.use('/webhooks/email', inboundEmailRouter)
if (storage.mode === 'local') {
app.use('/uploads', express.static(UPLOAD_DIR, {
fallthrough: false,
maxAge: '1h',
setHeaders: (res, filePath) => {
// Uploads are served from the app origin in local-storage mode, so a
// file the browser renders as active content (HTML/SVG/XML) would run
// with the app's origin and could read the localStorage session token.
// Never let the browser sniff a type, and force everything that isn't
// a previewable raster image to download instead of render.
res.setHeader('X-Content-Type-Options', 'nosniff')
const ext = filePath.slice(filePath.lastIndexOf('.') + 1).toLowerCase()
const inlineOk = ext === 'png' || ext === 'jpg' || ext === 'jpeg' || ext === 'webp' || ext === 'gif'
if (!inlineOk) res.setHeader('Content-Disposition', 'attachment')
},
}))
}
app.use((req, res, next) => {
const t = Date.now()
res.on('finish', () => {
const ms = Date.now() - t
if (ms > 250 || res.statusCode >= 400) {
console.log(`[http] ${req.method} ${req.url} → ${res.statusCode} ${ms}ms`)
}
})
next()
})
app.use('/api', api)
// Per-pod agent runtime API — JWT-authed, completely separate from the
// human /api/* surface. That surface reads Authorization: Bearer or
// x-session-token; it has no cookie session, and authMiddleware does not
// reject. See agents/runtime/server.ts.
app.use('/runtime', runtimeRouter)
// ============== Host gating ==============
// api.cumora.ai is JSON-only. The /api, /runtime, /uploads mounts above
// already handle the legit traffic; anything reaching this middleware on
// the api.* host is a stray GET that would otherwise fall through to the
// SPA shell, which we don't want — return 404 instead. admin.* and app.*
// hosts continue through to the SPA, which self-routes client-side.
app.use((req, res, next) => {
const host = (req.hostname || '').toLowerCase()
if (host.startsWith('api.')) {
res.status(404).json({ error: 'not found' })
return
}
next()
})
// ============== Web SPA bundle ==============
// The same Docker image bakes in the frontend `dist/` next to the server
// source, so this single pod serves both api.cumora.ai (JSON) and
// app.cumora.ai (HTML+JS). The two hostnames hit the SAME GKE Service —
// same-origin from the browser's POV when on app.cumora.ai means zero
// CORS for the renderer. Local-dev keeps using Vite (port 5173) and the
// existing /api proxy, so the static handler is skipped when dist/ isn't
// present.
//
// Mount order matters: /api, /runtime, /uploads are matched FIRST above,
// so the SPA fallback below only catches non-API paths. The regex on the
// SPA route explicitly excludes those prefixes too, as a belt-and-braces
// against future routes that might fall through to here.
// Dev (NODE_ENV !== 'production') always uses Vite on :5173 — we
// explicitly skip the static handler so a stale dist/ from a previous
// `npm run build` doesn't get served on :5181 and confuse the
// localhost flow.
const DIST_DIR = resolve(process.cwd(), 'dist')
const INDEX_HTML = join(DIST_DIR, 'index.html')
const hasDist = env.NODE_ENV === 'production' && existsSync(INDEX_HTML)
if (hasDist) {
app.use(express.static(DIST_DIR, {
index: false,
// Hashed JS/CSS get long cache; index.html is served via the SPA
// fallback below with no-cache headers, so deploys are picked up
// instantly even when the static-asset CDN caches aggressively.
maxAge: '1h',
setHeaders: (res, filePath) => {
if (filePath.endsWith('index.html')) {
res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
}
},
}))
// SPA fallback — any GET that isn't an API / runtime / uploads / ws path
// returns index.html so client-side routes like /invite/<token> work on
// first load + on refresh. POST/PUT/DELETE never fall through to here.
app.get(/^(?!\/(api|runtime|uploads|ws)(\/|$)).*/, (_req, res) => {
res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
res.sendFile(INDEX_HTML)
})
console.log(`[boot] serving SPA from ${DIST_DIR}`)
} else {
// No bundle present (typical for local-dev where Vite serves the
// renderer on :5173). Keep `/` informational so dev probes still work.
app.get('/', (_req, res) => res.json({
name: 'cumora', instance: env.INSTANCE_ID, spa: 'dev (served by vite)',
}))
}
// App-level error handler — catches anything the router-level handler missed
// (body-parse errors, middleware exceptions, etc.) and ALWAYS returns a JSON
// body so the client error display gets something actionable instead of an
// empty 500 page. Must come last in the middleware chain.
app.use((err: unknown, req: express.Request, res: express.Response, _next: express.NextFunction) => {
const msg = err instanceof Error ? err.message : String(err)
const stack = err instanceof Error ? err.stack : undefined
console.error(`[app] uncaught on ${req.method} ${req.url}:`, msg)
if (stack) console.error(stack)
if (res.headersSent) return
const status = (err as { status?: number; statusCode?: number })?.status
?? (err as { statusCode?: number })?.statusCode
?? 500
res.status(status).json({ error: msg || 'internal server error' })
})
const server = http.createServer(app)
const wss = attachWebSocket(server)
// Cross-instance Y.Doc fan-out — the room manager subscribes to the
// doc redis channels here so two server instances stay convergent.
bootDocumentBus()
server.listen(env.PORT, () => {
console.log(`[boot] cumora server :${env.PORT} · instance ${env.INSTANCE_ID} · model ${env.OPENAI_MODEL}`)
})
// Seed and backfill after listen. They used to run first, so a rollout
// could not become ready until every replica had scanned the tables.
// One replica takes the lock; the others skip. The filesystem sweep is
// per machine and stays outside that lock.
void sweepStaleNamespaces().catch((error) => {
console.warn('[boot] sweepStaleNamespaces crashed', error instanceof Error ? error.message : error)
})
void runBootBackfill().catch((error) => {
console.warn('[boot] backfill crashed', error instanceof Error ? error.message : error)
})
void backfillMemoryEmbeddings().catch((e) =>
console.warn('[embed:backfill] crashed', e instanceof Error ? e.message : String(e)),
)
// Demote any 'avail' humans left over from the previous run; real
// presence will be re-asserted as WS clients reconnect. Run AFTER
// server.listen() and fire-and-forget — this used to be `await`'d
// before the listen, which blocked startup. On first deploy of the
// presence feature there were thousands of stale rows, and the
// sequential publish loop (one Redis publish per row) made the
// healthcheck miss its window, timing out the k8s rollout. The
// UPDATE is the durable state; the publishes are a "tell live
// clients now" optimization and at boot there are zero connected
// clients, so we can defer it safely.
void resetHumanPresenceOnBoot().catch((err) =>
console.warn('[boot] resetHumanPresenceOnBoot crashed in background:',
err instanceof Error ? err.message : String(err)),
)
// Mailbox scheduler: subscribes to CH_MESSAGE_NEW and runs an agent turn for
// every conversation member who isn't the author. Replaces the old
// server-side classifier/cascade — every agent decides for itself via its
// own LLM call whether to reply / react / dm / ack.
startScheduler()
// Start generic background scans for agents that explicitly have
// the background.scan capability.
// Safe across replicas: each tick takes an advisory lock, so exactly one
// instance runs the pass, and wakes are claimed in Redis with a 24h TTL so
// the lock moving elsewhere doesn't re-wake agents for activity already
// scanned. See agents/scanner.ts.
if (process.env.ENABLE_SCANNER !== 'false') {
const handle = startScanner(env.SCANNER_INTERVAL_MS)
console.log(`[boot] background scanner running every ${env.SCANNER_INTERVAL_MS}ms`)
handle.unref()
}
// Idle scheduler — gives agents a chance to spontaneously initiate when
// nothing is incoming. Defaults to 15min cadence; set IDLE_INTERVAL_MS=0
// to disable. Each tick takes an advisory lock, so a second replica does
// not run a second agenda classifier and a second agent turn. See agents/idle.ts.
if (process.env.ENABLE_IDLE !== 'false' && env.IDLE_INTERVAL_MS > 0) {
const handle = startIdleScheduler(env.IDLE_INTERVAL_MS)
if (handle) {
console.log(`[boot] idle scheduler running every ${env.IDLE_INTERVAL_MS}ms (min quiet ${env.IDLE_MIN_QUIET_MIN}min)`)
handle.unref()
}
}
// Outbound email retry loop — reclaims transport_status='failed' rows.
// SKIP LOCKED keeps multi-replica deploys safe; setting
// EMAIL_RETRY_INTERVAL_MS=0 disables it (useful when actively testing
// the failed-state branch and not wanting silent recoveries).
startEmailRetryWorker()
// Email-attachment GC — periodic sweep that deletes storage objects no
// longer referenced by any email_attachments row (DB CASCADE doesn't
// touch object storage on its own). Disabled with EMAIL_GC_INTERVAL_MS=0.
startEmailGcWorker()
startDbGcWorker()
// Mobile Pro-trial expiry — hourly sweep that downgrades lapsed trials
// back to free (mirrors sub2api via the same path the admin UI uses).
startTrialSweepWorker()
// Calendar dispatcher — once a minute, scan calendar_events for due
// occurrences and post the scheduled prompt into the target conversation
// (which wakes the assignee agent via the same mailbox path).
startCalendarScheduler()
// Poll expiration sweeper — once a minute, close polls past their
// expiresAt and broadcast the close event. Safe across replicas: each
// close goes through a per-poll transaction with FOR UPDATE, so a
// racing replica seeing the same row finds closedAt already set and
// bails idempotently.
startPollExpirationSweeper(env.POLL_SWEEP_INTERVAL_MS)
// Observability pre-aggregation refresher — keeps llm_calls_rollup current
// so the admin Observability page reads ~30k hourly buckets (~230ms) instead
// of scanning the whole 470k-row llm_calls table 6× per load (5-25s). First
// tick backfills; steady-state upserts the recent few hours. Advisory-locked
// so one replica refreshes. LLM_ROLLUP_INTERVAL_MS=0 disables.
startLlmRollupRefresher()
// Product shipping loop maintenance — promotes missed production
// readbacks to visible overdue friction and mines repeated agent completion
// failures into deduplicated improvement signals.
startShippingMaintenance()
// PostgreSQL owns command completion; Redis delivery is retried here after
// commit. Start after the schema ensure so the outbox table is guaranteed.
startRealtimeOutboxWorker()
startWorkspaceCleanupWorker()
// Agent-pod garbage collection — sweep Succeeded/Failed/Unknown
// agent pods older than 5min. Plain Pods don't have TTL-after-
// finished, so without this leftover idle-exit pods accumulate
// indefinitely. ENABLE_AGENT_POD_GC=false disables (e.g. local dev
// without kubectl).
if (process.env.ENABLE_AGENT_POD_GC !== 'false') {
startCompletedPodGc()
console.log('[boot] agent-pod GC running every 60s')
}
// Chrome-profile PVC garbage collection — reclaims volumes for
// off-boarded agents and for agents idle > CHROME_PVC_GC_IDLE_DAYS.
// At $0.10/GB·month per PVC this becomes the main cost driver as
// the agent population grows; without it, abandoned profiles
// accumulate forever. ENABLE_CHROME_PVC_GC=false disables (e.g.
// emptyDir-mode clusters that never create PVCs in the first place).
if (process.env.ENABLE_CHROME_PVC_GC !== 'false') {
startChromeProfilePvcGc({
intervalMs: env.CHROME_PVC_GC_INTERVAL_MS,
idleThresholdMs: env.CHROME_PVC_GC_IDLE_DAYS * 24 * 60 * 60_000,
})
}
// Cluster fuse-pressure monitor — periodic check that fires
// notifyAlert when pending agent pods stay above 20 (or fuse
// utilization ≥ 95%) for 5min sustained. ENABLE_CLUSTER_MONITOR=false
// disables. Pairs with the FUSE admission control in ensurePod.
if (process.env.ENABLE_CLUSTER_MONITOR !== 'false') {
startClusterFuseMonitor()
console.log('[boot] cluster fuse-pressure monitor running every 60s')
}
// Agent run orphan sweeper — if a pod finishes during server
// rolling restart / NEG cutover, its final /runtime/runs/:id/finish
// call can be lost after all semantic work is done. Active turns
// update agent_runs.updated_at through status heartbeats/events, so
// a 10min stale running row is an orphan, not real work.
if (process.env.ENABLE_AGENT_RUN_SWEEPER !== 'false') {
startStaleAgentRunSweeper()
console.log('[boot] stale agent-run sweeper running every 60s')
}
// BYOA computer offline sweeper — flip paired computers to 'offline' once
// their daemon heartbeat goes stale, broadcasting the transition so the
// Computers panel + agent chips update live.
{
const handle = setInterval(() => {
void sweepOfflineComputers().catch((e) =>
console.warn('[computer] offline sweep failed', e instanceof Error ? e.message : e))
}, 30_000)
handle.unref()
console.log('[boot] computer offline sweeper running every 30s')
}
// Graceful shutdown. Close sockets with a going-away frame, stop the
// workers that keep using the pool, then give Postgres ten seconds.
const shutdown = async (sig: string) => {
console.log(`[shutdown] ${sig}`)
stopRealtimeOutboxWorker()
stopWorkspaceCleanupWorker()
stopEmailGcWorker()
stopDbGcWorker()
stopCalendarScheduler()
stopPollExpirationSweeper()
stopLlmRollupRefresher()
stopTrialSweepWorker()
for (const socket of wss.clients) {
try { socket.close(1001, 'server shutting down') } catch { /* ignore */ }
}
wss.close()
await new Promise<void>((resolve) => server.close(() => resolve()))
await Promise.race([
Promise.all([
pool.end().catch(() => undefined),
healthPool.end().catch(() => undefined),
]),
new Promise((resolve) => setTimeout(resolve, 10_000)),
])
try { redis.disconnect() } catch { /* ignore */ }
process.exit(0)
}
process.on('SIGINT', () => shutdown('SIGINT'))
process.on('SIGTERM', () => shutdown('SIGTERM'))
}
// Process-level safety nets. On modern Node (≥ 15) an unhandled
// promise rejection terminates the process by default — for cumora
// that would knock out the wake bus + scheduler + HTTP API and every
// agent goes silent until the pod restarts. Most of our async paths
// either await with try/catch or `.catch()`-suffixed fire-and-forget;
// these handlers catch the cases we missed (the recordEvent /
// claimAndWake stack is a known offender) so a single transient
// upstream blip can't drop the whole server.
process.on('unhandledRejection', (reason, _p) => {
// notifyAlert never throws and is fire-and-forget — we still log
// synchronously through it so a webhook outage doesn't lose the
// stderr record.
void notifyAlert({ label: 'server.unhandledRejection', error: reason })
})
process.on('uncaughtException', (err) => {
void notifyAlert({ label: 'server.uncaughtException', error: err })
})
main().catch((err) => {
console.error('[boot] fatal', err)
process.exit(1)
})