Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Merge batch 11-03 bk: #2698 (#2699)

0xken151e3674da1cff
diff --git a/backend/src/lib/memoryStats.test.ts b/backend/src/lib/memoryStats.test.ts
new file mode 100644
index 000000000..a2377ad9b
--- /dev/null
+++ b/backend/src/lib/memoryStats.test.ts
@@ -0,0 +1,30 @@
+import { afterEach, expect, it, vi } from 'vitest'
+import { startMemoryStats } from './memoryStats.js'
+
+afterEach(() => vi.useRealTimers())
+
+const MB = 1024 * 1024
+
+it('logs the process memory in MB and what the Swarm read cache holds, every minute (prod 2026-11-03: 2 -> 6 pods on memory)', () => {
+ vi.useFakeTimers()
+ const log = vi.fn()
+ const usage = () => ({ rss: 700.4 * MB, heapTotal: 400 * MB, heapUsed: 350.6 * MB, external: 30 * MB, arrayBuffers: 12 * MB })
+ const stop = startMemoryStats({ everyMs: 60_000, log, usage, cache: () => ({ entries: 120, bytes: 50 * MB }) })
+ expect(log).not.toHaveBeenCalled()
+ vi.advanceTimersByTime(60_000)
+ expect(log).toHaveBeenCalledWith('memory', { rssMb: 700, heapTotalMb: 400, heapUsedMb: 351, externalMb: 30, arrayBuffersMb: 12,
+ cache: { entries: 120, mb: 50 } })
+ stop()
+ vi.advanceTimersByTime(60_000)
+ expect(log).toHaveBeenCalledTimes(1)
+})
+
+it('says no cache when the process has none (SWARM_READ_CACHE=off, or no stamps)', () => {
+ vi.useFakeTimers()
+ const log = vi.fn()
+ const stop = startMemoryStats({ everyMs: 60_000, log, cache: () => null })
+ vi.advanceTimersByTime(60_000)
+ expect(log.mock.calls[0]![1]).not.toHaveProperty('cache')
+ expect(log.mock.calls[0]![1].rssMb).toBeGreaterThan(0)
+ stop()
+})
diff --git a/backend/src/lib/memoryStats.ts b/backend/src/lib/memoryStats.ts
new file mode 100644
index 000000000..f8b5434ea
--- /dev/null
+++ b/backend/src/lib/memoryStats.ts
@@ -0,0 +1,29 @@
+/**
+ * The process's memory, every minute (`memory`): what the HPA's memory metric sees from inside. On v1.3.87 (2026-11-03) pods
+ * held ~750 MB each and a 70% memory target took the backend from 2 to 6 pods at 22% CPU, with nothing in the logs to say
+ * what held it. The Swarm read cache's own count (its values' JSON size, cache.ts freeze) sits beside the heap, so the two
+ * can be compared.
+ */
+import { logger } from '../utils/logger.js'
+
+const MB = 1024 * 1024
+const mb = (bytes: number) => Math.round(bytes / MB)
+
+type Usage = Pick<NodeJS.MemoryUsage, 'rss' | 'heapTotal' | 'heapUsed' | 'external' | 'arrayBuffers'>
+
+export function startMemoryStats({ everyMs = 60_000, log = (message: string, context: Record<string, unknown>) => logger.info(message, context),
+ usage = () => process.memoryUsage(), cache }: {
+ everyMs?: number
+ log?: (message: string, context: Record<string, unknown>) => void
+ usage?: () => Usage
+ /** What the read cache holds now, or null when the process has none. */
+ cache: () => { entries: number; bytes: number } | null
+}): () => void {
+ const timer = setInterval(() => {
+ const now = usage(), held = cache()
+ log('memory', { rssMb: mb(now.rss), heapTotalMb: mb(now.heapTotal), heapUsedMb: mb(now.heapUsed), externalMb: mb(now.external),
+ arrayBuffersMb: mb(now.arrayBuffers), ...held ? { cache: { entries: held.entries, mb: mb(held.bytes) } } : {} })
+ }, everyMs)
+ timer.unref()
+ return () => clearInterval(timer)
+}
diff --git a/backend/src/server.ts b/backend/src/server.ts
index 687637186..a935161c7 100644
--- a/backend/src/server.ts
+++ b/backend/src/server.ts
@@ -23,9 +23,10 @@ import { startTurnCredentialRefresh } from './lib/turnCredentials.js'
import { closeBus, pub, redisCount, redisSsoProfileStore, redisSwarmStamps } from './lib/bus.js'
import { prisma } from './lib/prisma.js'
import { backendReadinessChecks, createReadiness, drainBeforeClosing, processDrain } from './lib/readiness.js'
-import { configureSwarmStamps } from './superTabs/cache.js'
+import { configureSwarmStamps, swarmReadCache } from './superTabs/cache.js'
import { startCacheStats } from './superTabs/cacheStats.js'
import { startDeliveryCalls } from './lib/deliveryCalls.js'
+import { startMemoryStats } from './lib/memoryStats.js'
import { readDelta } from './superTabs/cachedReads.js'
import { watchGroupIndexes } from './superTabs/parts/indexed.js'
import { startMessageReadStats } from './superTabs/messageReadStats.js'
@@ -156,6 +157,8 @@ async function start(): Promise<void> {
const stopDeliveryCalls = startDeliveryCalls()
// Every minute: how the Swarm read cache served reads (hit, patched, loaded in full and why) and what its checks found.
const stopCacheStats = startCacheStats()
+ // Every minute: the process's memory and what the read cache holds of it (lib/memoryStats.ts; the HPA scales on memory).
+ const stopMemoryStats = startMemoryStats({ cache: () => swarmReadCache()?.size ?? null })
const stopRouteLatency = startRouteLatency(registerRouteLatency(app))
// Every minute: who reads messages how, with the rows each page decoded and showed (superTabs/messageReadStats.ts).
const stopMessageReadStats = startMessageReadStats()
@@ -248,6 +251,7 @@ async function start(): Promise<void> {
stopRetryStats()
stopDeliveryCalls()
stopCacheStats()
+ stopMemoryStats()
stopRouteLatency()
stopMachineReportLog()
stopMessageReadStats()