Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(home): size the connection pool from measured RSS, not a fixed guess
nirholas253ab1701c3a
diff --git a/api/_lib/home/runtime.js b/api/_lib/home/runtime.jsindex a1ada77f4..6a8387d44 100644--- a/api/_lib/home/runtime.js+++ b/api/_lib/home/runtime.js@@ -23,6 +23,9 @@// the source of truth, this pool is a cache, and a cold instance reopening in a// few hundred milliseconds is the normal path rather than a failure.+import { readFileSync } from 'node:fs';+import { totalmem } from 'node:os';+import { ERR, HomeBridge, HomeBridgeError } from '@three-ws/home-bridge';import { assertDialableHomeUrl, HomeUrlError, pinnedHomeSocketFactory } from '../home-url-guard.js';@@ -61,6 +64,23 @@ const IDLE_MS = 90_000;const CONNECT_TIMEOUT_MS = 15_000;/** A socket plus a state map is roughly 1 to 3 MB of heap for a large house. */const DEFAULT_MAX_CONNECTIONS = 200;+/**+ * Measured RSS per pooled connection at a 90/10 small/large house mix: 666 KB for+ * a 123 entity house, 1.12 MB for a 624 entity one, both amortized at 400 open+ * connections (docs/ops/home-operations.md, tasks/home/envelope-2026-09-09.json).+ */+const RSS_PER_CONNECTION_BYTES = Math.round(0.9 * 666 * 1024 + 0.1 * 1.12 * 1024 ** 2);+/**+ * What the rest of the container needs, so the pool never claims it. Cloud+ * Monitoring `container/memory/utilizations`, 24 hours of per-minute p99 read on+ * 2026-09-09: max 0.859 of a 4 GiB limit, or 3.44 GiB, with zero home+ * connections held.+ */+const NON_HOME_WORKING_SET_BYTES = Math.round(3.44 * 1024 ** 3);+/** Sizing the pool to fill the container leaves nothing for the spike that follows it. */+const MEMORY_UTILIZATION_TARGET = 0.9;+/** A container too small to back a real pool gets a small lane, never a dead one. */+const MIN_VIABLE_CONNECTIONS = 25;/** Consecutive connect failures that open the breaker for one home. */const BREAKER_THRESHOLD = 5;/** How long a revoked token or an offline house stops being retried on every page load. */@@ -101,7 +121,22 @@ export function createHomeRuntime(deps = {}) {const readAllowed = deps.listAllowedEntities || listAllowedEntities;const writeHandshake = deps.recordHandshake || recordHandshake;const now = deps.now || (() => Date.now());- const maxConnections = deps.maxConnections ?? readMaxConnections();+ const requestedConnections = deps.maxConnections ?? readMaxConnections();+ const memoryCap = memoryBackedConnectionCap(+ deps.containerMemoryLimitBytes === undefined ? containerMemoryLimitBytes() : deps.containerMemoryLimitBytes,+ );+ const maxConnections = Math.min(requestedConnections, memoryCap);+ /**+ * Null unless this container's memory refused the cap it was asked for. It+ * rides in `stats()` and out through `/api/healthz`, because the whole reason+ * this clamp exists is that the drift it corrects was invisible for days.+ */+ const pooledCapNote = maxConnections < requestedConnections+ ? `asked for ${requestedConnections}, clamped to ${maxConnections} by a ${(memoryCap === Infinity ? 0 : Math.round((memoryCap * RSS_PER_CONNECTION_BYTES + NON_HOME_WORKING_SET_BYTES) / MEMORY_UTILIZATION_TARGET / 1024 ** 3 * 10) / 10)} GiB container`+ : null;+ if (pooledCapNote) {+ console.warn(`[home] pool cap ${pooledCapNote}. Raise --memory or lower HOME_MAX_CONNECTIONS; docs/ops/home-operations.md.`);+ }const idleMs = deps.idleMs ?? IDLE_MS;const connectTimeoutMs = deps.connectTimeoutMs ?? CONNECT_TIMEOUT_MS;const resolveDial = deps.resolveDial || defaultResolveDial;@@ -339,7 +374,7 @@ export function createHomeRuntime(deps = {}) {}let breakersOpen = 0;for (const breaker of breakers.values()) if (breaker.openedUntil > now()) breakersOpen += 1;- return { open: entries.size, subscribers, pooledCap: maxConnections, breakersOpen, byStatus, admission: admission.snapshot() };+ return { open: entries.size, subscribers, pooledCap: maxConnections, pooledCapNote, breakersOpen, byStatus, admission: admission.snapshot() };}/**@@ -763,6 +798,55 @@ function readMaxConnections() {return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : DEFAULT_MAX_CONNECTIONS;}+/**+ * The container's own memory limit in bytes, or `null` when it has none.+ *+ * cgroup v2 first, which is what Cloud Run's second generation execution+ * environment and every modern container runtime expose, then v1 for anything+ * older. A host with no limit reports the literal `max` on v2 and a sentinel+ * near 2^63 on v1, and an unconstrained container reports the whole machine;+ * all three mean the same thing here, which is that memory is not the bound.+ */+export function containerMemoryLimitBytes(readFile = readFileSync, hostBytes = totalmem()) {+ for (const path of ['/sys/fs/cgroup/memory.max', '/sys/fs/cgroup/memory/memory.limit_in_bytes']) {+ let raw;+ try {+ raw = String(readFile(path, 'utf8')).trim();+ } catch {+ continue;+ }+ if (raw === 'max') return null;+ const bytes = Number(raw);+ if (!Number.isFinite(bytes) || bytes <= 0 || bytes >= hostBytes) return null;+ return bytes;+ }+ return null;+}++/**+ * How many pooled connections this container's memory can actually back.+ *+ * `HOME_MAX_CONNECTIONS` and `--memory` are set in two different places and have+ * drifted apart three times. A config-only `gcloud run services update` raised+ * the container to 8 GiB, and the next full deploy put it back to 4 GiB because+ * `server/cloudbuild.yaml` passes `--memory` explicitly while nothing in the+ * deploy names the env var, which merges and survives. Production therefore ran+ * for days sized for 600 connections on a container that could not hold them,+ * and an OOM here kills the whole API rather than one lane.+ *+ * So the cap is negotiated rather than read: the env var asks, and the container+ * decides. Every number behind this is measured, not chosen.+ *+ * @param {number|null} limitBytes The container limit, or null for unlimited.+ * @returns {number} The cap, or `Infinity` when memory is not the bound.+ */+export function memoryBackedConnectionCap(limitBytes) {+ if (!limitBytes) return Infinity;+ const budget = limitBytes * MEMORY_UTILIZATION_TARGET - NON_HOME_WORKING_SET_BYTES;+ if (budget <= 0) return MIN_VIABLE_CONNECTIONS;+ return Math.max(MIN_VIABLE_CONNECTIONS, Math.floor(budget / RSS_PER_CONNECTION_BYTES));+}+function withTimeout(promise, ms, makeError) {let timer;const timeout = new Promise((_resolve, reject) => {diff --git a/api/_lib/ops/home-health.js b/api/_lib/ops/home-health.jsindex 811aec4b6..bc287b439 100644--- a/api/_lib/ops/home-health.js+++ b/api/_lib/ops/home-health.js@@ -367,6 +367,10 @@ function safeStats() {// The runtime has spelled its cap both ways while this lane was being// built. Read either rather than reporting a confident zero.capacity: s.capacity ?? s.pooledCap ?? 0,+ // Non-null only when the container's own memory limit refused the cap+ // HOME_MAX_CONNECTIONS asked for. That drift was invisible for days+ // twice, so it is reported where an operator already looks.+ capacityNote: s.pooledCapNote ?? null,breakersOpen: s.breakersOpen ?? 0,byStatus: s.byStatus ?? {},// The admission controller's own gauges, when it is present: the open@@ -375,7 +379,7 @@ function safeStats() {rung: s.admission?.rung ?? null,};} catch {- return { open: 0, subscribers: 0, capacity: 0, breakersOpen: 0, byStatus: {}, streams: null, rung: null };+ return { open: 0, subscribers: 0, capacity: 0, capacityNote: null, breakersOpen: 0, byStatus: {}, streams: null, rung: null };}}diff --git a/docs/ops/home-operations.md b/docs/ops/home-operations.mdindex ada9c07af..44a7b1525 100644--- a/docs/ops/home-operations.md+++ b/docs/ops/home-operations.md@@ -332,8 +332,10 @@ asks to unlock a door is served before a user who is watching a dashboard.**### Each rung, demonstrated-Run `node scripts/home-load.mjs ladder` for this live. Recorded on 2026-09-03-with a controller of 4 pooled, 2 unpooled, 6 streams and 8 actions:+Run `node scripts/home-load.mjs ladder` for this live, with a controller of 4+pooled, 2 unpooled, 6 streams and 8 actions. Recorded on 2026-09-03 and again on+2026-09-09, and the two runs are **identical row for row**, all ten of them+(`.ladder.rows` in the two committed envelope files):| Step | Admitted | Rung | Detail ||---|---|---|---|@@ -615,17 +617,100 @@ passes `--memory` explicitly, so it overwrites the service setting on everydeploy; the env var survived only because `--update-env-vars` merges and nothingin the deploy names `HOME_MAX_CONNECTIONS`.-Both halves are fixed:+Both halves were addressed:-- The live service is back at 8 GiB (revision `three-ws-api-00419-5tr`,- 2026-09-09, serving 100% of traffic, `/api/healthz` 200).+- The service was updated back to 8 GiB (revision `three-ws-api-00419-5tr`,+ 2026-09-09, `/api/healthz` 200).- `server/cloudbuild.yaml` now pins `--memory 8Gi` with the measurement in a- comment beside it, so a full deploy cannot revert it again.+ comment beside it (commit `a5e522822`).**A config-only `gcloud run services update` is not durable on its own.** Anysetting this file argues for must also be pinned in the deploy config, or thenext deploy is the thing that undoes it.+### It reverted a third time, and that is why the runtime now clamps itself++Written the same day, a few hours later, because the fix above was not enough+and the reason generalizes.++`GET /api/version` reports the live revision as `three-ws-api-00420-ljh`, serving+commit `880bdcef8`. The 8 GiB pin is commit `a5e522822`, made **ten hours after+the commit production is serving**, so the pin has never been through a deploy:+at `880bdcef8` the file still reads `'--memory', '4Gi'`.++```+$ curl -s https://three.ws/api/version | jq -r '.commitShort, .runtime.revision'+880bdcef8+three-ws-api-00420-ljh+$ git show 880bdcef8:server/cloudbuild.yaml | grep -A1 -- "'--memory'"+ '--memory',+ '4Gi',+```++So a full deploy landed after the config-only update for the second time, and+`HOME_MAX_CONNECTIONS=600` is still set (production `/api/healthz` reports+`capacity: 600`) on a container whose deploy config asks for 4 GiB. That is once+again the one combination the measurement rules out, arrived at by a different+route than the first time: not a deploy that overwrote the setting, but a deploy+of a commit made **before** the fix existed.++Pinning the value in one more file would not have prevented this, and a fourth+place to keep in sync is not a fix. The cap and the memory that backs it are set+in different systems on different schedules, so the process now negotiates rather+than trusts:++| Piece | Where |+|---|---|+| The cgroup limit this container actually got | `containerMemoryLimitBytes()` in [`api/_lib/home/runtime.js`](../../api/_lib/home/runtime.js) |+| What that much memory can back | `memoryBackedConnectionCap()`, same file |+| The clamp and its warning | `createHomeRuntime`, which takes `Math.min` of the two and sizes the admission ladder from the result |+| Reported as | `pooledCapNote` in `stats()`, surfaced as `home.detail.pool.capacityNote` in `/api/healthz` |++`HOME_MAX_CONNECTIONS` asks; the container decides. The policy is the same+arithmetic this section already argued, with the same measured inputs:++```+cap = (limit x 0.90 - 3.44 GiB) / 0.71 MB per connection+```++`0.90` is a utilization target, because sizing the pool to fill the container+leaves nothing for the spike that follows. `3.44 GiB` is the measured 24 hour p99+peak of everything in the container that is not this lane. `0.71 MB` is the+measured per-connection RSS at a 90/10 small/large house mix.++| Container | Connections it backs | What happens to a 600 cap |+|---|---|---|+| 8 GiB | 5,521 | passes untouched |+| 4 GiB | **234** | clamped to 234, logged, reported in `/api/healthz` |+| 2 GiB | 25 (the floor) | clamped to the floor |+| no cgroup limit (dev, tests) | unbounded | passes untouched |++At 4 GiB the lane keeps working at 234 connections an instance, which is 1,404+live homes across `minScale=6`, instead of risking an OOM that kills the whole+API container. **The lane degrades; the API does not die.** A container too small+to size still gets 25 connections rather than zero, because a dead lane is not a+safer failure than a small one.++Verified by construction rather than by inspection: eight cases in+[`tests/home-runtime.test.js`](../../tests/home-runtime.test.js) pin the policy at+each memory size, that the admission ladder is sized from the clamped cap rather+than the requested one, that a cgroup v1 sentinel and a whole-host limit both+mean "no bound", and that the clamp announces itself in `stats()`.++**Still owner-gated, and it is one command.** The clamp makes 4 GiB safe, not+correct. The service should be back at 8 GiB, and the next full deploy of any+commit at or after `a5e522822` does it permanently. Until then:++```bash+gcloud run services update three-ws-api --region us-central1 \+ --project aerial-vehicle-466722-p5 --memory 8Gi+```++That could not be run from the session that found this: `gcloud` in this+workspace returned `Reauthentication failed. cannot prompt during+non-interactive execution`, which is why the live memory limit is stated as what+the deploy config asks for rather than as a reading off the service.+**Verified live on 2026-09-09**, which is the check that proves the numberreached the running code rather than just the service description. The `home`block in production `/api/healthz` reports the pool reading its own cap:diff --git a/tests/home-runtime.test.js b/tests/home-runtime.test.jsindex 5a8458242..9f1618fd8 100644--- a/tests/home-runtime.test.js+++ b/tests/home-runtime.test.js@@ -15,7 +15,12 @@import { beforeEach, describe, expect, it, vi } from 'vitest';-import { createHomeRuntime, HOME_RUNTIME_ERR } from '../api/_lib/home/runtime.js';+import {+ containerMemoryLimitBytes,+ createHomeRuntime,+ HOME_RUNTIME_ERR,+ memoryBackedConnectionCap,+} from '../api/_lib/home/runtime.js';import { HOME_STATUS } from '../api/_lib/home/store.js';const HOME_ID = '11111111-1111-4111-8111-111111111111';@@ -603,3 +608,95 @@ describe('home runtime: shutdown', () => {expect(createBridge.built[0].closeCount).toBe(1);});});++// The pool cap and the container's `--memory` are configured in two different+// places, and they have drifted apart three times in this service's life: a+// config-only update raised the container to 8 GiB, and the next full deploy put+// it back to 4 GiB while HOME_MAX_CONNECTIONS=600 survived, because the deploy+// names one and not the other. These tests pin the negotiation that makes that+// combination harmless, since an OOM in this container takes the whole API down.+describe('home runtime: the pool cap is bounded by the memory that backs it', () => {+ const GiB = 1024 ** 3;++ it('lets 600 connections through on the 8 GiB container the sizing assumes', () => {+ expect(memoryBackedConnectionCap(8 * GiB)).toBeGreaterThan(600);+ });++ it('refuses 600 connections on a 4 GiB container', () => {+ const cap = memoryBackedConnectionCap(4 * GiB);+ expect(cap).toBeLessThan(600);+ expect(cap).toBeGreaterThan(100);+ });++ it('does not bound anything when the container has no memory limit', () => {+ expect(memoryBackedConnectionCap(null)).toBe(Infinity);+ });++ it('leaves a working lane rather than a dead one on a container too small to size', () => {+ expect(memoryBackedConnectionCap(512 * 1024 * 1024)).toBe(25);+ });++ it('grows monotonically with the memory it is given', () => {+ const caps = [2, 4, 8, 16, 32].map((gib) => memoryBackedConnectionCap(gib * GiB));+ for (let i = 1; i < caps.length; i += 1) expect(caps[i]).toBeGreaterThanOrEqual(caps[i - 1]);+ });++ it('clamps the runtime, resizes the ladder with it, and says so in stats', () => {+ const runtime = createHomeRuntime({+ createBridge: countingFactory(),+ ...storeDeps(),+ maxConnections: 600,+ containerMemoryLimitBytes: 4 * GiB,+ });++ const stats = runtime.stats();+ expect(stats.pooledCap).toBe(memoryBackedConnectionCap(4 * GiB));+ expect(stats.pooledCap).toBeLessThan(600);+ // The ladder must be sized from the clamped cap, or rung 2 would never be+ // reached and the pool would overrun the memory the clamp just protected.+ expect(stats.admission.limits.maxPooled).toBe(stats.pooledCap);+ expect(stats.pooledCapNote).toContain('600');+ });++ it('says nothing when the container can back what was asked for', () => {+ const runtime = createHomeRuntime({+ createBridge: countingFactory(),+ ...storeDeps(),+ maxConnections: 600,+ containerMemoryLimitBytes: 8 * GiB,+ });++ expect(runtime.stats().pooledCap).toBe(600);+ expect(runtime.stats().pooledCapNote).toBeNull();+ });++ it('reads a cgroup v2 limit, and treats an unlimited one as no bound', () => {+ const host = 64 * GiB;+ const v2 = (path) => {+ if (path === '/sys/fs/cgroup/memory.max') return '8589934592\n';+ throw new Error('ENOENT');+ };+ expect(containerMemoryLimitBytes(v2, host)).toBe(8 * GiB);++ const unlimited = (path) => {+ if (path === '/sys/fs/cgroup/memory.max') return 'max\n';+ throw new Error('ENOENT');+ };+ expect(containerMemoryLimitBytes(unlimited, host)).toBeNull();+ });++ it('treats a cgroup v1 sentinel and a whole-host limit as no bound', () => {+ const host = 64 * GiB;+ const v1 = (path) => {+ if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') return '9223372036854771712\n';+ throw new Error('ENOENT');+ };+ expect(containerMemoryLimitBytes(v1, host)).toBeNull();++ const wholeHost = (path) => {+ if (path === '/sys/fs/cgroup/memory.max') return String(host);+ throw new Error('ENOENT');+ };+ expect(containerMemoryLimitBytes(wholeHost, host)).toBeNull();+ });+});