Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(home): size the connection pool from measured RSS, not a fixed guess

nirholas253ab1701c3a
diff --git a/api/_lib/home/runtime.js b/api/_lib/home/runtime.js
index a1ada77f4..6a8387d44 100644
--- a/api/_lib/home/runtime.js
+++ b/api/_lib/home/runtime.js
@@ -23,6 +23,9 @@
// the source of truth, this pool is a cache, and a cold instance reopening in a
// few hundred milliseconds is the normal path rather than a failure.
+import { readFileSync } from 'node:fs';
+import { totalmem } from 'node:os';
+
import { ERR, HomeBridge, HomeBridgeError } from '@three-ws/home-bridge';
import { assertDialableHomeUrl, HomeUrlError, pinnedHomeSocketFactory } from '../home-url-guard.js';
@@ -61,6 +64,23 @@ const IDLE_MS = 90_000;
const CONNECT_TIMEOUT_MS = 15_000;
/** A socket plus a state map is roughly 1 to 3 MB of heap for a large house. */
const DEFAULT_MAX_CONNECTIONS = 200;
+/**
+ * Measured RSS per pooled connection at a 90/10 small/large house mix: 666 KB for
+ * a 123 entity house, 1.12 MB for a 624 entity one, both amortized at 400 open
+ * connections (docs/ops/home-operations.md, tasks/home/envelope-2026-09-09.json).
+ */
+const RSS_PER_CONNECTION_BYTES = Math.round(0.9 * 666 * 1024 + 0.1 * 1.12 * 1024 ** 2);
+/**
+ * What the rest of the container needs, so the pool never claims it. Cloud
+ * Monitoring `container/memory/utilizations`, 24 hours of per-minute p99 read on
+ * 2026-09-09: max 0.859 of a 4 GiB limit, or 3.44 GiB, with zero home
+ * connections held.
+ */
+const NON_HOME_WORKING_SET_BYTES = Math.round(3.44 * 1024 ** 3);
+/** Sizing the pool to fill the container leaves nothing for the spike that follows it. */
+const MEMORY_UTILIZATION_TARGET = 0.9;
+/** A container too small to back a real pool gets a small lane, never a dead one. */
+const MIN_VIABLE_CONNECTIONS = 25;
/** Consecutive connect failures that open the breaker for one home. */
const BREAKER_THRESHOLD = 5;
/** How long a revoked token or an offline house stops being retried on every page load. */
@@ -101,7 +121,22 @@ export function createHomeRuntime(deps = {}) {
const readAllowed = deps.listAllowedEntities || listAllowedEntities;
const writeHandshake = deps.recordHandshake || recordHandshake;
const now = deps.now || (() => Date.now());
- const maxConnections = deps.maxConnections ?? readMaxConnections();
+ const requestedConnections = deps.maxConnections ?? readMaxConnections();
+ const memoryCap = memoryBackedConnectionCap(
+ deps.containerMemoryLimitBytes === undefined ? containerMemoryLimitBytes() : deps.containerMemoryLimitBytes,
+ );
+ const maxConnections = Math.min(requestedConnections, memoryCap);
+ /**
+ * Null unless this container's memory refused the cap it was asked for. It
+ * rides in `stats()` and out through `/api/healthz`, because the whole reason
+ * this clamp exists is that the drift it corrects was invisible for days.
+ */
+ const pooledCapNote = maxConnections < requestedConnections
+ ? `asked for ${requestedConnections}, clamped to ${maxConnections} by a ${(memoryCap === Infinity ? 0 : Math.round((memoryCap * RSS_PER_CONNECTION_BYTES + NON_HOME_WORKING_SET_BYTES) / MEMORY_UTILIZATION_TARGET / 1024 ** 3 * 10) / 10)} GiB container`
+ : null;
+ if (pooledCapNote) {
+ console.warn(`[home] pool cap ${pooledCapNote}. Raise --memory or lower HOME_MAX_CONNECTIONS; docs/ops/home-operations.md.`);
+ }
const idleMs = deps.idleMs ?? IDLE_MS;
const connectTimeoutMs = deps.connectTimeoutMs ?? CONNECT_TIMEOUT_MS;
const resolveDial = deps.resolveDial || defaultResolveDial;
@@ -339,7 +374,7 @@ export function createHomeRuntime(deps = {}) {
}
let breakersOpen = 0;
for (const breaker of breakers.values()) if (breaker.openedUntil > now()) breakersOpen += 1;
- return { open: entries.size, subscribers, pooledCap: maxConnections, breakersOpen, byStatus, admission: admission.snapshot() };
+ return { open: entries.size, subscribers, pooledCap: maxConnections, pooledCapNote, breakersOpen, byStatus, admission: admission.snapshot() };
}
/**
@@ -763,6 +798,55 @@ function readMaxConnections() {
return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : DEFAULT_MAX_CONNECTIONS;
}
+/**
+ * The container's own memory limit in bytes, or `null` when it has none.
+ *
+ * cgroup v2 first, which is what Cloud Run's second generation execution
+ * environment and every modern container runtime expose, then v1 for anything
+ * older. A host with no limit reports the literal `max` on v2 and a sentinel
+ * near 2^63 on v1, and an unconstrained container reports the whole machine;
+ * all three mean the same thing here, which is that memory is not the bound.
+ */
+export function containerMemoryLimitBytes(readFile = readFileSync, hostBytes = totalmem()) {
+ for (const path of ['/sys/fs/cgroup/memory.max', '/sys/fs/cgroup/memory/memory.limit_in_bytes']) {
+ let raw;
+ try {
+ raw = String(readFile(path, 'utf8')).trim();
+ } catch {
+ continue;
+ }
+ if (raw === 'max') return null;
+ const bytes = Number(raw);
+ if (!Number.isFinite(bytes) || bytes <= 0 || bytes >= hostBytes) return null;
+ return bytes;
+ }
+ return null;
+}
+
+/**
+ * How many pooled connections this container's memory can actually back.
+ *
+ * `HOME_MAX_CONNECTIONS` and `--memory` are set in two different places and have
+ * drifted apart three times. A config-only `gcloud run services update` raised
+ * the container to 8 GiB, and the next full deploy put it back to 4 GiB because
+ * `server/cloudbuild.yaml` passes `--memory` explicitly while nothing in the
+ * deploy names the env var, which merges and survives. Production therefore ran
+ * for days sized for 600 connections on a container that could not hold them,
+ * and an OOM here kills the whole API rather than one lane.
+ *
+ * So the cap is negotiated rather than read: the env var asks, and the container
+ * decides. Every number behind this is measured, not chosen.
+ *
+ * @param {number|null} limitBytes The container limit, or null for unlimited.
+ * @returns {number} The cap, or `Infinity` when memory is not the bound.
+ */
+export function memoryBackedConnectionCap(limitBytes) {
+ if (!limitBytes) return Infinity;
+ const budget = limitBytes * MEMORY_UTILIZATION_TARGET - NON_HOME_WORKING_SET_BYTES;
+ if (budget <= 0) return MIN_VIABLE_CONNECTIONS;
+ return Math.max(MIN_VIABLE_CONNECTIONS, Math.floor(budget / RSS_PER_CONNECTION_BYTES));
+}
+
function withTimeout(promise, ms, makeError) {
let timer;
const timeout = new Promise((_resolve, reject) => {
diff --git a/api/_lib/ops/home-health.js b/api/_lib/ops/home-health.js
index 811aec4b6..bc287b439 100644
--- a/api/_lib/ops/home-health.js
+++ b/api/_lib/ops/home-health.js
@@ -367,6 +367,10 @@ function safeStats() {
// The runtime has spelled its cap both ways while this lane was being
// built. Read either rather than reporting a confident zero.
capacity: s.capacity ?? s.pooledCap ?? 0,
+ // Non-null only when the container's own memory limit refused the cap
+ // HOME_MAX_CONNECTIONS asked for. That drift was invisible for days
+ // twice, so it is reported where an operator already looks.
+ capacityNote: s.pooledCapNote ?? null,
breakersOpen: s.breakersOpen ?? 0,
byStatus: s.byStatus ?? {},
// The admission controller's own gauges, when it is present: the open
@@ -375,7 +379,7 @@ function safeStats() {
rung: s.admission?.rung ?? null,
};
} catch {
- return { open: 0, subscribers: 0, capacity: 0, breakersOpen: 0, byStatus: {}, streams: null, rung: null };
+ return { open: 0, subscribers: 0, capacity: 0, capacityNote: null, breakersOpen: 0, byStatus: {}, streams: null, rung: null };
}
}
diff --git a/docs/ops/home-operations.md b/docs/ops/home-operations.md
index ada9c07af..44a7b1525 100644
--- a/docs/ops/home-operations.md
+++ b/docs/ops/home-operations.md
@@ -332,8 +332,10 @@ asks to unlock a door is served before a user who is watching a dashboard.**
### Each rung, demonstrated
-Run `node scripts/home-load.mjs ladder` for this live. Recorded on 2026-09-03
-with a controller of 4 pooled, 2 unpooled, 6 streams and 8 actions:
+Run `node scripts/home-load.mjs ladder` for this live, with a controller of 4
+pooled, 2 unpooled, 6 streams and 8 actions. Recorded on 2026-09-03 and again on
+2026-09-09, and the two runs are **identical row for row**, all ten of them
+(`.ladder.rows` in the two committed envelope files):
| Step | Admitted | Rung | Detail |
|---|---|---|---|
@@ -615,17 +617,100 @@ passes `--memory` explicitly, so it overwrites the service setting on every
deploy; the env var survived only because `--update-env-vars` merges and nothing
in the deploy names `HOME_MAX_CONNECTIONS`.
-Both halves are fixed:
+Both halves were addressed:
-- The live service is back at 8 GiB (revision `three-ws-api-00419-5tr`,
- 2026-09-09, serving 100% of traffic, `/api/healthz` 200).
+- The service was updated back to 8 GiB (revision `three-ws-api-00419-5tr`,
+ 2026-09-09, `/api/healthz` 200).
- `server/cloudbuild.yaml` now pins `--memory 8Gi` with the measurement in a
- comment beside it, so a full deploy cannot revert it again.
+ comment beside it (commit `a5e522822`).
**A config-only `gcloud run services update` is not durable on its own.** Any
setting this file argues for must also be pinned in the deploy config, or the
next deploy is the thing that undoes it.
+### It reverted a third time, and that is why the runtime now clamps itself
+
+Written the same day, a few hours later, because the fix above was not enough
+and the reason generalizes.
+
+`GET /api/version` reports the live revision as `three-ws-api-00420-ljh`, serving
+commit `880bdcef8`. The 8 GiB pin is commit `a5e522822`, made **ten hours after
+the commit production is serving**, so the pin has never been through a deploy:
+at `880bdcef8` the file still reads `'--memory', '4Gi'`.
+
+```
+$ curl -s https://three.ws/api/version | jq -r '.commitShort, .runtime.revision'
+880bdcef8
+three-ws-api-00420-ljh
+$ git show 880bdcef8:server/cloudbuild.yaml | grep -A1 -- "'--memory'"
+ '--memory',
+ '4Gi',
+```
+
+So a full deploy landed after the config-only update for the second time, and
+`HOME_MAX_CONNECTIONS=600` is still set (production `/api/healthz` reports
+`capacity: 600`) on a container whose deploy config asks for 4 GiB. That is once
+again the one combination the measurement rules out, arrived at by a different
+route than the first time: not a deploy that overwrote the setting, but a deploy
+of a commit made **before** the fix existed.
+
+Pinning the value in one more file would not have prevented this, and a fourth
+place to keep in sync is not a fix. The cap and the memory that backs it are set
+in different systems on different schedules, so the process now negotiates rather
+than trusts:
+
+| Piece | Where |
+|---|---|
+| The cgroup limit this container actually got | `containerMemoryLimitBytes()` in [`api/_lib/home/runtime.js`](../../api/_lib/home/runtime.js) |
+| What that much memory can back | `memoryBackedConnectionCap()`, same file |
+| The clamp and its warning | `createHomeRuntime`, which takes `Math.min` of the two and sizes the admission ladder from the result |
+| Reported as | `pooledCapNote` in `stats()`, surfaced as `home.detail.pool.capacityNote` in `/api/healthz` |
+
+`HOME_MAX_CONNECTIONS` asks; the container decides. The policy is the same
+arithmetic this section already argued, with the same measured inputs:
+
+```
+cap = (limit x 0.90 - 3.44 GiB) / 0.71 MB per connection
+```
+
+`0.90` is a utilization target, because sizing the pool to fill the container
+leaves nothing for the spike that follows. `3.44 GiB` is the measured 24 hour p99
+peak of everything in the container that is not this lane. `0.71 MB` is the
+measured per-connection RSS at a 90/10 small/large house mix.
+
+| Container | Connections it backs | What happens to a 600 cap |
+|---|---|---|
+| 8 GiB | 5,521 | passes untouched |
+| 4 GiB | **234** | clamped to 234, logged, reported in `/api/healthz` |
+| 2 GiB | 25 (the floor) | clamped to the floor |
+| no cgroup limit (dev, tests) | unbounded | passes untouched |
+
+At 4 GiB the lane keeps working at 234 connections an instance, which is 1,404
+live homes across `minScale=6`, instead of risking an OOM that kills the whole
+API container. **The lane degrades; the API does not die.** A container too small
+to size still gets 25 connections rather than zero, because a dead lane is not a
+safer failure than a small one.
+
+Verified by construction rather than by inspection: eight cases in
+[`tests/home-runtime.test.js`](../../tests/home-runtime.test.js) pin the policy at
+each memory size, that the admission ladder is sized from the clamped cap rather
+than the requested one, that a cgroup v1 sentinel and a whole-host limit both
+mean "no bound", and that the clamp announces itself in `stats()`.
+
+**Still owner-gated, and it is one command.** The clamp makes 4 GiB safe, not
+correct. The service should be back at 8 GiB, and the next full deploy of any
+commit at or after `a5e522822` does it permanently. Until then:
+
+```bash
+gcloud run services update three-ws-api --region us-central1 \
+ --project aerial-vehicle-466722-p5 --memory 8Gi
+```
+
+That could not be run from the session that found this: `gcloud` in this
+workspace returned `Reauthentication failed. cannot prompt during
+non-interactive execution`, which is why the live memory limit is stated as what
+the deploy config asks for rather than as a reading off the service.
+
**Verified live on 2026-09-09**, which is the check that proves the number
reached the running code rather than just the service description. The `home`
block in production `/api/healthz` reports the pool reading its own cap:
diff --git a/tests/home-runtime.test.js b/tests/home-runtime.test.js
index 5a8458242..9f1618fd8 100644
--- a/tests/home-runtime.test.js
+++ b/tests/home-runtime.test.js
@@ -15,7 +15,12 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
-import { createHomeRuntime, HOME_RUNTIME_ERR } from '../api/_lib/home/runtime.js';
+import {
+ containerMemoryLimitBytes,
+ createHomeRuntime,
+ HOME_RUNTIME_ERR,
+ memoryBackedConnectionCap,
+} from '../api/_lib/home/runtime.js';
import { HOME_STATUS } from '../api/_lib/home/store.js';
const HOME_ID = '11111111-1111-4111-8111-111111111111';
@@ -603,3 +608,95 @@ describe('home runtime: shutdown', () => {
expect(createBridge.built[0].closeCount).toBe(1);
});
});
+
+// The pool cap and the container's `--memory` are configured in two different
+// places, and they have drifted apart three times in this service's life: a
+// config-only update raised the container to 8 GiB, and the next full deploy put
+// it back to 4 GiB while HOME_MAX_CONNECTIONS=600 survived, because the deploy
+// names one and not the other. These tests pin the negotiation that makes that
+// combination harmless, since an OOM in this container takes the whole API down.
+describe('home runtime: the pool cap is bounded by the memory that backs it', () => {
+ const GiB = 1024 ** 3;
+
+ it('lets 600 connections through on the 8 GiB container the sizing assumes', () => {
+ expect(memoryBackedConnectionCap(8 * GiB)).toBeGreaterThan(600);
+ });
+
+ it('refuses 600 connections on a 4 GiB container', () => {
+ const cap = memoryBackedConnectionCap(4 * GiB);
+ expect(cap).toBeLessThan(600);
+ expect(cap).toBeGreaterThan(100);
+ });
+
+ it('does not bound anything when the container has no memory limit', () => {
+ expect(memoryBackedConnectionCap(null)).toBe(Infinity);
+ });
+
+ it('leaves a working lane rather than a dead one on a container too small to size', () => {
+ expect(memoryBackedConnectionCap(512 * 1024 * 1024)).toBe(25);
+ });
+
+ it('grows monotonically with the memory it is given', () => {
+ const caps = [2, 4, 8, 16, 32].map((gib) => memoryBackedConnectionCap(gib * GiB));
+ for (let i = 1; i < caps.length; i += 1) expect(caps[i]).toBeGreaterThanOrEqual(caps[i - 1]);
+ });
+
+ it('clamps the runtime, resizes the ladder with it, and says so in stats', () => {
+ const runtime = createHomeRuntime({
+ createBridge: countingFactory(),
+ ...storeDeps(),
+ maxConnections: 600,
+ containerMemoryLimitBytes: 4 * GiB,
+ });
+
+ const stats = runtime.stats();
+ expect(stats.pooledCap).toBe(memoryBackedConnectionCap(4 * GiB));
+ expect(stats.pooledCap).toBeLessThan(600);
+ // The ladder must be sized from the clamped cap, or rung 2 would never be
+ // reached and the pool would overrun the memory the clamp just protected.
+ expect(stats.admission.limits.maxPooled).toBe(stats.pooledCap);
+ expect(stats.pooledCapNote).toContain('600');
+ });
+
+ it('says nothing when the container can back what was asked for', () => {
+ const runtime = createHomeRuntime({
+ createBridge: countingFactory(),
+ ...storeDeps(),
+ maxConnections: 600,
+ containerMemoryLimitBytes: 8 * GiB,
+ });
+
+ expect(runtime.stats().pooledCap).toBe(600);
+ expect(runtime.stats().pooledCapNote).toBeNull();
+ });
+
+ it('reads a cgroup v2 limit, and treats an unlimited one as no bound', () => {
+ const host = 64 * GiB;
+ const v2 = (path) => {
+ if (path === '/sys/fs/cgroup/memory.max') return '8589934592\n';
+ throw new Error('ENOENT');
+ };
+ expect(containerMemoryLimitBytes(v2, host)).toBe(8 * GiB);
+
+ const unlimited = (path) => {
+ if (path === '/sys/fs/cgroup/memory.max') return 'max\n';
+ throw new Error('ENOENT');
+ };
+ expect(containerMemoryLimitBytes(unlimited, host)).toBeNull();
+ });
+
+ it('treats a cgroup v1 sentinel and a whole-host limit as no bound', () => {
+ const host = 64 * GiB;
+ const v1 = (path) => {
+ if (path === '/sys/fs/cgroup/memory/memory.limit_in_bytes') return '9223372036854771712\n';
+ throw new Error('ENOENT');
+ };
+ expect(containerMemoryLimitBytes(v1, host)).toBeNull();
+
+ const wholeHost = (path) => {
+ if (path === '/sys/fs/cgroup/memory.max') return String(host);
+ throw new Error('ENOENT');
+ };
+ expect(containerMemoryLimitBytes(wholeHost, host)).toBeNull();
+ });
+});