Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
#!/usr/bin/env node
/**
* The Home lane's load harness.
*
* Everything in docs/ops/home-operations.md is a number this script printed. It
* opens real WebSocket connections, with real long-lived access tokens, to real
* Home Assistant containers started by scripts/home-fleet.mjs, and it measures
* what they cost this process.
*
* It will only ever talk to the fleet manifest's containers, which live on
* 127.0.0.1. Never point a load harness at somebody's house.
*
* node scripts/home-fleet.mjs up --homes 12 --big 1
* node --expose-gc scripts/home-load.mjs all --out tasks/home/envelope-2026-09-03.json
*
* Individual measurements, all of which `all` runs:
*
* coldstart process start to first connected home, on a cold process
* connections heap, RSS, file descriptors and action p95 at a connection count
* burst CPU absorbed by a 100-entity update burst, and the coalescing
* sse the cost of one SSE subscriber, separated from the connection
* gate under saturation, a guarded action still demands a human
*/
import { spawn } from 'node:child_process';
import { readdirSync } from 'node:fs';
import { mkdir, writeFile } from 'node:fs/promises';
import { createServer } from 'node:http';
import os from 'node:os';
import path from 'node:path';
import process from 'node:process';
import { fileURLToPath } from 'node:url';
import { createAdmissionController, requiresConfirmation } from '../api/_lib/home/admission.js';
import { HomeBridge } from '../packages/home-bridge/src/index.js';
import { classifyCall } from '../packages/home-bridge/src/safety.js';
import { readFleet } from './home-fleet.mjs';
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// ---------------------------------------------------------------- instruments
/**
* A heap reading that means something.
*
* Without a forced collection this measures garbage as readily as it measures
* connections, and the answer moves by tens of megabytes between runs. The
* script refuses to report a heap number at all if it was started without
* --expose-gc, because a number nobody can reproduce is worse than no number.
*/
function heapNow() {
if (typeof global.gc !== 'function') {
throw new Error('run this with --expose-gc, or the heap numbers are noise: node --expose-gc scripts/home-load.mjs ...');
}
global.gc();
global.gc();
const mem = process.memoryUsage();
return { heapUsed: mem.heapUsed, rss: mem.rss, external: mem.external, arrayBuffers: mem.arrayBuffers };
}
/** Open file descriptors held by this process, which is where sockets show up. */
function fdCount() {
try {
return readdirSync('/proc/self/fd').length;
} catch {
return null;
}
}
function percentile(sorted, p) {
if (!sorted.length) return null;
const index = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
return sorted[Math.max(0, index)];
}
function summarize(samples) {
const sorted = [...samples].sort((a, b) => a - b);
return {
n: sorted.length,
min: sorted[0] ?? null,
p50: percentile(sorted, 50),
p95: percentile(sorted, 95),
p99: percentile(sorted, 99),
max: sorted[sorted.length - 1] ?? null,
mean: sorted.length ? Number((sorted.reduce((a, b) => a + b, 0) / sorted.length).toFixed(2)) : null,
};
}
const bytesPer = (delta, n) => (n ? Math.round(delta / n) : null);
// ---------------------------------------------------------------- fleet helpers
function pickHouses(fleet, { big = false } = {}) {
const houses = fleet.houses.filter((h) => Boolean(h.big) === big);
if (!houses.length) throw new Error(`the fleet has no ${big ? 'large' : 'small'} house. Re-run home-fleet.mjs up with --big 1.`);
return houses;
}
/**
* Open `count` bridges spread round robin across `houses`.
*
* Round robin rather than one-house-per-connection because the thing being
* measured is what a connection costs THIS process. Which container answers is
* the fixture; how many sockets, states maps and room graphs we hold is the
* product. Where a measurement does depend on the far side (action latency,
* chaos isolation) the script says so.
*/
async function openBridges(houses, count, { onOpen } = {}) {
const bridges = [];
const failures = [];
// A storm of simultaneous handshakes measures the handshake, not the steady
// state. Twenty at a time keeps the containers honest without serializing.
const WAVE = 20;
for (let i = 0; i < count; i += WAVE) {
const wave = [];
for (let j = i; j < Math.min(i + WAVE, count); j++) {
const house = houses[j % houses.length];
const bridge = new HomeBridge({ baseUrl: house.baseUrl, token: house.token });
wave.push(
bridge.connect().then(
() => {
bridges.push(bridge);
onOpen?.(bridges.length);
},
(err) => {
failures.push(err.message);
bridge.close();
},
),
);
}
await Promise.all(wave);
}
return { bridges, failures };
}
function closeAll(bridges) {
for (const bridge of bridges) {
try {
bridge.close();
} catch {
// A bridge whose socket already died still has to be released.
}
}
}
function firstEntity(bridge, prefix) {
return Object.keys(bridge.states).find((id) => id.startsWith(prefix)) || null;
}
// ---------------------------------------------------------------- measurements
/**
* Cold start: how long a brand new instance takes to have one live home.
*
* This is the number that decides whether the lane needs minScale above zero, so
* it is measured on a genuinely cold process (the caller re-execs this script)
* and split into the two halves that have different fixes: our module graph
* loading, and the Home Assistant handshake.
*/
async function measureColdStart(fleet) {
const house = pickHouses(fleet)[0];
const processStart = performance.now() - Math.round(process.uptime() * 1000);
const moduleReady = performance.now();
const t0 = performance.now();
const bridge = new HomeBridge({ baseUrl: house.baseUrl, token: house.token });
await bridge.connect();
const connectMs = performance.now() - t0;
const entities = Object.keys(bridge.states).length;
const rooms = bridge.graph.rooms.length;
bridge.close();
return {
bootToModulesReadyMs: Number((moduleReady - processStart).toFixed(1)),
connectMs: Number(connectMs.toFixed(1)),
totalToFirstConnectedHomeMs: Number((moduleReady - processStart + connectMs).toFixed(1)),
entities,
rooms,
note: 'Measured in a freshly spawned node process. bootToModulesReadyMs covers node start plus this harness module graph; the API container loads more than this and its own cold start is measured separately in docs/ops/home-operations.md.',
};
}
/**
* The per-connection cost at one connection count.
*
* Reports the delta from an idle baseline taken in the same process moments
* earlier, so the answer is "what did these connections add", not "how big is
* node".
*/
async function measureConnections(fleet, { count, big = false, actionSamples = 40 }) {
const houses = pickHouses(fleet, { big });
const before = heapNow();
const fdBefore = fdCount();
const cpuBefore = process.cpuUsage();
const t0 = performance.now();
const { bridges, failures } = await openBridges(houses, count);
const openMs = performance.now() - t0;
// Let the state subscriptions settle and the graph coalescing quiesce, so the
// heap reading is of a steady connection rather than of a handshake.
await sleep(4000);
const after = heapNow();
const fdAfter = fdCount();
const idleCpu = process.cpuUsage(cpuBefore);
// Steady-state cost: what these connections consume while nothing is asked of
// them. The demo integration pushes state on its own, so this is not zero and
// should not be.
const idleWindowStart = process.cpuUsage();
await sleep(10_000);
const idleWindow = process.cpuUsage(idleWindowStart);
const entities = bridges.reduce((n, b) => n + Object.keys(b.states).length, 0);
const rooms = bridges.reduce((n, b) => n + b.graph.rooms.length, 0);
const actions = await measureActionLatency(bridges, actionSamples);
// Drop every reference before the residual reading, or this array keeps all
// N bridges (and their entity states and room graphs) alive and the harness
// reports its own scope as a leak. That is exactly what the first run did:
// 188KB per connection "retained" was this variable.
closeAll(bridges);
const connected = bridges.length;
bridges.length = 0;
// Two collections and a real pause, because a socket's buffers are released
// on a later tick than its object graph. Reading straight after close reports
// a leak that is not there.
await sleep(4000);
heapNow();
await sleep(1000);
const afterClose = heapNow();
return {
requested: count,
connected,
// Per measurement, not per run: this box is shared, and a p95 taken at load
// 30 is a different number from the same p95 taken at load 8. A reader who
// cannot see that cannot tell contention from a regression.
loadAverage: os.loadavg().map((n) => Number(n.toFixed(2))),
failures: failures.slice(0, 5),
house: big ? 'large' : 'small',
entitiesHeld: entities,
entitiesPerConnection: connected ? Math.round(entities / connected) : null,
roomsHeld: rooms,
openMs: Number(openMs.toFixed(1)),
openMsPerConnection: connected ? Number((openMs / connected).toFixed(2)) : null,
heapUsedDelta: after.heapUsed - before.heapUsed,
heapPerConnection: bytesPer(after.heapUsed - before.heapUsed, connected),
rssDelta: after.rss - before.rss,
rssPerConnection: bytesPer(after.rss - before.rss, connected),
externalDelta: after.external - before.external,
fdBefore,
fdAfter,
fdPerConnection: connected && fdAfter != null && fdBefore != null ? Number(((fdAfter - fdBefore) / connected).toFixed(2)) : null,
openCpuMs: Number(((idleCpu.user + idleCpu.system) / 1000).toFixed(1)),
idleCpuMsPer10s: Number(((idleWindow.user + idleWindow.system) / 1000).toFixed(1)),
idleCpuMsPerConnectionPerMinute: connected
? Number((((idleWindow.user + idleWindow.system) / 1000 / connected) * 6).toFixed(3))
: null,
actionLatencyMs: actions,
heapAfterCloseDelta: afterClose.heapUsed - before.heapUsed,
heapRetainedPerConnection: bytesPer(afterClose.heapUsed - before.heapUsed, connected),
fdAfterClose: fdCount(),
};
}
/**
* Action latency, measured the way a user experiences it: the call goes out over
* the same socket the state comes back on, and the sample is the round trip Home
* Assistant took to acknowledge it.
*/
async function measureActionLatency(bridges, samples) {
const usable = bridges.filter((b) => firstEntity(b, 'light.'));
if (!usable.length) return { n: 0, note: 'no light entity in this fleet' };
const durations = [];
for (let i = 0; i < samples; i++) {
const bridge = usable[i % usable.length];
const entityId = firstEntity(bridge, 'light.');
const t = performance.now();
try {
await bridge.call('light', 'turn_on', { entity_id: entityId, brightness: 20 + (i % 200) });
durations.push(Number((performance.now() - t).toFixed(2)));
} catch {
// A refused or failed call is not a latency sample.
}
}
return summarize(durations);
}
/**
* CPU absorbed by a burst of 100 entity updates.
*
* The big house carries 500 helper entities precisely so this can drive a known
* number of real state changes rather than waiting for the demo integration to
* produce some. The interesting output is not only the CPU: it is how many graph
* rebuilds 100 updates collapsed into, which is the coalescing window doing its
* job or failing to.
*/
async function measureBurst(fleet, { updates = 100, connections = 1 } = {}) {
const houses = pickHouses(fleet, { big: true });
const { bridges } = await openBridges(houses, connections);
if (!bridges.length) throw new Error('could not open a bridge to the large house');
const driver = bridges[0];
await sleep(2000);
const targets = Object.keys(driver.states)
.filter((id) => id.startsWith('input_number.load_level_'))
.slice(0, updates);
if (targets.length < updates) {
throw new Error(`the large house has ${targets.length} drivable helpers, need ${updates}. Re-run home-fleet.mjs up --big 1.`);
}
let rebuilds = 0;
const stops = bridges.map((b) => b.on('graph', () => rebuilds++));
const seen = new Set();
const watchStops = bridges.map((b) =>
b.on('graph', () => {
for (const id of targets) if (b.states[id]?.state !== undefined) seen.add(id);
}),
);
global.gc();
const cpuBefore = process.cpuUsage();
const heapBefore = process.memoryUsage().heapUsed;
const t0 = performance.now();
// The value must DIFFER from the entity's current one or Home Assistant
// records no state change, pushes nothing, and the burst measures an empty
// socket. A first run happens to differ; a second run against the same house
// writes the same numbers back and silently measures nothing, which is how
// this harness first reported zero graph rebuilds for a hundred updates.
await Promise.all(
targets.map((entityId) => {
const current = Number(driver.states[entityId]?.state ?? 0);
return driver.call('input_number', 'set_value', { entity_id: entityId, value: (current % 100) + 1 });
}),
);
const dispatchMs = performance.now() - t0;
// Wait for the graph to stop moving, so the CPU number includes the rebuilds
// the burst caused rather than only the calls that caused them.
await sleep(3000);
const cpu = process.cpuUsage(cpuBefore);
const absorbMs = performance.now() - t0;
global.gc();
const heapAfter = process.memoryUsage().heapUsed;
for (const stop of [...stops, ...watchStops]) stop();
const entityCount = Object.keys(driver.states).length;
const roomCount = driver.graph.rooms.length;
// The pure cost of one rebuild, separated from everything around it, so the
// extrapolation later has a real per-rebuild unit rather than a blended one.
const { buildHomeGraph } = await import('../packages/home-bridge/src/rooms.js');
const registries = driver.registries;
const rebuildSamples = [];
for (let i = 0; i < 30; i++) {
const t = performance.now();
buildHomeGraph({ ...registries, states: driver.states });
rebuildSamples.push(Number((performance.now() - t).toFixed(3)));
}
closeAll(bridges);
return {
connectionsHeld: bridges.length,
houseEntities: entityCount,
houseRooms: roomCount,
updates,
dispatchMs: Number(dispatchMs.toFixed(1)),
absorbMs: Number(absorbMs.toFixed(1)),
cpuMs: Number(((cpu.user + cpu.system) / 1000).toFixed(1)),
cpuMsPerUpdate: Number(((cpu.user + cpu.system) / 1000 / updates).toFixed(3)),
graphRebuilds: rebuilds,
coalescingRatio: Number((updates / Math.max(1, rebuilds)).toFixed(2)),
heapDeltaAcrossBurst: heapAfter - heapBefore,
graphRebuildMs: summarize(rebuildSamples),
note: 'cpuMs covers the whole absorb window: the 100 outbound service calls, the state pushes they caused, and every graph rebuild that survived coalescing.',
};
}
/**
* The cost of an SSE subscriber, separated from the cost of a connection.
*
* These are two different resources and the envelope has to price them apart: a
* house with twelve people watching it is one WebSocket and twelve HTTP
* responses held open. This measures the second half, with a real HTTP server,
* real client sockets and the real serialization of a real room graph, fed by a
* real house. The production endpoint is api/home/*; the cost measured here is
* the shape it has, which is what the sizing decision needs.
*/
async function measureSse(fleet, { subscribers = 200, seconds = 10 } = {}) {
const houses = pickHouses(fleet);
const { bridges } = await openBridges(houses, 1);
if (!bridges.length) throw new Error('could not open a bridge for the SSE measurement');
const source = bridges[0];
await sleep(1500);
const clients = new Set();
let framesWritten = 0;
let bytesWritten = 0;
const server = createServer((req, res) => {
res.writeHead(200, {
'content-type': 'text/event-stream',
'cache-control': 'no-cache',
connection: 'keep-alive',
});
res.write(`event: ready\ndata: ${JSON.stringify({ rooms: source.graph.rooms.length })}\n\n`);
clients.add(res);
req.on('close', () => clients.delete(res));
});
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
const port = server.address().port;
const stopFanout = source.on('graph', (graph) => {
const frame = `event: graph\ndata: ${JSON.stringify(graph)}\n\n`;
for (const res of clients) {
res.write(frame);
framesWritten++;
bytesWritten += frame.length;
}
});
const before = heapNow();
const fdBefore = fdCount();
const controllers = [];
for (let i = 0; i < subscribers; i++) {
const controller = new AbortController();
controllers.push(controller);
fetch(`http://127.0.0.1:${port}/home/stream`, { signal: controller.signal })
.then((res) => res.body.pipeTo(new WritableStream({ write() {} }), { signal: controller.signal }))
.catch(() => {
// An aborted stream at teardown is the expected end, not a failure.
});
}
// Give every subscriber time to actually be connected before reading a heap.
for (let i = 0; i < 100 && clients.size < subscribers; i++) await sleep(100);
const attached = clients.size;
const after = heapNow();
const fdAfter = fdCount();
// Drive real state changes for the whole window. Without this the fixture
// house simply does not change during a ten second sample, no graph event
// fires, and the measurement reports the cost of holding a socket while
// silently omitting the cost of writing to it: the first run of this measured
// exactly zero frames and looked like a bargain.
const driveTargets = Object.keys(source.states).filter((id) => id.startsWith('light.'));
const cpuBefore = process.cpuUsage();
const driveUntil = Date.now() + seconds * 1000;
let drives = 0;
while (Date.now() < driveUntil) {
const entityId = driveTargets[drives % driveTargets.length];
await source.call('light', 'turn_on', { entity_id: entityId, brightness: 20 + (drives * 17) % 220 }).catch(() => {});
drives++;
await sleep(500);
}
const cpu = process.cpuUsage(cpuBefore);
stopFanout();
for (const controller of controllers) controller.abort();
for (const res of clients) res.end();
await new Promise((resolve) => server.close(resolve));
closeAll(bridges);
return {
subscribersRequested: subscribers,
subscribersAttached: attached,
heapDelta: after.heapUsed - before.heapUsed,
heapPerSubscriber: bytesPer(after.heapUsed - before.heapUsed, attached),
rssDelta: after.rss - before.rss,
rssPerSubscriber: bytesPer(after.rss - before.rss, attached),
// Two descriptors per subscriber here because both ends of every socket are
// in this one process. In production the far end is a browser, so the
// server side of a subscriber costs one.
fdPerSubscriberBothEnds: attached && fdAfter != null ? Number(((fdAfter - fdBefore) / attached).toFixed(2)) : null,
fdPerSubscriberServerSide: attached && fdAfter != null ? Number((((fdAfter - fdBefore) / attached) / 2).toFixed(2)) : null,
stateChangesDriven: drives,
framesWritten,
framesPerSubscriber: attached ? Number((framesWritten / attached).toFixed(2)) : null,
bytesWritten,
bytesPerFrame: framesWritten ? Math.round(bytesWritten / framesWritten) : null,
windowSeconds: seconds,
cpuMsInWindow: Number(((cpu.user + cpu.system) / 1000).toFixed(1)),
cpuMsPerSubscriberPerMinute: attached ? Number((((cpu.user + cpu.system) / 1000 / attached) * (60 / seconds)).toFixed(3)) : null,
note: 'One live house fanning its real room graph to N real HTTP subscribers over a real socket each, with the house being changed throughout so frames actually flow. Both halves of the cost, the held response and the per-frame serialization, are in these numbers. Both ends of every socket are in this process, so the descriptor and RSS figures are an upper bound on what a server pays.',
};
}
/**
* The safety assertion: saturation must never buy an agent a free unlock.
*
* Drives the admission controller from empty to fully shed while a real guarded
* call is classified against a real lock in a real house at every step, and
* asserts that at no point does the verdict stop demanding a human.
*/
async function measureGateUnderSaturation(fleet, { steps = 400 } = {}) {
const houses = pickHouses(fleet);
const { bridges } = await openBridges(houses, 1);
if (!bridges.length) throw new Error('could not open a bridge for the gate assertion');
const bridge = bridges[0];
const lock = firstEntity(bridge, 'lock.');
if (!lock) throw new Error('the fixture house has no lock; the gate assertion needs one');
const admission = createAdmissionController({ maxPooled: 20, maxUnpooled: 5, maxStreams: 30, maxInflightActions: 40 });
const rungsSeen = new Set();
const violations = [];
let refusedByGate = 0;
let shedByLoad = 0;
for (let i = 0; i < steps; i++) {
// Climb: take connection slots, open streams, start actions, and never
// finish them, so the controller walks the whole ladder.
if (i % 3 === 0) admission.acquire();
if (i % 2 === 0) admission.admitStream();
if (i % 5 === 0) admission.setDatabaseHealthy(false);
const verdict = classifyCall({ domain: 'lock', service: 'unlock', entityId: lock, attributes: bridge.states[lock]?.attributes });
const decision = admission.admitAction({ guarded: verdict.guarded, confirmed: false, allowed: bridge.allowList.has(lock) });
rungsSeen.add(decision.rung);
if (!decision.requiresConfirmation) {
violations.push({ step: i, rung: decision.rung, snapshot: admission.snapshot() });
}
if (decision.admitted) {
// The gate is what stops it, not the load: an admitted guarded action
// still has to come back with requiresConfirmation and go no further.
refusedByGate++;
admission.finishAction();
// Re-take the slot without releasing, to climb toward the shed rung.
if (i > steps / 3) admission.admitAction({ guarded: false });
} else {
shedByLoad++;
}
}
// The real call, at the top of the ladder, against the real lock: it must
// still refuse, and the lock must still be locked afterwards.
const stateBefore = bridge.states[lock]?.state;
let liveRefusal = null;
try {
await bridge.call('lock', 'unlock', { entity_id: lock });
liveRefusal = 'NOT REFUSED';
} catch (err) {
liveRefusal = err.code;
}
await sleep(1200);
const stateAfter = bridge.states[lock]?.state;
closeAll(bridges);
return {
steps,
lock,
rungsSeen: [...rungsSeen].sort(),
guardedActionsSeen: steps,
everWavedThrough: violations.length,
violations: violations.slice(0, 3),
refusedByGateWhileAdmitted: refusedByGate,
shedByLoad,
liveCallOutcome: liveRefusal,
lockStateBefore: stateBefore,
lockStateAfter: stateAfter,
passed: violations.length === 0 && liveRefusal === 'needs_confirmation' && stateAfter === stateBefore,
};
}
/**
* What Cloud Run's CPU throttling actually does to a held SSE stream.
*
* This is the setting most likely to be wrong for this workload and the order it
* comes from is explicit that it must be tested rather than reasoned about. It
* cannot be tested on this machine: throttling is a Cloud Run behaviour, not a
* container one. So it is tested against the running production service, using a
* streaming endpoint that is already deployed there and already emits a ping on
* a fixed interval (api/pump/trades-stream.js, PING_INTERVAL_MS = 15000).
*
* That ping is a background timer inside a held request, which is exactly the
* shape order 03's home stream has. If `cpu-throttling=true` starved timers
* while a stream is open, the interval between pings would drift; if the request
* timeout cut the stream, it would end at a measurable second. Both are read off
* the wire here rather than off the documentation.
*
* @param {{ url?: string, seconds?: number, expectedIntervalMs?: number }} options
*/
async function measureCloudRunStream({ url = 'https://three.ws/api/pump/trades-stream', seconds = 960, expectedIntervalMs = 15_000 } = {}) {
const controller = new AbortController();
const startedAt = Date.now();
const events = [];
let bytes = 0;
let endedReason = 'still open at the deadline';
const timer = setTimeout(() => {
endedReason = 'closed by this probe at the deadline';
controller.abort();
}, seconds * 1000);
const response = await fetch(url, { headers: { accept: 'text/event-stream' }, signal: controller.signal });
const status = response.status;
const served = {
status,
server: response.headers.get('server'),
contentType: response.headers.get('content-type'),
cacheControl: response.headers.get('cache-control'),
};
try {
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
for (;;) {
const { value, done } = await reader.read();
if (done) {
endedReason = 'the server ended the stream';
break;
}
bytes += value.byteLength;
buffer += decoder.decode(value, { stream: true });
let index;
while ((index = buffer.indexOf('\n\n')) !== -1) {
const frame = buffer.slice(0, index);
buffer = buffer.slice(index + 2);
const name = (frame.match(/^event:\s*(.+)$/m) || [null, 'message'])[1].trim();
events.push({ atMs: Date.now() - startedAt, event: name });
}
}
} catch (err) {
if (err.name !== 'AbortError') endedReason = `the stream failed: ${err.message}`;
} finally {
clearTimeout(timer);
}
const heldMs = Date.now() - startedAt;
// Whichever event this endpoint uses for its heartbeat is the one to time.
// The name differs between streams ("ping" here, "heartbeat" there) and the
// probe should not need editing to point at a different one.
const histogram = {};
for (const e of events) histogram[e.event] = (histogram[e.event] || 0) + 1;
const heartbeatName = Object.entries(histogram).sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
const pings = events.filter((e) => e.event === heartbeatName);
const gaps = pings.slice(1).map((p, i) => p.atMs - pings[i].atMs);
const drift = gaps.map((g) => g - expectedIntervalMs);
return {
url,
requestedSeconds: seconds,
heldSeconds: Number((heldMs / 1000).toFixed(1)),
served,
endedReason,
bytes,
framesReceived: events.length,
frameHistogram: histogram,
heartbeatEvent: heartbeatName,
pingsReceived: pings.length,
expectedIntervalMs,
pingGapMs: summarize(gaps),
pingDriftMs: summarize(drift),
worstLatePingMs: drift.length ? Math.max(...drift) : null,
note: 'A ping arriving on schedule for the whole hold means the interval timer inside a held streaming request kept its CPU. The end reason and heldSeconds say whether the platform request timeout cut the stream before this probe did.',
};
}
/**
* Does a held SSE stream occupy a Cloud Run concurrency slot?
*
* This decides the stream cap, and it is not a detail: `containerConcurrency` is
* 160 on three-ws-api, so if a stream holds a slot for its whole life then the
* platform starts refusing requests long before an admission cap of several
* hundred would ever fire, and the ladder would be decoration.
*
* It is measured rather than reasoned about, against the running service, using
* a streaming endpoint already deployed there. Cloud Run publishes the observed
* concurrency per instance as `container/max_request_concurrencies`, which is
* the platform's own accounting of what it counts as an in-flight request. If
* holding N streams moves that number, a stream is a concurrency slot; if it
* does not, a stream is free and the cap can be sized on memory alone.
*
* Reading that metric rather than watching for a scaling event is deliberate:
* forcing this service to autoscale would take more than a thousand concurrent
* streams, which is a load test on production, and the concurrency gauge answers
* the same question with two hundred.
*
* @param {{ url?: string, streams?: number, seconds?: number, project?: string, service?: string }} options
*/
async function measureStreamConcurrency({
url = 'https://three.ws/api/feed-stream',
streams = 200,
seconds = 240,
project = 'aerial-vehicle-466722-p5',
service = 'three-ws-api',
} = {}) {
const before = {
concurrency: await readConcurrency({ project, service, minutesBack: 20 }),
instances: await readInstanceCount({ project, service, minutesBack: 20 }),
};
const controllers = [];
let opened = 0;
let failed = 0;
const openStream = async () => {
const controller = new AbortController();
controllers.push(controller);
try {
const response = await fetch(url, { headers: { accept: 'text/event-stream' }, signal: controller.signal });
if (!response.ok) {
failed++;
return;
}
opened++;
// Drain and discard. A stream nobody reads is a stream the kernel
// eventually stalls, which would measure our own back pressure instead
// of the platform's.
response.body.pipeTo(new WritableStream({ write() {} }), { signal: controller.signal }).catch(() => {});
} catch {
failed++;
}
};
// Open in waves, and keep reopening: the endpoint caps its own streams at 275
// seconds, so a fixed set would drain away mid measurement.
const started = Date.now();
const keepAlive = setInterval(() => {
const shortfall = streams - opened + failed;
for (let i = 0; i < Math.min(20, Math.max(0, shortfall)); i++) openStream();
}, 10_000);
for (let i = 0; i < streams; i += 25) {
await Promise.all(Array.from({ length: Math.min(25, streams - i) }, openStream));
await sleep(500);
}
await sleep(Math.max(0, seconds * 1000 - (Date.now() - started)));
const during = {
concurrency: await readConcurrency({ project, service, minutesBack: 6 }),
instances: await readInstanceCount({ project, service, minutesBack: 6 }),
};
clearInterval(keepAlive);
for (const controller of controllers) controller.abort();
const rise = during.concurrency.peak != null && before.concurrency.peak != null
? Number((during.concurrency.peak - before.concurrency.peak).toFixed(1))
: null;
const instancesDuring = during.instances.peakActive ?? null;
return {
url,
streamsRequested: streams,
streamsOpened: opened,
streamsRefused: failed,
heldSeconds: seconds,
concurrencyBefore: before.concurrency,
concurrencyDuring: during.concurrency,
concurrencyRise: rise,
instancesBefore: before.instances,
instancesDuring: during.instances,
expectedRiseIfStreamsHoldSlots: instancesDuring ? Number((opened / instancesDuring).toFixed(1)) : null,
occupiesConcurrencySlot: rise != null ? rise > 2 : null,
note: 'Cloud Run counts an in-flight request against containerConcurrency. Holding N streams and watching the platform own concurrency gauge rise by roughly N divided by the instance count is the platform saying a held stream is one of those requests.',
};
}
/** Observed concurrent requests per instance, from Cloud Run's own gauge. */
async function readConcurrency({ project, service, minutesBack }) {
const series = await readTimeSeries({
project,
service,
metric: 'run.googleapis.com/container/max_request_concurrencies',
minutesBack,
aligner: 'ALIGN_PERCENTILE_99',
});
if (series.error) return { error: series.error, peak: null };
const values = (series.timeSeries[0]?.points || []).map((p) => Number(p.value.doubleValue ?? p.value.int64Value ?? 0));
if (!values.length) return { peak: null, points: 0 };
return {
windowMinutes: minutesBack,
points: values.length,
peak: Number(Math.max(...values).toFixed(1)),
median: Number(values.slice().sort((a, b) => a - b)[Math.floor(values.length / 2)].toFixed(1)),
};
}
/** Cloud Run instance counts, straight from Cloud Monitoring. */
async function readInstanceCount({ project, service, minutesBack }) {
const series = await readTimeSeries({
project,
service,
metric: 'run.googleapis.com/container/instance_count',
minutesBack,
aligner: 'ALIGN_MEAN',
});
if (series.error) return { error: series.error, peakActive: null };
const body = series;
const byState = {};
for (const series of body.timeSeries || []) {
const state = series.metric.labels.state;
const values = series.points.map((p) => Number(p.value.doubleValue ?? p.value.int64Value ?? 0));
byState[state] = { peak: Math.max(...values, 0), mean: Number((values.reduce((a, b) => a + b, 0) / Math.max(1, values.length)).toFixed(2)), points: values.length };
}
return { windowMinutes: minutesBack, byState, peakActive: byState.active?.peak ?? null };
}
/** One Cloud Monitoring read, with the access token gcloud already holds. */
async function readTimeSeries({ project, service, metric, minutesBack, aligner }) {
const token = await new Promise((resolve, reject) => {
const child = spawn('gcloud', ['auth', 'print-access-token'], { stdio: ['ignore', 'pipe', 'ignore'] });
let out = '';
child.stdout.on('data', (d) => (out += d));
child.on('error', reject);
child.on('close', (code) => (code === 0 ? resolve(out.trim()) : reject(new Error('gcloud auth print-access-token failed'))));
});
const end = new Date();
const start = new Date(end.getTime() - minutesBack * 60_000);
const params = new URLSearchParams({
filter: `metric.type="${metric}" AND resource.labels.service_name="${service}"`,
'interval.startTime': start.toISOString(),
'interval.endTime': end.toISOString(),
'aggregation.alignmentPeriod': '60s',
'aggregation.perSeriesAligner': aligner,
});
const response = await fetch(`https://monitoring.googleapis.com/v3/projects/${project}/timeSeries?${params}`, {
headers: { authorization: `Bearer ${token}` },
});
const body = await response.json();
if (body.error) return { error: body.error.message };
return { timeSeries: body.timeSeries || [] };
}
/** Each rung of the ladder, demonstrated rather than described. */
function demonstrateLadder() {
const admission = createAdmissionController({ maxPooled: 4, maxUnpooled: 2, maxStreams: 6, maxInflightActions: 8, streamYieldRatio: 0.5 });
const rows = [];
const record = (label, verdict) =>
rows.push({ label, admitted: verdict.admitted, rung: verdict.rung, connection: verdict.connection ?? null, source: verdict.source ?? null, requiresConfirmation: verdict.requiresConfirmation ?? null, retryAfterSeconds: verdict.retryAfterSeconds ?? null, reason: verdict.reason || null });
record('rung 1: acquire under the cap', admission.acquire());
for (let i = 0; i < 3; i++) admission.acquire();
record('rung 2: acquire at the cap', admission.acquire());
record('rung 2: read is still from the database', admission.admitRead());
admission.setDatabaseHealthy(false);
record('rung 3: read with the database down', admission.admitRead());
rows.push({ label: 'rung 3: write policy with the database down', ...admission.writePolicy() });
admission.setDatabaseHealthy(true);
record('rung 4: stream while 3 of 8 actions are in flight', (() => {
for (let i = 0; i < 3; i++) admission.admitAction({ guarded: false });
return admission.admitStream();
})());
record('rung 4: stream once actions pass the yield floor', (() => {
for (let i = 0; i < 2; i++) admission.admitAction({ guarded: false });
return admission.admitStream();
})());
record('rung 4: an action at the same moment', admission.admitAction({ guarded: false }));
record('rung 5: an action with every slot taken', (() => {
for (let i = 0; i < 8; i++) admission.admitAction({ guarded: false });
return admission.admitAction({ guarded: false });
})());
record('rung 5: a GUARDED action with every slot taken', admission.admitAction({ guarded: true, confirmed: false }));
return { rows, finalSnapshot: admission.snapshot() };
}
// ---------------------------------------------------------------- runner
/**
* Run one measurement in a brand new process and return its JSON.
*
* Heap deltas do not survive being taken back to back in one process: the
* garbage from a 400 connection tier is still resident when the next tier reads
* its baseline, and the arithmetic comes out negative. Measuring the large house
* that way produced -6.8MB per connection, which is not a number, it is an
* artefact. Every heap figure in the envelope is therefore taken against a
* baseline read moments earlier in a process that has done nothing else.
*/
function measureInChildProcess(args) {
return new Promise((resolve, reject) => {
const child = spawn(process.execPath, ['--expose-gc', fileURLToPath(import.meta.url), ...args], {
stdio: ['ignore', 'pipe', 'inherit'],
env: process.env,
});
let out = '';
child.stdout.on('data', (d) => (out += d));
child.on('error', reject);
child.on('close', (code) => {
if (code !== 0) return reject(new Error(`${args.join(' ')} exited ${code}`));
try {
resolve(JSON.parse(out));
} catch (err) {
reject(new Error(`${args.join(' ')} did not print JSON: ${err.message}`));
}
});
});
}
async function all({ out, counts, sseSubscribers }) {
const fleet = await readFleet();
const machine = {
node: process.version,
platform: `${os.platform()} ${os.release()}`,
cpus: os.cpus().length,
cpuModel: os.cpus()[0]?.model ?? null,
totalMemBytes: os.totalmem(),
freeMemBytes: os.freemem(),
ulimitNofile: Number(process.report?.getReport()?.userLimits?.open_files?.soft ?? 0) || null,
// This box is shared with other work. Load average is recorded so a reader
// can tell a real latency number from one taken while sixteen cores were
// already busy, instead of having to trust that the run was quiet.
loadAverage: os.loadavg().map((n) => Number(n.toFixed(2))),
};
const result = {
measuredAt: new Date().toISOString(),
machine,
fleet: {
image: fleet.image,
houses: fleet.houses.length,
small: fleet.houses.filter((h) => !h.big).length,
large: fleet.houses.filter((h) => h.big).length,
entitiesPerSmallHouse: fleet.houses.find((h) => !h.big)?.entityCount ?? null,
entitiesPerLargeHouse: fleet.houses.find((h) => h.big)?.entityCount ?? null,
haBootMs: summarize(fleet.houses.map((h) => h.bootMs)),
realNote: 'Every house is a real Home Assistant container with a real long-lived access token, started by scripts/home-fleet.mjs. Connections are real WebSocket sessions through home-assistant-js-websocket, the same client the product uses.',
},
coldStart: await measureColdStart(fleet),
connections: [],
ladder: demonstrateLadder(),
};
for (const count of counts) {
process.stderr.write(` measuring ${count} connections (fresh process)...\n`);
result.connections.push(await measureInChildProcess(['connections', '--n', String(count)]));
}
process.stderr.write(' measuring the large house (fresh process)...\n');
result.largeHouse = await measureInChildProcess(['connections', '--n', '10', '--big']);
process.stderr.write(' measuring a 100 entity update burst...\n');
result.burst = await measureInChildProcess(['burst', '--updates', '100']);
process.stderr.write(' measuring SSE subscribers...\n');
result.sse = await measureInChildProcess(['sse', '--subscribers', String(sseSubscribers)]);
process.stderr.write(' asserting the gate under saturation...\n');
result.gate = await measureInChildProcess(['gate']);
result.machineAtEnd = { loadAverage: os.loadavg().map((n) => Number(n.toFixed(2))), freeMemBytes: os.freemem() };
if (out) {
const file = path.resolve(ROOT, out);
await mkdir(path.dirname(file), { recursive: true });
await writeFile(file, `${JSON.stringify(result, null, 2)}\n`);
process.stderr.write(`\nwrote ${path.relative(ROOT, file)}\n`);
} else {
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
}
return result;
}
function parseArgs(argv) {
const out = { _: [] };
for (let i = 0; i < argv.length; i++) {
const arg = argv[i];
if (arg.startsWith('--')) out[arg.slice(2)] = argv[i + 1]?.startsWith('--') || argv[i + 1] === undefined ? true : argv[++i];
else out._.push(arg);
}
return out;
}
export { demonstrateLadder, measureBurst, measureCloudRunStream, measureColdStart, measureStreamConcurrency, measureConnections, measureGateUnderSaturation, measureSse, openBridges, closeAll, pickHouses, summarize, requiresConfirmation };
const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
if (isMain) {
const args = parseArgs(process.argv.slice(2));
const command = args._[0] || 'all';
const counts = String(args.counts || '10,50,100,200,400')
.split(',')
.map((n) => Number(n.trim()))
.filter(Boolean);
const commands = {
all: () => all({ out: typeof args.out === 'string' ? args.out : null, counts, sseSubscribers: Number(args.subscribers || 200) }),
coldstart: async () => measureColdStart(await readFleet()),
connections: async () => measureConnections(await readFleet(), { count: Number(args.n || 50), big: Boolean(args.big) }),
burst: async () => measureBurst(await readFleet(), { updates: Number(args.updates || 100) }),
sse: async () => measureSse(await readFleet(), { subscribers: Number(args.subscribers || 200) }),
gate: async () => measureGateUnderSaturation(await readFleet()),
ladder: async () => demonstrateLadder(),
cloudrun: async () => measureCloudRunStream({ seconds: Number(args.seconds || 960), url: typeof args.url === 'string' ? args.url : undefined }),
concurrency: async () => measureStreamConcurrency({ streams: Number(args.streams || 200), seconds: Number(args.seconds || 240) }),
};
const run = commands[command];
if (!run) {
console.error(`unknown command "${command}". Use: ${Object.keys(commands).join(' | ')}`);
process.exit(2);
}
run().then(
(value) => {
if (command !== 'all' || !args.out) process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
process.exit(0);
},
(err) => {
console.error(err.stack || err.message);
process.exit(1);
},
);
}