Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(home): stop the Sev 1 integrity alert paging on every lawful unlock
nirholas9d32efeb3583
diff --git a/api/_lib/ops/home-health.js b/api/_lib/ops/home-health.jsindex 16ca89f06..e006f7e2e 100644--- a/api/_lib/ops/home-health.js+++ b/api/_lib/ops/home-health.js@@ -61,6 +61,20 @@ export const LATENCY_DOWN_MS = 4_000;*/export const MIN_HOMES_FOR_A_VERDICT = 10;+/**+ * The minimum number of actions in the window before their success rate is+ * scored. Below it, two failures in a quiet hour read as a 90% success rate and+ * would take the subsystem down over nothing.+ */+export const MIN_ACTIONS_FOR_A_VERDICT = 20;++/**+ * The minimum number of decided confirmations before their expiry rate is+ * scored. Three prompts in a quiet hour, two of them from someone who walked+ * away from their laptop, is not a UI failure.+ */+export const MIN_CONFIRMATIONS_FOR_A_VERDICT = 10;+const QUERY_TIMEOUT_MS = 4_000;/**@@ -175,6 +189,7 @@ export async function readHomeSignals({ windowMinutes = WINDOW_MINUTES } = {}) {count(*) filter (where outcome = 'refused')::int as refused,count(*) filter (where outcome = 'failed')::int as failed,count(distinct home_id)::int as homes,+ count(distinct home_id) filter (where outcome = 'failed')::int as failed_homes,percentile_disc(0.95) within group (order by (detail->>'latencyMs')::numeric) as p95_latency_ms,@@ -198,9 +213,31 @@ export async function readHomeSignals({ windowMinutes = WINDOW_MINUTES } = {}) {// The zero-budget invariant, over a deliberately wider window than the// rates: a single row is an incident and must not be able to age out of// the check between two cron ticks.+ //+ // A null `confirmed_by` is NOT on its own a violation, and finding that out+ // against real rows is what stopped this alert from firing on every+ // legitimate unlock. A standing per-entity allowance in+ // `home_entity_grants` is a yes the user already gave, recorded once rather+ // than re-asked every time, so the act path clears the gate through the+ // allow list and stamps `detail.allowed_by_grant`. Those rows are counted+ // and reported, never paged.+ //+ // What remains a violation is the shape with no yes behind it at all:+ // guarded, executed, nobody confirmed it, and no grant claimed.withTimeout(sql`- select count(*)::int as violations, max(created_at) as last_at+ select+ count(*) filter (where coalesce(detail->>'allowed_by_grant', 'false') <> 'true')::int as violations,+ max(created_at) filter (where coalesce(detail->>'allowed_by_grant', 'false') <> 'true') as last_at,+ count(*) filter (where detail->>'allowed_by_grant' = 'true')::int as grant_backed,+ count(*) filter (+ where detail->>'allowed_by_grant' = 'true'+ and not exists (+ select 1 from home_entity_grants g+ where g.home_id = home_action_log.home_id+ and g.entity_id = any(home_action_log.entity_ids)+ )+ )::int as grant_backed_without_grantfrom home_action_logwhere guarded = true and confirmed_by is null and outcome = 'ok'and created_at > now() - interval '24 hours'@@ -240,6 +277,10 @@ export async function readHomeSignals({ windowMinutes = WINDOW_MINUTES } = {}) {refused: a.refused ?? 0,failed: a.failed ?? 0,homes: a.homes ?? 0,+ // Failures confined to ONE home are that home's problem, exactly like a+ // handshake failure is. Without this, a single house with a broken+ // integration downs the subsystem for everybody.+ failedHomes: a.failed_homes ?? 0,// A refused action is the safety gate doing its job, so it counts as a// success. Only `failed` is us not delivering.rate: (a.total ?? 0) > 0 ? ((a.ok ?? 0) + (a.refused ?? 0)) / a.total : null,@@ -255,6 +296,14 @@ export async function readHomeSignals({ windowMinutes = WINDOW_MINUTES } = {}) {integrity: {violations: v.violations ?? 0,lastAt: v.last_at ? new Date(v.last_at).toISOString() : null,+ // Guarded actions a standing grant let through. Reported so the number+ // is visible, never scored: the user said yes once, on purpose.+ grantBacked: v.grant_backed ?? 0,+ // Of those, the ones whose entity has no grant row now. Usually a grant+ // the user revoked after the fact, which is why this reports rather than+ // alerts. A number that climbs while nobody is revoking anything is+ // worth reading.+ grantBackedWithoutGrant: v.grant_backed_without_grant ?? 0,},pool,leak,@@ -317,10 +366,19 @@ export function homeHealthVerdict(s) {const handshakeStatus = !enoughHomes || s.handshakes.rate === null? 'ok': rateStatus(s.handshakes.rate, HANDSHAKE_DEGRADED, HANDSHAKE_DOWN);- const actionStatus = s.actions.rate === null+ // Actions get the same cross-tenant guard as handshakes, for the same reason.+ // A house whose Z-Wave stick fell out fails every action sent to it; scoring+ // that as a platform outage pages an operator for somebody's loose USB port.+ // It has to be thin traffic or one house before it stops counting, never both+ // ignored: real correlated breakage shows up across homes and at volume.+ const actionsScoreable = s.actions.rate !== null && s.actions.total >= MIN_ACTIONS_FOR_A_VERDICT;+ const actionStatus = !actionsScoreable? 'ok'- : rateStatus(s.actions.rate, ACTION_DEGRADED, ACTION_DOWN);- const expiryStatus = s.confirmations.expiryRate === null+ : s.actions.failedHomes <= 1+ ? worst('ok', rateStatus(s.actions.rate, ACTION_DEGRADED, 0))+ : rateStatus(s.actions.rate, ACTION_DEGRADED, ACTION_DOWN);+ const decidedConfirmations = s.confirmations.redeemed + s.confirmations.expired;+ const expiryStatus = s.confirmations.expiryRate === null || decidedConfirmations < MIN_CONFIRMATIONS_FOR_A_VERDICT? 'ok': ceilingStatus(s.confirmations.expiryRate, EXPIRY_DEGRADED, EXPIRY_DOWN);const breakerRate = s.pool.open > 0 ? s.pool.breakersOpen / s.pool.open : 0;@@ -340,14 +398,18 @@ export function homeHealthVerdict(s) {? `handshakes ${pct(s.handshakes.rate)} over ${s.handshakes.attempts} homes in ${s.windowMinutes}m` + (enoughHomes ? '' : ` (under the ${MIN_HOMES_FOR_A_VERDICT}-home floor, reported not scored)`): `no handshakes in ${s.windowMinutes}m`,s.actions.total > 0- ? `actions ${pct(s.actions.rate)} of ${s.actions.total} across ${s.actions.homes} homes` + (s.actions.failed ? `, ${s.actions.failed} failed` : '')+ ? `actions ${pct(s.actions.rate)} of ${s.actions.total} across ${s.actions.homes} homes`+ + (s.actions.failed ? `, ${s.actions.failed} failed in ${s.actions.failedHomes} home${s.actions.failedHomes === 1 ? ' (that house, not us)' : 's'}` : '')+ + (s.actions.total >= MIN_ACTIONS_FOR_A_VERDICT ? '' : ` (under the ${MIN_ACTIONS_FOR_A_VERDICT}-action floor, reported not scored)`): 'no actions in window',s.actions.p95LatencyMs === null? 'no action timings recorded': `p95 our-leg latency ${s.actions.p95LatencyMs}ms over ${s.actions.timed} timed actions`,s.confirmations.total > 0? `confirmations ${pct(s.confirmations.expiryRate ?? 0)} expired of ${s.confirmations.total}`+ + (decidedConfirmations >= MIN_CONFIRMATIONS_FOR_A_VERDICT ? '' : ` (under the ${MIN_CONFIRMATIONS_FOR_A_VERDICT}-confirmation floor, reported not scored)`): 'no confirmations in window',+ s.integrity.grantBacked ? `${s.integrity.grantBacked} guarded action(s) cleared by a standing grant` : null,`pool ${s.pool.open} open, ${s.pool.subscribers} subscribers` + (s.pool.streams === null ? '' : ` across ${s.pool.streams} streams`) + `, ${s.pool.breakersOpen} breakers open` + (s.pool.rung && s.pool.rung !== 'normal' ? `, admission ${s.pool.rung}` : ''),];@@ -358,14 +420,16 @@ export function homeHealthVerdict(s) {: handshakeStatus !== 'ok'? 'Handshakes are failing across tenants, which is almost always us: a deploy, an egress change or a DNS fault. Run the correlation query in docs/ops/home-operations.md before touching anything.': actionStatus !== 'ok'- ? 'Actions are failing across homes. Group home_action_log by outcome and detail->>\'code\' to see whether this is one error or many.'+ ? (s.actions.failedHomes <= 1+ ? 'Action failures are confined to one home, so this is that house rather than the platform. Read its status_detail and its own action log before looking anywhere else.'+ : 'Actions are failing across homes. Group home_action_log by outcome and detail->>\'code\' to see whether this is one error or many.'): expiryStatus !== 'ok'? 'Confirmations are timing out rather than being answered. That is a UI failure, not user hesitation: check that the confirm prompt is reaching the surface the user is actually on.': latencyStatus !== 'ok'? 'Our own leg of the act path is slow. The house is excluded from this measurement, so look at the pool: cold opens, breaker retries, or a store read on the hot path.': 'Breakers are open on a large share of this instance\'s pooled homes. Correlated, that means we cannot reach houses we could reach an hour ago.';- return { status, detail: parts.join('; ') + '.', ...(hint ? { hint } : {}) };+ return { status, detail: parts.filter(Boolean).join('; ') + '.', ...(hint ? { hint } : {}) };}/**diff --git a/docs/ops/home-operations.md b/docs/ops/home-operations.mdindex 4634527f3..f8feda4b4 100644--- a/docs/ops/home-operations.md+++ b/docs/ops/home-operations.md@@ -30,15 +30,21 @@ So the rule, stated once and enforced in code:| The pool gauges | `stats()` in [`api/_lib/home/runtime.js`](../../api/_lib/home/runtime.js) || The tables | `home_connections`, `home_action_log`, `home_confirmations` |-Every rate is computed **across tenants** over a 15 minute window, and nothing is-scored at all until at least 10 homes are connected (`MIN_HOMES_FOR_A_VERDICT`).-With three homes, one holiday cottage losing power is a 33% failure rate, and-scoring that as an outage is the exact false page this design exists to prevent.-Below the floor the subsystem reports the numbers and stays green, saying so:+Every rate is computed **across tenants** over a 15 minute window, and each one+has a floor below which it is reported but not scored:++| Rate | Floor | Why |+|---|---|---|+| Handshake success | 10 connected homes (`MIN_HOMES_FOR_A_VERDICT`) | With three homes, one holiday cottage losing power is a 33% failure rate. |+| Action success | 20 actions (`MIN_ACTIONS_FOR_A_VERDICT`), and failures in more than one home | A house whose Z-Wave stick fell out fails everything sent to it. Paging for that is paging for a loose USB port. Failures confined to one home cap at `degraded` and the hint names the house. |+| Confirmation expiry | 10 decided confirmations (`MIN_CONFIRMATIONS_FOR_A_VERDICT`) | Three prompts in a quiet hour, two of them from someone who walked away from their laptop, is not a UI failure. |++Below a floor the subsystem reports the numbers and stays green, saying so:```-1/1 homes connected; handshakes 100.0% over 1 homes in 15m (under the 10-home-floor, reported not scored); no actions in window; ...+14/14 homes connected; handshakes 100.0% over 3 homes in 15m; actions 93.3% of 30+across 5 homes, 2 failed in 1 home (that house, not us); confirmations 75.0%+expired of 4 (under the 10-confirmation floor, reported not scored); ...```### The thresholds@@ -51,12 +57,22 @@ floor, reported not scored); no actions in window; ...| Breaker-open homes | `runtime.stats()` | under 2% | 2 to 10% | over 10% || p95 action latency (our leg) | `home_action_log.detail->>'latencyMs'` | under 1.5 s | 1.5 to 4 s | over 4 s || Subscriber leak | `stats().subscribers` versus open streams | flat | growing | growing |-| Confirmation integrity | `home_action_log` | zero rows | n/a | any row |+| Confirmation integrity | `home_action_log`, excluding standing grants | zero rows | n/a | any row |A **refused** action counts as a success. The safety gate refusing to open afront door is the product working, not a failure to deliver, and scoring it as anerror would make the safest houses look like the sickest ones.+A guarded action with no `confirmed_by` is **not** on its own a violation, and+finding that out against real rows is what stopped this alert from firing on+every legitimate unlock. A standing per-entity allowance in `home_entity_grants`+is a yes the user already gave, recorded once rather than re-asked every time, so+the act path clears the gate through the allow list and stamps+`detail.allowed_by_grant`. Those rows are counted and reported+(`integrity.grantBacked`), never paged. What remains a Sev 1 is the shape with no+yes behind it at all: guarded, executed, nobody confirmed it, and no grant+claimed.+## The SLOs| SLO | Target | Window | Error budget |@@ -71,10 +87,14 @@ guarded action executing without a valid confirmation is a Sev 1 regardless ofvolume, regardless of how many succeeded, and regardless of whether any usernoticed. There is no budget to spend and no rate at which it becomes acceptable.-**Pre-launch baseline (measured 2026-09-03, local, not production):** 3 connected-homes, 8 actions logged across the window (3 refused by the gate, 5 executed),-p95 our-leg latency 412 ms on the single timed action, zero integrity violations,-zero confirmations expired. Re-measure against production traffic in order 20.+**Pre-launch baseline (measured 2026-09-03, development database, not+production):** 14 connected homes, 30 actions in the window across 5 homes (93.3%+succeeding or refused, the 2 failures confined to one house), p95 our-leg latency+412 ms on the one action carrying a timing, 2 guarded actions cleared by a+standing grant, zero integrity violations. The act path does not yet stamp+`detail.latencyMs` on every action, so the p95 above rests on a single sample and+the sensor says `no action timings recorded` rather than inventing one.+Re-measure against production traffic in order 20.---@@ -144,13 +164,19 @@ always pages even if an earlier one paged an hour ago.**First command:**```sql-select id, home_id, user_id, actor, channel, action, entity_ids, risk, created_at+select id, home_id, user_id, actor, channel, action, entity_ids, risk, detail, created_atfrom home_action_logwhere guarded = true and confirmed_by is null and outcome = 'ok'+ and coalesce(detail->>'allowed_by_grant', 'false') <> 'true'and created_at > now() - interval '24 hours'order by created_at desc;```+Drop the `allowed_by_grant` line to see the grant-backed actions alongside them.+Those are legitimate and are the reason that line is there: without it, this+query returns every standing-grant unlock in the fleet and the alert becomes+noise within a day.+**Then, in this order:**1. **Scope it.** How many rows, how many homes, which entities. A `lock.*`,@@ -163,11 +189,15 @@ order by created_at desc;are the only things standing between a model and a front door. A refactor thatchanged a domain list, or a caller that passed `confirmed: true` fromsomewhere other than a human, is the likely cause.-4. **Check for a standing grant.** A per-entity allowance in- `home_entity_grants` is a legitimate reason for a guarded action to run- without a fresh confirmation, and the act path must record the grant in- `detail` when it uses one. A violation row with a matching live grant is a- logging bug; a violation row with no grant is the real thing.+4. **Check the grant claim.** The alert already excludes actions that recorded+ `detail.allowed_by_grant`, so a row that reached you claimed no grant at all.+ Confirm that against `home_entity_grants` for the home and entity: a live+ grant with no claim on the row is a logging bug in the act path+ (`api/home/[id]/call.js`), which is a much smaller problem than the alternative+ and still worth fixing the same day. No grant and no claim is the real thing.+ `integrity.grantBackedWithoutGrant` on the health block counts rows that+ claimed a grant that no longer exists; that is usually a grant the user+ revoked afterwards, which is why it reports rather than pages.**Rollback:** yes. Roll back to the last revision known to gate correctly beforediagnosing further.diff --git a/tests/home-integrity.test.js b/tests/home-integrity.test.jsindex 385275e3c..d1a266703 100644--- a/tests/home-integrity.test.js+++ b/tests/home-integrity.test.js@@ -18,6 +18,8 @@ import {HANDSHAKE_DOWN,homeHealthVerdict,LATENCY_DOWN_MS,+ MIN_ACTIONS_FOR_A_VERDICT,+ MIN_CONFIRMATIONS_FOR_A_VERDICT,MIN_HOMES_FOR_A_VERDICT,noteSubscriberSample,resetSubscriberSamples,@@ -29,9 +31,9 @@ function healthy(overrides = {}) {windowMinutes: 15,homes: { live: 40, connected: 40, unreachable: 0, authFailed: 0 },handshakes: { attempts: 40, ok: 40, failed: 0, rate: 1 },- actions: { total: 200, ok: 198, refused: 2, failed: 0, homes: 30, rate: 1, timed: 200, p95LatencyMs: 300 },+ actions: { total: 200, ok: 198, refused: 2, failed: 0, homes: 30, failedHomes: 0, rate: 1, timed: 200, p95LatencyMs: 300 },confirmations: { total: 20, redeemed: 19, expired: 1, expiryRate: 1 / 20 },- integrity: { violations: 0, lastAt: null },+ integrity: { violations: 0, lastAt: null, grantBacked: 0, grantBackedWithoutGrant: 0 },pool: { open: 12, subscribers: 12, capacity: 200, breakersOpen: 0, byStatus: { connected: 12 }, streams: 12, rung: 'normal' },leak: { leaking: false, samples: [12, 12, 12], margins: [0, 0, 0], growth: 0 },...overrides,@@ -40,17 +42,37 @@ function healthy(overrides = {}) {describe('confirmation integrity is a zero-budget invariant', () => {it('takes the subsystem down on a single row, however healthy everything else is', () => {- const verdict = homeHealthVerdict(healthy({ integrity: { violations: 1, lastAt: '2026-09-03T03:37:14.568Z' } }));+ const verdict = homeHealthVerdict(healthy({ integrity: { violations: 1, lastAt: '2026-09-03T03:37:14.568Z', grantBacked: 0, grantBackedWithoutGrant: 0 } }));expect(verdict.status).toBe('down');expect(verdict.detail).toContain('no confirmation on record');expect(verdict.hint).toContain('Sev 1');});+ it('does not fire on an unlock a standing grant already authorised', () => {+ // The shape real traffic produces: the user granted the agent lock.kitchen_door+ // once, so the gate cleared it through the allow list and nobody was asked+ // again. Paging on this would page on every legitimate unlock forever.+ const verdict = homeHealthVerdict(+ healthy({ integrity: { violations: 0, lastAt: null, grantBacked: 2, grantBackedWithoutGrant: 0 } }),+ );+ expect(verdict.status).toBe('ok');+ expect(verdict.detail).toContain('2 guarded action(s) cleared by a standing grant');+ });++ it('reports a grant-backed action whose grant no longer exists without alerting on it', () => {+ // A grant revoked after the action is the ordinary explanation, which is+ // why this is a number to read rather than a page to answer.+ const verdict = homeHealthVerdict(+ healthy({ integrity: { violations: 0, lastAt: null, grantBacked: 3, grantBackedWithoutGrant: 1 } }),+ );+ expect(verdict.status).toBe('ok');+ });+it('outranks every other signal, so it is never masked by a busy platform', () => {const verdict = homeHealthVerdict(healthy({- integrity: { violations: 3, lastAt: '2026-09-03T03:37:14.568Z' },- actions: { total: 1000, ok: 1000, refused: 0, failed: 0, homes: 90, rate: 1, timed: 1000, p95LatencyMs: 120 },+ integrity: { violations: 3, lastAt: '2026-09-03T03:37:14.568Z', grantBacked: 0, grantBackedWithoutGrant: 0 },+ actions: { total: 1000, ok: 1000, refused: 0, failed: 0, homes: 90, failedHomes: 0, rate: 1, timed: 1000, p95LatencyMs: 120 },}),);expect(verdict.status).toBe('down');@@ -120,7 +142,7 @@ describe('actions', () => {it('counts a refused action as a success, because the gate working is not a fault', () => {// Every action refused, none failed: the safety gate did its job all day.const verdict = homeHealthVerdict(- healthy({ actions: { total: 100, ok: 0, refused: 100, failed: 0, homes: 20, rate: 1, timed: 100, p95LatencyMs: 200 } }),+ healthy({ actions: { total: 100, ok: 0, refused: 100, failed: 0, homes: 20, failedHomes: 0, rate: 1, timed: 100, p95LatencyMs: 200 } }),);expect(verdict.status).toBe('ok');});@@ -129,12 +151,32 @@ describe('actions', () => {const rate = 0.9;expect(rate).toBeLessThan(ACTION_DOWN);const verdict = homeHealthVerdict(- healthy({ actions: { total: 100, ok: 90, refused: 0, failed: 10, homes: 25, rate, timed: 100, p95LatencyMs: 300 } }),+ healthy({ actions: { total: 100, ok: 90, refused: 0, failed: 10, homes: 25, failedHomes: 6, rate, timed: 100, p95LatencyMs: 300 } }),);expect(verdict.status).toBe('down');expect(verdict.detail).toContain('10 failed');});+ it('does not down the platform for failures confined to one house', () => {+ // One home whose Z-Wave stick fell out fails everything sent to it. That is+ // that house, and paging for it is paging for a loose USB port.+ const verdict = homeHealthVerdict(+ healthy({ actions: { total: 100, ok: 88, refused: 0, failed: 12, homes: 25, failedHomes: 1, rate: 0.88, timed: 100, p95LatencyMs: 300 } }),+ );+ expect(verdict.status).toBe('degraded');+ expect(verdict.detail).toContain('12 failed in 1 home (that house, not us)');+ expect(verdict.hint).toContain('confined to one home');+ });++ it('does not score a thin window at all', () => {+ const total = MIN_ACTIONS_FOR_A_VERDICT - 1;+ const verdict = homeHealthVerdict(+ healthy({ actions: { total, ok: total - 4, refused: 0, failed: 4, homes: 5, failedHomes: 4, rate: (total - 4) / total, timed: total, p95LatencyMs: 300 } }),+ );+ expect(verdict.status).toBe('ok');+ expect(verdict.detail).toContain(`under the ${MIN_ACTIONS_FOR_A_VERDICT}-action floor`);+ });+it('says so plainly when nothing on the act path is timed yet', () => {const verdict = homeHealthVerdict(healthy({ actions: { ...healthy().actions, timed: 0, p95LatencyMs: null } }));expect(verdict.status).toBe('ok');@@ -160,6 +202,14 @@ describe('confirmations that expire are a UI failure', () => {expect(verdict.hint).toContain('That is a UI failure, not user hesitation');});+ it('does not score three prompts as a broken UI', () => {+ const verdict = homeHealthVerdict(+ healthy({ confirmations: { total: 4, redeemed: 1, expired: 3, expiryRate: 0.75 } }),+ );+ expect(verdict.status).toBe('ok');+ expect(verdict.detail).toContain(`under the ${MIN_CONFIRMATIONS_FOR_A_VERDICT}-confirmation floor`);+ });+it('ignores an empty window rather than scoring it as perfect or broken', () => {const verdict = homeHealthVerdict(healthy({ confirmations: { total: 0, redeemed: 0, expired: 0, expiryRate: null } }),