Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(home): stop the Sev 1 integrity alert paging on every lawful unlock

nirholas9d32efeb3583
diff --git a/api/_lib/ops/home-health.js b/api/_lib/ops/home-health.js
index 16ca89f06..e006f7e2e 100644
--- a/api/_lib/ops/home-health.js
+++ b/api/_lib/ops/home-health.js
@@ -61,6 +61,20 @@ export const LATENCY_DOWN_MS = 4_000;
*/
export const MIN_HOMES_FOR_A_VERDICT = 10;
+/**
+ * The minimum number of actions in the window before their success rate is
+ * scored. Below it, two failures in a quiet hour read as a 90% success rate and
+ * would take the subsystem down over nothing.
+ */
+export const MIN_ACTIONS_FOR_A_VERDICT = 20;
+
+/**
+ * The minimum number of decided confirmations before their expiry rate is
+ * scored. Three prompts in a quiet hour, two of them from someone who walked
+ * away from their laptop, is not a UI failure.
+ */
+export const MIN_CONFIRMATIONS_FOR_A_VERDICT = 10;
+
const QUERY_TIMEOUT_MS = 4_000;
/**
@@ -175,6 +189,7 @@ export async function readHomeSignals({ windowMinutes = WINDOW_MINUTES } = {}) {
count(*) filter (where outcome = 'refused')::int as refused,
count(*) filter (where outcome = 'failed')::int as failed,
count(distinct home_id)::int as homes,
+ count(distinct home_id) filter (where outcome = 'failed')::int as failed_homes,
percentile_disc(0.95) within group (
order by (detail->>'latencyMs')::numeric
) as p95_latency_ms,
@@ -198,9 +213,31 @@ export async function readHomeSignals({ windowMinutes = WINDOW_MINUTES } = {}) {
// The zero-budget invariant, over a deliberately wider window than the
// rates: a single row is an incident and must not be able to age out of
// the check between two cron ticks.
+ //
+ // A null `confirmed_by` is NOT on its own a violation, and finding that out
+ // against real rows is what stopped this alert from firing on every
+ // legitimate unlock. A standing per-entity allowance in
+ // `home_entity_grants` is a yes the user already gave, recorded once rather
+ // than re-asked every time, so the act path clears the gate through the
+ // allow list and stamps `detail.allowed_by_grant`. Those rows are counted
+ // and reported, never paged.
+ //
+ // What remains a violation is the shape with no yes behind it at all:
+ // guarded, executed, nobody confirmed it, and no grant claimed.
withTimeout(
sql`
- select count(*)::int as violations, max(created_at) as last_at
+ select
+ count(*) filter (where coalesce(detail->>'allowed_by_grant', 'false') <> 'true')::int as violations,
+ max(created_at) filter (where coalesce(detail->>'allowed_by_grant', 'false') <> 'true') as last_at,
+ count(*) filter (where detail->>'allowed_by_grant' = 'true')::int as grant_backed,
+ count(*) filter (
+ where detail->>'allowed_by_grant' = 'true'
+ and not exists (
+ select 1 from home_entity_grants g
+ where g.home_id = home_action_log.home_id
+ and g.entity_id = any(home_action_log.entity_ids)
+ )
+ )::int as grant_backed_without_grant
from home_action_log
where guarded = true and confirmed_by is null and outcome = 'ok'
and created_at > now() - interval '24 hours'
@@ -240,6 +277,10 @@ export async function readHomeSignals({ windowMinutes = WINDOW_MINUTES } = {}) {
refused: a.refused ?? 0,
failed: a.failed ?? 0,
homes: a.homes ?? 0,
+ // Failures confined to ONE home are that home's problem, exactly like a
+ // handshake failure is. Without this, a single house with a broken
+ // integration downs the subsystem for everybody.
+ failedHomes: a.failed_homes ?? 0,
// A refused action is the safety gate doing its job, so it counts as a
// success. Only `failed` is us not delivering.
rate: (a.total ?? 0) > 0 ? ((a.ok ?? 0) + (a.refused ?? 0)) / a.total : null,
@@ -255,6 +296,14 @@ export async function readHomeSignals({ windowMinutes = WINDOW_MINUTES } = {}) {
integrity: {
violations: v.violations ?? 0,
lastAt: v.last_at ? new Date(v.last_at).toISOString() : null,
+ // Guarded actions a standing grant let through. Reported so the number
+ // is visible, never scored: the user said yes once, on purpose.
+ grantBacked: v.grant_backed ?? 0,
+ // Of those, the ones whose entity has no grant row now. Usually a grant
+ // the user revoked after the fact, which is why this reports rather than
+ // alerts. A number that climbs while nobody is revoking anything is
+ // worth reading.
+ grantBackedWithoutGrant: v.grant_backed_without_grant ?? 0,
},
pool,
leak,
@@ -317,10 +366,19 @@ export function homeHealthVerdict(s) {
const handshakeStatus = !enoughHomes || s.handshakes.rate === null
? 'ok'
: rateStatus(s.handshakes.rate, HANDSHAKE_DEGRADED, HANDSHAKE_DOWN);
- const actionStatus = s.actions.rate === null
+ // Actions get the same cross-tenant guard as handshakes, for the same reason.
+ // A house whose Z-Wave stick fell out fails every action sent to it; scoring
+ // that as a platform outage pages an operator for somebody's loose USB port.
+ // It has to be thin traffic or one house before it stops counting, never both
+ // ignored: real correlated breakage shows up across homes and at volume.
+ const actionsScoreable = s.actions.rate !== null && s.actions.total >= MIN_ACTIONS_FOR_A_VERDICT;
+ const actionStatus = !actionsScoreable
? 'ok'
- : rateStatus(s.actions.rate, ACTION_DEGRADED, ACTION_DOWN);
- const expiryStatus = s.confirmations.expiryRate === null
+ : s.actions.failedHomes <= 1
+ ? worst('ok', rateStatus(s.actions.rate, ACTION_DEGRADED, 0))
+ : rateStatus(s.actions.rate, ACTION_DEGRADED, ACTION_DOWN);
+ const decidedConfirmations = s.confirmations.redeemed + s.confirmations.expired;
+ const expiryStatus = s.confirmations.expiryRate === null || decidedConfirmations < MIN_CONFIRMATIONS_FOR_A_VERDICT
? 'ok'
: ceilingStatus(s.confirmations.expiryRate, EXPIRY_DEGRADED, EXPIRY_DOWN);
const breakerRate = s.pool.open > 0 ? s.pool.breakersOpen / s.pool.open : 0;
@@ -340,14 +398,18 @@ export function homeHealthVerdict(s) {
? `handshakes ${pct(s.handshakes.rate)} over ${s.handshakes.attempts} homes in ${s.windowMinutes}m` + (enoughHomes ? '' : ` (under the ${MIN_HOMES_FOR_A_VERDICT}-home floor, reported not scored)`)
: `no handshakes in ${s.windowMinutes}m`,
s.actions.total > 0
- ? `actions ${pct(s.actions.rate)} of ${s.actions.total} across ${s.actions.homes} homes` + (s.actions.failed ? `, ${s.actions.failed} failed` : '')
+ ? `actions ${pct(s.actions.rate)} of ${s.actions.total} across ${s.actions.homes} homes`
+ + (s.actions.failed ? `, ${s.actions.failed} failed in ${s.actions.failedHomes} home${s.actions.failedHomes === 1 ? ' (that house, not us)' : 's'}` : '')
+ + (s.actions.total >= MIN_ACTIONS_FOR_A_VERDICT ? '' : ` (under the ${MIN_ACTIONS_FOR_A_VERDICT}-action floor, reported not scored)`)
: 'no actions in window',
s.actions.p95LatencyMs === null
? 'no action timings recorded'
: `p95 our-leg latency ${s.actions.p95LatencyMs}ms over ${s.actions.timed} timed actions`,
s.confirmations.total > 0
? `confirmations ${pct(s.confirmations.expiryRate ?? 0)} expired of ${s.confirmations.total}`
+ + (decidedConfirmations >= MIN_CONFIRMATIONS_FOR_A_VERDICT ? '' : ` (under the ${MIN_CONFIRMATIONS_FOR_A_VERDICT}-confirmation floor, reported not scored)`)
: 'no confirmations in window',
+ s.integrity.grantBacked ? `${s.integrity.grantBacked} guarded action(s) cleared by a standing grant` : null,
`pool ${s.pool.open} open, ${s.pool.subscribers} subscribers` + (s.pool.streams === null ? '' : ` across ${s.pool.streams} streams`) + `, ${s.pool.breakersOpen} breakers open` + (s.pool.rung && s.pool.rung !== 'normal' ? `, admission ${s.pool.rung}` : ''),
];
@@ -358,14 +420,16 @@ export function homeHealthVerdict(s) {
: handshakeStatus !== 'ok'
? 'Handshakes are failing across tenants, which is almost always us: a deploy, an egress change or a DNS fault. Run the correlation query in docs/ops/home-operations.md before touching anything.'
: actionStatus !== 'ok'
- ? 'Actions are failing across homes. Group home_action_log by outcome and detail->>\'code\' to see whether this is one error or many.'
+ ? (s.actions.failedHomes <= 1
+ ? 'Action failures are confined to one home, so this is that house rather than the platform. Read its status_detail and its own action log before looking anywhere else.'
+ : 'Actions are failing across homes. Group home_action_log by outcome and detail->>\'code\' to see whether this is one error or many.')
: expiryStatus !== 'ok'
? 'Confirmations are timing out rather than being answered. That is a UI failure, not user hesitation: check that the confirm prompt is reaching the surface the user is actually on.'
: latencyStatus !== 'ok'
? 'Our own leg of the act path is slow. The house is excluded from this measurement, so look at the pool: cold opens, breaker retries, or a store read on the hot path.'
: 'Breakers are open on a large share of this instance\'s pooled homes. Correlated, that means we cannot reach houses we could reach an hour ago.';
- return { status, detail: parts.join('; ') + '.', ...(hint ? { hint } : {}) };
+ return { status, detail: parts.filter(Boolean).join('; ') + '.', ...(hint ? { hint } : {}) };
}
/**
diff --git a/docs/ops/home-operations.md b/docs/ops/home-operations.md
index 4634527f3..f8feda4b4 100644
--- a/docs/ops/home-operations.md
+++ b/docs/ops/home-operations.md
@@ -30,15 +30,21 @@ So the rule, stated once and enforced in code:
| The pool gauges | `stats()` in [`api/_lib/home/runtime.js`](../../api/_lib/home/runtime.js) |
| The tables | `home_connections`, `home_action_log`, `home_confirmations` |
-Every rate is computed **across tenants** over a 15 minute window, and nothing is
-scored at all until at least 10 homes are connected (`MIN_HOMES_FOR_A_VERDICT`).
-With three homes, one holiday cottage losing power is a 33% failure rate, and
-scoring that as an outage is the exact false page this design exists to prevent.
-Below the floor the subsystem reports the numbers and stays green, saying so:
+Every rate is computed **across tenants** over a 15 minute window, and each one
+has a floor below which it is reported but not scored:
+
+| Rate | Floor | Why |
+|---|---|---|
+| Handshake success | 10 connected homes (`MIN_HOMES_FOR_A_VERDICT`) | With three homes, one holiday cottage losing power is a 33% failure rate. |
+| Action success | 20 actions (`MIN_ACTIONS_FOR_A_VERDICT`), and failures in more than one home | A house whose Z-Wave stick fell out fails everything sent to it. Paging for that is paging for a loose USB port. Failures confined to one home cap at `degraded` and the hint names the house. |
+| Confirmation expiry | 10 decided confirmations (`MIN_CONFIRMATIONS_FOR_A_VERDICT`) | Three prompts in a quiet hour, two of them from someone who walked away from their laptop, is not a UI failure. |
+
+Below a floor the subsystem reports the numbers and stays green, saying so:
```
-1/1 homes connected; handshakes 100.0% over 1 homes in 15m (under the 10-home
-floor, reported not scored); no actions in window; ...
+14/14 homes connected; handshakes 100.0% over 3 homes in 15m; actions 93.3% of 30
+across 5 homes, 2 failed in 1 home (that house, not us); confirmations 75.0%
+expired of 4 (under the 10-confirmation floor, reported not scored); ...
```
### The thresholds
@@ -51,12 +57,22 @@ floor, reported not scored); no actions in window; ...
| Breaker-open homes | `runtime.stats()` | under 2% | 2 to 10% | over 10% |
| p95 action latency (our leg) | `home_action_log.detail->>'latencyMs'` | under 1.5 s | 1.5 to 4 s | over 4 s |
| Subscriber leak | `stats().subscribers` versus open streams | flat | growing | growing |
-| Confirmation integrity | `home_action_log` | zero rows | n/a | any row |
+| Confirmation integrity | `home_action_log`, excluding standing grants | zero rows | n/a | any row |
A **refused** action counts as a success. The safety gate refusing to open a
front door is the product working, not a failure to deliver, and scoring it as an
error would make the safest houses look like the sickest ones.
+A guarded action with no `confirmed_by` is **not** on its own a violation, and
+finding that out against real rows is what stopped this alert from firing on
+every legitimate unlock. A standing per-entity allowance in `home_entity_grants`
+is a yes the user already gave, recorded once rather than re-asked every time, so
+the act path clears the gate through the allow list and stamps
+`detail.allowed_by_grant`. Those rows are counted and reported
+(`integrity.grantBacked`), never paged. What remains a Sev 1 is the shape with no
+yes behind it at all: guarded, executed, nobody confirmed it, and no grant
+claimed.
+
## The SLOs
| SLO | Target | Window | Error budget |
@@ -71,10 +87,14 @@ guarded action executing without a valid confirmation is a Sev 1 regardless of
volume, regardless of how many succeeded, and regardless of whether any user
noticed. There is no budget to spend and no rate at which it becomes acceptable.
-**Pre-launch baseline (measured 2026-09-03, local, not production):** 3 connected
-homes, 8 actions logged across the window (3 refused by the gate, 5 executed),
-p95 our-leg latency 412 ms on the single timed action, zero integrity violations,
-zero confirmations expired. Re-measure against production traffic in order 20.
+**Pre-launch baseline (measured 2026-09-03, development database, not
+production):** 14 connected homes, 30 actions in the window across 5 homes (93.3%
+succeeding or refused, the 2 failures confined to one house), p95 our-leg latency
+412 ms on the one action carrying a timing, 2 guarded actions cleared by a
+standing grant, zero integrity violations. The act path does not yet stamp
+`detail.latencyMs` on every action, so the p95 above rests on a single sample and
+the sensor says `no action timings recorded` rather than inventing one.
+Re-measure against production traffic in order 20.
---
@@ -144,13 +164,19 @@ always pages even if an earlier one paged an hour ago.
**First command:**
```sql
-select id, home_id, user_id, actor, channel, action, entity_ids, risk, created_at
+select id, home_id, user_id, actor, channel, action, entity_ids, risk, detail, created_at
from home_action_log
where guarded = true and confirmed_by is null and outcome = 'ok'
+ and coalesce(detail->>'allowed_by_grant', 'false') <> 'true'
and created_at > now() - interval '24 hours'
order by created_at desc;
```
+Drop the `allowed_by_grant` line to see the grant-backed actions alongside them.
+Those are legitimate and are the reason that line is there: without it, this
+query returns every standing-grant unlock in the fleet and the alert becomes
+noise within a day.
+
**Then, in this order:**
1. **Scope it.** How many rows, how many homes, which entities. A `lock.*`,
@@ -163,11 +189,15 @@ order by created_at desc;
are the only things standing between a model and a front door. A refactor that
changed a domain list, or a caller that passed `confirmed: true` from
somewhere other than a human, is the likely cause.
-4. **Check for a standing grant.** A per-entity allowance in
- `home_entity_grants` is a legitimate reason for a guarded action to run
- without a fresh confirmation, and the act path must record the grant in
- `detail` when it uses one. A violation row with a matching live grant is a
- logging bug; a violation row with no grant is the real thing.
+4. **Check the grant claim.** The alert already excludes actions that recorded
+ `detail.allowed_by_grant`, so a row that reached you claimed no grant at all.
+ Confirm that against `home_entity_grants` for the home and entity: a live
+ grant with no claim on the row is a logging bug in the act path
+ (`api/home/[id]/call.js`), which is a much smaller problem than the alternative
+ and still worth fixing the same day. No grant and no claim is the real thing.
+ `integrity.grantBackedWithoutGrant` on the health block counts rows that
+ claimed a grant that no longer exists; that is usually a grant the user
+ revoked afterwards, which is why it reports rather than pages.
**Rollback:** yes. Roll back to the last revision known to gate correctly before
diagnosing further.
diff --git a/tests/home-integrity.test.js b/tests/home-integrity.test.js
index 385275e3c..d1a266703 100644
--- a/tests/home-integrity.test.js
+++ b/tests/home-integrity.test.js
@@ -18,6 +18,8 @@ import {
HANDSHAKE_DOWN,
homeHealthVerdict,
LATENCY_DOWN_MS,
+ MIN_ACTIONS_FOR_A_VERDICT,
+ MIN_CONFIRMATIONS_FOR_A_VERDICT,
MIN_HOMES_FOR_A_VERDICT,
noteSubscriberSample,
resetSubscriberSamples,
@@ -29,9 +31,9 @@ function healthy(overrides = {}) {
windowMinutes: 15,
homes: { live: 40, connected: 40, unreachable: 0, authFailed: 0 },
handshakes: { attempts: 40, ok: 40, failed: 0, rate: 1 },
- actions: { total: 200, ok: 198, refused: 2, failed: 0, homes: 30, rate: 1, timed: 200, p95LatencyMs: 300 },
+ actions: { total: 200, ok: 198, refused: 2, failed: 0, homes: 30, failedHomes: 0, rate: 1, timed: 200, p95LatencyMs: 300 },
confirmations: { total: 20, redeemed: 19, expired: 1, expiryRate: 1 / 20 },
- integrity: { violations: 0, lastAt: null },
+ integrity: { violations: 0, lastAt: null, grantBacked: 0, grantBackedWithoutGrant: 0 },
pool: { open: 12, subscribers: 12, capacity: 200, breakersOpen: 0, byStatus: { connected: 12 }, streams: 12, rung: 'normal' },
leak: { leaking: false, samples: [12, 12, 12], margins: [0, 0, 0], growth: 0 },
...overrides,
@@ -40,17 +42,37 @@ function healthy(overrides = {}) {
describe('confirmation integrity is a zero-budget invariant', () => {
it('takes the subsystem down on a single row, however healthy everything else is', () => {
- const verdict = homeHealthVerdict(healthy({ integrity: { violations: 1, lastAt: '2026-09-03T03:37:14.568Z' } }));
+ const verdict = homeHealthVerdict(healthy({ integrity: { violations: 1, lastAt: '2026-09-03T03:37:14.568Z', grantBacked: 0, grantBackedWithoutGrant: 0 } }));
expect(verdict.status).toBe('down');
expect(verdict.detail).toContain('no confirmation on record');
expect(verdict.hint).toContain('Sev 1');
});
+ it('does not fire on an unlock a standing grant already authorised', () => {
+ // The shape real traffic produces: the user granted the agent lock.kitchen_door
+ // once, so the gate cleared it through the allow list and nobody was asked
+ // again. Paging on this would page on every legitimate unlock forever.
+ const verdict = homeHealthVerdict(
+ healthy({ integrity: { violations: 0, lastAt: null, grantBacked: 2, grantBackedWithoutGrant: 0 } }),
+ );
+ expect(verdict.status).toBe('ok');
+ expect(verdict.detail).toContain('2 guarded action(s) cleared by a standing grant');
+ });
+
+ it('reports a grant-backed action whose grant no longer exists without alerting on it', () => {
+ // A grant revoked after the action is the ordinary explanation, which is
+ // why this is a number to read rather than a page to answer.
+ const verdict = homeHealthVerdict(
+ healthy({ integrity: { violations: 0, lastAt: null, grantBacked: 3, grantBackedWithoutGrant: 1 } }),
+ );
+ expect(verdict.status).toBe('ok');
+ });
+
it('outranks every other signal, so it is never masked by a busy platform', () => {
const verdict = homeHealthVerdict(
healthy({
- integrity: { violations: 3, lastAt: '2026-09-03T03:37:14.568Z' },
- actions: { total: 1000, ok: 1000, refused: 0, failed: 0, homes: 90, rate: 1, timed: 1000, p95LatencyMs: 120 },
+ integrity: { violations: 3, lastAt: '2026-09-03T03:37:14.568Z', grantBacked: 0, grantBackedWithoutGrant: 0 },
+ actions: { total: 1000, ok: 1000, refused: 0, failed: 0, homes: 90, failedHomes: 0, rate: 1, timed: 1000, p95LatencyMs: 120 },
}),
);
expect(verdict.status).toBe('down');
@@ -120,7 +142,7 @@ describe('actions', () => {
it('counts a refused action as a success, because the gate working is not a fault', () => {
// Every action refused, none failed: the safety gate did its job all day.
const verdict = homeHealthVerdict(
- healthy({ actions: { total: 100, ok: 0, refused: 100, failed: 0, homes: 20, rate: 1, timed: 100, p95LatencyMs: 200 } }),
+ healthy({ actions: { total: 100, ok: 0, refused: 100, failed: 0, homes: 20, failedHomes: 0, rate: 1, timed: 100, p95LatencyMs: 200 } }),
);
expect(verdict.status).toBe('ok');
});
@@ -129,12 +151,32 @@ describe('actions', () => {
const rate = 0.9;
expect(rate).toBeLessThan(ACTION_DOWN);
const verdict = homeHealthVerdict(
- healthy({ actions: { total: 100, ok: 90, refused: 0, failed: 10, homes: 25, rate, timed: 100, p95LatencyMs: 300 } }),
+ healthy({ actions: { total: 100, ok: 90, refused: 0, failed: 10, homes: 25, failedHomes: 6, rate, timed: 100, p95LatencyMs: 300 } }),
);
expect(verdict.status).toBe('down');
expect(verdict.detail).toContain('10 failed');
});
+ it('does not down the platform for failures confined to one house', () => {
+ // One home whose Z-Wave stick fell out fails everything sent to it. That is
+ // that house, and paging for it is paging for a loose USB port.
+ const verdict = homeHealthVerdict(
+ healthy({ actions: { total: 100, ok: 88, refused: 0, failed: 12, homes: 25, failedHomes: 1, rate: 0.88, timed: 100, p95LatencyMs: 300 } }),
+ );
+ expect(verdict.status).toBe('degraded');
+ expect(verdict.detail).toContain('12 failed in 1 home (that house, not us)');
+ expect(verdict.hint).toContain('confined to one home');
+ });
+
+ it('does not score a thin window at all', () => {
+ const total = MIN_ACTIONS_FOR_A_VERDICT - 1;
+ const verdict = homeHealthVerdict(
+ healthy({ actions: { total, ok: total - 4, refused: 0, failed: 4, homes: 5, failedHomes: 4, rate: (total - 4) / total, timed: total, p95LatencyMs: 300 } }),
+ );
+ expect(verdict.status).toBe('ok');
+ expect(verdict.detail).toContain(`under the ${MIN_ACTIONS_FOR_A_VERDICT}-action floor`);
+ });
+
it('says so plainly when nothing on the act path is timed yet', () => {
const verdict = homeHealthVerdict(healthy({ actions: { ...healthy().actions, timed: 0, p95LatencyMs: null } }));
expect(verdict.status).toBe('ok');
@@ -160,6 +202,14 @@ describe('confirmations that expire are a UI failure', () => {
expect(verdict.hint).toContain('That is a UI failure, not user hesitation');
});
+ it('does not score three prompts as a broken UI', () => {
+ const verdict = homeHealthVerdict(
+ healthy({ confirmations: { total: 4, redeemed: 1, expired: 3, expiryRate: 0.75 } }),
+ );
+ expect(verdict.status).toBe('ok');
+ expect(verdict.detail).toContain(`under the ${MIN_CONFIRMATIONS_FOR_A_VERDICT}-confirmation floor`);
+ });
+
it('ignores an empty window rather than scoring it as perfect or broken', () => {
const verdict = homeHealthVerdict(
healthy({ confirmations: { total: 0, redeemed: 0, expired: 0, expiryRate: null } }),