Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M18-003): Redis CACHE client must fail fast: /track cannot block for 42 seconds on a dependency the ingest path already treats as optional

ralph1d213a3d0cd6
diff --git a/apps/api/e2e/README.md b/apps/api/e2e/README.md
index a766e19d2..5487526fe 100644
--- a/apps/api/e2e/README.md
+++ b/apps/api/e2e/README.md
@@ -349,3 +349,22 @@ dev Redis's live queues, and it refuses to start if the event/group buffers are
non-empty (the real `flushEvents` handler it runs would drain them). The
un-namespaced keys' byte-identity is pinned separately by
`packages/core/src/jobs/naming.test.ts`.
+
+## `redis-fail-fast.ts` / `redis-command-cost.ts` — the M18-003 fault proof
+
+Also independent of the session harnesses and of `pnpm dev`. `redis-fail-fast.ts`
+spawns its own API on `:3399` pointed at a TCP proxy it can stop (`docker stop`)
+or freeze (`docker pause`), and measures what `/track` costs while the Redis
+**cache** is unreachable — sustained outage, 1 s blip under load, frozen server,
+and a boot with Redis already down. `redis-command-cost.ts` measures the cost of
+a *legitimate* command, which is what bounds `CACHE_COMMAND_TIMEOUT_MS`.
+
+```bash
+cd apps/api
+LOG_LEVEL=warn dotenv -e ../../.env -- bun e2e/redis-fail-fast.ts # M18_QUICK=1 for phases A-C
+dotenv -e ../../.env -- bun e2e/redis-command-cost.ts
+```
+
+Neither touches the shared dev stack: the proxied API uses Redis database 3 and
+its own Kafka topic, the cost bench database 4. The before/after numbers both
+produced, and the reasoning they justify, are in **`redis-fail-fast.md`**.
diff --git a/apps/api/e2e/redis-command-cost.ts b/apps/api/e2e/redis-command-cost.ts
new file mode 100644
index 000000000..bf82ec850
--- /dev/null
+++ b/apps/api/e2e/redis-command-cost.ts
@@ -0,0 +1,181 @@
+/**
+ * M18-003 — what does a LEGITIMATE Redis command on the cache client cost?
+ *
+ * `commandTimeout` on the cache client is a deadline every command on the
+ * ingest path now runs under, including the event buffer's writes and reads.
+ * Picking it needs the cost of the heaviest honest command, not a guess: if
+ * the deadline is anywhere near a busy flush, a healthy server starts looking
+ * like a down one and M18-001's shutdown path starts refusing to commit
+ * offsets it should have committed.
+ *
+ * Measures the four shapes the cache client actually issues, worst first:
+ * - `MULTI` of `EVENT_BUFFER_BATCH_SIZE` rpush — the shutdown flush of a
+ * full local buffer (`event-buffer.ts:writeToRedis`).
+ * - `MULTI` of the micro-batch size — the steady-state ingest write.
+ * - `LRANGE 0..batchSize-1` — the cron flush read (`processBuffer`).
+ * - `GET` — every cache/auth lookup.
+ *
+ * Runs against the local server in its own database with its own key prefix.
+ *
+ * dotenv -e ../../.env -- bun e2e/redis-command-cost.ts
+ */
+
+import { Redis } from '@openpanel/redis';
+
+/** Its own database: this must not touch anything the dev stack is using. */
+const BENCH_REDIS_DB = 4;
+const KEY = 'm18003:command-cost';
+
+/** `event-buffer.ts` — DEFAULT_BATCH_SIZE / DEFAULT_MICRO_BATCH_SIZE. */
+const EVENT_BUFFER_BATCH_SIZE = 4000;
+const EVENT_BUFFER_MICRO_BATCH_SIZE = 100;
+
+const GET_ITERATIONS = 2000;
+const MICRO_BATCH_ITERATIONS = 200;
+const FULL_BATCH_ITERATIONS = 20;
+const LRANGE_ITERATIONS = 100;
+const WARMUP_ITERATIONS = 5;
+
+/** A row shaped like the JSONEachRow lines the buffer actually stores. */
+function sampleEventRow(index: number): string {
+ return JSON.stringify({
+ id: crypto.randomUUID(),
+ name: 'screen_view',
+ device_id: crypto.randomUUID(),
+ profile_id: '',
+ project_id: 'e2e1e2e1-0000-4000-8000-000000000001',
+ session_id: crypto.randomUUID(),
+ path: `/pricing/plan/${index}`,
+ origin: 'https://openpanel.dev',
+ referrer: 'https://www.google.com/',
+ referrer_name: 'Google',
+ referrer_type: 'search',
+ duration: 0,
+ created_at: new Date().toISOString(),
+ country: 'SE',
+ city: 'Stockholm',
+ region: 'Stockholm County',
+ longitude: 18.0686,
+ latitude: 59.3293,
+ os: 'Mac OS',
+ os_version: '10.15.7',
+ browser: 'Chrome',
+ browser_version: '148.0.0.0',
+ device: 'desktop',
+ brand: 'Apple',
+ model: '',
+ imported_at: null,
+ sdk_name: '@openpanel/web',
+ sdk_version: '1.0.9',
+ properties: {
+ __hash: '',
+ __query:
+ 'utm_source=newsletter&utm_medium=email&utm_campaign=launch-week',
+ __title: 'Pricing — OpenPanel, the open source analytics platform',
+ __reason: 'navigation',
+ plan: 'business',
+ seats: 12,
+ experiment_bucket: 'variant-b',
+ },
+ });
+}
+
+interface Timing {
+ label: string;
+ samples: number[];
+ bytes?: number;
+}
+
+function report({ label, samples, bytes }: Timing): string {
+ const sorted = [...samples].sort((a, b) => a - b);
+ const at = (quantile: number) =>
+ sorted[Math.min(sorted.length - 1, Math.floor(quantile * sorted.length))] ??
+ 0;
+ const size = bytes === undefined ? '' : ` payload=${bytes}B`;
+ return `${label.padEnd(38)} n=${samples.length}${size} p50=${at(0.5).toFixed(2)}ms p95=${at(0.95).toFixed(2)}ms p99=${at(0.99).toFixed(2)}ms max=${(sorted.at(-1) ?? 0).toFixed(2)}ms`;
+}
+
+async function time(iterations: number, run: () => Promise<unknown>) {
+ for (let index = 0; index < WARMUP_ITERATIONS; index++) {
+ await run();
+ }
+ const samples: number[] = [];
+ for (let index = 0; index < iterations; index++) {
+ const startedAt = performance.now();
+ await run();
+ samples.push(performance.now() - startedAt);
+ }
+ return samples;
+}
+
+async function main() {
+ const url = new URL(process.env.REDIS_URL || 'redis://127.0.0.1:6379');
+ url.pathname = `/${BENCH_REDIS_DB}`;
+ const redis = new Redis(url.toString());
+ const rows = Array.from({ length: EVENT_BUFFER_BATCH_SIZE }, (_, index) =>
+ sampleEventRow(index)
+ );
+ const rowBytes = Math.round(
+ rows.reduce((total, row) => total + Buffer.byteLength(row), 0) / rows.length
+ );
+
+ console.log(
+ `redis ${url.host} db ${BENCH_REDIS_DB}, mean row ${rowBytes}B, batch ${EVENT_BUFFER_BATCH_SIZE}`
+ );
+
+ await redis.del(KEY);
+
+ const pushMulti = async (count: number) => {
+ const multi = redis.multi();
+ for (let index = 0; index < count; index++) {
+ multi.rpush(KEY, rows[index] as string);
+ }
+ await multi.exec();
+ await redis.del(KEY);
+ };
+
+ const fullBatch = await time(FULL_BATCH_ITERATIONS, () =>
+ pushMulti(EVENT_BUFFER_BATCH_SIZE)
+ );
+ const microBatch = await time(MICRO_BATCH_ITERATIONS, () =>
+ pushMulti(EVENT_BUFFER_MICRO_BATCH_SIZE)
+ );
+
+ await redis.del(KEY);
+ await redis.rpush(KEY, ...rows);
+ const lrange = await time(LRANGE_ITERATIONS, () =>
+ redis.lrange(KEY, 0, EVENT_BUFFER_BATCH_SIZE - 1)
+ );
+ await redis.del(KEY);
+
+ await redis.set(`${KEY}:get`, rows[0] as string);
+ const get = await time(GET_ITERATIONS, () => redis.get(`${KEY}:get`));
+ await redis.del(`${KEY}:get`);
+
+ console.log(
+ report({
+ label: `MULTI ${EVENT_BUFFER_BATCH_SIZE}x rpush (shutdown flush)`,
+ samples: fullBatch,
+ bytes: rowBytes * EVENT_BUFFER_BATCH_SIZE,
+ })
+ );
+ console.log(
+ report({
+ label: `MULTI ${EVENT_BUFFER_MICRO_BATCH_SIZE}x rpush (steady state)`,
+ samples: microBatch,
+ bytes: rowBytes * EVENT_BUFFER_MICRO_BATCH_SIZE,
+ })
+ );
+ console.log(
+ report({
+ label: `LRANGE 0..${EVENT_BUFFER_BATCH_SIZE - 1} (cron flush)`,
+ samples: lrange,
+ bytes: rowBytes * EVENT_BUFFER_BATCH_SIZE,
+ })
+ );
+ console.log(report({ label: 'GET (auth / cache lookup)', samples: get }));
+
+ redis.disconnect();
+}
+
+await main();
diff --git a/apps/api/e2e/redis-fail-fast.md b/apps/api/e2e/redis-fail-fast.md
new file mode 100644
index 000000000..8e1ab8a84
--- /dev/null
+++ b/apps/api/e2e/redis-fail-fast.md
@@ -0,0 +1,290 @@
+# M18-003 — the cache client's fail-fast, measured
+
+What `/track` costs when the Redis **cache** is unreachable, before and after
+the change in `packages/redis/redis.ts`. Every number below is a paste from a
+run on this box; nothing here is estimated.
+
+**Box** — 4 cores, 15 GB, Linux 6.8.0-137; Redis 7.2.5 on `127.0.0.1:6379`;
+Postgres, ClickHouse and Redpanda local. **Date** 2026-09-13. **Base revision**
+`714dae9d`. "Before" is that revision's `packages/redis/redis.ts` verbatim;
+"after" is the same tree with this task's change applied and nothing else.
+
+## Reproducing
+
+```bash
+cd apps/api
+# the fault harness — spawns its own API on :3399 behind a TCP proxy for Redis
+LOG_LEVEL=warn dotenv -e ../../.env -- bun e2e/redis-fail-fast.ts
+# phases A-C only (the recovery-window comparison)
+LOG_LEVEL=warn M18_QUICK=1 M18_OUTAGE_MS=60000 dotenv -e ../../.env -- bun e2e/redis-fail-fast.ts
+# what a legitimate command costs, which is what bounds commandTimeout
+dotenv -e ../../.env -- bun e2e/redis-command-cost.ts
+```
+
+The comparison runs ("before", `commandTimeout` only, ioredis's default backoff)
+are the same harness against an edited `createFailFastCacheClient` — the option
+under test removed, nothing else — and the tree was restored to the committed
+version afterwards, verified by hash.
+
+The harness never touches the shared server's data: the API under test is
+pointed at a local TCP proxy it can stop (`refuse`, a `docker stop`) or freeze
+(`blackhole`, a `docker pause`), on Redis database 3 and its own Kafka topic.
+The operator's dev stack keeps its own connection to `:6379` throughout.
+
+## The headline
+
+| Fault | Shape | Before | After |
+|---|---|---|---|
+| Redis stopped | warm-auth `/track` | **2 of 3 no answer in 30 s**, the third 500 at 30,001 ms | **200 × 10, p50 10.0 ms** |
+| Redis stopped | cold-auth `/track` | 2 of 3 no answer in 30 s, one 500 at 30,001 ms | **500 × 10, p50 1.3 ms** |
+| Redis stopped | browser-origin `/track` | 500 at **12,041 ms**, one no answer in 30 s | 500 × 10, p50 1.0 ms |
+| Redis stopped | `/healthcheck` | no answer in 30 s | **503 in 17 ms** |
+| Redis frozen | warm-auth `/track` | 5 of 5 no answer in 30 s | **200 × 5 at 1,010 ms** |
+| Redis frozen | browser-origin `/track` | 5 of 5 no answer in 30 s | 500 × 5 at 502 ms |
+| booted with Redis down | first `/track` | no answer in 30 s | 500 in 516 ms |
+
+The before column reproduces drill 02 (`verification/drills/02-redis-unavailable.md`):
+12,041 ms here against its 12,039 ms, and the same "half the requests never
+answered" shape. The 41,978 ms outlier drill 02 saw is above this harness's
+30 s client timeout, so it shows up as `no-answer(AbortError)` instead.
+
+The behavioural acceptance criterion is the first two rows: **warm auth keeps
+its normal latency and still answers 200** (10.0 ms p50 against a 11.7 ms p50
+healthy baseline — the auth resolves from the process-local LRU and the session
+lookup degrades to the deterministic id), and **cold auth fails promptly**
+(1.3 ms) instead of hanging.
+
+## Raw output
+
+### Before — `714dae9d` unmodified (`M18_OUTAGE_PROBES=3`)
+
+```
+── A. healthy baseline ──
+warm-secret /track n=20 200×20 p50=10.8ms p95=12.1ms max=12.1ms
+browser-origin /track n=20 200×20 p50=11.0ms p95=14.5ms max=14.5ms
+
+── B. sustained outage (20s, refuse) ──
+warm-secret /track n=3 no-answer(AbortError)×2 500×1 p50=30001.2ms p95=30001.6ms max=30001.6ms
+cold-unknown /track n=3 no-answer(AbortError)×2 500×1 p50=30001.2ms p95=30001.7ms max=30001.7ms
+browser-origin /track n=3 500×2 no-answer(AbortError)×1 p50=12041.4ms p95=30001.5ms max=30001.5ms
+/healthz/live status=200 1.2ms
+/healthz/ready status=200 4.5ms
+/healthcheck status=no-answer(AbortError) 30001.3ms
+
+── C. recovery ──
+first browser-origin 200 after restore: 64ms (status 200)
+browser-origin /track n=20 200×20 p50=10.7ms p95=18.8ms max=18.8ms
+
+── D. 1000ms blip under load ──
+warm-secret, control n=2745 200×2745 p50=10.8ms p95=12.8ms max=18.1ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=3
+warm-secret, blip n=2313 200×2313 p50=10.7ms p95=12.2ms max=1074.2ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=3
+browser-origin, control n=2490 200×2490 p50=12.0ms p95=13.9ms max=23.2ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=2
+browser-origin, blip n=2030 200×2030 p50=12.2ms p95=13.9ms max=1069.8ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=2
+
+── E. blackhole (frozen dependency) ──
+warm-secret /track n=5 no-answer(AbortError)×5 p50=30000.9ms p95=30001.0ms max=30001.0ms
+browser-origin /track n=5 no-answer(AbortError)×5 p50=30000.8ms p95=30001.0ms max=30001.0ms
+
+── F. boot with Redis down ──
+booted with redis unreachable: true (2235ms)
+ /healthz/live status=200 4.7ms
+ /healthz/ready status=200 2.0ms
+ /healthcheck status=503 10552.1ms
+ cold /track (never had redis) n=1 no-answer(AbortError)×1 p50=30001.0ms p95=30001.0ms max=30001.0ms
+ /track once redis returns: status=200 15.2ms
+ /healthz/ready status=200 1.1ms
+ /healthcheck status=200 12.5ms
+```
+
+### After — with the change
+
+```
+── A. healthy baseline ──
+warm-secret /track n=20 200×20 p50=11.7ms p95=13.0ms max=13.0ms
+browser-origin /track n=20 200×20 p50=11.8ms p95=14.6ms max=14.6ms
+
+── B. sustained outage (20s, refuse) ──
+warm-secret /track n=10 200×10 p50=10.0ms p95=11.8ms max=11.8ms
+cold-unknown /track n=10 500×10 p50=1.3ms p95=3.0ms max=3.0ms
+browser-origin /track n=10 500×10 p50=1.0ms p95=1.4ms max=1.4ms
+/healthz/live status=200 0.9ms
+/healthz/ready status=200 2.4ms
+/healthcheck status=503 17.1ms
+
+── C. recovery ──
+first browser-origin 200 after restore: 336ms (status 200)
+browser-origin /track n=20 200×20 p50=11.0ms p95=12.1ms max=12.1ms
+
+── D. 1000ms blip under load ──
+warm-secret, control n=2620 200×2620 p50=11.3ms p95=13.5ms max=19.0ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=2
+warm-secret, blip n=2830 200×2830 p50=10.8ms p95=12.7ms max=18.4ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=2
+browser-origin, control n=2320 200×2320 p50=12.5ms p95=18.2ms max=26.5ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=3
+browser-origin, blip n=6840 200×1980 500×4860 p50=1.2ms p95=13.0ms max=17.8ms
+ failures=4860 over 1220ms of wall clock, distinct sessionIds=3
+
+── E. blackhole (frozen dependency) ──
+warm-secret /track n=5 200×5 p50=1009.7ms p95=1010.1ms max=1010.1ms
+browser-origin /track n=5 500×5 p50=501.8ms p95=501.9ms max=501.9ms
+
+── F. boot with Redis down ──
+booted with redis unreachable: true (2224ms)
+ /healthz/live status=200 2.5ms
+ /healthz/ready status=200 3.7ms
+ /healthcheck status=503 503.5ms
+ cold /track (never had redis) n=1 500×1 p50=515.8ms p95=515.8ms max=515.8ms
+ /track once redis returns: status=200 74.5ms
+ /healthz/ready status=200 0.7ms
+ /healthcheck status=200 6.7ms
+```
+
+## The mechanism, and what it costs
+
+Two settings, because there are two faults, and `commandTimeout` alone does not
+fix the one that matters.
+
+**1. The offline queue goes off once the client has connected** — a command
+issued while the socket is known-down is rejected in microseconds instead of
+queued behind a 20-attempt reconnect cycle. This is what produces the 10.0 ms
+warm `/track` and the 1.3 ms cold refusal in phase B. `commandTimeout` on its
+own would have made those 500 ms each, not 1 ms.
+
+**2. `commandTimeout: 500`** — the offline queue cannot see the fault where the
+socket is UP and the server answers nothing. Phase E is that fault: 30 s of
+nothing before, 1,010 ms after for the warm shape (two cache calls in series,
+each cut off at the 500 ms deadline) and 502 ms for the browser shape.
+
+**The cost is the blip** (phase D — a 1 s outage under 5 concurrent clients,
+each shape run twice, once without the fault as a control):
+
+| Shape | Before | After |
+|---|---|---|
+| warm-auth | 0 failures, one request stretched to 1,074 ms | 0 failures, max 18.4 ms |
+| browser-origin | 0 failures, one request stretched to 1,070 ms | **4,860 × 500 over 1,220 ms** |
+
+That is the whole trade, stated plainly: a 1 s Redis blip used to cost the
+browser-shaped `/track` a ~1 s stall and no errors; it now costs it ~1.2 s of
+500s. The warm-auth shape pays nothing either way — it degrades to the
+deterministic session id. `distinct sessionIds` is unchanged between control
+and blip in every run, so no session was split by the faster failure.
+
+### The third configuration, measured too
+
+`commandTimeout` alone — the offline queue left ON — is a real middle ground and
+was measured as its own run, same harness, same box:
+
+| | before (neither) | `commandTimeout` only | fail-fast + `commandTimeout` (chosen) |
+|---|---|---|---|
+| outage, warm-auth `/track` | 2 of 3 unanswered in 30 s | 200 at **1,011 ms** | 200 at **10.0 ms** |
+| outage, cold-auth `/track` | 2 of 3 unanswered in 30 s | 500 at 502 ms | 500 at 1.3 ms |
+| outage, `/healthcheck` | unanswered in 30 s | 503 in 504 ms | 503 in 17 ms |
+| 1 s blip, browser-origin | 0 failures, one 1,070 ms stall | **5 failures over 504 ms** | 4,860 failures over 1,220 ms |
+| frozen server, warm-auth | unanswered in 30 s | 200 at 1,010 ms | 200 at 1,010 ms |
+
+`commandTimeout` alone is much the gentler option on the blip — ioredis arms the
+deadline in `sendCommand`, before the writable check, so a queued command is
+under it too, and only the five requests that waited past 500 ms failed. But it
+does not meet this task's behavioural criterion: a warm-auth `/track` during the
+outage takes 1,011 ms, not "tens of ms", because every request still pays the
+full deadline twice over for a dependency whose answer it is going to discard.
+Fail-fast is what turns that back into 10.0 ms. The chosen configuration is the
+combination, and the blip is what it costs.
+
+Its phases B and D, verbatim:
+
+```
+── B. sustained outage (20s, refuse) ──
+warm-secret /track n=10 200×10 p50=1011.3ms p95=1011.9ms max=1011.9ms
+cold-unknown /track n=10 500×10 p50=502.1ms p95=504.3ms max=504.3ms
+browser-origin /track n=10 500×10 p50=502.2ms p95=502.5ms max=502.5ms
+/healthz/live status=200 0.9ms
+/healthz/ready status=200 3.8ms
+/healthcheck status=503 503.9ms
+
+── D. 1000ms blip under load ──
+warm-secret, control n=2520 200×2520 p50=11.8ms p95=13.8ms max=21.9ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=3
+warm-secret, blip n=2118 200×2118 p50=11.6ms p95=13.8ms max=1010.9ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=2
+browser-origin, control n=2355 200×2355 p50=12.8ms p95=14.9ms max=20.7ms
+ failures=0 over 0ms of wall clock, distinct sessionIds=2
+browser-origin, blip n=1950 200×1945 500×5 p50=12.8ms p95=14.7ms max=514.5ms
+ failures=5 over 504ms of wall clock, distinct sessionIds=3
+```
+
+### Why the reconnect backoff is capped at 500 ms
+
+With the offline queue off, the reconnect backoff becomes user-visible: while
+ioredis waits out its next attempt, every command is rejected, so the failures
+outlive the outage. ioredis's default is `min(attempt * 50, 2000)` ms, and the
+attempt counter keeps climbing for as long as the server is away (measured
+directly: attempt 28, next delay 1,400 ms, 19 s into an outage). Time from
+"Redis is back" to "`/track` answers 200 again", phase C, three runs each at
+20 s and two each at 60 s:
+
+| Outage | ioredis default cap (2,000 ms) | this change (500 ms) |
+|---|---|---|
+| 20 s | 390 / 395 / 396 ms | 349 / 340 / 342 ms |
+| 60 s | 1,137 / 1,135 ms | 416 / 413 ms |
+
+The cap costs nothing (a reconnect to a live local server is sub-millisecond)
+and bounds that window at roughly half a second however long the outage was.
+
+### Why 500 ms and not 100 ms
+
+`commandTimeout` applies to every command the cache client issues, including
+the event buffer's. If it is anywhere near a busy flush, a healthy server
+starts failing as if it were down — and under M18-001 a failed flush refuses to
+commit Kafka offsets. `bun e2e/redis-command-cost.ts`, same box, same day:
+
+```
+redis 127.0.0.1:6379 db 4, mean row 998B, batch 4000
+MULTI 4000x rpush (shutdown flush) n=20 payload=3992000B p50=21.04ms p95=33.95ms p99=33.95ms max=33.95ms
+MULTI 100x rpush (steady state) n=200 payload=99800B p50=0.70ms p95=0.81ms p99=1.77ms max=1.86ms
+LRANGE 0..3999 (cron flush) n=100 payload=3992000B p50=5.68ms p95=12.86ms p99=21.26ms max=21.26ms
+GET (auth / cache lookup) n=2000 p50=0.11ms p95=0.14ms p99=0.18ms max=4.45ms
+```
+
+500 ms is ~15x the heaviest legitimate command (a 4 MB `MULTI` of 4,000 rpush,
+34 ms) and ~2,800x a `GET`. 100 ms would be only 3x the flush, which is too
+close for a box under load.
+
+## Boot ordering — the one real risk
+
+With the offline queue off from construction, ioredis rejects every command
+issued before the first connect completes, so boot order would decide whether a
+command works. The queue therefore **stays on until the client's first `ready`**;
+a process that has never reached Redis falls back to `commandTimeout`, which is
+bounded. Phase F is the proof: an API booted with Redis already unreachable
+comes up in 2,224 ms (2,235 ms before, so booting is not slower), `/healthz/live`
+and `/healthz/ready` answer 200 in single-digit ms, `/healthcheck` reports
+**503 in 504 ms** instead of hanging for 10.5 s, the first `/track` fails in
+516 ms instead of never answering, and once Redis returns the same process
+serves 200s again (74.5 ms) with `/healthcheck` back to 200. `packages/redis/redis.test.ts`
+pins the same property as a unit test: a command issued in the same tick as the
+client's construction resolves rather than throwing "Stream isn't writeable".
+
+## Interaction with M18-001
+
+M18-001's shutdown flushes the event buffer's micro-batch to Redis and refuses
+to commit offsets if that flush fails. Fail-fast makes that flush fail *faster*,
+which is correct only if a fast rejection still reads as a failure. ioredis
+**resolves** a `MULTI` whose individual commands failed, so the property that
+matters is that a fail-fast client *rejects* the `exec()` — pinned by
+`packages/redis/redis.test.ts` ("MULTI under fail-fast", rejects in under
+100 ms once the server is gone). The rest of the chain is already covered:
+`event-buffer.test.ts:370` (the buffer rethrows and keeps the events) and
+`shutdown.test.ts:113` (a rejected `flushEventBuffer` returns
+`SHUTDOWN_EXIT_FAILED`, so nothing is committed).
+
+## Not covered here
+
+The duplicate from drill 02's finding (1) — an event landing in ClickHouse more
+than once after a handler failure plus a rebalance — is out of scope for
+M18-003 and untouched by this change.
diff --git a/apps/api/e2e/redis-fail-fast.ts b/apps/api/e2e/redis-fail-fast.ts
new file mode 100644
index 000000000..53d0e52fe
--- /dev/null
+++ b/apps/api/e2e/redis-fail-fast.ts
@@ -0,0 +1,544 @@
+/**
+ * M18-003 — how long does `/track` take when the Redis CACHE is unreachable?
+ *
+ * Drill 02 measured 9,427 / 12,039 / 41,978 ms and 10-of-20 no-answer. This
+ * harness reproduces that measurement in-repo so the fix can be justified with
+ * a before/after on the same box, and re-run by anyone.
+ *
+ * It never touches the shared Redis server: the API under test is pointed at a
+ * local TCP proxy which is then stopped (`refuse`, docker-stop shaped) or
+ * frozen (`blackhole`, docker-pause shaped). The operator's stack keeps its
+ * own connection to :6379 throughout.
+ *
+ * dotenv -e ../../.env -- bun e2e/redis-fail-fast.ts
+ */
+
+import { spawn } from 'node:child_process';
+import net from 'node:net';
+
+const UPSTREAM_HOST = '127.0.0.1';
+const UPSTREAM_PORT = 6379;
+const PROXY_PORT = Number(process.env.M18_PROXY_PORT ?? 6399);
+/** Private Redis database + Kafka topic: this harness must not perturb :6379/0. */
+const PROXY_REDIS_DB = 3;
+const API_PORT = Number(process.env.M18_API_PORT ?? 3399);
+const BOOT_API_PORT = API_PORT + 1;
+const KAFKA_TOPIC = 'events-m18003-failfast';
+
+const SECRET_CLIENT_ID = 'e2e1e2e1-0000-4000-8000-000000000002';
+const CLIENT_SECRET = 'e2e-saturation-secret';
+const CORS_ORIGIN = 'https://e2e.test';
+const UA =
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';
+
+/** What drill 02's ledger client waited before recording "no response". */
+const CLIENT_TIMEOUT_MS = Number(process.env.M18_CLIENT_TIMEOUT_MS ?? 30_000);
+const BOOT_TIMEOUT_MS = 60_000;
+const WARMUP_REQUESTS = 6;
+const BASELINE_REQUESTS = Number(process.env.M18_BASELINE_REQUESTS ?? 20);
+const OUTAGE_PROBE_REQUESTS = Number(process.env.M18_OUTAGE_PROBES ?? 10);
+const BLIP_DURATION_MS = 1000;
+const BLIP_CONCURRENCY = 5;
+const BLIP_TOTAL_MS = 6000;
+/** One device for the whole blip, so a session split shows up as a second id. */
+const BLIP_CLIENT_IP = '10.18.3.200';
+const BLACKHOLE_PROBES = 5;
+const SUSTAINED_OUTAGE_MS = Number(process.env.M18_OUTAGE_MS ?? 20_000);
+/**
+ * Phases A-C only. The recovery window (C) is the one number that has to be
+ * compared across several runs to be worth anything, and re-running the blip,
+ * the blackhole and the second boot to get it costs minutes per sample.
+ */
+const QUICK_RUN = process.env.M18_QUICK === '1';
+
+type ProxyMode = 'pass' | 'refuse' | 'blackhole';
+
+// ── The proxy ───────────────────────────────────────────────────────────────
+
+class RedisFaultProxy {
+ private mode: ProxyMode = 'pass';
+ private server: net.Server | null = null;
+ private readonly sockets = new Set<net.Socket>();
+
+ async start(): Promise<void> {
+ await this.listen();
+ }
+
+ private listen(): Promise<void> {
+ return new Promise((resolve, reject) => {
+ const server = net.createServer((client) => this.onConnection(client));
+ server.on('error', reject);
+ server.listen(PROXY_PORT, UPSTREAM_HOST, () => {
+ this.server = server;
+ resolve();
+ });
+ });
+ }
+
+ private onConnection(client: net.Socket) {
+ this.sockets.add(client);
+ client.on('error', () => client.destroy());
+ client.on('close', () => this.sockets.delete(client));
+
+ // Frozen dependency: the handshake completes, nothing ever answers.
+ if (this.mode === 'blackhole') {
+ client.resume();
+ return;
+ }
+
+ const upstream = net.connect(UPSTREAM_PORT, UPSTREAM_HOST);
+ this.sockets.add(upstream);
+ upstream.on('error', () => {
+ upstream.destroy();
+ client.destroy();
+ });
+ upstream.on('close', () => {
+ this.sockets.delete(upstream);
+ client.destroy();
+ });
+ client.on('data', (chunk) => {
+ if (this.mode === 'pass') {
+ upstream.write(chunk);
+ }
+ });
+ upstream.on('data', (chunk) => {
+ if (this.mode === 'pass') {
+ client.write(chunk);
+ }
+ });
+ client.on('close', () => upstream.destroy());
+ }
+
+ /** docker stop: the port stops answering and open sockets die. */
+ async refuse(): Promise<void> {
+ this.mode = 'refuse';
+ // Sockets first: `server.close()` does not call back until every open
+ // connection has gone, so closing before destroying deadlocks.
+ this.destroyAll();
+ await this.closeServer();
+ }
+
+ /** docker pause: sockets stay open, bytes stop moving. */
+ async blackhole(): Promise<void> {
+ if (!this.server) {
+ await this.listen();
+ }
+ this.mode = 'blackhole';
+ }
+
+ async pass(): Promise<void> {
+ if (!this.server) {
+ await this.listen();
+ }
+ // Sockets opened while frozen have no upstream; drop them so ioredis
+ // rebuilds a real one.
+ if (this.mode === 'blackhole') {
+ this.destroyAll();
+ }
+ this.mode = 'pass';
+ }
+
+ private destroyAll() {
+ for (const socket of this.sockets) {
+ socket.destroy();
+ }
+ this.sockets.clear();
+ }
+
+ private closeServer(): Promise<void> {
+ const server = this.server;
+ this.server = null;
+ if (!server) {
+ return Promise.resolve();
+ }
+ return new Promise((resolve) => server.close(() => resolve()));
+ }
+
+ async stop(): Promise<void> {
+ this.destroyAll();
+ await this.closeServer();
+ }
+}
+
+// ── Probes ──────────────────────────────────────────────────────────────────
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+type Shape = 'warm-secret' | 'cold-unknown' | 'browser-origin';
+
+let deviceCounter = 0;
+let payloadCounter = 0;
+
+function headersFor(shape: Shape): Record<string, string> {
+ const base: Record<string, string> = {
+ 'content-type': 'application/json',
+ 'user-agent': UA,
+ 'x-client-ip': `10.18.3.${(deviceCounter++ % 250) + 1}`,
+ };
+ if (shape === 'warm-secret') {
+ return {
+ ...base,
+ 'openpanel-client-id': SECRET_CLIENT_ID,
+ 'openpanel-client-secret': CLIENT_SECRET,
+ };
+ }
+ if (shape === 'browser-origin') {
+ return {
+ ...base,
+ 'openpanel-client-id': SECRET_CLIENT_ID,
+ origin: CORS_ORIGIN,
+ };
+ }
+ return {
+ ...base,
+ 'openpanel-client-id': crypto.randomUUID(),
+ 'openpanel-client-secret': CLIENT_SECRET,
+ };
+}
+
+interface Probe {
+ status: number | null;
+ ms: number;
+ error?: string;
+ /** ms since the process started, so a phase can locate its failure window. */
+ sentAt: number;
+ /** `/track`'s answer, when it gave one — the blip's real cost is a session split. */
+ sessionId?: string;
+}
+
+/**
+ * A client that gives up after `CLIENT_TIMEOUT_MS`, the way drill 02's ledger
+ * client did. An explicit controller rather than `AbortSignal.timeout`, which
+ * does not interrupt an in-flight Bun fetch here.
+ */
+async function timedFetch(url: string, init: RequestInit = {}): Promise<Probe> {
+ const controller = new AbortController();
+ const giveUp = setTimeout(() => controller.abort(), CLIENT_TIMEOUT_MS);
+ const startedAt = performance.now();
+ try {
+ const res = await fetch(url, {
+ ...init,
+ signal: controller.signal,
+ // No keep-alive: a socket left behind by an abandoned request must not
+ // put the next probe behind it and inflate its latency.
+ headers: {
+ ...(init.headers as Record<string, string>),
+ connection: 'close',
+ },
+ });
+ const text = await res.text();
+ let sessionId: string | undefined;
+ try {
+ sessionId = (JSON.parse(text) as { sessionId?: string }).sessionId;
+ } catch {
+ sessionId = undefined;
+ }
+ return {
+ status: res.status,
+ ms: performance.now() - startedAt,
+ sentAt: startedAt,
+ sessionId,
+ };
+ } catch (error) {
+ return {
+ status: null,
+ ms: performance.now() - startedAt,
+ sentAt: startedAt,
+ error: (error as Error).name,
+ };
+ } finally {
+ clearTimeout(giveUp);
+ }
+}
+
+function trackOnce(
+ shape: Shape,
+ port = API_PORT,
+ fixedIp?: string
+): Promise<Probe> {
+ const headers = headersFor(shape);
+ if (fixedIp) {
+ headers['x-client-ip'] = fixedIp;
+ }
+ return timedFetch(`http://127.0.0.1:${port}/track`, {
+ method: 'POST',
+ headers,
+ body: JSON.stringify({
+ type: 'track',
+ payload: {
+ name: 'm18003_failfast',
+ // Unique per request: an identical payload from the same ip+origin is
+ // collapsed by the dedupe lock into `200 "Duplicate event"`, which
+ // would make the browser-shaped phases measure the lock and nothing
+ // past it.
+ properties: {
+ __path: 'https://e2e.test/m18003',
+ seq: payloadCounter++,
+ },
+ },
+ }),
+ });
+}
+
+function probeUrl(path: string, port = API_PORT): Promise<Probe> {
+ return timedFetch(`http://127.0.0.1:${port}${path}`);
+}
+
+function summarize(label: string, probes: Probe[]): string {
+ const statuses = new Map<string, number>();
+ for (const p of probes) {
+ const key = p.status === null ? `no-answer(${p.error})` : String(p.status);
+ statuses.set(key, (statuses.get(key) ?? 0) + 1);
+ }
+ const sorted = probes.map((p) => p.ms).sort((a, b) => a - b);
+ const at = (q: number) =>
+ sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] ?? 0;
+ return `${label.padEnd(34)} n=${probes.length} ${[...statuses]
+ .map(([k, v]) => `${k}×${v}`)
+ .join(' ')
+ .padEnd(
+ 22
+ )} p50=${at(0.5).toFixed(1)}ms p95=${at(0.95).toFixed(1)}ms max=${(sorted.at(-1) ?? 0).toFixed(1)}ms`;
+}
+
+// ── The API under test ──────────────────────────────────────────────────────
+
+function spawnApi(port: number) {
+ const child = spawn('bun', ['src/main.ts'], {
+ cwd: new URL('..', import.meta.url).pathname,
+ env: {
+ ...process.env,
+ ROLE: 'api',
+ API_PORT: String(port),
+ REDIS_URL: `redis://${UPSTREAM_HOST}:${PROXY_PORT}/${PROXY_REDIS_DB}`,
+ KAFKA_EVENTS_TOPIC: KAFKA_TOPIC,
+ LOG_LEVEL: process.env.M18_LOG_LEVEL ?? 'warn',
+ },
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+ const lines: string[] = [];
+ child.stdout?.on('data', (d) => lines.push(String(d)));
+ child.stderr?.on('data', (d) => lines.push(String(d)));
+ return { child, lines };
+}
+
+async function waitForLive(port: number, timeoutMs = BOOT_TIMEOUT_MS) {
+ const deadline = Date.now() + timeoutMs;
+ while (Date.now() < deadline) {
+ const res = await fetch(`http://127.0.0.1:${port}/healthz/live`).catch(
+ () => null
+ );
+ if (res?.ok) {
+ return true;
+ }
+ await sleep(200);
+ }
+ return false;
+}
+
+async function main() {
+ const proxy = new RedisFaultProxy();
+ await proxy.start();
+ console.log(
+ `proxy 127.0.0.1:${PROXY_PORT} → ${UPSTREAM_HOST}:${UPSTREAM_PORT} (db ${PROXY_REDIS_DB}), api :${API_PORT}, topic ${KAFKA_TOPIC}`
+ );
+
+ const { child, lines } = spawnApi(API_PORT);
+ const cleanup = async () => {
+ child.kill('SIGKILL');
+ await proxy.stop();
+ };
+
+ try {
+ if (!(await waitForLive(API_PORT))) {
+ console.log(lines.join(''));
+ throw new Error('api did not become live');
+ }
+
+ // ── warm-up: the L1 caches the criterion turns on ──────────────────────
+ for (let i = 0; i < WARMUP_REQUESTS; i++) {
+ await trackOnce('warm-secret');
+ await trackOnce('browser-origin');
+ }
+
+ console.log('\n── A. healthy baseline ──');
+ const baseline: Probe[] = [];
+ for (let i = 0; i < BASELINE_REQUESTS; i++) {
+ baseline.push(await trackOnce('warm-secret'));
+ }
+ console.log(summarize('warm-secret /track', baseline));
+ const baselineBrowser: Probe[] = [];
+ for (let i = 0; i < BASELINE_REQUESTS; i++) {
+ baselineBrowser.push(await trackOnce('browser-origin'));
+ }
+ console.log(summarize('browser-origin /track', baselineBrowser));
+
+ // ── B. sustained outage ───────────────────────────────────────────────
+ console.log(
+ `\n── B. sustained outage (${SUSTAINED_OUTAGE_MS / 1000}s, refuse) ──`
+ );
+ const outageStart = Date.now();
+ await proxy.refuse();
+ await sleep(500);
+
+ const warmDuringOutage: Probe[] = [];
+ for (let i = 0; i < OUTAGE_PROBE_REQUESTS; i++) {
+ warmDuringOutage.push(await trackOnce('warm-secret'));
+ }
+ console.log(summarize('warm-secret /track', warmDuringOutage));
+
+ const coldDuringOutage: Probe[] = [];
+ for (let i = 0; i < OUTAGE_PROBE_REQUESTS; i++) {
+ coldDuringOutage.push(await trackOnce('cold-unknown'));
+ }
+ console.log(summarize('cold-unknown /track', coldDuringOutage));
+
+ const browserDuringOutage: Probe[] = [];
+ for (let i = 0; i < OUTAGE_PROBE_REQUESTS; i++) {
+ browserDuringOutage.push(await trackOnce('browser-origin'));
+ }
+ console.log(summarize('browser-origin /track', browserDuringOutage));
+
+ for (const path of ['/healthz/live', '/healthz/ready', '/healthcheck']) {
+ const p = await probeUrl(path);
+ console.log(
+ `${path.padEnd(34)} status=${p.status ?? `no-answer(${p.error})`} ${p.ms.toFixed(1)}ms`
+ );
+ }
+
+ const remaining = SUSTAINED_OUTAGE_MS - (Date.now() - outageStart);
+ if (remaining > 0) {
+ await sleep(remaining);
+ }
+
+ // ── C. recovery ───────────────────────────────────────────────────────
+ console.log('\n── C. recovery ──');
+ await proxy.pass();
+ const recoveredAt = Date.now();
+ // Browser-shaped on purpose: it is the shape that NEEDS Redis (the dedupe
+ // lock), so it measures when the client is actually usable again. The
+ // warm-secret shape answers 200 by degrading and would report 0 ms.
+ let recovery: Probe = { status: null, ms: 0, sentAt: 0 };
+ while (Date.now() - recoveredAt < 15_000) {
+ recovery = await trackOnce('browser-origin');
+ if (recovery.status === 200) {
+ break;
+ }
+ }
+ console.log(
+ `first browser-origin 200 after restore: ${Date.now() - recoveredAt}ms (status ${recovery.status})`
+ );
+ const afterRestore: Probe[] = [];
+ for (let i = 0; i < BASELINE_REQUESTS; i++) {
+ afterRestore.push(await trackOnce('browser-origin'));
+ }
+ console.log(summarize('browser-origin /track', afterRestore));
+
+ if (QUICK_RUN) {
+ return;
+ }
+
+ // ── D. the blip ───────────────────────────────────────────────────────
+ // The cost side of fail-fast: a reconnect blip the offline queue used to
+ // absorb. Both shapes, because they degrade differently — the warm-secret
+ // one only loses its session lookup, the browser one loses its dedupe lock.
+ console.log(`\n── D. ${BLIP_DURATION_MS}ms blip under load ──`);
+ for (const shape of ['warm-secret', 'browser-origin'] as const) {
+ // A control pass first: the same load, same duration, no fault. Without
+ // it a session id that changed for some unrelated reason would be read
+ // as a cost of the blip.
+ for (const withFault of [false, true]) {
+ const probes: Probe[] = [];
+ const runEnd = Date.now() + BLIP_TOTAL_MS;
+ const workers = Array.from({ length: BLIP_CONCURRENCY }, async () => {
+ while (Date.now() < runEnd) {
+ probes.push(await trackOnce(shape, API_PORT, BLIP_CLIENT_IP));
+ }
+ });
+ await sleep(2000);
+ if (withFault) {
+ await proxy.refuse();
+ await sleep(BLIP_DURATION_MS);
+ await proxy.pass();
+ }
+ await Promise.all(workers);
+
+ const label = withFault ? 'blip' : 'control';
+ console.log(summarize(`${shape}, ${label}`, probes));
+ const failures = probes.filter((p) => p.status !== 200);
+ const failureWindowMs =
+ failures.length === 0
+ ? 0
+ : (failures.at(-1)?.sentAt ?? 0) +
+ (failures.at(-1)?.ms ?? 0) -
+ (failures[0]?.sentAt ?? 0);
+ const sessionIds = new Set(
+ probes.filter((p) => p.sessionId).map((p) => p.sessionId)
+ );
+ console.log(
+ `${''.padEnd(34)} failures=${failures.length} over ${failureWindowMs.toFixed(0)}ms of wall clock, distinct sessionIds=${sessionIds.size}`
+ );
+ await sleep(1000);
+ }
+ }
+
+ // ── E. blackhole ──────────────────────────────────────────────────────
+ console.log('\n── E. blackhole (frozen dependency) ──');
+ await sleep(1500);
+ await proxy.blackhole();
+ await sleep(300);
+ const frozen: Probe[] = [];
+ for (let i = 0; i < BLACKHOLE_PROBES; i++) {
+ frozen.push(await trackOnce('warm-secret'));
+ }
+ console.log(summarize('warm-secret /track', frozen));
+ const frozenBrowser: Probe[] = [];
+ for (let i = 0; i < BLACKHOLE_PROBES; i++) {
+ frozenBrowser.push(await trackOnce('browser-origin'));
+ }
+ console.log(summarize('browser-origin /track', frozenBrowser));
+ await proxy.pass();
+ await sleep(1500);
+
+ // ── F. boot with Redis already down ───────────────────────────────────
+ console.log('\n── F. boot with Redis down ──');
+ await proxy.refuse();
+ const boot = spawnApi(BOOT_API_PORT);
+ const bootedAt = Date.now();
+ const live = await waitForLive(BOOT_API_PORT, 30_000);
+ console.log(
+ `booted with redis unreachable: ${live} (${Date.now() - bootedAt}ms)`
+ );
+ if (live) {
+ for (const path of ['/healthz/live', '/healthz/ready', '/healthcheck']) {
+ const p = await probeUrl(path, BOOT_API_PORT);
+ console.log(
+ ` ${path.padEnd(32)} status=${p.status ?? `no-answer(${p.error})`} ${p.ms.toFixed(1)}ms`
+ );
+ }
+ console.log(
+ ` ${summarize('cold /track (never had redis)', [await trackOnce('warm-secret', BOOT_API_PORT)])}`
+ );
+ await proxy.pass();
+ await sleep(2000);
+ const after = await trackOnce('warm-secret', BOOT_API_PORT);
+ console.log(
+ ` /track once redis returns: status=${after.status} ${after.ms.toFixed(1)}ms`
+ );
+ for (const path of ['/healthz/ready', '/healthcheck']) {
+ const p = await probeUrl(path, BOOT_API_PORT);
+ console.log(
+ ` ${path.padEnd(32)} status=${p.status ?? `no-answer(${p.error})`} ${p.ms.toFixed(1)}ms`
+ );
+ }
+ } else {
+ console.log(boot.lines.join('').slice(-4000));
+ }
+ boot.child.kill('SIGKILL');
+ } finally {
+ await cleanup();
+ }
+}
+
+await main();
+process.exit(0);
diff --git a/packages/core/src/modules/ingest/src/device-id.test.ts b/packages/core/src/modules/ingest/src/device-id.test.ts
index 330ad64a1..089394beb 100644
--- a/packages/core/src/modules/ingest/src/device-id.test.ts
+++ b/packages/core/src/modules/ingest/src/device-id.test.ts
@@ -139,6 +139,42 @@ describe('getDeviceId — session resolution', () => {
expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes
});
+ /**
+ * M18-003 made the cache client fail fast, so this read now rejects in
+ * microseconds instead of after ioredis's 20-attempt reconnect cycle. The
+ * degradation itself must be untouched: the api and the worker only agree on
+ * a session because they derive the SAME id from project/device/time bucket,
+ * so a faster failure that changed the id would split every session an
+ * outage touched.
+ */
+ it('mints the SAME deterministic id whether the session read misses or rejects', async () => {
+ const { buffer: emptyStore } = stubBuffer(null);
+ const rejecting = {
+ getExistingSession: mock(() =>
+ Promise.reject(
+ new Error(
+ "Stream isn't writeable and enableOfflineQueue options is false"
+ )
+ )
+ ),
+ } as unknown as SessionBufferReader;
+
+ const onMiss = await getDeviceId({
+ ...BASE,
+ overrideDeviceId: 'cookie-abc',
+ sessionBuffer: emptyStore,
+ });
+ const onRejection = await getDeviceId({
+ ...BASE,
+ overrideDeviceId: 'cookie-abc',
+ sessionBuffer: rejecting,
+ });
+
+ expect(onRejection).toEqual(onMiss);
+ // Pinned to the value this derivation produced before the fail-fast change.
+ expect(onRejection.sessionId).toBe('wizrZalfI_J7u-QhdRttmQ');
+ });
+
it('logs a failed session read through the caller logger, not console', async () => {
const { error, logger } = stubLogger();
const failing = {
diff --git a/packages/redis/redis.test.ts b/packages/redis/redis.test.ts
new file mode 100644
index 000000000..d4c9899da
--- /dev/null
+++ b/packages/redis/redis.test.ts
@@ -0,0 +1,290 @@
+/**
+ * M18-003 — the CACHE client must reject promptly when Redis is unreachable.
+ *
+ * Drill 02 (rev e004e199) measured `/track` blocking 9,427 / 12,039 /
+ * 41,978 ms on a stopped Redis, with 10 of 20 requests never answered inside
+ * 30 s, because the command was queued and waited out ioredis's 20-attempt
+ * reconnect cycle. These tests pin the three properties that fixed it, and the
+ * one that must NOT change (the queue client).
+ *
+ * No real Redis: an unreachable port covers "never connected", and a fake RESP
+ * server covers "connected, then gone".
+ */
+
+import net from 'node:net';
+import { afterEach, describe, expect, it } from 'vitest';
+import {
+ CACHE_COMMAND_TIMEOUT_MS,
+ createFailFastCacheClient,
+ type ExtendedRedis,
+ QUEUE_CLIENT_OPTIONS,
+} from './redis';
+
+/** ioredis's own cycle is tens of seconds; anything near it is the bug back. */
+const PROMPT_WHEN_NEVER_CONNECTED_MS = CACHE_COMMAND_TIMEOUT_MS * 4;
+/** A known-down socket must not wait for the timeout at all. */
+const INSTANT_WHEN_DISCONNECTED_MS = 100;
+const READY_TIMEOUT_MS = 5000;
+
+// ── A fake Redis, just enough of RESP for ioredis to reach `ready` ──────────
+
+const INFO_BODY = '# Server\r\nredis_version:7.4.0\r\nloading:0\r\n';
+
+/** Pull every COMPLETE `*N` array of bulk strings out of `buffered`. */
+function parseCommands(buffered: string): {
+ commands: string[][];
+ rest: string;
+} {
+ const commands: string[][] = [];
+ let offset = 0;
+ while (offset < buffered.length && buffered[offset] === '*') {
+ const headerEnd = buffered.indexOf('\r\n', offset);
+ if (headerEnd < 0) {
+ break;
+ }
+ const argCount = Number(buffered.slice(offset + 1, headerEnd));
+ let cursor = headerEnd + 2;
+ const args: string[] = [];
+ let complete = true;
+ for (let index = 0; index < argCount; index++) {
+ const lengthEnd = buffered.indexOf('\r\n', cursor);
+ if (buffered[cursor] !== '$' || lengthEnd < 0) {
+ complete = false;
+ break;
+ }
+ const length = Number(buffered.slice(cursor + 1, lengthEnd));
+ const start = lengthEnd + 2;
+ if (buffered.length < start + length + 2) {
+ complete = false;
+ break;
+ }
+ args.push(buffered.slice(start, start + length));
+ cursor = start + length + 2;
+ }
+ if (!complete) {
+ break;
+ }
+ commands.push(args);
+ offset = cursor;
+ }
+ return { commands, rest: buffered.slice(offset) };
+}
+
+function replyTo(command: string[]): string {
+ const name = (command[0] ?? '').toLowerCase();
+ if (name === 'info') {
+ return `$${INFO_BODY.length}\r\n${INFO_BODY}\r\n`;
+ }
+ if (name === 'ping') {
+ return '+PONG\r\n';
+ }
+ if (name === 'get') {
+ return '$-1\r\n';
+ }
+ return '+OK\r\n';
+}
+
+interface FakeRedis {
+ port: number;
+ /** Stop listening and drop every open socket — a `docker stop`, not a blip. */
+ vanish(): Promise<void>;
+}
+
+function startFakeRedis(): Promise<FakeRedis> {
+ const sockets = new Set<net.Socket>();
+ const server = net.createServer((socket) => {
+ sockets.add(socket);
+ let buffered = '';
+ socket.on('error', () => socket.destroy());
+ socket.on('close', () => sockets.delete(socket));
+ socket.on('data', (chunk) => {
+ buffered += chunk.toString('latin1');
+ const { commands, rest } = parseCommands(buffered);
+ buffered = rest;
+ for (const command of commands) {
+ socket.write(replyTo(command), 'latin1');
+ }
+ });
+ });
+
+ return new Promise((resolve, reject) => {
+ server.on('error', reject);
+ server.listen(0, '127.0.0.1', () => {
+ const address = server.address() as net.AddressInfo;
+ resolve({
+ port: address.port,
+ vanish: () =>
+ new Promise((done) => {
+ for (const socket of sockets) {
+ socket.destroy();
+ }
+ sockets.clear();
+ server.close(() => done());
+ }),
+ });
+ });
+ });
+}
+
+/** A port nothing is listening on — every connect gets ECONNREFUSED. */
+function unusedPort(): Promise<number> {
+ return new Promise((resolve, reject) => {
+ const probe = net.createServer();
+ probe.on('error', reject);
+ probe.listen(0, '127.0.0.1', () => {
+ const { port } = probe.address() as net.AddressInfo;
+ probe.close(() => resolve(port));
+ });
+ });
+}
+
+/**
+ * ioredis learns a socket died from the event loop, and a command issued in
+ * the same tick is still written to the dead socket and bounded by
+ * `commandTimeout` instead. Wait for the client to notice before measuring the
+ * instant-rejection property.
+ */
+async function waitUntilDisconnected(client: ExtendedRedis): Promise<void> {
+ const deadline = Date.now() + READY_TIMEOUT_MS;
+ while (client.status === 'ready' && Date.now() < deadline) {
+ await new Promise((resolve) => setTimeout(resolve, 5));
+ }
+}
+
+async function timeRejection(run: () => Promise<unknown>): Promise<{
+ ms: number;
+ error: Error | null;
+}> {
+ const startedAt = performance.now();
+ try {
+ await run();
+ return { ms: performance.now() - startedAt, error: null };
+ } catch (error) {
+ return { ms: performance.now() - startedAt, error: error as Error };
+ }
+}
+
+const clients: ExtendedRedis[] = [];
+function track(client: ExtendedRedis): ExtendedRedis {
+ clients.push(client);
+ return client;
+}
+
+afterEach(() => {
+ while (clients.length > 0) {
+ clients.pop()?.disconnect();
+ }
+});
+
+describe('cache client fail-fast (M18-003)', () => {
+ it('rejects within the command timeout when Redis was never reachable', async () => {
+ const port = await unusedPort();
+ const client = track(
+ createFailFastCacheClient(
+ 'test-never-reachable',
+ `redis://127.0.0.1:${port}`
+ )
+ );
+
+ const { ms, error } = await timeRejection(() => client.get('any-key'));
+
+ expect(error).toBeInstanceOf(Error);
+ expect(ms).toBeLessThan(PROMPT_WHEN_NEVER_CONNECTED_MS);
+ });
+
+ it('does NOT reject commands issued before the first connect completes', async () => {
+ const fake = await startFakeRedis();
+ try {
+ const client = track(
+ createFailFastCacheClient(
+ 'test-boot-order',
+ `redis://127.0.0.1:${fake.port}`
+ )
+ );
+
+ // Same tick as construction — the boot-ordering case. With the offline
+ // queue off from the start ioredis would answer "Stream isn't writeable".
+ await expect(client.get('any-key')).resolves.toBeNull();
+ } finally {
+ await fake.vanish();
+ }
+ });
+
+ it('rejects instantly once a connected server goes away', async () => {
+ const fake = await startFakeRedis();
+ const client = track(
+ createFailFastCacheClient(
+ 'test-connected-then-gone',
+ `redis://127.0.0.1:${fake.port}`
+ )
+ );
+
+ await new Promise<void>((resolve, reject) => {
+ const timer = setTimeout(
+ () => reject(new Error('client never became ready')),
+ READY_TIMEOUT_MS
+ );
+ client.once('ready', () => {
+ clearTimeout(timer);
+ resolve();
+ });
+ });
+ await expect(client.get('any-key')).resolves.toBeNull();
+
+ await fake.vanish();
+ await waitUntilDisconnected(client);
+
+ const { ms, error } = await timeRejection(() => client.get('any-key'));
+
+ expect(error).toBeInstanceOf(Error);
+ // Not "within the timeout" — a known-down socket must not wait at all.
+ expect(ms).toBeLessThan(INSTANT_WHEN_DISCONNECTED_MS);
+ });
+
+ it('leaves the queue client on BullMQ’s mandated settings', () => {
+ expect(QUEUE_CLIENT_OPTIONS.maxRetriesPerRequest).toBeNull();
+ expect(QUEUE_CLIENT_OPTIONS.enableOfflineQueue).toBe(true);
+ expect(QUEUE_CLIENT_OPTIONS.enableReadyCheck).toBe(false);
+ expect(QUEUE_CLIENT_OPTIONS.commandTimeout).toBeUndefined();
+ });
+});
+
+describe('MULTI under fail-fast (the M18-001 shutdown contract)', () => {
+ /**
+ * `event-buffer.ts` decides whether the events it holds are durable from
+ * whether `multi().exec()` threw — and `apps/api/src/shutdown.ts` refuses to
+ * finish the shutdown when it did, so Kafka redelivers instead of the events
+ * being lost. ioredis RESOLVES a MULTI whose individual commands failed, so
+ * "rejects" is the property that keeps a fast refusal from reading as a
+ * successful flush.
+ */
+ it('rejects rather than resolving with per-command errors', async () => {
+ const fake = await startFakeRedis();
+ const client = track(
+ createFailFastCacheClient(
+ 'test-multi-after-vanish',
+ `redis://127.0.0.1:${fake.port}`
+ )
+ );
+ await new Promise<void>((resolve, reject) => {
+ const timer = setTimeout(
+ () => reject(new Error('client never became ready')),
+ READY_TIMEOUT_MS
+ );
+ client.once('ready', () => {
+ clearTimeout(timer);
+ resolve();
+ });
+ });
+
+ await fake.vanish();
+ await waitUntilDisconnected(client);
+
+ const { ms, error } = await timeRejection(() =>
+ client.multi().rpush('event_buffer:queue', 'an-event').exec()
+ );
+
+ expect(error).toBeInstanceOf(Error);
+ expect(ms).toBeLessThan(INSTANT_WHEN_DISCONNECTED_MS);
+ });
+});
diff --git a/packages/redis/redis.ts b/packages/redis/redis.ts
index 1655db6d0..d66ab5f1f 100644
--- a/packages/redis/redis.ts
+++ b/packages/redis/redis.ts
@@ -6,6 +6,51 @@ const options: RedisOptions = {
connectTimeout: 10000,
};
+/**
+ * The cache client's tail bound, for the fault the offline queue cannot see:
+ * a server whose socket is UP and which answers nothing (a frozen or
+ * partitioned node). Nothing else in the request path sets a deadline there —
+ * `/track` never answered inside 30 s in that fault before this.
+ *
+ * It has to clear the heaviest LEGITIMATE command by a wide margin, or a busy
+ * event-buffer flush would start failing as if Redis were down — and under
+ * M18-001 a failed flush refuses to commit Kafka offsets. Measured on this box
+ * (`apps/api/e2e/redis-command-cost.ts`): the event buffer's worst case, a
+ * MULTI of 4,000 rpush (~4 MB), p99 33.95 ms; `LRANGE 0..3999` p99 21.26 ms;
+ * `GET` p99 0.18 ms. 500 ms is ~15x the worst of those.
+ *
+ * Every number behind this file: `apps/api/e2e/redis-fail-fast.md`.
+ */
+export const CACHE_COMMAND_TIMEOUT_MS = 500;
+
+/**
+ * Fail-fast makes the RECONNECT BACKOFF user-visible. While ioredis is waiting
+ * out its next attempt every command is rejected instead of queued, so the
+ * backoff is a window of hard failures that lasts past the moment Redis comes
+ * back, and it grows with the length of the outage — ioredis's default is
+ * `min(attempt * 50, 2000)` ms and the attempt counter never resets while the
+ * server is away. Time from "Redis is back" to "`/track` answers 200 again"
+ * (`apps/api/e2e/redis-fail-fast.ts`, phase C): after a 20 s outage 390/395/396
+ * ms on the default cap vs 349/340/342 ms on this one, after a 60 s outage
+ * 1,137/1,135 ms vs 416/413 ms.
+ */
+const CACHE_RECONNECT_DELAY_STEP_MS = 50;
+const CACHE_RECONNECT_MAX_DELAY_MS = 500;
+
+/**
+ * BullMQ mandates `maxRetriesPerRequest: null` and an offline queue: a blocking
+ * BRPOPLPUSH must not be given up on, and a queued job must not be dropped
+ * because the socket blinked. M18-003 made the CACHE client fail fast and
+ * deliberately left this alone (Carl, 2026-09-13: "for cache we should have
+ * fail fast, for queue I think we should have the settings we have already").
+ */
+export const QUEUE_CLIENT_OPTIONS: RedisOptions = {
+ ...options,
+ enableReadyCheck: false,
+ maxRetriesPerRequest: null,
+ enableOfflineQueue: true,
+};
+
export { Redis };
const REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379';
@@ -63,10 +108,64 @@ const createRedisClient = (
return client;
};
+/**
+ * The CACHE client fails fast (M18-003). Drill 02 measured `/track` blocking
+ * 9,427 / 12,039 / 41,978 ms on a stopped Redis, and 10 of 20 requests getting
+ * no answer inside 30 s, because on ioredis's defaults the command is QUEUED
+ * and waits out 20 reconnect attempts (backoff `min(times * 50, 2000)` ms)
+ * before `MaxRetriesPerRequestError`. Everything on the ingest path already
+ * treats these reads as optional — `device-id.ts` catches the session-buffer
+ * failure and mints the deterministic session id — so the wait bought nothing
+ * but a slower failure.
+ *
+ * Two mechanisms, because there are two faults:
+ *
+ * - **The offline queue goes off**, so a command issued while the socket is
+ * known-down is rejected in microseconds rather than queued. This is what
+ * keeps a warm-cache `/track` at its normal latency during an outage.
+ * - **`commandTimeout`** bounds the case the offline queue cannot see, where
+ * the socket is up and the server answers nothing.
+ *
+ * Reproduced and measured before/after on this box, with the API pointed at a
+ * TCP proxy that can stop or freeze: with Redis stopped, a warm-auth `/track`
+ * goes from 2-of-3 requests never answered inside 30 s to 200 × 10 at p50
+ * 10.0 ms, and a cold-auth one from the same 30 s hang to 500 at p50 1.3 ms.
+ * `commandTimeout` on its own, with the queue left on, was measured as well and
+ * is not enough: it bounds the tail but leaves that warm `/track` at 1,011 ms,
+ * because ioredis arms the deadline before the writable check, so the queued
+ * command pays it in full. The cost of going further, and why the deadline is
+ * 500 ms: `apps/api/e2e/redis-fail-fast.md`.
+ *
+ * THE OFFLINE QUEUE STAYS ON UNTIL THE CLIENT HAS CONNECTED ONCE. ioredis
+ * rejects every command issued before the first connect completes when the
+ * queue is off — including the ones a process issues while it is still
+ * booting — which would make boot order decide whether a command works. A
+ * process that has never reached Redis therefore still falls back to
+ * `commandTimeout`, which is bounded; one that has connected fails instantly.
+ */
+export function createFailFastCacheClient(
+ name: string,
+ url: string
+): ExtendedRedis {
+ const client = createRedisClient(name, url, {
+ ...options,
+ commandTimeout: CACHE_COMMAND_TIMEOUT_MS,
+ retryStrategy: (attempt) =>
+ Math.min(
+ attempt * CACHE_RECONNECT_DELAY_STEP_MS,
+ CACHE_RECONNECT_MAX_DELAY_MS
+ ),
+ });
+ client.once('ready', () => {
+ client.options.enableOfflineQueue = false;
+ });
+ return client;
+}
+
let redisCache: ExtendedRedis;
export function getRedisCache() {
if (!redisCache) {
- redisCache = createRedisClient('redis-cache', REDIS_URL, options);
+ redisCache = createFailFastCacheClient('redis-cache', REDIS_URL);
}
return redisCache;
@@ -99,12 +198,11 @@ let redisQueue: ExtendedRedis;
export function getRedisQueue() {
if (!redisQueue) {
// Use different redis for queues (self-hosting will re-use the same redis instance)
- redisQueue = createRedisClient('redis-queue', REDIS_URL, {
- ...options,
- enableReadyCheck: false,
- maxRetriesPerRequest: null,
- enableOfflineQueue: true,
- });
+ redisQueue = createRedisClient(
+ 'redis-queue',
+ REDIS_URL,
+ QUEUE_CLIENT_OPTIONS
+ );
}
return redisQueue;