OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M20-001): Dead-letter failed events to a capped Redis list and drop them unconditionally, replacing a Kafka DLQ produce that has never once succeeded
ralph99449f170e0b
diff --git a/apps/api/src/config/env.test.ts b/apps/api/src/config/env.test.tsindex c44465db1..3fea74a6c 100644--- a/apps/api/src/config/env.test.ts+++ b/apps/api/src/config/env.test.ts@@ -245,6 +245,21 @@ describe('derived values, computed once in the transform', () => {).toBe('own');});+ it('the dead-letter list cap defaults to 1000 and takes a positive integer', () => {+ // The Kafka DLQ topic above is no longer the dead-letter destination+ // (M20-001) — it is kept only so the seam can be swapped back.+ expect(loadConfig(base).INGEST_DEAD_LETTER_MAX_ENTRIES).toBe(1000);+ expect(+ loadConfig({ ...base, INGEST_DEAD_LETTER_MAX_ENTRIES: '25' })+ .INGEST_DEAD_LETTER_MAX_ENTRIES+ ).toBe(25);+ // Same doctrine as its Kafka neighbours: a cap of 0 or -5 would silently+ // keep nothing, so it fails boot rather than becoming a surprise.+ expect(() =>+ loadConfig({ ...base, INGEST_DEAD_LETTER_MAX_ENTRIES: '0' })+ ).toThrow('INGEST_DEAD_LETTER_MAX_ENTRIES');+ });+it('the log exporter and the stdout interception verdict', () => {expect(loadConfig(base).core.logging).toMatchObject({exporter: 'stdout',diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.tsindex 30d0005f0..33cb4782b 100644--- a/apps/api/src/config/env.ts+++ b/apps/api/src/config/env.ts@@ -74,6 +74,12 @@ const DEFAULT_KAFKA_PRODUCER_BATCH_SIZE = 25;*/const DEFAULT_KAFKA_PRODUCER_BATCH_LINGER_MS = 5;const DEFAULT_KAFKA_HANDLER_MAX_ATTEMPTS = 3;+/**+ * How many dead-lettered events the capped Redis list keeps (M20-001). A+ * debugging sample, not a recovery mechanism — the volume is in+ * `kafka_events_dead_lettered_total`. Carl's "only keep the last N events".+ */+const DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES = 1000;const DEFAULT_KAFKA_HANDLER_RETRY_INITIAL_MS = 100;const DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS = 1000;@@ -449,6 +455,11 @@ const rawSchema = z.object({DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS),+ // --- ingest dead-letter ---+ INGEST_DEAD_LETTER_MAX_ENTRIES: positiveIntWithDefault(+ DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES+ ),+// --- buffers ---BUFFER_ASYNC_INSERTS: definedIsTrueSchema,BUFFER_CH_INSERT_CONCURRENCY: optionalPositiveInt,@@ -795,6 +806,13 @@ export interface Config {DISABLE_BULLBOARD: boolean;COOKIE_SECRET: string;SHUTDOWN_FORCE_EXIT_MS: number;+ /**+ * The dead-letter list's cap (M20-001). It sits here rather than in+ * `core.kafka` because the destination is no longer Kafka: `apps/api`+ * chooses the dead-letter sink, and `packages/core` only knows the+ * `sendToDeadLetter` seam it is handed.+ */+ INGEST_DEAD_LETTER_MAX_ENTRIES: number;/** The CORS delegator's origin allowlist, in V1's own order. */dashboardOrigins: string[];/** V1's `ENABLE_VERBOSE_LOGGING`, already split. */@@ -834,6 +852,7 @@ const envSchema = rawSchemaDISABLE_BULLBOARD: raw.DISABLE_BULLBOARD,COOKIE_SECRET: raw.COOKIE_SECRET,SHUTDOWN_FORCE_EXIT_MS: raw.SHUTDOWN_FORCE_EXIT_MS,+ INGEST_DEAD_LETTER_MAX_ENTRIES: raw.INGEST_DEAD_LETTER_MAX_ENTRIES,dashboardOrigins: [dashboardUrl,...splitTokens(raw.API_CORS_ORIGINS),diff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex efb5f9fa0..ba8dd1fe6 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -62,7 +62,6 @@ import {kafkaLogger,markEventsActivity,opsRoutes,- produceDeadLetterEvent,publicApiRoutes,type QueueDefinition,type QueueProducerHandle,@@ -84,7 +83,12 @@ import {} from '@openpanel/core';import { ch } from '@openpanel/db/src/clickhouse/client';import { db, Prisma } from '@openpanel/db/src/prisma-client';-import { getRedisCache, getRedisPub, getRedisQueue } from '@openpanel/redis';+import {+ createDeadLetterRecorder,+ getRedisCache,+ getRedisPub,+ getRedisQueue,+} from '@openpanel/redis';import { Elysia } from 'elysia';import pino from 'pino';import { type Config, KAFKA_QUEUE_TOKEN, loadConfig } from './config/env';@@ -269,8 +273,17 @@ async function startIngestConsumer(handleEvent: createIncomingEventHandler(deps, {checkNotificationRulesForEvent,}),- sendToDeadLetter: (message) =>- produceDeadLetterEvent(config.core, message),+ // M20-001: a capped Redis list, and the event is DROPPED whether or not+ // the record lands. The Kafka DLQ it replaced+ // (`produceDeadLetterEvent`, still exported from core) produced to a+ // topic nothing creates — 1,592 failures, 0 successes in drill 08 — and+ // every failure held the offset back, which was the redelivery loop.+ // The CACHE client, not the queue client: only that one fails fast, and+ // failing fast is the point here.+ sendToDeadLetter: createDeadLetterRecorder({+ client: deps.redis,+ maxEntries: config.INGEST_DEAD_LETTER_MAX_ENTRIES,+ }),// Opened before the batch's first handler and closed before its first// resolved offset; the gate THROWS if the rpush did not land.// `pendingEvents` is the only in-process buffer state in the tree —diff --git a/packages/core/src/modules/ingest/src/consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.tsindex 8f1d3e26e..fc449500a 100644--- a/packages/core/src/modules/ingest/src/consumer.test.ts+++ b/packages/core/src/modules/ingest/src/consumer.test.ts@@ -10,6 +10,12 @@*/import { describe, expect, mock, test } from 'bun:test';+import {+ createDeadLetterRecorder,+ type DeadLetterMulti,+ type DeadLetterRecord,+ type DeadLetterRedisClient,+} from '@openpanel/redis';import type { EachBatchPayload, KafkaMessage } from 'kafkajs';import {createEventsBatchHandler,@@ -300,9 +306,9 @@ describe('handler failure', () => {expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2']);});- test('leaves the offset unresolved when the dead-letter produce fails', async () => {+ test('resolves the offset even when the dead-letter write fails', async () => {const { deps, handleEvent, metrics } = makeDeps({- sendToDeadLetter: mock(() => Promise.reject(new Error('broker down'))),+ sendToDeadLetter: mock(() => Promise.reject(new Error('redis down'))),});handleEvent.mockImplementation(async (p: { projectId: string }) => {if (p.projectId === 'project-1') {@@ -318,9 +324,11 @@ describe('handler failure', () => {await handler.eachBatch(payload);+ // Dropped WITHOUT being recorded — and still acked. Holding the offset+ // back here is what rebuilt the redelivery loop (M20-001, gate M20).expect(metrics.deadLetterFailed).toHaveBeenCalledWith(String(PARTITION));- // Nothing at or after the failed offset is acked — it is redelivered.- expect(resolvedOffsets(resolveOffset)).toEqual(['0']);+ expect(metrics.deadLettered).not.toHaveBeenCalled();+ expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2']);});});@@ -670,3 +678,201 @@ describe('durability before commit', () => {expect(flushBufferedEvents).toHaveBeenCalledTimes(1);});});++/**+ * M20-001 end to end: the consumer wired to the REAL capped-list recorder+ * (`@openpanel/redis`'s `createDeadLetterRecorder`) instead of a mock, against+ * an in-memory stand-in for the one MULTI it issues. Redis itself is out of+ * the picture; the LPUSH/LTRIM arithmetic is not.+ */+describe('dead letter to a capped redis list', () => {+ const LIST_KEY = 'dead_letter:events-test';+ const MAX_ENTRIES = 3;+ const OK: [Error | null, unknown] = [null, 1];++ /** Enough of a Redis client for one MULTI, with real list semantics. */+ function fakeRedis(execError?: Error) {+ const entries: string[] = [];+ let roundTrips = 0;+ const client: DeadLetterRedisClient = {+ multi() {+ const queued: (() => void)[] = [];+ const chain: DeadLetterMulti = {+ lpush(_key, value) {+ queued.push(() => entries.unshift(value));+ return chain;+ },+ ltrim(_key, start, stop) {+ queued.push(() => entries.splice(stop + 1 - start));+ return chain;+ },+ async exec() {+ roundTrips += 1;+ if (execError) {+ throw execError;+ }+ for (const run of queued) {+ run();+ }+ return queued.map(() => OK);+ },+ };+ return chain;+ },+ };+ return {+ client,+ records: () => entries.map((raw) => JSON.parse(raw) as DeadLetterRecord),+ roundTrips: () => roundTrips,+ };+ }++ function makeRecordingDeps(execError?: Error) {+ const redis = fakeRedis(execError);+ const built = makeDeps({+ sendToDeadLetter: createDeadLetterRecorder({+ client: redis.client,+ maxEntries: MAX_ENTRIES,+ key: LIST_KEY,+ }),+ });+ return { ...built, redis };+ }++ test('a handler that exhausts its attempts leaves ONE record and a RESOLVED offset', async () => {+ const { deps, handleEvent, metrics, redis } = makeRecordingDeps();+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));+ const handler = createEventsBatchHandler(deps);+ const { payload, resolveOffset } = makeBatch([message(0, 'a')]);++ await handler.eachBatch(payload);++ expect(handleEvent).toHaveBeenCalledTimes(3);+ const records = redis.records();+ expect(records).toHaveLength(1);+ expect(records[0]).toMatchObject({+ reason: 'handler_error',+ error: 'boom',+ offset: '0',+ partition: PARTITION,+ topic: TOPIC,+ key: 'a',+ });+ expect(JSON.parse(records[0]?.value ?? '')).toMatchObject({+ projectId: 'project-0',+ });+ // One MULTI, so one round trip per dropped event.+ expect(redis.roundTrips()).toBe(1);+ expect(metrics.deadLettered).toHaveBeenCalledWith(+ String(PARTITION),+ 'handler_error'+ );+ expect(resolvedOffsets(resolveOffset)).toEqual(['0']);+ });++ test('a DEAD REDIS still resolves the offset and records nothing', async () => {+ // Drill 02's scenario exactly: the dependency that fails the handler is+ // the dependency the dead letter is written to. If this offset is left+ // unresolved the redelivery loop is back, in a new place.+ const { deps, handleEvent, metrics, redis } = makeRecordingDeps(+ new Error(+ "Stream isn't writeable and enableOfflineQueue options is false"+ )+ );+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));+ const handler = createEventsBatchHandler(deps);+ const { payload, resolveOffset } = makeBatch([+ message(0, 'a'),+ message(1, 'a'),+ message(2, 'a'),+ ]);++ await handler.eachBatch(payload);++ expect(redis.records()).toEqual([]);+ expect(metrics.deadLetterFailed).toHaveBeenCalledTimes(3);+ expect(metrics.deadLettered).not.toHaveBeenCalled();+ expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2']);+ });++ test('a per-command error inside the MULTI counts as not recorded', async () => {+ // WRONGTYPE is drill 08's own fault shape, and a MULTI reports it in+ // `exec()`'s results rather than by rejecting.+ const redis = fakeRedis();+ const wrongType: [Error | null, unknown] = [+ new Error(+ 'WRONGTYPE Operation against a key holding the wrong kind of value'+ ),+ null,+ ];+ const failing: DeadLetterRedisClient = {+ multi() {+ const chain: DeadLetterMulti = {+ lpush: () => chain,+ ltrim: () => chain,+ exec: () => Promise.resolve([wrongType, OK]),+ };+ return chain;+ },+ };+ const { deps, handleEvent, metrics } = makeDeps({+ sendToDeadLetter: createDeadLetterRecorder({+ client: failing,+ maxEntries: MAX_ENTRIES,+ key: LIST_KEY,+ }),+ });+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));+ const handler = createEventsBatchHandler(deps);+ const { payload, resolveOffset } = makeBatch([message(0, 'a')]);++ await handler.eachBatch(payload);++ expect(redis.records()).toEqual([]);+ expect(metrics.deadLetterFailed).toHaveBeenCalledWith(String(PARTITION));+ expect(metrics.deadLettered).not.toHaveBeenCalled();+ expect(resolvedOffsets(resolveOffset)).toEqual(['0']);+ });++ test('the list never exceeds N and keeps the most recent drops', async () => {+ const { deps, handleEvent, redis } = makeRecordingDeps();+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));+ const handler = createEventsBatchHandler(deps);+ // One key, so one serial group: the drop order is the offset order.+ const { payload, resolveOffset } = makeBatch(+ [0, 1, 2, 3, 4].map((offset) => message(offset, 'a'))+ );++ await handler.eachBatch(payload);++ const records = redis.records();+ expect(records).toHaveLength(MAX_ENTRIES);+ // Newest first, and the two oldest drops fell off the tail.+ expect(records.map((record) => record.offset)).toEqual(['4', '3', '2']);+ expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2', '3', '4']);+ });++ test('a parse failure takes the same path as a handler failure', async () => {+ const { deps, handleEvent, metrics, redis } = makeRecordingDeps();+ const handler = createEventsBatchHandler(deps);+ const { payload, resolveOffset } = makeBatch([+ message(0, 'a', 'not json at all'),+ ]);++ await handler.eachBatch(payload);++ expect(handleEvent).not.toHaveBeenCalled();+ const records = redis.records();+ expect(records).toHaveLength(1);+ expect(records[0]).toMatchObject({+ reason: 'parse_error',+ offset: '0',+ value: 'not json at all',+ });+ expect(metrics.deadLettered).toHaveBeenCalledWith(+ String(PARTITION),+ 'parse_error'+ );+ expect(resolvedOffsets(resolveOffset)).toEqual(['0']);+ });+});diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.tsindex ea6ec287f..47146f83b 100644--- a/packages/core/src/modules/ingest/src/consumer.ts+++ b/packages/core/src/modules/ingest/src/consumer.ts@@ -1,7 +1,10 @@// Ported from apps/worker/src/jobs/events.kafka-consumer.ts (M8-003). The// delivery contract is ADR-004's and is unchanged line for line: per-key// serial groups, the ascending contiguous-prefix offset walk, at-least-once,-// bounded in-consumer retry, and a dead-letter topic on exhaustion.+// bounded in-consumer retry, and a dead-letter on exhaustion. Only the+// dead-letter DESTINATION and its failure behaviour changed (M20-001): the+// record goes to a capped Redis list, and the event is dropped whether or not+// that write lands.//// The kafkajs client, the topic/consumer-group names, the DLQ producer and// the retry bounds are all INJECTED, and stay so now that they live one@@ -57,10 +60,15 @@ const DURABILITY_RETRY_MAX_MS = 5000;export type DeadLetterReason = 'parse_error' | 'handler_error';/**- * What the consumer hands to the dead-letter producer (./kafka.ts's- * `produceDeadLetterEvent` is the one implementation): the value stays the- * producer's original bytes, so a DLQ message can be replayed onto the events- * topic unchanged.+ * What the consumer hands to the dead-letter sink. The value stays the+ * producer's original bytes, so the record is a faithful copy of what arrived.+ *+ * Two implementations satisfy this, and the seam exists so the choice is one+ * line in `apps/api`'s wiring: `@openpanel/redis`'s `createDeadLetterRecorder`+ * (the capped list, in use since M20-001) and ./kafka.ts's+ * `produceDeadLetterEvent` (the DLQ topic, kept because Carl's decision was+ * explicitly "for now"; a DLQ message produced from these bytes is replayable+ * onto the events topic unchanged).*/export interface DeadLetterMessage {key: Buffer | null;@@ -82,7 +90,12 @@ export interface ConsumerLogger {export interface ConsumerMetrics {reprocessed: (partition: string) => void;handlerFailed: (partition: string) => void;+ /** The message was recorded in the dead-letter list and DROPPED. */deadLettered: (partition: string, reason: DeadLetterReason) => void;+ /**+ * The message was DROPPED WITHOUT being recorded — the dead-letter write+ * failed. Not "will be retried": nothing retries it (M20-001).+ */deadLetterFailed: (partition: string) => void;}@@ -91,6 +104,10 @@ export interface EventsBatchHandlerDeps {payload: IncomingEventPayload,meta: { partition: number; offset: string }) => Promise<unknown>;+ /**+ * Park a poison message somewhere an operator can read it. It may reject —+ * the caller drops the message and resolves the offset regardless.+ */sendToDeadLetter: (message: DeadLetterMessage) => Promise<void>;/*** Opens a durability window over the event buffer and returns the gate that@@ -140,7 +157,10 @@ export interface EventsBatchHandler {** Delivery contract (ADR-004): at-least-once. An offset is resolved only once* its message has been handled, dead-lettered, or deliberately skipped —- * never merely because it failed.+ * never merely because it failed. Since M20-001 a dead-letter always counts as+ * finished, because the message is dropped either way; the one place an+ * unresolved offset is still correct is a failed DURABILITY flush, which+ * leaves the whole batch for redelivery (M18-001).*/export function createEventsBatchHandler(deps: EventsBatchHandlerDeps@@ -155,12 +175,31 @@ export function createEventsBatchHandler(// post-rebalance redelivery from the last committed offset doesn't trip it.const resolvedHWM = new Map<string, number>();+ /**+ * Park the message and DROP it. The drop is unconditional: whether the+ * record was stored or not, this message's offset is resolved by the caller+ * and the event is gone from the pipeline (M20-001, gate M20).+ *+ * There is deliberately no retry, no backoff and no unresolved offset here.+ * The dead-letter destination is Redis, and Redis being unavailable is+ * exactly when handlers fail — so a failed write that held the offset back+ * would rebuild the very redelivery loop this replaced (drill 02's re-run 2:+ * 78-79 laps per message at ~0.62 s, no backoff; drill 08: 1,592 failed+ * dead-letters, 0 successes). Same lesson as M18-007's `pendingEvents`+ * re-queue: once something else owns the outcome, the safety net underneath+ * it is the bug.+ *+ * The two counters are what carry the volume, since a capped list makes+ * 50,000 drops look like 12: `deadLettered` = recorded and dropped,+ * `deadLetterFailed` = dropped WITHOUT being recorded. Neither means+ * "will be retried".+ */const deadLetter = async (message: KafkaMessage,partition: number,reason: DeadLetterReason,err: unknown- ): Promise<boolean> => {+ ): Promise<void> => {const label = String(partition);try {await deps.sendToDeadLetter({@@ -174,21 +213,27 @@ export function createEventsBatchHandler(error: errorMessage(err),});} catch (dlqErr) {- // The event is not lost: leaving the offset unresolved stops the- // contiguous-prefix walk here, so the broker redelivers this message.deps.metrics.deadLetterFailed(label);+ // The WRITE failure goes in `err` — it is the news, and only that field+ // gets the logger's error serializer. What the message originally did+ // wrong rides along as text.deps.logger.error(- { err: dlqErr, partition, offset: message.offset, reason },- 'kafka dead-letter produce failed — offset left unresolved for redelivery'+ {+ err: dlqErr,+ cause: errorMessage(err),+ partition,+ offset: message.offset,+ reason,+ },+ 'kafka message DROPPED without being recorded — dead-letter write failed');- return false;+ return;}deps.metrics.deadLettered(label, reason);deps.logger.error({ err, partition, offset: message.offset, reason },- 'kafka message dead-lettered'+ 'kafka message dead-lettered — recorded and dropped');- return true;};const runHandlerWithRetry = async (@@ -228,11 +273,15 @@ export function createEventsBatchHandler(return { ok: false, err: lastErr };};- /** @returns true when the offset may be resolved. */+ /**+ * Always returns: the offset is resolvable either way. A message is handled,+ * or it is dead-lettered and dropped — there is no third outcome, which is+ * why the batch loop below has no unresolvable-message branch.+ */const processMessage = async (message: KafkaMessage,partition: number- ): Promise<boolean> => {+ ): Promise<void> => {let payload: IncomingEventPayload;try {if (!message.value) {@@ -245,7 +294,8 @@ export function createEventsBatchHandler('kafka message parse failed');// Deterministic: retrying identical bytes cannot succeed.- return deadLetter(message, partition, 'parse_error', err);+ await deadLetter(message, partition, 'parse_error', err);+ return;}const result = await runHandlerWithRetry(@@ -254,9 +304,9 @@ export function createEventsBatchHandler(message.offset);if (result.ok) {- return true;+ return;}- return deadLetter(message, partition, 'handler_error', result.err);+ await deadLetter(message, partition, 'handler_error', result.err);};// Consecutive durability failures per `topic-partition`, for the backoff@@ -379,12 +429,12 @@ export function createEventsBatchHandler();}- const resolvable = await processMessage(m, batch.partition);- if (!resolvable) {- // Stop this key-group: continuing past an unresolvable offset only- // adds duplicates, since the whole tail is redelivered anyway.- return;- }+ // No "unresolvable" branch: a message is handled or dropped, and a+ // dropped one is still finished (M20-001). The only gap this walk+ // can now stop at is the isRunning/isStale early return above, and+ // the only offsets left uncommitted are a whole batch whose+ // durability flush failed — see below.+ await processMessage(m, batch.partition);processed.add(m.offset);processedCount += 1;diff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.tsindex 36fb0b4eb..b671f3339 100644--- a/packages/core/src/modules/ingest/src/ingest.metrics.ts+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts@@ -29,20 +29,25 @@ export const kafkaHandlerFailuresTotal = new client.Counter({registers: [registry],});-// Messages written to the dead-letter topic after their attempts were-// exhausted (or after they failed to parse). One increment per message.+// Messages recorded in the dead-letter list after their attempts were+// exhausted (or after they failed to parse), then DROPPED. One increment per+// message. The list is capped at the last N, so this counter — not the list —+// is the only thing that tells 50,000 drops apart from 12 (M20-001). The+// series name is unchanged: the destination moved, the meaning did not.export const kafkaDeadLetteredTotal = new client.Counter({name: 'kafka_events_dead_lettered_total',- help: 'Kafka event messages produced to the dead-letter topic',+ help: 'Kafka event messages recorded in the dead-letter list and dropped',labelNames: ['partition', 'reason'],registers: [registry],});-// The dead-letter produce itself failed. The offset is then left unresolved-// and the message is redelivered, so this is a stuck partition, not a loss.+// The dead-letter write itself failed, so the message was DROPPED WITHOUT+// being recorded — Redis unreachable, most likely, which is also when handlers+// fail. This is real loss, not a stuck partition: the offset is resolved+// anyway and nothing retries the message (M20-001, gate M20).export const kafkaDeadLetterFailedTotal = new client.Counter({name: 'kafka_events_dead_letter_failed_total',- help: 'Failed attempts to produce a Kafka event message to the dead-letter topic',+ help: 'Kafka event messages dropped without being recorded (dead-letter write failed)',labelNames: ['partition'],registers: [registry],});diff --git a/packages/redis/dead-letter.test.ts b/packages/redis/dead-letter.test.tsnew file mode 100644index 000000000..c39b820fd--- /dev/null+++ b/packages/redis/dead-letter.test.ts@@ -0,0 +1,192 @@+/**+ * M20-001 — the capped dead-letter list, at the Redis boundary.+ *+ * What the consumer does with a rejection is `packages/core`'s+ * `consumer.test.ts`; what this file pins is the write itself: one MULTI, the+ * record's shape, and the two ways a MULTI can fail without rejecting.+ *+ * No real Redis: an in-memory stand-in with real list semantics. The same+ * assertions were run against a local Redis (`redis://127.0.0.1:6379/15`) on+ * 2026-09-14 before this file was written — LLEN 3 after 7 pushes at N=3,+ * newest first, and a key holding a string rejecting with `WRONGTYPE`.+ */++import { describe, expect, it } from 'vitest';+import {+ createDeadLetterRecorder,+ DEAD_LETTER_LIST_KEY,+ type DeadLetterMulti,+ type DeadLetterRecord,+ type DeadLetterRedisClient,+} from './dead-letter';++const OK: [Error | null, unknown] = [null, 1];+const MAX_ENTRIES = 3;++const input = (offset: string, overrides: Record<string, unknown> = {}) => ({+ key: Buffer.from('device-1'),+ value: Buffer.from(JSON.stringify({ projectId: 'p1', offset })),+ headers: {+ 'request-id': Buffer.from('req-1'),+ attempts: ['1', '2'],+ plain: 'text',+ },+ topic: 'events',+ partition: 7,+ offset,+ reason: 'handler_error',+ error: 'handler exploded',+ ...overrides,+});++function fakeRedis() {+ const entries: string[] = [];+ const keysWritten: string[] = [];+ const trims: [number, number][] = [];+ let execs = 0;+ const client: DeadLetterRedisClient = {+ multi() {+ const queued: (() => void)[] = [];+ const chain: DeadLetterMulti = {+ lpush(key, value) {+ keysWritten.push(key);+ queued.push(() => entries.unshift(value));+ return chain;+ },+ ltrim(_key, start, stop) {+ trims.push([start, stop]);+ queued.push(() => entries.splice(stop + 1 - start));+ return chain;+ },+ async exec() {+ execs += 1;+ for (const run of queued) {+ run();+ }+ return queued.map(() => OK);+ },+ };+ return chain;+ },+ };+ return {+ client,+ keysWritten,+ trims,+ execs: () => execs,+ records: () => entries.map((raw) => JSON.parse(raw) as DeadLetterRecord),+ };+}++describe('createDeadLetterRecorder', () => {+ it('writes one record per event in a single round trip', async () => {+ const redis = fakeRedis();+ const record = createDeadLetterRecorder({+ client: redis.client,+ maxEntries: MAX_ENTRIES,+ });++ await record(input('100'));++ expect(redis.execs()).toBe(1);+ expect(redis.keysWritten).toEqual([DEAD_LETTER_LIST_KEY]);+ expect(redis.trims).toEqual([[0, MAX_ENTRIES - 1]]);+ });++ it('stores the coordinates, the reason, the error and the raw payload', async () => {+ const redis = fakeRedis();+ const record = createDeadLetterRecorder({+ client: redis.client,+ maxEntries: MAX_ENTRIES,+ });++ await record(input('100'));++ const [stored] = redis.records();+ expect(stored).toMatchObject({+ topic: 'events',+ partition: 7,+ offset: '100',+ reason: 'handler_error',+ error: 'handler exploded',+ key: 'device-1',+ // Buffer and array header values come back as text, not as `{"type":"Buffer"}`.+ headers: { 'request-id': 'req-1', attempts: '1,2', plain: 'text' },+ });+ expect(JSON.parse(stored?.value ?? '')).toMatchObject({ projectId: 'p1' });+ expect(Number.isNaN(Date.parse(stored?.recordedAt ?? ''))).toBe(false);+ });++ it('keeps a message with no key and no value at all', async () => {+ const redis = fakeRedis();+ const record = createDeadLetterRecorder({+ client: redis.client,+ maxEntries: MAX_ENTRIES,+ });++ await record(input('100', { key: null, value: null, headers: undefined }));++ expect(redis.records()[0]).toMatchObject({+ key: null,+ value: null,+ headers: {},+ });+ });++ it('keeps only the last N, newest first', async () => {+ const redis = fakeRedis();+ const record = createDeadLetterRecorder({+ client: redis.client,+ maxEntries: MAX_ENTRIES,+ });++ for (const offset of ['1', '2', '3', '4', '5']) {+ await record(input(offset));+ }++ expect(redis.records().map((stored) => stored.offset)).toEqual([+ '5',+ '4',+ '3',+ ]);+ });++ it('rejects on a per-command error, which a MULTI does not itself reject for', async () => {+ const wrongType = new Error('WRONGTYPE Operation against a key holding…');+ const client: DeadLetterRedisClient = {+ multi() {+ const chain: DeadLetterMulti = {+ lpush: () => chain,+ ltrim: () => chain,+ exec: () => Promise.resolve([[wrongType, null], OK]),+ };+ return chain;+ },+ };+ const record = createDeadLetterRecorder({+ client,+ maxEntries: MAX_ENTRIES,+ });++ await expect(record(input('100'))).rejects.toThrow(wrongType);+ });++ it('rejects when the MULTI is discarded', async () => {+ const client: DeadLetterRedisClient = {+ multi() {+ const chain: DeadLetterMulti = {+ lpush: () => chain,+ ltrim: () => chain,+ exec: () => Promise.resolve(null),+ };+ return chain;+ },+ };+ const record = createDeadLetterRecorder({+ client,+ maxEntries: MAX_ENTRIES,+ });++ await expect(record(input('100'))).rejects.toThrow('discarded by Redis');+ });+});diff --git a/packages/redis/dead-letter.ts b/packages/redis/dead-letter.tsnew file mode 100644index 000000000..53ced4472--- /dev/null+++ b/packages/redis/dead-letter.ts@@ -0,0 +1,160 @@+/**+ * The ingest consumer's dead-letter destination: a capped Redis list (M20-001).+ *+ * Carl, 2026-09-14: *"For now, we can just store errors in redis and drop the+ * event, push it to a redis list and only keep the last N events."*+ *+ * It replaces `produceDeadLetterEvent`, which produced to+ * `${KAFKA_EVENTS_TOPIC}-dlq` — a topic nothing in the tree ever creates.+ * Drill 08 measured that produce failing 1,592 times with 0 successes+ * (`Number of partitions is invalid`), and the failure left the offset+ * unresolved, which is the unbounded handler-error redelivery loop drill 02's+ * re-run 2 measured at 78-79 laps per message. The Kafka producer is still+ * exported from `@openpanel/core`, and the consumer's `sendToDeadLetter` seam+ * is unchanged, so swapping back is one line in `apps/api`'s wiring.+ *+ * THE RECORD IS A DEBUGGING SAMPLE, NOT A RECOVERY MECHANISM. The list keeps+ * the last N and nothing more; the event itself is gone from the pipeline. The+ * volume lives in `kafka_events_dead_lettered_total` / `_dead_letter_failed_total`,+ * because a capped list makes 50,000 drops look exactly like 12.+ *+ * Why here and not in `packages/core`'s ingest module: the whole of it is+ * Redis mechanics — one MULTI, one LPUSH, one LTRIM — and core's only export+ * door is its barrel, which M20-001's scope does not cover. The input type is+ * structural, so `DeadLetterMessage` (kafkajs `Buffer`s and `IHeaders`)+ * satisfies it without this package depending on kafkajs or on core.+ */++/** The one list. Not configurable: an operator has to be able to find it. */+export const DEAD_LETTER_LIST_KEY = 'dead_letter:events';++/** A `Buffer`, or anything else the caller hands us as raw message bytes. */+interface BinaryLike {+ toString(): string;+}++/**+ * Structurally what `@openpanel/core`'s `DeadLetterMessage` is — the same+ * fields `produceDeadLetterEvent` put in its DLQ headers, so a record carries+ * everything needed to find the message on the source topic.+ */+export interface DeadLetterInput {+ key: BinaryLike | null;+ value: BinaryLike | null;+ headers?: Record<string, unknown> | undefined;+ topic: string;+ partition: number;+ offset: string;+ reason: string;+ error: string;+}++/** One entry of the list, as stored (JSON). */+export interface DeadLetterRecord {+ recordedAt: string;+ topic: string;+ partition: number;+ offset: string;+ reason: string;+ error: string;+ key: string | null;+ /** The producer's original payload, as text — events on this topic are JSON. */+ value: string | null;+ headers: Record<string, string>;+}++/**+ * The narrow slice of a Redis client this needs. Structural rather than+ * ioredis's `Redis`, so the contract is one MULTI and a test can satisfy it+ * without a server.+ */+export interface DeadLetterMulti {+ lpush(key: string, value: string): DeadLetterMulti;+ ltrim(key: string, start: number, stop: number): DeadLetterMulti;+ exec(): Promise<[Error | null, unknown][] | null>;+}++export interface DeadLetterRedisClient {+ multi(): DeadLetterMulti;+}++export interface DeadLetterRecorderOptions {+ client: DeadLetterRedisClient;+ /** How many records the list keeps. The oldest fall off the tail. */+ maxEntries: number;+ /** Overridable for tests; production uses the constant above. */+ key?: string;+}++const toText = (value: unknown): string => {+ if (typeof value === 'string') {+ return value;+ }+ if (Array.isArray(value)) {+ return value.map(toText).join(',');+ }+ if (value === null || value === undefined) {+ return '';+ }+ return String(value);+};++const normalizeHeaders = (+ headers: Record<string, unknown> | undefined+): Record<string, string> => {+ const normalized: Record<string, string> = {};+ for (const [name, value] of Object.entries(headers ?? {})) {+ normalized[name] = toText(value);+ }+ return normalized;+};++export const toDeadLetterRecord = (+ input: DeadLetterInput,+ recordedAt: Date = new Date()+): DeadLetterRecord => ({+ recordedAt: recordedAt.toISOString(),+ topic: input.topic,+ partition: input.partition,+ offset: input.offset,+ reason: input.reason,+ error: input.error,+ key: input.key === null ? null : input.key.toString(),+ value: input.value === null ? null : input.value.toString(),+ headers: normalizeHeaders(input.headers),+});++/**+ * ONE round trip per dropped event: LPUSH and LTRIM travel in a single MULTI,+ * so the list is never observed above `maxEntries` and a dropped event never+ * costs the ingest path two waits.+ *+ * It REJECTS when the write did not land — including a per-command error such+ * as `WRONGTYPE`, which a MULTI reports inside `exec()`'s results rather than+ * by rejecting. The caller counts that as "dropped without being recorded" and+ * resolves the offset anyway: Redis being unavailable is exactly when handlers+ * fail, and an unresolved offset there is the redelivery loop this replaces.+ */+export function createDeadLetterRecorder(+ options: DeadLetterRecorderOptions+): (input: DeadLetterInput) => Promise<void> {+ const key = options.key ?? DEAD_LETTER_LIST_KEY;+ const lastIndexKept = options.maxEntries - 1;++ return async (input: DeadLetterInput): Promise<void> => {+ const results = await options.client+ .multi()+ .lpush(key, JSON.stringify(toDeadLetterRecord(input)))+ .ltrim(key, 0, lastIndexKept)+ .exec();++ if (results === null) {+ throw new Error(`dead-letter MULTI on ${key} was discarded by Redis`);+ }+ for (const [error] of results) {+ if (error) {+ throw error;+ }+ }+ };+}diff --git a/packages/redis/index.ts b/packages/redis/index.tsindex eae3c08a4..59f2651b4 100644--- a/packages/redis/index.ts+++ b/packages/redis/index.ts@@ -1,4 +1,5 @@export * from './redis';export * from './cachable';+export * from './dead-letter';export * from './run-every';export * from './publisher';