Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M20-001): Dead-letter failed events to a capped Redis list and drop them unconditionally, replacing a Kafka DLQ produce that has never once succeeded

ralph99449f170e0b
diff --git a/apps/api/src/config/env.test.ts b/apps/api/src/config/env.test.ts
index c44465db1..3fea74a6c 100644
--- a/apps/api/src/config/env.test.ts
+++ b/apps/api/src/config/env.test.ts
@@ -245,6 +245,21 @@ describe('derived values, computed once in the transform', () => {
).toBe('own');
});
+ it('the dead-letter list cap defaults to 1000 and takes a positive integer', () => {
+ // The Kafka DLQ topic above is no longer the dead-letter destination
+ // (M20-001) — it is kept only so the seam can be swapped back.
+ expect(loadConfig(base).INGEST_DEAD_LETTER_MAX_ENTRIES).toBe(1000);
+ expect(
+ loadConfig({ ...base, INGEST_DEAD_LETTER_MAX_ENTRIES: '25' })
+ .INGEST_DEAD_LETTER_MAX_ENTRIES
+ ).toBe(25);
+ // Same doctrine as its Kafka neighbours: a cap of 0 or -5 would silently
+ // keep nothing, so it fails boot rather than becoming a surprise.
+ expect(() =>
+ loadConfig({ ...base, INGEST_DEAD_LETTER_MAX_ENTRIES: '0' })
+ ).toThrow('INGEST_DEAD_LETTER_MAX_ENTRIES');
+ });
+
it('the log exporter and the stdout interception verdict', () => {
expect(loadConfig(base).core.logging).toMatchObject({
exporter: 'stdout',
diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts
index 30d0005f0..33cb4782b 100644
--- a/apps/api/src/config/env.ts
+++ b/apps/api/src/config/env.ts
@@ -74,6 +74,12 @@ const DEFAULT_KAFKA_PRODUCER_BATCH_SIZE = 25;
*/
const DEFAULT_KAFKA_PRODUCER_BATCH_LINGER_MS = 5;
const DEFAULT_KAFKA_HANDLER_MAX_ATTEMPTS = 3;
+/**
+ * How many dead-lettered events the capped Redis list keeps (M20-001). A
+ * debugging sample, not a recovery mechanism — the volume is in
+ * `kafka_events_dead_lettered_total`. Carl's "only keep the last N events".
+ */
+const DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES = 1000;
const DEFAULT_KAFKA_HANDLER_RETRY_INITIAL_MS = 100;
const DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS = 1000;
@@ -449,6 +455,11 @@ const rawSchema = z.object({
DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS
),
+ // --- ingest dead-letter ---
+ INGEST_DEAD_LETTER_MAX_ENTRIES: positiveIntWithDefault(
+ DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES
+ ),
+
// --- buffers ---
BUFFER_ASYNC_INSERTS: definedIsTrueSchema,
BUFFER_CH_INSERT_CONCURRENCY: optionalPositiveInt,
@@ -795,6 +806,13 @@ export interface Config {
DISABLE_BULLBOARD: boolean;
COOKIE_SECRET: string;
SHUTDOWN_FORCE_EXIT_MS: number;
+ /**
+ * The dead-letter list's cap (M20-001). It sits here rather than in
+ * `core.kafka` because the destination is no longer Kafka: `apps/api`
+ * chooses the dead-letter sink, and `packages/core` only knows the
+ * `sendToDeadLetter` seam it is handed.
+ */
+ INGEST_DEAD_LETTER_MAX_ENTRIES: number;
/** The CORS delegator's origin allowlist, in V1's own order. */
dashboardOrigins: string[];
/** V1's `ENABLE_VERBOSE_LOGGING`, already split. */
@@ -834,6 +852,7 @@ const envSchema = rawSchema
DISABLE_BULLBOARD: raw.DISABLE_BULLBOARD,
COOKIE_SECRET: raw.COOKIE_SECRET,
SHUTDOWN_FORCE_EXIT_MS: raw.SHUTDOWN_FORCE_EXIT_MS,
+ INGEST_DEAD_LETTER_MAX_ENTRIES: raw.INGEST_DEAD_LETTER_MAX_ENTRIES,
dashboardOrigins: [
dashboardUrl,
...splitTokens(raw.API_CORS_ORIGINS),
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index efb5f9fa0..ba8dd1fe6 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -62,7 +62,6 @@ import {
kafkaLogger,
markEventsActivity,
opsRoutes,
- produceDeadLetterEvent,
publicApiRoutes,
type QueueDefinition,
type QueueProducerHandle,
@@ -84,7 +83,12 @@ import {
} from '@openpanel/core';
import { ch } from '@openpanel/db/src/clickhouse/client';
import { db, Prisma } from '@openpanel/db/src/prisma-client';
-import { getRedisCache, getRedisPub, getRedisQueue } from '@openpanel/redis';
+import {
+ createDeadLetterRecorder,
+ getRedisCache,
+ getRedisPub,
+ getRedisQueue,
+} from '@openpanel/redis';
import { Elysia } from 'elysia';
import pino from 'pino';
import { type Config, KAFKA_QUEUE_TOKEN, loadConfig } from './config/env';
@@ -269,8 +273,17 @@ async function startIngestConsumer(
handleEvent: createIncomingEventHandler(deps, {
checkNotificationRulesForEvent,
}),
- sendToDeadLetter: (message) =>
- produceDeadLetterEvent(config.core, message),
+ // M20-001: a capped Redis list, and the event is DROPPED whether or not
+ // the record lands. The Kafka DLQ it replaced
+ // (`produceDeadLetterEvent`, still exported from core) produced to a
+ // topic nothing creates — 1,592 failures, 0 successes in drill 08 — and
+ // every failure held the offset back, which was the redelivery loop.
+ // The CACHE client, not the queue client: only that one fails fast, and
+ // failing fast is the point here.
+ sendToDeadLetter: createDeadLetterRecorder({
+ client: deps.redis,
+ maxEntries: config.INGEST_DEAD_LETTER_MAX_ENTRIES,
+ }),
// Opened before the batch's first handler and closed before its first
// resolved offset; the gate THROWS if the rpush did not land.
// `pendingEvents` is the only in-process buffer state in the tree —
diff --git a/packages/core/src/modules/ingest/src/consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.ts
index 8f1d3e26e..fc449500a 100644
--- a/packages/core/src/modules/ingest/src/consumer.test.ts
+++ b/packages/core/src/modules/ingest/src/consumer.test.ts
@@ -10,6 +10,12 @@
*/
import { describe, expect, mock, test } from 'bun:test';
+import {
+ createDeadLetterRecorder,
+ type DeadLetterMulti,
+ type DeadLetterRecord,
+ type DeadLetterRedisClient,
+} from '@openpanel/redis';
import type { EachBatchPayload, KafkaMessage } from 'kafkajs';
import {
createEventsBatchHandler,
@@ -300,9 +306,9 @@ describe('handler failure', () => {
expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2']);
});
- test('leaves the offset unresolved when the dead-letter produce fails', async () => {
+ test('resolves the offset even when the dead-letter write fails', async () => {
const { deps, handleEvent, metrics } = makeDeps({
- sendToDeadLetter: mock(() => Promise.reject(new Error('broker down'))),
+ sendToDeadLetter: mock(() => Promise.reject(new Error('redis down'))),
});
handleEvent.mockImplementation(async (p: { projectId: string }) => {
if (p.projectId === 'project-1') {
@@ -318,9 +324,11 @@ describe('handler failure', () => {
await handler.eachBatch(payload);
+ // Dropped WITHOUT being recorded — and still acked. Holding the offset
+ // back here is what rebuilt the redelivery loop (M20-001, gate M20).
expect(metrics.deadLetterFailed).toHaveBeenCalledWith(String(PARTITION));
- // Nothing at or after the failed offset is acked — it is redelivered.
- expect(resolvedOffsets(resolveOffset)).toEqual(['0']);
+ expect(metrics.deadLettered).not.toHaveBeenCalled();
+ expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2']);
});
});
@@ -670,3 +678,201 @@ describe('durability before commit', () => {
expect(flushBufferedEvents).toHaveBeenCalledTimes(1);
});
});
+
+/**
+ * M20-001 end to end: the consumer wired to the REAL capped-list recorder
+ * (`@openpanel/redis`'s `createDeadLetterRecorder`) instead of a mock, against
+ * an in-memory stand-in for the one MULTI it issues. Redis itself is out of
+ * the picture; the LPUSH/LTRIM arithmetic is not.
+ */
+describe('dead letter to a capped redis list', () => {
+ const LIST_KEY = 'dead_letter:events-test';
+ const MAX_ENTRIES = 3;
+ const OK: [Error | null, unknown] = [null, 1];
+
+ /** Enough of a Redis client for one MULTI, with real list semantics. */
+ function fakeRedis(execError?: Error) {
+ const entries: string[] = [];
+ let roundTrips = 0;
+ const client: DeadLetterRedisClient = {
+ multi() {
+ const queued: (() => void)[] = [];
+ const chain: DeadLetterMulti = {
+ lpush(_key, value) {
+ queued.push(() => entries.unshift(value));
+ return chain;
+ },
+ ltrim(_key, start, stop) {
+ queued.push(() => entries.splice(stop + 1 - start));
+ return chain;
+ },
+ async exec() {
+ roundTrips += 1;
+ if (execError) {
+ throw execError;
+ }
+ for (const run of queued) {
+ run();
+ }
+ return queued.map(() => OK);
+ },
+ };
+ return chain;
+ },
+ };
+ return {
+ client,
+ records: () => entries.map((raw) => JSON.parse(raw) as DeadLetterRecord),
+ roundTrips: () => roundTrips,
+ };
+ }
+
+ function makeRecordingDeps(execError?: Error) {
+ const redis = fakeRedis(execError);
+ const built = makeDeps({
+ sendToDeadLetter: createDeadLetterRecorder({
+ client: redis.client,
+ maxEntries: MAX_ENTRIES,
+ key: LIST_KEY,
+ }),
+ });
+ return { ...built, redis };
+ }
+
+ test('a handler that exhausts its attempts leaves ONE record and a RESOLVED offset', async () => {
+ const { deps, handleEvent, metrics, redis } = makeRecordingDeps();
+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));
+ const handler = createEventsBatchHandler(deps);
+ const { payload, resolveOffset } = makeBatch([message(0, 'a')]);
+
+ await handler.eachBatch(payload);
+
+ expect(handleEvent).toHaveBeenCalledTimes(3);
+ const records = redis.records();
+ expect(records).toHaveLength(1);
+ expect(records[0]).toMatchObject({
+ reason: 'handler_error',
+ error: 'boom',
+ offset: '0',
+ partition: PARTITION,
+ topic: TOPIC,
+ key: 'a',
+ });
+ expect(JSON.parse(records[0]?.value ?? '')).toMatchObject({
+ projectId: 'project-0',
+ });
+ // One MULTI, so one round trip per dropped event.
+ expect(redis.roundTrips()).toBe(1);
+ expect(metrics.deadLettered).toHaveBeenCalledWith(
+ String(PARTITION),
+ 'handler_error'
+ );
+ expect(resolvedOffsets(resolveOffset)).toEqual(['0']);
+ });
+
+ test('a DEAD REDIS still resolves the offset and records nothing', async () => {
+ // Drill 02's scenario exactly: the dependency that fails the handler is
+ // the dependency the dead letter is written to. If this offset is left
+ // unresolved the redelivery loop is back, in a new place.
+ const { deps, handleEvent, metrics, redis } = makeRecordingDeps(
+ new Error(
+ "Stream isn't writeable and enableOfflineQueue options is false"
+ )
+ );
+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));
+ const handler = createEventsBatchHandler(deps);
+ const { payload, resolveOffset } = makeBatch([
+ message(0, 'a'),
+ message(1, 'a'),
+ message(2, 'a'),
+ ]);
+
+ await handler.eachBatch(payload);
+
+ expect(redis.records()).toEqual([]);
+ expect(metrics.deadLetterFailed).toHaveBeenCalledTimes(3);
+ expect(metrics.deadLettered).not.toHaveBeenCalled();
+ expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2']);
+ });
+
+ test('a per-command error inside the MULTI counts as not recorded', async () => {
+ // WRONGTYPE is drill 08's own fault shape, and a MULTI reports it in
+ // `exec()`'s results rather than by rejecting.
+ const redis = fakeRedis();
+ const wrongType: [Error | null, unknown] = [
+ new Error(
+ 'WRONGTYPE Operation against a key holding the wrong kind of value'
+ ),
+ null,
+ ];
+ const failing: DeadLetterRedisClient = {
+ multi() {
+ const chain: DeadLetterMulti = {
+ lpush: () => chain,
+ ltrim: () => chain,
+ exec: () => Promise.resolve([wrongType, OK]),
+ };
+ return chain;
+ },
+ };
+ const { deps, handleEvent, metrics } = makeDeps({
+ sendToDeadLetter: createDeadLetterRecorder({
+ client: failing,
+ maxEntries: MAX_ENTRIES,
+ key: LIST_KEY,
+ }),
+ });
+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));
+ const handler = createEventsBatchHandler(deps);
+ const { payload, resolveOffset } = makeBatch([message(0, 'a')]);
+
+ await handler.eachBatch(payload);
+
+ expect(redis.records()).toEqual([]);
+ expect(metrics.deadLetterFailed).toHaveBeenCalledWith(String(PARTITION));
+ expect(metrics.deadLettered).not.toHaveBeenCalled();
+ expect(resolvedOffsets(resolveOffset)).toEqual(['0']);
+ });
+
+ test('the list never exceeds N and keeps the most recent drops', async () => {
+ const { deps, handleEvent, redis } = makeRecordingDeps();
+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));
+ const handler = createEventsBatchHandler(deps);
+ // One key, so one serial group: the drop order is the offset order.
+ const { payload, resolveOffset } = makeBatch(
+ [0, 1, 2, 3, 4].map((offset) => message(offset, 'a'))
+ );
+
+ await handler.eachBatch(payload);
+
+ const records = redis.records();
+ expect(records).toHaveLength(MAX_ENTRIES);
+ // Newest first, and the two oldest drops fell off the tail.
+ expect(records.map((record) => record.offset)).toEqual(['4', '3', '2']);
+ expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2', '3', '4']);
+ });
+
+ test('a parse failure takes the same path as a handler failure', async () => {
+ const { deps, handleEvent, metrics, redis } = makeRecordingDeps();
+ const handler = createEventsBatchHandler(deps);
+ const { payload, resolveOffset } = makeBatch([
+ message(0, 'a', 'not json at all'),
+ ]);
+
+ await handler.eachBatch(payload);
+
+ expect(handleEvent).not.toHaveBeenCalled();
+ const records = redis.records();
+ expect(records).toHaveLength(1);
+ expect(records[0]).toMatchObject({
+ reason: 'parse_error',
+ offset: '0',
+ value: 'not json at all',
+ });
+ expect(metrics.deadLettered).toHaveBeenCalledWith(
+ String(PARTITION),
+ 'parse_error'
+ );
+ expect(resolvedOffsets(resolveOffset)).toEqual(['0']);
+ });
+});
diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.ts
index ea6ec287f..47146f83b 100644
--- a/packages/core/src/modules/ingest/src/consumer.ts
+++ b/packages/core/src/modules/ingest/src/consumer.ts
@@ -1,7 +1,10 @@
// Ported from apps/worker/src/jobs/events.kafka-consumer.ts (M8-003). The
// delivery contract is ADR-004's and is unchanged line for line: per-key
// serial groups, the ascending contiguous-prefix offset walk, at-least-once,
-// bounded in-consumer retry, and a dead-letter topic on exhaustion.
+// bounded in-consumer retry, and a dead-letter on exhaustion. Only the
+// dead-letter DESTINATION and its failure behaviour changed (M20-001): the
+// record goes to a capped Redis list, and the event is dropped whether or not
+// that write lands.
//
// The kafkajs client, the topic/consumer-group names, the DLQ producer and
// the retry bounds are all INJECTED, and stay so now that they live one
@@ -57,10 +60,15 @@ const DURABILITY_RETRY_MAX_MS = 5000;
export type DeadLetterReason = 'parse_error' | 'handler_error';
/**
- * What the consumer hands to the dead-letter producer (./kafka.ts's
- * `produceDeadLetterEvent` is the one implementation): the value stays the
- * producer's original bytes, so a DLQ message can be replayed onto the events
- * topic unchanged.
+ * What the consumer hands to the dead-letter sink. The value stays the
+ * producer's original bytes, so the record is a faithful copy of what arrived.
+ *
+ * Two implementations satisfy this, and the seam exists so the choice is one
+ * line in `apps/api`'s wiring: `@openpanel/redis`'s `createDeadLetterRecorder`
+ * (the capped list, in use since M20-001) and ./kafka.ts's
+ * `produceDeadLetterEvent` (the DLQ topic, kept because Carl's decision was
+ * explicitly "for now"; a DLQ message produced from these bytes is replayable
+ * onto the events topic unchanged).
*/
export interface DeadLetterMessage {
key: Buffer | null;
@@ -82,7 +90,12 @@ export interface ConsumerLogger {
export interface ConsumerMetrics {
reprocessed: (partition: string) => void;
handlerFailed: (partition: string) => void;
+ /** The message was recorded in the dead-letter list and DROPPED. */
deadLettered: (partition: string, reason: DeadLetterReason) => void;
+ /**
+ * The message was DROPPED WITHOUT being recorded — the dead-letter write
+ * failed. Not "will be retried": nothing retries it (M20-001).
+ */
deadLetterFailed: (partition: string) => void;
}
@@ -91,6 +104,10 @@ export interface EventsBatchHandlerDeps {
payload: IncomingEventPayload,
meta: { partition: number; offset: string }
) => Promise<unknown>;
+ /**
+ * Park a poison message somewhere an operator can read it. It may reject —
+ * the caller drops the message and resolves the offset regardless.
+ */
sendToDeadLetter: (message: DeadLetterMessage) => Promise<void>;
/**
* Opens a durability window over the event buffer and returns the gate that
@@ -140,7 +157,10 @@ export interface EventsBatchHandler {
*
* Delivery contract (ADR-004): at-least-once. An offset is resolved only once
* its message has been handled, dead-lettered, or deliberately skipped —
- * never merely because it failed.
+ * never merely because it failed. Since M20-001 a dead-letter always counts as
+ * finished, because the message is dropped either way; the one place an
+ * unresolved offset is still correct is a failed DURABILITY flush, which
+ * leaves the whole batch for redelivery (M18-001).
*/
export function createEventsBatchHandler(
deps: EventsBatchHandlerDeps
@@ -155,12 +175,31 @@ export function createEventsBatchHandler(
// post-rebalance redelivery from the last committed offset doesn't trip it.
const resolvedHWM = new Map<string, number>();
+ /**
+ * Park the message and DROP it. The drop is unconditional: whether the
+ * record was stored or not, this message's offset is resolved by the caller
+ * and the event is gone from the pipeline (M20-001, gate M20).
+ *
+ * There is deliberately no retry, no backoff and no unresolved offset here.
+ * The dead-letter destination is Redis, and Redis being unavailable is
+ * exactly when handlers fail — so a failed write that held the offset back
+ * would rebuild the very redelivery loop this replaced (drill 02's re-run 2:
+ * 78-79 laps per message at ~0.62 s, no backoff; drill 08: 1,592 failed
+ * dead-letters, 0 successes). Same lesson as M18-007's `pendingEvents`
+ * re-queue: once something else owns the outcome, the safety net underneath
+ * it is the bug.
+ *
+ * The two counters are what carry the volume, since a capped list makes
+ * 50,000 drops look like 12: `deadLettered` = recorded and dropped,
+ * `deadLetterFailed` = dropped WITHOUT being recorded. Neither means
+ * "will be retried".
+ */
const deadLetter = async (
message: KafkaMessage,
partition: number,
reason: DeadLetterReason,
err: unknown
- ): Promise<boolean> => {
+ ): Promise<void> => {
const label = String(partition);
try {
await deps.sendToDeadLetter({
@@ -174,21 +213,27 @@ export function createEventsBatchHandler(
error: errorMessage(err),
});
} catch (dlqErr) {
- // The event is not lost: leaving the offset unresolved stops the
- // contiguous-prefix walk here, so the broker redelivers this message.
deps.metrics.deadLetterFailed(label);
+ // The WRITE failure goes in `err` — it is the news, and only that field
+ // gets the logger's error serializer. What the message originally did
+ // wrong rides along as text.
deps.logger.error(
- { err: dlqErr, partition, offset: message.offset, reason },
- 'kafka dead-letter produce failed — offset left unresolved for redelivery'
+ {
+ err: dlqErr,
+ cause: errorMessage(err),
+ partition,
+ offset: message.offset,
+ reason,
+ },
+ 'kafka message DROPPED without being recorded — dead-letter write failed'
);
- return false;
+ return;
}
deps.metrics.deadLettered(label, reason);
deps.logger.error(
{ err, partition, offset: message.offset, reason },
- 'kafka message dead-lettered'
+ 'kafka message dead-lettered — recorded and dropped'
);
- return true;
};
const runHandlerWithRetry = async (
@@ -228,11 +273,15 @@ export function createEventsBatchHandler(
return { ok: false, err: lastErr };
};
- /** @returns true when the offset may be resolved. */
+ /**
+ * Always returns: the offset is resolvable either way. A message is handled,
+ * or it is dead-lettered and dropped — there is no third outcome, which is
+ * why the batch loop below has no unresolvable-message branch.
+ */
const processMessage = async (
message: KafkaMessage,
partition: number
- ): Promise<boolean> => {
+ ): Promise<void> => {
let payload: IncomingEventPayload;
try {
if (!message.value) {
@@ -245,7 +294,8 @@ export function createEventsBatchHandler(
'kafka message parse failed'
);
// Deterministic: retrying identical bytes cannot succeed.
- return deadLetter(message, partition, 'parse_error', err);
+ await deadLetter(message, partition, 'parse_error', err);
+ return;
}
const result = await runHandlerWithRetry(
@@ -254,9 +304,9 @@ export function createEventsBatchHandler(
message.offset
);
if (result.ok) {
- return true;
+ return;
}
- return deadLetter(message, partition, 'handler_error', result.err);
+ await deadLetter(message, partition, 'handler_error', result.err);
};
// Consecutive durability failures per `topic-partition`, for the backoff
@@ -379,12 +429,12 @@ export function createEventsBatchHandler(
);
}
- const resolvable = await processMessage(m, batch.partition);
- if (!resolvable) {
- // Stop this key-group: continuing past an unresolvable offset only
- // adds duplicates, since the whole tail is redelivered anyway.
- return;
- }
+ // No "unresolvable" branch: a message is handled or dropped, and a
+ // dropped one is still finished (M20-001). The only gap this walk
+ // can now stop at is the isRunning/isStale early return above, and
+ // the only offsets left uncommitted are a whole batch whose
+ // durability flush failed — see below.
+ await processMessage(m, batch.partition);
processed.add(m.offset);
processedCount += 1;
diff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.ts
index 36fb0b4eb..b671f3339 100644
--- a/packages/core/src/modules/ingest/src/ingest.metrics.ts
+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts
@@ -29,20 +29,25 @@ export const kafkaHandlerFailuresTotal = new client.Counter({
registers: [registry],
});
-// Messages written to the dead-letter topic after their attempts were
-// exhausted (or after they failed to parse). One increment per message.
+// Messages recorded in the dead-letter list after their attempts were
+// exhausted (or after they failed to parse), then DROPPED. One increment per
+// message. The list is capped at the last N, so this counter — not the list —
+// is the only thing that tells 50,000 drops apart from 12 (M20-001). The
+// series name is unchanged: the destination moved, the meaning did not.
export const kafkaDeadLetteredTotal = new client.Counter({
name: 'kafka_events_dead_lettered_total',
- help: 'Kafka event messages produced to the dead-letter topic',
+ help: 'Kafka event messages recorded in the dead-letter list and dropped',
labelNames: ['partition', 'reason'],
registers: [registry],
});
-// The dead-letter produce itself failed. The offset is then left unresolved
-// and the message is redelivered, so this is a stuck partition, not a loss.
+// The dead-letter write itself failed, so the message was DROPPED WITHOUT
+// being recorded — Redis unreachable, most likely, which is also when handlers
+// fail. This is real loss, not a stuck partition: the offset is resolved
+// anyway and nothing retries the message (M20-001, gate M20).
export const kafkaDeadLetterFailedTotal = new client.Counter({
name: 'kafka_events_dead_letter_failed_total',
- help: 'Failed attempts to produce a Kafka event message to the dead-letter topic',
+ help: 'Kafka event messages dropped without being recorded (dead-letter write failed)',
labelNames: ['partition'],
registers: [registry],
});
diff --git a/packages/redis/dead-letter.test.ts b/packages/redis/dead-letter.test.ts
new file mode 100644
index 000000000..c39b820fd
--- /dev/null
+++ b/packages/redis/dead-letter.test.ts
@@ -0,0 +1,192 @@
+/**
+ * M20-001 — the capped dead-letter list, at the Redis boundary.
+ *
+ * What the consumer does with a rejection is `packages/core`'s
+ * `consumer.test.ts`; what this file pins is the write itself: one MULTI, the
+ * record's shape, and the two ways a MULTI can fail without rejecting.
+ *
+ * No real Redis: an in-memory stand-in with real list semantics. The same
+ * assertions were run against a local Redis (`redis://127.0.0.1:6379/15`) on
+ * 2026-09-14 before this file was written — LLEN 3 after 7 pushes at N=3,
+ * newest first, and a key holding a string rejecting with `WRONGTYPE`.
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+ createDeadLetterRecorder,
+ DEAD_LETTER_LIST_KEY,
+ type DeadLetterMulti,
+ type DeadLetterRecord,
+ type DeadLetterRedisClient,
+} from './dead-letter';
+
+const OK: [Error | null, unknown] = [null, 1];
+const MAX_ENTRIES = 3;
+
+const input = (offset: string, overrides: Record<string, unknown> = {}) => ({
+ key: Buffer.from('device-1'),
+ value: Buffer.from(JSON.stringify({ projectId: 'p1', offset })),
+ headers: {
+ 'request-id': Buffer.from('req-1'),
+ attempts: ['1', '2'],
+ plain: 'text',
+ },
+ topic: 'events',
+ partition: 7,
+ offset,
+ reason: 'handler_error',
+ error: 'handler exploded',
+ ...overrides,
+});
+
+function fakeRedis() {
+ const entries: string[] = [];
+ const keysWritten: string[] = [];
+ const trims: [number, number][] = [];
+ let execs = 0;
+ const client: DeadLetterRedisClient = {
+ multi() {
+ const queued: (() => void)[] = [];
+ const chain: DeadLetterMulti = {
+ lpush(key, value) {
+ keysWritten.push(key);
+ queued.push(() => entries.unshift(value));
+ return chain;
+ },
+ ltrim(_key, start, stop) {
+ trims.push([start, stop]);
+ queued.push(() => entries.splice(stop + 1 - start));
+ return chain;
+ },
+ async exec() {
+ execs += 1;
+ for (const run of queued) {
+ run();
+ }
+ return queued.map(() => OK);
+ },
+ };
+ return chain;
+ },
+ };
+ return {
+ client,
+ keysWritten,
+ trims,
+ execs: () => execs,
+ records: () => entries.map((raw) => JSON.parse(raw) as DeadLetterRecord),
+ };
+}
+
+describe('createDeadLetterRecorder', () => {
+ it('writes one record per event in a single round trip', async () => {
+ const redis = fakeRedis();
+ const record = createDeadLetterRecorder({
+ client: redis.client,
+ maxEntries: MAX_ENTRIES,
+ });
+
+ await record(input('100'));
+
+ expect(redis.execs()).toBe(1);
+ expect(redis.keysWritten).toEqual([DEAD_LETTER_LIST_KEY]);
+ expect(redis.trims).toEqual([[0, MAX_ENTRIES - 1]]);
+ });
+
+ it('stores the coordinates, the reason, the error and the raw payload', async () => {
+ const redis = fakeRedis();
+ const record = createDeadLetterRecorder({
+ client: redis.client,
+ maxEntries: MAX_ENTRIES,
+ });
+
+ await record(input('100'));
+
+ const [stored] = redis.records();
+ expect(stored).toMatchObject({
+ topic: 'events',
+ partition: 7,
+ offset: '100',
+ reason: 'handler_error',
+ error: 'handler exploded',
+ key: 'device-1',
+ // Buffer and array header values come back as text, not as `{"type":"Buffer"}`.
+ headers: { 'request-id': 'req-1', attempts: '1,2', plain: 'text' },
+ });
+ expect(JSON.parse(stored?.value ?? '')).toMatchObject({ projectId: 'p1' });
+ expect(Number.isNaN(Date.parse(stored?.recordedAt ?? ''))).toBe(false);
+ });
+
+ it('keeps a message with no key and no value at all', async () => {
+ const redis = fakeRedis();
+ const record = createDeadLetterRecorder({
+ client: redis.client,
+ maxEntries: MAX_ENTRIES,
+ });
+
+ await record(input('100', { key: null, value: null, headers: undefined }));
+
+ expect(redis.records()[0]).toMatchObject({
+ key: null,
+ value: null,
+ headers: {},
+ });
+ });
+
+ it('keeps only the last N, newest first', async () => {
+ const redis = fakeRedis();
+ const record = createDeadLetterRecorder({
+ client: redis.client,
+ maxEntries: MAX_ENTRIES,
+ });
+
+ for (const offset of ['1', '2', '3', '4', '5']) {
+ await record(input(offset));
+ }
+
+ expect(redis.records().map((stored) => stored.offset)).toEqual([
+ '5',
+ '4',
+ '3',
+ ]);
+ });
+
+ it('rejects on a per-command error, which a MULTI does not itself reject for', async () => {
+ const wrongType = new Error('WRONGTYPE Operation against a key holding…');
+ const client: DeadLetterRedisClient = {
+ multi() {
+ const chain: DeadLetterMulti = {
+ lpush: () => chain,
+ ltrim: () => chain,
+ exec: () => Promise.resolve([[wrongType, null], OK]),
+ };
+ return chain;
+ },
+ };
+ const record = createDeadLetterRecorder({
+ client,
+ maxEntries: MAX_ENTRIES,
+ });
+
+ await expect(record(input('100'))).rejects.toThrow(wrongType);
+ });
+
+ it('rejects when the MULTI is discarded', async () => {
+ const client: DeadLetterRedisClient = {
+ multi() {
+ const chain: DeadLetterMulti = {
+ lpush: () => chain,
+ ltrim: () => chain,
+ exec: () => Promise.resolve(null),
+ };
+ return chain;
+ },
+ };
+ const record = createDeadLetterRecorder({
+ client,
+ maxEntries: MAX_ENTRIES,
+ });
+
+ await expect(record(input('100'))).rejects.toThrow('discarded by Redis');
+ });
+});
diff --git a/packages/redis/dead-letter.ts b/packages/redis/dead-letter.ts
new file mode 100644
index 000000000..53ced4472
--- /dev/null
+++ b/packages/redis/dead-letter.ts
@@ -0,0 +1,160 @@
+/**
+ * The ingest consumer's dead-letter destination: a capped Redis list (M20-001).
+ *
+ * Carl, 2026-09-14: *"For now, we can just store errors in redis and drop the
+ * event, push it to a redis list and only keep the last N events."*
+ *
+ * It replaces `produceDeadLetterEvent`, which produced to
+ * `${KAFKA_EVENTS_TOPIC}-dlq` — a topic nothing in the tree ever creates.
+ * Drill 08 measured that produce failing 1,592 times with 0 successes
+ * (`Number of partitions is invalid`), and the failure left the offset
+ * unresolved, which is the unbounded handler-error redelivery loop drill 02's
+ * re-run 2 measured at 78-79 laps per message. The Kafka producer is still
+ * exported from `@openpanel/core`, and the consumer's `sendToDeadLetter` seam
+ * is unchanged, so swapping back is one line in `apps/api`'s wiring.
+ *
+ * THE RECORD IS A DEBUGGING SAMPLE, NOT A RECOVERY MECHANISM. The list keeps
+ * the last N and nothing more; the event itself is gone from the pipeline. The
+ * volume lives in `kafka_events_dead_lettered_total` / `_dead_letter_failed_total`,
+ * because a capped list makes 50,000 drops look exactly like 12.
+ *
+ * Why here and not in `packages/core`'s ingest module: the whole of it is
+ * Redis mechanics — one MULTI, one LPUSH, one LTRIM — and core's only export
+ * door is its barrel, which M20-001's scope does not cover. The input type is
+ * structural, so `DeadLetterMessage` (kafkajs `Buffer`s and `IHeaders`)
+ * satisfies it without this package depending on kafkajs or on core.
+ */
+
+/** The one list. Not configurable: an operator has to be able to find it. */
+export const DEAD_LETTER_LIST_KEY = 'dead_letter:events';
+
+/** A `Buffer`, or anything else the caller hands us as raw message bytes. */
+interface BinaryLike {
+ toString(): string;
+}
+
+/**
+ * Structurally what `@openpanel/core`'s `DeadLetterMessage` is — the same
+ * fields `produceDeadLetterEvent` put in its DLQ headers, so a record carries
+ * everything needed to find the message on the source topic.
+ */
+export interface DeadLetterInput {
+ key: BinaryLike | null;
+ value: BinaryLike | null;
+ headers?: Record<string, unknown> | undefined;
+ topic: string;
+ partition: number;
+ offset: string;
+ reason: string;
+ error: string;
+}
+
+/** One entry of the list, as stored (JSON). */
+export interface DeadLetterRecord {
+ recordedAt: string;
+ topic: string;
+ partition: number;
+ offset: string;
+ reason: string;
+ error: string;
+ key: string | null;
+ /** The producer's original payload, as text — events on this topic are JSON. */
+ value: string | null;
+ headers: Record<string, string>;
+}
+
+/**
+ * The narrow slice of a Redis client this needs. Structural rather than
+ * ioredis's `Redis`, so the contract is one MULTI and a test can satisfy it
+ * without a server.
+ */
+export interface DeadLetterMulti {
+ lpush(key: string, value: string): DeadLetterMulti;
+ ltrim(key: string, start: number, stop: number): DeadLetterMulti;
+ exec(): Promise<[Error | null, unknown][] | null>;
+}
+
+export interface DeadLetterRedisClient {
+ multi(): DeadLetterMulti;
+}
+
+export interface DeadLetterRecorderOptions {
+ client: DeadLetterRedisClient;
+ /** How many records the list keeps. The oldest fall off the tail. */
+ maxEntries: number;
+ /** Overridable for tests; production uses the constant above. */
+ key?: string;
+}
+
+const toText = (value: unknown): string => {
+ if (typeof value === 'string') {
+ return value;
+ }
+ if (Array.isArray(value)) {
+ return value.map(toText).join(',');
+ }
+ if (value === null || value === undefined) {
+ return '';
+ }
+ return String(value);
+};
+
+const normalizeHeaders = (
+ headers: Record<string, unknown> | undefined
+): Record<string, string> => {
+ const normalized: Record<string, string> = {};
+ for (const [name, value] of Object.entries(headers ?? {})) {
+ normalized[name] = toText(value);
+ }
+ return normalized;
+};
+
+export const toDeadLetterRecord = (
+ input: DeadLetterInput,
+ recordedAt: Date = new Date()
+): DeadLetterRecord => ({
+ recordedAt: recordedAt.toISOString(),
+ topic: input.topic,
+ partition: input.partition,
+ offset: input.offset,
+ reason: input.reason,
+ error: input.error,
+ key: input.key === null ? null : input.key.toString(),
+ value: input.value === null ? null : input.value.toString(),
+ headers: normalizeHeaders(input.headers),
+});
+
+/**
+ * ONE round trip per dropped event: LPUSH and LTRIM travel in a single MULTI,
+ * so the list is never observed above `maxEntries` and a dropped event never
+ * costs the ingest path two waits.
+ *
+ * It REJECTS when the write did not land — including a per-command error such
+ * as `WRONGTYPE`, which a MULTI reports inside `exec()`'s results rather than
+ * by rejecting. The caller counts that as "dropped without being recorded" and
+ * resolves the offset anyway: Redis being unavailable is exactly when handlers
+ * fail, and an unresolved offset there is the redelivery loop this replaces.
+ */
+export function createDeadLetterRecorder(
+ options: DeadLetterRecorderOptions
+): (input: DeadLetterInput) => Promise<void> {
+ const key = options.key ?? DEAD_LETTER_LIST_KEY;
+ const lastIndexKept = options.maxEntries - 1;
+
+ return async (input: DeadLetterInput): Promise<void> => {
+ const results = await options.client
+ .multi()
+ .lpush(key, JSON.stringify(toDeadLetterRecord(input)))
+ .ltrim(key, 0, lastIndexKept)
+ .exec();
+
+ if (results === null) {
+ throw new Error(`dead-letter MULTI on ${key} was discarded by Redis`);
+ }
+ for (const [error] of results) {
+ if (error) {
+ throw error;
+ }
+ }
+ };
+}
diff --git a/packages/redis/index.ts b/packages/redis/index.ts
index eae3c08a4..59f2651b4 100644
--- a/packages/redis/index.ts
+++ b/packages/redis/index.ts
@@ -1,4 +1,5 @@
export * from './redis';
export * from './cachable';
+export * from './dead-letter';
export * from './run-every';
export * from './publisher';