Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M21-001): Delete the offset watermark duplicate detector and mark duplicates with an event-keyed Redis key instead - mark only, never drop

ralphcae38290efe7
diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts
index 33cb4782b..cafbd7fb5 100644
--- a/apps/api/src/config/env.ts
+++ b/apps/api/src/config/env.ts
@@ -83,6 +83,22 @@ const DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES = 1000;
const DEFAULT_KAFKA_HANDLER_RETRY_INITIAL_MS = 100;
const DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS = 1000;
+/**
+ * How long a duplicate marker outlives its event (M21-001). It only has to
+ * outlive the REPLAY window, not the data: drill 08's eviction rejoin — the
+ * one reassignment that costs duplicate rows — was 3,094 ms, and drill 06's
+ * lost-ACK replay landed ~30 s after the produce. Two minutes is ~40x the
+ * first and 4x the second, and covers a process restart resuming from its last
+ * committed offset.
+ *
+ * It is the knob that decides the key count, which is `events/s x TTL`: at the
+ * 5,000 events/s the scaling requirement targets that is 600,000 live markers.
+ * Measured on this box's Redis 2026-09-14 — 200,000 real markers cost
+ * 137.6 bytes each including the expires dict — so ~79 MB. Longer is only
+ * worth buying if a redelivery window longer than this is ever observed.
+ */
+const DEFAULT_INGEST_DUPLICATE_MARKER_TTL_MS = 120_000;
+
/**
* The Kafka events consumer's token. Renamed from `events_kafka` — there is
* one events transport now, so there is one token for it (ADR-004 rec 5/6,
@@ -460,6 +476,11 @@ const rawSchema = z.object({
DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES
),
+ // --- ingest duplicate marker ---
+ INGEST_DUPLICATE_MARKER_TTL_MS: positiveIntWithDefault(
+ DEFAULT_INGEST_DUPLICATE_MARKER_TTL_MS
+ ),
+
// --- buffers ---
BUFFER_ASYNC_INSERTS: definedIsTrueSchema,
BUFFER_CH_INSERT_CONCURRENCY: optionalPositiveInt,
@@ -813,6 +834,13 @@ export interface Config {
* `sendToDeadLetter` seam it is handed.
*/
INGEST_DEAD_LETTER_MAX_ENTRIES: number;
+ /**
+ * How long the ingest consumer's duplicate marker key lives. Same reason it
+ * is not on `core.kafka`: the marker is keyed on the EVENT, not on a Kafka
+ * coordinate, and `packages/core` only knows the `markDuplicateEvent` seam
+ * it is handed.
+ */
+ INGEST_DUPLICATE_MARKER_TTL_MS: number;
/** The CORS delegator's origin allowlist, in V1's own order. */
dashboardOrigins: string[];
/** V1's `ENABLE_VERBOSE_LOGGING`, already split. */
@@ -853,6 +881,7 @@ const envSchema = rawSchema
COOKIE_SECRET: raw.COOKIE_SECRET,
SHUTDOWN_FORCE_EXIT_MS: raw.SHUTDOWN_FORCE_EXIT_MS,
INGEST_DEAD_LETTER_MAX_ENTRIES: raw.INGEST_DEAD_LETTER_MAX_ENTRIES,
+ INGEST_DUPLICATE_MARKER_TTL_MS: raw.INGEST_DUPLICATE_MARKER_TTL_MS,
dashboardOrigins: [
dashboardUrl,
...splitTokens(raw.API_CORS_ORIGINS),
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index ba8dd1fe6..9578f4f5b 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -85,6 +85,7 @@ import { ch } from '@openpanel/db/src/clickhouse/client';
import { db, Prisma } from '@openpanel/db/src/prisma-client';
import {
createDeadLetterRecorder,
+ createDuplicateEventMarker,
getRedisCache,
getRedisPub,
getRedisQueue,
@@ -272,6 +273,15 @@ async function startIngestConsumer(
// slot in its signature.
handleEvent: createIncomingEventHandler(deps, {
checkNotificationRulesForEvent,
+ // M21-001: MARKS a redelivery, never drops one — the event is
+ // inserted either way. The CACHE client, again, and for the same
+ // reason: a handler that blocks on Redis is what got drill 02's
+ // consumer evicted past its session timeout, and an eviction is the
+ // one reassignment M19 measured costing duplicate rows.
+ markDuplicateEvent: createDuplicateEventMarker({
+ client: deps.redis,
+ ttlMs: config.INGEST_DUPLICATE_MARKER_TTL_MS,
+ }),
}),
// M20-001: a capped Redis list, and the event is DROPPED whether or not
// the record lands. The Kafka DLQ it replaced
diff --git a/packages/core/src/modules/ingest/src/consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.ts
index fc449500a..a80523238 100644
--- a/packages/core/src/modules/ingest/src/consumer.test.ts
+++ b/packages/core/src/modules/ingest/src/consumer.test.ts
@@ -78,7 +78,6 @@ type DepOverrides = Partial<
function makeDeps(overrides: DepOverrides = {}) {
const metrics: EventsBatchHandlerDeps['metrics'] = {
- reprocessed: mock(() => undefined),
handlerFailed: mock(() => undefined),
deadLettered: mock(() => undefined),
deadLetterFailed: mock(() => undefined),
@@ -372,59 +371,72 @@ describe('unparseable messages', () => {
});
});
-describe('reprocess counter', () => {
- test('does not fire for intra-batch out-of-order processing', async () => {
- const { deps, metrics } = makeDeps();
+/**
+ * M21-001 deleted the offset watermark, whose only outputs were the
+ * `reprocessed` counter and a warn line — drill 08 measured 161 real
+ * duplicates and 0 of those lines, because `resetWatermarks()` fired on
+ * GROUP_JOIN and a failed durability flush never advanced it.
+ *
+ * These are the regression tests for the half of that claim that matters: the
+ * functional statement in the resolve loop was always `resolveOffset(m.offset)`
+ * and never read the watermark, so a redelivery must resolve and commit
+ * exactly what it did before.
+ */
+describe('a redelivery after the watermark was deleted', () => {
+ test('resolves every offset again, in ascending order', async () => {
+ const { deps } = makeDeps();
const handler = createEventsBatchHandler(deps);
- const { payload } = makeBatch([
- message(0, 'a'),
- message(1, 'b'),
- message(2, 'a'),
- ]);
+ const messages = [message(0, 'a'), message(1, 'a')];
- await handler.eachBatch(payload);
+ const first = makeBatch(messages);
+ await handler.eachBatch(first.payload);
+ expect(resolvedOffsets(first.resolveOffset)).toEqual(['0', '1']);
- expect(metrics.reprocessed).not.toHaveBeenCalled();
+ const redelivered = makeBatch(messages);
+ await handler.eachBatch(redelivered.payload);
+ expect(resolvedOffsets(redelivered.resolveOffset)).toEqual(['0', '1']);
});
- test('fires once per redelivered offset in a later batch', async () => {
- const { deps, metrics } = makeDeps();
+ test('hands the redelivered message to the handler again', async () => {
+ const { deps, handleEvent } = makeDeps();
const handler = createEventsBatchHandler(deps);
- const messages = [message(0, 'a'), message(1, 'a')];
+ const messages = [message(0, 'a')];
await handler.eachBatch(makeBatch(messages).payload);
- expect(metrics.reprocessed).not.toHaveBeenCalled();
-
await handler.eachBatch(makeBatch(messages).payload);
- expect(metrics.reprocessed).toHaveBeenCalledTimes(2);
- expect(metrics.reprocessed).toHaveBeenCalledWith(String(PARTITION));
+
+ // At-least-once, unchanged: the consumer does not suppress a redelivery,
+ // and nothing in this file now decides whether one is a duplicate — the
+ // marker on the event id does (incoming-event-handler.ts).
+ expect(handleEvent).toHaveBeenCalledTimes(2);
});
- test('does not fire after a rebalance clears the watermarks', async () => {
- const { deps, metrics } = makeDeps();
+ test('still resolves a redelivered offset that was dead-lettered', async () => {
+ const { deps, handleEvent } = makeDeps();
+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));
const handler = createEventsBatchHandler(deps);
- const messages = [message(0, 'a'), message(1, 'a')];
+ const messages = [message(0, 'a')];
- await handler.eachBatch(makeBatch(messages).payload);
- handler.resetWatermarks();
- await handler.eachBatch(makeBatch(messages).payload);
+ const first = makeBatch(messages);
+ await handler.eachBatch(first.payload);
+ expect(resolvedOffsets(first.resolveOffset)).toEqual(['0']);
- expect(metrics.reprocessed).not.toHaveBeenCalled();
+ const redelivered = makeBatch(messages);
+ await handler.eachBatch(redelivered.payload);
+ expect(resolvedOffsets(redelivered.resolveOffset)).toEqual(['0']);
});
- test('does not fire for a dead-lettered offset re-seen only within the batch', async () => {
- // A dead-lettered offset is resolved, so it must move the watermark just
- // like a handled one — otherwise the next batch would flag it as a
- // reprocess.
- const { deps, handleEvent, metrics } = makeDeps();
- handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));
+ test('resetDurabilityBackoff on GROUP_JOIN changes no offset behaviour', async () => {
+ const { deps } = makeDeps();
const handler = createEventsBatchHandler(deps);
+ const messages = [message(0, 'a'), message(1, 'b')];
- await handler.eachBatch(makeBatch([message(0, 'a')]).payload);
- expect(metrics.reprocessed).not.toHaveBeenCalled();
+ await handler.eachBatch(makeBatch(messages).payload);
+ handler.resetDurabilityBackoff();
- await handler.eachBatch(makeBatch([message(1, 'a')]).payload);
- expect(metrics.reprocessed).not.toHaveBeenCalled();
+ const afterRebalance = makeBatch(messages);
+ await handler.eachBatch(afterRebalance.payload);
+ expect(resolvedOffsets(afterRebalance.resolveOffset)).toEqual(['0', '1']);
});
});
@@ -504,9 +516,9 @@ describe('durability before commit', () => {
expect(onActivity).toHaveBeenCalledTimes(1);
});
- test('redelivery after a failed flush commits, and is not flagged a duplicate', async () => {
+ test('redelivery after a failed flush commits', async () => {
let redisIsDown = true;
- const { deps, metrics } = makeDeps({
+ const { deps } = makeDeps({
flushBufferedEvents: mock(async () => {
if (redisIsDown) {
throw new Error('redis is unreachable');
@@ -524,9 +536,6 @@ describe('durability before commit', () => {
await handler.eachBatch(redelivered.payload);
expect(resolvedOffsets(redelivered.resolveOffset)).toEqual(['0', '1']);
- // The failed batch never advanced the watermark, so the replay it asked
- // for is not then reported as an at-least-once duplicate.
- expect(metrics.reprocessed).not.toHaveBeenCalled();
});
test('opens the durability window before the first handler buffers anything', async () => {
diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.ts
index 47146f83b..f6ddd05f8 100644
--- a/packages/core/src/modules/ingest/src/consumer.ts
+++ b/packages/core/src/modules/ingest/src/consumer.ts
@@ -88,7 +88,6 @@ export interface ConsumerLogger {
}
export interface ConsumerMetrics {
- reprocessed: (partition: string) => void;
handlerFailed: (partition: string) => void;
/** The message was recorded in the dead-letter list and DROPPED. */
deadLettered: (partition: string, reason: DeadLetterReason) => void;
@@ -146,8 +145,11 @@ const errorMessage = (err: unknown): string =>
export interface EventsBatchHandler {
eachBatch: (payload: EachBatchPayload) => Promise<void>;
- /** Called on GROUP_JOIN: a new assignment invalidates the watermarks. */
- resetWatermarks: () => void;
+ /**
+ * Called on GROUP_JOIN: a new assignment may hand this member different
+ * partitions, so the per-partition durability backoff below starts over.
+ */
+ resetDurabilityBackoff: () => void;
}
/**
@@ -167,14 +169,6 @@ export function createEventsBatchHandler(
): EventsBatchHandler {
const sleep = deps.sleep ?? defaultSleep;
- // Highest Kafka offset we have *resolved* (committed) per `topic-partition`,
- // tracked across batches. Used purely for the reprocess detector below: if we
- // ever see an offset at or below this watermark again, the message is being
- // redelivered (at-least-once duplicate) outside of a rebalance — which is the
- // signature of an offset-handling bug. Cleared on GROUP_JOIN so legitimate
- // post-rebalance redelivery from the last committed offset doesn't trip it.
- const resolvedHWM = new Map<string, number>();
-
/**
* Park the message and DROP it. The drop is unconditional: whether the
* record was stored or not, this message's offset is resolved by the caller
@@ -314,8 +308,8 @@ export function createEventsBatchHandler(
const durabilityFailures = new Map<string, number>();
/**
- * @returns false when the flush failed, which leaves the watermark where it
- * was so the redelivered batch is not then reported as a duplicate.
+ * @returns false when the flush failed, which resolves no offset and leaves
+ * the whole batch for the broker to redeliver.
*/
const bufferedEventsAreDurable = async (
partition: number,
@@ -374,10 +368,6 @@ export function createEventsBatchHandler(
// for THIS batch's events and not for whatever else was pending when it
// finished (M18-007).
const closeDurabilityWindow = deps.openDurabilityWindow();
- // Watermark from *previous* batches. Anything at or below this that we see
- // now is a redelivery. Captured before this batch so intra-batch
- // out-of-order processing (normal, see below) is never counted.
- const priorHWM = resolvedHWM.get(pk) ?? -1;
// Group by partition key (= deviceId or `${projectId}:${profileId}`).
// Same-key messages stay serial so sessionBuffer/session-end-job state
@@ -414,21 +404,6 @@ export function createEventsBatchHandler(
return;
}
- // Reprocess detector: only fires for offsets already resolved in a
- // PRIOR batch (redelivery). Intra-batch out-of-order processing
- // across key-groups is expected and is not flagged.
- if (Number(m.offset) <= priorHWM) {
- deps.metrics.reprocessed(String(batch.partition));
- deps.logger.warn(
- {
- partition: batch.partition,
- offset: m.offset,
- resolvedHighWaterMark: priorHWM,
- },
- 'kafka offset REPROCESSED — at-least-once duplicate (outside rebalance)'
- );
- }
-
// No "unresolvable" branch: a message is handled or dropped, and a
// dropped one is still finished (M20-001). The only gap this walk
// can now stop at is the isRunning/isStale early return above, and
@@ -457,15 +432,12 @@ export function createEventsBatchHandler(
// Resolve in strict ascending offset order, stopping at the first offset
// that did not finish (e.g. an isStale/isRunning early-return mid-batch).
// batch.messages is already ordered by offset.
- let newHWM = priorHWM;
for (const m of batch.messages) {
if (!processed.has(m.offset)) {
break;
}
resolveOffset(m.offset);
- newHWM = Math.max(newHWM, Number(m.offset));
}
- resolvedHWM.set(pk, newHWM);
} else if (isRunning() && !isStale()) {
// Not while shutting down: the redelivery is the next process's problem
// and the wait would only eat the shutdown budget.
@@ -478,8 +450,7 @@ export function createEventsBatchHandler(
return {
eachBatch,
- resetWatermarks: () => {
- resolvedHWM.clear();
+ resetDurabilityBackoff: () => {
durabilityFailures.clear();
},
};
@@ -519,10 +490,10 @@ export async function startKafkaEventsConsumer(
// We had no logging for partition reassignment before; without it a rebalance
// storm (a common source of at-least-once duplicates) is invisible.
consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {
- // A new assignment means partitions may have moved between members; reset
- // the reprocess watermarks so legitimate resume-from-committed-offset after
- // a rebalance is not flagged as a duplicate.
- handler.resetWatermarks();
+ // A new assignment means partitions may have moved between members, so a
+ // consecutive-failure count carried over from the old one would pick the
+ // wrong backoff for the new one.
+ handler.resetDurabilityBackoff();
logger.info(
{
memberId: payload.memberId,
diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts
index 3a99cf1c9..3db311fcf 100644
--- a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts
+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts
@@ -11,6 +11,7 @@
// mapping are still pinned here, not re-implemented by the test.
import { beforeEach, describe, expect, mock, test } from 'bun:test';
+import { createDuplicateEventMarker } from '@openpanel/redis';
import { formatClickhouseDate } from '../../event/src/dates';
import type { IClickhouseSession } from '../../session/session.service';
import {
@@ -157,6 +158,10 @@ function makeDeps() {
const enqueueSessionEnd = mock<IncomingEventDeps['enqueueSessionEnd']>(
async () => undefined
);
+ const markDuplicate = mock<IncomingEventDeps['markDuplicate']>(
+ async () => false
+ );
+ const duplicateMarked = mock(() => undefined);
const deps: IncomingEventDeps = {
logger: noopLogger,
sessions: { getExistingSession, ingest },
@@ -167,12 +172,22 @@ function makeDeps() {
markFirstEvent: mock(async () => undefined),
},
enqueueSessionEnd,
+ markDuplicate,
metrics: {
sessionStarted: mock(() => undefined),
sessionEndEnqueued: mock(() => undefined),
+ duplicateMarked,
},
};
- return { deps, createEvent, getExistingSession, ingest, enqueueSessionEnd };
+ return {
+ deps,
+ createEvent,
+ getExistingSession,
+ ingest,
+ enqueueSessionEnd,
+ markDuplicate,
+ duplicateMarked,
+ };
}
/** Every `createEvent` call's first argument, in order. */
@@ -454,3 +469,167 @@ describe('incomingEvent', () => {
expect(enqueueSessionEnd).not.toHaveBeenCalled();
});
});
+
+/**
+ * M21-001. The marker replaces the offset watermark, which drill 08 showed
+ * could not see any of the 161 duplicates it measured. MARK MEANS COUNT AND
+ * LOG: every test below asserts the event was still INSERTED.
+ */
+describe('duplicate marker', () => {
+ const TTL_MS = 120_000;
+ const OTHER_EVENT_ID = '99999999-8888-4777-8666-555555555555';
+
+ /**
+ * Enough of Redis for `SET key NX PX`, with a hand-driven clock so the
+ * expiry is asserted rather than slept through. The REAL marker runs against
+ * it, so the key shape and the `NX` semantics are under test too.
+ */
+ function fakeRedis() {
+ const keys = new Map<string, number>();
+ let nowMs = 0;
+ return {
+ advance: (ms: number) => {
+ nowMs += ms;
+ },
+ keyCount: () => {
+ for (const [key, expiresAt] of keys) {
+ if (expiresAt <= nowMs) {
+ keys.delete(key);
+ }
+ }
+ return keys.size;
+ },
+ set: (key: string, _value: string, _px: 'PX', ttlMs: number) => {
+ const expiresAt = keys.get(key);
+ if (expiresAt !== undefined && expiresAt > nowMs) {
+ return Promise.resolve(null);
+ }
+ keys.set(key, nowMs + ttlMs);
+ return Promise.resolve('OK');
+ },
+ };
+ }
+
+ function depsWithRealMarker() {
+ const redis = fakeRedis();
+ const built = makeDeps();
+ built.deps.markDuplicate = createDuplicateEventMarker({
+ client: redis,
+ ttlMs: TTL_MS,
+ });
+ return { ...built, redis };
+ }
+
+ test('marks the second delivery of one id, and inserts BOTH', async () => {
+ const { deps, createEvent, ingest, duplicateMarked } = depsWithRealMarker();
+ ingest.mockResolvedValue({ kind: 'extend', current: makeSession() });
+ const envelope = buildJobData({ id: producedEventId });
+
+ await incomingEvent(envelope, deps);
+ expect(duplicateMarked).not.toHaveBeenCalled();
+
+ await incomingEvent(envelope, deps);
+ expect(duplicateMarked).toHaveBeenCalledTimes(1);
+
+ // The whole point: the redelivery is COUNTED, not suppressed. Two
+ // inserts, both carrying the producer's id.
+ const inserted = createdEvents(createEvent);
+ expect(inserted).toHaveLength(2);
+ expect(inserted.every((event) => event.id === producedEventId)).toBe(true);
+ });
+
+ test('never marks distinct ids', async () => {
+ const { deps, createEvent, ingest, duplicateMarked } = depsWithRealMarker();
+ ingest.mockResolvedValue({ kind: 'extend', current: makeSession() });
+
+ await incomingEvent(buildJobData({ id: producedEventId }), deps);
+ await incomingEvent(buildJobData({ id: OTHER_EVENT_ID }), deps);
+
+ expect(duplicateMarked).not.toHaveBeenCalled();
+ expect(createdEvents(createEvent)).toHaveLength(2);
+ });
+
+ test('the key expires, so a replay past the TTL is unmarked', async () => {
+ const { deps, ingest, duplicateMarked, redis } = depsWithRealMarker();
+ ingest.mockResolvedValue({ kind: 'extend', current: makeSession() });
+ const envelope = buildJobData({ id: producedEventId });
+
+ await incomingEvent(envelope, deps);
+ expect(redis.keyCount()).toBe(1);
+
+ redis.advance(TTL_MS + 1);
+ expect(redis.keyCount()).toBe(0);
+
+ await incomingEvent(envelope, deps);
+ expect(duplicateMarked).not.toHaveBeenCalled();
+ });
+
+ test('a payload with no producer-minted id costs no Redis call', async () => {
+ const { deps, createEvent, ingest, markDuplicate } = ctx;
+ ingest.mockResolvedValueOnce({ kind: 'extend', current: makeSession() });
+
+ await incomingEvent(buildJobData(), deps);
+
+ expect(markDuplicate).not.toHaveBeenCalled();
+ expect(createdEvents(createEvent)).toHaveLength(1);
+ });
+
+ /**
+ * THE REGRESSION TEST THAT MATTERS. A blocking Redis call in this path is
+ * how drill 02's consumer was evicted past its 30 s session timeout, and
+ * M19 proved an eviction is the only reassignment that costs duplicate
+ * rows. M18-003 removed that trigger; this must not put it back.
+ */
+ test('Redis unavailable: the event is processed normally and nothing throws', async () => {
+ const { deps, createEvent, ingest, markDuplicate, duplicateMarked } = ctx;
+ ingest.mockResolvedValueOnce({ kind: 'extend', current: makeSession() });
+ markDuplicate.mockImplementation(() =>
+ Promise.reject(
+ new Error(
+ "Stream isn't writeable and enableOfflineQueue options is false"
+ )
+ )
+ );
+
+ const event = await incomingEvent(
+ buildJobData({ id: producedEventId }),
+ deps
+ );
+
+ expect(event).not.toBeNull();
+ expect(createdEvents(createEvent)).toHaveLength(1);
+ expect(createdEvents(createEvent)[0]!.id).toBe(producedEventId);
+ // Fail open means fail SILENT to the counter: an unwritten marker is not
+ // evidence of a duplicate.
+ expect(duplicateMarked).not.toHaveBeenCalled();
+ });
+
+ test('the marker runs alongside the handler, never in front of it', async () => {
+ // "Not slowed" asserted structurally rather than with a stopwatch: the
+ // marker is started BEFORE the handler's own Redis work and awaited
+ // AFTER it, so a slow marker overlaps a slow session read instead of
+ // adding to it. A serial check would give mark:start, mark:end,
+ // ingest:start — which is the shape that would put M18-003's eviction
+ // trigger back.
+ const timeline: string[] = [];
+ const slow = () => new Promise((resolve) => setTimeout(resolve, 20));
+ const { deps, ingest, markDuplicate } = ctx;
+ markDuplicate.mockImplementation(async () => {
+ timeline.push('mark:start');
+ await slow();
+ timeline.push('mark:end');
+ return false;
+ });
+ ingest.mockImplementation(async () => {
+ timeline.push('ingest:start');
+ await slow();
+ timeline.push('ingest:end');
+ return { kind: 'extend', current: makeSession() };
+ });
+
+ await incomingEvent(buildJobData({ id: producedEventId }), deps);
+
+ expect(timeline.slice(0, 2)).toEqual(['mark:start', 'ingest:start']);
+ expect(timeline).toContain('mark:end');
+ });
+});
diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.ts
index 14987b513..e2c48e397 100644
--- a/packages/core/src/modules/ingest/src/incoming-event-handler.ts
+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.ts
@@ -33,7 +33,7 @@ import type { IClickhouseSession } from '../../session/session.service';
import { sessionEndsEnqueued } from '../../session/src/session.metrics';
import type { EnqueueSessionEndInput } from '../../session/src/session-end';
import type { IncomingEventPayload } from './incoming-event';
-import { sessionsStarted } from './ingest.metrics';
+import { duplicateEventsMarkedTotal, sessionsStarted } from './ingest.metrics';
const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];
@@ -74,6 +74,8 @@ export interface IncomingEventSessions {
export interface IncomingEventMetrics {
sessionStarted(kind: 'new' | 'boundary'): void;
sessionEndEnqueued(source: 'boundary'): void;
+ /** One event whose producer-minted id had already been seen (M21-001). */
+ duplicateMarked(): void;
}
export interface IncomingEventDeps {
@@ -93,6 +95,14 @@ export interface IncomingEventDeps {
markFirstEvent(projectId: string): Promise<unknown>;
};
enqueueSessionEnd(input: EnqueueSessionEndInput): Promise<unknown>;
+ /**
+ * Claim the producer-minted event id. Resolves true when the id has been
+ * seen before — a redelivery — and false on its first sighting.
+ *
+ * MARK, NOT DEDUPE: the caller counts and logs, and inserts the event
+ * either way. It may reject; the caller then processes the event normally.
+ */
+ markDuplicate(eventId: string): Promise<boolean>;
metrics: IncomingEventMetrics;
}
@@ -115,6 +125,14 @@ export interface IncomingEventBindings {
deps: Ctx,
payload: IServiceCreateEventPayload
): Promise<unknown>;
+ /**
+ * `@openpanel/redis`'s `createDuplicateEventMarker`, already bound to the
+ * TTL `apps/api` parsed. Injected rather than built here because the TTL is
+ * configuration and `packages/core` reads no environment (ADR-022 R7), and
+ * because building it once at the composition root keeps the key prefix and
+ * the TTL out of the per-message path.
+ */
+ markDuplicateEvent(eventId: string): Promise<boolean>;
}
/**
@@ -142,9 +160,11 @@ export function createIncomingEventDeps(
},
},
enqueueSessionEnd: (input) => ctx.services.session.enqueueSessionEnd(input),
+ markDuplicate: (eventId) => bindings.markDuplicateEvent(eventId),
metrics: {
sessionStarted: (kind) => sessionsStarted.inc({ kind }),
sessionEndEnqueued: (source) => sessionEndsEnqueued.inc({ source }),
+ duplicateMarked: () => duplicateEventsMarkedTotal.inc(),
},
};
}
@@ -227,12 +247,91 @@ const parseRevenue = (revenue: unknown): number | undefined => {
return undefined;
};
+/**
+ * Claim the event id and report a redelivery. THE EVENT IS INSERTED EITHER
+ * WAY — this counts and logs, it does not drop, and nothing downstream reads
+ * its answer. A false positive on a suppressing check would be silent data
+ * loss, and whether the events table ever gets dedupe is not decided here.
+ *
+ * It replaces the offset watermark, which could not see a duplicate across a
+ * rebalance, an eviction, a durability redelivery or a restart — drill 08
+ * measured 161 real duplicates and 0 of them flagged. It catches redeliveries
+ * of the SAME id: crash, eviction, durability redelivery. It does NOT catch
+ * drill 06's lost ACK, where the SDK re-sends and the producer mints a fresh
+ * id per request; SDK-supplied ids are what closes that, later.
+ *
+ * FAILS OPEN, AND NEVER REJECTS. Redis being away is exactly when redeliveries
+ * happen, so an observability counter must not be the thing that delays or
+ * loses an event — and a handler blocked on Redis is how drill 02's consumer
+ * was evicted past its session timeout, which is the one reassignment M19
+ * measured costing duplicate rows.
+ */
+function startDuplicateMark(
+ eventId: string | undefined,
+ projectId: string,
+ logger: Logger,
+ deps: IncomingEventDeps
+): Promise<void> {
+ if (!eventId) {
+ return Promise.resolve();
+ }
+ return deps
+ .markDuplicate(eventId)
+ .then((seenBefore) => {
+ if (!seenBefore) {
+ return;
+ }
+ deps.metrics.duplicateMarked();
+ logger.warn(
+ { eventId, projectId },
+ 'DUPLICATE event id — already ingested, inserting it anyway'
+ );
+ })
+ .catch((error) => {
+ logger.warn(
+ { err: error, eventId },
+ 'could not mark duplicate — this event is unchecked'
+ );
+ });
+}
+
export async function incomingEvent(
jobPayload: IncomingEventPayload,
deps: IncomingEventDeps,
// Logged so a duplicate row in ClickHouse can be traced back to the exact
// partition/offset that produced it.
meta?: IncomingEventDelivery
+) {
+ // `requestId` rides in on `deps.logger`, which the consumer scopes to this
+ // envelope's id (ADR-018 R1 renames V1's `reqId`); only the delivery
+ // coordinates are per-message news.
+ const logger = meta
+ ? deps.logger.child({
+ kafkaPartition: meta.partition,
+ kafkaOffset: meta.offset,
+ })
+ : deps.logger;
+ // Started BEFORE the handler's own Redis work and awaited AFTER it, so the
+ // marker's round trip overlaps the session buffer's GET and EVAL on the same
+ // client: healthy it costs the handler nothing measurable, and under a
+ // frozen Redis its bound is one the handler was already paying.
+ const duplicateMark = startDuplicateMark(
+ jobPayload.id,
+ jobPayload.projectId,
+ logger,
+ deps
+ );
+ try {
+ return await ingestIncomingEvent(jobPayload, deps, logger);
+ } finally {
+ await duplicateMark;
+ }
+}
+
+async function ingestIncomingEvent(
+ jobPayload: IncomingEventPayload,
+ deps: IncomingEventDeps,
+ logger: Logger
) {
const {
geo,
@@ -248,15 +347,6 @@ export async function incomingEvent(
id: eventId,
} = jobPayload;
const properties: Record<string, unknown> = body.properties ?? {};
- // `requestId` rides in on `deps.logger`, which the consumer scopes to this
- // envelope's id (ADR-018 R1 renames V1's `reqId`); only the delivery
- // coordinates are per-message news.
- const logger = meta
- ? deps.logger.child({
- kafkaPartition: meta.partition,
- kafkaOffset: meta.offset,
- })
- : deps.logger;
const getProperty = (name: string): string | undefined => {
// replace thing is just for older sdks when we didn't have `__`
// remove when kiddokitchen app (24.09.02) is not used anymore
diff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.ts
index b671f3339..827129fd7 100644
--- a/packages/core/src/modules/ingest/src/ingest.metrics.ts
+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts
@@ -9,13 +9,19 @@ import client from 'prom-client';
import { registry } from '../../../metrics';
import type { ConsumerMetrics, DeadLetterReason } from './consumer';
-// Kafka event messages reprocessed (same offset redelivered outside a
-// rebalance). Should stay ~0. A sustained non-zero rate means the consumer is
-// re-delivering messages it already handled — an offset-handling/duplicate bug.
-export const kafkaReprocessedTotal = new client.Counter({
- name: 'kafka_events_reprocessed_total',
- help: 'Kafka event messages reprocessed (offset redelivered outside a rebalance)',
- labelNames: ['partition'],
+// Events whose producer-minted id had already been seen (M21-001). This is a
+// MARKER, not a dedupe: the event is inserted either way and this counter is
+// the whole point of the check. It replaces `kafka_events_reprocessed_total`,
+// which was keyed on a Kafka offset in one process's heap and therefore blind
+// to every duplicate that mattered — drill 08 measured 161 real duplicates and
+// 0 reprocessed increments.
+//
+// It UNDERCOUNTS by design: a duplicate whose marker could not be written
+// (Redis away) is not counted, because the check fails open rather than
+// delaying the event.
+export const duplicateEventsMarkedTotal = new client.Counter({
+ name: 'ingest_duplicate_events_marked_total',
+ help: 'Events whose producer-minted id had already been seen (marked, still inserted)',
registers: [registry],
});
@@ -65,7 +71,6 @@ export const sessionsStarted = new client.Counter({
* — a metric is a module's business, not the entrypoint's.
*/
export const ingestConsumerMetrics = {
- reprocessed: (partition: string) => kafkaReprocessedTotal.inc({ partition }),
handlerFailed: (partition: string) =>
kafkaHandlerFailuresTotal.inc({ partition }),
deadLettered: (partition: string, reason: DeadLetterReason) =>
diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.ts
index 7dea1f7ff..012af2639 100644
--- a/packages/core/test/request-id-end-to-end.test.ts
+++ b/packages/core/test/request-id-end-to-end.test.ts
@@ -579,6 +579,9 @@ function ingestDeps() {
const ingestBindings: IncomingEventBindings = {
checkNotificationRulesForEvent: () => Promise.resolve(null),
+ // Never a duplicate here: this suite is about the requestId, and the marker
+ // must not decide anything about the row either way (M21-001).
+ markDuplicateEvent: () => Promise.resolve(false),
};
function ingestEnvelope(requestId?: string): IncomingEventPayload {
diff --git a/packages/redis/duplicate-marker.test.ts b/packages/redis/duplicate-marker.test.ts
new file mode 100644
index 000000000..e92f8776b
--- /dev/null
+++ b/packages/redis/duplicate-marker.test.ts
@@ -0,0 +1,139 @@
+/**
+ * M21-001 — the event-keyed duplicate marker, at the Redis boundary.
+ *
+ * What the ingest handler does with the answer is `packages/core`'s
+ * `incoming-event-handler.test.ts`; what this file pins is the command: one
+ * `SET key NX PX`, the key shape, and which reply means "seen before".
+ *
+ * No real Redis here: an in-memory stand-in with real `NX` and expiry
+ * semantics. The same four answers were driven through THIS marker against the
+ * local Redis (`redis://127.0.0.1:6379/14`) on 2026-09-14 before this file was
+ * written, at an 800 ms TTL — first sighting false, `PTTL` 800, second
+ * sighting true, a distinct id false, and the same id false again 900 ms
+ * later.
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+ createDuplicateEventMarker,
+ DUPLICATE_MARKER_KEY_PREFIX,
+ type DuplicateMarkerRedisClient,
+} from './duplicate-marker';
+
+const TTL_MS = 120_000;
+const EVENT_ID = '11111111-2222-4333-8444-555555555555';
+const OTHER_EVENT_ID = '99999999-8888-4777-8666-555555555555';
+
+function fakeRedis() {
+ const expiries = new Map<string, number>();
+ const commands: unknown[][] = [];
+ let nowMs = 0;
+
+ const client: DuplicateMarkerRedisClient = {
+ set(key, value, expiryMode, ttlMs, setMode) {
+ commands.push([key, value, expiryMode, ttlMs, setMode]);
+ const expiresAt = expiries.get(key);
+ if (expiresAt !== undefined && expiresAt > nowMs) {
+ return Promise.resolve(null);
+ }
+ expiries.set(key, nowMs + ttlMs);
+ return Promise.resolve('OK');
+ },
+ };
+
+ return {
+ client,
+ commands,
+ advance: (ms: number) => {
+ nowMs += ms;
+ },
+ };
+}
+
+describe('createDuplicateEventMarker', () => {
+ it('reports the first sighting of an id as not-a-duplicate', async () => {
+ const redis = fakeRedis();
+ const mark = createDuplicateEventMarker({
+ client: redis.client,
+ ttlMs: TTL_MS,
+ });
+
+ expect(await mark(EVENT_ID)).toBe(false);
+ });
+
+ it('reports the second sighting of the same id as a duplicate', async () => {
+ const redis = fakeRedis();
+ const mark = createDuplicateEventMarker({
+ client: redis.client,
+ ttlMs: TTL_MS,
+ });
+
+ await mark(EVENT_ID);
+
+ expect(await mark(EVENT_ID)).toBe(true);
+ });
+
+ it('keeps distinct ids independent', async () => {
+ const redis = fakeRedis();
+ const mark = createDuplicateEventMarker({
+ client: redis.client,
+ ttlMs: TTL_MS,
+ });
+
+ await mark(EVENT_ID);
+
+ expect(await mark(OTHER_EVENT_ID)).toBe(false);
+ });
+
+ it('forgets an id once its key has expired', async () => {
+ const redis = fakeRedis();
+ const mark = createDuplicateEventMarker({
+ client: redis.client,
+ ttlMs: TTL_MS,
+ });
+
+ await mark(EVENT_ID);
+ redis.advance(TTL_MS + 1);
+
+ expect(await mark(EVENT_ID)).toBe(false);
+ });
+
+ it('costs exactly one command per event, a prefixed SET NX PX', async () => {
+ const redis = fakeRedis();
+ const mark = createDuplicateEventMarker({
+ client: redis.client,
+ ttlMs: TTL_MS,
+ });
+
+ await mark(EVENT_ID);
+
+ expect(redis.commands).toEqual([
+ [`${DUPLICATE_MARKER_KEY_PREFIX}${EVENT_ID}`, '1', 'PX', TTL_MS, 'NX'],
+ ]);
+ });
+
+ it('takes a key prefix override without touching the default', async () => {
+ const redis = fakeRedis();
+ const mark = createDuplicateEventMarker({
+ client: redis.client,
+ ttlMs: TTL_MS,
+ keyPrefix: 'test:dup:',
+ });
+
+ await mark(EVENT_ID);
+
+ expect(redis.commands[0]![0]).toBe(`test:dup:${EVENT_ID}`);
+ expect(DUPLICATE_MARKER_KEY_PREFIX).toBe('ingest:duplicate:');
+ });
+
+ it('rejects when Redis does not answer, leaving the decision to the caller', async () => {
+ const mark = createDuplicateEventMarker({
+ client: {
+ set: () => Promise.reject(new Error('Stream is not writeable')),
+ },
+ ttlMs: TTL_MS,
+ });
+
+ await expect(mark(EVENT_ID)).rejects.toThrow('Stream is not writeable');
+ });
+});
diff --git a/packages/redis/duplicate-marker.ts b/packages/redis/duplicate-marker.ts
new file mode 100644
index 000000000..902d21aa9
--- /dev/null
+++ b/packages/redis/duplicate-marker.ts
@@ -0,0 +1,95 @@
+/**
+ * The ingest consumer's duplicate MARKER (M21-001).
+ *
+ * Carl, 2026-09-14, on the offset watermark this replaces: *"are they purely
+ * for logging reprocessed? Then we should remove them, i dont care. we can use
+ * the redis check we talked about earlier to mark if it was a duplicate
+ * event?"*
+ *
+ * It is keyed on the EVENT, not on a Kafka offset. The event `id` is
+ * producer-minted, so a redelivered message re-presents the same id — drill 08
+ * confirmed all 161 of its duplicates were byte-identical replays sharing one
+ * `id`, one `created_at` and one `session_id`. A key therefore survives
+ * restarts, rebalances, evictions and process boundaries, none of which the
+ * in-process watermark survived: drill 08 measured 161 real duplicates and 0
+ * `REPROCESSED` lines.
+ *
+ * MARK MEANS COUNT AND LOG. IT DOES NOT MEAN DROP. This tells the caller the
+ * id has been seen before; the caller inserts the event either way. A false
+ * positive on a suppressing check is silent data loss, and whether the events
+ * table ever gets dedupe is Carl's decision, not this file's.
+ *
+ * ONE round trip, `SET key NX PX`: the set and the "did it already exist"
+ * answer are the same command, and there is no clean seam to fold it into —
+ * the session buffer's own GET and Lua EVAL live in `packages/core`'s
+ * `buffers/`, which this change does not own.
+ *
+ * IT MUST BE GIVEN THE CACHE CLIENT. That one fails fast (M18-003): offline
+ * queue off once connected, `commandTimeout` 500 ms. `getRedisQueue()` keeps
+ * `maxRetriesPerRequest: null` for BullMQ and would block a handler for tens
+ * of seconds — which is how drill 02's consumer was evicted past its 30 s
+ * session timeout, and an eviction is the only reassignment M19 measured
+ * costing duplicate rows.
+ *
+ * Why here and not in `packages/core`'s ingest module: the whole of it is
+ * Redis mechanics, and core's only export door is its barrel, which this
+ * task's scope does not cover — the same reason `dead-letter.ts` lives here.
+ */
+
+/** Live key prefix. An operator has to be able to find these. */
+export const DUPLICATE_MARKER_KEY_PREFIX = 'ingest:duplicate:';
+
+/**
+ * The narrow slice of a Redis client this needs. Structural rather than
+ * ioredis's `Redis`, so a test can satisfy it without a server.
+ */
+export interface DuplicateMarkerRedisClient {
+ set(
+ key: string,
+ value: string,
+ expiryMode: 'PX',
+ ttlMs: number,
+ setMode: 'NX'
+ ): Promise<string | null>;
+}
+
+export interface DuplicateEventMarkerOptions {
+ client: DuplicateMarkerRedisClient;
+ /**
+ * How long a marker outlives its event. It only has to outlive the REPLAY
+ * window, not the data.
+ */
+ ttlMs: number;
+ /** Overridable for tests; production uses the constant above. */
+ keyPrefix?: string;
+}
+
+/** The marker's only stored value — presence is the whole signal. */
+const MARKER_VALUE = '1';
+
+/**
+ * @returns true when this event id has been seen before, false when this call
+ * is the first sighting.
+ *
+ * REJECTS when Redis does not answer. The caller decides what that means, and
+ * on the ingest path the answer is always "process the event normally" — an
+ * observability counter must never be the reason an event is delayed or lost.
+ */
+export function createDuplicateEventMarker(
+ options: DuplicateEventMarkerOptions
+): (eventId: string) => Promise<boolean> {
+ const keyPrefix = options.keyPrefix ?? DUPLICATE_MARKER_KEY_PREFIX;
+
+ return async (eventId: string): Promise<boolean> => {
+ const stored = await options.client.set(
+ `${keyPrefix}${eventId}`,
+ MARKER_VALUE,
+ 'PX',
+ options.ttlMs,
+ 'NX'
+ );
+ // `SET NX` answers 'OK' when it wrote the key and null when one was
+ // already there — so null IS the duplicate.
+ return stored === null;
+ };
+}
diff --git a/packages/redis/index.ts b/packages/redis/index.ts
index 59f2651b4..709bee287 100644
--- a/packages/redis/index.ts
+++ b/packages/redis/index.ts
@@ -1,5 +1,6 @@
-export * from './redis';
export * from './cachable';
export * from './dead-letter';
-export * from './run-every';
+export * from './duplicate-marker';
export * from './publisher';
+export * from './redis';
+export * from './run-every';