OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M21-001): Delete the offset watermark duplicate detector and mark duplicates with an event-keyed Redis key instead - mark only, never drop
ralphcae38290efe7
diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.tsindex 33cb4782b..cafbd7fb5 100644--- a/apps/api/src/config/env.ts+++ b/apps/api/src/config/env.ts@@ -83,6 +83,22 @@ const DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES = 1000;const DEFAULT_KAFKA_HANDLER_RETRY_INITIAL_MS = 100;const DEFAULT_KAFKA_HANDLER_RETRY_MAX_MS = 1000;+/**+ * How long a duplicate marker outlives its event (M21-001). It only has to+ * outlive the REPLAY window, not the data: drill 08's eviction rejoin — the+ * one reassignment that costs duplicate rows — was 3,094 ms, and drill 06's+ * lost-ACK replay landed ~30 s after the produce. Two minutes is ~40x the+ * first and 4x the second, and covers a process restart resuming from its last+ * committed offset.+ *+ * It is the knob that decides the key count, which is `events/s x TTL`: at the+ * 5,000 events/s the scaling requirement targets that is 600,000 live markers.+ * Measured on this box's Redis 2026-09-14 — 200,000 real markers cost+ * 137.6 bytes each including the expires dict — so ~79 MB. Longer is only+ * worth buying if a redelivery window longer than this is ever observed.+ */+const DEFAULT_INGEST_DUPLICATE_MARKER_TTL_MS = 120_000;+/*** The Kafka events consumer's token. Renamed from `events_kafka` — there is* one events transport now, so there is one token for it (ADR-004 rec 5/6,@@ -460,6 +476,11 @@ const rawSchema = z.object({DEFAULT_INGEST_DEAD_LETTER_MAX_ENTRIES),+ // --- ingest duplicate marker ---+ INGEST_DUPLICATE_MARKER_TTL_MS: positiveIntWithDefault(+ DEFAULT_INGEST_DUPLICATE_MARKER_TTL_MS+ ),+// --- buffers ---BUFFER_ASYNC_INSERTS: definedIsTrueSchema,BUFFER_CH_INSERT_CONCURRENCY: optionalPositiveInt,@@ -813,6 +834,13 @@ export interface Config {* `sendToDeadLetter` seam it is handed.*/INGEST_DEAD_LETTER_MAX_ENTRIES: number;+ /**+ * How long the ingest consumer's duplicate marker key lives. Same reason it+ * is not on `core.kafka`: the marker is keyed on the EVENT, not on a Kafka+ * coordinate, and `packages/core` only knows the `markDuplicateEvent` seam+ * it is handed.+ */+ INGEST_DUPLICATE_MARKER_TTL_MS: number;/** The CORS delegator's origin allowlist, in V1's own order. */dashboardOrigins: string[];/** V1's `ENABLE_VERBOSE_LOGGING`, already split. */@@ -853,6 +881,7 @@ const envSchema = rawSchemaCOOKIE_SECRET: raw.COOKIE_SECRET,SHUTDOWN_FORCE_EXIT_MS: raw.SHUTDOWN_FORCE_EXIT_MS,INGEST_DEAD_LETTER_MAX_ENTRIES: raw.INGEST_DEAD_LETTER_MAX_ENTRIES,+ INGEST_DUPLICATE_MARKER_TTL_MS: raw.INGEST_DUPLICATE_MARKER_TTL_MS,dashboardOrigins: [dashboardUrl,...splitTokens(raw.API_CORS_ORIGINS),diff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex ba8dd1fe6..9578f4f5b 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -85,6 +85,7 @@ import { ch } from '@openpanel/db/src/clickhouse/client';import { db, Prisma } from '@openpanel/db/src/prisma-client';import {createDeadLetterRecorder,+ createDuplicateEventMarker,getRedisCache,getRedisPub,getRedisQueue,@@ -272,6 +273,15 @@ async function startIngestConsumer(// slot in its signature.handleEvent: createIncomingEventHandler(deps, {checkNotificationRulesForEvent,+ // M21-001: MARKS a redelivery, never drops one — the event is+ // inserted either way. The CACHE client, again, and for the same+ // reason: a handler that blocks on Redis is what got drill 02's+ // consumer evicted past its session timeout, and an eviction is the+ // one reassignment M19 measured costing duplicate rows.+ markDuplicateEvent: createDuplicateEventMarker({+ client: deps.redis,+ ttlMs: config.INGEST_DUPLICATE_MARKER_TTL_MS,+ }),}),// M20-001: a capped Redis list, and the event is DROPPED whether or not// the record lands. The Kafka DLQ it replaceddiff --git a/packages/core/src/modules/ingest/src/consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.tsindex fc449500a..a80523238 100644--- a/packages/core/src/modules/ingest/src/consumer.test.ts+++ b/packages/core/src/modules/ingest/src/consumer.test.ts@@ -78,7 +78,6 @@ type DepOverrides = Partial<function makeDeps(overrides: DepOverrides = {}) {const metrics: EventsBatchHandlerDeps['metrics'] = {- reprocessed: mock(() => undefined),handlerFailed: mock(() => undefined),deadLettered: mock(() => undefined),deadLetterFailed: mock(() => undefined),@@ -372,59 +371,72 @@ describe('unparseable messages', () => {});});-describe('reprocess counter', () => {- test('does not fire for intra-batch out-of-order processing', async () => {- const { deps, metrics } = makeDeps();+/**+ * M21-001 deleted the offset watermark, whose only outputs were the+ * `reprocessed` counter and a warn line — drill 08 measured 161 real+ * duplicates and 0 of those lines, because `resetWatermarks()` fired on+ * GROUP_JOIN and a failed durability flush never advanced it.+ *+ * These are the regression tests for the half of that claim that matters: the+ * functional statement in the resolve loop was always `resolveOffset(m.offset)`+ * and never read the watermark, so a redelivery must resolve and commit+ * exactly what it did before.+ */+describe('a redelivery after the watermark was deleted', () => {+ test('resolves every offset again, in ascending order', async () => {+ const { deps } = makeDeps();const handler = createEventsBatchHandler(deps);- const { payload } = makeBatch([- message(0, 'a'),- message(1, 'b'),- message(2, 'a'),- ]);+ const messages = [message(0, 'a'), message(1, 'a')];- await handler.eachBatch(payload);+ const first = makeBatch(messages);+ await handler.eachBatch(first.payload);+ expect(resolvedOffsets(first.resolveOffset)).toEqual(['0', '1']);- expect(metrics.reprocessed).not.toHaveBeenCalled();+ const redelivered = makeBatch(messages);+ await handler.eachBatch(redelivered.payload);+ expect(resolvedOffsets(redelivered.resolveOffset)).toEqual(['0', '1']);});- test('fires once per redelivered offset in a later batch', async () => {- const { deps, metrics } = makeDeps();+ test('hands the redelivered message to the handler again', async () => {+ const { deps, handleEvent } = makeDeps();const handler = createEventsBatchHandler(deps);- const messages = [message(0, 'a'), message(1, 'a')];+ const messages = [message(0, 'a')];await handler.eachBatch(makeBatch(messages).payload);- expect(metrics.reprocessed).not.toHaveBeenCalled();-await handler.eachBatch(makeBatch(messages).payload);- expect(metrics.reprocessed).toHaveBeenCalledTimes(2);- expect(metrics.reprocessed).toHaveBeenCalledWith(String(PARTITION));++ // At-least-once, unchanged: the consumer does not suppress a redelivery,+ // and nothing in this file now decides whether one is a duplicate — the+ // marker on the event id does (incoming-event-handler.ts).+ expect(handleEvent).toHaveBeenCalledTimes(2);});- test('does not fire after a rebalance clears the watermarks', async () => {- const { deps, metrics } = makeDeps();+ test('still resolves a redelivered offset that was dead-lettered', async () => {+ const { deps, handleEvent } = makeDeps();+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));const handler = createEventsBatchHandler(deps);- const messages = [message(0, 'a'), message(1, 'a')];+ const messages = [message(0, 'a')];- await handler.eachBatch(makeBatch(messages).payload);- handler.resetWatermarks();- await handler.eachBatch(makeBatch(messages).payload);+ const first = makeBatch(messages);+ await handler.eachBatch(first.payload);+ expect(resolvedOffsets(first.resolveOffset)).toEqual(['0']);- expect(metrics.reprocessed).not.toHaveBeenCalled();+ const redelivered = makeBatch(messages);+ await handler.eachBatch(redelivered.payload);+ expect(resolvedOffsets(redelivered.resolveOffset)).toEqual(['0']);});- test('does not fire for a dead-lettered offset re-seen only within the batch', async () => {- // A dead-lettered offset is resolved, so it must move the watermark just- // like a handled one — otherwise the next batch would flag it as a- // reprocess.- const { deps, handleEvent, metrics } = makeDeps();- handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));+ test('resetDurabilityBackoff on GROUP_JOIN changes no offset behaviour', async () => {+ const { deps } = makeDeps();const handler = createEventsBatchHandler(deps);+ const messages = [message(0, 'a'), message(1, 'b')];- await handler.eachBatch(makeBatch([message(0, 'a')]).payload);- expect(metrics.reprocessed).not.toHaveBeenCalled();+ await handler.eachBatch(makeBatch(messages).payload);+ handler.resetDurabilityBackoff();- await handler.eachBatch(makeBatch([message(1, 'a')]).payload);- expect(metrics.reprocessed).not.toHaveBeenCalled();+ const afterRebalance = makeBatch(messages);+ await handler.eachBatch(afterRebalance.payload);+ expect(resolvedOffsets(afterRebalance.resolveOffset)).toEqual(['0', '1']);});});@@ -504,9 +516,9 @@ describe('durability before commit', () => {expect(onActivity).toHaveBeenCalledTimes(1);});- test('redelivery after a failed flush commits, and is not flagged a duplicate', async () => {+ test('redelivery after a failed flush commits', async () => {let redisIsDown = true;- const { deps, metrics } = makeDeps({+ const { deps } = makeDeps({flushBufferedEvents: mock(async () => {if (redisIsDown) {throw new Error('redis is unreachable');@@ -524,9 +536,6 @@ describe('durability before commit', () => {await handler.eachBatch(redelivered.payload);expect(resolvedOffsets(redelivered.resolveOffset)).toEqual(['0', '1']);- // The failed batch never advanced the watermark, so the replay it asked- // for is not then reported as an at-least-once duplicate.- expect(metrics.reprocessed).not.toHaveBeenCalled();});test('opens the durability window before the first handler buffers anything', async () => {diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.tsindex 47146f83b..f6ddd05f8 100644--- a/packages/core/src/modules/ingest/src/consumer.ts+++ b/packages/core/src/modules/ingest/src/consumer.ts@@ -88,7 +88,6 @@ export interface ConsumerLogger {}export interface ConsumerMetrics {- reprocessed: (partition: string) => void;handlerFailed: (partition: string) => void;/** The message was recorded in the dead-letter list and DROPPED. */deadLettered: (partition: string, reason: DeadLetterReason) => void;@@ -146,8 +145,11 @@ const errorMessage = (err: unknown): string =>export interface EventsBatchHandler {eachBatch: (payload: EachBatchPayload) => Promise<void>;- /** Called on GROUP_JOIN: a new assignment invalidates the watermarks. */- resetWatermarks: () => void;+ /**+ * Called on GROUP_JOIN: a new assignment may hand this member different+ * partitions, so the per-partition durability backoff below starts over.+ */+ resetDurabilityBackoff: () => void;}/**@@ -167,14 +169,6 @@ export function createEventsBatchHandler(): EventsBatchHandler {const sleep = deps.sleep ?? defaultSleep;- // Highest Kafka offset we have *resolved* (committed) per `topic-partition`,- // tracked across batches. Used purely for the reprocess detector below: if we- // ever see an offset at or below this watermark again, the message is being- // redelivered (at-least-once duplicate) outside of a rebalance — which is the- // signature of an offset-handling bug. Cleared on GROUP_JOIN so legitimate- // post-rebalance redelivery from the last committed offset doesn't trip it.- const resolvedHWM = new Map<string, number>();-/*** Park the message and DROP it. The drop is unconditional: whether the* record was stored or not, this message's offset is resolved by the caller@@ -314,8 +308,8 @@ export function createEventsBatchHandler(const durabilityFailures = new Map<string, number>();/**- * @returns false when the flush failed, which leaves the watermark where it- * was so the redelivered batch is not then reported as a duplicate.+ * @returns false when the flush failed, which resolves no offset and leaves+ * the whole batch for the broker to redeliver.*/const bufferedEventsAreDurable = async (partition: number,@@ -374,10 +368,6 @@ export function createEventsBatchHandler(// for THIS batch's events and not for whatever else was pending when it// finished (M18-007).const closeDurabilityWindow = deps.openDurabilityWindow();- // Watermark from *previous* batches. Anything at or below this that we see- // now is a redelivery. Captured before this batch so intra-batch- // out-of-order processing (normal, see below) is never counted.- const priorHWM = resolvedHWM.get(pk) ?? -1;// Group by partition key (= deviceId or `${projectId}:${profileId}`).// Same-key messages stay serial so sessionBuffer/session-end-job state@@ -414,21 +404,6 @@ export function createEventsBatchHandler(return;}- // Reprocess detector: only fires for offsets already resolved in a- // PRIOR batch (redelivery). Intra-batch out-of-order processing- // across key-groups is expected and is not flagged.- if (Number(m.offset) <= priorHWM) {- deps.metrics.reprocessed(String(batch.partition));- deps.logger.warn(- {- partition: batch.partition,- offset: m.offset,- resolvedHighWaterMark: priorHWM,- },- 'kafka offset REPROCESSED — at-least-once duplicate (outside rebalance)'- );- }-// No "unresolvable" branch: a message is handled or dropped, and a// dropped one is still finished (M20-001). The only gap this walk// can now stop at is the isRunning/isStale early return above, and@@ -457,15 +432,12 @@ export function createEventsBatchHandler(// Resolve in strict ascending offset order, stopping at the first offset// that did not finish (e.g. an isStale/isRunning early-return mid-batch).// batch.messages is already ordered by offset.- let newHWM = priorHWM;for (const m of batch.messages) {if (!processed.has(m.offset)) {break;}resolveOffset(m.offset);- newHWM = Math.max(newHWM, Number(m.offset));}- resolvedHWM.set(pk, newHWM);} else if (isRunning() && !isStale()) {// Not while shutting down: the redelivery is the next process's problem// and the wait would only eat the shutdown budget.@@ -478,8 +450,7 @@ export function createEventsBatchHandler(return {eachBatch,- resetWatermarks: () => {- resolvedHWM.clear();+ resetDurabilityBackoff: () => {durabilityFailures.clear();},};@@ -519,10 +490,10 @@ export async function startKafkaEventsConsumer(// We had no logging for partition reassignment before; without it a rebalance// storm (a common source of at-least-once duplicates) is invisible.consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {- // A new assignment means partitions may have moved between members; reset- // the reprocess watermarks so legitimate resume-from-committed-offset after- // a rebalance is not flagged as a duplicate.- handler.resetWatermarks();+ // A new assignment means partitions may have moved between members, so a+ // consecutive-failure count carried over from the old one would pick the+ // wrong backoff for the new one.+ handler.resetDurabilityBackoff();logger.info({memberId: payload.memberId,diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.test.tsindex 3a99cf1c9..3db311fcf 100644--- a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts@@ -11,6 +11,7 @@// mapping are still pinned here, not re-implemented by the test.import { beforeEach, describe, expect, mock, test } from 'bun:test';+import { createDuplicateEventMarker } from '@openpanel/redis';import { formatClickhouseDate } from '../../event/src/dates';import type { IClickhouseSession } from '../../session/session.service';import {@@ -157,6 +158,10 @@ function makeDeps() {const enqueueSessionEnd = mock<IncomingEventDeps['enqueueSessionEnd']>(async () => undefined);+ const markDuplicate = mock<IncomingEventDeps['markDuplicate']>(+ async () => false+ );+ const duplicateMarked = mock(() => undefined);const deps: IncomingEventDeps = {logger: noopLogger,sessions: { getExistingSession, ingest },@@ -167,12 +172,22 @@ function makeDeps() {markFirstEvent: mock(async () => undefined),},enqueueSessionEnd,+ markDuplicate,metrics: {sessionStarted: mock(() => undefined),sessionEndEnqueued: mock(() => undefined),+ duplicateMarked,},};- return { deps, createEvent, getExistingSession, ingest, enqueueSessionEnd };+ return {+ deps,+ createEvent,+ getExistingSession,+ ingest,+ enqueueSessionEnd,+ markDuplicate,+ duplicateMarked,+ };}/** Every `createEvent` call's first argument, in order. */@@ -454,3 +469,167 @@ describe('incomingEvent', () => {expect(enqueueSessionEnd).not.toHaveBeenCalled();});});++/**+ * M21-001. The marker replaces the offset watermark, which drill 08 showed+ * could not see any of the 161 duplicates it measured. MARK MEANS COUNT AND+ * LOG: every test below asserts the event was still INSERTED.+ */+describe('duplicate marker', () => {+ const TTL_MS = 120_000;+ const OTHER_EVENT_ID = '99999999-8888-4777-8666-555555555555';++ /**+ * Enough of Redis for `SET key NX PX`, with a hand-driven clock so the+ * expiry is asserted rather than slept through. The REAL marker runs against+ * it, so the key shape and the `NX` semantics are under test too.+ */+ function fakeRedis() {+ const keys = new Map<string, number>();+ let nowMs = 0;+ return {+ advance: (ms: number) => {+ nowMs += ms;+ },+ keyCount: () => {+ for (const [key, expiresAt] of keys) {+ if (expiresAt <= nowMs) {+ keys.delete(key);+ }+ }+ return keys.size;+ },+ set: (key: string, _value: string, _px: 'PX', ttlMs: number) => {+ const expiresAt = keys.get(key);+ if (expiresAt !== undefined && expiresAt > nowMs) {+ return Promise.resolve(null);+ }+ keys.set(key, nowMs + ttlMs);+ return Promise.resolve('OK');+ },+ };+ }++ function depsWithRealMarker() {+ const redis = fakeRedis();+ const built = makeDeps();+ built.deps.markDuplicate = createDuplicateEventMarker({+ client: redis,+ ttlMs: TTL_MS,+ });+ return { ...built, redis };+ }++ test('marks the second delivery of one id, and inserts BOTH', async () => {+ const { deps, createEvent, ingest, duplicateMarked } = depsWithRealMarker();+ ingest.mockResolvedValue({ kind: 'extend', current: makeSession() });+ const envelope = buildJobData({ id: producedEventId });++ await incomingEvent(envelope, deps);+ expect(duplicateMarked).not.toHaveBeenCalled();++ await incomingEvent(envelope, deps);+ expect(duplicateMarked).toHaveBeenCalledTimes(1);++ // The whole point: the redelivery is COUNTED, not suppressed. Two+ // inserts, both carrying the producer's id.+ const inserted = createdEvents(createEvent);+ expect(inserted).toHaveLength(2);+ expect(inserted.every((event) => event.id === producedEventId)).toBe(true);+ });++ test('never marks distinct ids', async () => {+ const { deps, createEvent, ingest, duplicateMarked } = depsWithRealMarker();+ ingest.mockResolvedValue({ kind: 'extend', current: makeSession() });++ await incomingEvent(buildJobData({ id: producedEventId }), deps);+ await incomingEvent(buildJobData({ id: OTHER_EVENT_ID }), deps);++ expect(duplicateMarked).not.toHaveBeenCalled();+ expect(createdEvents(createEvent)).toHaveLength(2);+ });++ test('the key expires, so a replay past the TTL is unmarked', async () => {+ const { deps, ingest, duplicateMarked, redis } = depsWithRealMarker();+ ingest.mockResolvedValue({ kind: 'extend', current: makeSession() });+ const envelope = buildJobData({ id: producedEventId });++ await incomingEvent(envelope, deps);+ expect(redis.keyCount()).toBe(1);++ redis.advance(TTL_MS + 1);+ expect(redis.keyCount()).toBe(0);++ await incomingEvent(envelope, deps);+ expect(duplicateMarked).not.toHaveBeenCalled();+ });++ test('a payload with no producer-minted id costs no Redis call', async () => {+ const { deps, createEvent, ingest, markDuplicate } = ctx;+ ingest.mockResolvedValueOnce({ kind: 'extend', current: makeSession() });++ await incomingEvent(buildJobData(), deps);++ expect(markDuplicate).not.toHaveBeenCalled();+ expect(createdEvents(createEvent)).toHaveLength(1);+ });++ /**+ * THE REGRESSION TEST THAT MATTERS. A blocking Redis call in this path is+ * how drill 02's consumer was evicted past its 30 s session timeout, and+ * M19 proved an eviction is the only reassignment that costs duplicate+ * rows. M18-003 removed that trigger; this must not put it back.+ */+ test('Redis unavailable: the event is processed normally and nothing throws', async () => {+ const { deps, createEvent, ingest, markDuplicate, duplicateMarked } = ctx;+ ingest.mockResolvedValueOnce({ kind: 'extend', current: makeSession() });+ markDuplicate.mockImplementation(() =>+ Promise.reject(+ new Error(+ "Stream isn't writeable and enableOfflineQueue options is false"+ )+ )+ );++ const event = await incomingEvent(+ buildJobData({ id: producedEventId }),+ deps+ );++ expect(event).not.toBeNull();+ expect(createdEvents(createEvent)).toHaveLength(1);+ expect(createdEvents(createEvent)[0]!.id).toBe(producedEventId);+ // Fail open means fail SILENT to the counter: an unwritten marker is not+ // evidence of a duplicate.+ expect(duplicateMarked).not.toHaveBeenCalled();+ });++ test('the marker runs alongside the handler, never in front of it', async () => {+ // "Not slowed" asserted structurally rather than with a stopwatch: the+ // marker is started BEFORE the handler's own Redis work and awaited+ // AFTER it, so a slow marker overlaps a slow session read instead of+ // adding to it. A serial check would give mark:start, mark:end,+ // ingest:start — which is the shape that would put M18-003's eviction+ // trigger back.+ const timeline: string[] = [];+ const slow = () => new Promise((resolve) => setTimeout(resolve, 20));+ const { deps, ingest, markDuplicate } = ctx;+ markDuplicate.mockImplementation(async () => {+ timeline.push('mark:start');+ await slow();+ timeline.push('mark:end');+ return false;+ });+ ingest.mockImplementation(async () => {+ timeline.push('ingest:start');+ await slow();+ timeline.push('ingest:end');+ return { kind: 'extend', current: makeSession() };+ });++ await incomingEvent(buildJobData({ id: producedEventId }), deps);++ expect(timeline.slice(0, 2)).toEqual(['mark:start', 'ingest:start']);+ expect(timeline).toContain('mark:end');+ });+});diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.tsindex 14987b513..e2c48e397 100644--- a/packages/core/src/modules/ingest/src/incoming-event-handler.ts+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.ts@@ -33,7 +33,7 @@ import type { IClickhouseSession } from '../../session/session.service';import { sessionEndsEnqueued } from '../../session/src/session.metrics';import type { EnqueueSessionEndInput } from '../../session/src/session-end';import type { IncomingEventPayload } from './incoming-event';-import { sessionsStarted } from './ingest.metrics';+import { duplicateEventsMarkedTotal, sessionsStarted } from './ingest.metrics';const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];@@ -74,6 +74,8 @@ export interface IncomingEventSessions {export interface IncomingEventMetrics {sessionStarted(kind: 'new' | 'boundary'): void;sessionEndEnqueued(source: 'boundary'): void;+ /** One event whose producer-minted id had already been seen (M21-001). */+ duplicateMarked(): void;}export interface IncomingEventDeps {@@ -93,6 +95,14 @@ export interface IncomingEventDeps {markFirstEvent(projectId: string): Promise<unknown>;};enqueueSessionEnd(input: EnqueueSessionEndInput): Promise<unknown>;+ /**+ * Claim the producer-minted event id. Resolves true when the id has been+ * seen before — a redelivery — and false on its first sighting.+ *+ * MARK, NOT DEDUPE: the caller counts and logs, and inserts the event+ * either way. It may reject; the caller then processes the event normally.+ */+ markDuplicate(eventId: string): Promise<boolean>;metrics: IncomingEventMetrics;}@@ -115,6 +125,14 @@ export interface IncomingEventBindings {deps: Ctx,payload: IServiceCreateEventPayload): Promise<unknown>;+ /**+ * `@openpanel/redis`'s `createDuplicateEventMarker`, already bound to the+ * TTL `apps/api` parsed. Injected rather than built here because the TTL is+ * configuration and `packages/core` reads no environment (ADR-022 R7), and+ * because building it once at the composition root keeps the key prefix and+ * the TTL out of the per-message path.+ */+ markDuplicateEvent(eventId: string): Promise<boolean>;}/**@@ -142,9 +160,11 @@ export function createIncomingEventDeps(},},enqueueSessionEnd: (input) => ctx.services.session.enqueueSessionEnd(input),+ markDuplicate: (eventId) => bindings.markDuplicateEvent(eventId),metrics: {sessionStarted: (kind) => sessionsStarted.inc({ kind }),sessionEndEnqueued: (source) => sessionEndsEnqueued.inc({ source }),+ duplicateMarked: () => duplicateEventsMarkedTotal.inc(),},};}@@ -227,12 +247,91 @@ const parseRevenue = (revenue: unknown): number | undefined => {return undefined;};+/**+ * Claim the event id and report a redelivery. THE EVENT IS INSERTED EITHER+ * WAY — this counts and logs, it does not drop, and nothing downstream reads+ * its answer. A false positive on a suppressing check would be silent data+ * loss, and whether the events table ever gets dedupe is not decided here.+ *+ * It replaces the offset watermark, which could not see a duplicate across a+ * rebalance, an eviction, a durability redelivery or a restart — drill 08+ * measured 161 real duplicates and 0 of them flagged. It catches redeliveries+ * of the SAME id: crash, eviction, durability redelivery. It does NOT catch+ * drill 06's lost ACK, where the SDK re-sends and the producer mints a fresh+ * id per request; SDK-supplied ids are what closes that, later.+ *+ * FAILS OPEN, AND NEVER REJECTS. Redis being away is exactly when redeliveries+ * happen, so an observability counter must not be the thing that delays or+ * loses an event — and a handler blocked on Redis is how drill 02's consumer+ * was evicted past its session timeout, which is the one reassignment M19+ * measured costing duplicate rows.+ */+function startDuplicateMark(+ eventId: string | undefined,+ projectId: string,+ logger: Logger,+ deps: IncomingEventDeps+): Promise<void> {+ if (!eventId) {+ return Promise.resolve();+ }+ return deps+ .markDuplicate(eventId)+ .then((seenBefore) => {+ if (!seenBefore) {+ return;+ }+ deps.metrics.duplicateMarked();+ logger.warn(+ { eventId, projectId },+ 'DUPLICATE event id — already ingested, inserting it anyway'+ );+ })+ .catch((error) => {+ logger.warn(+ { err: error, eventId },+ 'could not mark duplicate — this event is unchecked'+ );+ });+}+export async function incomingEvent(jobPayload: IncomingEventPayload,deps: IncomingEventDeps,// Logged so a duplicate row in ClickHouse can be traced back to the exact// partition/offset that produced it.meta?: IncomingEventDelivery+) {+ // `requestId` rides in on `deps.logger`, which the consumer scopes to this+ // envelope's id (ADR-018 R1 renames V1's `reqId`); only the delivery+ // coordinates are per-message news.+ const logger = meta+ ? deps.logger.child({+ kafkaPartition: meta.partition,+ kafkaOffset: meta.offset,+ })+ : deps.logger;+ // Started BEFORE the handler's own Redis work and awaited AFTER it, so the+ // marker's round trip overlaps the session buffer's GET and EVAL on the same+ // client: healthy it costs the handler nothing measurable, and under a+ // frozen Redis its bound is one the handler was already paying.+ const duplicateMark = startDuplicateMark(+ jobPayload.id,+ jobPayload.projectId,+ logger,+ deps+ );+ try {+ return await ingestIncomingEvent(jobPayload, deps, logger);+ } finally {+ await duplicateMark;+ }+}++async function ingestIncomingEvent(+ jobPayload: IncomingEventPayload,+ deps: IncomingEventDeps,+ logger: Logger) {const {geo,@@ -248,15 +347,6 @@ export async function incomingEvent(id: eventId,} = jobPayload;const properties: Record<string, unknown> = body.properties ?? {};- // `requestId` rides in on `deps.logger`, which the consumer scopes to this- // envelope's id (ADR-018 R1 renames V1's `reqId`); only the delivery- // coordinates are per-message news.- const logger = meta- ? deps.logger.child({- kafkaPartition: meta.partition,- kafkaOffset: meta.offset,- })- : deps.logger;const getProperty = (name: string): string | undefined => {// replace thing is just for older sdks when we didn't have `__`// remove when kiddokitchen app (24.09.02) is not used anymorediff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.tsindex b671f3339..827129fd7 100644--- a/packages/core/src/modules/ingest/src/ingest.metrics.ts+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts@@ -9,13 +9,19 @@ import client from 'prom-client';import { registry } from '../../../metrics';import type { ConsumerMetrics, DeadLetterReason } from './consumer';-// Kafka event messages reprocessed (same offset redelivered outside a-// rebalance). Should stay ~0. A sustained non-zero rate means the consumer is-// re-delivering messages it already handled — an offset-handling/duplicate bug.-export const kafkaReprocessedTotal = new client.Counter({- name: 'kafka_events_reprocessed_total',- help: 'Kafka event messages reprocessed (offset redelivered outside a rebalance)',- labelNames: ['partition'],+// Events whose producer-minted id had already been seen (M21-001). This is a+// MARKER, not a dedupe: the event is inserted either way and this counter is+// the whole point of the check. It replaces `kafka_events_reprocessed_total`,+// which was keyed on a Kafka offset in one process's heap and therefore blind+// to every duplicate that mattered — drill 08 measured 161 real duplicates and+// 0 reprocessed increments.+//+// It UNDERCOUNTS by design: a duplicate whose marker could not be written+// (Redis away) is not counted, because the check fails open rather than+// delaying the event.+export const duplicateEventsMarkedTotal = new client.Counter({+ name: 'ingest_duplicate_events_marked_total',+ help: 'Events whose producer-minted id had already been seen (marked, still inserted)',registers: [registry],});@@ -65,7 +71,6 @@ export const sessionsStarted = new client.Counter({* — a metric is a module's business, not the entrypoint's.*/export const ingestConsumerMetrics = {- reprocessed: (partition: string) => kafkaReprocessedTotal.inc({ partition }),handlerFailed: (partition: string) =>kafkaHandlerFailuresTotal.inc({ partition }),deadLettered: (partition: string, reason: DeadLetterReason) =>diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.tsindex 7dea1f7ff..012af2639 100644--- a/packages/core/test/request-id-end-to-end.test.ts+++ b/packages/core/test/request-id-end-to-end.test.ts@@ -579,6 +579,9 @@ function ingestDeps() {const ingestBindings: IncomingEventBindings = {checkNotificationRulesForEvent: () => Promise.resolve(null),+ // Never a duplicate here: this suite is about the requestId, and the marker+ // must not decide anything about the row either way (M21-001).+ markDuplicateEvent: () => Promise.resolve(false),};function ingestEnvelope(requestId?: string): IncomingEventPayload {diff --git a/packages/redis/duplicate-marker.test.ts b/packages/redis/duplicate-marker.test.tsnew file mode 100644index 000000000..e92f8776b--- /dev/null+++ b/packages/redis/duplicate-marker.test.ts@@ -0,0 +1,139 @@+/**+ * M21-001 — the event-keyed duplicate marker, at the Redis boundary.+ *+ * What the ingest handler does with the answer is `packages/core`'s+ * `incoming-event-handler.test.ts`; what this file pins is the command: one+ * `SET key NX PX`, the key shape, and which reply means "seen before".+ *+ * No real Redis here: an in-memory stand-in with real `NX` and expiry+ * semantics. The same four answers were driven through THIS marker against the+ * local Redis (`redis://127.0.0.1:6379/14`) on 2026-09-14 before this file was+ * written, at an 800 ms TTL — first sighting false, `PTTL` 800, second+ * sighting true, a distinct id false, and the same id false again 900 ms+ * later.+ */++import { describe, expect, it } from 'vitest';+import {+ createDuplicateEventMarker,+ DUPLICATE_MARKER_KEY_PREFIX,+ type DuplicateMarkerRedisClient,+} from './duplicate-marker';++const TTL_MS = 120_000;+const EVENT_ID = '11111111-2222-4333-8444-555555555555';+const OTHER_EVENT_ID = '99999999-8888-4777-8666-555555555555';++function fakeRedis() {+ const expiries = new Map<string, number>();+ const commands: unknown[][] = [];+ let nowMs = 0;++ const client: DuplicateMarkerRedisClient = {+ set(key, value, expiryMode, ttlMs, setMode) {+ commands.push([key, value, expiryMode, ttlMs, setMode]);+ const expiresAt = expiries.get(key);+ if (expiresAt !== undefined && expiresAt > nowMs) {+ return Promise.resolve(null);+ }+ expiries.set(key, nowMs + ttlMs);+ return Promise.resolve('OK');+ },+ };++ return {+ client,+ commands,+ advance: (ms: number) => {+ nowMs += ms;+ },+ };+}++describe('createDuplicateEventMarker', () => {+ it('reports the first sighting of an id as not-a-duplicate', async () => {+ const redis = fakeRedis();+ const mark = createDuplicateEventMarker({+ client: redis.client,+ ttlMs: TTL_MS,+ });++ expect(await mark(EVENT_ID)).toBe(false);+ });++ it('reports the second sighting of the same id as a duplicate', async () => {+ const redis = fakeRedis();+ const mark = createDuplicateEventMarker({+ client: redis.client,+ ttlMs: TTL_MS,+ });++ await mark(EVENT_ID);++ expect(await mark(EVENT_ID)).toBe(true);+ });++ it('keeps distinct ids independent', async () => {+ const redis = fakeRedis();+ const mark = createDuplicateEventMarker({+ client: redis.client,+ ttlMs: TTL_MS,+ });++ await mark(EVENT_ID);++ expect(await mark(OTHER_EVENT_ID)).toBe(false);+ });++ it('forgets an id once its key has expired', async () => {+ const redis = fakeRedis();+ const mark = createDuplicateEventMarker({+ client: redis.client,+ ttlMs: TTL_MS,+ });++ await mark(EVENT_ID);+ redis.advance(TTL_MS + 1);++ expect(await mark(EVENT_ID)).toBe(false);+ });++ it('costs exactly one command per event, a prefixed SET NX PX', async () => {+ const redis = fakeRedis();+ const mark = createDuplicateEventMarker({+ client: redis.client,+ ttlMs: TTL_MS,+ });++ await mark(EVENT_ID);++ expect(redis.commands).toEqual([+ [`${DUPLICATE_MARKER_KEY_PREFIX}${EVENT_ID}`, '1', 'PX', TTL_MS, 'NX'],+ ]);+ });++ it('takes a key prefix override without touching the default', async () => {+ const redis = fakeRedis();+ const mark = createDuplicateEventMarker({+ client: redis.client,+ ttlMs: TTL_MS,+ keyPrefix: 'test:dup:',+ });++ await mark(EVENT_ID);++ expect(redis.commands[0]![0]).toBe(`test:dup:${EVENT_ID}`);+ expect(DUPLICATE_MARKER_KEY_PREFIX).toBe('ingest:duplicate:');+ });++ it('rejects when Redis does not answer, leaving the decision to the caller', async () => {+ const mark = createDuplicateEventMarker({+ client: {+ set: () => Promise.reject(new Error('Stream is not writeable')),+ },+ ttlMs: TTL_MS,+ });++ await expect(mark(EVENT_ID)).rejects.toThrow('Stream is not writeable');+ });+});diff --git a/packages/redis/duplicate-marker.ts b/packages/redis/duplicate-marker.tsnew file mode 100644index 000000000..902d21aa9--- /dev/null+++ b/packages/redis/duplicate-marker.ts@@ -0,0 +1,95 @@+/**+ * The ingest consumer's duplicate MARKER (M21-001).+ *+ * Carl, 2026-09-14, on the offset watermark this replaces: *"are they purely+ * for logging reprocessed? Then we should remove them, i dont care. we can use+ * the redis check we talked about earlier to mark if it was a duplicate+ * event?"*+ *+ * It is keyed on the EVENT, not on a Kafka offset. The event `id` is+ * producer-minted, so a redelivered message re-presents the same id — drill 08+ * confirmed all 161 of its duplicates were byte-identical replays sharing one+ * `id`, one `created_at` and one `session_id`. A key therefore survives+ * restarts, rebalances, evictions and process boundaries, none of which the+ * in-process watermark survived: drill 08 measured 161 real duplicates and 0+ * `REPROCESSED` lines.+ *+ * MARK MEANS COUNT AND LOG. IT DOES NOT MEAN DROP. This tells the caller the+ * id has been seen before; the caller inserts the event either way. A false+ * positive on a suppressing check is silent data loss, and whether the events+ * table ever gets dedupe is Carl's decision, not this file's.+ *+ * ONE round trip, `SET key NX PX`: the set and the "did it already exist"+ * answer are the same command, and there is no clean seam to fold it into —+ * the session buffer's own GET and Lua EVAL live in `packages/core`'s+ * `buffers/`, which this change does not own.+ *+ * IT MUST BE GIVEN THE CACHE CLIENT. That one fails fast (M18-003): offline+ * queue off once connected, `commandTimeout` 500 ms. `getRedisQueue()` keeps+ * `maxRetriesPerRequest: null` for BullMQ and would block a handler for tens+ * of seconds — which is how drill 02's consumer was evicted past its 30 s+ * session timeout, and an eviction is the only reassignment M19 measured+ * costing duplicate rows.+ *+ * Why here and not in `packages/core`'s ingest module: the whole of it is+ * Redis mechanics, and core's only export door is its barrel, which this+ * task's scope does not cover — the same reason `dead-letter.ts` lives here.+ */++/** Live key prefix. An operator has to be able to find these. */+export const DUPLICATE_MARKER_KEY_PREFIX = 'ingest:duplicate:';++/**+ * The narrow slice of a Redis client this needs. Structural rather than+ * ioredis's `Redis`, so a test can satisfy it without a server.+ */+export interface DuplicateMarkerRedisClient {+ set(+ key: string,+ value: string,+ expiryMode: 'PX',+ ttlMs: number,+ setMode: 'NX'+ ): Promise<string | null>;+}++export interface DuplicateEventMarkerOptions {+ client: DuplicateMarkerRedisClient;+ /**+ * How long a marker outlives its event. It only has to outlive the REPLAY+ * window, not the data.+ */+ ttlMs: number;+ /** Overridable for tests; production uses the constant above. */+ keyPrefix?: string;+}++/** The marker's only stored value — presence is the whole signal. */+const MARKER_VALUE = '1';++/**+ * @returns true when this event id has been seen before, false when this call+ * is the first sighting.+ *+ * REJECTS when Redis does not answer. The caller decides what that means, and+ * on the ingest path the answer is always "process the event normally" — an+ * observability counter must never be the reason an event is delayed or lost.+ */+export function createDuplicateEventMarker(+ options: DuplicateEventMarkerOptions+): (eventId: string) => Promise<boolean> {+ const keyPrefix = options.keyPrefix ?? DUPLICATE_MARKER_KEY_PREFIX;++ return async (eventId: string): Promise<boolean> => {+ const stored = await options.client.set(+ `${keyPrefix}${eventId}`,+ MARKER_VALUE,+ 'PX',+ options.ttlMs,+ 'NX'+ );+ // `SET NX` answers 'OK' when it wrote the key and null when one was+ // already there — so null IS the duplicate.+ return stored === null;+ };+}diff --git a/packages/redis/index.ts b/packages/redis/index.tsindex 59f2651b4..709bee287 100644--- a/packages/redis/index.ts+++ b/packages/redis/index.ts@@ -1,5 +1,6 @@-export * from './redis';export * from './cachable';export * from './dead-letter';-export * from './run-every';+export * from './duplicate-marker';export * from './publisher';+export * from './redis';+export * from './run-every';