OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M11-003): packages/queue into core: Kafka transport, notification dispatch and the legacy queue definitions; main.ts imports none of it from @openpanel/queue
ralph3deada3955b3
diff --git a/apps/api/e2e/lag-monitor.ts b/apps/api/e2e/lag-monitor.tsindex 3925cb876..904e391eb 100644--- a/apps/api/e2e/lag-monitor.ts+++ b/apps/api/e2e/lag-monitor.ts@@ -1,8 +1,9 @@/*** Kafka consumer-group lag sampling for the stress harness — the backpressure* story a throughput number alone hides. Wraps the admin-API primitives in- * `@openpanel/queue`; a fast API can outrun its consumer for a while without- * a single failed request, and this is the only place that would show it.+ * `@openpanel/core`'s ingest module; a fast API can outrun its consumer for a+ * while without a single failed request, and this is the only place that would+ * show it.*/import {@@ -12,7 +13,7 @@ import {KAFKA_CONSUMER_GROUP,KAFKA_EVENTS_TOPIC,sampleConsumerGroupLag,-} from '@openpanel/queue';+} from '@openpanel/core';export interface LagSummary {sampleCount: number;diff --git a/apps/api/e2e/legacy-job-proof.ts b/apps/api/e2e/legacy-job-proof.tsindex fb4100b47..4227508dc 100644--- a/apps/api/e2e/legacy-job-proof.ts+++ b/apps/api/e2e/legacy-job-proof.ts@@ -40,7 +40,6 @@ import {queues,startWorkers,} from '@openpanel/core';-import { produceIncomingEvent } from '@openpanel/queue';import { Redis } from '@openpanel/redis';import pino from 'pino';@@ -57,7 +56,7 @@ interface LegacyCase {queue: QueueKey;/** The BullMQ job name V1 put on the wire. */wireName: string;- /** V1's job data, verbatim (packages/queue/src/queues.ts). */+ /** V1's job data, verbatim (was packages/queue/src/queues.ts). */data: unknown;/** The V2 job the compat hook must resolve it to. */expectJob: string;@@ -286,7 +285,6 @@ async function main() {clients: undefined as unknown as AppDeps['clients'],buffers,producers,- produceIncomingEvent,logger,config: { selfHosted: false },};diff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex df234d1db..f03cfdb7e 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -34,16 +34,19 @@ import { createHmac } from 'node:crypto';import {type AppDeps,appRouter,+ assertKafkaConfigured,BULL_BOARD_BASE_PATH,type BufferDeps,bullBoardRoutes,COOKIE_OPTIONS,+ checkNotificationRulesForEvent,clearProjectByIdCache,corsDelegator,createBuffers,createClients,createIncomingEventHandler,createInitialSalts,+ createKafkaEventsConsumer,createProducers,createTrpcFetchHandler,dashboardRoutes,@@ -53,9 +56,16 @@ import {getProjectByIdCached,type HttpCtx,ingestConsumerMetrics,+ KAFKA_EVENTS_TOPIC,+ KAFKA_HANDLER_MAX_ATTEMPTS,+ KAFKA_HANDLER_RETRY_INITIAL_MS,+ KAFKA_HANDLER_RETRY_MAX_MS,+ KAFKA_PARTITIONS_CONCURRENT,type KafkaConsumerHandle,+ kafkaLogger,markEventsActivity,opsRoutes,+ produceDeadLetterEvent,publicApiRoutes,type QueueDefinition,type QueueProducerHandle,@@ -79,19 +89,6 @@ import {} from '@openpanel/core';import { ch } from '@openpanel/db/src/clickhouse/client';import { db } from '@openpanel/db/src/prisma-client';-import {- assertKafkaConfigured,- createKafkaEventsConsumer,- KAFKA_EVENTS_TOPIC,- KAFKA_HANDLER_MAX_ATTEMPTS,- KAFKA_HANDLER_RETRY_INITIAL_MS,- KAFKA_HANDLER_RETRY_MAX_MS,- KAFKA_PARTITIONS_CONCURRENT,- kafkaLogger,- produceDeadLetterEvent,- produceIncomingEvent,-} from '@openpanel/queue';-import { checkNotificationRulesForEvent } from '@openpanel/queue/src/notification-dispatch';import { getRedisCache, getRedisPub, getRedisQueue } from '@openpanel/redis';import { Elysia } from 'elysia';import pino from 'pino';@@ -206,9 +203,6 @@ function buildDeps(): AppDeps {clients: createClients(),buffers: createBuffers(bufferDeps(producers)),producers,- // @openpanel/queue's Kafka producer, injected because core cannot import- // it back (M8-002). It dies with that package at M9-003.- produceIncomingEvent,logger,config: { selfHosted: config.SELF_HOSTED },};@@ -277,9 +271,9 @@ function bootServiceDeps(/*** The Kafka events consumer. The kafkajs client, the topic, the consumer- * group, the DLQ producer and the retry bounds all come from- * `@openpanel/queue`, which is what keeps those names byte-identical across- * the cutover — core never spells one (see modules/ingest/src/consumer.ts).+ * group, the DLQ producer and the retry bounds all come from core's own+ * `modules/ingest/src/kafka.ts` (M11-003) — still passed in as arguments, so+ * `consumer.ts` spells none of those names itself.*/async function startIngestConsumer(deps: AppDeps@@ -296,9 +290,9 @@ async function startIngestConsumer(batch: {// One Ctx per message, scoped to the requestId the producer stamped// into the envelope (M10-006). The two bindings a work scope cannot- // supply are resolved here, once: the notification dispatch lives in- // @openpanel/queue (core cannot import it back) and the project cache- // is the boot-registered singleton ingest/http/mcp share.+ // supply are resolved here, once: the notification dispatch has no Ctx+ // slot in its signature, and the project cache is the boot-registered+ // singleton ingest/http/mcp share.handleEvent: createIncomingEventHandler(deps, {checkNotificationRulesForEvent,getCachedProject: getProjectByIdCached,diff --git a/docs/TECH_DEBT.md b/docs/TECH_DEBT.mdindex 1a16395d0..188ac999c 100644--- a/docs/TECH_DEBT.md+++ b/docs/TECH_DEBT.md@@ -992,3 +992,119 @@ existed to leak.implement one.- `verification/harness stop`, then `verification/full.sh` — green, with thedocumented pre-P11 `contracts/sdk/dist-gate.sh` exception.++### 2026-09-06 — M11-003, `packages/queue`'s Kafka transport and notification dispatch move into core; `queues.ts` / `buffers.ts` do NOT (out of scope)++Ran by: ralph (M11-003 implement task).++**Survey, before the move** — every import of `@openpanel/queue` outside+`packages/queue` (`grep -rnE "from '@openpanel/queue" --include=*.ts`):++| File | Symbols |+|---|---|+| `apps/api/src/main.ts` | `assertKafkaConfigured`, `createKafkaEventsConsumer`, `KAFKA_EVENTS_TOPIC`, `KAFKA_HANDLER_MAX_ATTEMPTS`, `KAFKA_HANDLER_RETRY_INITIAL_MS`, `KAFKA_HANDLER_RETRY_MAX_MS`, `KAFKA_PARTITIONS_CONCURRENT`, `kafkaLogger`, `produceDeadLetterEvent`, `produceIncomingEvent` (10, from the barrel) + `checkNotificationRulesForEvent` (deep, `@openpanel/queue/src/notification-dispatch`) |+| `apps/api/e2e/lag-monitor.ts` | `createKafkaAdmin`, `disconnectKafka`, `KAFKA_BROKERS`, `KAFKA_CONSUMER_GROUP`, `KAFKA_EVENTS_TOPIC`, `sampleConsumerGroupLag` |+| `apps/api/e2e/legacy-job-proof.ts` | `produceIncomingEvent` |+| `packages/core/src/modules/session/session.jobs.ts` | `checkNotificationRulesForSessionEnd` (lazy `import('@openpanel/queue/src/notification-dispatch')`) |+| `packages/core/src/buffers/lazy-db-buffers.ts` | the buffer singletons (lazy `import('@openpanel/queue/src/buffers')`), reached only by `modules/widget/widget.rpc.ts` |+| `packages/db/scripts/check-sessions.ts` | `sessionsQueue` |+| `packages/db/scripts/drain-old-session-jobs.ts` | `sessionsQueue` |+| `packages/db/scripts/migrate-sessions.ts` | `sessionsQueue`, `EventsQueuePayloadCreateSessionEnd` |+| `packages/trpc/src/routers/cohort.ts` | `cohortComputeQueue` |+| `packages/trpc/src/routers/gsc.ts` | `gscQueue` |+| `packages/trpc/src/routers/import.ts` | `importQueue` |+| `packages/trpc/src/routers/overview.ts` | `eventBuffer` (deep, `@openpanel/queue/src/buffers`) |+| `packages/trpc/src/routers/widget.ts` | `eventBuffer` (deep, `@openpanel/queue/src/buffers`) |++**What moved.** `src/kafka.ts` → `packages/core/src/modules/ingest/src/kafka.ts`+and `src/notification-dispatch.ts` →+`packages/core/src/modules/notification/src/notification-dispatch.ts`. Both are+the same file: `diff` against `HEAD` shows only repointed imports, plus two+edits with reasons —++- `DeadLetterMessage` was declared twice (once in `kafka.ts`, once in+ `consumer.ts`, structurally identical by design); now that they are siblings+ `kafka.ts` imports the consumer's.+- `produceIncomingEvent`'s parameter changed from+ `EventsQueuePayloadIncomingEvent['payload']` to core's own+ `IncomingEventPayload`. The wire format is unchanged: a temporary+ `[A] extends [B] ? [B] extends [A]` assertion compiled clean in both+ directions against `@openpanel/queue`'s type.+- `triggerNotification` enqueues through core's registry+ (`queues.notification.sendNotification`) rather than a second+ `new Queue('notification')` — same Redis key, same job name, same+ `{payload, meta}` envelope (`golden/queue-keys/check.sh` still matches the+ V1 goldens), and it became `async` because the registry producer is.++Every Kafka constant is byte-identical after the move — topic+(`process.env.KAFKA_EVENTS_TOPIC || 'events'`), DLQ+(`|| \`${KAFKA_EVENTS_TOPIC}-dlq\``), group+(`|| 'openpanel-events'`), and the 15 `Number.parseInt` bounds — verified by+`diff -u` of the two files' bodies.++**The `AppDeps.produceIncomingEvent` injection is retired.** It existed only+because `@openpanel/queue` imports `@openpanel/core` for its logger, so core+could not import the producer back (`context.ts`'s own comment). With+`kafka.ts` in core, `ingest.routes.ts` imports the sibling directly. The+producer stays an *argument* to `ingest.service.ts` — that seam is now about+letting a test assert what was produced without a broker, not about a package+cycle, and `ingest.service.test.ts` / `legacy-event.test.ts` still use it.++**`buffers/lazy-db-buffers.ts` is deleted**; its one core caller+(`widget.rpc.ts`'s `loadEventBuffer`) reads `ctx.buffers.event`.++**The three orphan `packages/db/scripts` are deleted** — `check-sessions.ts`,+`drain-old-session-jobs.ts`, `migrate-sessions.ts`, all V1 session-migration+tooling. `grep -rn 'check-sessions\|drain-old-session-jobs\|migrate-sessions'`+over `*.json`/`*.ts`/`*.md`/`*.sh` returns nothing outside the files+themselves; `packages/db/package.json`'s only `scripts/*` entry is+`duplicate-events` → `find-duplicate-events.ts`, untouched.++**NOT done, and why — `packages/queue/src/queues.ts` and `src/buffers.ts` stay.**+The task asked for both to be deleted. They cannot be, inside this task's+declared scope (`packages/core/**`, `packages/queue/**`,+`packages/db/scripts/**`, `apps/api/**`, `docs/**`): five+`packages/trpc/src/routers/*` files import them (the last five rows of the+survey above), and `packages/trpc/**` is not in scope. Deleting the two files+without touching those routers fails `pnpm run typecheck`; touching them is+the out-of-scope edit. **This needs either a scope extension to+`packages/trpc/src/routers/{cohort,gsc,import,overview,widget}.ts` or a+follow-up task**, and it is not a mechanical repoint — the three queue call+sites become `ctx.services.cohort.enqueueCompute` / `ctx.queues.gsc...` /+`ctx.services.import.enqueue`, which changes `import.ts`'s `jobId` handling+and `cohort.ts`'s procedure signatures. `packages/queue/index.ts` no longer+re-exports `./src/kafka`; what remains behind the barrel is exactly what trpc+still consumes.++Also still declared, unused: `@openpanel/queue` in `apps/api/package.json`,+`packages/core/package.json` and `packages/db/package.json` — no file in any+of the three imports it any more. Removing the manifest entries belongs with+M11-004's deletion of the package.++**Verification, all run by ralph on 2026-09-06:**++- `pnpm run typecheck` — all 25 workspaces, `Done`.+- `pnpm test` — 13 files, **205 passed**.+- `cd packages/core && bun test` (bare) — **1466 pass, 12 skip, 0 fail**,+ `Ran 1478 tests across 151 files. [12.59s]`, `real 0m12.652s`, and it+ EXITS: moving `kafka.ts` into core constructs no Kafka client at import+ time (`getKafka()` is lazy; only `assertKafkaConfigured` reads+ `KAFKA_BROKERS`, and nothing calls it at module scope).+- `pnpm run check:deps` — `no dependency violations found (2321 modules,+ 10072 dependencies cruised)`, both rules still at `error`.+- `bash apps/api/e2e/boot-proof.sh` — all checks passed, all three ROLEs.+- `verification/golden/queue-keys/check.sh` — `OK - 7x2 queue keys and 20+ scheduler ids derived from the V2 registry match the V1 goldens`.+- `verification/harness start` → `cd apps/api && pnpm run e2e:sessions`:+ **29/29 checks passed** (Kafka produce → consume is the path under test) →+ `pnpm run e2e:legacy-jobs`: **all checks passed** → `harness stop`.+- `verification/full.sh` — `FULL: green`.++One incidental find, recorded not fixed: `usage.ts`'s static+`import { getOrganizationBillingEventsCount } from '../../../v1-compat'` had+to become part of the existing lazy `loadProjectService()` call. `jobs.registry.ts`+reaches this file (`session.jobs.ts` → `session-end` → `usage`), and that one+static edge pulls `services.ts` and all 36 services into the registry's import+graph, which builds a ClickHouse client at import time — `queue-keys/check.sh`+fails outright without the change (confirmed by reverting the file alone and+re-running it).diff --git a/packages/core/src/buffers/lazy-db-buffers.ts b/packages/core/src/buffers/lazy-db-buffers.tsdeleted file mode 100644index 3f9d1f747..000000000--- a/packages/core/src/buffers/lazy-db-buffers.ts+++ /dev/null@@ -1,10 +0,0 @@-// The seven buffer singletons are built once by @openpanel/queue's-// src/buffers.ts (moved from @openpanel/db, M9-CLEANUP-001 — see that file's-// header for why @openpanel/queue and not here) and, once V2 owns boot,-// handed to AppDeps by createBuffers directly. Until then, every core module-// that needs a buffer ahead of that boot wiring reaches it through THIS one-// dynamic import() — one edge, one file, same reasoning as-// session.service.ts's loadProfileService comment.-export function loadDbBuffers() {- return import('@openpanel/queue/src/buffers');-}diff --git a/packages/core/src/context.test.ts b/packages/core/src/context.test.tsindex fd25e8e56..2125f9dcb 100644--- a/packages/core/src/context.test.ts+++ b/packages/core/src/context.test.ts@@ -81,7 +81,6 @@ function stubDeps(): {return surface;},},- produceIncomingEvent: () => Promise.resolve(),logger: stubLogger(),config: { selfHosted: true },},diff --git a/packages/core/src/context.ts b/packages/core/src/context.tsindex 4bc1b3681..7e464bf4a 100644--- a/packages/core/src/context.ts+++ b/packages/core/src/context.ts@@ -8,7 +8,6 @@ import type { Buffers } from './buffers/create-buffers';import type { QueueProducerHandle, QueueProducers } from './jobs.registry';import type { Logger } from './logger';import type { SessionValidationResult } from './modules/auth/src/login-session';-import type { IncomingEventProducer } from './modules/ingest/src/incoming-event';import { createServices, type Services } from './services';import type { CookieJar, CookieOptions } from './shared/cookie';@@ -53,14 +52,6 @@ export interface AppDeps {clients: ServiceClients;buffers: Buffers;producers: QueueProducerHandle;- /**- * The events transport. `produceIncomingEvent` still lives in- * @openpanel/queue, which imports core for its logger, so core cannot- * import it back (M8-002); main.ts supplies it. It stays injected until- * @openpanel/queue dies at P9 — M8-003 moved the consumer, not the- * producer, and the consumer takes its kafkajs client the same way.- */- produceIncomingEvent: IncomingEventProducer;logger: Logger;config: RuntimeFlags;}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 8baa54df4..610009416 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -533,15 +533,6 @@ export {extractToken,handleStatelessMcpRequest,} from './modules/mcp/mcp.service';-// Dissolved from @openpanel/db's services/notification.service.ts (M6-005,-// "rules + dispatch stay together") — packages/trpc's notification router-// and apps/worker's notification job call these directly, the same way V1-// reaches every other dissolved service here. `createNotification` /-// `triggerNotification` / `checkNotificationRulesForEvent` /-// `checkNotificationRulesForSessionEnd` — the BullMQ-producer orchestration-// around a rule match — moved to `@openpanel/queue/src/notification-dispatch.ts`-// (M9-CLEANUP-001, packages/db's final-surface shrink), not here: core still-// cannot import `@openpanel/queue` back without a real package cycle.export type {INotificationPayload,INotificationRuleCached,@@ -561,6 +552,14 @@ export {notificationTemplateEvent,notificationTemplateFunnel,} from './modules/notification/notification.service';+// Dissolved from @openpanel/db's services/notification.service.ts (M6-005,+// "rules + dispatch stay together"). `createNotification` /+// `triggerNotification` / `checkNotificationRulesForEvent` /+// `checkNotificationRulesForSessionEnd` — the BullMQ-producer orchestration+// around a rule match — came home from @openpanel/queue at M11-003 and now+// live in this module's own src/notification-dispatch.ts. Only the one the+// Kafka message handler is bound to at boot needs to leave the package.+export { checkNotificationRulesForEvent } from './modules/notification/src/notification-dispatch';// New module (M6-003) — the onboarding-project mutation and the onboarding// email drip, neither of which had a packages/db/src/services/* home to move// from. packages/trpc's onboarding router and apps/worker's onboarding cron@@ -882,10 +881,10 @@ export {summarizeBotSignals,validateIngestRequest,} from './modules/ingest/ingest.service';-// The Kafka consumer and its per-message handler (M8-003). apps/worker's-// events.kafka-consumer.ts / events.incoming-event.ts are thin delegates that-// hand in the kafkajs client, the topic/group names and their own registry's-// counters — none of which this package spells, so they stay byte-identical.+// The Kafka consumer and its per-message handler (M8-003), which take the+// kafkajs client, the topic/group names and the retry bounds as arguments —+// the injection seam that used to cross a package boundary and now just+// crosses two files.export type {ConsumerLogger,ConsumerMetrics,@@ -921,6 +920,45 @@ export {incomingEvent,} from './modules/ingest/src/incoming-event-handler';export { ingestConsumerMetrics } from './modules/ingest/src/ingest.metrics';+// The Kafka transport itself (M11-003): producer, consumer factory, admin/lag+// sampling and every topic/group/retry constant, moved from+// @openpanel/queue's kafka.ts unchanged. It constructs no client at import+// time, so the barrel stays offline-importable.+export type {+ Admin,+ ConsumerGroupLag,+ EachBatchPayload,+ KafkaMessage,+ PartitionLag,+} from './modules/ingest/src/kafka';+export {+ assertKafkaConfigured,+ createKafkaAdmin,+ createKafkaEventsConsumer,+ disconnectKafka,+ KAFKA_BROKERS,+ KAFKA_CONNECTION_TIMEOUT_MS,+ KAFKA_CONSUMER_GROUP,+ KAFKA_EVENTS_DLQ_TOPIC,+ KAFKA_EVENTS_TOPIC,+ KAFKA_HANDLER_MAX_ATTEMPTS,+ KAFKA_HANDLER_RETRY_INITIAL_MS,+ KAFKA_HANDLER_RETRY_MAX_MS,+ KAFKA_HEARTBEAT_INTERVAL_MS,+ KAFKA_MAX_MESSAGES_PER_PARTITION,+ KAFKA_MAX_WAIT_MS,+ KAFKA_MIN_MESSAGES,+ KAFKA_PARTITIONS_CONCURRENT,+ KAFKA_PRODUCER_INITIAL_RETRY_MS,+ KAFKA_PRODUCER_MAX_RETRY_MS,+ KAFKA_PRODUCER_RETRIES,+ KAFKA_REQUEST_TIMEOUT_MS,+ KAFKA_SESSION_TIMEOUT_MS,+ kafkaLogger,+ produceDeadLetterEvent,+ produceIncomingEvent,+ sampleConsumerGroupLag,+} from './modules/ingest/src/kafka';export type {DailyInsightCandidate,GetReferrerSpikesInput,diff --git a/packages/core/src/jobs/workers.test.ts b/packages/core/src/jobs/workers.test.tsindex 691076498..438615cf9 100644--- a/packages/core/src/jobs/workers.test.ts+++ b/packages/core/src/jobs/workers.test.ts@@ -30,7 +30,6 @@ function stubDeps(): AppDeps {clients: {} as AppDeps['clients'],buffers: {} as Buffers,producers: createRecordingProducers(queues),- produceIncomingEvent: () => Promise.resolve(),logger: stubLogger(),config: { selfHosted: true },};diff --git a/packages/core/src/modules/ingest/ingest.routes.ts b/packages/core/src/modules/ingest/ingest.routes.tsindex 5f98ad6b1..956b4a2f3 100644--- a/packages/core/src/modules/ingest/ingest.routes.ts+++ b/packages/core/src/modules/ingest/ingest.routes.ts@@ -43,6 +43,10 @@ import {} from './ingest.service';import { toIngestHeaders } from './src/headers';import { recordLegacyEventRequest } from './src/ingest.metrics';+// The Kafka producer is a sibling now (M11-003), not an `AppDeps` field:+// kafka.ts moved into this module, so there is nothing left for main.ts to+// inject. It still constructs no client at import time.+import { produceIncomingEvent } from './src/kafka';const TAGS = ['Track'];const LEGACY_EVENT_TAGS = ['Event'];@@ -174,7 +178,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>},{buffers: ctx.buffers,- produceIncomingEvent: deps.produceIncomingEvent,+ produceIncomingEvent,deps: ctx,});@@ -244,7 +248,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>},{buffers: ctx.buffers,- produceIncomingEvent: deps.produceIncomingEvent,+ produceIncomingEvent,deps: ctx,});diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.tsindex ca1180a2d..abedc9af8 100644--- a/packages/core/src/modules/ingest/ingest.service.ts+++ b/packages/core/src/modules/ingest/ingest.service.ts@@ -8,11 +8,10 @@// codes and bodies, so the pipeline returns a `TrackOutcome` and V1 maps it// back onto the exact `HttpError`s and replies it produced before.//-// The Kafka producer is INJECTED. `produceIncomingEvent` still lives in-// @openpanel/queue, which imports @openpanel/core for its logger — core-// cannot import it back without a real package cycle (see-// cohort.service.ts's header). M8-003 moves the producer into core and the-// injection point becomes an AppDeps field.+// The Kafka producer is INJECTED. Since M11-003 `produceIncomingEvent` is a+// sibling (./src/kafka.ts), so the injection is no longer a package-cycle+// workaround and no longer travels through `AppDeps` — it stays an argument+// only so a test can assert on what was produced without a broker.import { generateId } from '@openpanel/common';import { assocPath, pathOr, pick } from 'ramda';diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.tsindex 7408baa29..a2fa53277 100644--- a/packages/core/src/modules/ingest/src/consumer.ts+++ b/packages/core/src/modules/ingest/src/consumer.ts@@ -4,10 +4,9 @@// bounded in-consumer retry, and a dead-letter topic on exhaustion.//// The kafkajs client, the topic/consumer-group names, the DLQ producer and-// the retry bounds are all INJECTED. They still live in @openpanel/queue,-// which imports @openpanel/core for its logger, so core cannot import it back-// without a package cycle (see ingest.service.ts's header). Injection is also-// what keeps those names byte-identical: this file never spells one.+// the retry bounds are all INJECTED, and stay so now that they live one+// directory away in ./kafka.ts (M11-003): injection is what keeps those names+// byte-identical, because this file never spells one.import type {Consumer,@@ -32,10 +31,10 @@ const RETRY_BACKOFF_FACTOR = 2;export type DeadLetterReason = 'parse_error' | 'handler_error';/**- * What the consumer hands to the dead-letter producer. Structurally identical- * to @openpanel/queue's `DeadLetterMessage`: the value stays the producer's- * original bytes, so a DLQ message can be replayed onto the events topic- * unchanged.+ * What the consumer hands to the dead-letter producer (./kafka.ts's+ * `produceDeadLetterEvent` is the one implementation): the value stays the+ * producer's original bytes, so a DLQ message can be replayed onto the events+ * topic unchanged.*/export interface DeadLetterMessage {key: Buffer | null;diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.tsindex 848dd64f6..e701837b9 100644--- a/packages/core/src/modules/ingest/src/incoming-event-handler.ts+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.ts@@ -101,9 +101,9 @@ export interface IncomingEventDeps {*/export interface IncomingEventBindings {/**- * BullMQ-producer orchestration, which lives in `@openpanel/queue` —- * a package that imports `@openpanel/core` back, so core cannot reach it- * (notification.service.ts's header).+ * BullMQ-producer orchestration (notification/src/notification-dispatch.ts).+ * Injected rather than imported so the hot path's per-message handler keeps+ * the notification module out of its own import graph.*/checkNotificationRulesForEvent(payload: IServiceCreateEventPayloaddiff --git a/packages/core/src/modules/ingest/src/incoming-event.ts b/packages/core/src/modules/ingest/src/incoming-event.tsindex 5a80de335..be476e2e5 100644--- a/packages/core/src/modules/ingest/src/incoming-event.ts+++ b/packages/core/src/modules/ingest/src/incoming-event.ts@@ -2,9 +2,9 @@//// This is a WIRE FORMAT, not an internal type: a rolling deploy overlaps a V1// producer with a V2 consumer on the same topic, so it stays byte-compatible-// with @openpanel/queue's `EventsQueuePayloadIncomingEvent['payload']`-// (ADR-004). A leaf file so `context.ts` can name the producer without-// reaching into the ingest service.+// with V1's `EventsQueuePayloadIncomingEvent['payload']` (ADR-004). A leaf+// file, so both ./kafka.ts's producer and the consumer can name the payload+// without reaching into the ingest service.import type { GeoLocation } from '../../../clients/geo';import type { parseUserAgent } from '../../../shared/parser-user-agent';diff --git a/packages/queue/src/kafka.ts b/packages/core/src/modules/ingest/src/kafka.tssimilarity index 90%rename from packages/queue/src/kafka.tsrename to packages/core/src/modules/ingest/src/kafka.tsindex cc575d86e..dae90966d 100644--- a/packages/queue/src/kafka.ts+++ b/packages/core/src/modules/ingest/src/kafka.ts@@ -1,4 +1,16 @@-import { createLogger } from '@openpanel/core';+// Moved from packages/queue/src/kafka.ts (M11-003), byte-for-byte apart from+// the three repointed imports and the de-duplicated `DeadLetterMessage` below.+// What kept it out of core was the import direction, not the code: it lived+// in @openpanel/queue, a package that imports @openpanel/core for its logger,+// so core could not import it back. Moving the file takes it out of that+// cycle — `createLogger` is a sibling now, and both the producer and the+// consumer sit in the module that owns the transport (ADR-004: Kafka is the+// sole events transport — no topic, group or envelope changes with the move).+//+// NOTHING here constructs a Kafka client at import time: `getKafka()` is lazy+// and only `assertKafkaConfigured` reads the broker list eagerly, so core+// stays importable with no broker and `bun test` still runs offline.+import {type Admin,type Consumer,@@ -7,7 +19,11 @@ import {logLevel,type Producer,} from 'kafkajs';-import type { EventsQueuePayloadIncomingEvent } from './queues';+import { createLogger } from '../../../clients/logger';+// One definition, not two: the consumer already declares the dead-letter+// message shape it hands to this producer, and both files are now siblings.+import type { DeadLetterMessage } from './consumer';+import type { IncomingEventPayload } from './incoming-event';export type { Admin, EachBatchPayload, KafkaMessage } from 'kafkajs';@@ -241,7 +257,7 @@ const send = async (topic: string, message: OutgoingMessage): Promise<void> => {};export const produceIncomingEvent = async (- payload: EventsQueuePayloadIncomingEvent['payload'],+ payload: IncomingEventPayload,partitionKey: string): Promise<void> =>send(KAFKA_EVENTS_TOPIC, {@@ -252,17 +268,6 @@ export const produceIncomingEvent = async (// Why the reason/error/coordinates travel as headers and not in the value: the// value stays the producer's original bytes, so a DLQ message can be replayed// onto the events topic unchanged.-export interface DeadLetterMessage {- key: Buffer | null;- value: Buffer | null;- headers?: IHeaders;- topic: string;- partition: number;- offset: string;- reason: string;- error: string;-}-export const produceDeadLetterEvent = async (message: DeadLetterMessage): Promise<void> =>diff --git a/packages/core/src/modules/notification/notification.service.ts b/packages/core/src/modules/notification/notification.service.tsindex 7f6e0b2ad..b1a57ea79 100644--- a/packages/core/src/modules/notification/notification.service.ts+++ b/packages/core/src/modules/notification/notification.service.ts@@ -3,16 +3,12 @@// moves here too, from apps/worker/src/jobs/notification.ts (DELEGATE// PATTERN: that file becomes a thin wrapper calling `deliverNotification`).//-// What does NOT move: `createNotification` / `triggerNotification` /-// `checkNotificationRulesForEvent` / `checkNotificationRulesForSessionEnd` —-// the BullMQ-producer orchestration around a rule match. `@openpanel/queue`-// imports `@openpanel/core` for its logger (packages/queue/src/queues.ts), so-// core cannot import `@openpanel/queue` back without a real package cycle —-// same constraint gsc.service.ts and cohort.service.ts document for their own-// enqueue wrappers. Those four functions stay in-// packages/db/src/services/notification.service.ts, now built on the rule-// matching, templates, cache and constants exported below instead of holding-// their own copies.+// `createNotification` / `triggerNotification` / `checkNotificationRulesForEvent`+// / `checkNotificationRulesForSessionEnd` — the BullMQ-producer orchestration+// around a rule match — live in this module's own `src/notification-dispatch.ts`+// since M11-003, built on the rule matching, templates, cache and constants+// exported below. They are a separate file only so the enqueue side is not+// dragged into every import of this one.//// db/ch access is LAZY (`load*` below), not a static top-level import — see// insight.service.ts's header for the full reasoning (jobs.registry.ts anddiff --git a/packages/queue/src/notification-dispatch.ts b/packages/core/src/modules/notification/src/notification-dispatch.tssimilarity index 76%rename from packages/queue/src/notification-dispatch.tsrename to packages/core/src/modules/notification/src/notification-dispatch.tsindex e744ae5fa..0999c3c15 100644Binary files a/packages/queue/src/notification-dispatch.ts and b/packages/core/src/modules/notification/src/notification-dispatch.ts differdiff --git a/packages/core/src/modules/session/session.jobs.ts b/packages/core/src/modules/session/session.jobs.tsindex 913750b81..1c966399b 100644--- a/packages/core/src/modules/session/session.jobs.ts+++ b/packages/core/src/modules/session/session.jobs.ts@@ -65,11 +65,10 @@ const sessionEndPayload = z.custom<SessionEndJobData>(/*** The session-end job's dependencies, bound to the run's own ctx.*- * Two lookups stay dynamic. `checkNotificationRulesForSessionEnd` lives in- * `@openpanel/queue`, which imports `@openpanel/core` back — a real package- * cycle (notification.service.ts's header). The rule cache is reached the same- * way only to keep notification.service.ts out of jobs.registry.ts's eager- * import graph, which every core test file walks.+ * Two lookups stay dynamic, both for the same reason now that+ * notification-dispatch.ts has moved into core (M11-003): keeping+ * notification.service.ts — and the dispatch module built on it — out of+ * jobs.registry.ts's eager import graph, which every core test file walks.*/async function sessionEndDeps(ctx: Ctx,@@ -78,7 +77,7 @@ async function sessionEndDeps(const [notifications, { checkNotificationRulesForSessionEnd }] =await Promise.all([import('../notification/notification.service'),- import('@openpanel/queue/src/notification-dispatch'),+ import('../notification/src/notification-dispatch'),]);return {diff --git a/packages/core/src/modules/session/src/session-end.ts b/packages/core/src/modules/session/src/session-end.tsindex 830c802cf..84fe8f063 100644--- a/packages/core/src/modules/session/src/session-end.ts+++ b/packages/core/src/modules/session/src/session-end.ts@@ -109,7 +109,8 @@ export interface SessionEndDeps extends SessionRuntime {notifications: {getRules(projectId: string): Promise<INotificationRuleCached[]>;hasFunnelRules(rules: INotificationRuleCached[]): boolean;- /** Lives in @openpanel/queue — it enqueues through the notification queue. */+ /** notification/src/notification-dispatch.ts — it enqueues through the+ * notification queue. */checkFunnelRules(events: IServiceEvent[]): Promise<unknown>;};}diff --git a/packages/core/src/modules/session/src/usage.ts b/packages/core/src/modules/session/src/usage.tsindex 4dcd270b7..f5a838edc 100644--- a/packages/core/src/modules/session/src/usage.ts+++ b/packages/core/src/modules/session/src/usage.ts@@ -8,13 +8,19 @@ import { cacheable } from '@openpanel/redis';import { sendEmail } from '../../../clients/email';import { createLogger, type ILogger } from '../../../clients/logger';import type { Logger } from '../../../logger';-import { getOrganizationBillingEventsCount } from '../../../v1-compat';// project.service.ts's `getProjectEventsCount` takes `ServiceDeps` now// (M10-004); this file has none (its own db access is the lazy `loadDb()`// below), so it reaches the bare, v1-compat-wrapped spelling instead.// GENUINE CYCLE, kept lazy: services.ts -> session.service.ts -> this file// -> v1-compat.ts -> services.ts.+//+// EVERY v1-compat symbol this file uses goes through here, never a static+// import: jobs.registry.ts reaches this file (session.jobs.ts -> session-end+// -> usage), and one static edge to v1-compat pulls services.ts and all 36+// services into the registry's import graph — which builds a ClickHouse+// client at import time and breaks the offline contract+// verification/golden/queue-keys/check.sh depends on.function loadProjectService() {return import('../../../v1-compat');}@@ -84,9 +90,10 @@ export const updateEventsCount = cacheable(async function updateEventsCount(return;}+ const { getOrganizationBillingEventsCount, getProjectEventsCount } =+ await loadProjectService();const organizationEventsCount =await getOrganizationBillingEventsCount(organization);- const { getProjectEventsCount } = await loadProjectService();const projectEventsCount = await getProjectEventsCount(projectId);if (projectEventsCount) {diff --git a/packages/core/src/modules/widget/widget.rpc.ts b/packages/core/src/modules/widget/widget.rpc.tsindex 90bbe45e5..c1b3ce277 100644--- a/packages/core/src/modules/widget/widget.rpc.ts+++ b/packages/core/src/modules/widget/widget.rpc.ts@@ -25,7 +25,6 @@import { clix } from '@openpanel/db/src/clickhouse/query-builder';import ShortUniqueId from 'short-unique-id';import { z } from 'zod';-import { loadDbBuffers } from '../../buffers/lazy-db-buffers';import type { Ctx } from '../../context';import {createTRPCRouter,@@ -40,10 +39,6 @@ import { zWidgetOptions, zWidgetType } from '../report/report.constants';const uid = new ShortUniqueId({ length: 6 });const BADGE_CACHE_TTL_SECONDS = 5 * 60; // queries 30 days of data-function loadEventBuffer() {- return loadDbBuffers().then((m) => m.eventBuffer);-}-function loadCache() {return import('@openpanel/redis').then((m) => m.getCache);}@@ -191,7 +186,7 @@ export const widgetRouter = createTRPCRouter({throw new TRPCNotFoundError('Invalid widget type');}- const eventBuffer = await loadEventBuffer();+ const eventBuffer = ctx.buffers.event;return {projectId: widget.projectId,diff --git a/packages/core/test/http-fixtures.ts b/packages/core/test/http-fixtures.tsindex 18cd04a9f..df38bfcb7 100644--- a/packages/core/test/http-fixtures.ts+++ b/packages/core/test/http-fixtures.ts@@ -44,7 +44,6 @@ export function stubAppDeps(): AppDepsStub {clients: {} as AppDeps['clients'],buffers: {} as Buffers,producers,- produceIncomingEvent: () => Promise.resolve(),logger,config: { selfHosted: false },},diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.tsindex be3d38efc..6fb01742e 100644--- a/packages/core/test/request-id-end-to-end.test.ts+++ b/packages/core/test/request-id-end-to-end.test.ts@@ -186,7 +186,6 @@ function stubDeps() {clients: {} as AppDeps['clients'],buffers: {} as Buffers,producers: producers as unknown as QueueProducerHandle,- produceIncomingEvent: () => Promise.resolve(),logger: bindingLogger(lines),config: { selfHosted: false },};@@ -368,7 +367,6 @@ function chartDeps() {clients: {} as AppDeps['clients'],buffers: {} as Buffers,producers: producers as unknown as QueueProducerHandle,- produceIncomingEvent: () => Promise.resolve(),logger: chartLogger(lines),config: { selfHosted: false },};@@ -554,7 +552,6 @@ function ingestDeps() {clients: {} as AppDeps['clients'],buffers,producers: producers as unknown as QueueProducerHandle,- produceIncomingEvent: () => Promise.resolve(),logger: ingestLogger(lines),config: { selfHosted: false },};diff --git a/packages/core/test/rpc-fixtures.ts b/packages/core/test/rpc-fixtures.tsindex 28d8f6781..bccf79346 100644--- a/packages/core/test/rpc-fixtures.ts+++ b/packages/core/test/rpc-fixtures.ts@@ -93,7 +93,6 @@ export function stubHttpCtx(clients: {} as AppDeps['clients'],buffers: {} as Buffers,producers: createRecordingProducers(queues),- produceIncomingEvent: () => Promise.resolve(),logger,config: { selfHosted: false },};diff --git a/packages/db/scripts/check-sessions.ts b/packages/db/scripts/check-sessions.tsdeleted file mode 100644index b05b89972..000000000--- a/packages/db/scripts/check-sessions.ts+++ /dev/null@@ -1,424 +0,0 @@-// Read-only health check for the session lifecycle pipeline.-//-// Inspects Redis state, the sessions BullMQ queue, both internal buffers,-// and ClickHouse — then reconciles them to surface anything stuck or-// drifting. Safe to run anytime; never mutates state.-//-// Usage:-// pnpm with-env tsx scripts/check-sessions.ts-// pnpm with-env tsx scripts/check-sessions.ts --verbose-// pnpm with-env tsx scripts/check-sessions.ts --project beatchurn-website-// pnpm with-env tsx scripts/check-sessions.ts --hours 48--import { getSafeJson } from '@openpanel/json';-import type { IClickhouseSession } from '@openpanel/core';-import { sessionsQueue } from '@openpanel/queue';-import { getRedisCache } from '@openpanel/redis';-import { ch } from '../src/clickhouse/client';--const DEADMAN_MS = Number.parseInt(- process.env.SESSION_REAPER_WALLCLOCK_DEADMAN_MS || String(30 * 60 * 1000),- 10-);-const REAPER_INTERVAL_MS = 5 * 60 * 1000;-// "Eligible" = past the deadman, reaper is expected to close shortly.-// "Stuck" = past 3x reaper interval beyond the deadman — reaper is behind.-const STUCK_GRACE_MS = REAPER_INTERVAL_MS * 3;-const STUCK_CUTOFF_MS = DEADMAN_MS + STUCK_GRACE_MS;-const SAMPLE_SIZE_PER_PROJECT = 100;--type ProjectStat = {- project: string;- active: number;- eligible: number;- stuck: number;- orphans: number;- oldestAgeMin: number;-};--type ChCounts = {- starts1h: number;- ends1h: number;- starts24h: number;- ends24h: number;-};--function fmt(n: number, w: number) {- return String(n).padStart(w);-}--function header(title: string) {- console.log('');- console.log('─'.repeat(70));- console.log(title);- console.log('─'.repeat(70));-}--async function inspectRedis(projects: string[]): Promise<{- total: number;- totalEligible: number;- totalStuck: number;- totalOrphans: number;- stats: ProjectStat[];-}> {- const redis = getRedisCache();- const now = Date.now();- const eligibleCutoff = now - DEADMAN_MS;- const stuckCutoff = now - STUCK_CUTOFF_MS;-- const stats: ProjectStat[] = [];- let total = 0;- let totalEligible = 0;- let totalStuck = 0;- let totalOrphans = 0;-- for (const project of projects) {- const wallclockKey = `session:wallclock:${project}`;-- const [active, eligible, stuck, oldestEntry] = await Promise.all([- redis.zcard(wallclockKey),- redis.zcount(wallclockKey, '-inf', eligibleCutoff),- redis.zcount(wallclockKey, '-inf', stuckCutoff),- redis.zrange(wallclockKey, 0, 0, 'WITHSCORES'),- ]);-- const oldestAgeMin =- oldestEntry.length === 2 ? (now - Number(oldestEntry[1])) / 60_000 : 0;-- // Sample N oldest entries and verify their blob exists.- const sampleSize = Math.min(SAMPLE_SIZE_PER_PROJECT, active);- let orphans = 0;- if (sampleSize > 0) {- const sample = await redis.zrange(wallclockKey, 0, sampleSize - 1);- const multi = redis.multi();- for (const did of sample) {- multi.exists(`session:${project}:${did}`);- }- const results = await multi.exec();- for (const entry of results ?? []) {- if (Number(entry?.[1] ?? 0) === 0) orphans++;- }- }-- stats.push({ project, active, eligible, stuck, orphans, oldestAgeMin });- total += active;- totalEligible += eligible;- totalStuck += stuck;- totalOrphans += orphans;- }-- return { total, totalEligible, totalStuck, totalOrphans, stats };-}--async function inspectQueue() {- const [waiting, delayed, active, failed, completed] = await Promise.all([- sessionsQueue.getWaitingCount(),- sessionsQueue.getDelayedCount(),- sessionsQueue.getActiveCount(),- sessionsQueue.getFailedCount(),- sessionsQueue.getCompletedCount(),- ]);-- const failedSample = failed > 0 ? await sessionsQueue.getFailed(0, 4) : [];-- return { waiting, delayed, active, failed, completed, failedSample };-}--async function inspectBuffers() {- const redis = getRedisCache();- const [eventCounter, sessionCounter, eventListLen, sessionListLen] =- await Promise.all([- redis.get('buffer-counter:event'),- redis.get('buffer-counter:session'),- redis.llen('event-buffer'),- redis.llen('session-buffer'),- ]);-- return {- event: {- counter: Number(eventCounter ?? 0),- listLen: Number(eventListLen ?? 0),- },- session: {- counter: Number(sessionCounter ?? 0),- listLen: Number(sessionListLen ?? 0),- },- };-}--async function inspectClickhouse(- projects: string[] | null,- hours: number-): Promise<Map<string, ChCounts>> {- const projectClause = projects?.length- ? `AND project_id IN (${projects.map((p) => `'${p.replace(/'/g, "''")}'`).join(',')})`- : '';- const query = `- SELECT- project_id,- name,- sum(if(created_at > now() - INTERVAL 1 HOUR, 1, 0)) AS h1,- sum(if(created_at > now() - INTERVAL ${hours} HOUR, 1, 0)) AS hN- FROM events- WHERE name IN ('session_start', 'session_end')- AND created_at > now() - INTERVAL ${hours} HOUR- ${projectClause}- GROUP BY project_id, name- `;- const res = await ch.query({ query, format: 'JSONEachRow' });- const rows = await res.json<{- project_id: string;- name: string;- h1: string;- hN: string;- }>();-- const byProject = new Map<string, ChCounts>();- for (const r of rows) {- const cur = byProject.get(r.project_id) ?? {- starts1h: 0,- ends1h: 0,- starts24h: 0,- ends24h: 0,- };- if (r.name === 'session_start') {- cur.starts1h = Number(r.h1);- cur.starts24h = Number(r.hN);- } else {- cur.ends1h = Number(r.h1);- cur.ends24h = Number(r.hN);- }- byProject.set(r.project_id, cur);- }- return byProject;-}--async function inspectSessionEndClaims() {- const redis = getRedisCache();- let cursor = '0';- let count = 0;- do {- const [next, batch] = await redis.scan(- cursor,- 'MATCH',- 'session:end:emitted:*',- 'COUNT',- 1000- );- cursor = next;- count += batch.length;- } while (cursor !== '0' && count < 100_000);- return count;-}--async function sampleOldestSession(- projectId: string,- deviceId: string-): Promise<IClickhouseSession | null> {- const blob = await getRedisCache().get(`session:${projectId}:${deviceId}`);- return blob ? getSafeJson<IClickhouseSession>(blob) : null;-}--async function main() {- const args = process.argv.slice(2);- const verbose = args.includes('--verbose');- const projectIdx = args.indexOf('--project');- const projectFilter = projectIdx >= 0 ? args[projectIdx + 1] : null;- const hoursIdx = args.indexOf('--hours');- const hours = hoursIdx >= 0 ? Number(args[hoursIdx + 1]) : 24;-- const redis = getRedisCache();-- console.log('═'.repeat(70));- console.log('SESSION LIFECYCLE HEALTH CHECK');- console.log(`time: ${new Date().toISOString()}`);- console.log(`deadman: ${DEADMAN_MS / 60_000} min`);- console.log(`stuck cutoff: ${STUCK_CUTOFF_MS / 60_000} min (deadman + 3 reaper ticks)`);- console.log(`ch window: ${hours}h`);- if (projectFilter) console.log(`project: ${projectFilter}`);- console.log('═'.repeat(70));-- // 1. Redis inventory- const allProjects = await redis.smembers('session:projects');- const projects = projectFilter- ? [projectFilter]- : allProjects.slice().sort();-- header('REDIS — active sessions');- console.log(` ${allProjects.length} projects in session:projects`);-- const redisInfo = await inspectRedis(projects);- console.log(` ${redisInfo.total} total active sessions`);- console.log(` ${redisInfo.totalEligible} past deadman (expected to reap soon)`);- console.log(- ` ${redisInfo.totalStuck} past deadman + ${STUCK_GRACE_MS / 60_000}min (stuck — reaper is behind)`- );- console.log(- ` ${redisInfo.totalOrphans} wallclock entries with no blob (in sampled ${SAMPLE_SIZE_PER_PROJECT}/project)`- );-- if (verbose && redisInfo.stats.length > 0) {- console.log('');- console.log(- ` ${'project'.padEnd(40)} ${'active'.padStart(8)} ${'eligible'.padStart(10)} ${'stuck'.padStart(8)} ${'orphans'.padStart(8)} ${'oldest'.padStart(10)}`- );- for (const s of redisInfo.stats.sort((a, b) => b.active - a.active)) {- console.log(- ` ${s.project.padEnd(40)} ${fmt(s.active, 8)} ${fmt(s.eligible, 10)} ${fmt(s.stuck, 8)} ${fmt(s.orphans, 8)} ${s.oldestAgeMin.toFixed(1).padStart(8)}m`- );- }- }-- // 2. Sessions queue- header('SESSIONS QUEUE — createSessionEnd jobs');- const queue = await inspectQueue();- console.log(` waiting: ${queue.waiting}`);- console.log(` delayed: ${queue.delayed}`);- console.log(` active: ${queue.active}`);- console.log(` failed: ${queue.failed}`);- console.log(` completed: ${queue.completed} (removed-on-complete)`);- if (queue.failedSample.length > 0) {- console.log('');- console.log(' Latest failed jobs:');- for (const job of queue.failedSample) {- const when = new Date(job.timestamp).toISOString();- const reason = (job.failedReason ?? '').slice(0, 200);- console.log(` - [${when}] ${job.id}`);- console.log(` ${reason}`);- }- }-- // 3. Buffers- header('BUFFERS — pending writes to ClickHouse');- const buffers = await inspectBuffers();- console.log(- ` event buffer: counter=${buffers.event.counter} list=${buffers.event.listLen}`- );- console.log(- ` session buffer: counter=${buffers.session.counter} list=${buffers.session.listLen}`- );-- // 4. Dedup claims- header('IDEMPOTENCY — session:end:emitted:* claim keys');- const claimCount = await inspectSessionEndClaims();- console.log(- ` ${claimCount} active claim keys (TTL 2h; bounds at ~last 2h of closes)`- );-- // 5. ClickHouse reconciliation- header(`CLICKHOUSE — session_start vs session_end (last ${hours}h)`);- const chCounts = await inspectClickhouse(- projectFilter ? [projectFilter] : null,- hours- );-- let totalStarts1h = 0;- let totalEnds1h = 0;- let totalStartsN = 0;- let totalEndsN = 0;-- console.log('');- console.log(- ` ${'project'.padEnd(40)} ${'start 1h'.padStart(9)} ${'end 1h'.padStart(8)} ${'start ' + hours + 'h'}`.padEnd(58) +- `${('end ' + hours + 'h').padStart(8)} ${'open'.padStart(8)}`- );- for (const [project, c] of [...chCounts.entries()].sort()) {- const open = c.starts24h - c.ends24h;- console.log(- ` ${project.padEnd(40)} ${fmt(c.starts1h, 9)} ${fmt(c.ends1h, 8)} ${fmt(c.starts24h, 10)} ${fmt(c.ends24h, 8)} ${fmt(open, 8)}`- );- totalStarts1h += c.starts1h;- totalEnds1h += c.ends1h;- totalStartsN += c.starts24h;- totalEndsN += c.ends24h;- }- const openInCh = totalStartsN - totalEndsN;- console.log('');- console.log(- ` ${'TOTAL'.padEnd(40)} ${fmt(totalStarts1h, 9)} ${fmt(totalEnds1h, 8)} ${fmt(totalStartsN, 10)} ${fmt(totalEndsN, 8)} ${fmt(openInCh, 8)}`- );-- // 6. Reconciliation- header('RECONCILIATION — Redis active vs CH open');- const diff = redisInfo.total - openInCh;- const diffPct =- openInCh > 0 ? ((Math.abs(diff) / openInCh) * 100).toFixed(1) : '∞';- console.log(` Redis active sessions: ${redisInfo.total}`);- console.log(` CH starts - ends in ${hours}h window: ${openInCh}`);- console.log(` Diff: ${diff} (${diffPct}%)`);- console.log(- ` Note: diff > 0 means some "open" sessions in Redis were started before the ${hours}h window.`- );-- // 7. Sample the oldest session in Redis if anything looks suspicious- if (verbose && redisInfo.stats.some((s) => s.oldestAgeMin > 60 * 6)) {- header('SAMPLE — oldest session per project (any > 6h)');- for (const s of redisInfo.stats.filter((x) => x.oldestAgeMin > 60 * 6)) {- const wallclockKey = `session:wallclock:${s.project}`;- const [oldest] = await redis.zrange(wallclockKey, 0, 0);- if (oldest) {- const blob = await sampleOldestSession(s.project, oldest);- if (blob) {- console.log(` ${s.project} / ${oldest}`);- console.log(` id: ${blob.id}`);- console.log(` created_at: ${blob.created_at}`);- console.log(` ended_at: ${blob.ended_at}`);- console.log(` event_count: ${blob.event_count}`);- }- }- }- }-- // 8. Summary- header('SUMMARY');- const issues: string[] = [];- if (redisInfo.totalStuck > 0) {- issues.push(- `${redisInfo.totalStuck} sessions past the stuck cutoff — reaper may be unhealthy`- );- }- if (redisInfo.totalOrphans > 0) {- issues.push(- `${redisInfo.totalOrphans} orphan wallclock entries — cleanup() raced or vacuum is overdue`- );- }- if (queue.failed > 0) {- issues.push(`${queue.failed} failed session_end jobs`);- }- if (queue.waiting + queue.active > 5000) {- issues.push(- `session_end backlog (waiting=${queue.waiting} active=${queue.active})`- );- }- if (buffers.event.listLen > 50_000) {- issues.push(`event buffer is backed up (${buffers.event.listLen} items)`);- }- if (buffers.session.listLen > 50_000) {- issues.push(- `session buffer is backed up (${buffers.session.listLen} items)`- );- }- // The Redis-vs-CH diff is INFORMATIONAL, not an issue on its own:- // diff > 0: more in Redis than CH's window — older sessions still active- // diff < 0: more starts in CH than Redis active — legacy starts without ends- // (e.g. from before a bug fix) that will never close- // Only flag if both reaper is behind AND the diff is growing — which we can't- // know from a single run. So just print it; the reaper-behind and stuck- // counters above will catch live issues.-- if (issues.length === 0) {- console.log(' [OK] No issues detected.');- } else {- console.log(' [WARN] Issues:');- for (const i of issues) console.log(` - ${i}`);- }-- await redis.quit();- await ch.close?.();- process.exit(issues.length > 0 ? 1 : 0);-}--main().catch((e) => {- console.error(e);- process.exit(1);-});diff --git a/packages/db/scripts/drain-old-session-jobs.ts b/packages/db/scripts/drain-old-session-jobs.tsdeleted file mode 100644index 3fd539170..000000000--- a/packages/db/scripts/drain-old-session-jobs.ts+++ /dev/null@@ -1,79 +0,0 @@-// One-off cleanup: drain legacy `sessionEnd:{projectId}:{deviceId}` delayed-// jobs from the `sessions` queue. Run AFTER:-//-// 1. The new code is deployed (worker emits reaper-enqueued jobs with the-// `sessionEnd:v2:*` jobId prefix).-// 2. The `events.incoming-event` queue has been resumed and you've verified-// session metrics look healthy under the new flow.-//-// This removes only jobs whose jobId is `sessionEnd:*` and NOT `sessionEnd:v2:*`,-// leaving any reaper-enqueued v2 jobs intact. After this script runs you can-// safely resume the `sessions` queue.-//-// Idempotent: re-running on an already-drained queue is a no-op.-//-// Usage (from packages/db):-// pnpm with-env tsx scripts/drain-old-session-jobs.ts-// pnpm with-env tsx scripts/drain-old-session-jobs.ts --dry-run--import { sessionsQueue } from '@openpanel/queue';--async function main() {- const dryRun = process.argv.includes('--dry-run');-- console.log(- `[drain-old-session-jobs] starting${dryRun ? ' (DRY RUN — no removes)' : ''}`- );-- const jobs = await sessionsQueue.getJobs(['delayed', 'waiting']);-- const legacy: typeof jobs = [];- const v2: typeof jobs = [];-- for (const job of jobs) {- if (!job.id) continue;- if (job.id.startsWith('sessionEnd:v2:')) {- v2.push(job);- } else if (job.id.startsWith('sessionEnd:')) {- legacy.push(job);- }- }-- console.log(- `[drain-old-session-jobs] found ${jobs.length} jobs total ` +- `(${legacy.length} legacy, ${v2.length} v2, will keep v2)`- );-- if (dryRun) {- console.log('[drain-old-session-jobs] dry run — exiting');- process.exit(0);- }-- let removed = 0;- let failed = 0;-- for (const job of legacy) {- try {- await job.remove();- removed++;- if (removed % 1000 === 0) {- console.log(`[drain-old-session-jobs] removed ${removed}/${legacy.length}`);- }- } catch (error) {- failed++;- console.error('[drain-old-session-jobs] failed to remove job', job.id, error);- }- }-- console.log('[drain-old-session-jobs] done', {- removed,- failed,- kept_v2: v2.length,- });- process.exit(0);-}--main().catch((error) => {- console.error('[drain-old-session-jobs] fatal error:', error);- process.exit(1);-});diff --git a/packages/db/scripts/migrate-sessions.ts b/packages/db/scripts/migrate-sessions.tsdeleted file mode 100644index baf33da6d..000000000--- a/packages/db/scripts/migrate-sessions.ts+++ /dev/null@@ -1,121 +0,0 @@-// One-off migration: convert legacy `session:{sessionId}` Redis blobs +-// delayed BullMQ `sessionEnd:{projectId}:{deviceId}` jobs into the new-// session-buffer key scheme:-//-// session:{projectId}:{deviceId} -> full session JSON blob (no TTL)-// session:profile:{projectId}:{profileId} -> deviceId pointer (no TTL)-// session:wallclock:{projectId} -> ZSET scored by wall-clock (ms)-// session:projects -> SET of project_ids with active sessions-//-// Source of truth: the delayed `sessionsQueue` jobs — one per (projectId,-// deviceId) pair currently considered "active" by the legacy system. For-// each, we read the corresponding `session:{sessionId}` blob (still alive-// under its 1h TTL) and rewrite it into the new structure.-//-// Idempotent: re-running the script is safe. Existing new-scheme keys are-// preserved via SET NX semantics inside the multi pipeline.-//-// Usage (from packages/db, env loaded from repo root .env):-// pnpm with-env tsx scripts/migrate-sessions.ts-// pnpm with-env tsx scripts/migrate-sessions.ts --dry-run-//-// Runbook step 2: pause `events.incoming-event` AND `sessions` queues first,-// wait for in-flight jobs to drain (~30s), THEN run this script. Resume-// `events.incoming-event` after deploy; keep `sessions` paused until step 7-// (drain-old-session-jobs.ts) finishes.--import type { IClickhouseSession } from '@openpanel/core';-import { getSafeJson } from '@openpanel/json';-import {- type EventsQueuePayloadCreateSessionEnd,- sessionsQueue,-} from '@openpanel/queue';-import { getRedisCache } from '@openpanel/redis';--const sessionKey = (projectId: string, deviceId: string) =>- `session:${projectId}:${deviceId}`;-const profileIndexKey = (projectId: string, profileId: string) =>- `session:profile:${projectId}:${profileId}`;-const wallclockSetKey = (projectId: string) =>- `session:wallclock:${projectId}`;-const PROJECTS_SET_KEY = 'session:projects';--async function main() {- const dryRun = process.argv.includes('--dry-run');- const redis = getRedisCache();-- console.log(- `[migrate-sessions] starting${dryRun ? ' (DRY RUN — no writes)' : ''}`- );-- const jobs = await sessionsQueue.getJobs(['delayed', 'waiting']);- console.log(`[migrate-sessions] found ${jobs.length} legacy session jobs`);-- const counts = {- migrated: 0,- blobMissing: 0,- alreadyMigrated: 0,- parseFailed: 0,- };-- for (const job of jobs) {- const data = job.data as EventsQueuePayloadCreateSessionEnd | undefined;- const payload = data?.payload;- if (!payload?.projectId || !payload.deviceId || !payload.sessionId) {- counts.parseFailed++;- continue;- }- const { projectId, deviceId, sessionId } = payload;-- const existsAlready = await redis.exists(sessionKey(projectId, deviceId));- if (existsAlready) {- counts.alreadyMigrated++;- continue;- }-- const blob = await redis.get(`session:${sessionId}`);- if (!blob) {- counts.blobMissing++;- continue;- }-- const session = getSafeJson<IClickhouseSession>(blob);- if (!session) {- counts.parseFailed++;- continue;- }-- const nowMs = Date.now();-- if (dryRun) {- counts.migrated++;- continue;- }-- const multi = redis.multi();- multi.set(sessionKey(projectId, deviceId), blob);- multi.zadd(wallclockSetKey(projectId), nowMs.toString(), deviceId);- multi.sadd(PROJECTS_SET_KEY, projectId);- if (session.profile_id && session.profile_id !== deviceId) {- multi.set(profileIndexKey(projectId, session.profile_id), deviceId);- }- await multi.exec();-- counts.migrated++;-- if (counts.migrated % 1000 === 0) {- console.log(- `[migrate-sessions] progress: ${counts.migrated} migrated`- );- }- }-- console.log('[migrate-sessions] done', counts);- await redis.quit();- process.exit(0);-}--main().catch((error) => {- console.error('[migrate-sessions] fatal error:', error);- process.exit(1);-});diff --git a/packages/queue/index.ts b/packages/queue/index.tsindex 3dc13f492..197f47f0a 100644--- a/packages/queue/index.ts+++ b/packages/queue/index.ts@@ -1,4 +1,11 @@-export * from './src/queues';-export type * from './src/queues';-export * from './src/kafka';+// The Kafka transport and the notification dispatch left this package at+// M11-003 for @openpanel/core (modules/ingest/src/kafka.ts and+// modules/notification/src/notification-dispatch.ts). What stays is V1's+// BullMQ queue definitions and the buffer singleton built on `cronQueue`,+// both still imported by packages/trpc's routers — a package this task's+// scope does not cover, so their deletion waits for the follow-up that may+// touch those call sites (see docs/TECH_DEBT.md).+export type { JobsOptions } from 'bullmq';+export type * from './src/queues';+export * from './src/queues';diff --git a/packages/queue/src/buffers.ts b/packages/queue/src/buffers.tsindex 4e652fb6c..a313cca3e 100644--- a/packages/queue/src/buffers.ts+++ b/packages/queue/src/buffers.ts@@ -3,21 +3,22 @@// (M8-001): `createBuffers(deps)` builds them once and V2's main.ts hangs the// result on `AppDeps.buffers` (ADR-007). This file is the ONE process-wide// singleton instance every call site that predates that ctx wiring still-// reaches — packages/trpc's overview/widget routers and the several-// `@openpanel/core` modules that lazy-import this specifier-// (`loadDbBuffers`) rather than threading `ctx.buffers` through. Repointing-// those call sites onto `ctx.buffers` is the larger, separately-scoped-// follow-up docs/TECH_DEBT.md tracks (the 93 lazy db imports bypassing-// ctx/requestId) — not this task.+// reaches. Since M11-003 that is packages/trpc's overview/widget routers+// alone — core's last lazy importer (widget.rpc.ts, via `loadDbBuffers`)+// now reads `ctx.buffers`. Repointing the two remaining trpc call sites is+// the larger, separately-scoped follow-up docs/TECH_DEBT.md tracks (the 93+// lazy db imports bypassing ctx/requestId) — not this task, whose scope+// stops at packages/queue.//// Lives in @openpanel/queue rather than @openpanel/core because it needs// `cronQueue` (ADR-005's BullMQ escape hatch: pausing `cron` from bull-board// halts all buffer flushing, preserved deliberately — docs/ANSWERS.md §3:// "known!"), and core must not import `@openpanel/queue` back without a real-// package cycle (see notification-dispatch.ts). `@openpanel/queue` already-// depends on `@openpanel/core`, so this is the one side of that boundary free-// of a cycle. Also keeps core's own rule intact: buffers are boot singletons-// on AppDeps, never a module-level singleton in core itself (M8-005).+// package cycle (see queues.ts, which imports core for its logger).+// `@openpanel/queue` already depends on `@openpanel/core`, so this is the one+// side of that boundary free of a cycle. Also keeps core's own rule intact:+// buffers are boot singletons on AppDeps, never a module-level singleton in+// core itself (M8-005).import { createBuffers, createLogger } from '@openpanel/core';import { cronQueue } from './queues';