Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M11-003): packages/queue into core: Kafka transport, notification dispatch and the legacy queue definitions; main.ts imports none of it from @openpanel/queue

ralph3deada3955b3
diff --git a/apps/api/e2e/lag-monitor.ts b/apps/api/e2e/lag-monitor.ts
index 3925cb876..904e391eb 100644
--- a/apps/api/e2e/lag-monitor.ts
+++ b/apps/api/e2e/lag-monitor.ts
@@ -1,8 +1,9 @@
/**
* Kafka consumer-group lag sampling for the stress harness — the backpressure
* story a throughput number alone hides. Wraps the admin-API primitives in
- * `@openpanel/queue`; a fast API can outrun its consumer for a while without
- * a single failed request, and this is the only place that would show it.
+ * `@openpanel/core`'s ingest module; a fast API can outrun its consumer for a
+ * while without a single failed request, and this is the only place that would
+ * show it.
*/
import {
@@ -12,7 +13,7 @@ import {
KAFKA_CONSUMER_GROUP,
KAFKA_EVENTS_TOPIC,
sampleConsumerGroupLag,
-} from '@openpanel/queue';
+} from '@openpanel/core';
export interface LagSummary {
sampleCount: number;
diff --git a/apps/api/e2e/legacy-job-proof.ts b/apps/api/e2e/legacy-job-proof.ts
index fb4100b47..4227508dc 100644
--- a/apps/api/e2e/legacy-job-proof.ts
+++ b/apps/api/e2e/legacy-job-proof.ts
@@ -40,7 +40,6 @@ import {
queues,
startWorkers,
} from '@openpanel/core';
-import { produceIncomingEvent } from '@openpanel/queue';
import { Redis } from '@openpanel/redis';
import pino from 'pino';
@@ -57,7 +56,7 @@ interface LegacyCase {
queue: QueueKey;
/** The BullMQ job name V1 put on the wire. */
wireName: string;
- /** V1's job data, verbatim (packages/queue/src/queues.ts). */
+ /** V1's job data, verbatim (was packages/queue/src/queues.ts). */
data: unknown;
/** The V2 job the compat hook must resolve it to. */
expectJob: string;
@@ -286,7 +285,6 @@ async function main() {
clients: undefined as unknown as AppDeps['clients'],
buffers,
producers,
- produceIncomingEvent,
logger,
config: { selfHosted: false },
};
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index df234d1db..f03cfdb7e 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -34,16 +34,19 @@ import { createHmac } from 'node:crypto';
import {
type AppDeps,
appRouter,
+ assertKafkaConfigured,
BULL_BOARD_BASE_PATH,
type BufferDeps,
bullBoardRoutes,
COOKIE_OPTIONS,
+ checkNotificationRulesForEvent,
clearProjectByIdCache,
corsDelegator,
createBuffers,
createClients,
createIncomingEventHandler,
createInitialSalts,
+ createKafkaEventsConsumer,
createProducers,
createTrpcFetchHandler,
dashboardRoutes,
@@ -53,9 +56,16 @@ import {
getProjectByIdCached,
type HttpCtx,
ingestConsumerMetrics,
+ KAFKA_EVENTS_TOPIC,
+ KAFKA_HANDLER_MAX_ATTEMPTS,
+ KAFKA_HANDLER_RETRY_INITIAL_MS,
+ KAFKA_HANDLER_RETRY_MAX_MS,
+ KAFKA_PARTITIONS_CONCURRENT,
type KafkaConsumerHandle,
+ kafkaLogger,
markEventsActivity,
opsRoutes,
+ produceDeadLetterEvent,
publicApiRoutes,
type QueueDefinition,
type QueueProducerHandle,
@@ -79,19 +89,6 @@ import {
} from '@openpanel/core';
import { ch } from '@openpanel/db/src/clickhouse/client';
import { db } from '@openpanel/db/src/prisma-client';
-import {
- assertKafkaConfigured,
- createKafkaEventsConsumer,
- KAFKA_EVENTS_TOPIC,
- KAFKA_HANDLER_MAX_ATTEMPTS,
- KAFKA_HANDLER_RETRY_INITIAL_MS,
- KAFKA_HANDLER_RETRY_MAX_MS,
- KAFKA_PARTITIONS_CONCURRENT,
- kafkaLogger,
- produceDeadLetterEvent,
- produceIncomingEvent,
-} from '@openpanel/queue';
-import { checkNotificationRulesForEvent } from '@openpanel/queue/src/notification-dispatch';
import { getRedisCache, getRedisPub, getRedisQueue } from '@openpanel/redis';
import { Elysia } from 'elysia';
import pino from 'pino';
@@ -206,9 +203,6 @@ function buildDeps(): AppDeps {
clients: createClients(),
buffers: createBuffers(bufferDeps(producers)),
producers,
- // @openpanel/queue's Kafka producer, injected because core cannot import
- // it back (M8-002). It dies with that package at M9-003.
- produceIncomingEvent,
logger,
config: { selfHosted: config.SELF_HOSTED },
};
@@ -277,9 +271,9 @@ function bootServiceDeps(
/**
* The Kafka events consumer. The kafkajs client, the topic, the consumer
- * group, the DLQ producer and the retry bounds all come from
- * `@openpanel/queue`, which is what keeps those names byte-identical across
- * the cutover — core never spells one (see modules/ingest/src/consumer.ts).
+ * group, the DLQ producer and the retry bounds all come from core's own
+ * `modules/ingest/src/kafka.ts` (M11-003) — still passed in as arguments, so
+ * `consumer.ts` spells none of those names itself.
*/
async function startIngestConsumer(
deps: AppDeps
@@ -296,9 +290,9 @@ async function startIngestConsumer(
batch: {
// One Ctx per message, scoped to the requestId the producer stamped
// into the envelope (M10-006). The two bindings a work scope cannot
- // supply are resolved here, once: the notification dispatch lives in
- // @openpanel/queue (core cannot import it back) and the project cache
- // is the boot-registered singleton ingest/http/mcp share.
+ // supply are resolved here, once: the notification dispatch has no Ctx
+ // slot in its signature, and the project cache is the boot-registered
+ // singleton ingest/http/mcp share.
handleEvent: createIncomingEventHandler(deps, {
checkNotificationRulesForEvent,
getCachedProject: getProjectByIdCached,
diff --git a/docs/TECH_DEBT.md b/docs/TECH_DEBT.md
index 1a16395d0..188ac999c 100644
--- a/docs/TECH_DEBT.md
+++ b/docs/TECH_DEBT.md
@@ -992,3 +992,119 @@ existed to leak.
implement one.
- `verification/harness stop`, then `verification/full.sh` — green, with the
documented pre-P11 `contracts/sdk/dist-gate.sh` exception.
+
+### 2026-09-06 — M11-003, `packages/queue`'s Kafka transport and notification dispatch move into core; `queues.ts` / `buffers.ts` do NOT (out of scope)
+
+Ran by: ralph (M11-003 implement task).
+
+**Survey, before the move** — every import of `@openpanel/queue` outside
+`packages/queue` (`grep -rnE "from '@openpanel/queue" --include=*.ts`):
+
+| File | Symbols |
+|---|---|
+| `apps/api/src/main.ts` | `assertKafkaConfigured`, `createKafkaEventsConsumer`, `KAFKA_EVENTS_TOPIC`, `KAFKA_HANDLER_MAX_ATTEMPTS`, `KAFKA_HANDLER_RETRY_INITIAL_MS`, `KAFKA_HANDLER_RETRY_MAX_MS`, `KAFKA_PARTITIONS_CONCURRENT`, `kafkaLogger`, `produceDeadLetterEvent`, `produceIncomingEvent` (10, from the barrel) + `checkNotificationRulesForEvent` (deep, `@openpanel/queue/src/notification-dispatch`) |
+| `apps/api/e2e/lag-monitor.ts` | `createKafkaAdmin`, `disconnectKafka`, `KAFKA_BROKERS`, `KAFKA_CONSUMER_GROUP`, `KAFKA_EVENTS_TOPIC`, `sampleConsumerGroupLag` |
+| `apps/api/e2e/legacy-job-proof.ts` | `produceIncomingEvent` |
+| `packages/core/src/modules/session/session.jobs.ts` | `checkNotificationRulesForSessionEnd` (lazy `import('@openpanel/queue/src/notification-dispatch')`) |
+| `packages/core/src/buffers/lazy-db-buffers.ts` | the buffer singletons (lazy `import('@openpanel/queue/src/buffers')`), reached only by `modules/widget/widget.rpc.ts` |
+| `packages/db/scripts/check-sessions.ts` | `sessionsQueue` |
+| `packages/db/scripts/drain-old-session-jobs.ts` | `sessionsQueue` |
+| `packages/db/scripts/migrate-sessions.ts` | `sessionsQueue`, `EventsQueuePayloadCreateSessionEnd` |
+| `packages/trpc/src/routers/cohort.ts` | `cohortComputeQueue` |
+| `packages/trpc/src/routers/gsc.ts` | `gscQueue` |
+| `packages/trpc/src/routers/import.ts` | `importQueue` |
+| `packages/trpc/src/routers/overview.ts` | `eventBuffer` (deep, `@openpanel/queue/src/buffers`) |
+| `packages/trpc/src/routers/widget.ts` | `eventBuffer` (deep, `@openpanel/queue/src/buffers`) |
+
+**What moved.** `src/kafka.ts` → `packages/core/src/modules/ingest/src/kafka.ts`
+and `src/notification-dispatch.ts` →
+`packages/core/src/modules/notification/src/notification-dispatch.ts`. Both are
+the same file: `diff` against `HEAD` shows only repointed imports, plus two
+edits with reasons —
+
+- `DeadLetterMessage` was declared twice (once in `kafka.ts`, once in
+ `consumer.ts`, structurally identical by design); now that they are siblings
+ `kafka.ts` imports the consumer's.
+- `produceIncomingEvent`'s parameter changed from
+ `EventsQueuePayloadIncomingEvent['payload']` to core's own
+ `IncomingEventPayload`. The wire format is unchanged: a temporary
+ `[A] extends [B] ? [B] extends [A]` assertion compiled clean in both
+ directions against `@openpanel/queue`'s type.
+- `triggerNotification` enqueues through core's registry
+ (`queues.notification.sendNotification`) rather than a second
+ `new Queue('notification')` — same Redis key, same job name, same
+ `{payload, meta}` envelope (`golden/queue-keys/check.sh` still matches the
+ V1 goldens), and it became `async` because the registry producer is.
+
+Every Kafka constant is byte-identical after the move — topic
+(`process.env.KAFKA_EVENTS_TOPIC || 'events'`), DLQ
+(`|| \`${KAFKA_EVENTS_TOPIC}-dlq\``), group
+(`|| 'openpanel-events'`), and the 15 `Number.parseInt` bounds — verified by
+`diff -u` of the two files' bodies.
+
+**The `AppDeps.produceIncomingEvent` injection is retired.** It existed only
+because `@openpanel/queue` imports `@openpanel/core` for its logger, so core
+could not import the producer back (`context.ts`'s own comment). With
+`kafka.ts` in core, `ingest.routes.ts` imports the sibling directly. The
+producer stays an *argument* to `ingest.service.ts` — that seam is now about
+letting a test assert what was produced without a broker, not about a package
+cycle, and `ingest.service.test.ts` / `legacy-event.test.ts` still use it.
+
+**`buffers/lazy-db-buffers.ts` is deleted**; its one core caller
+(`widget.rpc.ts`'s `loadEventBuffer`) reads `ctx.buffers.event`.
+
+**The three orphan `packages/db/scripts` are deleted** — `check-sessions.ts`,
+`drain-old-session-jobs.ts`, `migrate-sessions.ts`, all V1 session-migration
+tooling. `grep -rn 'check-sessions\|drain-old-session-jobs\|migrate-sessions'`
+over `*.json`/`*.ts`/`*.md`/`*.sh` returns nothing outside the files
+themselves; `packages/db/package.json`'s only `scripts/*` entry is
+`duplicate-events` → `find-duplicate-events.ts`, untouched.
+
+**NOT done, and why — `packages/queue/src/queues.ts` and `src/buffers.ts` stay.**
+The task asked for both to be deleted. They cannot be, inside this task's
+declared scope (`packages/core/**`, `packages/queue/**`,
+`packages/db/scripts/**`, `apps/api/**`, `docs/**`): five
+`packages/trpc/src/routers/*` files import them (the last five rows of the
+survey above), and `packages/trpc/**` is not in scope. Deleting the two files
+without touching those routers fails `pnpm run typecheck`; touching them is
+the out-of-scope edit. **This needs either a scope extension to
+`packages/trpc/src/routers/{cohort,gsc,import,overview,widget}.ts` or a
+follow-up task**, and it is not a mechanical repoint — the three queue call
+sites become `ctx.services.cohort.enqueueCompute` / `ctx.queues.gsc...` /
+`ctx.services.import.enqueue`, which changes `import.ts`'s `jobId` handling
+and `cohort.ts`'s procedure signatures. `packages/queue/index.ts` no longer
+re-exports `./src/kafka`; what remains behind the barrel is exactly what trpc
+still consumes.
+
+Also still declared, unused: `@openpanel/queue` in `apps/api/package.json`,
+`packages/core/package.json` and `packages/db/package.json` — no file in any
+of the three imports it any more. Removing the manifest entries belongs with
+M11-004's deletion of the package.
+
+**Verification, all run by ralph on 2026-09-06:**
+
+- `pnpm run typecheck` — all 25 workspaces, `Done`.
+- `pnpm test` — 13 files, **205 passed**.
+- `cd packages/core && bun test` (bare) — **1466 pass, 12 skip, 0 fail**,
+ `Ran 1478 tests across 151 files. [12.59s]`, `real 0m12.652s`, and it
+ EXITS: moving `kafka.ts` into core constructs no Kafka client at import
+ time (`getKafka()` is lazy; only `assertKafkaConfigured` reads
+ `KAFKA_BROKERS`, and nothing calls it at module scope).
+- `pnpm run check:deps` — `no dependency violations found (2321 modules,
+ 10072 dependencies cruised)`, both rules still at `error`.
+- `bash apps/api/e2e/boot-proof.sh` — all checks passed, all three ROLEs.
+- `verification/golden/queue-keys/check.sh` — `OK - 7x2 queue keys and 20
+ scheduler ids derived from the V2 registry match the V1 goldens`.
+- `verification/harness start` → `cd apps/api && pnpm run e2e:sessions`:
+ **29/29 checks passed** (Kafka produce → consume is the path under test) →
+ `pnpm run e2e:legacy-jobs`: **all checks passed** → `harness stop`.
+- `verification/full.sh` — `FULL: green`.
+
+One incidental find, recorded not fixed: `usage.ts`'s static
+`import { getOrganizationBillingEventsCount } from '../../../v1-compat'` had
+to become part of the existing lazy `loadProjectService()` call. `jobs.registry.ts`
+reaches this file (`session.jobs.ts` → `session-end` → `usage`), and that one
+static edge pulls `services.ts` and all 36 services into the registry's import
+graph, which builds a ClickHouse client at import time — `queue-keys/check.sh`
+fails outright without the change (confirmed by reverting the file alone and
+re-running it).
diff --git a/packages/core/src/buffers/lazy-db-buffers.ts b/packages/core/src/buffers/lazy-db-buffers.ts
deleted file mode 100644
index 3f9d1f747..000000000
--- a/packages/core/src/buffers/lazy-db-buffers.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-// The seven buffer singletons are built once by @openpanel/queue's
-// src/buffers.ts (moved from @openpanel/db, M9-CLEANUP-001 — see that file's
-// header for why @openpanel/queue and not here) and, once V2 owns boot,
-// handed to AppDeps by createBuffers directly. Until then, every core module
-// that needs a buffer ahead of that boot wiring reaches it through THIS one
-// dynamic import() — one edge, one file, same reasoning as
-// session.service.ts's loadProfileService comment.
-export function loadDbBuffers() {
- return import('@openpanel/queue/src/buffers');
-}
diff --git a/packages/core/src/context.test.ts b/packages/core/src/context.test.ts
index fd25e8e56..2125f9dcb 100644
--- a/packages/core/src/context.test.ts
+++ b/packages/core/src/context.test.ts
@@ -81,7 +81,6 @@ function stubDeps(): {
return surface;
},
},
- produceIncomingEvent: () => Promise.resolve(),
logger: stubLogger(),
config: { selfHosted: true },
},
diff --git a/packages/core/src/context.ts b/packages/core/src/context.ts
index 4bc1b3681..7e464bf4a 100644
--- a/packages/core/src/context.ts
+++ b/packages/core/src/context.ts
@@ -8,7 +8,6 @@ import type { Buffers } from './buffers/create-buffers';
import type { QueueProducerHandle, QueueProducers } from './jobs.registry';
import type { Logger } from './logger';
import type { SessionValidationResult } from './modules/auth/src/login-session';
-import type { IncomingEventProducer } from './modules/ingest/src/incoming-event';
import { createServices, type Services } from './services';
import type { CookieJar, CookieOptions } from './shared/cookie';
@@ -53,14 +52,6 @@ export interface AppDeps {
clients: ServiceClients;
buffers: Buffers;
producers: QueueProducerHandle;
- /**
- * The events transport. `produceIncomingEvent` still lives in
- * @openpanel/queue, which imports core for its logger, so core cannot
- * import it back (M8-002); main.ts supplies it. It stays injected until
- * @openpanel/queue dies at P9 — M8-003 moved the consumer, not the
- * producer, and the consumer takes its kafkajs client the same way.
- */
- produceIncomingEvent: IncomingEventProducer;
logger: Logger;
config: RuntimeFlags;
}
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 8baa54df4..610009416 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -533,15 +533,6 @@ export {
extractToken,
handleStatelessMcpRequest,
} from './modules/mcp/mcp.service';
-// Dissolved from @openpanel/db's services/notification.service.ts (M6-005,
-// "rules + dispatch stay together") — packages/trpc's notification router
-// and apps/worker's notification job call these directly, the same way V1
-// reaches every other dissolved service here. `createNotification` /
-// `triggerNotification` / `checkNotificationRulesForEvent` /
-// `checkNotificationRulesForSessionEnd` — the BullMQ-producer orchestration
-// around a rule match — moved to `@openpanel/queue/src/notification-dispatch.ts`
-// (M9-CLEANUP-001, packages/db's final-surface shrink), not here: core still
-// cannot import `@openpanel/queue` back without a real package cycle.
export type {
INotificationPayload,
INotificationRuleCached,
@@ -561,6 +552,14 @@ export {
notificationTemplateEvent,
notificationTemplateFunnel,
} from './modules/notification/notification.service';
+// Dissolved from @openpanel/db's services/notification.service.ts (M6-005,
+// "rules + dispatch stay together"). `createNotification` /
+// `triggerNotification` / `checkNotificationRulesForEvent` /
+// `checkNotificationRulesForSessionEnd` — the BullMQ-producer orchestration
+// around a rule match — came home from @openpanel/queue at M11-003 and now
+// live in this module's own src/notification-dispatch.ts. Only the one the
+// Kafka message handler is bound to at boot needs to leave the package.
+export { checkNotificationRulesForEvent } from './modules/notification/src/notification-dispatch';
// New module (M6-003) — the onboarding-project mutation and the onboarding
// email drip, neither of which had a packages/db/src/services/* home to move
// from. packages/trpc's onboarding router and apps/worker's onboarding cron
@@ -882,10 +881,10 @@ export {
summarizeBotSignals,
validateIngestRequest,
} from './modules/ingest/ingest.service';
-// The Kafka consumer and its per-message handler (M8-003). apps/worker's
-// events.kafka-consumer.ts / events.incoming-event.ts are thin delegates that
-// hand in the kafkajs client, the topic/group names and their own registry's
-// counters — none of which this package spells, so they stay byte-identical.
+// The Kafka consumer and its per-message handler (M8-003), which take the
+// kafkajs client, the topic/group names and the retry bounds as arguments —
+// the injection seam that used to cross a package boundary and now just
+// crosses two files.
export type {
ConsumerLogger,
ConsumerMetrics,
@@ -921,6 +920,45 @@ export {
incomingEvent,
} from './modules/ingest/src/incoming-event-handler';
export { ingestConsumerMetrics } from './modules/ingest/src/ingest.metrics';
+// The Kafka transport itself (M11-003): producer, consumer factory, admin/lag
+// sampling and every topic/group/retry constant, moved from
+// @openpanel/queue's kafka.ts unchanged. It constructs no client at import
+// time, so the barrel stays offline-importable.
+export type {
+ Admin,
+ ConsumerGroupLag,
+ EachBatchPayload,
+ KafkaMessage,
+ PartitionLag,
+} from './modules/ingest/src/kafka';
+export {
+ assertKafkaConfigured,
+ createKafkaAdmin,
+ createKafkaEventsConsumer,
+ disconnectKafka,
+ KAFKA_BROKERS,
+ KAFKA_CONNECTION_TIMEOUT_MS,
+ KAFKA_CONSUMER_GROUP,
+ KAFKA_EVENTS_DLQ_TOPIC,
+ KAFKA_EVENTS_TOPIC,
+ KAFKA_HANDLER_MAX_ATTEMPTS,
+ KAFKA_HANDLER_RETRY_INITIAL_MS,
+ KAFKA_HANDLER_RETRY_MAX_MS,
+ KAFKA_HEARTBEAT_INTERVAL_MS,
+ KAFKA_MAX_MESSAGES_PER_PARTITION,
+ KAFKA_MAX_WAIT_MS,
+ KAFKA_MIN_MESSAGES,
+ KAFKA_PARTITIONS_CONCURRENT,
+ KAFKA_PRODUCER_INITIAL_RETRY_MS,
+ KAFKA_PRODUCER_MAX_RETRY_MS,
+ KAFKA_PRODUCER_RETRIES,
+ KAFKA_REQUEST_TIMEOUT_MS,
+ KAFKA_SESSION_TIMEOUT_MS,
+ kafkaLogger,
+ produceDeadLetterEvent,
+ produceIncomingEvent,
+ sampleConsumerGroupLag,
+} from './modules/ingest/src/kafka';
export type {
DailyInsightCandidate,
GetReferrerSpikesInput,
diff --git a/packages/core/src/jobs/workers.test.ts b/packages/core/src/jobs/workers.test.ts
index 691076498..438615cf9 100644
--- a/packages/core/src/jobs/workers.test.ts
+++ b/packages/core/src/jobs/workers.test.ts
@@ -30,7 +30,6 @@ function stubDeps(): AppDeps {
clients: {} as AppDeps['clients'],
buffers: {} as Buffers,
producers: createRecordingProducers(queues),
- produceIncomingEvent: () => Promise.resolve(),
logger: stubLogger(),
config: { selfHosted: true },
};
diff --git a/packages/core/src/modules/ingest/ingest.routes.ts b/packages/core/src/modules/ingest/ingest.routes.ts
index 5f98ad6b1..956b4a2f3 100644
--- a/packages/core/src/modules/ingest/ingest.routes.ts
+++ b/packages/core/src/modules/ingest/ingest.routes.ts
@@ -43,6 +43,10 @@ import {
} from './ingest.service';
import { toIngestHeaders } from './src/headers';
import { recordLegacyEventRequest } from './src/ingest.metrics';
+// The Kafka producer is a sibling now (M11-003), not an `AppDeps` field:
+// kafka.ts moved into this module, so there is nothing left for main.ts to
+// inject. It still constructs no client at import time.
+import { produceIncomingEvent } from './src/kafka';
const TAGS = ['Track'];
const LEGACY_EVENT_TAGS = ['Event'];
@@ -174,7 +178,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>
},
{
buffers: ctx.buffers,
- produceIncomingEvent: deps.produceIncomingEvent,
+ produceIncomingEvent,
deps: ctx,
}
);
@@ -244,7 +248,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>
},
{
buffers: ctx.buffers,
- produceIncomingEvent: deps.produceIncomingEvent,
+ produceIncomingEvent,
deps: ctx,
}
);
diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.ts
index ca1180a2d..abedc9af8 100644
--- a/packages/core/src/modules/ingest/ingest.service.ts
+++ b/packages/core/src/modules/ingest/ingest.service.ts
@@ -8,11 +8,10 @@
// codes and bodies, so the pipeline returns a `TrackOutcome` and V1 maps it
// back onto the exact `HttpError`s and replies it produced before.
//
-// The Kafka producer is INJECTED. `produceIncomingEvent` still lives in
-// @openpanel/queue, which imports @openpanel/core for its logger — core
-// cannot import it back without a real package cycle (see
-// cohort.service.ts's header). M8-003 moves the producer into core and the
-// injection point becomes an AppDeps field.
+// The Kafka producer is INJECTED. Since M11-003 `produceIncomingEvent` is a
+// sibling (./src/kafka.ts), so the injection is no longer a package-cycle
+// workaround and no longer travels through `AppDeps` — it stays an argument
+// only so a test can assert on what was produced without a broker.
import { generateId } from '@openpanel/common';
import { assocPath, pathOr, pick } from 'ramda';
diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.ts
index 7408baa29..a2fa53277 100644
--- a/packages/core/src/modules/ingest/src/consumer.ts
+++ b/packages/core/src/modules/ingest/src/consumer.ts
@@ -4,10 +4,9 @@
// bounded in-consumer retry, and a dead-letter topic on exhaustion.
//
// The kafkajs client, the topic/consumer-group names, the DLQ producer and
-// the retry bounds are all INJECTED. They still live in @openpanel/queue,
-// which imports @openpanel/core for its logger, so core cannot import it back
-// without a package cycle (see ingest.service.ts's header). Injection is also
-// what keeps those names byte-identical: this file never spells one.
+// the retry bounds are all INJECTED, and stay so now that they live one
+// directory away in ./kafka.ts (M11-003): injection is what keeps those names
+// byte-identical, because this file never spells one.
import type {
Consumer,
@@ -32,10 +31,10 @@ const RETRY_BACKOFF_FACTOR = 2;
export type DeadLetterReason = 'parse_error' | 'handler_error';
/**
- * What the consumer hands to the dead-letter producer. Structurally identical
- * to @openpanel/queue's `DeadLetterMessage`: the value stays the producer's
- * original bytes, so a DLQ message can be replayed onto the events topic
- * unchanged.
+ * What the consumer hands to the dead-letter producer (./kafka.ts's
+ * `produceDeadLetterEvent` is the one implementation): the value stays the
+ * producer's original bytes, so a DLQ message can be replayed onto the events
+ * topic unchanged.
*/
export interface DeadLetterMessage {
key: Buffer | null;
diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.ts
index 848dd64f6..e701837b9 100644
--- a/packages/core/src/modules/ingest/src/incoming-event-handler.ts
+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.ts
@@ -101,9 +101,9 @@ export interface IncomingEventDeps {
*/
export interface IncomingEventBindings {
/**
- * BullMQ-producer orchestration, which lives in `@openpanel/queue` —
- * a package that imports `@openpanel/core` back, so core cannot reach it
- * (notification.service.ts's header).
+ * BullMQ-producer orchestration (notification/src/notification-dispatch.ts).
+ * Injected rather than imported so the hot path's per-message handler keeps
+ * the notification module out of its own import graph.
*/
checkNotificationRulesForEvent(
payload: IServiceCreateEventPayload
diff --git a/packages/core/src/modules/ingest/src/incoming-event.ts b/packages/core/src/modules/ingest/src/incoming-event.ts
index 5a80de335..be476e2e5 100644
--- a/packages/core/src/modules/ingest/src/incoming-event.ts
+++ b/packages/core/src/modules/ingest/src/incoming-event.ts
@@ -2,9 +2,9 @@
//
// This is a WIRE FORMAT, not an internal type: a rolling deploy overlaps a V1
// producer with a V2 consumer on the same topic, so it stays byte-compatible
-// with @openpanel/queue's `EventsQueuePayloadIncomingEvent['payload']`
-// (ADR-004). A leaf file so `context.ts` can name the producer without
-// reaching into the ingest service.
+// with V1's `EventsQueuePayloadIncomingEvent['payload']` (ADR-004). A leaf
+// file, so both ./kafka.ts's producer and the consumer can name the payload
+// without reaching into the ingest service.
import type { GeoLocation } from '../../../clients/geo';
import type { parseUserAgent } from '../../../shared/parser-user-agent';
diff --git a/packages/queue/src/kafka.ts b/packages/core/src/modules/ingest/src/kafka.ts
similarity index 90%
rename from packages/queue/src/kafka.ts
rename to packages/core/src/modules/ingest/src/kafka.ts
index cc575d86e..dae90966d 100644
--- a/packages/queue/src/kafka.ts
+++ b/packages/core/src/modules/ingest/src/kafka.ts
@@ -1,4 +1,16 @@
-import { createLogger } from '@openpanel/core';
+// Moved from packages/queue/src/kafka.ts (M11-003), byte-for-byte apart from
+// the three repointed imports and the de-duplicated `DeadLetterMessage` below.
+// What kept it out of core was the import direction, not the code: it lived
+// in @openpanel/queue, a package that imports @openpanel/core for its logger,
+// so core could not import it back. Moving the file takes it out of that
+// cycle — `createLogger` is a sibling now, and both the producer and the
+// consumer sit in the module that owns the transport (ADR-004: Kafka is the
+// sole events transport — no topic, group or envelope changes with the move).
+//
+// NOTHING here constructs a Kafka client at import time: `getKafka()` is lazy
+// and only `assertKafkaConfigured` reads the broker list eagerly, so core
+// stays importable with no broker and `bun test` still runs offline.
+
import {
type Admin,
type Consumer,
@@ -7,7 +19,11 @@ import {
logLevel,
type Producer,
} from 'kafkajs';
-import type { EventsQueuePayloadIncomingEvent } from './queues';
+import { createLogger } from '../../../clients/logger';
+// One definition, not two: the consumer already declares the dead-letter
+// message shape it hands to this producer, and both files are now siblings.
+import type { DeadLetterMessage } from './consumer';
+import type { IncomingEventPayload } from './incoming-event';
export type { Admin, EachBatchPayload, KafkaMessage } from 'kafkajs';
@@ -241,7 +257,7 @@ const send = async (topic: string, message: OutgoingMessage): Promise<void> => {
};
export const produceIncomingEvent = async (
- payload: EventsQueuePayloadIncomingEvent['payload'],
+ payload: IncomingEventPayload,
partitionKey: string
): Promise<void> =>
send(KAFKA_EVENTS_TOPIC, {
@@ -252,17 +268,6 @@ export const produceIncomingEvent = async (
// Why the reason/error/coordinates travel as headers and not in the value: the
// value stays the producer's original bytes, so a DLQ message can be replayed
// onto the events topic unchanged.
-export interface DeadLetterMessage {
- key: Buffer | null;
- value: Buffer | null;
- headers?: IHeaders;
- topic: string;
- partition: number;
- offset: string;
- reason: string;
- error: string;
-}
-
export const produceDeadLetterEvent = async (
message: DeadLetterMessage
): Promise<void> =>
diff --git a/packages/core/src/modules/notification/notification.service.ts b/packages/core/src/modules/notification/notification.service.ts
index 7f6e0b2ad..b1a57ea79 100644
--- a/packages/core/src/modules/notification/notification.service.ts
+++ b/packages/core/src/modules/notification/notification.service.ts
@@ -3,16 +3,12 @@
// moves here too, from apps/worker/src/jobs/notification.ts (DELEGATE
// PATTERN: that file becomes a thin wrapper calling `deliverNotification`).
//
-// What does NOT move: `createNotification` / `triggerNotification` /
-// `checkNotificationRulesForEvent` / `checkNotificationRulesForSessionEnd` —
-// the BullMQ-producer orchestration around a rule match. `@openpanel/queue`
-// imports `@openpanel/core` for its logger (packages/queue/src/queues.ts), so
-// core cannot import `@openpanel/queue` back without a real package cycle —
-// same constraint gsc.service.ts and cohort.service.ts document for their own
-// enqueue wrappers. Those four functions stay in
-// packages/db/src/services/notification.service.ts, now built on the rule
-// matching, templates, cache and constants exported below instead of holding
-// their own copies.
+// `createNotification` / `triggerNotification` / `checkNotificationRulesForEvent`
+// / `checkNotificationRulesForSessionEnd` — the BullMQ-producer orchestration
+// around a rule match — live in this module's own `src/notification-dispatch.ts`
+// since M11-003, built on the rule matching, templates, cache and constants
+// exported below. They are a separate file only so the enqueue side is not
+// dragged into every import of this one.
//
// db/ch access is LAZY (`load*` below), not a static top-level import — see
// insight.service.ts's header for the full reasoning (jobs.registry.ts and
diff --git a/packages/queue/src/notification-dispatch.ts b/packages/core/src/modules/notification/src/notification-dispatch.ts
similarity index 76%
rename from packages/queue/src/notification-dispatch.ts
rename to packages/core/src/modules/notification/src/notification-dispatch.ts
index e744ae5fa..0999c3c15 100644
Binary files a/packages/queue/src/notification-dispatch.ts and b/packages/core/src/modules/notification/src/notification-dispatch.ts differ
diff --git a/packages/core/src/modules/session/session.jobs.ts b/packages/core/src/modules/session/session.jobs.ts
index 913750b81..1c966399b 100644
--- a/packages/core/src/modules/session/session.jobs.ts
+++ b/packages/core/src/modules/session/session.jobs.ts
@@ -65,11 +65,10 @@ const sessionEndPayload = z.custom<SessionEndJobData>(
/**
* The session-end job's dependencies, bound to the run's own ctx.
*
- * Two lookups stay dynamic. `checkNotificationRulesForSessionEnd` lives in
- * `@openpanel/queue`, which imports `@openpanel/core` back — a real package
- * cycle (notification.service.ts's header). The rule cache is reached the same
- * way only to keep notification.service.ts out of jobs.registry.ts's eager
- * import graph, which every core test file walks.
+ * Two lookups stay dynamic, both for the same reason now that
+ * notification-dispatch.ts has moved into core (M11-003): keeping
+ * notification.service.ts — and the dispatch module built on it — out of
+ * jobs.registry.ts's eager import graph, which every core test file walks.
*/
async function sessionEndDeps(
ctx: Ctx,
@@ -78,7 +77,7 @@ async function sessionEndDeps(
const [notifications, { checkNotificationRulesForSessionEnd }] =
await Promise.all([
import('../notification/notification.service'),
- import('@openpanel/queue/src/notification-dispatch'),
+ import('../notification/src/notification-dispatch'),
]);
return {
diff --git a/packages/core/src/modules/session/src/session-end.ts b/packages/core/src/modules/session/src/session-end.ts
index 830c802cf..84fe8f063 100644
--- a/packages/core/src/modules/session/src/session-end.ts
+++ b/packages/core/src/modules/session/src/session-end.ts
@@ -109,7 +109,8 @@ export interface SessionEndDeps extends SessionRuntime {
notifications: {
getRules(projectId: string): Promise<INotificationRuleCached[]>;
hasFunnelRules(rules: INotificationRuleCached[]): boolean;
- /** Lives in @openpanel/queue — it enqueues through the notification queue. */
+ /** notification/src/notification-dispatch.ts — it enqueues through the
+ * notification queue. */
checkFunnelRules(events: IServiceEvent[]): Promise<unknown>;
};
}
diff --git a/packages/core/src/modules/session/src/usage.ts b/packages/core/src/modules/session/src/usage.ts
index 4dcd270b7..f5a838edc 100644
--- a/packages/core/src/modules/session/src/usage.ts
+++ b/packages/core/src/modules/session/src/usage.ts
@@ -8,13 +8,19 @@ import { cacheable } from '@openpanel/redis';
import { sendEmail } from '../../../clients/email';
import { createLogger, type ILogger } from '../../../clients/logger';
import type { Logger } from '../../../logger';
-import { getOrganizationBillingEventsCount } from '../../../v1-compat';
// project.service.ts's `getProjectEventsCount` takes `ServiceDeps` now
// (M10-004); this file has none (its own db access is the lazy `loadDb()`
// below), so it reaches the bare, v1-compat-wrapped spelling instead.
// GENUINE CYCLE, kept lazy: services.ts -> session.service.ts -> this file
// -> v1-compat.ts -> services.ts.
+//
+// EVERY v1-compat symbol this file uses goes through here, never a static
+// import: jobs.registry.ts reaches this file (session.jobs.ts -> session-end
+// -> usage), and one static edge to v1-compat pulls services.ts and all 36
+// services into the registry's import graph — which builds a ClickHouse
+// client at import time and breaks the offline contract
+// verification/golden/queue-keys/check.sh depends on.
function loadProjectService() {
return import('../../../v1-compat');
}
@@ -84,9 +90,10 @@ export const updateEventsCount = cacheable(async function updateEventsCount(
return;
}
+ const { getOrganizationBillingEventsCount, getProjectEventsCount } =
+ await loadProjectService();
const organizationEventsCount =
await getOrganizationBillingEventsCount(organization);
- const { getProjectEventsCount } = await loadProjectService();
const projectEventsCount = await getProjectEventsCount(projectId);
if (projectEventsCount) {
diff --git a/packages/core/src/modules/widget/widget.rpc.ts b/packages/core/src/modules/widget/widget.rpc.ts
index 90bbe45e5..c1b3ce277 100644
--- a/packages/core/src/modules/widget/widget.rpc.ts
+++ b/packages/core/src/modules/widget/widget.rpc.ts
@@ -25,7 +25,6 @@
import { clix } from '@openpanel/db/src/clickhouse/query-builder';
import ShortUniqueId from 'short-unique-id';
import { z } from 'zod';
-import { loadDbBuffers } from '../../buffers/lazy-db-buffers';
import type { Ctx } from '../../context';
import {
createTRPCRouter,
@@ -40,10 +39,6 @@ import { zWidgetOptions, zWidgetType } from '../report/report.constants';
const uid = new ShortUniqueId({ length: 6 });
const BADGE_CACHE_TTL_SECONDS = 5 * 60; // queries 30 days of data
-function loadEventBuffer() {
- return loadDbBuffers().then((m) => m.eventBuffer);
-}
-
function loadCache() {
return import('@openpanel/redis').then((m) => m.getCache);
}
@@ -191,7 +186,7 @@ export const widgetRouter = createTRPCRouter({
throw new TRPCNotFoundError('Invalid widget type');
}
- const eventBuffer = await loadEventBuffer();
+ const eventBuffer = ctx.buffers.event;
return {
projectId: widget.projectId,
diff --git a/packages/core/test/http-fixtures.ts b/packages/core/test/http-fixtures.ts
index 18cd04a9f..df38bfcb7 100644
--- a/packages/core/test/http-fixtures.ts
+++ b/packages/core/test/http-fixtures.ts
@@ -44,7 +44,6 @@ export function stubAppDeps(): AppDepsStub {
clients: {} as AppDeps['clients'],
buffers: {} as Buffers,
producers,
- produceIncomingEvent: () => Promise.resolve(),
logger,
config: { selfHosted: false },
},
diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.ts
index be3d38efc..6fb01742e 100644
--- a/packages/core/test/request-id-end-to-end.test.ts
+++ b/packages/core/test/request-id-end-to-end.test.ts
@@ -186,7 +186,6 @@ function stubDeps() {
clients: {} as AppDeps['clients'],
buffers: {} as Buffers,
producers: producers as unknown as QueueProducerHandle,
- produceIncomingEvent: () => Promise.resolve(),
logger: bindingLogger(lines),
config: { selfHosted: false },
};
@@ -368,7 +367,6 @@ function chartDeps() {
clients: {} as AppDeps['clients'],
buffers: {} as Buffers,
producers: producers as unknown as QueueProducerHandle,
- produceIncomingEvent: () => Promise.resolve(),
logger: chartLogger(lines),
config: { selfHosted: false },
};
@@ -554,7 +552,6 @@ function ingestDeps() {
clients: {} as AppDeps['clients'],
buffers,
producers: producers as unknown as QueueProducerHandle,
- produceIncomingEvent: () => Promise.resolve(),
logger: ingestLogger(lines),
config: { selfHosted: false },
};
diff --git a/packages/core/test/rpc-fixtures.ts b/packages/core/test/rpc-fixtures.ts
index 28d8f6781..bccf79346 100644
--- a/packages/core/test/rpc-fixtures.ts
+++ b/packages/core/test/rpc-fixtures.ts
@@ -93,7 +93,6 @@ export function stubHttpCtx(
clients: {} as AppDeps['clients'],
buffers: {} as Buffers,
producers: createRecordingProducers(queues),
- produceIncomingEvent: () => Promise.resolve(),
logger,
config: { selfHosted: false },
};
diff --git a/packages/db/scripts/check-sessions.ts b/packages/db/scripts/check-sessions.ts
deleted file mode 100644
index b05b89972..000000000
--- a/packages/db/scripts/check-sessions.ts
+++ /dev/null
@@ -1,424 +0,0 @@
-// Read-only health check for the session lifecycle pipeline.
-//
-// Inspects Redis state, the sessions BullMQ queue, both internal buffers,
-// and ClickHouse — then reconciles them to surface anything stuck or
-// drifting. Safe to run anytime; never mutates state.
-//
-// Usage:
-// pnpm with-env tsx scripts/check-sessions.ts
-// pnpm with-env tsx scripts/check-sessions.ts --verbose
-// pnpm with-env tsx scripts/check-sessions.ts --project beatchurn-website
-// pnpm with-env tsx scripts/check-sessions.ts --hours 48
-
-import { getSafeJson } from '@openpanel/json';
-import type { IClickhouseSession } from '@openpanel/core';
-import { sessionsQueue } from '@openpanel/queue';
-import { getRedisCache } from '@openpanel/redis';
-import { ch } from '../src/clickhouse/client';
-
-const DEADMAN_MS = Number.parseInt(
- process.env.SESSION_REAPER_WALLCLOCK_DEADMAN_MS || String(30 * 60 * 1000),
- 10
-);
-const REAPER_INTERVAL_MS = 5 * 60 * 1000;
-// "Eligible" = past the deadman, reaper is expected to close shortly.
-// "Stuck" = past 3x reaper interval beyond the deadman — reaper is behind.
-const STUCK_GRACE_MS = REAPER_INTERVAL_MS * 3;
-const STUCK_CUTOFF_MS = DEADMAN_MS + STUCK_GRACE_MS;
-const SAMPLE_SIZE_PER_PROJECT = 100;
-
-type ProjectStat = {
- project: string;
- active: number;
- eligible: number;
- stuck: number;
- orphans: number;
- oldestAgeMin: number;
-};
-
-type ChCounts = {
- starts1h: number;
- ends1h: number;
- starts24h: number;
- ends24h: number;
-};
-
-function fmt(n: number, w: number) {
- return String(n).padStart(w);
-}
-
-function header(title: string) {
- console.log('');
- console.log('─'.repeat(70));
- console.log(title);
- console.log('─'.repeat(70));
-}
-
-async function inspectRedis(projects: string[]): Promise<{
- total: number;
- totalEligible: number;
- totalStuck: number;
- totalOrphans: number;
- stats: ProjectStat[];
-}> {
- const redis = getRedisCache();
- const now = Date.now();
- const eligibleCutoff = now - DEADMAN_MS;
- const stuckCutoff = now - STUCK_CUTOFF_MS;
-
- const stats: ProjectStat[] = [];
- let total = 0;
- let totalEligible = 0;
- let totalStuck = 0;
- let totalOrphans = 0;
-
- for (const project of projects) {
- const wallclockKey = `session:wallclock:${project}`;
-
- const [active, eligible, stuck, oldestEntry] = await Promise.all([
- redis.zcard(wallclockKey),
- redis.zcount(wallclockKey, '-inf', eligibleCutoff),
- redis.zcount(wallclockKey, '-inf', stuckCutoff),
- redis.zrange(wallclockKey, 0, 0, 'WITHSCORES'),
- ]);
-
- const oldestAgeMin =
- oldestEntry.length === 2 ? (now - Number(oldestEntry[1])) / 60_000 : 0;
-
- // Sample N oldest entries and verify their blob exists.
- const sampleSize = Math.min(SAMPLE_SIZE_PER_PROJECT, active);
- let orphans = 0;
- if (sampleSize > 0) {
- const sample = await redis.zrange(wallclockKey, 0, sampleSize - 1);
- const multi = redis.multi();
- for (const did of sample) {
- multi.exists(`session:${project}:${did}`);
- }
- const results = await multi.exec();
- for (const entry of results ?? []) {
- if (Number(entry?.[1] ?? 0) === 0) orphans++;
- }
- }
-
- stats.push({ project, active, eligible, stuck, orphans, oldestAgeMin });
- total += active;
- totalEligible += eligible;
- totalStuck += stuck;
- totalOrphans += orphans;
- }
-
- return { total, totalEligible, totalStuck, totalOrphans, stats };
-}
-
-async function inspectQueue() {
- const [waiting, delayed, active, failed, completed] = await Promise.all([
- sessionsQueue.getWaitingCount(),
- sessionsQueue.getDelayedCount(),
- sessionsQueue.getActiveCount(),
- sessionsQueue.getFailedCount(),
- sessionsQueue.getCompletedCount(),
- ]);
-
- const failedSample = failed > 0 ? await sessionsQueue.getFailed(0, 4) : [];
-
- return { waiting, delayed, active, failed, completed, failedSample };
-}
-
-async function inspectBuffers() {
- const redis = getRedisCache();
- const [eventCounter, sessionCounter, eventListLen, sessionListLen] =
- await Promise.all([
- redis.get('buffer-counter:event'),
- redis.get('buffer-counter:session'),
- redis.llen('event-buffer'),
- redis.llen('session-buffer'),
- ]);
-
- return {
- event: {
- counter: Number(eventCounter ?? 0),
- listLen: Number(eventListLen ?? 0),
- },
- session: {
- counter: Number(sessionCounter ?? 0),
- listLen: Number(sessionListLen ?? 0),
- },
- };
-}
-
-async function inspectClickhouse(
- projects: string[] | null,
- hours: number
-): Promise<Map<string, ChCounts>> {
- const projectClause = projects?.length
- ? `AND project_id IN (${projects.map((p) => `'${p.replace(/'/g, "''")}'`).join(',')})`
- : '';
- const query = `
- SELECT
- project_id,
- name,
- sum(if(created_at > now() - INTERVAL 1 HOUR, 1, 0)) AS h1,
- sum(if(created_at > now() - INTERVAL ${hours} HOUR, 1, 0)) AS hN
- FROM events
- WHERE name IN ('session_start', 'session_end')
- AND created_at > now() - INTERVAL ${hours} HOUR
- ${projectClause}
- GROUP BY project_id, name
- `;
- const res = await ch.query({ query, format: 'JSONEachRow' });
- const rows = await res.json<{
- project_id: string;
- name: string;
- h1: string;
- hN: string;
- }>();
-
- const byProject = new Map<string, ChCounts>();
- for (const r of rows) {
- const cur = byProject.get(r.project_id) ?? {
- starts1h: 0,
- ends1h: 0,
- starts24h: 0,
- ends24h: 0,
- };
- if (r.name === 'session_start') {
- cur.starts1h = Number(r.h1);
- cur.starts24h = Number(r.hN);
- } else {
- cur.ends1h = Number(r.h1);
- cur.ends24h = Number(r.hN);
- }
- byProject.set(r.project_id, cur);
- }
- return byProject;
-}
-
-async function inspectSessionEndClaims() {
- const redis = getRedisCache();
- let cursor = '0';
- let count = 0;
- do {
- const [next, batch] = await redis.scan(
- cursor,
- 'MATCH',
- 'session:end:emitted:*',
- 'COUNT',
- 1000
- );
- cursor = next;
- count += batch.length;
- } while (cursor !== '0' && count < 100_000);
- return count;
-}
-
-async function sampleOldestSession(
- projectId: string,
- deviceId: string
-): Promise<IClickhouseSession | null> {
- const blob = await getRedisCache().get(`session:${projectId}:${deviceId}`);
- return blob ? getSafeJson<IClickhouseSession>(blob) : null;
-}
-
-async function main() {
- const args = process.argv.slice(2);
- const verbose = args.includes('--verbose');
- const projectIdx = args.indexOf('--project');
- const projectFilter = projectIdx >= 0 ? args[projectIdx + 1] : null;
- const hoursIdx = args.indexOf('--hours');
- const hours = hoursIdx >= 0 ? Number(args[hoursIdx + 1]) : 24;
-
- const redis = getRedisCache();
-
- console.log('═'.repeat(70));
- console.log('SESSION LIFECYCLE HEALTH CHECK');
- console.log(`time: ${new Date().toISOString()}`);
- console.log(`deadman: ${DEADMAN_MS / 60_000} min`);
- console.log(`stuck cutoff: ${STUCK_CUTOFF_MS / 60_000} min (deadman + 3 reaper ticks)`);
- console.log(`ch window: ${hours}h`);
- if (projectFilter) console.log(`project: ${projectFilter}`);
- console.log('═'.repeat(70));
-
- // 1. Redis inventory
- const allProjects = await redis.smembers('session:projects');
- const projects = projectFilter
- ? [projectFilter]
- : allProjects.slice().sort();
-
- header('REDIS — active sessions');
- console.log(` ${allProjects.length} projects in session:projects`);
-
- const redisInfo = await inspectRedis(projects);
- console.log(` ${redisInfo.total} total active sessions`);
- console.log(` ${redisInfo.totalEligible} past deadman (expected to reap soon)`);
- console.log(
- ` ${redisInfo.totalStuck} past deadman + ${STUCK_GRACE_MS / 60_000}min (stuck — reaper is behind)`
- );
- console.log(
- ` ${redisInfo.totalOrphans} wallclock entries with no blob (in sampled ${SAMPLE_SIZE_PER_PROJECT}/project)`
- );
-
- if (verbose && redisInfo.stats.length > 0) {
- console.log('');
- console.log(
- ` ${'project'.padEnd(40)} ${'active'.padStart(8)} ${'eligible'.padStart(10)} ${'stuck'.padStart(8)} ${'orphans'.padStart(8)} ${'oldest'.padStart(10)}`
- );
- for (const s of redisInfo.stats.sort((a, b) => b.active - a.active)) {
- console.log(
- ` ${s.project.padEnd(40)} ${fmt(s.active, 8)} ${fmt(s.eligible, 10)} ${fmt(s.stuck, 8)} ${fmt(s.orphans, 8)} ${s.oldestAgeMin.toFixed(1).padStart(8)}m`
- );
- }
- }
-
- // 2. Sessions queue
- header('SESSIONS QUEUE — createSessionEnd jobs');
- const queue = await inspectQueue();
- console.log(` waiting: ${queue.waiting}`);
- console.log(` delayed: ${queue.delayed}`);
- console.log(` active: ${queue.active}`);
- console.log(` failed: ${queue.failed}`);
- console.log(` completed: ${queue.completed} (removed-on-complete)`);
- if (queue.failedSample.length > 0) {
- console.log('');
- console.log(' Latest failed jobs:');
- for (const job of queue.failedSample) {
- const when = new Date(job.timestamp).toISOString();
- const reason = (job.failedReason ?? '').slice(0, 200);
- console.log(` - [${when}] ${job.id}`);
- console.log(` ${reason}`);
- }
- }
-
- // 3. Buffers
- header('BUFFERS — pending writes to ClickHouse');
- const buffers = await inspectBuffers();
- console.log(
- ` event buffer: counter=${buffers.event.counter} list=${buffers.event.listLen}`
- );
- console.log(
- ` session buffer: counter=${buffers.session.counter} list=${buffers.session.listLen}`
- );
-
- // 4. Dedup claims
- header('IDEMPOTENCY — session:end:emitted:* claim keys');
- const claimCount = await inspectSessionEndClaims();
- console.log(
- ` ${claimCount} active claim keys (TTL 2h; bounds at ~last 2h of closes)`
- );
-
- // 5. ClickHouse reconciliation
- header(`CLICKHOUSE — session_start vs session_end (last ${hours}h)`);
- const chCounts = await inspectClickhouse(
- projectFilter ? [projectFilter] : null,
- hours
- );
-
- let totalStarts1h = 0;
- let totalEnds1h = 0;
- let totalStartsN = 0;
- let totalEndsN = 0;
-
- console.log('');
- console.log(
- ` ${'project'.padEnd(40)} ${'start 1h'.padStart(9)} ${'end 1h'.padStart(8)} ${'start ' + hours + 'h'}`.padEnd(58) +
- `${('end ' + hours + 'h').padStart(8)} ${'open'.padStart(8)}`
- );
- for (const [project, c] of [...chCounts.entries()].sort()) {
- const open = c.starts24h - c.ends24h;
- console.log(
- ` ${project.padEnd(40)} ${fmt(c.starts1h, 9)} ${fmt(c.ends1h, 8)} ${fmt(c.starts24h, 10)} ${fmt(c.ends24h, 8)} ${fmt(open, 8)}`
- );
- totalStarts1h += c.starts1h;
- totalEnds1h += c.ends1h;
- totalStartsN += c.starts24h;
- totalEndsN += c.ends24h;
- }
- const openInCh = totalStartsN - totalEndsN;
- console.log('');
- console.log(
- ` ${'TOTAL'.padEnd(40)} ${fmt(totalStarts1h, 9)} ${fmt(totalEnds1h, 8)} ${fmt(totalStartsN, 10)} ${fmt(totalEndsN, 8)} ${fmt(openInCh, 8)}`
- );
-
- // 6. Reconciliation
- header('RECONCILIATION — Redis active vs CH open');
- const diff = redisInfo.total - openInCh;
- const diffPct =
- openInCh > 0 ? ((Math.abs(diff) / openInCh) * 100).toFixed(1) : '∞';
- console.log(` Redis active sessions: ${redisInfo.total}`);
- console.log(` CH starts - ends in ${hours}h window: ${openInCh}`);
- console.log(` Diff: ${diff} (${diffPct}%)`);
- console.log(
- ` Note: diff > 0 means some "open" sessions in Redis were started before the ${hours}h window.`
- );
-
- // 7. Sample the oldest session in Redis if anything looks suspicious
- if (verbose && redisInfo.stats.some((s) => s.oldestAgeMin > 60 * 6)) {
- header('SAMPLE — oldest session per project (any > 6h)');
- for (const s of redisInfo.stats.filter((x) => x.oldestAgeMin > 60 * 6)) {
- const wallclockKey = `session:wallclock:${s.project}`;
- const [oldest] = await redis.zrange(wallclockKey, 0, 0);
- if (oldest) {
- const blob = await sampleOldestSession(s.project, oldest);
- if (blob) {
- console.log(` ${s.project} / ${oldest}`);
- console.log(` id: ${blob.id}`);
- console.log(` created_at: ${blob.created_at}`);
- console.log(` ended_at: ${blob.ended_at}`);
- console.log(` event_count: ${blob.event_count}`);
- }
- }
- }
- }
-
- // 8. Summary
- header('SUMMARY');
- const issues: string[] = [];
- if (redisInfo.totalStuck > 0) {
- issues.push(
- `${redisInfo.totalStuck} sessions past the stuck cutoff — reaper may be unhealthy`
- );
- }
- if (redisInfo.totalOrphans > 0) {
- issues.push(
- `${redisInfo.totalOrphans} orphan wallclock entries — cleanup() raced or vacuum is overdue`
- );
- }
- if (queue.failed > 0) {
- issues.push(`${queue.failed} failed session_end jobs`);
- }
- if (queue.waiting + queue.active > 5000) {
- issues.push(
- `session_end backlog (waiting=${queue.waiting} active=${queue.active})`
- );
- }
- if (buffers.event.listLen > 50_000) {
- issues.push(`event buffer is backed up (${buffers.event.listLen} items)`);
- }
- if (buffers.session.listLen > 50_000) {
- issues.push(
- `session buffer is backed up (${buffers.session.listLen} items)`
- );
- }
- // The Redis-vs-CH diff is INFORMATIONAL, not an issue on its own:
- // diff > 0: more in Redis than CH's window — older sessions still active
- // diff < 0: more starts in CH than Redis active — legacy starts without ends
- // (e.g. from before a bug fix) that will never close
- // Only flag if both reaper is behind AND the diff is growing — which we can't
- // know from a single run. So just print it; the reaper-behind and stuck
- // counters above will catch live issues.
-
- if (issues.length === 0) {
- console.log(' [OK] No issues detected.');
- } else {
- console.log(' [WARN] Issues:');
- for (const i of issues) console.log(` - ${i}`);
- }
-
- await redis.quit();
- await ch.close?.();
- process.exit(issues.length > 0 ? 1 : 0);
-}
-
-main().catch((e) => {
- console.error(e);
- process.exit(1);
-});
diff --git a/packages/db/scripts/drain-old-session-jobs.ts b/packages/db/scripts/drain-old-session-jobs.ts
deleted file mode 100644
index 3fd539170..000000000
--- a/packages/db/scripts/drain-old-session-jobs.ts
+++ /dev/null
@@ -1,79 +0,0 @@
-// One-off cleanup: drain legacy `sessionEnd:{projectId}:{deviceId}` delayed
-// jobs from the `sessions` queue. Run AFTER:
-//
-// 1. The new code is deployed (worker emits reaper-enqueued jobs with the
-// `sessionEnd:v2:*` jobId prefix).
-// 2. The `events.incoming-event` queue has been resumed and you've verified
-// session metrics look healthy under the new flow.
-//
-// This removes only jobs whose jobId is `sessionEnd:*` and NOT `sessionEnd:v2:*`,
-// leaving any reaper-enqueued v2 jobs intact. After this script runs you can
-// safely resume the `sessions` queue.
-//
-// Idempotent: re-running on an already-drained queue is a no-op.
-//
-// Usage (from packages/db):
-// pnpm with-env tsx scripts/drain-old-session-jobs.ts
-// pnpm with-env tsx scripts/drain-old-session-jobs.ts --dry-run
-
-import { sessionsQueue } from '@openpanel/queue';
-
-async function main() {
- const dryRun = process.argv.includes('--dry-run');
-
- console.log(
- `[drain-old-session-jobs] starting${dryRun ? ' (DRY RUN — no removes)' : ''}`
- );
-
- const jobs = await sessionsQueue.getJobs(['delayed', 'waiting']);
-
- const legacy: typeof jobs = [];
- const v2: typeof jobs = [];
-
- for (const job of jobs) {
- if (!job.id) continue;
- if (job.id.startsWith('sessionEnd:v2:')) {
- v2.push(job);
- } else if (job.id.startsWith('sessionEnd:')) {
- legacy.push(job);
- }
- }
-
- console.log(
- `[drain-old-session-jobs] found ${jobs.length} jobs total ` +
- `(${legacy.length} legacy, ${v2.length} v2, will keep v2)`
- );
-
- if (dryRun) {
- console.log('[drain-old-session-jobs] dry run — exiting');
- process.exit(0);
- }
-
- let removed = 0;
- let failed = 0;
-
- for (const job of legacy) {
- try {
- await job.remove();
- removed++;
- if (removed % 1000 === 0) {
- console.log(`[drain-old-session-jobs] removed ${removed}/${legacy.length}`);
- }
- } catch (error) {
- failed++;
- console.error('[drain-old-session-jobs] failed to remove job', job.id, error);
- }
- }
-
- console.log('[drain-old-session-jobs] done', {
- removed,
- failed,
- kept_v2: v2.length,
- });
- process.exit(0);
-}
-
-main().catch((error) => {
- console.error('[drain-old-session-jobs] fatal error:', error);
- process.exit(1);
-});
diff --git a/packages/db/scripts/migrate-sessions.ts b/packages/db/scripts/migrate-sessions.ts
deleted file mode 100644
index baf33da6d..000000000
--- a/packages/db/scripts/migrate-sessions.ts
+++ /dev/null
@@ -1,121 +0,0 @@
-// One-off migration: convert legacy `session:{sessionId}` Redis blobs +
-// delayed BullMQ `sessionEnd:{projectId}:{deviceId}` jobs into the new
-// session-buffer key scheme:
-//
-// session:{projectId}:{deviceId} -> full session JSON blob (no TTL)
-// session:profile:{projectId}:{profileId} -> deviceId pointer (no TTL)
-// session:wallclock:{projectId} -> ZSET scored by wall-clock (ms)
-// session:projects -> SET of project_ids with active sessions
-//
-// Source of truth: the delayed `sessionsQueue` jobs — one per (projectId,
-// deviceId) pair currently considered "active" by the legacy system. For
-// each, we read the corresponding `session:{sessionId}` blob (still alive
-// under its 1h TTL) and rewrite it into the new structure.
-//
-// Idempotent: re-running the script is safe. Existing new-scheme keys are
-// preserved via SET NX semantics inside the multi pipeline.
-//
-// Usage (from packages/db, env loaded from repo root .env):
-// pnpm with-env tsx scripts/migrate-sessions.ts
-// pnpm with-env tsx scripts/migrate-sessions.ts --dry-run
-//
-// Runbook step 2: pause `events.incoming-event` AND `sessions` queues first,
-// wait for in-flight jobs to drain (~30s), THEN run this script. Resume
-// `events.incoming-event` after deploy; keep `sessions` paused until step 7
-// (drain-old-session-jobs.ts) finishes.
-
-import type { IClickhouseSession } from '@openpanel/core';
-import { getSafeJson } from '@openpanel/json';
-import {
- type EventsQueuePayloadCreateSessionEnd,
- sessionsQueue,
-} from '@openpanel/queue';
-import { getRedisCache } from '@openpanel/redis';
-
-const sessionKey = (projectId: string, deviceId: string) =>
- `session:${projectId}:${deviceId}`;
-const profileIndexKey = (projectId: string, profileId: string) =>
- `session:profile:${projectId}:${profileId}`;
-const wallclockSetKey = (projectId: string) =>
- `session:wallclock:${projectId}`;
-const PROJECTS_SET_KEY = 'session:projects';
-
-async function main() {
- const dryRun = process.argv.includes('--dry-run');
- const redis = getRedisCache();
-
- console.log(
- `[migrate-sessions] starting${dryRun ? ' (DRY RUN — no writes)' : ''}`
- );
-
- const jobs = await sessionsQueue.getJobs(['delayed', 'waiting']);
- console.log(`[migrate-sessions] found ${jobs.length} legacy session jobs`);
-
- const counts = {
- migrated: 0,
- blobMissing: 0,
- alreadyMigrated: 0,
- parseFailed: 0,
- };
-
- for (const job of jobs) {
- const data = job.data as EventsQueuePayloadCreateSessionEnd | undefined;
- const payload = data?.payload;
- if (!payload?.projectId || !payload.deviceId || !payload.sessionId) {
- counts.parseFailed++;
- continue;
- }
- const { projectId, deviceId, sessionId } = payload;
-
- const existsAlready = await redis.exists(sessionKey(projectId, deviceId));
- if (existsAlready) {
- counts.alreadyMigrated++;
- continue;
- }
-
- const blob = await redis.get(`session:${sessionId}`);
- if (!blob) {
- counts.blobMissing++;
- continue;
- }
-
- const session = getSafeJson<IClickhouseSession>(blob);
- if (!session) {
- counts.parseFailed++;
- continue;
- }
-
- const nowMs = Date.now();
-
- if (dryRun) {
- counts.migrated++;
- continue;
- }
-
- const multi = redis.multi();
- multi.set(sessionKey(projectId, deviceId), blob);
- multi.zadd(wallclockSetKey(projectId), nowMs.toString(), deviceId);
- multi.sadd(PROJECTS_SET_KEY, projectId);
- if (session.profile_id && session.profile_id !== deviceId) {
- multi.set(profileIndexKey(projectId, session.profile_id), deviceId);
- }
- await multi.exec();
-
- counts.migrated++;
-
- if (counts.migrated % 1000 === 0) {
- console.log(
- `[migrate-sessions] progress: ${counts.migrated} migrated`
- );
- }
- }
-
- console.log('[migrate-sessions] done', counts);
- await redis.quit();
- process.exit(0);
-}
-
-main().catch((error) => {
- console.error('[migrate-sessions] fatal error:', error);
- process.exit(1);
-});
diff --git a/packages/queue/index.ts b/packages/queue/index.ts
index 3dc13f492..197f47f0a 100644
--- a/packages/queue/index.ts
+++ b/packages/queue/index.ts
@@ -1,4 +1,11 @@
-export * from './src/queues';
-export type * from './src/queues';
-export * from './src/kafka';
+// The Kafka transport and the notification dispatch left this package at
+// M11-003 for @openpanel/core (modules/ingest/src/kafka.ts and
+// modules/notification/src/notification-dispatch.ts). What stays is V1's
+// BullMQ queue definitions and the buffer singleton built on `cronQueue`,
+// both still imported by packages/trpc's routers — a package this task's
+// scope does not cover, so their deletion waits for the follow-up that may
+// touch those call sites (see docs/TECH_DEBT.md).
+
export type { JobsOptions } from 'bullmq';
+export type * from './src/queues';
+export * from './src/queues';
diff --git a/packages/queue/src/buffers.ts b/packages/queue/src/buffers.ts
index 4e652fb6c..a313cca3e 100644
--- a/packages/queue/src/buffers.ts
+++ b/packages/queue/src/buffers.ts
@@ -3,21 +3,22 @@
// (M8-001): `createBuffers(deps)` builds them once and V2's main.ts hangs the
// result on `AppDeps.buffers` (ADR-007). This file is the ONE process-wide
// singleton instance every call site that predates that ctx wiring still
-// reaches — packages/trpc's overview/widget routers and the several
-// `@openpanel/core` modules that lazy-import this specifier
-// (`loadDbBuffers`) rather than threading `ctx.buffers` through. Repointing
-// those call sites onto `ctx.buffers` is the larger, separately-scoped
-// follow-up docs/TECH_DEBT.md tracks (the 93 lazy db imports bypassing
-// ctx/requestId) — not this task.
+// reaches. Since M11-003 that is packages/trpc's overview/widget routers
+// alone — core's last lazy importer (widget.rpc.ts, via `loadDbBuffers`)
+// now reads `ctx.buffers`. Repointing the two remaining trpc call sites is
+// the larger, separately-scoped follow-up docs/TECH_DEBT.md tracks (the 93
+// lazy db imports bypassing ctx/requestId) — not this task, whose scope
+// stops at packages/queue.
//
// Lives in @openpanel/queue rather than @openpanel/core because it needs
// `cronQueue` (ADR-005's BullMQ escape hatch: pausing `cron` from bull-board
// halts all buffer flushing, preserved deliberately — docs/ANSWERS.md §3:
// "known!"), and core must not import `@openpanel/queue` back without a real
-// package cycle (see notification-dispatch.ts). `@openpanel/queue` already
-// depends on `@openpanel/core`, so this is the one side of that boundary free
-// of a cycle. Also keeps core's own rule intact: buffers are boot singletons
-// on AppDeps, never a module-level singleton in core itself (M8-005).
+// package cycle (see queues.ts, which imports core for its logger).
+// `@openpanel/queue` already depends on `@openpanel/core`, so this is the one
+// side of that boundary free of a cycle. Also keeps core's own rule intact:
+// buffers are boot singletons on AppDeps, never a module-level singleton in
+// core itself (M8-005).
import { createBuffers, createLogger } from '@openpanel/core';
import { cronQueue } from './queues';