OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M8-001): buffers into core: createBuffers at boot on AppDeps + metrics instrumentation
ralphafbaece4236c
diff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex 42ef23077..181765a9d 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -3,24 +3,32 @@// keeps serving the golden harness untouched until `apps/worker` is deleted// at P9.//-// This is the M3 gate boot only: db/ch/redis/clients/buffers are not wired-// into `AppDeps` yet — their types stay the `unknown` stubs `context.ts`-// declares until P3/P4/P8 land the real clients — so only the ops surface-// (healthz + metrics) is mounted here. `dashboardRoutes`/`publicApiRoutes`-// land with their first module (see `rest.routes.ts`).+// This is the M3 gate boot only: db/ch/redis/clients are not wired into+// `AppDeps` yet — their types stay the `unknown` stubs `context.ts` declares+// until P3/P4 land the real clients — so only the ops surface (healthz ++// metrics) is mounted here. `dashboardRoutes`/`publicApiRoutes` land with+// their first module (see `rest.routes.ts`). The buffers ARE real from M8-001:+// `createBuffers` builds them once, here, and nowhere else.process.env.TZ = 'UTC';import {type AppDeps,+ type BufferDeps,+ createBuffers,opsRoutes,type QueueProducerHandle,queues,+ registerBufferMetrics,} from '@openpanel/core';import pino from 'pino';import { type Config, loadConfig } from './config/env';const SHUTDOWN_FORCE_EXIT_MS = 5000;+// The registry key and the Redis name are the same string for all seven queues+// (jobs.registry.ts); `queueKey` only braces it under `QUEUE_CLUSTER`, which no+// deployment sets.+const CRON_QUEUE_NAME = 'cron';// Used only to report a config error itself — the real level isn't known// until `config/env.ts` has validated `LOG_LEVEL`.const BOOTSTRAP_LOG_LEVEL = 'info';@@ -72,17 +80,37 @@ function stubProducers(): QueueProducerHandle {};}+/**+ * A named child of the boot logger per buffer, plus the buffers' one direct+ * BullMQ read — ADR-005's `bullQueues` escape hatch.+ * Pausing `cron` from bull-board halts ALL buffer flushing, preserved+ * deliberately (docs/ANSWERS.md §3: "known!"). The M3 producer stub owns no+ * queues yet, and a queue that does not exist has not been paused.+ */+function bufferDeps(producers: QueueProducerHandle): BufferDeps {+ return {+ createLogger: (name) => logger.child({ name }),+ isCronPaused: async () => {+ const cron = producers.bullQueues.find(+ (queue) => queue.name === CRON_QUEUE_NAME+ );+ return cron ? await cron.isPaused() : false;+ },+ };+}+function buildDeps(): AppDeps {+ const producers = stubProducers();return {- // db/ch/redis/clients/buffers are `unknown` stubs in context.ts until- // P3/P4/P8 land the real clients — reading one before its type lands is- // meant to be a compile error, not a runtime `undefined`.+ // db/ch/redis/clients are `unknown` stubs in context.ts until P3/P4 land+ // the real clients — reading one before its type lands is meant to be a+ // compile error, not a runtime `undefined`.db: undefined,ch: undefined,redis: undefined,clients: undefined,- buffers: undefined,- producers: stubProducers(),+ buffers: createBuffers(bufferDeps(producers)),+ producers,logger,config: { selfHosted: config.SELF_HOSTED },};@@ -93,6 +121,14 @@ async function main() {const port = config.API_PORT;const deps = buildDeps();+ // Scrape-time collectors register only where a role consumes queues: the+ // LLEN gauges cost one Redis round trip per buffer per scrape, and ten API+ // replicas exposing them would multiply that for no new information+ // (TARGET_ARCHITECTURE §18).+ if (role !== 'api') {+ registerBufferMetrics(deps.buffers);+ }+const app = opsRoutes(deps);if (role !== 'api') {diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.tsindex ebfbbbf62..59563c955 100644--- a/apps/worker/src/metrics.ts+++ b/apps/worker/src/metrics.ts@@ -1,3 +1,4 @@+import { registerBufferMetrics } from '@openpanel/core';import {botBuffer,eventBuffer,@@ -129,201 +130,23 @@ queues.forEach((queue) => {// Buffer metrics// ------------------------------------------------------------------------------const allBuffers = [- eventBuffer,- profileBuffer,- botBuffer,- sessionBuffer,- replayBuffer,- groupBuffer,- profileBackfillBuffer,-];--// Ground-truth LLEN of each buffer's main list. O(1) — no shadow counter.-for (const buf of allBuffers) {- register.registerMetric(- new client.Gauge({- name: `buffer_${buf.name.replace(/-/g, '_')}_count`,- help: 'Buffer size (LLEN of the Redis list)',- async collect() {- try {- this.set(await buf.getBufferSize());- } catch {- // ignore — scrape continues- }- },- })- );-}--// Number of events sitting in event-buffer's in-process micro-batch (pre-Redis).-// Other buffers don't have a local layer.-register.registerMetric(- new client.Gauge({- name: 'buffer_event_pending_local_count',- help: 'Events in event-buffer process-local micro-batch (not yet in Redis)',- collect() {- try {- this.set(eventBuffer.getPendingLocalCount());- } catch {- // ignore- }- },- })+// The twelve `buffer_*` series moved to @openpanel/core with the buffers+// themselves (M8-001). V1 keeps its own registry, so it hands one in: the+// names, labels, buckets and registration order are core's, and this body is+// byte-identical to what this file registered before the move.+registerBufferMetrics(+ {+ event: eventBuffer,+ profile: profileBuffer,+ bot: botBuffer,+ session: sessionBuffer,+ replay: replayBuffer,+ group: groupBuffer,+ profileBackfill: profileBackfillBuffer,+ },+ register);-// ---- Flush metrics (populated via flushObserver hooks) ------const flushDuration = new client.Histogram({- name: 'buffer_flush_duration_ms',- help: 'Wall time of a tryFlush call, including lock acquisition',- labelNames: ['buffer', 'result', 'trigger'],- buckets: [- 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,- ],-});-register.registerMetric(flushDuration);--const flushTotal = new client.Counter({- name: 'buffer_flush_total',- help: 'Count of tryFlush invocations by result (success/error/locked/paused) and trigger (add/cron)',- labelNames: ['buffer', 'result', 'trigger'],-});-register.registerMetric(flushTotal);--const flushRowsTotal = new client.Counter({- name: 'buffer_flush_rows_total',- help: 'Rows drained from the buffer per flush (sum)',- labelNames: ['buffer'],-});-register.registerMetric(flushRowsTotal);--// Per-phase Redis op timing on the flush hot path.-const redisOpDurationMs = new client.Histogram({- name: 'buffer_redis_op_duration_ms',- help: 'Duration of a Redis op during flush',- labelNames: ['buffer', 'op'],- buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000],-});-register.registerMetric(redisOpDurationMs);--const chInsertDurationMs = new client.Histogram({- name: 'buffer_ch_insert_duration_ms',- help: 'Duration of the ClickHouse insert(s) inside a single flush',- labelNames: ['buffer'],- buckets: [- 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,- ],-});-register.registerMetric(chInsertDurationMs);--// CH SELECT latency inside a flush (e.g. profile-buffer's fetch-existing-// profiles for merge). Separated from ch_insert because for some buffers-// the SELECT dominates total flush time. Only populated by buffers that-// actually read CH inside the flush path.-const chFetchDurationMs = new client.Histogram({- name: 'buffer_ch_fetch_duration_ms',- help: 'Duration of CH SELECT(s) inside a single flush (e.g. profile merge fetch)',- labelNames: ['buffer'],- buckets: [- 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,- ],-});-register.registerMetric(chFetchDurationMs);--const flushLlenAtStart = new client.Histogram({- name: 'buffer_flush_llen_at_start',- help: 'LLEN observed at the start of a flush attempt',- labelNames: ['buffer'],- buckets: [- 0, 10, 100, 500, 1000, 5000, 10_000, 50_000, 100_000, 500_000, 1_000_000,- 5_000_000,- ],-});-register.registerMetric(flushLlenAtStart);--// ---- Add-path metrics ------const addDurationMs = new client.Histogram({- name: 'buffer_add_duration_ms',- help: 'Duration of a single add() call (per-event ingest path)',- labelNames: ['buffer'],- buckets: [0.5, 1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 5000],-});-register.registerMetric(addDurationMs);--const addTotal = new client.Counter({- name: 'buffer_add_total',- help: 'Total add() calls per buffer that actually enqueued (excludes skipped).',- labelNames: ['buffer'],-});-register.registerMetric(addTotal);--const addSkippedTotal = new client.Counter({- name: 'buffer_add_skipped_total',- help: 'add() calls that short-circuited (no enqueue). Reason: cached, etc.',- labelNames: ['buffer', 'reason'],-});-register.registerMetric(addSkippedTotal);--for (const buf of allBuffers) {- buf.flushObserver = (obs) => {- flushTotal.inc({- buffer: obs.buffer,- result: obs.result,- trigger: obs.trigger,- });- flushDuration.observe(- { buffer: obs.buffer, result: obs.result, trigger: obs.trigger },- obs.totalMs- );-- if (obs.llenAtStart != null) {- flushLlenAtStart.observe({ buffer: obs.buffer }, obs.llenAtStart);- }-- if (obs.rowsProcessed != null && obs.rowsProcessed > 0) {- flushRowsTotal.inc({ buffer: obs.buffer }, obs.rowsProcessed);- }-- if (obs.phases?.lrangeMs != null) {- redisOpDurationMs.observe(- { buffer: obs.buffer, op: 'lrange' },- obs.phases.lrangeMs- );- }- if (obs.phases?.trimMs != null) {- redisOpDurationMs.observe(- { buffer: obs.buffer, op: 'trim' },- obs.phases.trimMs- );- }- if (obs.phases?.chFetchMs != null) {- chFetchDurationMs.observe({ buffer: obs.buffer }, obs.phases.chFetchMs);- }- if (obs.phases?.chInsertMs != null) {- chInsertDurationMs.observe({ buffer: obs.buffer }, obs.phases.chInsertMs);- }- };-- buf.addObserver = (obs) => {- if (obs.skipped) {- addSkippedTotal.inc({- buffer: obs.buffer,- reason: obs.skipReason ?? 'unknown',- });- // Don't pollute add-latency histogram with no-op fast paths- return;- }- addTotal.inc({ buffer: obs.buffer });- addDurationMs.observe({ buffer: obs.buffer }, obs.durationMs);- };-}--// Note: `buffer_replay_count` is already registered by the allBuffers loop-// above (replayBuffer is in that list), so no standalone gauge here — a second-// registration with the same name throws and crashes the worker on boot.-// -----------------------------------------------------------------------// Session lifecycle metrics (new session-buffer + reaper world)// -----------------------------------------------------------------------diff --git a/packages/core/package.json b/packages/core/package.jsonindex cd5ac6165..2098fbef4 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -47,6 +47,7 @@"csv-parse": "^6.1.0","date-fns": "^3.3.1","elysia": "1.4.30",+ "fast-deep-equal": "^3.1.3","ipaddr.js": "^2.5.0","lru-cache": "^11.2.4","mathjs": "^12.3.2",diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/core/src/buffers/base-buffer.tssimilarity index 92%rename from packages/db/src/buffers/base-buffer.tsrename to packages/core/src/buffers/base-buffer.tsindex d021f31fd..a02d0d24c 100644--- a/packages/db/src/buffers/base-buffer.ts+++ b/packages/core/src/buffers/base-buffer.ts@@ -1,9 +1,29 @@import { Readable } from 'node:stream';import type { ClickHouseSettings } from '@clickhouse/client';-import { generateSecureId } from '@openpanel/core';-import { cronQueue } from '@openpanel/queue';import { getRedisCache } from '@openpanel/redis';-import { createLogger, type ILogger } from '../logger';+import type { Logger } from '../logger';+import { generateSecureId } from '../shared/id';++/**+ * What a buffer needs from the boot scope. Buffers are built once by+ * `createBuffers(deps)` and hung on `AppDeps` — never module singletons, so a+ * test constructs its own and closes nothing shared (ADR-007).+ */+export interface BufferDeps {+ /**+ * Each buffer logs under its own name. Injected rather than built here so+ * core instantiates no pino (ADR-007 layout) and a test hands in a silent+ * stub instead of spawning one transport worker thread per buffer.+ */+ createLogger(name: string): Logger;+ /**+ * Asked before every flush. ADR-005's `ProducerHandle.bullQueues` escape+ * hatch, narrowed to the one thing the buffers ask BullMQ directly: pausing+ * `cron` from bull-board halts ALL buffer flushing, preserved deliberately+ * (docs/ANSWERS.md §3: "known!").+ */+ isCronPaused(): Promise<boolean>;+}export type FlushPhaseTimings = {lrangeMs?: number;@@ -57,7 +77,7 @@ export type AddObserver = (obs: AddObservation) => void;export class BaseBuffer {name: string;- logger: ILogger;+ logger: Logger;lockKey: string;lockTimeout = 60;onFlush: () => Promise<void> | void;@@ -98,12 +118,15 @@ export class BaseBuffer {/** Optional hook for instrumenting add() latency. */public addObserver: AddObserver | null = null;- constructor(options: {- name: string;- onFlush: () => Promise<void>;- enableParallelProcessing?: boolean;- }) {- this.logger = createLogger({ name: options.name });+ constructor(+ private readonly deps: BufferDeps,+ options: {+ name: string;+ onFlush: () => Promise<void>;+ enableParallelProcessing?: boolean;+ }+ ) {+ this.logger = deps.createLogger(options.name);this.name = options.name;this.lockKey = `lock:${this.name}`;this.onFlush = options.onFlush;@@ -387,7 +410,7 @@ export class BaseBuffer {// best-effort}- const isCronQueuePaused = await cronQueue.isPaused();+ const isCronQueuePaused = await this.deps.isCronPaused();if (isCronQueuePaused) {this.emitFlushObservation({buffer: this.name,diff --git a/packages/db/src/buffers/bot-buffer.ts b/packages/core/src/buffers/bot-buffer.tssimilarity index 88%rename from packages/db/src/buffers/bot-buffer.tsrename to packages/core/src/buffers/bot-buffer.tsindex 3224647c6..5d917c336 100644--- a/packages/db/src/buffers/bot-buffer.ts+++ b/packages/core/src/buffers/bot-buffer.ts@@ -1,7 +1,7 @@import { getRedisCache, type Redis } from '@openpanel/redis';-import { ch, TABLE_NAMES } from '../clickhouse/client';-import type { IClickhouseBotEvent } from '../services/event.service';-import { BaseBuffer } from './base-buffer';+import type { IClickhouseBotEvent } from '../modules/event/event.service';+import { BaseBuffer, type BufferDeps } from './base-buffer';+import { loadClickHouse } from './clickhouse';export class BotBuffer extends BaseBuffer {private batchSize = process.env.BOT_BUFFER_BATCH_SIZE@@ -10,8 +10,8 @@ export class BotBuffer extends BaseBuffer {private readonly redisKey = 'bot-events-buffer';private redis: Redis;- constructor() {- super({+ constructor(deps: BufferDeps) {+ super(deps, {name: 'bot',onFlush: async () => {await this.processBuffer();@@ -60,6 +60,7 @@ export class BotBuffer extends BaseBuffer {// Raw passthrough: each Redis entry is already a valid JSONEachRow// line. Streaming raw strings skips JSON.parse + the client's// re-stringify on the hot path.+ const { ch, TABLE_NAMES } = await loadClickHouse();const chStart = performance.now();await ch.insert({table: TABLE_NAMES.events_bots,diff --git a/packages/core/src/buffers/buffer.metrics.ts b/packages/core/src/buffers/buffer.metrics.tsnew file mode 100644index 000000000..cdfb16963--- /dev/null+++ b/packages/core/src/buffers/buffer.metrics.ts@@ -0,0 +1,229 @@+// The twelve `buffer_*` series, moved verbatim from apps/worker/src/metrics.ts+// (TARGET_ARCHITECTURE §18: names and labels reproduced exactly). Registered+// per boot rather than per import, because the buffers themselves are per boot+// — and only where a role consumes queues, since the LLEN gauges cost one+// Redis round trip per buffer per scrape.++import client from 'prom-client';+import { registry } from '../metrics';+import type { Buffers } from './create-buffers';++const FLUSH_DURATION_BUCKETS_MS = [+ 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,+];+const REDIS_OP_BUCKETS_MS = [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000];+const CH_DURATION_BUCKETS_MS = [+ 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,+];+const LLEN_BUCKETS = [+ 0, 10, 100, 500, 1000, 5000, 10_000, 50_000, 100_000, 500_000, 1_000_000,+ 5_000_000,+];+const ADD_DURATION_BUCKETS_MS = [+ 0.5, 1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 5000,+];++/**+ * Registration order is V1's, because it is the order the series appear in the+ * `/metrics` body.+ */+function allBuffers(buffers: Buffers) {+ return [+ buffers.event,+ buffers.profile,+ buffers.bot,+ buffers.session,+ buffers.replay,+ buffers.group,+ buffers.profileBackfill,+ ];+}++/**+ * Registers the buffer metrics and installs the flush/add observers on the+ * given buffers. Call once per process: prom-client throws on a duplicate+ * series name, and each buffer holds exactly one observer slot.+ *+ * `register` defaults to the one core registry; V1's worker passes its own+ * dying registry so its `/metrics` body is unchanged (M8-001 delegation).+ */+export function registerBufferMetrics(+ buffers: Buffers,+ register: client.Registry = registry+): void {+ const bufferList = allBuffers(buffers);++ // Ground-truth LLEN of each buffer's main list. O(1) — no shadow counter.+ for (const buf of bufferList) {+ register.registerMetric(+ new client.Gauge({+ name: `buffer_${buf.name.replace(/-/g, '_')}_count`,+ help: 'Buffer size (LLEN of the Redis list)',+ async collect() {+ try {+ this.set(await buf.getBufferSize());+ } catch {+ // ignore — scrape continues+ }+ },+ })+ );+ }++ // Number of events sitting in event-buffer's in-process micro-batch+ // (pre-Redis). Other buffers don't have a local layer.+ register.registerMetric(+ new client.Gauge({+ name: 'buffer_event_pending_local_count',+ help: 'Events in event-buffer process-local micro-batch (not yet in Redis)',+ collect() {+ try {+ this.set(buffers.event.getPendingLocalCount());+ } catch {+ // ignore+ }+ },+ })+ );++ // ---- Flush metrics (populated via flushObserver hooks) ----++ const flushDuration = new client.Histogram({+ name: 'buffer_flush_duration_ms',+ help: 'Wall time of a tryFlush call, including lock acquisition',+ labelNames: ['buffer', 'result', 'trigger'],+ buckets: FLUSH_DURATION_BUCKETS_MS,+ });+ register.registerMetric(flushDuration);++ const flushTotal = new client.Counter({+ name: 'buffer_flush_total',+ help: 'Count of tryFlush invocations by result (success/error/locked/paused) and trigger (add/cron)',+ labelNames: ['buffer', 'result', 'trigger'],+ });+ register.registerMetric(flushTotal);++ const flushRowsTotal = new client.Counter({+ name: 'buffer_flush_rows_total',+ help: 'Rows drained from the buffer per flush (sum)',+ labelNames: ['buffer'],+ });+ register.registerMetric(flushRowsTotal);++ // Per-phase Redis op timing on the flush hot path.+ const redisOpDurationMs = new client.Histogram({+ name: 'buffer_redis_op_duration_ms',+ help: 'Duration of a Redis op during flush',+ labelNames: ['buffer', 'op'],+ buckets: REDIS_OP_BUCKETS_MS,+ });+ register.registerMetric(redisOpDurationMs);++ const chInsertDurationMs = new client.Histogram({+ name: 'buffer_ch_insert_duration_ms',+ help: 'Duration of the ClickHouse insert(s) inside a single flush',+ labelNames: ['buffer'],+ buckets: CH_DURATION_BUCKETS_MS,+ });+ register.registerMetric(chInsertDurationMs);++ // CH SELECT latency inside a flush (e.g. profile-buffer's fetch-existing+ // profiles for merge). Separated from ch_insert because for some buffers+ // the SELECT dominates total flush time.+ const chFetchDurationMs = new client.Histogram({+ name: 'buffer_ch_fetch_duration_ms',+ help: 'Duration of CH SELECT(s) inside a single flush (e.g. profile merge fetch)',+ labelNames: ['buffer'],+ buckets: CH_DURATION_BUCKETS_MS,+ });+ register.registerMetric(chFetchDurationMs);++ const flushLlenAtStart = new client.Histogram({+ name: 'buffer_flush_llen_at_start',+ help: 'LLEN observed at the start of a flush attempt',+ labelNames: ['buffer'],+ buckets: LLEN_BUCKETS,+ });+ register.registerMetric(flushLlenAtStart);++ // ---- Add-path metrics ----++ const addDurationMs = new client.Histogram({+ name: 'buffer_add_duration_ms',+ help: 'Duration of a single add() call (per-event ingest path)',+ labelNames: ['buffer'],+ buckets: ADD_DURATION_BUCKETS_MS,+ });+ register.registerMetric(addDurationMs);++ const addTotal = new client.Counter({+ name: 'buffer_add_total',+ help: 'Total add() calls per buffer that actually enqueued (excludes skipped).',+ labelNames: ['buffer'],+ });+ register.registerMetric(addTotal);++ const addSkippedTotal = new client.Counter({+ name: 'buffer_add_skipped_total',+ help: 'add() calls that short-circuited (no enqueue). Reason: cached, etc.',+ labelNames: ['buffer', 'reason'],+ });+ register.registerMetric(addSkippedTotal);++ for (const buf of bufferList) {+ buf.flushObserver = (obs) => {+ flushTotal.inc({+ buffer: obs.buffer,+ result: obs.result,+ trigger: obs.trigger,+ });+ flushDuration.observe(+ { buffer: obs.buffer, result: obs.result, trigger: obs.trigger },+ obs.totalMs+ );++ if (obs.llenAtStart != null) {+ flushLlenAtStart.observe({ buffer: obs.buffer }, obs.llenAtStart);+ }++ if (obs.rowsProcessed != null && obs.rowsProcessed > 0) {+ flushRowsTotal.inc({ buffer: obs.buffer }, obs.rowsProcessed);+ }++ if (obs.phases?.lrangeMs != null) {+ redisOpDurationMs.observe(+ { buffer: obs.buffer, op: 'lrange' },+ obs.phases.lrangeMs+ );+ }+ if (obs.phases?.trimMs != null) {+ redisOpDurationMs.observe(+ { buffer: obs.buffer, op: 'trim' },+ obs.phases.trimMs+ );+ }+ if (obs.phases?.chFetchMs != null) {+ chFetchDurationMs.observe({ buffer: obs.buffer }, obs.phases.chFetchMs);+ }+ if (obs.phases?.chInsertMs != null) {+ chInsertDurationMs.observe(+ { buffer: obs.buffer },+ obs.phases.chInsertMs+ );+ }+ };++ buf.addObserver = (obs) => {+ if (obs.skipped) {+ addSkippedTotal.inc({+ buffer: obs.buffer,+ reason: obs.skipReason ?? 'unknown',+ });+ // Don't pollute add-latency histogram with no-op fast paths+ return;+ }+ addTotal.inc({ buffer: obs.buffer });+ addDurationMs.observe({ buffer: obs.buffer }, obs.durationMs);+ };+ }+}diff --git a/packages/core/src/buffers/clickhouse.ts b/packages/core/src/buffers/clickhouse.tsnew file mode 100644index 000000000..a6824616e--- /dev/null+++ b/packages/core/src/buffers/clickhouse.ts@@ -0,0 +1,15 @@+// @openpanel/db's clickhouse/client.ts builds a real client and a real pino+// logger (with a pino-pretty worker thread) the moment it is imported, and+// `createBuffers` sits on the curated barrel — a static import here would pay+// that once per `bun test --isolate` file. Same choice insight.service.ts and+// event.service.ts made, for the same reason. Resolved once per process; a+// flush pays one already-settled promise.++type ClickHouseModule = typeof import('@openpanel/db/src/clickhouse/client');++let pending: Promise<ClickHouseModule> | undefined;++export function loadClickHouse(): Promise<ClickHouseModule> {+ pending ??= import('@openpanel/db/src/clickhouse/client');+ return pending;+}diff --git a/packages/core/src/buffers/create-buffers.ts b/packages/core/src/buffers/create-buffers.tsnew file mode 100644index 000000000..f34a031f9--- /dev/null+++ b/packages/core/src/buffers/create-buffers.ts@@ -0,0 +1,36 @@+// Built ONCE at boot and hung on `AppDeps.buffers` — never module singletons+// (ADR-007: "buffers stay boot singletons on AppDeps"). V1's+// `packages/db/src/buffers/index.ts` is a thin delegate over one call to this+// factory, so a V1 process and a V2 process reach the same seven objects and+// the same Redis lists.++import type { BufferDeps } from './base-buffer';+import { BotBuffer } from './bot-buffer';+import { EventBuffer } from './event-buffer';+import { GroupBuffer } from './group-buffer';+import { ProfileBackfillBuffer } from './profile-backfill-buffer';+import { ProfileBuffer } from './profile-buffer';+import { ReplayBuffer } from './replay-buffer';+import { SessionBuffer } from './session-buffer';++export interface Buffers {+ event: EventBuffer;+ profile: ProfileBuffer;+ profileBackfill: ProfileBackfillBuffer;+ bot: BotBuffer;+ session: SessionBuffer;+ replay: ReplayBuffer;+ group: GroupBuffer;+}++export function createBuffers(deps: BufferDeps): Buffers {+ return {+ event: new EventBuffer(deps),+ profile: new ProfileBuffer(deps),+ profileBackfill: new ProfileBackfillBuffer(deps),+ bot: new BotBuffer(deps),+ session: new SessionBuffer(deps),+ replay: new ReplayBuffer(deps),+ group: new GroupBuffer(deps),+ };+}diff --git a/packages/db/src/buffers/event-buffer.test.ts b/packages/core/src/buffers/event-buffer.test.tssimilarity index 81%rename from packages/db/src/buffers/event-buffer.test.tsrename to packages/core/src/buffers/event-buffer.test.tsindex 66e95f130..910ced559 100644--- a/packages/db/src/buffers/event-buffer.test.ts+++ b/packages/core/src/buffers/event-buffer.test.ts@@ -1,13 +1,26 @@+import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';import type { Readable } from 'node:stream';import { getRedisCache } from '@openpanel/redis';-import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';-import * as chClient from '../clickhouse/client';-const { ch } = chClient;+import { stubBufferDeps } from '../../test/buffer-fixtures';-// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer-vi.mock('../services/event.service', () => ({}));+// `./clickhouse` is the one seam the buffers reach ClickHouse through, and it+// is the right thing to replace: mocking `@openpanel/db/src/clickhouse/client`+// instead would need the real module imported first (to spread and restore it,+// since `mock.module` replaces a specifier process-wide without `--isolate`),+// and importing it builds a client and a pino transport worker thread per test+// file. This module has one export, so the replacement is complete rather than+// partial — and it is restored in afterAll regardless.+const realClickhouse = { ...(await import('./clickhouse')) };-import { EventBuffer, extractProjectId } from './event-buffer';+const chInsert = mock(async (_options: unknown): Promise<unknown> => undefined);+const chQuery = mock(async (_sql: string): Promise<unknown[]> => []);++mock.module('./clickhouse', () => ({+ loadClickHouse: () => Promise.resolve({ ch: { insert: chInsert }, chQuery }),+}));++const { EventBuffer, extractProjectId } = await import('./event-buffer');+type EventBuffer = InstanceType<typeof EventBuffer>;/** Drain an object-mode Readable into an array of line strings. event-buffer* yields each JSONEachRow row as its own string chunk; the @clickhouse/client@@ -25,20 +38,25 @@ const redis = getRedisCache();beforeEach(async () => {const keys = await redis.keys('event*');- if (keys.length > 0) await redis.del(...keys);+ if (keys.length > 0) {+ await redis.del(...keys);+ }+ chInsert.mockClear();+ chQuery.mockClear();});-afterAll(async () => {- try {- await redis.quit();- } catch {}+// The shared `getRedisCache()` client is deliberately NOT quit here: bun runs+// every file in one process, and closing the singleton takes it away from the+// files that run next (V1's vitest isolated per file, so it could).+afterAll(() => {+ mock.module('./clickhouse', () => realClickhouse);});describe('EventBuffer', () => {let eventBuffer: EventBuffer;beforeEach(() => {- eventBuffer = new EventBuffer();+ eventBuffer = new EventBuffer(stubBufferDeps);});it('adds regular event directly to buffer queue', async () => {@@ -174,14 +192,14 @@ describe('EventBuffer', () => {expect(await eventBuffer.getBufferSize()).toBe(2);- const insertSpy = vi- .spyOn(ch, 'insert')- .mockResolvedValueOnce(undefined as any);-await eventBuffer.processBuffer();- expect(insertSpy).toHaveBeenCalled();- const callArgs = insertSpy.mock.calls[0]![0];+ expect(chInsert).toHaveBeenCalled();+ const callArgs = chInsert.mock.calls[0]![0] as {+ format: string;+ table: string;+ values: unknown;+ };expect(callArgs.format).toBe('JSONEachRow');expect(callArgs.table).toBe('events');// After the raw-passthrough optimisation, `values` is an@@ -201,14 +219,12 @@ describe('EventBuffer', () => {expect(JSON.parse(lines[1]!).name).toBe('event2');expect(await eventBuffer.getBufferSize()).toBe(0);-- insertSpy.mockRestore();});it('processes buffer with chunking', async () => {const prev = process.env.EVENT_BUFFER_CHUNK_SIZE;process.env.EVENT_BUFFER_CHUNK_SIZE = '2';- const eb = new EventBuffer();+ const eb = new EventBuffer(stubBufferDeps);for (let i = 0; i < 4; i++) {eb.add({@@ -219,39 +235,32 @@ describe('EventBuffer', () => {}await eb.flush();- const insertSpy = vi- .spyOn(ch, 'insert')- .mockResolvedValue(undefined as any);-await eb.processBuffer();- expect(insertSpy).toHaveBeenCalledTimes(2);+ expect(chInsert).toHaveBeenCalledTimes(2);const call1 = await streamToLines(- insertSpy.mock.calls[0]![0].values as Readable,+ (chInsert.mock.calls[0]![0] as { values: Readable }).values);const call2 = await streamToLines(- insertSpy.mock.calls[1]![0].values as Readable,+ (chInsert.mock.calls[1]![0] as { values: Readable }).values);expect(call1.length).toBe(2);expect(call2.length).toBe(2);- if (prev === undefined) delete process.env.EVENT_BUFFER_CHUNK_SIZE;- else process.env.EVENT_BUFFER_CHUNK_SIZE = prev;-- insertSpy.mockRestore();+ if (prev === undefined) {+ delete process.env.EVENT_BUFFER_CHUNK_SIZE;+ } else {+ process.env.EVENT_BUFFER_CHUNK_SIZE = prev;+ }});it('tracks active visitors', async () => {- const querySpy = vi- .spyOn(chClient, 'chQuery')- .mockResolvedValueOnce([{ count: 2 }] as any);+ chQuery.mockResolvedValueOnce([{ count: 2 }]);const count = await eventBuffer.getActiveVisitorCount('p9');expect(count).toBe(2);- expect(querySpy).toHaveBeenCalledOnce();- expect(querySpy.mock.calls[0]![0]).toContain("project_id = 'p9'");-- querySpy.mockRestore();+ expect(chQuery).toHaveBeenCalledTimes(1);+ expect(chQuery.mock.calls[0]![0]).toContain("project_id = 'p9'");});it('handles multiple sessions independently — all events go to buffer', async () => {@@ -313,18 +322,14 @@ describe('EventBuffer', () => {} as any);await eventBuffer.flush();- const insertSpy = vi- .spyOn(ch, 'insert')- .mockRejectedValueOnce(new Error('ClickHouse unavailable'));+ chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));// Errors propagate to tryFlush (which resyncs the counter). The safety// property — queue preserved on CH failure — still holds.await expect(eventBuffer.processBuffer()).rejects.toThrow(- 'ClickHouse unavailable',+ 'ClickHouse unavailable');expect(await eventBuffer.getBufferSize()).toBe(1);-- insertSpy.mockRestore();});});diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/core/src/buffers/event-buffer.tssimilarity index 93%rename from packages/db/src/buffers/event-buffer.tsrename to packages/core/src/buffers/event-buffer.tsindex d29003f63..896107dd2 100644--- a/packages/db/src/buffers/event-buffer.ts+++ b/packages/core/src/buffers/event-buffer.ts@@ -1,7 +1,7 @@import { getRedisCache, publishEvent } from '@openpanel/redis';-import { ch, chQuery } from '../clickhouse/client';-import type { IClickhouseEvent } from '../services/event.service';-import { BaseBuffer } from './base-buffer';+import type { IClickhouseEvent } from '../modules/event/event.service';+import { BaseBuffer, type BufferDeps } from './base-buffer';+import { loadClickHouse } from './clickhouse';const PROJECT_ID_NEEDLE = '"project_id":"';@@ -26,9 +26,14 @@ const PROJECT_ID_NEEDLE = '"project_id":"';*/export function extractProjectId(line: string): string | null {const first = line.indexOf(PROJECT_ID_NEEDLE);- if (first < 0) return null;+ if (first < 0) {+ return null;+ }- const second = line.indexOf(PROJECT_ID_NEEDLE, first + PROJECT_ID_NEEDLE.length);+ const second = line.indexOf(+ PROJECT_ID_NEEDLE,+ first + PROJECT_ID_NEEDLE.length+ );if (second >= 0) {try {const obj = JSON.parse(line) as { project_id?: unknown };@@ -43,7 +48,9 @@ export function extractProjectId(line: string): string | null {// valueEnd === valueStart means the value is empty (`"project_id":""`)// — treat as missing so we don't pollute pub/sub counts with an empty// key. Matches the old regex's `[^"]+` (one-or-more) behavior.- if (valueEnd <= valueStart) return null;+ if (valueEnd <= valueStart) {+ return null;+ }return line.slice(valueStart, valueEnd);}@@ -70,8 +77,8 @@ export class EventBuffer extends BaseBuffer {private queueKey = 'event_buffer:queue';- constructor() {- super({+ constructor(deps: BufferDeps) {+ super(deps, {name: 'event',onFlush: async () => {await this.processBuffer();@@ -212,16 +219,14 @@ export class EventBuffer extends BaseBuffer {for (let i = 0; i < queueEvents.length; i++) {const projectId = extractProjectId(queueEvents[i]!);if (projectId) {- countByProject.set(- projectId,- (countByProject.get(projectId) ?? 0) + 1,- );+ countByProject.set(projectId, (countByProject.get(projectId) ?? 0) + 1);}if ((i + 1) % yieldEvery === 0) {await this.yieldToEventLoop();}}+ const { ch } = await loadClickHouse();const chStart = performance.now();await this.parallelLimit(this.chunks(queueEvents, this.chunkSize),@@ -233,7 +238,7 @@ export class EventBuffer extends BaseBuffer {values: this.jsonEachRowStream(chunk),format: 'JSONEachRow',clickhouse_settings: this.getClickhouseSettings(),- }),+ }));const chInsertMs = performance.now() - chStart;@@ -252,6 +257,7 @@ export class EventBuffer extends BaseBuffer {}public async getActiveVisitorCount(projectId: string): Promise<number> {+ const { chQuery } = await loadClickHouse();const rows = await chQuery<{ count: number }>(`SELECT uniq(profile_id) AS countFROM eventsdiff --git a/packages/db/src/buffers/group-buffer.ts b/packages/core/src/buffers/group-buffer.tssimilarity index 93%rename from packages/db/src/buffers/group-buffer.tsrename to packages/core/src/buffers/group-buffer.tsindex feb66fcfd..0462fb469 100644--- a/packages/db/src/buffers/group-buffer.ts+++ b/packages/core/src/buffers/group-buffer.ts@@ -1,15 +1,10 @@import { toDots } from '@openpanel/common';-import { getSafeJson } from '@openpanel/json';import { getRedisCache, type Redis } from '@openpanel/redis';import shallowEqual from 'fast-deep-equal';import sqlstring from 'sqlstring';-import {- ch,- chQuery,- formatClickhouseDate,- TABLE_NAMES,-} from '../clickhouse/client';-import { BaseBuffer } from './base-buffer';+import { getSafeJson } from '../shared/json';+import { BaseBuffer, type BufferDeps } from './base-buffer';+import { loadClickHouse } from './clickhouse';type IGroupBufferEntry = {project_id: string;@@ -55,8 +50,8 @@ export class GroupBuffer extends BaseBuffer {private redis: Redis;- constructor() {- super({+ constructor(deps: BufferDeps) {+ super(deps, {name: 'group',onFlush: async () => {await this.processBuffer();@@ -84,6 +79,7 @@ export class GroupBuffer extends BaseBuffer {projectId: string,id: string): Promise<IGroupCacheEntry | null> {+ const { chQuery, TABLE_NAMES } = await loadClickHouse();const rows = await chQuery<IGroupCacheEntry>(`SELECT project_id, id, type, name, properties, created_atFROM ${TABLE_NAMES.groups} FINAL@@ -97,6 +93,7 @@ export class GroupBuffer extends BaseBuffer {async add(input: IGroupBufferInput): Promise<void> {return this.timeAdd(async () => {try {+ const { formatClickhouseDate } = await loadClickHouse();const cacheKey = this.getCacheKey(input.projectId, input.id);const existing =@@ -179,6 +176,7 @@ export class GroupBuffer extends BaseBuffer {// Raw passthrough: each Redis entry is already a valid JSONEachRow// line. Streaming raw strings to CH skips JSON.parse + the client's// re-stringify on the hot path.+ const { ch, TABLE_NAMES } = await loadClickHouse();const chStart = performance.now();await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>ch.insert({@@ -186,7 +184,7 @@ export class GroupBuffer extends BaseBuffer {values: this.jsonEachRowStream(chunk),format: 'JSONEachRow',clickhouse_settings: this.getClickhouseSettings(),- }),+ }));const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/buffers/partial-json-match.ts b/packages/core/src/buffers/partial-json-match.tssimilarity index 90%rename from packages/db/src/buffers/partial-json-match.tsrename to packages/core/src/buffers/partial-json-match.tsindex 84e0340ac..8be454410 100644--- a/packages/db/src/buffers/partial-json-match.ts+++ b/packages/core/src/buffers/partial-json-match.ts@@ -22,10 +22,7 @@ export function isPartialMatch(source: any, partial: any): boolean {// Check each property in partialfor (const key in partial) {- if (- Object.prototype.hasOwnProperty.call(partial, key) &&- partial[key] !== undefined- ) {+ if (Object.hasOwn(partial, key) && partial[key] !== undefined) {// If property doesn't exist in source, no matchif (!(key in source)) {return false;diff --git a/packages/db/src/buffers/profile-backfill-buffer.ts b/packages/core/src/buffers/profile-backfill-buffer.tssimilarity index 79%rename from packages/db/src/buffers/profile-backfill-buffer.tsrename to packages/core/src/buffers/profile-backfill-buffer.tsindex 163137dfb..31b050ed6 100644--- a/packages/db/src/buffers/profile-backfill-buffer.ts+++ b/packages/core/src/buffers/profile-backfill-buffer.ts@@ -1,8 +1,8 @@-import { getSafeJson } from '@openpanel/json';-import { type Redis, getRedisCache } from '@openpanel/redis';+import { getRedisCache, type Redis } from '@openpanel/redis';import sqlstring from 'sqlstring';-import { TABLE_NAMES, ch, getReplicatedTableName } from '../clickhouse/client';-import { BaseBuffer } from './base-buffer';+import { getSafeJson } from '../shared/json';+import { BaseBuffer, type BufferDeps } from './base-buffer';+import { loadClickHouse } from './clickhouse';export interface ProfileBackfillEntry {projectId: string;@@ -21,8 +21,8 @@ export class ProfileBackfillBuffer extends BaseBuffer {private readonly redisKey = 'profile-backfill-buffer';private redis: Redis;- constructor() {- super({+ constructor(deps: BufferDeps) {+ super(deps, {name: 'profile-backfill',onFlush: async () => {await this.processBuffer();@@ -39,13 +39,12 @@ export class ProfileBackfillBuffer extends BaseBuffer {} catch (error) {this.logger.error({ err: error },- 'Failed to add profile backfill entry',+ 'Failed to add profile backfill entry');}});}-protected getRedisListKey(): string {return this.redisKey;}@@ -70,6 +69,7 @@ export class ProfileBackfillBuffer extends BaseBuffer {}const entries = Array.from(seen.values());+ const { ch, TABLE_NAMES, getReplicatedTableName } = await loadClickHouse();const table = getReplicatedTableName(TABLE_NAMES.events);const chunks = this.chunks(entries, CHUNK_SIZE);@@ -78,10 +78,16 @@ export class ProfileBackfillBuffer extends BaseBuffer {const chStart = performance.now();for (const chunk of chunks) {const caseClause = chunk- .map(({ sessionId, profileId }) => `WHEN ${sqlstring.escape(sessionId)} THEN ${sqlstring.escape(profileId)}`)+ .map(+ ({ sessionId, profileId }) =>+ `WHEN ${sqlstring.escape(sessionId)} THEN ${sqlstring.escape(profileId)}`+ ).join('\n');const tupleList = chunk- .map(({ projectId, sessionId }) => `(${sqlstring.escape(projectId)}, ${sqlstring.escape(sessionId)})`)+ .map(+ ({ projectId, sessionId }) =>+ `(${sqlstring.escape(projectId)}, ${sqlstring.escape(sessionId)})`+ ).join(',');const query = `@@ -96,7 +102,7 @@ export class ProfileBackfillBuffer extends BaseBuffer {query,clickhouse_settings: {mutations_sync: '0',- allow_experimental_lightweight_update: '1'+ allow_experimental_lightweight_update: '1',},});diff --git a/packages/db/src/buffers/profile-buffer.test.ts b/packages/core/src/buffers/profile-buffer.test.tssimilarity index 70%rename from packages/db/src/buffers/profile-buffer.test.tsrename to packages/core/src/buffers/profile-buffer.test.tsindex b0a9cb8d1..1257aa172 100644--- a/packages/db/src/buffers/profile-buffer.test.ts+++ b/packages/core/src/buffers/profile-buffer.test.ts@@ -1,24 +1,33 @@+import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';import { getRedisCache } from '@openpanel/redis';-import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';-import type { IClickhouseProfile } from '../services/profile.service';--// Mock chQuery to avoid hitting real ClickHouse-vi.mock('../clickhouse/client', () => ({- ch: {- insert: vi.fn().mockResolvedValue(undefined),- },- chQuery: vi.fn().mockResolvedValue([]),- TABLE_NAMES: {- profiles: 'profiles',- },+import { stubBufferDeps } from '../../test/buffer-fixtures';+import type { IClickhouseProfile } from '../modules/profile/profile.service';++// `./clickhouse` is the one seam the buffers reach ClickHouse through — see+// event-buffer.test.ts's header for why that, and not+// `@openpanel/db/src/clickhouse/client`, is what gets replaced.+const realClickhouse = { ...(await import('./clickhouse')) };++const chInsert = mock(async (_options: unknown): Promise<unknown> => undefined);+const chQuery = mock(async (_sql: string): Promise<IClickhouseProfile[]> => []);++mock.module('./clickhouse', () => ({+ loadClickHouse: () =>+ Promise.resolve({+ ch: { insert: chInsert },+ chQuery,+ TABLE_NAMES: { profiles: 'profiles' },+ }),}));-import { ProfileBuffer } from './profile-buffer';-import { chQuery } from '../clickhouse/client';+const { ProfileBuffer } = await import('./profile-buffer');+type ProfileBuffer = InstanceType<typeof ProfileBuffer>;const redis = getRedisCache();-function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile {+function makeProfile(+ overrides: Partial<IClickhouseProfile>+): IClickhouseProfile {const now = new Date().toISOString();return {id: 'profile-1',@@ -38,28 +47,36 @@ function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfilebeforeEach(async () => {const keys = [- ...await redis.keys('profile*'),- ...await redis.keys('lock:profile'),+ ...(await redis.keys('profile*')),+ ...(await redis.keys('lock:profile')),];- if (keys.length > 0) await redis.del(...keys);- vi.mocked(chQuery).mockResolvedValue([]);+ if (keys.length > 0) {+ await redis.del(...keys);+ }+ chInsert.mockClear();+ chQuery.mockReset();+ chQuery.mockResolvedValue([]);});-afterAll(async () => {- try {- await redis.quit();- } catch {}+// The shared `getRedisCache()` client is deliberately NOT quit here: bun runs+// every file in one process, and closing the singleton takes it away from the+// files that run next (V1's vitest isolated per file, so it could).+afterAll(() => {+ mock.module('./clickhouse', () => realClickhouse);});describe('ProfileBuffer', () => {let profileBuffer: ProfileBuffer;beforeEach(() => {- profileBuffer = new ProfileBuffer();+ profileBuffer = new ProfileBuffer(stubBufferDeps);});it('adds a profile to the buffer', async () => {- const profile = makeProfile({ first_name: 'John', email: '[email protected]' });+ const profile = makeProfile({+ first_name: 'John',+ email: '[email protected]',+ });const sizeBefore = await profileBuffer.getBufferSize();await profileBuffer.add(profile);@@ -143,7 +160,7 @@ describe('ProfileBuffer', () => {email: '[email protected]',groups: ['existing-group'],});- vi.mocked(chQuery).mockResolvedValue([existingInClickhouse]);+ chQuery.mockResolvedValue([existingInClickhouse]);const incomingProfile = makeProfile({first_name: '',@@ -173,36 +190,24 @@ describe('ProfileBuffer', () => {it('retains profiles in queue when ClickHouse insert fails', async () => {await profileBuffer.add(makeProfile({ first_name: 'John' }));- const { ch } = await import('../clickhouse/client');- const insertSpy = vi- .spyOn(ch, 'insert')- .mockRejectedValueOnce(new Error('ClickHouse unavailable'));+ chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));// Errors propagate to tryFlush (which resyncs the counter). The safety// property — queue preserved on CH failure — still holds.await expect(profileBuffer.processBuffer()).rejects.toThrow(- 'ClickHouse unavailable',+ 'ClickHouse unavailable');expect(await profileBuffer.getBufferSize()).toBe(1);-- insertSpy.mockRestore();});it('proceeds with insert when ClickHouse fetch fails (treats profiles as new)', async () => {- vi.mocked(chQuery).mockRejectedValueOnce(new Error('ClickHouse unavailable'));-- const { ch } = await import('../clickhouse/client');- const insertSpy = vi- .spyOn(ch, 'insert')- .mockResolvedValueOnce(undefined as any);+ chQuery.mockRejectedValueOnce(new Error('ClickHouse unavailable'));await profileBuffer.add(makeProfile({ first_name: 'John' }));await profileBuffer.processBuffer();// Insert must still have been called — no data loss even when fetch fails- expect(insertSpy).toHaveBeenCalled();+ expect(chInsert).toHaveBeenCalled();expect(await profileBuffer.getBufferSize()).toBe(0);-- insertSpy.mockRestore();});});diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/core/src/buffers/profile-buffer.tssimilarity index 96%rename from packages/db/src/buffers/profile-buffer.tsrename to packages/core/src/buffers/profile-buffer.tsindex 554467d4c..c16eccdf9 100644--- a/packages/db/src/buffers/profile-buffer.ts+++ b/packages/core/src/buffers/profile-buffer.ts@@ -1,11 +1,11 @@import { deepMergeObjects } from '@openpanel/common';-import { getSafeJson } from '@openpanel/json';import { getRedisCache, type Redis } from '@openpanel/redis';import { omit, uniq } from 'ramda';import sqlstring from 'sqlstring';-import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';-import type { IClickhouseProfile } from '../services/profile.service';-import { BaseBuffer } from './base-buffer';+import type { IClickhouseProfile } from '../modules/profile/profile.service';+import { getSafeJson } from '../shared/json';+import { BaseBuffer, type BufferDeps } from './base-buffer';+import { loadClickHouse } from './clickhouse';// Inlined to avoid a circular value-import with `profile.service.ts`// (which imports `profileBuffer` from `../buffers`). Keep this in sync@@ -62,8 +62,8 @@ export class ProfileBuffer extends BaseBuffer {private readonly redis: Redis;- constructor() {- super({+ constructor(deps: BufferDeps) {+ super(deps, {name: 'profile',onFlush: async () => {await this.processBuffer();@@ -190,6 +190,7 @@ export class ProfileBuffer extends BaseBuffer {private async batchFetchFromClickhouse(profiles: IClickhouseProfile[]): Promise<Map<string, IClickhouseProfile>> {+ const { chQuery, TABLE_NAMES } = await loadClickHouse();const result = new Map<string, IClickhouseProfile>();// Non-external (anonymous/device) profiles get a 2-day recency filter to@@ -276,11 +277,13 @@ export class ProfileBuffer extends BaseBuffer {const mergedInBatch = new Map<string, IClickhouseProfile>();for (let i = 0; i < rawProfiles.length; i++) {const profile = getSafeJson<IClickhouseProfile>(rawProfiles[i]!);- if (!profile) continue;+ if (!profile) {+ continue;+ }const key = `${profile.project_id}:${profile.id}`;mergedInBatch.set(key,- this.mergeProfiles(mergedInBatch.get(key) ?? null, profile),+ this.mergeProfiles(mergedInBatch.get(key) ?? null, profile));if ((i + 1) % mergeYieldEvery === 0) {await this.yieldToEventLoop();@@ -342,7 +345,7 @@ export class ProfileBuffer extends BaseBuffer {const key = `${profile.project_id}:${profile.id}`;const merged = this.mergeProfiles(existingByKey.get(key) ?? null,- profile,+ profile);toInsert.push(merged);multi.set(@@ -352,13 +355,14 @@ export class ProfileBuffer extends BaseBuffer {}),JSON.stringify(merged),'EX',- this.ttlInSeconds,+ this.ttlInSeconds);if ((i + 1) % finalYieldEvery === 0) {await this.yieldToEventLoop();}}+ const { ch, TABLE_NAMES } = await loadClickHouse();const chStart = performance.now();await this.parallelLimit(this.chunks(toInsert, this.chunkSize), (chunk) =>ch.insert({diff --git a/packages/db/src/buffers/replay-buffer.ts b/packages/core/src/buffers/replay-buffer.tssimilarity index 92%rename from packages/db/src/buffers/replay-buffer.tsrename to packages/core/src/buffers/replay-buffer.tsindex b2c1cefa2..a5aa08f21 100644--- a/packages/db/src/buffers/replay-buffer.ts+++ b/packages/core/src/buffers/replay-buffer.ts@@ -1,6 +1,6 @@import { getRedisCache } from '@openpanel/redis';-import { ch, TABLE_NAMES } from '../clickhouse/client';-import { BaseBuffer } from './base-buffer';+import { BaseBuffer, type BufferDeps } from './base-buffer';+import { loadClickHouse } from './clickhouse';export interface IClickhouseSessionReplayChunk {project_id: string;@@ -23,8 +23,8 @@ export class ReplayBuffer extends BaseBuffer {private readonly redisKey = 'replay-buffer';- constructor() {- super({+ constructor(deps: BufferDeps) {+ super(deps, {name: 'replay',onFlush: async () => {await this.processBuffer();@@ -76,6 +76,7 @@ export class ReplayBuffer extends BaseBuffer {// raw strings to CH skips JSON.parse × N on the worker AND the// client's internal JSON.stringify × N — significant because each// rrweb chunk's `payload` is 10–100KB.+ const { ch, TABLE_NAMES } = await loadClickHouse();const chStart = performance.now();await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>ch.insert({@@ -83,7 +84,7 @@ export class ReplayBuffer extends BaseBuffer {values: this.jsonEachRowStream(chunk),format: 'JSONEachRow',clickhouse_settings: this.getClickhouseSettings(),- }),+ }));const chInsertMs = performance.now() - chStart;diff --git a/packages/db/src/buffers/session-buffer.test.ts b/packages/core/src/buffers/session-buffer.test.tssimilarity index 88%rename from packages/db/src/buffers/session-buffer.test.tsrename to packages/core/src/buffers/session-buffer.test.tsindex 45d4b9faf..2bbf587ce 100644--- a/packages/db/src/buffers/session-buffer.test.ts+++ b/packages/core/src/buffers/session-buffer.test.ts@@ -1,4 +1,3 @@-import { getRedisCache } from '@openpanel/redis';import {afterAll,afterEach,@@ -6,21 +5,36 @@ import {describe,expect,it,- vi,-} from 'vitest';-import { ch } from '../clickhouse/client';--vi.mock('../clickhouse/client', () => ({- ch: {- insert: vi.fn().mockResolvedValue(undefined),- },- TABLE_NAMES: {- sessions: 'sessions',- },+ mock,+} from 'bun:test';+import { getRedisCache } from '@openpanel/redis';+import { stubBufferDeps } from '../../test/buffer-fixtures';+import type { IServiceCreateEventPayload } from '../modules/event/event.service';+import type { IClickhouseSession } from '../modules/session/session.service';++// `./clickhouse` is the one seam the buffers reach ClickHouse through — see+// event-buffer.test.ts's header for why that, and not+// `@openpanel/db/src/clickhouse/client`, is what gets replaced.+const realClickhouse = { ...(await import('./clickhouse')) };++const chInsert = mock(+ async (_options: {+ table: string;+ format: string;+ values: IClickhouseSession[];+ }): Promise<unknown> => undefined+);++mock.module('./clickhouse', () => ({+ loadClickHouse: () =>+ Promise.resolve({+ ch: { insert: chInsert },+ TABLE_NAMES: { sessions: 'sessions' },+ }),}));-import type { IServiceCreateEventPayload } from '../services/event.service';-import { SessionBuffer } from './session-buffer';+const { SessionBuffer } = await import('./session-buffer');+type SessionBuffer = InstanceType<typeof SessionBuffer>;const redis = getRedisCache();@@ -74,20 +88,22 @@ beforeEach(async () => {for (let i = 0; i < keys.length; i += BATCH) {await redis.del(...keys.slice(i, i + BATCH));}- vi.mocked(ch.insert).mockResolvedValue(undefined as any);+ chInsert.mockReset();+ chInsert.mockResolvedValue(undefined);});-afterAll(async () => {- try {- await redis.quit();- } catch {}+// The shared `getRedisCache()` client is deliberately NOT quit here: bun runs+// every file in one process, and closing the singleton takes it away from the+// files that run next (V1's vitest isolated per file, so it could).+afterAll(() => {+ mock.module('./clickhouse', () => realClickhouse);});describe('SessionBuffer', () => {let sessionBuffer: SessionBuffer;beforeEach(() => {- sessionBuffer = new SessionBuffer();+ sessionBuffer = new SessionBuffer(stubBufferDeps);});it('opens a new session and stores it at session:{pid}:{did}', async () => {@@ -302,18 +318,12 @@ describe('SessionBuffer', () => {it('processes buffer and inserts sessions into ClickHouse', async () => {await sessionBuffer.ingest(makePayload());- const insertSpy = vi- .spyOn(ch, 'insert')- .mockResolvedValueOnce(undefined as any);-await sessionBuffer.processBuffer();- expect(insertSpy).toHaveBeenCalledWith(+ expect(chInsert).toHaveBeenCalledWith(expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' }));expect(await sessionBuffer.getBufferSize()).toBe(0);-- insertSpy.mockRestore();});it('squash does not orphan the creation row when create + updates share a batch', async () => {@@ -331,14 +341,12 @@ describe('SessionBuffer', () => {}const inserted: Array<{ sign: number; version: number }> = [];- const insertSpy = vi- .spyOn(ch, 'insert')- .mockImplementation(async ({ values }: any) => {- for (const v of values) {- inserted.push({ sign: v.sign, version: v.version });- }- return undefined as any;- });+ chInsert.mockImplementation(async ({ values }) => {+ for (const v of values) {+ inserted.push({ sign: v.sign, version: v.version });+ }+ return await Promise.resolve(undefined);+ });await sessionBuffer.processBuffer();@@ -356,21 +364,19 @@ describe('SessionBuffer', () => {for (const { sign, version } of inserted) {netByVersion.set(version, (netByVersion.get(version) ?? 0) + sign);}- const survivors = [...netByVersion.entries()].filter(([, net]) => net !== 0);+ const survivors = [...netByVersion.entries()].filter(+ ([, net]) => net !== 0+ );// A fresh session must collapse to exactly one final +1 row, with no// orphaned negatives (the bug left a permanent (-1, v1)).expect(survivors).toEqual([[EVENTS, 1]]);-- insertSpy.mockRestore();});it('retains sessions in queue when ClickHouse insert fails', async () => {await sessionBuffer.ingest(makePayload());- const insertSpy = vi- .spyOn(ch, 'insert')- .mockRejectedValueOnce(new Error('ClickHouse unavailable'));+ chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));// Errors now propagate to tryFlush (which handles them by resyncing the// counter). processBuffer no longer swallows — we still verify the@@ -379,8 +385,6 @@ describe('SessionBuffer', () => {'ClickHouse unavailable');expect(await sessionBuffer.getBufferSize()).toBe(1);-- insertSpy.mockRestore();});// persist() is one Lua script, so a failure part-way through must leave thediff --git a/packages/db/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.tssimilarity index 92%rename from packages/db/src/buffers/session-buffer.tsrename to packages/core/src/buffers/session-buffer.tsindex b93684d06..e5309b5c3 100644--- a/packages/db/src/buffers/session-buffer.ts+++ b/packages/core/src/buffers/session-buffer.ts@@ -1,10 +1,10 @@import { DateTime } from '@openpanel/common';-import { getSafeJson } from '@openpanel/json';import { getRedisCache, type Redis } from '@openpanel/redis';-import { ch, TABLE_NAMES } from '../clickhouse/client';-import type { IServiceCreateEventPayload } from '../services/event.service';-import type { IClickhouseSession } from '../services/session.service';-import { BaseBuffer } from './base-buffer';+import type { IServiceCreateEventPayload } from '../modules/event/event.service';+import type { IClickhouseSession } from '../modules/session/session.service';+import { getSafeJson } from '../shared/json';+import { BaseBuffer, type BufferDeps } from './base-buffer';+import { loadClickHouse } from './clickhouse';// 30min of idle in event-time → session ends. Matches industry default.// Idle window for a session (boundary detection + the reaper's default deadman).@@ -18,8 +18,7 @@ const sessionKey = (projectId: string, deviceId: string) =>`session:${projectId}:${deviceId}`;const profileIndexKey = (projectId: string, profileId: string) =>`session:profile:${projectId}:${profileId}`;-const wallclockSetKey = (projectId: string) =>- `session:wallclock:${projectId}`;+const wallclockSetKey = (projectId: string) => `session:wallclock:${projectId}`;const PROJECTS_SET_KEY = 'session:projects';// Atomic id-gated cleanup. Only deletes the blob + sorted-set entry +@@ -91,7 +90,11 @@ return bufferLengthexport type SessionIngestResult =| { kind: 'new'; current: IClickhouseSession }| { kind: 'extend'; current: IClickhouseSession }- | { kind: 'boundary'; current: IClickhouseSession; closed: IClickhouseSession };+ | {+ kind: 'boundary';+ current: IClickhouseSession;+ closed: IClickhouseSession;+ };function toClickhouseDate(date: Date): string {return DateTime.fromJSDate(date)@@ -109,8 +112,9 @@ function pickUtm(payload: IServiceCreateEventPayload,key: 'utm_medium' | 'utm_source' | 'utm_campaign' | 'utm_content' | 'utm_term'): string {- const query = (payload.properties as { __query?: Record<string, unknown> } | undefined)- ?.__query;+ const query = (+ payload.properties as { __query?: Record<string, unknown> } | undefined+ )?.__query;const v = query?.[key];return v ? String(v) : '';}@@ -129,8 +133,8 @@ export class SessionBuffer extends BaseBuffer {private readonly redisKey = 'session-buffer';private redis: Redis;- constructor() {- super({+ constructor(deps: BufferDeps) {+ super(deps, {name: 'session',onFlush: async () => {await this.processBuffer();@@ -157,7 +161,9 @@ export class SessionBuffer extends BaseBuffer {const deviceId = await this.redis.get(profileIndexKey(options.projectId, options.profileId));- if (!deviceId) return null;+ if (!deviceId) {+ return null;+ }const hit = await this.redis.get(sessionKey(options.projectId, deviceId));return hit ? getSafeJson<IClickhouseSession>(hit) : null;}@@ -178,7 +184,9 @@ export class SessionBuffer extends BaseBuffer {async ingest(payload: IServiceCreateEventPayload): Promise<SessionIngestResult | null> {- if (!payload.projectId || !payload.deviceId) return null;+ if (!(payload.projectId && payload.deviceId)) {+ return null;+ }if (payload.name === 'session_start' || payload.name === 'session_end') {return null;}@@ -281,18 +289,29 @@ export class SessionBuffer extends BaseBuffer {if (eventTimeMs >= endMs) {current.ended_at = eventCh;- if (payload.path) current.exit_path = payload.path;- if (payload.origin) current.exit_origin = payload.origin;+ if (payload.path) {+ current.exit_path = payload.path;+ }+ if (payload.origin) {+ current.exit_origin = payload.origin;+ }}if (eventTimeMs < startMs) {current.created_at = eventCh;- if (payload.path) current.entry_path = payload.path;- if (payload.origin) current.entry_origin = payload.origin;+ if (payload.path) {+ current.entry_path = payload.path;+ }+ if (payload.origin) {+ current.entry_origin = payload.origin;+ }} else {- if (!current.entry_path && payload.path) current.entry_path = payload.path;- if (!current.entry_origin && payload.origin)+ if (!current.entry_path && payload.path) {+ current.entry_path = payload.path;+ }+ if (!current.entry_origin && payload.origin) {current.entry_origin = payload.origin;+ }}current.duration =@@ -520,12 +539,16 @@ export class SessionBuffer extends BaseBuffer {0,this.batchSize - 1);- if (events.length === 0) return;+ if (events.length === 0) {+ return;+ }const parsed: IClickhouseSession[] = [];for (const e of events) {const s = getSafeJson<IClickhouseSession>(e);- if (!s) continue;+ if (!s) {+ continue;+ }// Freshly parsed object — safe to clamp in place, no clone needed.s.duration = Math.max(0, s.duration || 0);parsed.push(s);@@ -543,6 +566,7 @@ export class SessionBuffer extends BaseBuffer {? this.squashSessionsByVersion(parsed): parsed;+ const { ch, TABLE_NAMES } = await loadClickHouse();for (const chunk of this.chunks(sessions, this.chunkSize)) {await ch.insert({table: TABLE_NAMES.sessions,diff --git a/packages/core/src/context.test.ts b/packages/core/src/context.test.tsindex 4f25eb2d2..628f390dc 100644--- a/packages/core/src/context.test.ts+++ b/packages/core/src/context.test.ts@@ -1,5 +1,5 @@import { beforeAll, expect, mock, test } from 'bun:test';-import type { AppDeps, Ctx } from './context';+import type { AppDeps, Buffers, Ctx } from './context';import type { JobMeta } from './jobs/envelope';import { createRecordingProducers } from './jobs/testing';import type { QueueProducers } from './jobs.registry';@@ -58,7 +58,7 @@ function stubDeps(): {ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,producers: {...producers,scope(meta) {diff --git a/packages/core/src/context.ts b/packages/core/src/context.tsindex 0855ac2f8..c738f9632 100644--- a/packages/core/src/context.ts+++ b/packages/core/src/context.ts@@ -5,21 +5,25 @@// (ADR-007 decision 18, ADR-018 R1).import type { SessionValidationResult } from '@openpanel/db';+import type { Buffers } from './buffers/create-buffers';import type { QueueProducerHandle, QueueProducers } from './jobs.registry';import type { Logger } from './logger';import { createServices, type Services } from './services';import type { CookieJar, CookieOptions } from './shared/cookie';// Handles core does not own yet. Each real type arrives with the phase that-// builds it (db/ch/redis P3, clients P4, buffers P8); they are named now so-// AppDeps and Ctx are written once in their final shape and replacing an alias-// moves no field. `unknown` is deliberate — reading one before its type lands-// is a compile error rather than a silent `any`.+// builds it (db/ch/redis P3, clients P4); they are named now so AppDeps and+// Ctx are written once in their final shape and replacing an alias moves no+// field. `unknown` is deliberate — reading one before its type lands is a+// compile error rather than a silent `any`.export type Db = unknown;export type ClickHouseClient = unknown;export type RedisClient = unknown;export type ServiceClients = unknown;-export type Buffers = unknown;++// Landed at M8-001: built once by `createBuffers(deps)` in main.ts, never a+// module singleton.+export type { Buffers } from './buffers/create-buffers';// The resolved session. Type-only, so nothing of `@openpanel/db` is loaded at// runtime — core stays importable with no database, which is what letsdiff --git a/packages/core/src/index.test.ts b/packages/core/src/index.test.tsindex b12340bd0..ad4e227d5 100644--- a/packages/core/src/index.test.ts+++ b/packages/core/src/index.test.ts@@ -35,3 +35,22 @@ test('the rpc base is on the barrel, so there is one tRPC instance', () => {test('a service, a client or a buffer is not on the barrel', () => {expect(Object.keys(core)).not.toContain('createServices');});++// The buffers are boot singletons on AppDeps (ADR-007), so main.ts needs the+// FACTORY and nothing else. A named instance here would be the module+// singleton the design refuses.+test('buffers reach the barrel as a factory, never as instances', () => {+ expect(core.createBuffers).toBeTypeOf('function');+ expect(core.registerBufferMetrics).toBeTypeOf('function');+ for (const instance of [+ 'eventBuffer',+ 'profileBuffer',+ 'profileBackfillBuffer',+ 'botBuffer',+ 'sessionBuffer',+ 'replayBuffer',+ 'groupBuffer',+ ]) {+ expect(Object.keys(core)).not.toContain(instance);+ }+});diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 0cb6e15c9..edc3aeed9 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -6,6 +6,32 @@// once and mount the three route surfaces plus the tRPC router over it; a// service, a client or a buffer is not reachable from here by design.+// The concrete pino implementation (dissolved from @openpanel/logger,+// M4-003). `./logger` above is the structural interface every module codes+// against; this is what apps/api, and the still-live apps/worker, call to+// build one.+// The seven buffers (moved from packages/db/src/buffers, M8-001). Only the+// FACTORY is on the barrel: they are boot singletons on `AppDeps`, built once+// by `main.ts`, never module singletons (ADR-007). V1's+// packages/db/src/buffers/index.ts calls this once and re-exports the+// instances, so both boots share one set.+export type {+ AddObservation,+ AddObserver,+ BufferDeps,+ FlushObservation,+ FlushObserver,+ FlushPhaseTimings,+ FlushTrigger,+} from './buffers/base-buffer';+export { registerBufferMetrics } from './buffers/buffer.metrics';+export type { Buffers } from './buffers/create-buffers';+export { createBuffers } from './buffers/create-buffers';+export type { IGroupBufferInput } from './buffers/group-buffer';+export type { ProfileBackfillEntry } from './buffers/profile-backfill-buffer';+export type { IClickhouseSessionReplayChunk } from './buffers/replay-buffer';+export type { SessionIngestResult } from './buffers/session-buffer';+export { SESSION_TIMEOUT_MS } from './buffers/session-buffer';// Dissolved from @openpanel/ai (M4-005) — the chat agent app, the filter// command bar, insight explanation/enrichment and the worker's digest and// win-back emails all call these directly.@@ -78,10 +104,6 @@ export {sendSlackNotification,slackInstaller,} from './clients/integrations/slack';-// The concrete pino implementation (dissolved from @openpanel/logger,-// M4-003). `./logger` above is the structural interface every module codes-// against; this is what apps/api, and the still-live apps/worker, call to-// build one.export type { ILogger } from './clients/logger';export {createLogger,@@ -118,15 +140,14 @@ export {// overview router call these.//// Loaded via dynamic import, NOT a static re-export like every other service-// on this barrel: packages/db/src/buffers/base-buffer.ts already imports-// `@openpanel/core` eagerly, and this module's own chain reaches deep into-// `@openpanel/db` (the Prisma-backed conversation store, ~30 analytics/report-// tool functions). A static re-export here would make THIS barrel's own-// evaluation re-enter `@openpanel/db` while it is still mid-evaluation of-// `./buffers` — `bot-buffer.ts extends BaseBuffer` then sees `BaseBuffer` as-// `undefined` (a live TDZ binding that never got the chance to fill in).-// `event-buffer.test.ts` et al. hit exactly this before this indirection was-// added. `chatApp` and `chatRunContext` are one-time-per-process values, so+// on this barrel: packages/db/src/buffers/index.ts imports `@openpanel/core`+// eagerly (it is the V1 delegate over `createBuffers`), and this module's own+// chain reaches deep into `@openpanel/db` (the Prisma-backed conversation+// store, ~30 analytics/report tool functions). A static re-export here would+// make THIS barrel's own evaluation re-enter `@openpanel/db` while that+// package is still mid-evaluation, and a class the re-entering module extends+// is then an `undefined` TDZ binding that never got the chance to fill in.+// `chatApp` and `chatRunContext` are one-time-per-process values, so// callers resolve them once (apps/api's Fastify wrapper does so inside its// own already-async route registration) and keep the reference.export type {@@ -1044,7 +1065,7 @@ export {// Loaded via dynamic import, NOT a static re-export, for the same reason// `getChatApp`/`runFilterCommand` above are: `mcp.service` reaches// `@openpanel/db` (auth's client lookup, every analytics tool), and-// `@openpanel/db/src/buffers/base-buffer.ts` already imports `@openpanel/core`+// `@openpanel/db/src/buffers/index.ts` already imports `@openpanel/core`// eagerly. A static export here would make this barrel's own evaluation// re-enter `@openpanel/db` mid-evaluation — observed as a `PagesService` TDZ// ReferenceError two modules away, in a tool file that never otherwise runsdiff --git a/packages/core/src/jobs/workers.test.ts b/packages/core/src/jobs/workers.test.tsindex 3d2eca95d..87274d36e 100644--- a/packages/core/src/jobs/workers.test.ts+++ b/packages/core/src/jobs/workers.test.ts@@ -1,6 +1,6 @@import { expect, test } from 'bun:test';import { z } from 'zod';-import type { AppDeps, JobCtx } from '../context';+import type { AppDeps, Buffers, JobCtx } from '../context';import { queues } from '../jobs.registry';import type { Logger } from '../logger';import { defineJob, defineQueue } from './define';@@ -28,7 +28,7 @@ function stubDeps(): AppDeps {ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,producers: createRecordingProducers(queues),logger: stubLogger(),config: { selfHosted: true },diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.tsindex 954b40271..676c3e2ca 100644--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts@@ -2,7 +2,7 @@// methods themselves are exercised in cohort.service.test.ts.import { expect, test } from 'bun:test';-import type { JobCtx } from '../../context';+import type { Buffers, JobCtx } from '../../context';import { createRecordingProducers } from '../../jobs/testing';import { queues } from '../../jobs.registry';import type { Logger } from '../../logger';@@ -50,7 +50,7 @@ function stubJobCtx(cohort: Partial<CohortService>, jobName = 'test'): JobCtx {ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: createRecordingProducers(queues).queues,services,diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.tsindex 8c63dcacb..cd33a4488 100644--- a/packages/core/src/modules/gsc/gsc.jobs.test.ts+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts@@ -2,7 +2,7 @@// methods themselves are exercised in gsc.service.test.ts.import { expect, test } from 'bun:test';-import type { JobCtx } from '../../context';+import type { Buffers, JobCtx } from '../../context';import { createRecordingProducers } from '../../jobs/testing';import { queues } from '../../jobs.registry';import type { Logger } from '../../logger';@@ -46,7 +46,7 @@ function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: createRecordingProducers(queues).queues,services,@@ -123,7 +123,7 @@ test('gscSync fans out one gscProjectSync enqueue per connected project', asyncch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: producers.queues,services: {diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.tsindex c6d3ad1aa..14e0aa846 100644--- a/packages/core/src/modules/gsc/gsc.service.test.ts+++ b/packages/core/src/modules/gsc/gsc.service.test.ts@@ -126,6 +126,20 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({chQuery,}));+// Bypasses the Redis cache-aside entirely — `getGscCannibalization`'s own+// logic is exercised directly, its caching is @openpanel/redis's concern.+// `getRedisCache` is unused here but included for the same cross-file+// mock.module reason as above.+const realRedis = { ...(await import('@openpanel/redis')) };+mock.module('@openpanel/redis', () => ({+ ...realRedis,+ cacheable: <T>(fn: T) => fn,+ getRedisCache: () => ({+ get: async () => null,+ setex: async () => undefined,+ }),+}));+// `chQuery` above is a fake stuck in place for the rest of the process once// this file's tests finish (`mock.module` has no per-file scope without// `--isolate` — see AGENTS.md): the mcp module's integration suite calls the@@ -137,20 +151,9 @@ afterAll(() => {'@openpanel/db/src/clickhouse/client',() => realClickhouseClient);+ mock.module('@openpanel/redis', () => realRedis);});-// Bypasses the Redis cache-aside entirely — `getGscCannibalization`'s own-// logic is exercised directly, its caching is @openpanel/redis's concern.-// `getRedisCache` is unused here but included for the same cross-file-// mock.module reason as above.-mock.module('@openpanel/redis', () => ({- cacheable: <T>(fn: T) => fn,- getRedisCache: () => ({- get: async () => null,- setex: async () => undefined,- }),-}));-const validateAuthorizationCode = mock(async () => ({accessToken: () => 'google-access-token',hasRefreshToken: () => true,diff --git a/packages/core/src/modules/import/import.jobs.test.ts b/packages/core/src/modules/import/import.jobs.test.tsindex a8f276c12..da8feca35 100644--- a/packages/core/src/modules/import/import.jobs.test.ts+++ b/packages/core/src/modules/import/import.jobs.test.ts@@ -2,7 +2,7 @@// themselves are exercised in import.service.test.ts.import { expect, test } from 'bun:test';-import type { JobCtx } from '../../context';+import type { Buffers, JobCtx } from '../../context';import { createRecordingProducers } from '../../jobs/testing';import { queues } from '../../jobs.registry';import type { Logger } from '../../logger';@@ -46,7 +46,7 @@ function stubJobCtx(importService: Partial<ImportService>): JobCtx {ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: createRecordingProducers(queues).queues,services,diff --git a/packages/core/src/modules/import/import.service.test.ts b/packages/core/src/modules/import/import.service.test.tsindex 3a9eaba7d..c98fbadd0 100644--- a/packages/core/src/modules/import/import.service.test.ts+++ b/packages/core/src/modules/import/import.service.test.ts@@ -5,6 +5,7 @@// (side-effect-free) import.import { beforeAll, expect, mock, test } from 'bun:test';+import type { Buffers } from '../../context';import { createRecordingProducers } from '../../jobs/testing';import { queues } from '../../jobs.registry';import type { Logger } from '../../logger';@@ -163,7 +164,7 @@ test('ImportService.enqueue adds the import job onto the import queue', async ()ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: producers.queues,});diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.tsindex a278b7266..cb5793fdf 100644--- a/packages/core/src/modules/insight/insight.jobs.test.ts+++ b/packages/core/src/modules/insight/insight.jobs.test.ts@@ -2,7 +2,7 @@// methods themselves are exercised in insight.service.test.ts.import { expect, test } from 'bun:test';-import type { JobCtx } from '../../context';+import type { Buffers, JobCtx } from '../../context';import { createRecordingProducers } from '../../jobs/testing';import { queues } from '../../jobs.registry';import type { Logger } from '../../logger';@@ -57,7 +57,7 @@ function stubJobCtx(ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: createRecordingProducers(queues).queues,services,@@ -117,7 +117,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId dech: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: producers.queues,services: {diff --git a/packages/core/src/modules/insight/insight.service.test.ts b/packages/core/src/modules/insight/insight.service.test.tsindex 198b09cd6..54bc75184 100644--- a/packages/core/src/modules/insight/insight.service.test.ts+++ b/packages/core/src/modules/insight/insight.service.test.ts@@ -5,6 +5,7 @@// (side-effect-free) import.import { beforeAll, expect, mock, test } from 'bun:test';+import type { Buffers } from '../../context';const listProjectIdsForCadence = mock(async () => ['p1', 'p2']);const getProjectCreatedAt = mock(async () => null);@@ -169,7 +170,7 @@ test('createInsightService binds every InsightService method', () => {ch: undefined,redis: undefined,clients: undefined,- buffers: undefined,+ buffers: undefined as unknown as Buffers,logger,queues:undefined as unknown as import('../../jobs.registry').QueueProducers,diff --git a/packages/core/src/modules/notification/notification.jobs.test.ts b/packages/core/src/modules/notification/notification.jobs.test.tsindex c27995020..5c328f3c7 100644--- a/packages/core/src/modules/notification/notification.jobs.test.ts+++ b/packages/core/src/modules/notification/notification.jobs.test.ts@@ -2,7 +2,7 @@// dispatch logic themselves are exercised in notification.service.test.ts.import { expect, test } from 'bun:test';-import type { JobCtx } from '../../context';+import type { Buffers, JobCtx } from '../../context';import { createRecordingProducers } from '../../jobs/testing';import { queues } from '../../jobs.registry';import type { Logger } from '../../logger';@@ -46,7 +46,7 @@ function stubJobCtx(notification: Partial<NotificationService>): JobCtx {ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: createRecordingProducers(queues).queues,services,diff --git a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts b/packages/core/src/modules/onboarding/onboarding.jobs.test.tsindex 45f9791ab..355dec217 100644--- a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts+++ b/packages/core/src/modules/onboarding/onboarding.jobs.test.ts@@ -2,7 +2,7 @@// method itself is exercised in onboarding.service.test.ts.import { expect, test } from 'bun:test';-import type { JobCtx } from '../../context';+import type { Buffers, JobCtx } from '../../context';import { createRecordingProducers } from '../../jobs/testing';import { queues } from '../../jobs.registry';import type { Logger } from '../../logger';@@ -49,7 +49,7 @@ function stubJobCtx(ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: createRecordingProducers(queues).queues,services,diff --git a/packages/core/src/modules/organization/organization.jobs.test.ts b/packages/core/src/modules/organization/organization.jobs.test.tsindex afade1295..0b22dfa20 100644--- a/packages/core/src/modules/organization/organization.jobs.test.ts+++ b/packages/core/src/modules/organization/organization.jobs.test.ts@@ -2,7 +2,7 @@// method itself is exercised in organization.service.test.ts.import { expect, test } from 'bun:test';-import type { JobCtx } from '../../context';+import type { Buffers, JobCtx } from '../../context';import { createRecordingProducers } from '../../jobs/testing';import { queues } from '../../jobs.registry';import type { Logger } from '../../logger';@@ -52,7 +52,7 @@ function stubJobCtx(ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,logger: stubLogger(),queues: createRecordingProducers(queues).queues,services,diff --git a/packages/core/src/modules/organization/organization.service.test.ts b/packages/core/src/modules/organization/organization.service.test.tsindex 21c2cbd71..744732fa9 100644--- a/packages/core/src/modules/organization/organization.service.test.ts+++ b/packages/core/src/modules/organization/organization.service.test.ts@@ -422,13 +422,6 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({ch: { command: chCommand },}));-afterAll(() => {- mock.module(- '@openpanel/db/src/clickhouse/client',- () => realClickhouseClient- );-});-// Bypasses the Redis cache-aside entirely — this module's own logic is// exercised directly, caching is @openpanel/redis's concern. `access.service.ts`// (reached via connectUserToOrganization's dynamic import) calls `.clear()` on@@ -451,7 +444,14 @@ function cacheableStub(set: () => async () => 'OK' as const,});}+// Spread the real module for the same reason the clickhouse/client mock above+// does: `mock.module` replaces this specifier process-wide. Surfaced by+// M8-001, which put `createBuffers` on core's barrel — event-buffer.ts+// value-imports `publishEvent` from here, so a partial factory turns this+// file's own barrel import into a SyntaxError.+const realRedis = { ...(await import('@openpanel/redis')) };mock.module('@openpanel/redis', () => ({+ ...realRedis,cacheable: cacheableStub,getRedisCache: () => ({get: async () => null,@@ -460,6 +460,14 @@ mock.module('@openpanel/redis', () => ({}),}));+afterAll(() => {+ mock.module(+ '@openpanel/db/src/clickhouse/client',+ () => realClickhouseClient+ );+ mock.module('@openpanel/redis', () => realRedis);+});+// Mocks core's own clients/email.ts wrapper, not @openpanel/email itself:// that package statically imports @openpanel/db's full barrel (Resend/SMTP// setup aside), which is exactly the eager side-effecting chaindiff --git a/packages/core/test/buffer-fixtures.ts b/packages/core/test/buffer-fixtures.tsnew file mode 100644index 000000000..22df0a879--- /dev/null+++ b/packages/core/test/buffer-fixtures.ts@@ -0,0 +1,25 @@+// Stubs shared by the buffer tests. Not a test file (no `*.test.ts` suffix),+// so the runner does not pick it up.++import type { BufferDeps } from '../src/buffers/base-buffer';+import type { Logger } from '../src/logger';++const silentLogger: Logger = {+ fatal: () => undefined,+ error: () => undefined,+ warn: () => undefined,+ info: () => undefined,+ debug: () => undefined,+ trace: () => undefined,+ child: () => silentLogger,+};++/**+ * A buffer's boot scope, silent and never paused. Silent matters: the real+ * `createLogger` spawns a pino transport worker thread per buffer, and these+ * suites build one buffer per test.+ */+export const stubBufferDeps: BufferDeps = {+ createLogger: () => silentLogger,+ isCronPaused: () => Promise.resolve(false),+};diff --git a/packages/core/test/http-fixtures.ts b/packages/core/test/http-fixtures.tsindex 99472948d..b7c8b510b 100644--- a/packages/core/test/http-fixtures.ts+++ b/packages/core/test/http-fixtures.ts@@ -1,7 +1,7 @@// Stubs shared by the http tests. Not a test file (no `*.test.ts` suffix), so// the runner does not pick it up.-import type { AppDeps } from '../src/context';+import type { AppDeps, Buffers } from '../src/context';import { createRecordingProducers } from '../src/jobs/testing';import { queues } from '../src/jobs.registry';import { type CapturedLogger, capturingLogger } from './rpc-fixtures';@@ -42,7 +42,7 @@ export function stubAppDeps(): AppDepsStub {ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,producers,logger,config: { selfHosted: false },diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.tsindex d6e662716..477ae8417 100644--- a/packages/core/test/request-id-end-to-end.test.ts+++ b/packages/core/test/request-id-end-to-end.test.ts@@ -11,7 +11,7 @@import { beforeAll, beforeEach, expect, mock, test } from 'bun:test';import { Elysia } from 'elysia';import { z } from 'zod';-import type { AppDeps } from '../src/context';+import type { AppDeps, Buffers } from '../src/context';import { defineJob, defineQueue, type Producers } from '../src/jobs/define';import { wrap } from '../src/jobs/envelope';import {@@ -144,7 +144,7 @@ function stubDeps() {ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,producers: producers as unknown as QueueProducerHandle,logger: bindingLogger(lines),config: { selfHosted: false },diff --git a/packages/core/test/rpc-fixtures.ts b/packages/core/test/rpc-fixtures.tsindex 9ebffdfd6..619092877 100644--- a/packages/core/test/rpc-fixtures.ts+++ b/packages/core/test/rpc-fixtures.ts@@ -3,6 +3,7 @@import {type AppDeps,+ type Buffers,createCtx,extendCtx,type HttpCtx,@@ -89,7 +90,7 @@ export function stubHttpCtx(ch: {},redis: {},clients: {},- buffers: {},+ buffers: {} as Buffers,producers: createRecordingProducers(queues),logger,config: { selfHosted: false },diff --git a/packages/db/src/buffers/index.ts b/packages/db/src/buffers/index.tsindex 46b5920bc..ac967c71f 100644--- a/packages/db/src/buffers/index.ts+++ b/packages/db/src/buffers/index.ts@@ -1,22 +1,35 @@-import { BotBuffer as BotBufferRedis } from './bot-buffer';-import { EventBuffer as EventBufferRedis } from './event-buffer';-import { GroupBuffer } from './group-buffer';-import { ProfileBackfillBuffer } from './profile-backfill-buffer';-import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';-import { ReplayBuffer } from './replay-buffer';-import { SessionBuffer } from './session-buffer';+// The seven buffers live in @openpanel/core now (M8-001):+// packages/core/src/buffers/*. They are no longer module singletons there —+// `createBuffers(deps)` builds them once and V2's main.ts hangs the result on+// `AppDeps.buffers` (ADR-007). This file is the V1 delegate: one call to that+// factory, its instances re-exported under the names apps/api, apps/worker,+// packages/trpc and core's own lazy `import('@openpanel/db/src/buffers')`+// call sites already use. Both boots therefore share ONE set of buffers and+// one set of Redis lists.+import { createBuffers } from '@openpanel/core';+import { cronQueue } from '@openpanel/queue';+import { createLogger } from '../logger';-export const eventBuffer = new EventBufferRedis();-export const profileBuffer = new ProfileBufferRedis();-export const botBuffer = new BotBufferRedis();-export const sessionBuffer = new SessionBuffer();-export const profileBackfillBuffer = new ProfileBackfillBuffer();-export const replayBuffer = new ReplayBuffer();-export const groupBuffer = new GroupBuffer();+const buffers = createBuffers({+ // packages/db's own pino, exactly as each buffer built it before the move —+ // same `LOG_PREFIX-<name>-<NODE_ENV>` service name on every line.+ createLogger: (name) => createLogger({ name }),+ // ADR-005's BullMQ escape hatch: pausing `cron` from bull-board halts all+ // buffer flushing, preserved deliberately (docs/ANSWERS.md §3: "known!").+ isCronPaused: () => cronQueue.isPaused(),+});-export type { ProfileBackfillEntry } from './profile-backfill-buffer';-export type { IClickhouseSessionReplayChunk } from './replay-buffer';-export {- SESSION_TIMEOUT_MS,- type SessionIngestResult,-} from './session-buffer';+export const eventBuffer = buffers.event;+export const profileBuffer = buffers.profile;+export const botBuffer = buffers.bot;+export const sessionBuffer = buffers.session;+export const profileBackfillBuffer = buffers.profileBackfill;+export const replayBuffer = buffers.replay;+export const groupBuffer = buffers.group;++export type {+ IClickhouseSessionReplayChunk,+ ProfileBackfillEntry,+ SessionIngestResult,+} from '@openpanel/core';+export { SESSION_TIMEOUT_MS } from '@openpanel/core';diff --git a/packages/trpc/src/trpc.ts b/packages/trpc/src/trpc.tsindex d5ff18ab6..fb0cbbef6 100644--- a/packages/trpc/src/trpc.ts+++ b/packages/trpc/src/trpc.ts@@ -119,9 +119,10 @@ export async function createContext({/*** The `Ctx` half of the context under V1.*- * `db`/`ch`/`redis`/`clients`/`buffers` are core's `unknown` stubs until P3-P8- * wire the real clients, and V1's routers reach for their own singletons- * regardless. `queues` and `services` throw rather than returning an empty+ * `db`/`ch`/`redis`/`clients` are core's `unknown` stubs until P3-P4 wire the+ * real clients, and V1's routers reach for their own singletons regardless —+ * including the buffers, which V1 reaches through `@openpanel/db`'s delegate+ * (M8-001) rather than through this context. `queues` and `services` throw rather than returning an empty* object: a V1 router that reached for either through core would otherwise* silently do nothing (`queues`) or hit `undefined` (`services`) instead of* the loud error that tells it to reach `@openpanel/db`/`@openpanel/core`@@ -136,7 +137,7 @@ function v1CtxScope(req: CreateFastifyContextOptions['req']): Ctx {ch: undefined,redis: undefined,clients: undefined,- buffers: undefined,+ buffers: undefined as unknown as Ctx['buffers'],logger: req.log,queues: NOT_WIRED_QUEUES,services: NOT_WIRED_SERVICES,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 855c84aa2..fbc2a05c1 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1163,6 +1163,9 @@ importers:elysia:specifier: 1.4.30version: 1.4.30(@sinclair/[email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])+ fast-deep-equal:+ specifier: ^3.1.3+ version: 3.1.3ipaddr.js:specifier: ^2.5.0version: 2.5.0