Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M8-001): buffers into core: createBuffers at boot on AppDeps + metrics instrumentation

ralphafbaece4236c
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index 42ef23077..181765a9d 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -3,24 +3,32 @@
// keeps serving the golden harness untouched until `apps/worker` is deleted
// at P9.
//
-// This is the M3 gate boot only: db/ch/redis/clients/buffers are not wired
-// into `AppDeps` yet — their types stay the `unknown` stubs `context.ts`
-// declares until P3/P4/P8 land the real clients — so only the ops surface
-// (healthz + metrics) is mounted here. `dashboardRoutes`/`publicApiRoutes`
-// land with their first module (see `rest.routes.ts`).
+// This is the M3 gate boot only: db/ch/redis/clients are not wired into
+// `AppDeps` yet — their types stay the `unknown` stubs `context.ts` declares
+// until P3/P4 land the real clients — so only the ops surface (healthz +
+// metrics) is mounted here. `dashboardRoutes`/`publicApiRoutes` land with
+// their first module (see `rest.routes.ts`). The buffers ARE real from M8-001:
+// `createBuffers` builds them once, here, and nowhere else.
process.env.TZ = 'UTC';
import {
type AppDeps,
+ type BufferDeps,
+ createBuffers,
opsRoutes,
type QueueProducerHandle,
queues,
+ registerBufferMetrics,
} from '@openpanel/core';
import pino from 'pino';
import { type Config, loadConfig } from './config/env';
const SHUTDOWN_FORCE_EXIT_MS = 5000;
+// The registry key and the Redis name are the same string for all seven queues
+// (jobs.registry.ts); `queueKey` only braces it under `QUEUE_CLUSTER`, which no
+// deployment sets.
+const CRON_QUEUE_NAME = 'cron';
// Used only to report a config error itself — the real level isn't known
// until `config/env.ts` has validated `LOG_LEVEL`.
const BOOTSTRAP_LOG_LEVEL = 'info';
@@ -72,17 +80,37 @@ function stubProducers(): QueueProducerHandle {
};
}
+/**
+ * A named child of the boot logger per buffer, plus the buffers' one direct
+ * BullMQ read — ADR-005's `bullQueues` escape hatch.
+ * Pausing `cron` from bull-board halts ALL buffer flushing, preserved
+ * deliberately (docs/ANSWERS.md §3: "known!"). The M3 producer stub owns no
+ * queues yet, and a queue that does not exist has not been paused.
+ */
+function bufferDeps(producers: QueueProducerHandle): BufferDeps {
+ return {
+ createLogger: (name) => logger.child({ name }),
+ isCronPaused: async () => {
+ const cron = producers.bullQueues.find(
+ (queue) => queue.name === CRON_QUEUE_NAME
+ );
+ return cron ? await cron.isPaused() : false;
+ },
+ };
+}
+
function buildDeps(): AppDeps {
+ const producers = stubProducers();
return {
- // db/ch/redis/clients/buffers are `unknown` stubs in context.ts until
- // P3/P4/P8 land the real clients — reading one before its type lands is
- // meant to be a compile error, not a runtime `undefined`.
+ // db/ch/redis/clients are `unknown` stubs in context.ts until P3/P4 land
+ // the real clients — reading one before its type lands is meant to be a
+ // compile error, not a runtime `undefined`.
db: undefined,
ch: undefined,
redis: undefined,
clients: undefined,
- buffers: undefined,
- producers: stubProducers(),
+ buffers: createBuffers(bufferDeps(producers)),
+ producers,
logger,
config: { selfHosted: config.SELF_HOSTED },
};
@@ -93,6 +121,14 @@ async function main() {
const port = config.API_PORT;
const deps = buildDeps();
+ // Scrape-time collectors register only where a role consumes queues: the
+ // LLEN gauges cost one Redis round trip per buffer per scrape, and ten API
+ // replicas exposing them would multiply that for no new information
+ // (TARGET_ARCHITECTURE §18).
+ if (role !== 'api') {
+ registerBufferMetrics(deps.buffers);
+ }
+
const app = opsRoutes(deps);
if (role !== 'api') {
diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.ts
index ebfbbbf62..59563c955 100644
--- a/apps/worker/src/metrics.ts
+++ b/apps/worker/src/metrics.ts
@@ -1,3 +1,4 @@
+import { registerBufferMetrics } from '@openpanel/core';
import {
botBuffer,
eventBuffer,
@@ -129,201 +130,23 @@ queues.forEach((queue) => {
// Buffer metrics
// -----------------------------------------------------------------------------
-const allBuffers = [
- eventBuffer,
- profileBuffer,
- botBuffer,
- sessionBuffer,
- replayBuffer,
- groupBuffer,
- profileBackfillBuffer,
-];
-
-// Ground-truth LLEN of each buffer's main list. O(1) — no shadow counter.
-for (const buf of allBuffers) {
- register.registerMetric(
- new client.Gauge({
- name: `buffer_${buf.name.replace(/-/g, '_')}_count`,
- help: 'Buffer size (LLEN of the Redis list)',
- async collect() {
- try {
- this.set(await buf.getBufferSize());
- } catch {
- // ignore — scrape continues
- }
- },
- })
- );
-}
-
-// Number of events sitting in event-buffer's in-process micro-batch (pre-Redis).
-// Other buffers don't have a local layer.
-register.registerMetric(
- new client.Gauge({
- name: 'buffer_event_pending_local_count',
- help: 'Events in event-buffer process-local micro-batch (not yet in Redis)',
- collect() {
- try {
- this.set(eventBuffer.getPendingLocalCount());
- } catch {
- // ignore
- }
- },
- })
+// The twelve `buffer_*` series moved to @openpanel/core with the buffers
+// themselves (M8-001). V1 keeps its own registry, so it hands one in: the
+// names, labels, buckets and registration order are core's, and this body is
+// byte-identical to what this file registered before the move.
+registerBufferMetrics(
+ {
+ event: eventBuffer,
+ profile: profileBuffer,
+ bot: botBuffer,
+ session: sessionBuffer,
+ replay: replayBuffer,
+ group: groupBuffer,
+ profileBackfill: profileBackfillBuffer,
+ },
+ register
);
-// ---- Flush metrics (populated via flushObserver hooks) ----
-
-const flushDuration = new client.Histogram({
- name: 'buffer_flush_duration_ms',
- help: 'Wall time of a tryFlush call, including lock acquisition',
- labelNames: ['buffer', 'result', 'trigger'],
- buckets: [
- 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,
- ],
-});
-register.registerMetric(flushDuration);
-
-const flushTotal = new client.Counter({
- name: 'buffer_flush_total',
- help: 'Count of tryFlush invocations by result (success/error/locked/paused) and trigger (add/cron)',
- labelNames: ['buffer', 'result', 'trigger'],
-});
-register.registerMetric(flushTotal);
-
-const flushRowsTotal = new client.Counter({
- name: 'buffer_flush_rows_total',
- help: 'Rows drained from the buffer per flush (sum)',
- labelNames: ['buffer'],
-});
-register.registerMetric(flushRowsTotal);
-
-// Per-phase Redis op timing on the flush hot path.
-const redisOpDurationMs = new client.Histogram({
- name: 'buffer_redis_op_duration_ms',
- help: 'Duration of a Redis op during flush',
- labelNames: ['buffer', 'op'],
- buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000],
-});
-register.registerMetric(redisOpDurationMs);
-
-const chInsertDurationMs = new client.Histogram({
- name: 'buffer_ch_insert_duration_ms',
- help: 'Duration of the ClickHouse insert(s) inside a single flush',
- labelNames: ['buffer'],
- buckets: [
- 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,
- ],
-});
-register.registerMetric(chInsertDurationMs);
-
-// CH SELECT latency inside a flush (e.g. profile-buffer's fetch-existing
-// profiles for merge). Separated from ch_insert because for some buffers
-// the SELECT dominates total flush time. Only populated by buffers that
-// actually read CH inside the flush path.
-const chFetchDurationMs = new client.Histogram({
- name: 'buffer_ch_fetch_duration_ms',
- help: 'Duration of CH SELECT(s) inside a single flush (e.g. profile merge fetch)',
- labelNames: ['buffer'],
- buckets: [
- 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,
- ],
-});
-register.registerMetric(chFetchDurationMs);
-
-const flushLlenAtStart = new client.Histogram({
- name: 'buffer_flush_llen_at_start',
- help: 'LLEN observed at the start of a flush attempt',
- labelNames: ['buffer'],
- buckets: [
- 0, 10, 100, 500, 1000, 5000, 10_000, 50_000, 100_000, 500_000, 1_000_000,
- 5_000_000,
- ],
-});
-register.registerMetric(flushLlenAtStart);
-
-// ---- Add-path metrics ----
-
-const addDurationMs = new client.Histogram({
- name: 'buffer_add_duration_ms',
- help: 'Duration of a single add() call (per-event ingest path)',
- labelNames: ['buffer'],
- buckets: [0.5, 1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 5000],
-});
-register.registerMetric(addDurationMs);
-
-const addTotal = new client.Counter({
- name: 'buffer_add_total',
- help: 'Total add() calls per buffer that actually enqueued (excludes skipped).',
- labelNames: ['buffer'],
-});
-register.registerMetric(addTotal);
-
-const addSkippedTotal = new client.Counter({
- name: 'buffer_add_skipped_total',
- help: 'add() calls that short-circuited (no enqueue). Reason: cached, etc.',
- labelNames: ['buffer', 'reason'],
-});
-register.registerMetric(addSkippedTotal);
-
-for (const buf of allBuffers) {
- buf.flushObserver = (obs) => {
- flushTotal.inc({
- buffer: obs.buffer,
- result: obs.result,
- trigger: obs.trigger,
- });
- flushDuration.observe(
- { buffer: obs.buffer, result: obs.result, trigger: obs.trigger },
- obs.totalMs
- );
-
- if (obs.llenAtStart != null) {
- flushLlenAtStart.observe({ buffer: obs.buffer }, obs.llenAtStart);
- }
-
- if (obs.rowsProcessed != null && obs.rowsProcessed > 0) {
- flushRowsTotal.inc({ buffer: obs.buffer }, obs.rowsProcessed);
- }
-
- if (obs.phases?.lrangeMs != null) {
- redisOpDurationMs.observe(
- { buffer: obs.buffer, op: 'lrange' },
- obs.phases.lrangeMs
- );
- }
- if (obs.phases?.trimMs != null) {
- redisOpDurationMs.observe(
- { buffer: obs.buffer, op: 'trim' },
- obs.phases.trimMs
- );
- }
- if (obs.phases?.chFetchMs != null) {
- chFetchDurationMs.observe({ buffer: obs.buffer }, obs.phases.chFetchMs);
- }
- if (obs.phases?.chInsertMs != null) {
- chInsertDurationMs.observe({ buffer: obs.buffer }, obs.phases.chInsertMs);
- }
- };
-
- buf.addObserver = (obs) => {
- if (obs.skipped) {
- addSkippedTotal.inc({
- buffer: obs.buffer,
- reason: obs.skipReason ?? 'unknown',
- });
- // Don't pollute add-latency histogram with no-op fast paths
- return;
- }
- addTotal.inc({ buffer: obs.buffer });
- addDurationMs.observe({ buffer: obs.buffer }, obs.durationMs);
- };
-}
-
-// Note: `buffer_replay_count` is already registered by the allBuffers loop
-// above (replayBuffer is in that list), so no standalone gauge here — a second
-// registration with the same name throws and crashes the worker on boot.
-
// -----------------------------------------------------------------------
// Session lifecycle metrics (new session-buffer + reaper world)
// -----------------------------------------------------------------------
diff --git a/packages/core/package.json b/packages/core/package.json
index cd5ac6165..2098fbef4 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -47,6 +47,7 @@
"csv-parse": "^6.1.0",
"date-fns": "^3.3.1",
"elysia": "1.4.30",
+ "fast-deep-equal": "^3.1.3",
"ipaddr.js": "^2.5.0",
"lru-cache": "^11.2.4",
"mathjs": "^12.3.2",
diff --git a/packages/db/src/buffers/base-buffer.ts b/packages/core/src/buffers/base-buffer.ts
similarity index 92%
rename from packages/db/src/buffers/base-buffer.ts
rename to packages/core/src/buffers/base-buffer.ts
index d021f31fd..a02d0d24c 100644
--- a/packages/db/src/buffers/base-buffer.ts
+++ b/packages/core/src/buffers/base-buffer.ts
@@ -1,9 +1,29 @@
import { Readable } from 'node:stream';
import type { ClickHouseSettings } from '@clickhouse/client';
-import { generateSecureId } from '@openpanel/core';
-import { cronQueue } from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
-import { createLogger, type ILogger } from '../logger';
+import type { Logger } from '../logger';
+import { generateSecureId } from '../shared/id';
+
+/**
+ * What a buffer needs from the boot scope. Buffers are built once by
+ * `createBuffers(deps)` and hung on `AppDeps` — never module singletons, so a
+ * test constructs its own and closes nothing shared (ADR-007).
+ */
+export interface BufferDeps {
+ /**
+ * Each buffer logs under its own name. Injected rather than built here so
+ * core instantiates no pino (ADR-007 layout) and a test hands in a silent
+ * stub instead of spawning one transport worker thread per buffer.
+ */
+ createLogger(name: string): Logger;
+ /**
+ * Asked before every flush. ADR-005's `ProducerHandle.bullQueues` escape
+ * hatch, narrowed to the one thing the buffers ask BullMQ directly: pausing
+ * `cron` from bull-board halts ALL buffer flushing, preserved deliberately
+ * (docs/ANSWERS.md §3: "known!").
+ */
+ isCronPaused(): Promise<boolean>;
+}
export type FlushPhaseTimings = {
lrangeMs?: number;
@@ -57,7 +77,7 @@ export type AddObserver = (obs: AddObservation) => void;
export class BaseBuffer {
name: string;
- logger: ILogger;
+ logger: Logger;
lockKey: string;
lockTimeout = 60;
onFlush: () => Promise<void> | void;
@@ -98,12 +118,15 @@ export class BaseBuffer {
/** Optional hook for instrumenting add() latency. */
public addObserver: AddObserver | null = null;
- constructor(options: {
- name: string;
- onFlush: () => Promise<void>;
- enableParallelProcessing?: boolean;
- }) {
- this.logger = createLogger({ name: options.name });
+ constructor(
+ private readonly deps: BufferDeps,
+ options: {
+ name: string;
+ onFlush: () => Promise<void>;
+ enableParallelProcessing?: boolean;
+ }
+ ) {
+ this.logger = deps.createLogger(options.name);
this.name = options.name;
this.lockKey = `lock:${this.name}`;
this.onFlush = options.onFlush;
@@ -387,7 +410,7 @@ export class BaseBuffer {
// best-effort
}
- const isCronQueuePaused = await cronQueue.isPaused();
+ const isCronQueuePaused = await this.deps.isCronPaused();
if (isCronQueuePaused) {
this.emitFlushObservation({
buffer: this.name,
diff --git a/packages/db/src/buffers/bot-buffer.ts b/packages/core/src/buffers/bot-buffer.ts
similarity index 88%
rename from packages/db/src/buffers/bot-buffer.ts
rename to packages/core/src/buffers/bot-buffer.ts
index 3224647c6..5d917c336 100644
--- a/packages/db/src/buffers/bot-buffer.ts
+++ b/packages/core/src/buffers/bot-buffer.ts
@@ -1,7 +1,7 @@
import { getRedisCache, type Redis } from '@openpanel/redis';
-import { ch, TABLE_NAMES } from '../clickhouse/client';
-import type { IClickhouseBotEvent } from '../services/event.service';
-import { BaseBuffer } from './base-buffer';
+import type { IClickhouseBotEvent } from '../modules/event/event.service';
+import { BaseBuffer, type BufferDeps } from './base-buffer';
+import { loadClickHouse } from './clickhouse';
export class BotBuffer extends BaseBuffer {
private batchSize = process.env.BOT_BUFFER_BATCH_SIZE
@@ -10,8 +10,8 @@ export class BotBuffer extends BaseBuffer {
private readonly redisKey = 'bot-events-buffer';
private redis: Redis;
- constructor() {
- super({
+ constructor(deps: BufferDeps) {
+ super(deps, {
name: 'bot',
onFlush: async () => {
await this.processBuffer();
@@ -60,6 +60,7 @@ export class BotBuffer extends BaseBuffer {
// Raw passthrough: each Redis entry is already a valid JSONEachRow
// line. Streaming raw strings skips JSON.parse + the client's
// re-stringify on the hot path.
+ const { ch, TABLE_NAMES } = await loadClickHouse();
const chStart = performance.now();
await ch.insert({
table: TABLE_NAMES.events_bots,
diff --git a/packages/core/src/buffers/buffer.metrics.ts b/packages/core/src/buffers/buffer.metrics.ts
new file mode 100644
index 000000000..cdfb16963
--- /dev/null
+++ b/packages/core/src/buffers/buffer.metrics.ts
@@ -0,0 +1,229 @@
+// The twelve `buffer_*` series, moved verbatim from apps/worker/src/metrics.ts
+// (TARGET_ARCHITECTURE §18: names and labels reproduced exactly). Registered
+// per boot rather than per import, because the buffers themselves are per boot
+// — and only where a role consumes queues, since the LLEN gauges cost one
+// Redis round trip per buffer per scrape.
+
+import client from 'prom-client';
+import { registry } from '../metrics';
+import type { Buffers } from './create-buffers';
+
+const FLUSH_DURATION_BUCKETS_MS = [
+ 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,
+];
+const REDIS_OP_BUCKETS_MS = [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000];
+const CH_DURATION_BUCKETS_MS = [
+ 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10_000, 30_000, 60_000,
+];
+const LLEN_BUCKETS = [
+ 0, 10, 100, 500, 1000, 5000, 10_000, 50_000, 100_000, 500_000, 1_000_000,
+ 5_000_000,
+];
+const ADD_DURATION_BUCKETS_MS = [
+ 0.5, 1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1000, 5000,
+];
+
+/**
+ * Registration order is V1's, because it is the order the series appear in the
+ * `/metrics` body.
+ */
+function allBuffers(buffers: Buffers) {
+ return [
+ buffers.event,
+ buffers.profile,
+ buffers.bot,
+ buffers.session,
+ buffers.replay,
+ buffers.group,
+ buffers.profileBackfill,
+ ];
+}
+
+/**
+ * Registers the buffer metrics and installs the flush/add observers on the
+ * given buffers. Call once per process: prom-client throws on a duplicate
+ * series name, and each buffer holds exactly one observer slot.
+ *
+ * `register` defaults to the one core registry; V1's worker passes its own
+ * dying registry so its `/metrics` body is unchanged (M8-001 delegation).
+ */
+export function registerBufferMetrics(
+ buffers: Buffers,
+ register: client.Registry = registry
+): void {
+ const bufferList = allBuffers(buffers);
+
+ // Ground-truth LLEN of each buffer's main list. O(1) — no shadow counter.
+ for (const buf of bufferList) {
+ register.registerMetric(
+ new client.Gauge({
+ name: `buffer_${buf.name.replace(/-/g, '_')}_count`,
+ help: 'Buffer size (LLEN of the Redis list)',
+ async collect() {
+ try {
+ this.set(await buf.getBufferSize());
+ } catch {
+ // ignore — scrape continues
+ }
+ },
+ })
+ );
+ }
+
+ // Number of events sitting in event-buffer's in-process micro-batch
+ // (pre-Redis). Other buffers don't have a local layer.
+ register.registerMetric(
+ new client.Gauge({
+ name: 'buffer_event_pending_local_count',
+ help: 'Events in event-buffer process-local micro-batch (not yet in Redis)',
+ collect() {
+ try {
+ this.set(buffers.event.getPendingLocalCount());
+ } catch {
+ // ignore
+ }
+ },
+ })
+ );
+
+ // ---- Flush metrics (populated via flushObserver hooks) ----
+
+ const flushDuration = new client.Histogram({
+ name: 'buffer_flush_duration_ms',
+ help: 'Wall time of a tryFlush call, including lock acquisition',
+ labelNames: ['buffer', 'result', 'trigger'],
+ buckets: FLUSH_DURATION_BUCKETS_MS,
+ });
+ register.registerMetric(flushDuration);
+
+ const flushTotal = new client.Counter({
+ name: 'buffer_flush_total',
+ help: 'Count of tryFlush invocations by result (success/error/locked/paused) and trigger (add/cron)',
+ labelNames: ['buffer', 'result', 'trigger'],
+ });
+ register.registerMetric(flushTotal);
+
+ const flushRowsTotal = new client.Counter({
+ name: 'buffer_flush_rows_total',
+ help: 'Rows drained from the buffer per flush (sum)',
+ labelNames: ['buffer'],
+ });
+ register.registerMetric(flushRowsTotal);
+
+ // Per-phase Redis op timing on the flush hot path.
+ const redisOpDurationMs = new client.Histogram({
+ name: 'buffer_redis_op_duration_ms',
+ help: 'Duration of a Redis op during flush',
+ labelNames: ['buffer', 'op'],
+ buckets: REDIS_OP_BUCKETS_MS,
+ });
+ register.registerMetric(redisOpDurationMs);
+
+ const chInsertDurationMs = new client.Histogram({
+ name: 'buffer_ch_insert_duration_ms',
+ help: 'Duration of the ClickHouse insert(s) inside a single flush',
+ labelNames: ['buffer'],
+ buckets: CH_DURATION_BUCKETS_MS,
+ });
+ register.registerMetric(chInsertDurationMs);
+
+ // CH SELECT latency inside a flush (e.g. profile-buffer's fetch-existing
+ // profiles for merge). Separated from ch_insert because for some buffers
+ // the SELECT dominates total flush time.
+ const chFetchDurationMs = new client.Histogram({
+ name: 'buffer_ch_fetch_duration_ms',
+ help: 'Duration of CH SELECT(s) inside a single flush (e.g. profile merge fetch)',
+ labelNames: ['buffer'],
+ buckets: CH_DURATION_BUCKETS_MS,
+ });
+ register.registerMetric(chFetchDurationMs);
+
+ const flushLlenAtStart = new client.Histogram({
+ name: 'buffer_flush_llen_at_start',
+ help: 'LLEN observed at the start of a flush attempt',
+ labelNames: ['buffer'],
+ buckets: LLEN_BUCKETS,
+ });
+ register.registerMetric(flushLlenAtStart);
+
+ // ---- Add-path metrics ----
+
+ const addDurationMs = new client.Histogram({
+ name: 'buffer_add_duration_ms',
+ help: 'Duration of a single add() call (per-event ingest path)',
+ labelNames: ['buffer'],
+ buckets: ADD_DURATION_BUCKETS_MS,
+ });
+ register.registerMetric(addDurationMs);
+
+ const addTotal = new client.Counter({
+ name: 'buffer_add_total',
+ help: 'Total add() calls per buffer that actually enqueued (excludes skipped).',
+ labelNames: ['buffer'],
+ });
+ register.registerMetric(addTotal);
+
+ const addSkippedTotal = new client.Counter({
+ name: 'buffer_add_skipped_total',
+ help: 'add() calls that short-circuited (no enqueue). Reason: cached, etc.',
+ labelNames: ['buffer', 'reason'],
+ });
+ register.registerMetric(addSkippedTotal);
+
+ for (const buf of bufferList) {
+ buf.flushObserver = (obs) => {
+ flushTotal.inc({
+ buffer: obs.buffer,
+ result: obs.result,
+ trigger: obs.trigger,
+ });
+ flushDuration.observe(
+ { buffer: obs.buffer, result: obs.result, trigger: obs.trigger },
+ obs.totalMs
+ );
+
+ if (obs.llenAtStart != null) {
+ flushLlenAtStart.observe({ buffer: obs.buffer }, obs.llenAtStart);
+ }
+
+ if (obs.rowsProcessed != null && obs.rowsProcessed > 0) {
+ flushRowsTotal.inc({ buffer: obs.buffer }, obs.rowsProcessed);
+ }
+
+ if (obs.phases?.lrangeMs != null) {
+ redisOpDurationMs.observe(
+ { buffer: obs.buffer, op: 'lrange' },
+ obs.phases.lrangeMs
+ );
+ }
+ if (obs.phases?.trimMs != null) {
+ redisOpDurationMs.observe(
+ { buffer: obs.buffer, op: 'trim' },
+ obs.phases.trimMs
+ );
+ }
+ if (obs.phases?.chFetchMs != null) {
+ chFetchDurationMs.observe({ buffer: obs.buffer }, obs.phases.chFetchMs);
+ }
+ if (obs.phases?.chInsertMs != null) {
+ chInsertDurationMs.observe(
+ { buffer: obs.buffer },
+ obs.phases.chInsertMs
+ );
+ }
+ };
+
+ buf.addObserver = (obs) => {
+ if (obs.skipped) {
+ addSkippedTotal.inc({
+ buffer: obs.buffer,
+ reason: obs.skipReason ?? 'unknown',
+ });
+ // Don't pollute add-latency histogram with no-op fast paths
+ return;
+ }
+ addTotal.inc({ buffer: obs.buffer });
+ addDurationMs.observe({ buffer: obs.buffer }, obs.durationMs);
+ };
+ }
+}
diff --git a/packages/core/src/buffers/clickhouse.ts b/packages/core/src/buffers/clickhouse.ts
new file mode 100644
index 000000000..a6824616e
--- /dev/null
+++ b/packages/core/src/buffers/clickhouse.ts
@@ -0,0 +1,15 @@
+// @openpanel/db's clickhouse/client.ts builds a real client and a real pino
+// logger (with a pino-pretty worker thread) the moment it is imported, and
+// `createBuffers` sits on the curated barrel — a static import here would pay
+// that once per `bun test --isolate` file. Same choice insight.service.ts and
+// event.service.ts made, for the same reason. Resolved once per process; a
+// flush pays one already-settled promise.
+
+type ClickHouseModule = typeof import('@openpanel/db/src/clickhouse/client');
+
+let pending: Promise<ClickHouseModule> | undefined;
+
+export function loadClickHouse(): Promise<ClickHouseModule> {
+ pending ??= import('@openpanel/db/src/clickhouse/client');
+ return pending;
+}
diff --git a/packages/core/src/buffers/create-buffers.ts b/packages/core/src/buffers/create-buffers.ts
new file mode 100644
index 000000000..f34a031f9
--- /dev/null
+++ b/packages/core/src/buffers/create-buffers.ts
@@ -0,0 +1,36 @@
+// Built ONCE at boot and hung on `AppDeps.buffers` — never module singletons
+// (ADR-007: "buffers stay boot singletons on AppDeps"). V1's
+// `packages/db/src/buffers/index.ts` is a thin delegate over one call to this
+// factory, so a V1 process and a V2 process reach the same seven objects and
+// the same Redis lists.
+
+import type { BufferDeps } from './base-buffer';
+import { BotBuffer } from './bot-buffer';
+import { EventBuffer } from './event-buffer';
+import { GroupBuffer } from './group-buffer';
+import { ProfileBackfillBuffer } from './profile-backfill-buffer';
+import { ProfileBuffer } from './profile-buffer';
+import { ReplayBuffer } from './replay-buffer';
+import { SessionBuffer } from './session-buffer';
+
+export interface Buffers {
+ event: EventBuffer;
+ profile: ProfileBuffer;
+ profileBackfill: ProfileBackfillBuffer;
+ bot: BotBuffer;
+ session: SessionBuffer;
+ replay: ReplayBuffer;
+ group: GroupBuffer;
+}
+
+export function createBuffers(deps: BufferDeps): Buffers {
+ return {
+ event: new EventBuffer(deps),
+ profile: new ProfileBuffer(deps),
+ profileBackfill: new ProfileBackfillBuffer(deps),
+ bot: new BotBuffer(deps),
+ session: new SessionBuffer(deps),
+ replay: new ReplayBuffer(deps),
+ group: new GroupBuffer(deps),
+ };
+}
diff --git a/packages/db/src/buffers/event-buffer.test.ts b/packages/core/src/buffers/event-buffer.test.ts
similarity index 81%
rename from packages/db/src/buffers/event-buffer.test.ts
rename to packages/core/src/buffers/event-buffer.test.ts
index 66e95f130..910ced559 100644
--- a/packages/db/src/buffers/event-buffer.test.ts
+++ b/packages/core/src/buffers/event-buffer.test.ts
@@ -1,13 +1,26 @@
+import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';
import type { Readable } from 'node:stream';
import { getRedisCache } from '@openpanel/redis';
-import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
-import * as chClient from '../clickhouse/client';
-const { ch } = chClient;
+import { stubBufferDeps } from '../../test/buffer-fixtures';
-// Break circular dep: event-buffer -> event.service -> buffers/index -> EventBuffer
-vi.mock('../services/event.service', () => ({}));
+// `./clickhouse` is the one seam the buffers reach ClickHouse through, and it
+// is the right thing to replace: mocking `@openpanel/db/src/clickhouse/client`
+// instead would need the real module imported first (to spread and restore it,
+// since `mock.module` replaces a specifier process-wide without `--isolate`),
+// and importing it builds a client and a pino transport worker thread per test
+// file. This module has one export, so the replacement is complete rather than
+// partial — and it is restored in afterAll regardless.
+const realClickhouse = { ...(await import('./clickhouse')) };
-import { EventBuffer, extractProjectId } from './event-buffer';
+const chInsert = mock(async (_options: unknown): Promise<unknown> => undefined);
+const chQuery = mock(async (_sql: string): Promise<unknown[]> => []);
+
+mock.module('./clickhouse', () => ({
+ loadClickHouse: () => Promise.resolve({ ch: { insert: chInsert }, chQuery }),
+}));
+
+const { EventBuffer, extractProjectId } = await import('./event-buffer');
+type EventBuffer = InstanceType<typeof EventBuffer>;
/** Drain an object-mode Readable into an array of line strings. event-buffer
* yields each JSONEachRow row as its own string chunk; the @clickhouse/client
@@ -25,20 +38,25 @@ const redis = getRedisCache();
beforeEach(async () => {
const keys = await redis.keys('event*');
- if (keys.length > 0) await redis.del(...keys);
+ if (keys.length > 0) {
+ await redis.del(...keys);
+ }
+ chInsert.mockClear();
+ chQuery.mockClear();
});
-afterAll(async () => {
- try {
- await redis.quit();
- } catch {}
+// The shared `getRedisCache()` client is deliberately NOT quit here: bun runs
+// every file in one process, and closing the singleton takes it away from the
+// files that run next (V1's vitest isolated per file, so it could).
+afterAll(() => {
+ mock.module('./clickhouse', () => realClickhouse);
});
describe('EventBuffer', () => {
let eventBuffer: EventBuffer;
beforeEach(() => {
- eventBuffer = new EventBuffer();
+ eventBuffer = new EventBuffer(stubBufferDeps);
});
it('adds regular event directly to buffer queue', async () => {
@@ -174,14 +192,14 @@ describe('EventBuffer', () => {
expect(await eventBuffer.getBufferSize()).toBe(2);
- const insertSpy = vi
- .spyOn(ch, 'insert')
- .mockResolvedValueOnce(undefined as any);
-
await eventBuffer.processBuffer();
- expect(insertSpy).toHaveBeenCalled();
- const callArgs = insertSpy.mock.calls[0]![0];
+ expect(chInsert).toHaveBeenCalled();
+ const callArgs = chInsert.mock.calls[0]![0] as {
+ format: string;
+ table: string;
+ values: unknown;
+ };
expect(callArgs.format).toBe('JSONEachRow');
expect(callArgs.table).toBe('events');
// After the raw-passthrough optimisation, `values` is an
@@ -201,14 +219,12 @@ describe('EventBuffer', () => {
expect(JSON.parse(lines[1]!).name).toBe('event2');
expect(await eventBuffer.getBufferSize()).toBe(0);
-
- insertSpy.mockRestore();
});
it('processes buffer with chunking', async () => {
const prev = process.env.EVENT_BUFFER_CHUNK_SIZE;
process.env.EVENT_BUFFER_CHUNK_SIZE = '2';
- const eb = new EventBuffer();
+ const eb = new EventBuffer(stubBufferDeps);
for (let i = 0; i < 4; i++) {
eb.add({
@@ -219,39 +235,32 @@ describe('EventBuffer', () => {
}
await eb.flush();
- const insertSpy = vi
- .spyOn(ch, 'insert')
- .mockResolvedValue(undefined as any);
-
await eb.processBuffer();
- expect(insertSpy).toHaveBeenCalledTimes(2);
+ expect(chInsert).toHaveBeenCalledTimes(2);
const call1 = await streamToLines(
- insertSpy.mock.calls[0]![0].values as Readable,
+ (chInsert.mock.calls[0]![0] as { values: Readable }).values
);
const call2 = await streamToLines(
- insertSpy.mock.calls[1]![0].values as Readable,
+ (chInsert.mock.calls[1]![0] as { values: Readable }).values
);
expect(call1.length).toBe(2);
expect(call2.length).toBe(2);
- if (prev === undefined) delete process.env.EVENT_BUFFER_CHUNK_SIZE;
- else process.env.EVENT_BUFFER_CHUNK_SIZE = prev;
-
- insertSpy.mockRestore();
+ if (prev === undefined) {
+ delete process.env.EVENT_BUFFER_CHUNK_SIZE;
+ } else {
+ process.env.EVENT_BUFFER_CHUNK_SIZE = prev;
+ }
});
it('tracks active visitors', async () => {
- const querySpy = vi
- .spyOn(chClient, 'chQuery')
- .mockResolvedValueOnce([{ count: 2 }] as any);
+ chQuery.mockResolvedValueOnce([{ count: 2 }]);
const count = await eventBuffer.getActiveVisitorCount('p9');
expect(count).toBe(2);
- expect(querySpy).toHaveBeenCalledOnce();
- expect(querySpy.mock.calls[0]![0]).toContain("project_id = 'p9'");
-
- querySpy.mockRestore();
+ expect(chQuery).toHaveBeenCalledTimes(1);
+ expect(chQuery.mock.calls[0]![0]).toContain("project_id = 'p9'");
});
it('handles multiple sessions independently — all events go to buffer', async () => {
@@ -313,18 +322,14 @@ describe('EventBuffer', () => {
} as any);
await eventBuffer.flush();
- const insertSpy = vi
- .spyOn(ch, 'insert')
- .mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+ chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
// Errors propagate to tryFlush (which resyncs the counter). The safety
// property — queue preserved on CH failure — still holds.
await expect(eventBuffer.processBuffer()).rejects.toThrow(
- 'ClickHouse unavailable',
+ 'ClickHouse unavailable'
);
expect(await eventBuffer.getBufferSize()).toBe(1);
-
- insertSpy.mockRestore();
});
});
diff --git a/packages/db/src/buffers/event-buffer.ts b/packages/core/src/buffers/event-buffer.ts
similarity index 93%
rename from packages/db/src/buffers/event-buffer.ts
rename to packages/core/src/buffers/event-buffer.ts
index d29003f63..896107dd2 100644
--- a/packages/db/src/buffers/event-buffer.ts
+++ b/packages/core/src/buffers/event-buffer.ts
@@ -1,7 +1,7 @@
import { getRedisCache, publishEvent } from '@openpanel/redis';
-import { ch, chQuery } from '../clickhouse/client';
-import type { IClickhouseEvent } from '../services/event.service';
-import { BaseBuffer } from './base-buffer';
+import type { IClickhouseEvent } from '../modules/event/event.service';
+import { BaseBuffer, type BufferDeps } from './base-buffer';
+import { loadClickHouse } from './clickhouse';
const PROJECT_ID_NEEDLE = '"project_id":"';
@@ -26,9 +26,14 @@ const PROJECT_ID_NEEDLE = '"project_id":"';
*/
export function extractProjectId(line: string): string | null {
const first = line.indexOf(PROJECT_ID_NEEDLE);
- if (first < 0) return null;
+ if (first < 0) {
+ return null;
+ }
- const second = line.indexOf(PROJECT_ID_NEEDLE, first + PROJECT_ID_NEEDLE.length);
+ const second = line.indexOf(
+ PROJECT_ID_NEEDLE,
+ first + PROJECT_ID_NEEDLE.length
+ );
if (second >= 0) {
try {
const obj = JSON.parse(line) as { project_id?: unknown };
@@ -43,7 +48,9 @@ export function extractProjectId(line: string): string | null {
// valueEnd === valueStart means the value is empty (`"project_id":""`)
// — treat as missing so we don't pollute pub/sub counts with an empty
// key. Matches the old regex's `[^"]+` (one-or-more) behavior.
- if (valueEnd <= valueStart) return null;
+ if (valueEnd <= valueStart) {
+ return null;
+ }
return line.slice(valueStart, valueEnd);
}
@@ -70,8 +77,8 @@ export class EventBuffer extends BaseBuffer {
private queueKey = 'event_buffer:queue';
- constructor() {
- super({
+ constructor(deps: BufferDeps) {
+ super(deps, {
name: 'event',
onFlush: async () => {
await this.processBuffer();
@@ -212,16 +219,14 @@ export class EventBuffer extends BaseBuffer {
for (let i = 0; i < queueEvents.length; i++) {
const projectId = extractProjectId(queueEvents[i]!);
if (projectId) {
- countByProject.set(
- projectId,
- (countByProject.get(projectId) ?? 0) + 1,
- );
+ countByProject.set(projectId, (countByProject.get(projectId) ?? 0) + 1);
}
if ((i + 1) % yieldEvery === 0) {
await this.yieldToEventLoop();
}
}
+ const { ch } = await loadClickHouse();
const chStart = performance.now();
await this.parallelLimit(
this.chunks(queueEvents, this.chunkSize),
@@ -233,7 +238,7 @@ export class EventBuffer extends BaseBuffer {
values: this.jsonEachRowStream(chunk),
format: 'JSONEachRow',
clickhouse_settings: this.getClickhouseSettings(),
- }),
+ })
);
const chInsertMs = performance.now() - chStart;
@@ -252,6 +257,7 @@ export class EventBuffer extends BaseBuffer {
}
public async getActiveVisitorCount(projectId: string): Promise<number> {
+ const { chQuery } = await loadClickHouse();
const rows = await chQuery<{ count: number }>(
`SELECT uniq(profile_id) AS count
FROM events
diff --git a/packages/db/src/buffers/group-buffer.ts b/packages/core/src/buffers/group-buffer.ts
similarity index 93%
rename from packages/db/src/buffers/group-buffer.ts
rename to packages/core/src/buffers/group-buffer.ts
index feb66fcfd..0462fb469 100644
--- a/packages/db/src/buffers/group-buffer.ts
+++ b/packages/core/src/buffers/group-buffer.ts
@@ -1,15 +1,10 @@
import { toDots } from '@openpanel/common';
-import { getSafeJson } from '@openpanel/json';
import { getRedisCache, type Redis } from '@openpanel/redis';
import shallowEqual from 'fast-deep-equal';
import sqlstring from 'sqlstring';
-import {
- ch,
- chQuery,
- formatClickhouseDate,
- TABLE_NAMES,
-} from '../clickhouse/client';
-import { BaseBuffer } from './base-buffer';
+import { getSafeJson } from '../shared/json';
+import { BaseBuffer, type BufferDeps } from './base-buffer';
+import { loadClickHouse } from './clickhouse';
type IGroupBufferEntry = {
project_id: string;
@@ -55,8 +50,8 @@ export class GroupBuffer extends BaseBuffer {
private redis: Redis;
- constructor() {
- super({
+ constructor(deps: BufferDeps) {
+ super(deps, {
name: 'group',
onFlush: async () => {
await this.processBuffer();
@@ -84,6 +79,7 @@ export class GroupBuffer extends BaseBuffer {
projectId: string,
id: string
): Promise<IGroupCacheEntry | null> {
+ const { chQuery, TABLE_NAMES } = await loadClickHouse();
const rows = await chQuery<IGroupCacheEntry>(`
SELECT project_id, id, type, name, properties, created_at
FROM ${TABLE_NAMES.groups} FINAL
@@ -97,6 +93,7 @@ export class GroupBuffer extends BaseBuffer {
async add(input: IGroupBufferInput): Promise<void> {
return this.timeAdd(async () => {
try {
+ const { formatClickhouseDate } = await loadClickHouse();
const cacheKey = this.getCacheKey(input.projectId, input.id);
const existing =
@@ -179,6 +176,7 @@ export class GroupBuffer extends BaseBuffer {
// Raw passthrough: each Redis entry is already a valid JSONEachRow
// line. Streaming raw strings to CH skips JSON.parse + the client's
// re-stringify on the hot path.
+ const { ch, TABLE_NAMES } = await loadClickHouse();
const chStart = performance.now();
await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>
ch.insert({
@@ -186,7 +184,7 @@ export class GroupBuffer extends BaseBuffer {
values: this.jsonEachRowStream(chunk),
format: 'JSONEachRow',
clickhouse_settings: this.getClickhouseSettings(),
- }),
+ })
);
const chInsertMs = performance.now() - chStart;
diff --git a/packages/db/src/buffers/partial-json-match.ts b/packages/core/src/buffers/partial-json-match.ts
similarity index 90%
rename from packages/db/src/buffers/partial-json-match.ts
rename to packages/core/src/buffers/partial-json-match.ts
index 84e0340ac..8be454410 100644
--- a/packages/db/src/buffers/partial-json-match.ts
+++ b/packages/core/src/buffers/partial-json-match.ts
@@ -22,10 +22,7 @@ export function isPartialMatch(source: any, partial: any): boolean {
// Check each property in partial
for (const key in partial) {
- if (
- Object.prototype.hasOwnProperty.call(partial, key) &&
- partial[key] !== undefined
- ) {
+ if (Object.hasOwn(partial, key) && partial[key] !== undefined) {
// If property doesn't exist in source, no match
if (!(key in source)) {
return false;
diff --git a/packages/db/src/buffers/profile-backfill-buffer.ts b/packages/core/src/buffers/profile-backfill-buffer.ts
similarity index 79%
rename from packages/db/src/buffers/profile-backfill-buffer.ts
rename to packages/core/src/buffers/profile-backfill-buffer.ts
index 163137dfb..31b050ed6 100644
--- a/packages/db/src/buffers/profile-backfill-buffer.ts
+++ b/packages/core/src/buffers/profile-backfill-buffer.ts
@@ -1,8 +1,8 @@
-import { getSafeJson } from '@openpanel/json';
-import { type Redis, getRedisCache } from '@openpanel/redis';
+import { getRedisCache, type Redis } from '@openpanel/redis';
import sqlstring from 'sqlstring';
-import { TABLE_NAMES, ch, getReplicatedTableName } from '../clickhouse/client';
-import { BaseBuffer } from './base-buffer';
+import { getSafeJson } from '../shared/json';
+import { BaseBuffer, type BufferDeps } from './base-buffer';
+import { loadClickHouse } from './clickhouse';
export interface ProfileBackfillEntry {
projectId: string;
@@ -21,8 +21,8 @@ export class ProfileBackfillBuffer extends BaseBuffer {
private readonly redisKey = 'profile-backfill-buffer';
private redis: Redis;
- constructor() {
- super({
+ constructor(deps: BufferDeps) {
+ super(deps, {
name: 'profile-backfill',
onFlush: async () => {
await this.processBuffer();
@@ -39,13 +39,12 @@ export class ProfileBackfillBuffer extends BaseBuffer {
} catch (error) {
this.logger.error(
{ err: error },
- 'Failed to add profile backfill entry',
+ 'Failed to add profile backfill entry'
);
}
});
}
-
protected getRedisListKey(): string {
return this.redisKey;
}
@@ -70,6 +69,7 @@ export class ProfileBackfillBuffer extends BaseBuffer {
}
const entries = Array.from(seen.values());
+ const { ch, TABLE_NAMES, getReplicatedTableName } = await loadClickHouse();
const table = getReplicatedTableName(TABLE_NAMES.events);
const chunks = this.chunks(entries, CHUNK_SIZE);
@@ -78,10 +78,16 @@ export class ProfileBackfillBuffer extends BaseBuffer {
const chStart = performance.now();
for (const chunk of chunks) {
const caseClause = chunk
- .map(({ sessionId, profileId }) => `WHEN ${sqlstring.escape(sessionId)} THEN ${sqlstring.escape(profileId)}`)
+ .map(
+ ({ sessionId, profileId }) =>
+ `WHEN ${sqlstring.escape(sessionId)} THEN ${sqlstring.escape(profileId)}`
+ )
.join('\n');
const tupleList = chunk
- .map(({ projectId, sessionId }) => `(${sqlstring.escape(projectId)}, ${sqlstring.escape(sessionId)})`)
+ .map(
+ ({ projectId, sessionId }) =>
+ `(${sqlstring.escape(projectId)}, ${sqlstring.escape(sessionId)})`
+ )
.join(',');
const query = `
@@ -96,7 +102,7 @@ export class ProfileBackfillBuffer extends BaseBuffer {
query,
clickhouse_settings: {
mutations_sync: '0',
- allow_experimental_lightweight_update: '1'
+ allow_experimental_lightweight_update: '1',
},
});
diff --git a/packages/db/src/buffers/profile-buffer.test.ts b/packages/core/src/buffers/profile-buffer.test.ts
similarity index 70%
rename from packages/db/src/buffers/profile-buffer.test.ts
rename to packages/core/src/buffers/profile-buffer.test.ts
index b0a9cb8d1..1257aa172 100644
--- a/packages/db/src/buffers/profile-buffer.test.ts
+++ b/packages/core/src/buffers/profile-buffer.test.ts
@@ -1,24 +1,33 @@
+import { afterAll, beforeEach, describe, expect, it, mock } from 'bun:test';
import { getRedisCache } from '@openpanel/redis';
-import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
-import type { IClickhouseProfile } from '../services/profile.service';
-
-// Mock chQuery to avoid hitting real ClickHouse
-vi.mock('../clickhouse/client', () => ({
- ch: {
- insert: vi.fn().mockResolvedValue(undefined),
- },
- chQuery: vi.fn().mockResolvedValue([]),
- TABLE_NAMES: {
- profiles: 'profiles',
- },
+import { stubBufferDeps } from '../../test/buffer-fixtures';
+import type { IClickhouseProfile } from '../modules/profile/profile.service';
+
+// `./clickhouse` is the one seam the buffers reach ClickHouse through — see
+// event-buffer.test.ts's header for why that, and not
+// `@openpanel/db/src/clickhouse/client`, is what gets replaced.
+const realClickhouse = { ...(await import('./clickhouse')) };
+
+const chInsert = mock(async (_options: unknown): Promise<unknown> => undefined);
+const chQuery = mock(async (_sql: string): Promise<IClickhouseProfile[]> => []);
+
+mock.module('./clickhouse', () => ({
+ loadClickHouse: () =>
+ Promise.resolve({
+ ch: { insert: chInsert },
+ chQuery,
+ TABLE_NAMES: { profiles: 'profiles' },
+ }),
}));
-import { ProfileBuffer } from './profile-buffer';
-import { chQuery } from '../clickhouse/client';
+const { ProfileBuffer } = await import('./profile-buffer');
+type ProfileBuffer = InstanceType<typeof ProfileBuffer>;
const redis = getRedisCache();
-function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile {
+function makeProfile(
+ overrides: Partial<IClickhouseProfile>
+): IClickhouseProfile {
const now = new Date().toISOString();
return {
id: 'profile-1',
@@ -38,28 +47,36 @@ function makeProfile(overrides: Partial<IClickhouseProfile>): IClickhouseProfile
beforeEach(async () => {
const keys = [
- ...await redis.keys('profile*'),
- ...await redis.keys('lock:profile'),
+ ...(await redis.keys('profile*')),
+ ...(await redis.keys('lock:profile')),
];
- if (keys.length > 0) await redis.del(...keys);
- vi.mocked(chQuery).mockResolvedValue([]);
+ if (keys.length > 0) {
+ await redis.del(...keys);
+ }
+ chInsert.mockClear();
+ chQuery.mockReset();
+ chQuery.mockResolvedValue([]);
});
-afterAll(async () => {
- try {
- await redis.quit();
- } catch {}
+// The shared `getRedisCache()` client is deliberately NOT quit here: bun runs
+// every file in one process, and closing the singleton takes it away from the
+// files that run next (V1's vitest isolated per file, so it could).
+afterAll(() => {
+ mock.module('./clickhouse', () => realClickhouse);
});
describe('ProfileBuffer', () => {
let profileBuffer: ProfileBuffer;
beforeEach(() => {
- profileBuffer = new ProfileBuffer();
+ profileBuffer = new ProfileBuffer(stubBufferDeps);
});
it('adds a profile to the buffer', async () => {
- const profile = makeProfile({ first_name: 'John', email: '[email protected]' });
+ const profile = makeProfile({
+ first_name: 'John',
+ email: '[email protected]',
+ });
const sizeBefore = await profileBuffer.getBufferSize();
await profileBuffer.add(profile);
@@ -143,7 +160,7 @@ describe('ProfileBuffer', () => {
groups: ['existing-group'],
});
- vi.mocked(chQuery).mockResolvedValue([existingInClickhouse]);
+ chQuery.mockResolvedValue([existingInClickhouse]);
const incomingProfile = makeProfile({
first_name: '',
@@ -173,36 +190,24 @@ describe('ProfileBuffer', () => {
it('retains profiles in queue when ClickHouse insert fails', async () => {
await profileBuffer.add(makeProfile({ first_name: 'John' }));
- const { ch } = await import('../clickhouse/client');
- const insertSpy = vi
- .spyOn(ch, 'insert')
- .mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+ chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
// Errors propagate to tryFlush (which resyncs the counter). The safety
// property — queue preserved on CH failure — still holds.
await expect(profileBuffer.processBuffer()).rejects.toThrow(
- 'ClickHouse unavailable',
+ 'ClickHouse unavailable'
);
expect(await profileBuffer.getBufferSize()).toBe(1);
-
- insertSpy.mockRestore();
});
it('proceeds with insert when ClickHouse fetch fails (treats profiles as new)', async () => {
- vi.mocked(chQuery).mockRejectedValueOnce(new Error('ClickHouse unavailable'));
-
- const { ch } = await import('../clickhouse/client');
- const insertSpy = vi
- .spyOn(ch, 'insert')
- .mockResolvedValueOnce(undefined as any);
+ chQuery.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
await profileBuffer.add(makeProfile({ first_name: 'John' }));
await profileBuffer.processBuffer();
// Insert must still have been called — no data loss even when fetch fails
- expect(insertSpy).toHaveBeenCalled();
+ expect(chInsert).toHaveBeenCalled();
expect(await profileBuffer.getBufferSize()).toBe(0);
-
- insertSpy.mockRestore();
});
});
diff --git a/packages/db/src/buffers/profile-buffer.ts b/packages/core/src/buffers/profile-buffer.ts
similarity index 96%
rename from packages/db/src/buffers/profile-buffer.ts
rename to packages/core/src/buffers/profile-buffer.ts
index 554467d4c..c16eccdf9 100644
--- a/packages/db/src/buffers/profile-buffer.ts
+++ b/packages/core/src/buffers/profile-buffer.ts
@@ -1,11 +1,11 @@
import { deepMergeObjects } from '@openpanel/common';
-import { getSafeJson } from '@openpanel/json';
import { getRedisCache, type Redis } from '@openpanel/redis';
import { omit, uniq } from 'ramda';
import sqlstring from 'sqlstring';
-import { ch, chQuery, TABLE_NAMES } from '../clickhouse/client';
-import type { IClickhouseProfile } from '../services/profile.service';
-import { BaseBuffer } from './base-buffer';
+import type { IClickhouseProfile } from '../modules/profile/profile.service';
+import { getSafeJson } from '../shared/json';
+import { BaseBuffer, type BufferDeps } from './base-buffer';
+import { loadClickHouse } from './clickhouse';
// Inlined to avoid a circular value-import with `profile.service.ts`
// (which imports `profileBuffer` from `../buffers`). Keep this in sync
@@ -62,8 +62,8 @@ export class ProfileBuffer extends BaseBuffer {
private readonly redis: Redis;
- constructor() {
- super({
+ constructor(deps: BufferDeps) {
+ super(deps, {
name: 'profile',
onFlush: async () => {
await this.processBuffer();
@@ -190,6 +190,7 @@ export class ProfileBuffer extends BaseBuffer {
private async batchFetchFromClickhouse(
profiles: IClickhouseProfile[]
): Promise<Map<string, IClickhouseProfile>> {
+ const { chQuery, TABLE_NAMES } = await loadClickHouse();
const result = new Map<string, IClickhouseProfile>();
// Non-external (anonymous/device) profiles get a 2-day recency filter to
@@ -276,11 +277,13 @@ export class ProfileBuffer extends BaseBuffer {
const mergedInBatch = new Map<string, IClickhouseProfile>();
for (let i = 0; i < rawProfiles.length; i++) {
const profile = getSafeJson<IClickhouseProfile>(rawProfiles[i]!);
- if (!profile) continue;
+ if (!profile) {
+ continue;
+ }
const key = `${profile.project_id}:${profile.id}`;
mergedInBatch.set(
key,
- this.mergeProfiles(mergedInBatch.get(key) ?? null, profile),
+ this.mergeProfiles(mergedInBatch.get(key) ?? null, profile)
);
if ((i + 1) % mergeYieldEvery === 0) {
await this.yieldToEventLoop();
@@ -342,7 +345,7 @@ export class ProfileBuffer extends BaseBuffer {
const key = `${profile.project_id}:${profile.id}`;
const merged = this.mergeProfiles(
existingByKey.get(key) ?? null,
- profile,
+ profile
);
toInsert.push(merged);
multi.set(
@@ -352,13 +355,14 @@ export class ProfileBuffer extends BaseBuffer {
}),
JSON.stringify(merged),
'EX',
- this.ttlInSeconds,
+ this.ttlInSeconds
);
if ((i + 1) % finalYieldEvery === 0) {
await this.yieldToEventLoop();
}
}
+ const { ch, TABLE_NAMES } = await loadClickHouse();
const chStart = performance.now();
await this.parallelLimit(this.chunks(toInsert, this.chunkSize), (chunk) =>
ch.insert({
diff --git a/packages/db/src/buffers/replay-buffer.ts b/packages/core/src/buffers/replay-buffer.ts
similarity index 92%
rename from packages/db/src/buffers/replay-buffer.ts
rename to packages/core/src/buffers/replay-buffer.ts
index b2c1cefa2..a5aa08f21 100644
--- a/packages/db/src/buffers/replay-buffer.ts
+++ b/packages/core/src/buffers/replay-buffer.ts
@@ -1,6 +1,6 @@
import { getRedisCache } from '@openpanel/redis';
-import { ch, TABLE_NAMES } from '../clickhouse/client';
-import { BaseBuffer } from './base-buffer';
+import { BaseBuffer, type BufferDeps } from './base-buffer';
+import { loadClickHouse } from './clickhouse';
export interface IClickhouseSessionReplayChunk {
project_id: string;
@@ -23,8 +23,8 @@ export class ReplayBuffer extends BaseBuffer {
private readonly redisKey = 'replay-buffer';
- constructor() {
- super({
+ constructor(deps: BufferDeps) {
+ super(deps, {
name: 'replay',
onFlush: async () => {
await this.processBuffer();
@@ -76,6 +76,7 @@ export class ReplayBuffer extends BaseBuffer {
// raw strings to CH skips JSON.parse × N on the worker AND the
// client's internal JSON.stringify × N — significant because each
// rrweb chunk's `payload` is 10–100KB.
+ const { ch, TABLE_NAMES } = await loadClickHouse();
const chStart = performance.now();
await this.parallelLimit(this.chunks(items, this.chunkSize), (chunk) =>
ch.insert({
@@ -83,7 +84,7 @@ export class ReplayBuffer extends BaseBuffer {
values: this.jsonEachRowStream(chunk),
format: 'JSONEachRow',
clickhouse_settings: this.getClickhouseSettings(),
- }),
+ })
);
const chInsertMs = performance.now() - chStart;
diff --git a/packages/db/src/buffers/session-buffer.test.ts b/packages/core/src/buffers/session-buffer.test.ts
similarity index 88%
rename from packages/db/src/buffers/session-buffer.test.ts
rename to packages/core/src/buffers/session-buffer.test.ts
index 45d4b9faf..2bbf587ce 100644
--- a/packages/db/src/buffers/session-buffer.test.ts
+++ b/packages/core/src/buffers/session-buffer.test.ts
@@ -1,4 +1,3 @@
-import { getRedisCache } from '@openpanel/redis';
import {
afterAll,
afterEach,
@@ -6,21 +5,36 @@ import {
describe,
expect,
it,
- vi,
-} from 'vitest';
-import { ch } from '../clickhouse/client';
-
-vi.mock('../clickhouse/client', () => ({
- ch: {
- insert: vi.fn().mockResolvedValue(undefined),
- },
- TABLE_NAMES: {
- sessions: 'sessions',
- },
+ mock,
+} from 'bun:test';
+import { getRedisCache } from '@openpanel/redis';
+import { stubBufferDeps } from '../../test/buffer-fixtures';
+import type { IServiceCreateEventPayload } from '../modules/event/event.service';
+import type { IClickhouseSession } from '../modules/session/session.service';
+
+// `./clickhouse` is the one seam the buffers reach ClickHouse through — see
+// event-buffer.test.ts's header for why that, and not
+// `@openpanel/db/src/clickhouse/client`, is what gets replaced.
+const realClickhouse = { ...(await import('./clickhouse')) };
+
+const chInsert = mock(
+ async (_options: {
+ table: string;
+ format: string;
+ values: IClickhouseSession[];
+ }): Promise<unknown> => undefined
+);
+
+mock.module('./clickhouse', () => ({
+ loadClickHouse: () =>
+ Promise.resolve({
+ ch: { insert: chInsert },
+ TABLE_NAMES: { sessions: 'sessions' },
+ }),
}));
-import type { IServiceCreateEventPayload } from '../services/event.service';
-import { SessionBuffer } from './session-buffer';
+const { SessionBuffer } = await import('./session-buffer');
+type SessionBuffer = InstanceType<typeof SessionBuffer>;
const redis = getRedisCache();
@@ -74,20 +88,22 @@ beforeEach(async () => {
for (let i = 0; i < keys.length; i += BATCH) {
await redis.del(...keys.slice(i, i + BATCH));
}
- vi.mocked(ch.insert).mockResolvedValue(undefined as any);
+ chInsert.mockReset();
+ chInsert.mockResolvedValue(undefined);
});
-afterAll(async () => {
- try {
- await redis.quit();
- } catch {}
+// The shared `getRedisCache()` client is deliberately NOT quit here: bun runs
+// every file in one process, and closing the singleton takes it away from the
+// files that run next (V1's vitest isolated per file, so it could).
+afterAll(() => {
+ mock.module('./clickhouse', () => realClickhouse);
});
describe('SessionBuffer', () => {
let sessionBuffer: SessionBuffer;
beforeEach(() => {
- sessionBuffer = new SessionBuffer();
+ sessionBuffer = new SessionBuffer(stubBufferDeps);
});
it('opens a new session and stores it at session:{pid}:{did}', async () => {
@@ -302,18 +318,12 @@ describe('SessionBuffer', () => {
it('processes buffer and inserts sessions into ClickHouse', async () => {
await sessionBuffer.ingest(makePayload());
- const insertSpy = vi
- .spyOn(ch, 'insert')
- .mockResolvedValueOnce(undefined as any);
-
await sessionBuffer.processBuffer();
- expect(insertSpy).toHaveBeenCalledWith(
+ expect(chInsert).toHaveBeenCalledWith(
expect.objectContaining({ table: 'sessions', format: 'JSONEachRow' })
);
expect(await sessionBuffer.getBufferSize()).toBe(0);
-
- insertSpy.mockRestore();
});
it('squash does not orphan the creation row when create + updates share a batch', async () => {
@@ -331,14 +341,12 @@ describe('SessionBuffer', () => {
}
const inserted: Array<{ sign: number; version: number }> = [];
- const insertSpy = vi
- .spyOn(ch, 'insert')
- .mockImplementation(async ({ values }: any) => {
- for (const v of values) {
- inserted.push({ sign: v.sign, version: v.version });
- }
- return undefined as any;
- });
+ chInsert.mockImplementation(async ({ values }) => {
+ for (const v of values) {
+ inserted.push({ sign: v.sign, version: v.version });
+ }
+ return await Promise.resolve(undefined);
+ });
await sessionBuffer.processBuffer();
@@ -356,21 +364,19 @@ describe('SessionBuffer', () => {
for (const { sign, version } of inserted) {
netByVersion.set(version, (netByVersion.get(version) ?? 0) + sign);
}
- const survivors = [...netByVersion.entries()].filter(([, net]) => net !== 0);
+ const survivors = [...netByVersion.entries()].filter(
+ ([, net]) => net !== 0
+ );
// A fresh session must collapse to exactly one final +1 row, with no
// orphaned negatives (the bug left a permanent (-1, v1)).
expect(survivors).toEqual([[EVENTS, 1]]);
-
- insertSpy.mockRestore();
});
it('retains sessions in queue when ClickHouse insert fails', async () => {
await sessionBuffer.ingest(makePayload());
- const insertSpy = vi
- .spyOn(ch, 'insert')
- .mockRejectedValueOnce(new Error('ClickHouse unavailable'));
+ chInsert.mockRejectedValueOnce(new Error('ClickHouse unavailable'));
// Errors now propagate to tryFlush (which handles them by resyncing the
// counter). processBuffer no longer swallows — we still verify the
@@ -379,8 +385,6 @@ describe('SessionBuffer', () => {
'ClickHouse unavailable'
);
expect(await sessionBuffer.getBufferSize()).toBe(1);
-
- insertSpy.mockRestore();
});
// persist() is one Lua script, so a failure part-way through must leave the
diff --git a/packages/db/src/buffers/session-buffer.ts b/packages/core/src/buffers/session-buffer.ts
similarity index 92%
rename from packages/db/src/buffers/session-buffer.ts
rename to packages/core/src/buffers/session-buffer.ts
index b93684d06..e5309b5c3 100644
--- a/packages/db/src/buffers/session-buffer.ts
+++ b/packages/core/src/buffers/session-buffer.ts
@@ -1,10 +1,10 @@
import { DateTime } from '@openpanel/common';
-import { getSafeJson } from '@openpanel/json';
import { getRedisCache, type Redis } from '@openpanel/redis';
-import { ch, TABLE_NAMES } from '../clickhouse/client';
-import type { IServiceCreateEventPayload } from '../services/event.service';
-import type { IClickhouseSession } from '../services/session.service';
-import { BaseBuffer } from './base-buffer';
+import type { IServiceCreateEventPayload } from '../modules/event/event.service';
+import type { IClickhouseSession } from '../modules/session/session.service';
+import { getSafeJson } from '../shared/json';
+import { BaseBuffer, type BufferDeps } from './base-buffer';
+import { loadClickHouse } from './clickhouse';
// 30min of idle in event-time → session ends. Matches industry default.
// Idle window for a session (boundary detection + the reaper's default deadman).
@@ -18,8 +18,7 @@ const sessionKey = (projectId: string, deviceId: string) =>
`session:${projectId}:${deviceId}`;
const profileIndexKey = (projectId: string, profileId: string) =>
`session:profile:${projectId}:${profileId}`;
-const wallclockSetKey = (projectId: string) =>
- `session:wallclock:${projectId}`;
+const wallclockSetKey = (projectId: string) => `session:wallclock:${projectId}`;
const PROJECTS_SET_KEY = 'session:projects';
// Atomic id-gated cleanup. Only deletes the blob + sorted-set entry +
@@ -91,7 +90,11 @@ return bufferLength
export type SessionIngestResult =
| { kind: 'new'; current: IClickhouseSession }
| { kind: 'extend'; current: IClickhouseSession }
- | { kind: 'boundary'; current: IClickhouseSession; closed: IClickhouseSession };
+ | {
+ kind: 'boundary';
+ current: IClickhouseSession;
+ closed: IClickhouseSession;
+ };
function toClickhouseDate(date: Date): string {
return DateTime.fromJSDate(date)
@@ -109,8 +112,9 @@ function pickUtm(
payload: IServiceCreateEventPayload,
key: 'utm_medium' | 'utm_source' | 'utm_campaign' | 'utm_content' | 'utm_term'
): string {
- const query = (payload.properties as { __query?: Record<string, unknown> } | undefined)
- ?.__query;
+ const query = (
+ payload.properties as { __query?: Record<string, unknown> } | undefined
+ )?.__query;
const v = query?.[key];
return v ? String(v) : '';
}
@@ -129,8 +133,8 @@ export class SessionBuffer extends BaseBuffer {
private readonly redisKey = 'session-buffer';
private redis: Redis;
- constructor() {
- super({
+ constructor(deps: BufferDeps) {
+ super(deps, {
name: 'session',
onFlush: async () => {
await this.processBuffer();
@@ -157,7 +161,9 @@ export class SessionBuffer extends BaseBuffer {
const deviceId = await this.redis.get(
profileIndexKey(options.projectId, options.profileId)
);
- if (!deviceId) return null;
+ if (!deviceId) {
+ return null;
+ }
const hit = await this.redis.get(sessionKey(options.projectId, deviceId));
return hit ? getSafeJson<IClickhouseSession>(hit) : null;
}
@@ -178,7 +184,9 @@ export class SessionBuffer extends BaseBuffer {
async ingest(
payload: IServiceCreateEventPayload
): Promise<SessionIngestResult | null> {
- if (!payload.projectId || !payload.deviceId) return null;
+ if (!(payload.projectId && payload.deviceId)) {
+ return null;
+ }
if (payload.name === 'session_start' || payload.name === 'session_end') {
return null;
}
@@ -281,18 +289,29 @@ export class SessionBuffer extends BaseBuffer {
if (eventTimeMs >= endMs) {
current.ended_at = eventCh;
- if (payload.path) current.exit_path = payload.path;
- if (payload.origin) current.exit_origin = payload.origin;
+ if (payload.path) {
+ current.exit_path = payload.path;
+ }
+ if (payload.origin) {
+ current.exit_origin = payload.origin;
+ }
}
if (eventTimeMs < startMs) {
current.created_at = eventCh;
- if (payload.path) current.entry_path = payload.path;
- if (payload.origin) current.entry_origin = payload.origin;
+ if (payload.path) {
+ current.entry_path = payload.path;
+ }
+ if (payload.origin) {
+ current.entry_origin = payload.origin;
+ }
} else {
- if (!current.entry_path && payload.path) current.entry_path = payload.path;
- if (!current.entry_origin && payload.origin)
+ if (!current.entry_path && payload.path) {
+ current.entry_path = payload.path;
+ }
+ if (!current.entry_origin && payload.origin) {
current.entry_origin = payload.origin;
+ }
}
current.duration =
@@ -520,12 +539,16 @@ export class SessionBuffer extends BaseBuffer {
0,
this.batchSize - 1
);
- if (events.length === 0) return;
+ if (events.length === 0) {
+ return;
+ }
const parsed: IClickhouseSession[] = [];
for (const e of events) {
const s = getSafeJson<IClickhouseSession>(e);
- if (!s) continue;
+ if (!s) {
+ continue;
+ }
// Freshly parsed object — safe to clamp in place, no clone needed.
s.duration = Math.max(0, s.duration || 0);
parsed.push(s);
@@ -543,6 +566,7 @@ export class SessionBuffer extends BaseBuffer {
? this.squashSessionsByVersion(parsed)
: parsed;
+ const { ch, TABLE_NAMES } = await loadClickHouse();
for (const chunk of this.chunks(sessions, this.chunkSize)) {
await ch.insert({
table: TABLE_NAMES.sessions,
diff --git a/packages/core/src/context.test.ts b/packages/core/src/context.test.ts
index 4f25eb2d2..628f390dc 100644
--- a/packages/core/src/context.test.ts
+++ b/packages/core/src/context.test.ts
@@ -1,5 +1,5 @@
import { beforeAll, expect, mock, test } from 'bun:test';
-import type { AppDeps, Ctx } from './context';
+import type { AppDeps, Buffers, Ctx } from './context';
import type { JobMeta } from './jobs/envelope';
import { createRecordingProducers } from './jobs/testing';
import type { QueueProducers } from './jobs.registry';
@@ -58,7 +58,7 @@ function stubDeps(): {
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
producers: {
...producers,
scope(meta) {
diff --git a/packages/core/src/context.ts b/packages/core/src/context.ts
index 0855ac2f8..c738f9632 100644
--- a/packages/core/src/context.ts
+++ b/packages/core/src/context.ts
@@ -5,21 +5,25 @@
// (ADR-007 decision 18, ADR-018 R1).
import type { SessionValidationResult } from '@openpanel/db';
+import type { Buffers } from './buffers/create-buffers';
import type { QueueProducerHandle, QueueProducers } from './jobs.registry';
import type { Logger } from './logger';
import { createServices, type Services } from './services';
import type { CookieJar, CookieOptions } from './shared/cookie';
// Handles core does not own yet. Each real type arrives with the phase that
-// builds it (db/ch/redis P3, clients P4, buffers P8); they are named now so
-// AppDeps and Ctx are written once in their final shape and replacing an alias
-// moves no field. `unknown` is deliberate — reading one before its type lands
-// is a compile error rather than a silent `any`.
+// builds it (db/ch/redis P3, clients P4); they are named now so AppDeps and
+// Ctx are written once in their final shape and replacing an alias moves no
+// field. `unknown` is deliberate — reading one before its type lands is a
+// compile error rather than a silent `any`.
export type Db = unknown;
export type ClickHouseClient = unknown;
export type RedisClient = unknown;
export type ServiceClients = unknown;
-export type Buffers = unknown;
+
+// Landed at M8-001: built once by `createBuffers(deps)` in main.ts, never a
+// module singleton.
+export type { Buffers } from './buffers/create-buffers';
// The resolved session. Type-only, so nothing of `@openpanel/db` is loaded at
// runtime — core stays importable with no database, which is what lets
diff --git a/packages/core/src/index.test.ts b/packages/core/src/index.test.ts
index b12340bd0..ad4e227d5 100644
--- a/packages/core/src/index.test.ts
+++ b/packages/core/src/index.test.ts
@@ -35,3 +35,22 @@ test('the rpc base is on the barrel, so there is one tRPC instance', () => {
test('a service, a client or a buffer is not on the barrel', () => {
expect(Object.keys(core)).not.toContain('createServices');
});
+
+// The buffers are boot singletons on AppDeps (ADR-007), so main.ts needs the
+// FACTORY and nothing else. A named instance here would be the module
+// singleton the design refuses.
+test('buffers reach the barrel as a factory, never as instances', () => {
+ expect(core.createBuffers).toBeTypeOf('function');
+ expect(core.registerBufferMetrics).toBeTypeOf('function');
+ for (const instance of [
+ 'eventBuffer',
+ 'profileBuffer',
+ 'profileBackfillBuffer',
+ 'botBuffer',
+ 'sessionBuffer',
+ 'replayBuffer',
+ 'groupBuffer',
+ ]) {
+ expect(Object.keys(core)).not.toContain(instance);
+ }
+});
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 0cb6e15c9..edc3aeed9 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -6,6 +6,32 @@
// once and mount the three route surfaces plus the tRPC router over it; a
// service, a client or a buffer is not reachable from here by design.
+// The concrete pino implementation (dissolved from @openpanel/logger,
+// M4-003). `./logger` above is the structural interface every module codes
+// against; this is what apps/api, and the still-live apps/worker, call to
+// build one.
+// The seven buffers (moved from packages/db/src/buffers, M8-001). Only the
+// FACTORY is on the barrel: they are boot singletons on `AppDeps`, built once
+// by `main.ts`, never module singletons (ADR-007). V1's
+// packages/db/src/buffers/index.ts calls this once and re-exports the
+// instances, so both boots share one set.
+export type {
+ AddObservation,
+ AddObserver,
+ BufferDeps,
+ FlushObservation,
+ FlushObserver,
+ FlushPhaseTimings,
+ FlushTrigger,
+} from './buffers/base-buffer';
+export { registerBufferMetrics } from './buffers/buffer.metrics';
+export type { Buffers } from './buffers/create-buffers';
+export { createBuffers } from './buffers/create-buffers';
+export type { IGroupBufferInput } from './buffers/group-buffer';
+export type { ProfileBackfillEntry } from './buffers/profile-backfill-buffer';
+export type { IClickhouseSessionReplayChunk } from './buffers/replay-buffer';
+export type { SessionIngestResult } from './buffers/session-buffer';
+export { SESSION_TIMEOUT_MS } from './buffers/session-buffer';
// Dissolved from @openpanel/ai (M4-005) — the chat agent app, the filter
// command bar, insight explanation/enrichment and the worker's digest and
// win-back emails all call these directly.
@@ -78,10 +104,6 @@ export {
sendSlackNotification,
slackInstaller,
} from './clients/integrations/slack';
-// The concrete pino implementation (dissolved from @openpanel/logger,
-// M4-003). `./logger` above is the structural interface every module codes
-// against; this is what apps/api, and the still-live apps/worker, call to
-// build one.
export type { ILogger } from './clients/logger';
export {
createLogger,
@@ -118,15 +140,14 @@ export {
// overview router call these.
//
// Loaded via dynamic import, NOT a static re-export like every other service
-// on this barrel: packages/db/src/buffers/base-buffer.ts already imports
-// `@openpanel/core` eagerly, and this module's own chain reaches deep into
-// `@openpanel/db` (the Prisma-backed conversation store, ~30 analytics/report
-// tool functions). A static re-export here would make THIS barrel's own
-// evaluation re-enter `@openpanel/db` while it is still mid-evaluation of
-// `./buffers` — `bot-buffer.ts extends BaseBuffer` then sees `BaseBuffer` as
-// `undefined` (a live TDZ binding that never got the chance to fill in).
-// `event-buffer.test.ts` et al. hit exactly this before this indirection was
-// added. `chatApp` and `chatRunContext` are one-time-per-process values, so
+// on this barrel: packages/db/src/buffers/index.ts imports `@openpanel/core`
+// eagerly (it is the V1 delegate over `createBuffers`), and this module's own
+// chain reaches deep into `@openpanel/db` (the Prisma-backed conversation
+// store, ~30 analytics/report tool functions). A static re-export here would
+// make THIS barrel's own evaluation re-enter `@openpanel/db` while that
+// package is still mid-evaluation, and a class the re-entering module extends
+// is then an `undefined` TDZ binding that never got the chance to fill in.
+// `chatApp` and `chatRunContext` are one-time-per-process values, so
// callers resolve them once (apps/api's Fastify wrapper does so inside its
// own already-async route registration) and keep the reference.
export type {
@@ -1044,7 +1065,7 @@ export {
// Loaded via dynamic import, NOT a static re-export, for the same reason
// `getChatApp`/`runFilterCommand` above are: `mcp.service` reaches
// `@openpanel/db` (auth's client lookup, every analytics tool), and
-// `@openpanel/db/src/buffers/base-buffer.ts` already imports `@openpanel/core`
+// `@openpanel/db/src/buffers/index.ts` already imports `@openpanel/core`
// eagerly. A static export here would make this barrel's own evaluation
// re-enter `@openpanel/db` mid-evaluation — observed as a `PagesService` TDZ
// ReferenceError two modules away, in a tool file that never otherwise runs
diff --git a/packages/core/src/jobs/workers.test.ts b/packages/core/src/jobs/workers.test.ts
index 3d2eca95d..87274d36e 100644
--- a/packages/core/src/jobs/workers.test.ts
+++ b/packages/core/src/jobs/workers.test.ts
@@ -1,6 +1,6 @@
import { expect, test } from 'bun:test';
import { z } from 'zod';
-import type { AppDeps, JobCtx } from '../context';
+import type { AppDeps, Buffers, JobCtx } from '../context';
import { queues } from '../jobs.registry';
import type { Logger } from '../logger';
import { defineJob, defineQueue } from './define';
@@ -28,7 +28,7 @@ function stubDeps(): AppDeps {
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
producers: createRecordingProducers(queues),
logger: stubLogger(),
config: { selfHosted: true },
diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.ts
index 954b40271..676c3e2ca 100644
--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts
+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts
@@ -2,7 +2,7 @@
// methods themselves are exercised in cohort.service.test.ts.
import { expect, test } from 'bun:test';
-import type { JobCtx } from '../../context';
+import type { Buffers, JobCtx } from '../../context';
import { createRecordingProducers } from '../../jobs/testing';
import { queues } from '../../jobs.registry';
import type { Logger } from '../../logger';
@@ -50,7 +50,7 @@ function stubJobCtx(cohort: Partial<CohortService>, jobName = 'test'): JobCtx {
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: createRecordingProducers(queues).queues,
services,
diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.ts
index 8c63dcacb..cd33a4488 100644
--- a/packages/core/src/modules/gsc/gsc.jobs.test.ts
+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts
@@ -2,7 +2,7 @@
// methods themselves are exercised in gsc.service.test.ts.
import { expect, test } from 'bun:test';
-import type { JobCtx } from '../../context';
+import type { Buffers, JobCtx } from '../../context';
import { createRecordingProducers } from '../../jobs/testing';
import { queues } from '../../jobs.registry';
import type { Logger } from '../../logger';
@@ -46,7 +46,7 @@ function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: createRecordingProducers(queues).queues,
services,
@@ -123,7 +123,7 @@ test('gscSync fans out one gscProjectSync enqueue per connected project', async
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: producers.queues,
services: {
diff --git a/packages/core/src/modules/gsc/gsc.service.test.ts b/packages/core/src/modules/gsc/gsc.service.test.ts
index c6d3ad1aa..14e0aa846 100644
--- a/packages/core/src/modules/gsc/gsc.service.test.ts
+++ b/packages/core/src/modules/gsc/gsc.service.test.ts
@@ -126,6 +126,20 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({
chQuery,
}));
+// Bypasses the Redis cache-aside entirely — `getGscCannibalization`'s own
+// logic is exercised directly, its caching is @openpanel/redis's concern.
+// `getRedisCache` is unused here but included for the same cross-file
+// mock.module reason as above.
+const realRedis = { ...(await import('@openpanel/redis')) };
+mock.module('@openpanel/redis', () => ({
+ ...realRedis,
+ cacheable: <T>(fn: T) => fn,
+ getRedisCache: () => ({
+ get: async () => null,
+ setex: async () => undefined,
+ }),
+}));
+
// `chQuery` above is a fake stuck in place for the rest of the process once
// this file's tests finish (`mock.module` has no per-file scope without
// `--isolate` — see AGENTS.md): the mcp module's integration suite calls the
@@ -137,20 +151,9 @@ afterAll(() => {
'@openpanel/db/src/clickhouse/client',
() => realClickhouseClient
);
+ mock.module('@openpanel/redis', () => realRedis);
});
-// Bypasses the Redis cache-aside entirely — `getGscCannibalization`'s own
-// logic is exercised directly, its caching is @openpanel/redis's concern.
-// `getRedisCache` is unused here but included for the same cross-file
-// mock.module reason as above.
-mock.module('@openpanel/redis', () => ({
- cacheable: <T>(fn: T) => fn,
- getRedisCache: () => ({
- get: async () => null,
- setex: async () => undefined,
- }),
-}));
-
const validateAuthorizationCode = mock(async () => ({
accessToken: () => 'google-access-token',
hasRefreshToken: () => true,
diff --git a/packages/core/src/modules/import/import.jobs.test.ts b/packages/core/src/modules/import/import.jobs.test.ts
index a8f276c12..da8feca35 100644
--- a/packages/core/src/modules/import/import.jobs.test.ts
+++ b/packages/core/src/modules/import/import.jobs.test.ts
@@ -2,7 +2,7 @@
// themselves are exercised in import.service.test.ts.
import { expect, test } from 'bun:test';
-import type { JobCtx } from '../../context';
+import type { Buffers, JobCtx } from '../../context';
import { createRecordingProducers } from '../../jobs/testing';
import { queues } from '../../jobs.registry';
import type { Logger } from '../../logger';
@@ -46,7 +46,7 @@ function stubJobCtx(importService: Partial<ImportService>): JobCtx {
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: createRecordingProducers(queues).queues,
services,
diff --git a/packages/core/src/modules/import/import.service.test.ts b/packages/core/src/modules/import/import.service.test.ts
index 3a9eaba7d..c98fbadd0 100644
--- a/packages/core/src/modules/import/import.service.test.ts
+++ b/packages/core/src/modules/import/import.service.test.ts
@@ -5,6 +5,7 @@
// (side-effect-free) import.
import { beforeAll, expect, mock, test } from 'bun:test';
+import type { Buffers } from '../../context';
import { createRecordingProducers } from '../../jobs/testing';
import { queues } from '../../jobs.registry';
import type { Logger } from '../../logger';
@@ -163,7 +164,7 @@ test('ImportService.enqueue adds the import job onto the import queue', async ()
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: producers.queues,
});
diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.ts
index a278b7266..cb5793fdf 100644
--- a/packages/core/src/modules/insight/insight.jobs.test.ts
+++ b/packages/core/src/modules/insight/insight.jobs.test.ts
@@ -2,7 +2,7 @@
// methods themselves are exercised in insight.service.test.ts.
import { expect, test } from 'bun:test';
-import type { JobCtx } from '../../context';
+import type { Buffers, JobCtx } from '../../context';
import { createRecordingProducers } from '../../jobs/testing';
import { queues } from '../../jobs.registry';
import type { Logger } from '../../logger';
@@ -57,7 +57,7 @@ function stubJobCtx(
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: createRecordingProducers(queues).queues,
services,
@@ -117,7 +117,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId de
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: producers.queues,
services: {
diff --git a/packages/core/src/modules/insight/insight.service.test.ts b/packages/core/src/modules/insight/insight.service.test.ts
index 198b09cd6..54bc75184 100644
--- a/packages/core/src/modules/insight/insight.service.test.ts
+++ b/packages/core/src/modules/insight/insight.service.test.ts
@@ -5,6 +5,7 @@
// (side-effect-free) import.
import { beforeAll, expect, mock, test } from 'bun:test';
+import type { Buffers } from '../../context';
const listProjectIdsForCadence = mock(async () => ['p1', 'p2']);
const getProjectCreatedAt = mock(async () => null);
@@ -169,7 +170,7 @@ test('createInsightService binds every InsightService method', () => {
ch: undefined,
redis: undefined,
clients: undefined,
- buffers: undefined,
+ buffers: undefined as unknown as Buffers,
logger,
queues:
undefined as unknown as import('../../jobs.registry').QueueProducers,
diff --git a/packages/core/src/modules/notification/notification.jobs.test.ts b/packages/core/src/modules/notification/notification.jobs.test.ts
index c27995020..5c328f3c7 100644
--- a/packages/core/src/modules/notification/notification.jobs.test.ts
+++ b/packages/core/src/modules/notification/notification.jobs.test.ts
@@ -2,7 +2,7 @@
// dispatch logic themselves are exercised in notification.service.test.ts.
import { expect, test } from 'bun:test';
-import type { JobCtx } from '../../context';
+import type { Buffers, JobCtx } from '../../context';
import { createRecordingProducers } from '../../jobs/testing';
import { queues } from '../../jobs.registry';
import type { Logger } from '../../logger';
@@ -46,7 +46,7 @@ function stubJobCtx(notification: Partial<NotificationService>): JobCtx {
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: createRecordingProducers(queues).queues,
services,
diff --git a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts b/packages/core/src/modules/onboarding/onboarding.jobs.test.ts
index 45f9791ab..355dec217 100644
--- a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts
+++ b/packages/core/src/modules/onboarding/onboarding.jobs.test.ts
@@ -2,7 +2,7 @@
// method itself is exercised in onboarding.service.test.ts.
import { expect, test } from 'bun:test';
-import type { JobCtx } from '../../context';
+import type { Buffers, JobCtx } from '../../context';
import { createRecordingProducers } from '../../jobs/testing';
import { queues } from '../../jobs.registry';
import type { Logger } from '../../logger';
@@ -49,7 +49,7 @@ function stubJobCtx(
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: createRecordingProducers(queues).queues,
services,
diff --git a/packages/core/src/modules/organization/organization.jobs.test.ts b/packages/core/src/modules/organization/organization.jobs.test.ts
index afade1295..0b22dfa20 100644
--- a/packages/core/src/modules/organization/organization.jobs.test.ts
+++ b/packages/core/src/modules/organization/organization.jobs.test.ts
@@ -2,7 +2,7 @@
// method itself is exercised in organization.service.test.ts.
import { expect, test } from 'bun:test';
-import type { JobCtx } from '../../context';
+import type { Buffers, JobCtx } from '../../context';
import { createRecordingProducers } from '../../jobs/testing';
import { queues } from '../../jobs.registry';
import type { Logger } from '../../logger';
@@ -52,7 +52,7 @@ function stubJobCtx(
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
logger: stubLogger(),
queues: createRecordingProducers(queues).queues,
services,
diff --git a/packages/core/src/modules/organization/organization.service.test.ts b/packages/core/src/modules/organization/organization.service.test.ts
index 21c2cbd71..744732fa9 100644
--- a/packages/core/src/modules/organization/organization.service.test.ts
+++ b/packages/core/src/modules/organization/organization.service.test.ts
@@ -422,13 +422,6 @@ mock.module('@openpanel/db/src/clickhouse/client', () => ({
ch: { command: chCommand },
}));
-afterAll(() => {
- mock.module(
- '@openpanel/db/src/clickhouse/client',
- () => realClickhouseClient
- );
-});
-
// Bypasses the Redis cache-aside entirely — this module's own logic is
// exercised directly, caching is @openpanel/redis's concern. `access.service.ts`
// (reached via connectUserToOrganization's dynamic import) calls `.clear()` on
@@ -451,7 +444,14 @@ function cacheableStub(
set: () => async () => 'OK' as const,
});
}
+// Spread the real module for the same reason the clickhouse/client mock above
+// does: `mock.module` replaces this specifier process-wide. Surfaced by
+// M8-001, which put `createBuffers` on core's barrel — event-buffer.ts
+// value-imports `publishEvent` from here, so a partial factory turns this
+// file's own barrel import into a SyntaxError.
+const realRedis = { ...(await import('@openpanel/redis')) };
mock.module('@openpanel/redis', () => ({
+ ...realRedis,
cacheable: cacheableStub,
getRedisCache: () => ({
get: async () => null,
@@ -460,6 +460,14 @@ mock.module('@openpanel/redis', () => ({
}),
}));
+afterAll(() => {
+ mock.module(
+ '@openpanel/db/src/clickhouse/client',
+ () => realClickhouseClient
+ );
+ mock.module('@openpanel/redis', () => realRedis);
+});
+
// Mocks core's own clients/email.ts wrapper, not @openpanel/email itself:
// that package statically imports @openpanel/db's full barrel (Resend/SMTP
// setup aside), which is exactly the eager side-effecting chain
diff --git a/packages/core/test/buffer-fixtures.ts b/packages/core/test/buffer-fixtures.ts
new file mode 100644
index 000000000..22df0a879
--- /dev/null
+++ b/packages/core/test/buffer-fixtures.ts
@@ -0,0 +1,25 @@
+// Stubs shared by the buffer tests. Not a test file (no `*.test.ts` suffix),
+// so the runner does not pick it up.
+
+import type { BufferDeps } from '../src/buffers/base-buffer';
+import type { Logger } from '../src/logger';
+
+const silentLogger: Logger = {
+ fatal: () => undefined,
+ error: () => undefined,
+ warn: () => undefined,
+ info: () => undefined,
+ debug: () => undefined,
+ trace: () => undefined,
+ child: () => silentLogger,
+};
+
+/**
+ * A buffer's boot scope, silent and never paused. Silent matters: the real
+ * `createLogger` spawns a pino transport worker thread per buffer, and these
+ * suites build one buffer per test.
+ */
+export const stubBufferDeps: BufferDeps = {
+ createLogger: () => silentLogger,
+ isCronPaused: () => Promise.resolve(false),
+};
diff --git a/packages/core/test/http-fixtures.ts b/packages/core/test/http-fixtures.ts
index 99472948d..b7c8b510b 100644
--- a/packages/core/test/http-fixtures.ts
+++ b/packages/core/test/http-fixtures.ts
@@ -1,7 +1,7 @@
// Stubs shared by the http tests. Not a test file (no `*.test.ts` suffix), so
// the runner does not pick it up.
-import type { AppDeps } from '../src/context';
+import type { AppDeps, Buffers } from '../src/context';
import { createRecordingProducers } from '../src/jobs/testing';
import { queues } from '../src/jobs.registry';
import { type CapturedLogger, capturingLogger } from './rpc-fixtures';
@@ -42,7 +42,7 @@ export function stubAppDeps(): AppDepsStub {
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
producers,
logger,
config: { selfHosted: false },
diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.ts
index d6e662716..477ae8417 100644
--- a/packages/core/test/request-id-end-to-end.test.ts
+++ b/packages/core/test/request-id-end-to-end.test.ts
@@ -11,7 +11,7 @@
import { beforeAll, beforeEach, expect, mock, test } from 'bun:test';
import { Elysia } from 'elysia';
import { z } from 'zod';
-import type { AppDeps } from '../src/context';
+import type { AppDeps, Buffers } from '../src/context';
import { defineJob, defineQueue, type Producers } from '../src/jobs/define';
import { wrap } from '../src/jobs/envelope';
import {
@@ -144,7 +144,7 @@ function stubDeps() {
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
producers: producers as unknown as QueueProducerHandle,
logger: bindingLogger(lines),
config: { selfHosted: false },
diff --git a/packages/core/test/rpc-fixtures.ts b/packages/core/test/rpc-fixtures.ts
index 9ebffdfd6..619092877 100644
--- a/packages/core/test/rpc-fixtures.ts
+++ b/packages/core/test/rpc-fixtures.ts
@@ -3,6 +3,7 @@
import {
type AppDeps,
+ type Buffers,
createCtx,
extendCtx,
type HttpCtx,
@@ -89,7 +90,7 @@ export function stubHttpCtx(
ch: {},
redis: {},
clients: {},
- buffers: {},
+ buffers: {} as Buffers,
producers: createRecordingProducers(queues),
logger,
config: { selfHosted: false },
diff --git a/packages/db/src/buffers/index.ts b/packages/db/src/buffers/index.ts
index 46b5920bc..ac967c71f 100644
--- a/packages/db/src/buffers/index.ts
+++ b/packages/db/src/buffers/index.ts
@@ -1,22 +1,35 @@
-import { BotBuffer as BotBufferRedis } from './bot-buffer';
-import { EventBuffer as EventBufferRedis } from './event-buffer';
-import { GroupBuffer } from './group-buffer';
-import { ProfileBackfillBuffer } from './profile-backfill-buffer';
-import { ProfileBuffer as ProfileBufferRedis } from './profile-buffer';
-import { ReplayBuffer } from './replay-buffer';
-import { SessionBuffer } from './session-buffer';
+// The seven buffers live in @openpanel/core now (M8-001):
+// packages/core/src/buffers/*. They are no longer module singletons there —
+// `createBuffers(deps)` builds them once and V2's main.ts hangs the result on
+// `AppDeps.buffers` (ADR-007). This file is the V1 delegate: one call to that
+// factory, its instances re-exported under the names apps/api, apps/worker,
+// packages/trpc and core's own lazy `import('@openpanel/db/src/buffers')`
+// call sites already use. Both boots therefore share ONE set of buffers and
+// one set of Redis lists.
+import { createBuffers } from '@openpanel/core';
+import { cronQueue } from '@openpanel/queue';
+import { createLogger } from '../logger';
-export const eventBuffer = new EventBufferRedis();
-export const profileBuffer = new ProfileBufferRedis();
-export const botBuffer = new BotBufferRedis();
-export const sessionBuffer = new SessionBuffer();
-export const profileBackfillBuffer = new ProfileBackfillBuffer();
-export const replayBuffer = new ReplayBuffer();
-export const groupBuffer = new GroupBuffer();
+const buffers = createBuffers({
+ // packages/db's own pino, exactly as each buffer built it before the move —
+ // same `LOG_PREFIX-<name>-<NODE_ENV>` service name on every line.
+ createLogger: (name) => createLogger({ name }),
+ // ADR-005's BullMQ escape hatch: pausing `cron` from bull-board halts all
+ // buffer flushing, preserved deliberately (docs/ANSWERS.md §3: "known!").
+ isCronPaused: () => cronQueue.isPaused(),
+});
-export type { ProfileBackfillEntry } from './profile-backfill-buffer';
-export type { IClickhouseSessionReplayChunk } from './replay-buffer';
-export {
- SESSION_TIMEOUT_MS,
- type SessionIngestResult,
-} from './session-buffer';
+export const eventBuffer = buffers.event;
+export const profileBuffer = buffers.profile;
+export const botBuffer = buffers.bot;
+export const sessionBuffer = buffers.session;
+export const profileBackfillBuffer = buffers.profileBackfill;
+export const replayBuffer = buffers.replay;
+export const groupBuffer = buffers.group;
+
+export type {
+ IClickhouseSessionReplayChunk,
+ ProfileBackfillEntry,
+ SessionIngestResult,
+} from '@openpanel/core';
+export { SESSION_TIMEOUT_MS } from '@openpanel/core';
diff --git a/packages/trpc/src/trpc.ts b/packages/trpc/src/trpc.ts
index d5ff18ab6..fb0cbbef6 100644
--- a/packages/trpc/src/trpc.ts
+++ b/packages/trpc/src/trpc.ts
@@ -119,9 +119,10 @@ export async function createContext({
/**
* The `Ctx` half of the context under V1.
*
- * `db`/`ch`/`redis`/`clients`/`buffers` are core's `unknown` stubs until P3-P8
- * wire the real clients, and V1's routers reach for their own singletons
- * regardless. `queues` and `services` throw rather than returning an empty
+ * `db`/`ch`/`redis`/`clients` are core's `unknown` stubs until P3-P4 wire the
+ * real clients, and V1's routers reach for their own singletons regardless —
+ * including the buffers, which V1 reaches through `@openpanel/db`'s delegate
+ * (M8-001) rather than through this context. `queues` and `services` throw rather than returning an empty
* object: a V1 router that reached for either through core would otherwise
* silently do nothing (`queues`) or hit `undefined` (`services`) instead of
* the loud error that tells it to reach `@openpanel/db`/`@openpanel/core`
@@ -136,7 +137,7 @@ function v1CtxScope(req: CreateFastifyContextOptions['req']): Ctx {
ch: undefined,
redis: undefined,
clients: undefined,
- buffers: undefined,
+ buffers: undefined as unknown as Ctx['buffers'],
logger: req.log,
queues: NOT_WIRED_QUEUES,
services: NOT_WIRED_SERVICES,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 855c84aa2..fbc2a05c1 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1163,6 +1163,9 @@ importers:
elysia:
specifier: 1.4.30
+ fast-deep-equal:
+ specifier: ^3.1.3
+ version: 3.1.3
ipaddr.js:
specifier: ^2.5.0
version: 2.5.0