Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M9-002): main.ts by ROLE: startWorkers + startSchedulers + ingest consumer + bull-board + one metrics registry

ralph6fc8851f2dd3
diff --git a/apps/api/e2e/boot-proof.sh b/apps/api/e2e/boot-proof.sh
index 1ef2c44eb..b999ae51c 100755
--- a/apps/api/e2e/boot-proof.sh
+++ b/apps/api/e2e/boot-proof.sh
@@ -1,12 +1,35 @@
#!/usr/bin/env bash
-# M3-CLOSE-002 verification: proves `bun run apps/api/src/main.ts` actually
-# boots, rather than merely typechecking. Three assertions:
+# Proves `bun run apps/api/src/main.ts` actually boots — every ROLE the ROLE
+# table declares — rather than merely typechecking (playbook rule 1).
#
-# 1. ROLE=api serves healthz 200 and /metrics 200 on a configurable port.
-# 2. An unknown ROLE fails boot loudly (non-zero exit, named value in the
-# message) — same doctrine as ENABLED_QUEUES.
+# M3-CLOSE-002 wrote parts 1-3; M9-002 added parts 4-5.
+#
+# 1. An unknown ROLE fails boot loudly (non-zero exit, named value in the
+# message) — and so does an unknown ENABLED_QUEUES token.
+# 2. ROLE=api serves /healthz/live, /healthz/ready and /metrics, runs NO
+# workers/schedulers/consumer, and mounts neither bull-board nor the
+# debug routes.
# 3. SIGTERM shuts the process down within 5s, exit code 0, no orphan left
# behind.
+# 4. ROLE=worker starts all seven BullMQ workers, the cron schedulers and
+# the Kafka ingest consumer; exposes the consuming-role scrape gauges;
+# serves bull-board behind the session guard (401, not 200 and not 404).
+# 5. ROLE=all does everything ROLE=worker does AND sets redis
+# keyspace-notify, which ROLE=worker must not.
+# 6. The two opt-outs: DISABLE_BULLBOARD unmounts the UI, DISABLE_WORKERS
+# leaves the ops surface up with nothing consuming.
+#
+# Parts 4-5 need the real backing services, so they source the repo `.env`
+# and are ISOLATED from whatever else uses this box:
+#
+# * QUEUE_NAMESPACE suffixes every BullMQ key, so the proof cannot consume
+# or prune the live `cron`/`sessions`/... keys. The un-namespaced keys
+# byte-identity is pinned separately by
+# packages/core/src/jobs/naming.test.ts and the controllers queue-keys
+# golden.
+# * KAFKA_CONSUMER_GROUP is a proof-only group, so joining does not
+# rebalance the real `openpanel-events` consumers. The topic is the real
+# one and is only read from `latest`.
#
# Run from anywhere: `bash apps/api/e2e/boot-proof.sh`.
@@ -25,12 +48,28 @@ fail() {
FAILURES=$((FAILURES + 1))
}
+assert_contains() {
+ if printf '%s' "$1" | grep -qF -- "$2"; then
+ pass "$3"
+ else
+ fail "$3 (missing: $2)"
+ fi
+}
+
+assert_absent() {
+ if printf '%s' "$1" | grep -qF -- "$2"; then
+ fail "$3 (unexpectedly present: $2)"
+ else
+ pass "$3"
+ fi
+}
+
# A real free port, not a guessed constant — this box runs other processes.
find_free_port() {
bun -e 'const s = Bun.listen({ port: 0, hostname: "127.0.0.1", socket: { data() {} } }); console.log(s.port); s.stop();'
}
-# --- Part 1: unknown ROLE fails boot loudly ------------------------------
+# --- Part 1: an unknown ROLE / ENABLED_QUEUES fails boot loudly ----------
UNKNOWN_LOG="$(mktemp)"
ROLE=bogus API_PORT=0 bun run "$MAIN_TS" >"$UNKNOWN_LOG" 2>&1
@@ -50,7 +89,20 @@ else
fi
rm -f "$UNKNOWN_LOG"
-# --- Part 2: ROLE=api boots and serves healthz + /metrics ----------------
+QUEUES_LOG="$(mktemp)"
+ROLE=worker ENABLED_QUEUES=events_kafka API_PORT=0 bun run "$MAIN_TS" >"$QUEUES_LOG" 2>&1
+QUEUES_EXIT=$?
+
+if [ "$QUEUES_EXIT" -ne 0 ] && grep -q 'events_kafka' "$QUEUES_LOG" \
+ && grep -q 'was renamed to' "$QUEUES_LOG"; then
+ pass "ENABLED_QUEUES=events_kafka exits non-zero and names the rename"
+else
+ fail "ENABLED_QUEUES=events_kafka should exit non-zero naming the rename (exit $QUEUES_EXIT) — log follows:"
+ cat "$QUEUES_LOG"
+fi
+rm -f "$QUEUES_LOG"
+
+# --- Part 2: ROLE=api boots and serves the ops surface -------------------
PORT="$(find_free_port)"
BOOT_LOG="$(mktemp)"
@@ -87,6 +139,13 @@ else
fail "GET /healthz/live -> $HEALTHZ_CODE, expected 200"
fi
+READY_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/healthz/ready")
+if [ "$READY_CODE" = "200" ]; then
+ pass "GET /healthz/ready -> 200 (ROLE=api: no consumer, so no heartbeat gate)"
+else
+ fail "GET /healthz/ready -> $READY_CODE, expected 200"
+fi
+
METRICS_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/metrics")
if [ "$METRICS_CODE" = "200" ]; then
pass "GET /metrics -> 200"
@@ -94,6 +153,44 @@ else
fail "GET /metrics -> $METRICS_CODE, expected 200"
fi
+API_METRICS="$(curl -s "http://127.0.0.1:$PORT/metrics")"
+assert_contains "$API_METRICS" 'process_cpu_seconds_total' \
+ "ROLE=api /metrics carries the default process metrics"
+assert_contains "$API_METRICS" 'http_request_duration_seconds' \
+ "ROLE=api /metrics carries the route histogram"
+assert_contains "$API_METRICS" 'http_request_summary_seconds' \
+ "ROLE=api /metrics carries the route summary"
+# One registry: the queue/buffer/session scrape collectors register ONLY where
+# the role consumes (TARGET_ARCHITECTURE 18).
+assert_absent "$API_METRICS" 'sessions_waiting_count' \
+ "ROLE=api does NOT register the queue gauges"
+assert_absent "$API_METRICS" 'sessions_active_total' \
+ "ROLE=api does NOT register the session scrape gauges"
+assert_absent "$API_METRICS" 'buffer_event_count' \
+ "ROLE=api does NOT register the buffer gauges"
+
+BOARD_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/bullboard")
+if [ "$BOARD_CODE" = "404" ]; then
+ pass "ROLE=api does NOT mount bull-board (GET /bullboard -> 404)"
+else
+ fail "ROLE=api should not mount bull-board, got $BOARD_CODE"
+fi
+
+if grep -q '"msg":"worker started"' "$BOOT_LOG"; then
+ fail "ROLE=api must not start workers — log follows:"
+ cat "$BOOT_LOG"
+else
+ pass "ROLE=api starts no workers, schedulers or ingest consumer"
+fi
+
+# The debug routes lived on the worker in V1; ROLE=api never had them.
+DEBUG_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/debug/cron")
+if [ "$DEBUG_CODE" = "404" ]; then
+ pass "ROLE=api does NOT mount the debug routes (GET /debug/cron -> 404)"
+else
+ fail "ROLE=api should not mount the debug routes, got $DEBUG_CODE"
+fi
+
# --- Part 3: SIGTERM shuts down within 5s, exit 0, no orphan -------------
kill -TERM "$BOOT_PID"
@@ -135,6 +232,289 @@ fi
rm -f "$BOOT_LOG"
+# --- Parts 4-5: the consuming roles --------------------------------------
+#
+# These need the real Redis/Kafka/Postgres, so they source `.env`. Isolated
+# by QUEUE_NAMESPACE + a proof-only Kafka consumer group (see the header).
+
+if [ ! -f "$REPO_ROOT/.env" ]; then
+ fail "$REPO_ROOT/.env is missing — parts 4-5 need REDIS_URL/KAFKA_BROKERS/DATABASE_URL"
+ echo
+ echo "boot-proof.sh: $FAILURES failure(s)"
+ exit 1
+fi
+
+PROOF_NAMESPACE="bootproof$$"
+PROOF_KAFKA_GROUP="openpanel-events-bootproof-$$"
+CONSUMING_READY_TIMEOUT_S=90
+
+# Boots one consuming role, waits for it to listen, and leaves $ROLE_PID and
+# $ROLE_LOG set for the caller's assertions.
+boot_consuming_role() { boot_consuming_role_with_env "$1"; }
+
+# `boot_consuming_role_with_env <role> [NAME=VALUE ...]`
+boot_consuming_role_with_env() {
+ local role="$1"
+ shift
+ ROLE_PORT="$(find_free_port)"
+ ROLE_LOG="$(mktemp)"
+
+ (
+ set -a
+ # shellcheck disable=SC1091
+ . "$REPO_ROOT/.env"
+ set +a
+ export ROLE="$role"
+ export API_PORT="$ROLE_PORT"
+ export QUEUE_NAMESPACE="$PROOF_NAMESPACE"
+ export KAFKA_CONSUMER_GROUP="$PROOF_KAFKA_GROUP"
+ for assignment in "$@"; do
+ export "${assignment?}"
+ done
+ exec bun run "$MAIN_TS"
+ ) >"$ROLE_LOG" 2>&1 &
+ ROLE_PID=$!
+
+ local waited=0
+ until curl -s -o /dev/null "http://127.0.0.1:$ROLE_PORT/healthz/live"; do
+ if ! kill -0 "$ROLE_PID" 2>/dev/null; then
+ fail "ROLE=$role exited before listening — log follows:"
+ cat "$ROLE_LOG"
+ return 1
+ fi
+ sleep 1
+ waited=$((waited + 1))
+ if [ "$waited" -ge "$CONSUMING_READY_TIMEOUT_S" ]; then
+ fail "ROLE=$role did not listen within ${CONSUMING_READY_TIMEOUT_S}s — log follows:"
+ cat "$ROLE_LOG"
+ kill -9 "$ROLE_PID" 2>/dev/null
+ return 1
+ fi
+ done
+ pass "ROLE=$role booted and is listening on :$ROLE_PORT (${waited}s)"
+ return 0
+}
+
+stop_consuming_role() {
+ local role="$1"
+ kill -TERM "$ROLE_PID" 2>/dev/null
+ local waited=0
+ while kill -0 "$ROLE_PID" 2>/dev/null; do
+ sleep 1
+ waited=$((waited + 1))
+ if [ "$waited" -ge 30 ]; then
+ break
+ fi
+ done
+ if kill -0 "$ROLE_PID" 2>/dev/null; then
+ fail "ROLE=$role still alive 30s after SIGTERM — killing it"
+ kill -9 "$ROLE_PID" 2>/dev/null
+ wait "$ROLE_PID" 2>/dev/null
+ else
+ wait "$ROLE_PID" 2>/dev/null
+ local exit_code=$?
+ if [ "$exit_code" -eq 0 ]; then
+ pass "ROLE=$role SIGTERM shutdown exit code 0 (${waited}s)"
+ else
+ fail "ROLE=$role SIGTERM shutdown exit code $exit_code, expected 0"
+ fi
+ fi
+}
+
+# The seven registry queues (jobs.registry.ts) — the same names as the Redis
+# keys, `cohortCompute` included.
+REGISTRY_QUEUES="sessions cron notification import insights gsc cohortCompute"
+
+assert_consuming_role() {
+ local role="$1" log="$2" port="$3"
+
+ local started
+ started=$(grep -c '"msg":"worker started"' "$log")
+ if [ "$started" -eq 7 ]; then
+ pass "ROLE=$role started all 7 BullMQ workers"
+ else
+ fail "ROLE=$role started $started workers, expected 7"
+ fi
+
+ for queue in $REGISTRY_QUEUES; do
+ if grep -q "\"key\":\"${queue}-${PROOF_NAMESPACE}\"" "$log"; then
+ pass "ROLE=$role worker on queue key ${queue}-${PROOF_NAMESPACE}"
+ else
+ fail "ROLE=$role has no worker for queue $queue"
+ fi
+ done
+
+ if grep -q '"msg":"updating cron jobs"' "$log"; then
+ pass "ROLE=$role upserted the cron schedulers"
+ else
+ fail "ROLE=$role did not run startSchedulers"
+ fi
+
+ if grep -q '"msg":"kafka events consumer running"' "$log"; then
+ pass "ROLE=$role started the Kafka ingest consumer"
+ else
+ fail "ROLE=$role did not start the Kafka ingest consumer"
+ fi
+
+ # M9-003 lands dataHealth/windDown/flushExports. Until then main.ts must SAY
+ # they are missing at boot rather than let each tick fail silently.
+ if grep -q 'cron schedulers have no handler' "$log"; then
+ pass "ROLE=$role names the three scheduler ids with no handler yet (M9-003)"
+ else
+ fail "ROLE=$role did not report the unhandled schedulers"
+ fi
+
+ local ready_code
+ ready_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/healthz/ready")
+ if [ "$ready_code" = "200" ]; then
+ pass "ROLE=$role GET /healthz/ready -> 200 (consumer heartbeat fresh)"
+ else
+ fail "ROLE=$role GET /healthz/ready -> $ready_code, expected 200"
+ fi
+
+ local metrics
+ metrics="$(curl -s "http://127.0.0.1:$port/metrics")"
+ assert_contains "$metrics" 'process_cpu_seconds_total' \
+ "ROLE=$role /metrics carries the default process metrics"
+ assert_contains "$metrics" 'http_request_duration_seconds' \
+ "ROLE=$role /metrics carries the route histogram"
+ assert_contains "$metrics" 'job_duration_ms' \
+ "ROLE=$role /metrics carries job_duration_ms"
+ assert_contains "$metrics" 'kafka_events_reprocessed_total' \
+ "ROLE=$role /metrics carries the kafka counters"
+ assert_contains "$metrics" 'sessions_active_total' \
+ "ROLE=$role /metrics carries the session scrape gauges"
+ assert_contains "$metrics" 'buffer_event_count' \
+ "ROLE=$role /metrics carries the buffer gauges"
+ for queue in $REGISTRY_QUEUES; do
+ assert_contains "$metrics" "${queue}_${PROOF_NAMESPACE}_waiting_count" \
+ "ROLE=$role /metrics carries the $queue queue gauges"
+ done
+
+ local board_code
+ board_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/bullboard")
+ if [ "$board_code" = "401" ]; then
+ pass "ROLE=$role mounts bull-board BEHIND the session guard (401, not 200)"
+ else
+ fail "ROLE=$role GET /bullboard -> $board_code, expected 401"
+ fi
+
+ local debug_code
+ debug_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/debug/cron")
+ if [ "$debug_code" = "200" ]; then
+ pass "ROLE=$role serves the dev-only debug routes (NODE_ENV is not production)"
+ else
+ fail "ROLE=$role GET /debug/cron -> $debug_code, expected 200"
+ fi
+}
+
+echo
+echo "--- Part 4: ROLE=worker ---"
+if boot_consuming_role worker; then
+ WORKER_LOG="$ROLE_LOG"
+ assert_consuming_role worker "$WORKER_LOG" "$ROLE_PORT"
+
+ if grep -q 'keyspace notifications configured' "$WORKER_LOG"; then
+ fail "ROLE=worker must NOT set redis keyspace notifications"
+ else
+ pass "ROLE=worker does NOT set redis keyspace notifications"
+ fi
+
+ stop_consuming_role worker
+ rm -f "$WORKER_LOG"
+fi
+
+echo
+echo "--- Part 5: ROLE=all ---"
+if boot_consuming_role all; then
+ ALL_LOG="$ROLE_LOG"
+ assert_consuming_role all "$ALL_LOG" "$ROLE_PORT"
+
+ if grep -q 'keyspace notifications configured' "$ALL_LOG"; then
+ pass "ROLE=all sets redis keyspace notifications (ROLE != worker)"
+ else
+ fail "ROLE=all did not set redis keyspace notifications"
+ fi
+
+ stop_consuming_role all
+ rm -f "$ALL_LOG"
+fi
+
+echo
+echo "--- Part 6: the DISABLE_* opt-outs ---"
+if boot_consuming_role_with_env worker DISABLE_BULLBOARD=1 DISABLE_WORKERS=1; then
+ OPTOUT_LOG="$ROLE_LOG"
+
+ board_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$ROLE_PORT/bullboard")
+ if [ "$board_code" = "404" ]; then
+ pass "DISABLE_BULLBOARD=1 unmounts bull-board (GET /bullboard -> 404)"
+ else
+ fail "DISABLE_BULLBOARD=1 should unmount bull-board, got $board_code"
+ fi
+
+ if grep -q '"msg":"worker started"' "$OPTOUT_LOG"; then
+ fail "DISABLE_WORKERS should start no workers"
+ else
+ pass "DISABLE_WORKERS starts no workers, schedulers or ingest consumer"
+ fi
+
+ if grep -q '"msg":"Workers are disabled"' "$OPTOUT_LOG"; then
+ pass "DISABLE_WORKERS says so at boot"
+ else
+ fail "DISABLE_WORKERS did not log why nothing is consuming"
+ fi
+
+ live_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$ROLE_PORT/healthz/live")
+ if [ "$live_code" = "200" ]; then
+ pass "DISABLE_WORKERS still serves the ops surface"
+ else
+ fail "DISABLE_WORKERS ops surface -> $live_code, expected 200"
+ fi
+
+ stop_consuming_role worker
+ rm -f "$OPTOUT_LOG"
+fi
+
+# The namespaced BullMQ keys and their job schedulers are this proof's litter;
+# an `every` scheduler left in Redis would keep firing forever.
+echo
+(
+ # From apps/api, whose node_modules resolves @openpanel/redis.
+ cd "$REPO_ROOT/apps/api" || exit 1
+ set -a
+ # shellcheck disable=SC1091
+ . "$REPO_ROOT/.env"
+ set +a
+ PROOF_NAMESPACE="$PROOF_NAMESPACE" bun -e '
+ const { Redis } = await import("@openpanel/redis");
+ const redis = new Redis(process.env.REDIS_URL ?? "redis://127.0.0.1:6379");
+ const pattern = `bull:*-${process.env.PROOF_NAMESPACE}:*`;
+ let cursor = "0";
+ let removed = 0;
+ do {
+ const [next, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000);
+ cursor = next;
+ if (keys.length > 0) {
+ removed += await redis.del(...keys);
+ }
+ } while (cursor !== "0");
+ console.log(`cleaned ${removed} proof queue keys`);
+ await redis.quit();
+ '
+) && pass "cleaned up the namespaced proof queue keys" \
+ || fail "could not clean up the namespaced proof queue keys"
+
+# Nothing this script started may outlive it.
+sleep 0.3
+LEFTOVER="$(pgrep -f "$MAIN_TS" || true)"
+if [ -z "$LEFTOVER" ]; then
+ pass "no boot-proof process left behind"
+else
+ fail "process(es) still running: $LEFTOVER"
+ # shellcheck disable=SC2086
+ kill -9 $LEFTOVER 2>/dev/null
+fi
+
echo
if [ "$FAILURES" -eq 0 ]; then
echo "boot-proof.sh: all checks passed"
diff --git a/apps/api/src/config/env.test.ts b/apps/api/src/config/env.test.ts
new file mode 100644
index 000000000..cbdab607b
--- /dev/null
+++ b/apps/api/src/config/env.test.ts
@@ -0,0 +1,128 @@
+import { describe, expect, it } from 'vitest';
+import {
+ concurrencyEnvKey,
+ concurrencyOverride,
+ ENABLED_QUEUE_VALUES,
+ loadConfig,
+} from './env';
+
+// `loadConfig` takes its source, so nothing here touches process.env.
+const base = { ROLE: 'worker' } as NodeJS.ProcessEnv;
+
+describe('ROLE', () => {
+ it('defaults to api and accepts the three roles', () => {
+ expect(loadConfig({}).ROLE).toBe('api');
+ for (const role of ['api', 'worker', 'all']) {
+ expect(loadConfig({ ROLE: role }).ROLE).toBe(role);
+ }
+ });
+
+ it('fails boot loudly on an unknown value, naming it', () => {
+ expect(() => loadConfig({ ROLE: 'wroker' })).toThrow(
+ /unknown value "wroker"/
+ );
+ });
+});
+
+describe('ENABLED_QUEUES', () => {
+ it('is every known consumer when unset', () => {
+ expect(loadConfig(base).ENABLED_QUEUES).toEqual([...ENABLED_QUEUE_VALUES]);
+ });
+
+ it('covers the Kafka token plus the seven registry queues', () => {
+ expect(ENABLED_QUEUE_VALUES).toEqual([
+ 'events',
+ 'sessions',
+ 'cron',
+ 'notification',
+ 'import',
+ 'insights',
+ 'gsc',
+ 'cohortCompute',
+ ]);
+ });
+
+ it('parses the cloud split (docs/ANSWERS.md 1.3)', () => {
+ expect(
+ loadConfig({ ...base, ENABLED_QUEUES: 'events' }).ENABLED_QUEUES
+ ).toEqual(['events']);
+ expect(
+ loadConfig({ ...base, ENABLED_QUEUES: ' sessions , cron ' })
+ .ENABLED_QUEUES
+ ).toEqual(['sessions', 'cron']);
+ });
+
+ it('rejects an unknown token instead of silently idling (V1 ignored it)', () => {
+ expect(() =>
+ loadConfig({ ...base, ENABLED_QUEUES: 'sessions,nope' })
+ ).toThrow(/unknown queue "nope"/);
+ });
+
+ it('names the rename when it sees the old events_kafka spelling', () => {
+ expect(() =>
+ loadConfig({ ...base, ENABLED_QUEUES: 'events_kafka' })
+ ).toThrow(/"events_kafka" was renamed to "events"/);
+ });
+});
+
+describe('<QUEUE>_CONCURRENCY', () => {
+ it('keeps V1 key derivation — cohortCompute reads COHORTCOMPUTE_CONCURRENCY', () => {
+ expect(concurrencyEnvKey('cohortCompute')).toBe(
+ 'COHORTCOMPUTE_CONCURRENCY'
+ );
+ const config = loadConfig({ ...base, COHORTCOMPUTE_CONCURRENCY: '9' });
+ expect(concurrencyOverride(config, 'cohortCompute')).toBe(9);
+ });
+
+ it('is undefined when unset, so the registry default stands', () => {
+ expect(concurrencyOverride(loadConfig(base), 'sessions')).toBeUndefined();
+ });
+
+ it('ignores a non-numeric or non-positive value, as V1 did', () => {
+ expect(
+ concurrencyOverride(
+ loadConfig({ ...base, SESSIONS_CONCURRENCY: 'lots' }),
+ 'sessions'
+ )
+ ).toBeUndefined();
+ expect(
+ concurrencyOverride(
+ loadConfig({ ...base, SESSIONS_CONCURRENCY: '0' }),
+ 'sessions'
+ )
+ ).toBeUndefined();
+ });
+});
+
+describe('the boot flags main.ts branches on', () => {
+ it('DISABLE_WORKERS is any defined value; DISABLE_BULLBOARD is 1 or true', () => {
+ expect(loadConfig(base).DISABLE_WORKERS).toBe(false);
+ expect(loadConfig({ ...base, DISABLE_WORKERS: '0' }).DISABLE_WORKERS).toBe(
+ true
+ );
+
+ expect(loadConfig(base).DISABLE_BULLBOARD).toBe(false);
+ expect(
+ loadConfig({ ...base, DISABLE_BULLBOARD: '1' }).DISABLE_BULLBOARD
+ ).toBe(true);
+ expect(
+ loadConfig({ ...base, DISABLE_BULLBOARD: 'true' }).DISABLE_BULLBOARD
+ ).toBe(true);
+ // V1 only ever accepted those two spellings.
+ expect(
+ loadConfig({ ...base, DISABLE_BULLBOARD: 'yes' }).DISABLE_BULLBOARD
+ ).toBe(false);
+ });
+
+ it('QUEUE_NAMESPACE and QUEUE_CLUSTER default to off, so queue keys are V1s', () => {
+ const config = loadConfig(base);
+ expect(config.QUEUE_CLUSTER).toBe(false);
+ expect(config.QUEUE_NAMESPACE).toBeUndefined();
+ });
+
+ it('reports every problem at once, never just the first', () => {
+ expect(() =>
+ loadConfig({ ROLE: 'nope', ENABLED_QUEUES: 'alsonope' })
+ ).toThrow(/unknown value "nope"[\s\S]*unknown queue "alsonope"/);
+ });
+});
diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.ts
index 744ffbe6e..34e95c560 100644
--- a/apps/api/src/config/env.ts
+++ b/apps/api/src/config/env.ts
@@ -2,12 +2,15 @@
// per TARGET_ARCHITECTURE §7 "apps/api — three files". `packages/db` and
// `packages/redis` keep reading their own env until they grow factories —
// an accepted pragmatic deviation (ADR-007 §7) — so this catalogue covers
-// only what `main.ts` itself reads today: TZ, ROLE, LOG_LEVEL, API_PORT,
-// SELF_HOSTED and QUEUE_CLUSTER.
+// only what `main.ts` itself reads today: TZ, ROLE, NODE_ENV, LOG_LEVEL,
+// API_PORT, API_HOST, SELF_HOSTED, QUEUE_CLUSTER, QUEUE_NAMESPACE,
+// ENABLED_QUEUES, the seven `<QUEUE>_CONCURRENCY` overrides, DISABLE_WORKERS,
+// DISABLE_BULLBOARD and SHUTDOWN_FORCE_EXIT_MS.
//
// Invalid config fails boot loudly, with every issue reported at once —
// never a fail-fast on the first bad var.
+import { queues } from '@openpanel/core';
import { z } from 'zod';
export const ROLE_VALUES = ['api', 'worker', 'all'] as const;
@@ -16,6 +19,38 @@ export type Role = (typeof ROLE_VALUES)[number];
const DEFAULT_API_PORT = 3000;
const DEFAULT_LOG_LEVEL = 'info';
const TRUE_STRING = 'true';
+const PRODUCTION = 'production';
+/** V1's worker deadline (apps/worker/src/boot-workers.ts). */
+const DEFAULT_SHUTDOWN_FORCE_EXIT_MS = 20_000;
+
+/**
+ * The Kafka events consumer's token. Renamed from `events_kafka` — there is
+ * one events transport now, so there is one token for it (ADR-004 rec 5/6,
+ * docs/ANSWERS.md §1.3). Carl updates the cloud env at cutover, which is why
+ * the old spelling gets its own message below rather than being aliased.
+ */
+export const KAFKA_QUEUE_TOKEN = 'events';
+
+/** `events` plus the seven registry keys, read from the registry itself so
+ * the accepted set cannot drift from what actually exists. */
+export const ENABLED_QUEUE_VALUES = [
+ KAFKA_QUEUE_TOKEN,
+ ...Object.keys(queues),
+] as const;
+
+export type EnabledQueue = (typeof ENABLED_QUEUE_VALUES)[number];
+
+const RENAMED_QUEUE_TOKENS: Record<string, string> = {
+ events_kafka: KAFKA_QUEUE_TOKEN,
+};
+
+/** V1's `getConcurrencyFor` key derivation, verbatim — `cohortCompute` keeps
+ * reading `COHORTCOMPUTE_CONCURRENCY` (ADR-005 acceptance note: no rename). */
+const NON_ENV_KEY_CHARS = /[^A-Z0-9]/g;
+
+export function concurrencyEnvKey(queueName: string): string {
+ return `${queueName.toUpperCase().replace(NON_ENV_KEY_CHARS, '_')}_CONCURRENCY`;
+}
// An unset or blank var should fall through to the default, matching the
// `value?.trim() || fallback` idiom the ported reads used before this file
@@ -52,8 +87,83 @@ const roleSchema = z.preprocess(
.transform((value) => value as Role)
);
+/**
+ * Same doctrine as ROLE: an unknown token fails boot loudly rather than
+ * leaving a worker silently idle (V1 ignored it — boot-workers.ts:78-87 —
+ * which docs/ANSWERS.md §1.3 rules must change). Unset means all of them.
+ */
+const enabledQueuesSchema = z.preprocess(
+ blankToUndefined,
+ z
+ .string()
+ .optional()
+ .superRefine((value, ctx) => {
+ if (value === undefined) {
+ return;
+ }
+ for (const token of splitTokens(value)) {
+ if ((ENABLED_QUEUE_VALUES as readonly string[]).includes(token)) {
+ continue;
+ }
+ const renamedTo = RENAMED_QUEUE_TOKENS[token];
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message: renamedTo
+ ? `ENABLED_QUEUES: "${token}" was renamed to "${renamedTo}" — update ENABLED_QUEUES to use the new name.`
+ : `ENABLED_QUEUES: unknown queue "${token}". Accepted values: ${ENABLED_QUEUE_VALUES.join(', ')}.`,
+ });
+ }
+ })
+ .transform((value) =>
+ value === undefined
+ ? [...ENABLED_QUEUE_VALUES]
+ : (splitTokens(value) as EnabledQueue[])
+ )
+);
+
+function splitTokens(value: string): string[] {
+ return value
+ .split(',')
+ .map((token) => token.trim())
+ .filter(Boolean);
+}
+
+/** V1 treated any defined value as "off" for workers and only `'1'`/`'true'`
+ * as "off" for bull-board. Both spellings port as-is. */
+const definedIsTrueSchema = z.preprocess(
+ blankToUndefined,
+ z
+ .string()
+ .optional()
+ .transform((value) => value !== undefined)
+);
+
+const oneOrTrueSchema = z.preprocess(
+ blankToUndefined,
+ z
+ .string()
+ .optional()
+ .transform((value) => value === '1' || value === TRUE_STRING)
+);
+
+/** One `<QUEUE>_CONCURRENCY` slot per registry queue. V1 ignored a
+ * non-numeric or non-positive value and kept the default; so does this. */
+const concurrencySchema = z.preprocess(
+ blankToUndefined,
+ z.coerce.number().int().positive().optional().catch(undefined)
+);
+
+const concurrencyShape = Object.fromEntries(
+ Object.keys(queues).map((name) => [
+ concurrencyEnvKey(name),
+ concurrencySchema,
+ ])
+) as Record<string, typeof concurrencySchema>;
+
const envSchema = z.object({
+ ...concurrencyShape,
ROLE: roleSchema,
+ NODE_ENV: z.preprocess(blankToUndefined, z.string().optional()),
LOG_LEVEL: z.preprocess(
blankToUndefined,
z.string().default(DEFAULT_LOG_LEVEL)
@@ -79,9 +189,55 @@ const envSchema = z.object({
.optional()
.transform((value) => value !== undefined)
),
+ /**
+ * Isolates two deployments — or a proof run and the dev box — sharing one
+ * Redis (`queueKey`'s `namespace`). UNSET in every real deployment: setting
+ * it renames every queue key, which orphans the jobs already in Redis.
+ */
+ QUEUE_NAMESPACE: z.preprocess(blankToUndefined, z.string().optional()),
+ ENABLED_QUEUES: enabledQueuesSchema,
+ DISABLE_WORKERS: definedIsTrueSchema,
+ DISABLE_BULLBOARD: oneOrTrueSchema,
+ /**
+ * Left UNSET means Bun's own default, `0.0.0.0`.
+ *
+ * Deliberate deviation from V1, which defaulted to `localhost` outside
+ * production (apps/api/src/index.ts): `Bun.serve({hostname:'localhost'})`
+ * binds `::1` ONLY, so a dev box's `127.0.0.1` request is refused. Making
+ * the loopback-only bind explicit rather than the accidental default is
+ * what keeps `bun run src/main.ts` reachable the way `node dist/index.js`
+ * was.
+ */
+ API_HOST: z.preprocess(blankToUndefined, z.string().optional()),
+ SHUTDOWN_FORCE_EXIT_MS: z.coerce
+ .number()
+ .int()
+ .positive()
+ .default(DEFAULT_SHUTDOWN_FORCE_EXIT_MS),
});
-export type Config = z.infer<typeof envSchema>;
+export type Config = z.infer<typeof envSchema> & {
+ [key: `${string}_CONCURRENCY`]: number | undefined;
+};
+
+/** V1's default host rule (apps/api/src/index.ts). */
+export function apiHost(config: Config): string {
+ return config.API_HOST ?? (isProduction(config) ? '0.0.0.0' : 'localhost');
+}
+
+export function isProduction(config: Config): boolean {
+ return config.NODE_ENV === PRODUCTION;
+}
+
+/** The `<QUEUE>_CONCURRENCY` override for one registry queue, if any. */
+export function concurrencyOverride(
+ config: Config,
+ queueName: string
+): number | undefined {
+ return (config as Record<string, unknown>)[concurrencyEnvKey(queueName)] as
+ | number
+ | undefined;
+}
function formatIssues(issues: z.ZodIssue[]): string {
return issues
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index f2a20c96d..236965740 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -1,43 +1,99 @@
// V2 boot entrypoint (ADR-007 §7 "apps/api — three files"; TARGET_ARCHITECTURE
// §7 main.ts boot order). Runs on Bun; the V1 Fastify boot at `./index.ts`
-// keeps serving the golden harness untouched until `apps/worker` is deleted
-// at P9.
+// keeps serving the golden harness untouched until it is deleted at M9-004.
//
-// This is the M3 gate boot only: db/ch/clients are not wired into `AppDeps`
-// yet — their types stay the `unknown` stubs `context.ts` declares until
-// P3/P4 land the real clients — so only the ops surface (healthz + metrics)
-// is mounted here. `dashboardRoutes`/`publicApiRoutes` land with their first
-// module (see `rest.routes.ts`). The buffers ARE real from M8-001:
-// `createBuffers` builds them once, here, and nowhere else. The producers are
-// real from M9-001: all seven queues, envelope on, byte-identical Redis keys.
-// Workers, schedulers and the ingest consumer are M9-002.
+// THE ROLE TABLE (TARGET_ARCHITECTURE §7). One process, three shapes:
+//
+// | | api | worker | all |
+// | ops HTTP + /metrics | x | x | x |
+// | producers | x | x | x |
+// | BullMQ workers | | x | x |
+// | schedulers (cron) | | x | x |
+// | Kafka ingest | | x | x |
+// | initial salts | | x | x |
+// | bull-board | | x | x |
+// | queue/buffer/session scrape gauges | | x | x |
+// | redis keyspace-notify | x | | x |
+//
+// An unknown ROLE fails boot loudly, naming the value — `config/env.ts`.
+// Inside a consuming role, ENABLED_QUEUES narrows *which* of the eight
+// consumers start (cloud runs 4 replicas on `events` and 6 on the rest,
+// docs/ANSWERS.md §1.3), and an unknown token there fails boot the same way.
+//
+// db/ch/clients are not on `AppDeps` yet — their types stay the `unknown`
+// stubs `context.ts` declares until P3/P4 land the real clients — so the
+// mounted HTTP surface is still ops-only; `dashboardRoutes`/`publicApiRoutes`
+// land at M9-004. The buffers are real from M8-001, the producers from
+// M9-001, and the workers/schedulers/consumer from here.
process.env.TZ = 'UTC';
import {
type AppDeps,
+ BULL_BOARD_BASE_PATH,
type BufferDeps,
+ bullBoardRoutes,
createBuffers,
+ createInitialSalts,
createProducers,
+ debugRoutes,
+ enableEventsHeartbeat,
+ incomingEvent,
+ ingestConsumerMetrics,
+ type KafkaConsumerHandle,
+ loadIncomingEventDeps,
+ markEventsActivity,
opsRoutes,
+ type QueueDefinition,
type QueueProducerHandle,
queueKey,
queues,
+ rawStderrWrite,
registerBufferMetrics,
+ registerDefaultMetrics,
+ registerQueueMetrics,
+ registerSessionScrapeMetrics,
+ type SessionMetricsRedis,
+ sessionEndEnqueueOptions,
+ sessionEndJobPayload,
+ setShuttingDown,
+ startKafkaEventsConsumer,
+ startSchedulers,
+ startWorkers,
+ type WorkerHandle,
} from '@openpanel/core';
-import { produceIncomingEvent } from '@openpanel/queue';
-import { getRedisQueue } from '@openpanel/redis';
+import {
+ assertKafkaConfigured,
+ createKafkaEventsConsumer,
+ KAFKA_EVENTS_TOPIC,
+ KAFKA_HANDLER_MAX_ATTEMPTS,
+ KAFKA_HANDLER_RETRY_INITIAL_MS,
+ KAFKA_HANDLER_RETRY_MAX_MS,
+ KAFKA_PARTITIONS_CONCURRENT,
+ kafkaLogger,
+ produceDeadLetterEvent,
+ produceIncomingEvent,
+} from '@openpanel/queue';
+import { getRedisCache, getRedisPub, getRedisQueue } from '@openpanel/redis';
import pino from 'pino';
-import { type Config, loadConfig } from './config/env';
+import {
+ type Config,
+ concurrencyOverride,
+ isProduction,
+ KAFKA_QUEUE_TOKEN,
+ loadConfig,
+} from './config/env';
-const SHUTDOWN_FORCE_EXIT_MS = 5000;
// The registry key and the Redis name are the same string for all seven queues
-// (jobs.registry.ts); `queueKey` only braces it under `QUEUE_CLUSTER`, which no
-// deployment sets.
+// (jobs.registry.ts); `queueKey` only braces it under `QUEUE_CLUSTER`.
const CRON_QUEUE_NAME = 'cron';
// Used only to report a config error itself — the real level isn't known
// until `config/env.ts` has validated `LOG_LEVEL`.
const BOOTSTRAP_LOG_LEVEL = 'info';
+// V1's cron drain bound (apps/worker/src/boot-workers.ts `waitForQueueToEmpty`).
+const CRON_DRAIN_TIMEOUT_MS = 60_000;
+const CRON_DRAIN_POLL_MS = 500;
+const FATAL_EXIT_DELAY_MS = 1000;
/**
* `config/env.ts` is the sole `process.env` reader on this path; an invalid
@@ -59,6 +115,10 @@ function loadConfigOrExit(): Config {
const config = loadConfigOrExit();
const logger = pino({ name: 'api', level: config.LOG_LEVEL });
+/** ROLE=api produces and serves; it never consumes. */
+const roleConsumes = config.ROLE !== 'api';
+const workersEnabled = roleConsumes && !config.DISABLE_WORKERS;
+
/**
* All seven queues, envelope on, on the Redis keys V1 already uses — so a job
* V1 enqueued before the cutover is read back by V2's `resolveJob` through
@@ -74,6 +134,7 @@ function buildProducerHandle(): QueueProducerHandle {
return createProducers(queues, {
connection: getRedisQueue(),
cluster: config.QUEUE_CLUSTER,
+ namespace: config.QUEUE_NAMESPACE,
logger,
});
}
@@ -85,18 +146,23 @@ function buildProducerHandle(): QueueProducerHandle {
* deliberately (docs/ANSWERS.md §3: "known!").
*/
function bufferDeps(producers: QueueProducerHandle): BufferDeps {
- // `bullQueues` are keyed by the Redis name, which `QUEUE_CLUSTER` braces.
- const cronKey = queueKey(CRON_QUEUE_NAME, { cluster: config.QUEUE_CLUSTER });
+ const cron = findBullQueue(producers, CRON_QUEUE_NAME);
return {
createLogger: (name) => logger.child({ name }),
- isCronPaused: async () => {
- const cron = producers.bullQueues.find((queue) => queue.name === cronKey);
- return cron ? await cron.isPaused() : false;
- },
+ isCronPaused: async () => (cron ? await cron.isPaused() : false),
};
}
+/** `bullQueues` are keyed by the Redis name, which cluster/namespace decide. */
+function findBullQueue(producers: QueueProducerHandle, name: string) {
+ const key = queueKey(name, {
+ cluster: config.QUEUE_CLUSTER,
+ namespace: config.QUEUE_NAMESPACE,
+ });
+ return producers.bullQueues.find((queue) => queue.name === key);
+}
+
function buildDeps(): AppDeps {
const producers = buildProducerHandle();
return {
@@ -110,39 +176,237 @@ function buildDeps(): AppDeps {
buffers: createBuffers(bufferDeps(producers)),
producers,
// @openpanel/queue's Kafka producer, injected because core cannot import
- // it back (M8-002). M8-003 replaces it with core's own.
+ // it back (M8-002). It dies with that package at M9-003.
produceIncomingEvent,
logger,
config: { selfHosted: config.SELF_HOSTED },
};
}
+/**
+ * The queues this process consumes, `<QUEUE>_CONCURRENCY` applied.
+ *
+ * ENABLED_QUEUES is the same allowlist V1 read, and its `events` token selects
+ * the Kafka consumer rather than a BullMQ queue — hence it is filtered out
+ * here and asked about separately.
+ */
+function consumedQueues(): Record<string, QueueDefinition> {
+ const enabled = new Set(config.ENABLED_QUEUES);
+ const selected: Record<string, QueueDefinition> = {};
+
+ for (const [name, definition] of Object.entries(queues)) {
+ if (!enabled.has(name)) {
+ continue;
+ }
+ const concurrency = concurrencyOverride(config, name);
+ selected[name] =
+ concurrency === undefined
+ ? definition
+ : { ...definition, worker: { ...definition.worker, concurrency } };
+ }
+
+ return selected;
+}
+
+/**
+ * Every declared scheduler must have a handler on the `cron` queue, or its
+ * tick enqueues a job the worker can only throw on. Reported at boot, once,
+ * rather than discovered N minutes later in a failed-job list.
+ */
+function warnOnUnhandledSchedulers(schedulerIds: string[]): void {
+ const handled = new Set(Object.keys(queues.cron.jobs));
+ const missing = schedulerIds.filter((id) => !handled.has(id));
+ if (missing.length > 0) {
+ logger.error(
+ { schedulers: missing },
+ 'cron schedulers have no handler on the cron queue — every tick will fail'
+ );
+ }
+}
+
+/**
+ * The Kafka events consumer. The kafkajs client, the topic, the consumer
+ * group, the DLQ producer and the retry bounds all come from
+ * `@openpanel/queue`, which is what keeps those names byte-identical across
+ * the cutover — core never spells one (see modules/ingest/src/consumer.ts).
+ */
+async function startIngestConsumer(
+ deps: AppDeps
+): Promise<KafkaConsumerHandle> {
+ assertKafkaConfigured();
+ enableEventsHeartbeat();
+
+ const eventDeps = await loadIncomingEventDeps(logger, async (input) => {
+ await deps.producers.queues.sessions.session.add(
+ sessionEndJobPayload(input),
+ sessionEndEnqueueOptions(input.closedSession.id)
+ );
+ });
+
+ return await startKafkaEventsConsumer({
+ createConsumer: createKafkaEventsConsumer,
+ logger,
+ kafkaLogger,
+ topic: KAFKA_EVENTS_TOPIC,
+ partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,
+ batch: {
+ handleEvent: (payload, meta) => incomingEvent(payload, eventDeps, meta),
+ sendToDeadLetter: produceDeadLetterEvent,
+ logger,
+ metrics: ingestConsumerMetrics,
+ onActivity: markEventsActivity,
+ maxAttempts: KAFKA_HANDLER_MAX_ATTEMPTS,
+ initialRetryMs: KAFKA_HANDLER_RETRY_INITIAL_MS,
+ maxRetryMs: KAFKA_HANDLER_RETRY_MAX_MS,
+ },
+ });
+}
+
+/**
+ * V1's shutdown drain (apps/worker/src/boot-workers.ts): let the cron queue
+ * finish what it started before closing the workers, because a cron job cut
+ * in half comes back as a stalled retry. One of ADR-005's four sanctioned
+ * `bullQueues` call sites.
+ */
+async function waitForCronToDrain(producers: QueueProducerHandle) {
+ const cron = findBullQueue(producers, CRON_QUEUE_NAME);
+ if (!cron) {
+ return;
+ }
+
+ const startedAt = performance.now();
+ while (true) {
+ const active = await cron.getActiveCount();
+ if (active === 0) {
+ return;
+ }
+ if (performance.now() - startedAt > CRON_DRAIN_TIMEOUT_MS) {
+ logger.warn(
+ { queue: cron.name, remainingCount: active },
+ 'Timeout reached while waiting for queue to empty'
+ );
+ return;
+ }
+ logger.info({ queue: cron.name, count: active }, 'Waiting for queue');
+ await new Promise((resolve) => setTimeout(resolve, CRON_DRAIN_POLL_MS));
+ }
+}
+
+function registerConsumerMetrics(deps: AppDeps): void {
+ // Scrape-time collectors register only where a role consumes: each is a
+ // Redis round trip per scrape, and ten API replicas exposing them would
+ // multiply that for no new information (TARGET_ARCHITECTURE §18).
+ registerBufferMetrics(deps.buffers);
+ registerQueueMetrics(deps.producers.bullQueues);
+ registerSessionScrapeMetrics(
+ () => getRedisCache() as unknown as SessionMetricsRedis
+ );
+}
+
+async function buildHttpApp(deps: AppDeps) {
+ const app = opsRoutes(deps);
+
+ // Local-only: trigger a cron job on demand instead of waiting for its
+ // schedule. Two conditions, both V1's: `NODE_ENV != production`, because it
+ // is an unauthenticated "run anything now" surface, and a consuming role,
+ // because in V1 these routes only ever existed on the worker.
+ if (roleConsumes && !isProduction(config)) {
+ app.use(debugRoutes(deps));
+ logger.info('Debug routes enabled at /debug/cron');
+ }
+
+ if (roleConsumes && !config.DISABLE_BULLBOARD) {
+ app.use(await bullBoardRoutes(deps, deps.producers.bullQueues));
+ logger.info({ path: BULL_BOARD_BASE_PATH }, 'bull-board mounted');
+ }
+
+ return app;
+}
+
+/**
+ * uncaughtException / unhandledRejection: the process state is corrupt, so log
+ * and exit fast rather than draining through a poisoned process.
+ *
+ * Fatals are mirrored to the REAL stderr, bypassing the output interceptor:
+ * the OTLP flush window is often lost on the way down, and `docker logs` must
+ * always show why we died (V1 did this in both apps).
+ *
+ * Installed in every role. V1's api only did so in production and its worker
+ * did so always; the merged process takes the worker's.
+ */
+function installFatalHandlers(): void {
+ process.on('uncaughtException', (error) => {
+ logger.fatal({ err: error }, 'Uncaught exception — exiting');
+ rawStderrWrite(`Uncaught exception — exiting: ${error?.stack ?? error}\n`);
+ setShuttingDown(true);
+ setTimeout(() => process.exit(1), FATAL_EXIT_DELAY_MS).unref();
+ });
+ process.on('unhandledRejection', (reason) => {
+ logger.fatal({ reason }, 'Unhandled rejection — exiting');
+ rawStderrWrite(
+ `Unhandled rejection — exiting: ${
+ reason instanceof Error ? reason.stack : String(reason)
+ }\n`
+ );
+ setShuttingDown(true);
+ setTimeout(() => process.exit(1), FATAL_EXIT_DELAY_MS).unref();
+ });
+}
+
async function main() {
const role = config.ROLE;
- const port = config.API_PORT;
const deps = buildDeps();
- // Scrape-time collectors register only where a role consumes queues: the
- // LLEN gauges cost one Redis round trip per buffer per scrape, and ten API
- // replicas exposing them would multiply that for no new information
- // (TARGET_ARCHITECTURE §18).
- if (role !== 'api') {
- registerBufferMetrics(deps.buffers);
+ // HTTP and default metrics register everywhere (TARGET_ARCHITECTURE §18).
+ registerDefaultMetrics();
+ if (roleConsumes) {
+ registerConsumerMetrics(deps);
}
- const app = opsRoutes(deps);
+ const app = await buildHttpApp(deps);
- if (role !== 'api') {
- logger.info(
- { role },
- 'ROLE=%s: worker/ingest bootstrap (workers, schedulers, Kafka consumer) is not implemented yet — running the HTTP ops surface only',
- role
- );
+ let workers: WorkerHandle | undefined;
+ let consumer: KafkaConsumerHandle | undefined;
+
+ if (workersEnabled) {
+ const definitions = consumedQueues();
+ workers = startWorkers({
+ definitions,
+ deps,
+ // BullMQ duplicates this for each worker's blocking fetch; the queue
+ // client itself stays packages/redis's singleton, as in V1.
+ createConnection: () => getRedisQueue(),
+ cluster: config.QUEUE_CLUSTER,
+ namespace: config.QUEUE_NAMESPACE,
+ });
+
+ if (definitions[CRON_QUEUE_NAME]) {
+ const cron = findBullQueue(deps.producers, CRON_QUEUE_NAME);
+ if (cron) {
+ await startSchedulers({
+ queue: cron,
+ flags: {
+ selfHosted: config.SELF_HOSTED,
+ production: isProduction(config),
+ },
+ logger,
+ });
+ warnOnUnhandledSchedulers(
+ (await cron.getJobSchedulers()).map((scheduler) => scheduler.key)
+ );
+ }
+ }
+
+ if (config.ENABLED_QUEUES.includes(KAFKA_QUEUE_TOKEN)) {
+ consumer = await startIngestConsumer(deps);
+ }
+
+ await createInitialSalts();
+ } else if (roleConsumes) {
+ logger.warn('Workers are disabled');
}
- app.listen(port, () => {
- logger.info({ role, port }, 'API listening');
- });
+ installFatalHandlers();
let shuttingDown = false;
const shutdown = async (signal: string) => {
@@ -150,16 +414,25 @@ async function main() {
return;
}
shuttingDown = true;
+ // Readiness answers 503 from here on, so the load balancer stops sending
+ // work before the drain below finishes rather than after.
+ setShuttingDown(true);
logger.info({ signal }, 'Starting graceful shutdown');
const forceExit = setTimeout(() => {
logger.error({ signal }, 'Graceful shutdown timed out — forcing exit');
process.exit(1);
- }, SHUTDOWN_FORCE_EXIT_MS);
+ }, config.SHUTDOWN_FORCE_EXIT_MS);
forceExit.unref();
try {
await app.stop();
+ // Only if THIS process consumes cron: waiting on a queue another replica
+ // owns would stall the whole shutdown budget on someone else's jobs.
+ if (workers && config.ENABLED_QUEUES.includes(CRON_QUEUE_NAME)) {
+ await waitForCronToDrain(deps.producers);
+ }
+ await Promise.all([workers?.close(), consumer?.stop()]);
await deps.producers.close();
logger.info('Graceful shutdown completed');
clearTimeout(forceExit);
@@ -177,6 +450,32 @@ async function main() {
process.on('SIGINT', () => {
shutdown('SIGINT');
});
+
+ // No `hostname` unless API_HOST says one: Bun's default is `0.0.0.0`, and
+ // its `localhost` binds IPv6-only (see config/env.ts's API_HOST).
+ const listen = config.API_HOST
+ ? { port: config.API_PORT, hostname: config.API_HOST }
+ : { port: config.API_PORT };
+ app.listen(listen, () => {
+ logger.info(
+ { role, port: config.API_PORT, hostname: config.API_HOST ?? '0.0.0.0' },
+ 'API listening'
+ );
+ });
+
+ // V1's api did this and its worker did not — the expiry notifications only
+ // matter where something subscribes (apps/api/src/index.ts).
+ if (role !== 'worker') {
+ try {
+ await getRedisPub().config('SET', 'notify-keyspace-events', 'Ex');
+ logger.info({ role }, 'redis keyspace notifications configured');
+ } catch (error) {
+ logger.warn({ err: error }, 'Failed to set redis notify-keyspace-events');
+ logger.warn(
+ 'If you use a managed Redis service, you may need to set this manually.'
+ );
+ }
+ }
}
main();
diff --git a/packages/core/docs/OPS_GRAFANA_MIGRATION.md b/packages/core/docs/OPS_GRAFANA_MIGRATION.md
new file mode 100644
index 000000000..b617f743c
--- /dev/null
+++ b/packages/core/docs/OPS_GRAFANA_MIGRATION.md
@@ -0,0 +1,225 @@
+# OPS: Grafana / alert migration for the V1 → V2 metrics cutover
+
+**Audience: whoever owns the OpenPanel dashboards and alerts. This is a written
+spec, not a change. No dashboard, alert or datasource was edited by the work
+that produced it — deliberately (M9 cutover discipline).**
+
+Written for M9-002. Sources: `packages/core/src/metrics.ts` (the one registry),
+`packages/core/src/http/http.metrics.ts`, `packages/core/src/jobs/jobs.metrics.ts`,
+`apps/api/src/main.ts` (the ROLE table), and — for the V1 side —
+`[email protected]/dist/fastify-metrics.js` and
+`apps/worker/src/metrics.ts`.
+
+> Where to put this file: it lives here because the task's scope globs cover
+> `packages/**` and not a repo-root `docs/`. Move it to `docs/OPS_GRAFANA_MIGRATION.md`
+> with one `git mv` if that is where you want it.
+
+---
+
+## 0. TL;DR — what a dashboard owner must do
+
+1. **No metric name changes.** Every panel and alert keeps working by name.
+2. **`route` label VALUES change** for a handful of routes (§2). Any panel or
+ alert that pins `route="…"` for one of those needs its literal updated.
+3. **Ten new `<queue>_*_count` series appear** (§3.1). Nothing to fix; you may
+ want new panels.
+4. **`buffer_add_*` for the replay, group and profile buffers steps up from
+ near-zero** (§3.3). It looks like a regression on a graph. It is a fix.
+5. **Bull-board moved** from `http://<worker>:3040/` to
+ `http://<host>:<port>/bullboard`, and is now behind the dashboard session
+ cookie (§4).
+6. **`/metrics` is unchanged**: same path, same container port 3000, served in
+ every ROLE.
+
+---
+
+## 1. Metric names and labels: unchanged, on purpose
+
+`fastify-metrics` is deleted with Fastify (ADR-002). Its two route series are
+re-created by one Elysia hook, reproducing the plugin's own definitions:
+
+| Series | Type | Labels | V1 source | V2 source |
+|---|---|---|---|---|
+| `http_request_duration_seconds` | histogram | `method`, `route`, `status_code` | `fastify-metrics` default | `core/src/http/http.metrics.ts` |
+| `http_request_summary_seconds` | summary | `method`, `route`, `status_code` | `fastify-metrics` default | same file |
+
+Both still observe **seconds**, not milliseconds — a millisecond observation on
+the same series would multiply every existing percentile by 1000, so it is
+pinned by a test (`http.metrics.test.ts`).
+
+Also unchanged in name and labels: the default process/GC metrics,
+`job_duration_ms{name,status}`, `kafka_events_*_total`, the twelve `buffer_*`
+series, the twelve session-lifecycle series, and the five
+`<queue>_{active,delayed,failed,completed,waiting}_count` gauges.
+
+`job_duration_ms` keeps its **failure-only** semantics: V1 observed it
+exclusively in the workers' `failed` listener, so every percentile ever computed
+from it is a percentile of failures. Success timings, if you ever want them,
+must get a **new** series name.
+
+---
+
+## 2. The `route` label: same name, different values
+
+This is the one thing that needs action.
+
+**V1's rule:** the label was Fastify's `routeOptions.url` — the declared route
+pattern *including* the plugin prefix, with Fastify's `:param` syntax.
+
+**V2's rule:** the label is Elysia's `route` — the matched route pattern, with
+Elysia's `:param` syntax.
+
+The two syntaxes agree, so **most values are byte-identical**: every
+`/insights/:projectId/...`, `/manage/...`, `/export/...`, `/import/events`,
+`/track/device-id`, `/healthz/live` and so on carries the same literal it does
+today. The differences are structural, and they are these:
+
+### 2.1 Routes whose label changes
+
+| What | V1 `route` label | V2 `route` label | Why |
+|---|---|---|---|
+| Track | `/track/` | `/track` | V1's label was `prefix` + the route `'/'`, giving a trailing slash. V2 declares the full path. |
+| Profile | `/profile/` | `/profile` | same |
+| Legacy event | `/event/` | `/event` | same (the route itself is KEPT — ADR-004/ADR-015 acceptance notes) |
+| tRPC | `/trpc/:path` | `/trpc/*` | the fetch adapter derives the path by slicing the prefix, and a **batched** request puts commas in that segment, which `:path` cannot match (ADR-009) |
+| OpenAPI UI | `/documentation` | `/openapi` | `@fastify/swagger-ui` → `@elysiajs/openapi` |
+| OpenAPI document | `/documentation/json` | `/openapi.json` | same |
+| Bull-board | *(separate worker port, `/`)* | `/bullboard`, and its sub-routes | §4 |
+| Debug cron | *(separate worker port, `/debug/cron/:type`)* | `/debug/cron/:type` on the main port | §4 |
+
+### 2.2 Routes whose label disappears
+
+| V1 `route` label | Why it goes |
+|---|---|
+| `/misc/og/clear`, `/misc/favicon/clear` | deleted (removal register entry 6) |
+| `/mcp` GET and DELETE | deleted (removal register entry 2); `POST /mcp` stays |
+| the whole worker-side `/` bull-board tree on :3040/:3050 | merged onto the main port |
+
+### 2.3 Behaviour at the edges, reproduced deliberately
+
+| Case | V1 (`fastify-metrics`) | V2 |
+|---|---|---|
+| `HEAD` / `OPTIONS` | not measured (`methodBlacklist` default) | not measured |
+| a request matching no route (404) | not measured (`registeredRoutesOnly` default true) | not measured |
+| a handler that threw and was mapped to 5xx | measured — `onResponse` fires | measured — the hook is `onAfterResponse`, which likewise fires |
+| status code grouping | ungrouped (`groupStatusCodes` default false) | ungrouped |
+
+### 2.4 One thing this table cannot promise yet
+
+At the time of writing, `main.ts` mounts only the **ops** surface; the dashboard
+and public-API surfaces mount at M9-004, when the V1 Fastify boot is deleted.
+The rules above are final, and §2.1 lists every difference in the route set that
+exists today — but **re-read §2.1 against the M9-004 commit before you edit a
+dashboard**, because that is the commit in which the values actually start being
+produced.
+
+Two route-set observations that belong to that commit, not to this one, and are
+recorded here so they are not discovered from a graph:
+
+* `packages/core/src/modules/tools/tools.routes.ts` declares `/site-checker`
+ and `/ip-lookup` at the root, where V1 served them under the router prefix
+ `/tools`. Whoever mounts the public-API surface has to restore the prefix;
+ until then both the URL and the metric label would differ.
+* `/manage/references*`, `/manage/projects/:projectId/dashboards*`,
+ `POST /event`, `/healthcheck` and `/trpc/*` are still V1-only routes.
+
+---
+
+## 3. Series that appear, disappear, or change level
+
+### 3.1 New: the queue gauges cover all seven queues
+
+V1 registered `<queue>_{active,delayed,failed,completed,waiting}_count` for
+**two** queues — `sessions` and `cron`. V2 registers them for all seven
+(ADR-018's continuity register). **Ten new series**, none of the existing ten
+changed:
+
+```
+notification_{active,delayed,failed,completed,waiting}_count
+import_{...}_count
+insights_{...}_count
+gsc_{...}_count
+cohortCompute_{...}_count
+```
+
+The `cohortCompute` spelling is intentional and permanent — the registry key
+was NOT renamed (ADR-005 acceptance note), so `COHORTCOMPUTE_CONCURRENCY` keeps
+working and the series name matches the queue.
+
+### 3.2 Gone: the GroupMQ shard series
+
+Removed with GroupMQ itself (ADR-004). Any panel or alert still selecting them
+will read "No data" rather than error. Delete those panels.
+
+### 3.3 Level change: `buffer_add_*` steps up
+
+V1 installed the buffer *add* observers only inside the worker process, so every
+API-side add — the replay, group and profile buffers on the `/track` path — was
+invisible. V2 has one registry per process and every role builds the buffers, so
+those adds now count.
+
+**On a graph this looks like a regression at the cutover. It is a fix.** Expect
+`buffer_add_*` for `replay`, `group` and `profile` to step up from near-zero;
+`event` and `session` are already counted in V1 and should not move.
+
+If you have an alert with an absolute threshold on any `buffer_add_*` series,
+re-baseline it after the cutover.
+
+### 3.4 Where each collector registers now
+
+`/metrics` is served in **every** role, but not every role registers every
+collector — a scrape-time gauge costs a Redis round trip per project, and ten
+API replicas exposing the session gauges would multiply that Redis load for no
+new information.
+
+| Collector | `ROLE=api` | `ROLE=worker` | `ROLE=all` |
+|---|---|---|---|
+| default process metrics | yes | yes | yes |
+| `http_request_*` | yes | yes | yes |
+| `<queue>_*_count` | **no** | yes | yes |
+| `buffer_*` | **no** | yes | yes |
+| `sessions_active_total`, `sessions_projects_active`, `sessions_hwm_lag_ms` | **no** | yes | yes |
+| `job_duration_ms`, `kafka_events_*` | declared, never observed | observed | observed |
+
+**Consequence for a dashboard:** a panel that today sums a queue or buffer gauge
+across every scrape target keeps working, because only the consuming replicas
+ever exposed a non-trivial value. A panel that *asserts presence* on an API
+target (e.g. `absent(buffer_event_count)`) will fire. Scope such alerts to the
+consuming job.
+
+---
+
+## 4. Two URLs moved (deploy note, not a dashboard note)
+
+The worker's second HTTP server is gone; one process serves one port in every
+role (ADR-007).
+
+| What | V1 | V2 |
+|---|---|---|
+| Bull-board | `http://<worker>:3040/` (and `:3050`), **unauthenticated on that port** | `http://<host>:<port>/bullboard`, **behind the dashboard session cookie** |
+| Debug cron trigger | `http://<worker>:3040/debug/cron/:type` | `/debug/cron/:type` on the main port, still `NODE_ENV != production` only |
+| `/metrics` | `:3030` (api), `:3040`/`:3050` (workers) | unchanged — same path, container port still 3000 |
+
+Bull-board is mounted only when the role consumes **and** `DISABLE_BULLBOARD` is
+unset (`'1'` and `'true'` are both still accepted as "disable"). It now returns
+**401** rather than the UI to an unauthenticated caller.
+
+Unchanged and deliberate: **pausing the `cron` queue from bull-board halts all
+buffer flushing** (docs/ANSWERS.md §3 — "known!").
+
+---
+
+## 5. Checklist
+
+- [ ] Search dashboards and alert rules for `route="/track/"`, `route="/profile/"`,
+ `route="/event/"`, `route="/trpc/:path"`, `route="/documentation"` and
+ update the literals per §2.1.
+- [ ] Delete panels selecting the GroupMQ shard series (§3.2).
+- [ ] Re-baseline absolute-threshold alerts on `buffer_add_*` for the replay,
+ group and profile buffers (§3.3).
+- [ ] Scope any `absent()`/`up`-style alert on a queue, buffer or session gauge
+ to the consuming job only (§3.4).
+- [ ] Optionally add panels for the five newly-covered queues (§3.1).
+- [ ] Update the bull-board bookmark and tell operators it now needs a dashboard
+ login (§4).
+- [ ] Re-read §2.1 against the M9-004 commit before editing anything (§2.4).
diff --git a/packages/core/package.json b/packages/core/package.json
index 528b482fd..39387f0ca 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -24,6 +24,8 @@
"@babel/parser": "^7.26.0",
"@better-agent/core": "^0.1.0-beta.3",
"@better-agent/providers": "^0.1.0-beta.3",
+ "@bull-board/api": "9.6.0",
+ "@bull-board/elysia": "9.6.0",
"@elysiajs/cors": "1.4.2",
"@elysiajs/openapi": "1.4.15",
"@google-cloud/storage": "^7.18.0",
diff --git a/packages/core/src/http/bull-board.ts b/packages/core/src/http/bull-board.ts
new file mode 100644
index 000000000..3a75ee50a
--- /dev/null
+++ b/packages/core/src/http/bull-board.ts
@@ -0,0 +1,79 @@
+// Bull-board, on the main port, behind the dashboard session guard
+// (ADR-005 recommendation 6). What dies with it is `@bull-board/express` and
+// the worker's second express server — nothing named `@bull-board/fastify`
+// ever existed. `@bull-board/api` + `@bull-board/elysia` are both pinned
+// exact 9.6.0 (ADR-016; ADR-017 register row 4 grants the bump+swap at P9).
+//
+// TWO things a reader must not mistake for accidents:
+//
+// 1. **The base path moved.** V1 served bull-board at `/` on the worker's own
+// express server (apps/worker/src/index.ts, `setBasePath('/')`). V2 has
+// one port for everything, so `/` would shadow every other route. It is
+// `/bullboard`; that is a deploy-note, written up in
+// `packages/core/docs/OPS_GRAFANA_MIGRATION.md`.
+// 2. **Pausing `cron` from this UI halts ALL buffer flushing.** Preserved
+// deliberately — docs/ANSWERS.md §3: "known!".
+
+import type { Queue as BullQueue } from 'bullmq';
+import { Elysia } from 'elysia';
+import type { AppDeps } from '../context';
+import { requestContext } from './context';
+
+export const BULL_BOARD_BASE_PATH = '/bullboard';
+
+const UNAUTHORIZED = 401;
+
+/**
+ * Mounted only where the role consumes and `DISABLE_BULLBOARD` is unset
+ * (main.ts owns both conditions — core reads no environment).
+ *
+ * The guard is `{ as: 'global' }` and re-checks the path itself rather than
+ * leaning on Elysia's hook scoping: `.use()`-ing a NAMED plugin
+ * (`@bull-board/elysia`) puts its routes in their own scope, where a `local`
+ * hook of the mounting instance would not run at all — and a guard that
+ * silently does not run is precisely the failure this must not have. The cost
+ * on every other route is one `startsWith`.
+ */
+export async function bullBoardRoutes(
+ deps: AppDeps,
+ queues: BullQueue[]
+): Promise<Elysia> {
+ // Imported here, not at module scope. `@bull-board/elysia` is CommonJS, so
+ // it `require()`s Elysia's CJS build, which `require()`s memoirist — and
+ // under Bun 1.4.0 that throws "require() async module ... is unsupported"
+ // if it happens while this package's module graph is still evaluating.
+ // Deferring it to the one call site that needs it puts the require after
+ // evaluation, where it resolves cleanly. Proven by
+ // `bun run apps/api/e2e/boot-proof.sh`'s ROLE=worker/all legs, which 401
+ // on /bullboard rather than failing to boot.
+ const [{ createBullBoard }, { BullMQAdapter }, { ElysiaAdapter }] =
+ await Promise.all([
+ import('@bull-board/api'),
+ import('@bull-board/api/bullMQAdapter'),
+ import('@bull-board/elysia'),
+ ]);
+
+ const serverAdapter = new ElysiaAdapter({
+ prefix: BULL_BOARD_BASE_PATH,
+ basePath: BULL_BOARD_BASE_PATH,
+ });
+
+ createBullBoard({
+ queues: queues.map((queue) => new BullMQAdapter(queue)),
+ serverAdapter,
+ });
+
+ const board = await serverAdapter.registerPlugin();
+
+ return new Elysia({ name: 'core/http/bull-board' })
+ .use(requestContext(deps))
+ .onBeforeHandle({ as: 'global' }, async ({ path, ctx, status }) => {
+ if (!path.startsWith(BULL_BOARD_BASE_PATH)) {
+ return;
+ }
+ if (!(await ctx.session())) {
+ return status(UNAUTHORIZED);
+ }
+ })
+ .use(board) as unknown as Elysia;
+}
diff --git a/packages/core/src/http/debug.routes.ts b/packages/core/src/http/debug.routes.ts
new file mode 100644
index 000000000..686c4907d
--- /dev/null
+++ b/packages/core/src/http/debug.routes.ts
@@ -0,0 +1,229 @@
+// V1's `apps/worker/src/boot-debug.ts`, ported onto Elysia and onto the ONE
+// port. Local-only: `main.ts` mounts this surface only outside production AND
+// only in a consuming role — the same two facts that were true of V1, where
+// the routes lived in the worker behind `NODE_ENV !== 'production'`. They are
+// what keeps an unauthenticated "run any cron job now" endpoint off a
+// production box.
+//
+// Three things changed on the way over, all deliberate:
+//
+// * The job list is `queues.cron.jobs`, not a hand-written array of the 20
+// `CronQueueType` strings. The array could drift from the registry; the
+// registry cannot drift from itself.
+// * A trigger runs through `runJob` with V1's `{ type }` data — the same
+// entry point the real worker uses, compat hook included — so what this
+// exercises is the production path and not a second dispatcher.
+// * `/debug/insights/:projectId?inline=1` summarises with the insight
+// service's own `listAllInsights` instead of V1's ad-hoc
+// `projectInsight.groupBy`, because core reaches Postgres through its
+// services, not through a Prisma client of its own.
+
+import { z } from 'zod';
+import { runJob } from '../jobs/workers';
+import { queues } from '../jobs.registry';
+import { defineRoutes } from './define';
+
+const BAD_REQUEST = 400;
+const INTERNAL_ERROR = 500;
+const DIGEST_INSIGHT_LIMIT = 15;
+
+const cronTypes = Object.keys(queues.cron.jobs);
+
+const HTML_ESCAPES: Record<string, string> = {
+ '&': '&amp;',
+ '<': '&lt;',
+ '>': '&gt;',
+ '"': '&quot;',
+};
+const HTML_ESCAPABLE = /[&<>"]/g;
+
+function escapeHtml(value: string): string {
+ return value.replace(HTML_ESCAPABLE, (char) => HTML_ESCAPES[char] ?? char);
+}
+
+// Browsers send an explicit `text/html` accept header; curl sends the
+// wildcard, which falls through to JSON — V1's `req.accepts(['json','html'])`
+// ordering, kept.
+function wantsHtml(headers: Headers): boolean {
+ return (headers.get('accept') ?? '').includes('text/html');
+}
+
+function buildJobs(request: Request) {
+ const base = new URL(request.url).origin;
+ return cronTypes.map((type) => ({
+ type,
+ url: `${base}/debug/cron/${type}`,
+ }));
+}
+
+function renderHtml(
+ jobs: { type: string; url: string }[],
+ message?: string
+): string {
+ const links = jobs
+ .map((job) => `<li><a href="${job.url}">${escapeHtml(job.type)}</a></li>`)
+ .join('');
+ const note = message
+ ? `<pre>${escapeHtml(message)}</pre>`
+ : '<p>Click a job to run it now.</p>';
+ return `<!doctype html>
+<html lang="en">
+<head><meta charset="utf-8"><title>Debug cron</title></head>
+<body style="font-family: ui-monospace, monospace; max-width: 40rem; margin: 2rem auto;">
+<h1>Trigger a cron job</h1>
+${note}
+<ul>${links}</ul>
+</body>
+</html>`;
+}
+
+const errorMessage = (error: unknown): string =>
+ error instanceof Error ? error.message : String(error);
+
+const projectIdParams = z.object({ projectId: z.string() });
+const HIDDEN = { hide: true } as const;
+
+export const debugRoutes = defineRoutes((app, deps) =>
+ app
+ .get(
+ '/debug/cron',
+ ({ request, set }) => {
+ const jobs = buildJobs(request);
+ if (wantsHtml(request.headers)) {
+ set.headers['content-type'] = 'text/html; charset=utf-8';
+ return renderHtml(jobs);
+ }
+ return {
+ message: 'Run a cron job now with GET or POST /debug/cron/:type',
+ jobs,
+ };
+ },
+ // Hidden from the OpenAPI document, like every other ops-surface route:
+ // in V1 these lived on the worker's own server and were never in it.
+ { detail: HIDDEN }
+ )
+ .all(
+ '/debug/cron/:type',
+ async ({ params, request, set, ctx }) => {
+ const { type } = params as { type: string };
+ const jobs = buildJobs(request);
+ const html = wantsHtml(request.headers);
+
+ if (!cronTypes.includes(type)) {
+ set.status = BAD_REQUEST;
+ const message = `Unknown cron type "${type}"`;
+ if (html) {
+ set.headers['content-type'] = 'text/html; charset=utf-8';
+ return renderHtml(jobs, message);
+ }
+ return { ok: false, error: message, jobs };
+ }
+
+ ctx.logger.info({ type }, 'Manually triggering cron job');
+
+ try {
+ // V1's wire data, so this goes through the `cron` compat hook.
+ await runJob(
+ queues.cron,
+ { name: type, data: { type }, attemptsMade: 0 },
+ deps
+ );
+ } catch (error) {
+ ctx.logger.error({ err: error, type }, 'Manual cron trigger failed');
+ set.status = INTERNAL_ERROR;
+ if (html) {
+ set.headers['content-type'] = 'text/html; charset=utf-8';
+ return renderHtml(
+ jobs,
+ `Error running ${type}: ${errorMessage(error)}`
+ );
+ }
+ return { ok: false, type, error: errorMessage(error) };
+ }
+
+ if (html) {
+ set.headers['content-type'] = 'text/html; charset=utf-8';
+ return renderHtml(jobs, `${type} ran. Check the logs for details.`);
+ }
+ return { ok: true, type };
+ },
+ { detail: HIDDEN }
+ )
+ // The insights pipeline for ONE project, bypassing the daily cron's
+ // eligibility filter. Default enqueues a real job so it shows up in
+ // bull-board; `?inline=1` runs it here and returns a summary.
+ .get(
+ '/debug/insights/:projectId',
+ async ({ params, query, set, ctx }) => {
+ const { projectId } = params;
+ const inline = query.inline === '1' || query.inline === 'true';
+ const date = new Date().toISOString().slice(0, 10);
+
+ try {
+ if (inline) {
+ ctx.logger.info({ projectId }, 'Manual insights run (inline)');
+ await ctx.services.insight.runProjectInsights({
+ projectId,
+ date,
+ });
+ return {
+ ok: true,
+ projectId,
+ date,
+ insights: await ctx.services.insight.listAllInsights({
+ projectId,
+ limit: DIGEST_INSIGHT_LIMIT,
+ }),
+ };
+ }
+
+ const jobId = await ctx.queues.insights.insightsProject.add(
+ { projectId, date },
+ // Unique id so repeated runs are not deduped by BullMQ.
+ { jobId: `debug:${projectId}:${Date.now()}` }
+ );
+ ctx.logger.info({ projectId, jobId }, 'Enqueued insights job');
+ return {
+ ok: true,
+ projectId,
+ jobId,
+ message:
+ 'Enqueued on the "insights" queue — watch it in bull-board. Add ?inline=1 to run synchronously.',
+ };
+ } catch (error) {
+ ctx.logger.error({ err: error, projectId }, 'Manual insights failed');
+ set.status = INTERNAL_ERROR;
+ return { ok: false, projectId, error: errorMessage(error) };
+ }
+ },
+ { params: projectIdParams, detail: HIDDEN }
+ )
+ // Weekly digest for ONE project, bypassing eligibility.
+ // [email protected] → send only to that address
+ // (no ?to) → return the assembled payload without sending
+ // ?force=1 → build even with 0 visitors this week
+ .get(
+ '/debug/weekly-digest/:projectId',
+ async ({ params, query, set, ctx }) => {
+ const { projectId } = params;
+ const to = typeof query.to === 'string' ? query.to : undefined;
+ const force = query.force === '1' || query.force === 'true';
+ ctx.logger.info({ projectId, to, force }, 'Manual weekly digest');
+
+ try {
+ return {
+ ok: true,
+ ...(await ctx.services.insight.previewWeeklyDigest(projectId, {
+ to,
+ force,
+ })),
+ };
+ } catch (error) {
+ ctx.logger.error({ err: error, projectId }, 'Manual digest failed');
+ set.status = INTERNAL_ERROR;
+ return { ok: false, projectId, error: errorMessage(error) };
+ }
+ },
+ { params: projectIdParams, detail: HIDDEN }
+ )
+);
diff --git a/packages/core/src/http/http.metrics.test.ts b/packages/core/src/http/http.metrics.test.ts
new file mode 100644
index 000000000..fa222c51f
--- /dev/null
+++ b/packages/core/src/http/http.metrics.test.ts
@@ -0,0 +1,81 @@
+import { beforeEach, expect, test } from 'bun:test';
+import client from 'prom-client';
+import { registry } from '../metrics';
+import { observeHttpRequest } from './http.metrics';
+
+const HISTOGRAM = 'http_request_duration_seconds';
+const SUMMARY = 'http_request_summary_seconds';
+
+async function samples(name: string) {
+ const metric = await registry.getSingleMetricAsString(name);
+ return metric;
+}
+
+beforeEach(() => {
+ registry.resetMetrics();
+});
+
+test('both series exist on the ONE registry, with V1s names and labels', () => {
+ const registered = registry.getMetricsAsArray().map((metric) => metric.name);
+ expect(registered).toContain(HISTOGRAM);
+ expect(registered).toContain(SUMMARY);
+
+ // A second registry anywhere would be the defect: prom-client's own global
+ // default must stay empty of these.
+ expect(client.register.getMetricsAsArray().map((m) => m.name)).not.toContain(
+ HISTOGRAM
+ );
+});
+
+test('observes in SECONDS, under method/route/status_code', async () => {
+ observeHttpRequest({
+ method: 'POST',
+ route: '/track',
+ statusCode: 202,
+ durationMs: 1500,
+ });
+
+ const text = await samples(HISTOGRAM);
+ expect(text).toContain('method="POST"');
+ expect(text).toContain('route="/track"');
+ expect(text).toContain('status_code="202"');
+ // 1500ms is 1.5s — a milliseconds observation would read 1500 here and
+ // multiply every existing percentile by 1000.
+ expect(text).toContain(
+ `${HISTOGRAM}_sum{method="POST",route="/track",status_code="202"} 1.5`
+ );
+});
+
+test('HEAD and OPTIONS are not measured (fastify-metrics methodBlacklist)', async () => {
+ observeHttpRequest({
+ method: 'OPTIONS',
+ route: '/track',
+ statusCode: 204,
+ durationMs: 1,
+ });
+ observeHttpRequest({
+ method: 'HEAD',
+ route: '/track',
+ statusCode: 200,
+ durationMs: 1,
+ });
+
+ expect(await samples(HISTOGRAM)).not.toContain('route="/track"');
+});
+
+test('an unmatched request produces no series (registeredRoutesOnly)', async () => {
+ observeHttpRequest({
+ method: 'GET',
+ route: '/*',
+ statusCode: 404,
+ durationMs: 1,
+ });
+ observeHttpRequest({
+ method: 'GET',
+ route: '',
+ statusCode: 404,
+ durationMs: 1,
+ });
+
+ expect(await samples(SUMMARY)).not.toContain('status_code="404"');
+});
diff --git a/packages/core/src/http/http.metrics.ts b/packages/core/src/http/http.metrics.ts
new file mode 100644
index 000000000..13e38d99e
--- /dev/null
+++ b/packages/core/src/http/http.metrics.ts
@@ -0,0 +1,109 @@
+// `fastify-metrics`'s two route series, re-created on Elysia
+// (TARGET_ARCHITECTURE §18 / ADR-018's metrics-continuity register: "kept,
+// name and labels reproduced exactly"). The plugin itself dies with Fastify
+// (ADR-002's deleted-plugin list), so these are ours now.
+//
+// What "reproduced exactly" pins, from fastify-metrics 12.1.0's own source
+// (dist/fastify-metrics.js:181-210, 258-290):
+//
+// * names `http_request_duration_seconds` (histogram) and
+// `http_request_summary_seconds` (summary)
+// * help "request duration in seconds" / "... summary"
+// * labels `method`, `route`, `status_code`, in that order
+// * unit SECONDS. prom-client's `startTimer()` observes seconds; a
+// milliseconds observation on the same series silently multiplies
+// every existing percentile by 1000.
+// * scope HEAD and OPTIONS are excluded (fastify-metrics' default
+// `methodBlacklist`), and only a MATCHED route is observed
+// (`registeredRoutesOnly` defaults true) — so a 404 produces no
+// series, exactly as today.
+//
+// The `route` label VALUE moves, because it is the framework's own route
+// pattern and the framework changed. That is the one thing a dashboard owner
+// has to act on; the old->new mapping is
+// `packages/core/docs/OPS_GRAFANA_MIGRATION.md`.
+
+import { Elysia } from 'elysia';
+import client from 'prom-client';
+import { registry } from '../metrics';
+
+const DURATION_HELP = 'request duration in seconds';
+const SUMMARY_HELP = 'request duration in seconds summary';
+const LABEL_NAMES = ['method', 'route', 'status_code'] as const;
+// fastify-metrics' `methodBlacklist` default (dist/fastify-metrics.js:63-66).
+const UNMEASURED_METHODS = new Set(['HEAD', 'OPTIONS']);
+// Elysia's `route` for a request that matched no route. Observing it would
+// create the unbounded-cardinality series `registeredRoutesOnly` avoids.
+const UNMATCHED_ROUTES = new Set(['', '/*']);
+const MS_PER_SECOND = 1000;
+// Elysia leaves `set.status` unset on a plain successful return.
+const DEFAULT_STATUS = 200;
+
+export const httpRequestDuration = new client.Histogram({
+ name: 'http_request_duration_seconds',
+ help: DURATION_HELP,
+ labelNames: LABEL_NAMES,
+ registers: [registry],
+});
+
+export const httpRequestSummary = new client.Summary({
+ name: 'http_request_summary_seconds',
+ help: SUMMARY_HELP,
+ labelNames: LABEL_NAMES,
+ registers: [registry],
+});
+
+/**
+ * Observes a single request. Exported so a test can assert the labels and the
+ * unit without standing up a server.
+ */
+export function observeHttpRequest(sample: {
+ method: string;
+ route: string;
+ statusCode: number;
+ durationMs: number;
+}): void {
+ if (UNMEASURED_METHODS.has(sample.method)) {
+ return;
+ }
+ if (UNMATCHED_ROUTES.has(sample.route)) {
+ return;
+ }
+
+ const labels = {
+ method: sample.method,
+ route: sample.route,
+ status_code: sample.statusCode,
+ };
+ const seconds = sample.durationMs / MS_PER_SECOND;
+
+ httpRequestDuration.observe(labels, seconds);
+ httpRequestSummary.observe(labels, seconds);
+}
+
+/**
+ * The one hook. `onAfterResponse` rather than `onAfterHandle` because
+ * fastify-metrics observed in `onResponse`: an error mapped to a 4xx/5xx by
+ * the error handler never reaches `onAfterHandle`, and dropping those samples
+ * would quietly remove every error-status bucket the existing panels group by.
+ *
+ * The start time rides on a `derive`, not on `store` — `store` is app-level
+ * state shared by every in-flight request, so a second concurrent request
+ * would overwrite the first one's clock.
+ */
+export function httpMetrics() {
+ return new Elysia({ name: 'core/http/metrics' })
+ .derive({ as: 'global' }, () => ({ metricsStartedAt: performance.now() }))
+ .onAfterResponse(
+ { as: 'global' },
+ ({ request, route, set, metricsStartedAt }) => {
+ observeHttpRequest({
+ method: request.method,
+ route,
+ statusCode:
+ typeof set.status === 'number' ? set.status : DEFAULT_STATUS,
+ durationMs: performance.now() - metricsStartedAt,
+ });
+ }
+ );
+}
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 9b9c9580d..8c963ae3f 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -141,7 +141,12 @@ export type {
Session,
} from './context';
export { createCtx, extendCtx } from './context';
+// The role-conditional HTTP surfaces main.ts mounts over the three in
+// rest.routes.ts (M9-002). `bullBoardRoutes` is async because the adapter
+// reads its own UI assets off disk before it can register.
+export { BULL_BOARD_BASE_PATH, bullBoardRoutes } from './http/bull-board';
export { requestContext, requestLogging } from './http/context';
+export { debugRoutes } from './http/debug.routes';
// The two runtime seams over the registry (ADR-005). `main.ts` builds
// producers in every role and workers only where the role consumes; both take
// the registry as data, so neither opens a connection until it is called.
@@ -151,6 +156,8 @@ export type {
QueueDefinition,
} from './jobs/define';
export type { JobEnvelope, JobMeta } from './jobs/envelope';
+export type { CountableQueue } from './jobs/jobs.metrics';
+export { registerQueueMetrics } from './jobs/jobs.metrics';
export { queueKey } from './jobs/naming';
export type { CreateProducersOptions } from './jobs/producers';
export { createProducers } from './jobs/producers';
@@ -168,7 +175,7 @@ export type {
TerminalFailure,
WorkerHandle,
} from './jobs/workers';
-export { startWorkers } from './jobs/workers';
+export { runJob, startWorkers } from './jobs/workers';
export type {
QueueProducerHandle,
QueueProducers,
@@ -181,6 +188,10 @@ export {
REQUEST_ID_LENGTH,
REQUEST_ID_LOG_FIELD,
} from './logger';
+// The one registry's own boot-time registrars. `registry` itself is NOT
+// exported: a caller that can reach it can start a second collector graph
+// somewhere other than a module, which is the thing §18 forbids.
+export { registerDefaultMetrics } from './metrics';
// Moved from apps/api/src/agents/* + packages/trpc/src/agents/filter-command.ts
// (M5-005) — apps/api's `/ai/agents/*` Fastify wrapper and packages/trpc's
// overview router call these.
@@ -1151,6 +1162,7 @@ export async function handleMcpRequest(
const token = extractToken(query, authHeader);
return handleStatelessMcpRequest(token, body);
}
+export { isShuttingDown, setShuttingDown } from './modules/health/src/shutdown';
// Dissolved from @openpanel/db's services/import.service.ts +
// apps/worker's job file + apps/api's /import controller (M5-004) —
// apps/worker's import job file and apps/api's import controller call these
@@ -1233,6 +1245,13 @@ export {
createEventsBatchHandler,
startKafkaEventsConsumer,
} from './modules/ingest/src/consumer';
+// The readiness probe's two inputs (M9-002). `markEventsActivity` is the
+// consumer's `onActivity`; `setShuttingDown` is main.ts's signal handler.
+export {
+ enableEventsHeartbeat,
+ getEventsHeartbeat,
+ markEventsActivity,
+} from './modules/ingest/src/heartbeat';
export type {
IncomingEventDelivery,
IncomingEventDeps,
@@ -1243,10 +1262,7 @@ export {
incomingEvent,
loadIncomingEventDeps,
} from './modules/ingest/src/incoming-event-handler';
-// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service
-// (M5-001) — apps/worker's insight job files and packages/trpc's insight
-// router call these directly, the same way V1 reaches every other dissolved
-// service here.
+export { ingestConsumerMetrics } from './modules/ingest/src/ingest.metrics';
export type {
DailyInsightCandidate,
GetReferrerSpikesInput,
@@ -1295,6 +1311,12 @@ export {
getSalts,
rotateSalt,
} from './modules/salt/salt.service';
+// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service
+// (M5-001) — apps/worker's insight job files and packages/trpc's insight
+// router call these directly, the same way V1 reaches every other dissolved
+// service here.
+export type { SessionMetricsRedis } from './modules/session/src/session.metrics';
+export { registerSessionScrapeMetrics } from './modules/session/src/session.metrics';
// New module (M7-008) — apps/api's tools controller calls these directly,
// the same way V1 reaches every other dissolved service here.
export type {
diff --git a/packages/core/src/jobs/jobs.metrics.test.ts b/packages/core/src/jobs/jobs.metrics.test.ts
new file mode 100644
index 000000000..cc0d4f06e
--- /dev/null
+++ b/packages/core/src/jobs/jobs.metrics.test.ts
@@ -0,0 +1,76 @@
+import { expect, test } from 'bun:test';
+import client from 'prom-client';
+import { queues } from '../jobs.registry';
+import { type CountableQueue, registerQueueMetrics } from './jobs.metrics';
+import { queueKey } from './naming';
+
+const COUNT_SUFFIXES = ['active', 'delayed', 'failed', 'completed', 'waiting'];
+
+function fakeQueue(name: string, value: number): CountableQueue {
+ return {
+ name,
+ getActiveCount: () => Promise.resolve(value),
+ getDelayedCount: () => Promise.resolve(value),
+ getFailedCount: () => Promise.resolve(value),
+ getCompletedCount: () => Promise.resolve(value),
+ getWaitingCount: () => Promise.resolve(value),
+ };
+}
+
+test('five gauges per queue, named from the Redis key exactly as V1 named them', () => {
+ const register = new client.Registry();
+ registerQueueMetrics(
+ Object.values(queues).map((definition) => fakeQueue(definition.name, 0)),
+ register
+ );
+
+ const names = register.getMetricsAsArray().map((metric) => metric.name);
+ expect(names).toHaveLength(
+ Object.keys(queues).length * COUNT_SUFFIXES.length
+ );
+ for (const suffix of COUNT_SUFFIXES) {
+ expect(names).toContain(`sessions_${suffix}_count`);
+ expect(names).toContain(`cron_${suffix}_count`);
+ // ADR-005 acceptance note: the registry key stays `cohortCompute`, so the
+ // series name does too.
+ expect(names).toContain(`cohortCompute_${suffix}_count`);
+ }
+});
+
+test('cluster braces are stripped from the series name, not from the queue key', () => {
+ const register = new client.Registry();
+ const key = queueKey('cron', { cluster: true });
+ expect(key).toBe('{cron}');
+
+ registerQueueMetrics([fakeQueue(key, 0)], register);
+
+ expect(register.getMetricsAsArray().map((metric) => metric.name)).toContain(
+ 'cron_active_count'
+ );
+});
+
+test('a QUEUE_NAMESPACE key stays a legal metric name', () => {
+ const register = new client.Registry();
+ const key = queueKey('cron', { namespace: 'proof-1' });
+ expect(key).toBe('cron-proof-1');
+
+ registerQueueMetrics([fakeQueue(key, 0)], register);
+
+ // `-` is not legal in a prom-client name; an unsanitised key throws at
+ // construction and takes boot down.
+ expect(register.getMetricsAsArray().map((metric) => metric.name)).toContain(
+ 'cron_proof_1_active_count'
+ );
+});
+
+test('a gauge whose Redis read throws does not fail the scrape', async () => {
+ const register = new client.Registry();
+ const broken: CountableQueue = {
+ ...fakeQueue('sessions', 7),
+ getActiveCount: () => Promise.reject(new Error('redis down')),
+ };
+ registerQueueMetrics([broken], register);
+
+ const text = await register.metrics();
+ expect(text).toContain('sessions_waiting_count 7');
+});
diff --git a/packages/core/src/jobs/jobs.metrics.ts b/packages/core/src/jobs/jobs.metrics.ts
new file mode 100644
index 000000000..ad56c1317
--- /dev/null
+++ b/packages/core/src/jobs/jobs.metrics.ts
@@ -0,0 +1,130 @@
+// The queue-side collectors, moved from apps/worker/src/metrics.ts onto the
+// one core registry (TARGET_ARCHITECTURE §18). Names, labels and buckets are
+// V1's; the worker-side copies die with apps/worker (P9).
+//
+// Three deliberate differences from V1, all listed in
+// `packages/core/docs/OPS_GRAFANA_MIGRATION.md`:
+//
+// 1. V1 registered the five `<queue>_*_count` gauges for TWO queues
+// (`sessionsQueue`, `cronQueue`). V2 registers them for all seven, which
+// ADR-018's continuity register asks for ("kept for the 7 BullMQ
+// queues"). Ten new series appear; none of the existing ten changes.
+// 2. They register only where the role consumes (main.ts), because each
+// gauge is one Redis round trip per scrape and ten api replicas exposing
+// them would multiply that for no new information.
+// 3. A gauge whose Redis read throws is skipped rather than failing the whole
+// scrape. V1 let it reject `registry.metrics()`, which turns one Redis
+// blip into a 500 on /metrics and a gap in every panel; core's buffer
+// gauges already swallow the same way (buffers/buffer.metrics.ts).
+
+import type { Queue as BullQueue } from 'bullmq';
+import client from 'prom-client';
+import { registry } from '../metrics';
+
+const JOB_DURATION_BUCKETS_MS = [
+ 10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10_000, 30_000,
+];
+
+// A Redis queue key is not a legal prom-client metric name: `{cron}`'s braces
+// and a `QUEUE_NAMESPACE`'s `-` are both rejected by prom-client, which throws
+// at construction and takes boot down. V1 stripped the braces the same way
+// (apps/worker/src/metrics.ts); the `_` substitution is the namespace half,
+// and it changes nothing for a deployment that sets no namespace.
+const QUEUE_BRACES = /[{}]/g;
+const NON_METRIC_NAME_CHARS = /[^a-zA-Z0-9_]/g;
+
+function metricPrefix(queueName: string): string {
+ return queueName
+ .replace(QUEUE_BRACES, '')
+ .replace(NON_METRIC_NAME_CHARS, '_');
+}
+
+/**
+ * FAILURE-ONLY, deliberately. V1 observed this exclusively in the workers'
+ * `failed` listener (apps/worker/src/boot-workers.ts), so every percentile
+ * ever computed from it is a percentile of failures. Success timings, if
+ * wanted, get a NEW series name — emitting them here silently rewrites every
+ * existing panel (ADR-018 risk 7).
+ */
+export const jobDurationMs = new client.Histogram({
+ name: 'job_duration_ms',
+ help: 'Duration of job processing (in ms)',
+ labelNames: ['name', 'status'],
+ buckets: JOB_DURATION_BUCKETS_MS,
+ registers: [registry],
+});
+
+/** The `Queue` surface these gauges need — narrowed so a test can fake it. */
+export interface CountableQueue {
+ name: string;
+ getActiveCount(): Promise<number>;
+ getDelayedCount(): Promise<number>;
+ getFailedCount(): Promise<number>;
+ getCompletedCount(): Promise<number>;
+ getWaitingCount(): Promise<number>;
+}
+
+const COUNTS: {
+ suffix: string;
+ help: string;
+ read: (queue: CountableQueue) => Promise<number>;
+}[] = [
+ {
+ suffix: 'active',
+ help: 'Active count',
+ read: (queue) => queue.getActiveCount(),
+ },
+ {
+ suffix: 'delayed',
+ help: 'Delayed count',
+ read: (queue) => queue.getDelayedCount(),
+ },
+ {
+ suffix: 'failed',
+ help: 'Failed count',
+ read: (queue) => queue.getFailedCount(),
+ },
+ {
+ suffix: 'completed',
+ help: 'Completed count',
+ read: (queue) => queue.getCompletedCount(),
+ },
+ {
+ suffix: 'waiting',
+ help: 'Waiting count',
+ read: (queue) => queue.getWaitingCount(),
+ },
+];
+
+/**
+ * Five scrape-time gauges per queue, named from the queue's Redis key exactly
+ * as V1 named them. Call once per process, only where the role consumes.
+ *
+ * `register` defaults to the one core registry; it is a parameter so a test
+ * can assert on an isolated one.
+ */
+export function registerQueueMetrics(
+ queues: (CountableQueue | BullQueue)[],
+ register: client.Registry = registry
+): void {
+ for (const queue of queues as CountableQueue[]) {
+ const prefix = metricPrefix(queue.name);
+ for (const count of COUNTS) {
+ // `registers` explicitly, never prom-client's global default: a gauge
+ // constructed without it lands on `client.register` too, which is a
+ // second registry by any other name.
+ new client.Gauge({
+ name: `${prefix}_${count.suffix}_count`,
+ help: count.help,
+ registers: [register],
+ async collect() {
+ try {
+ this.set(await count.read(queue));
+ } catch {
+ // ignore — a Redis blip must not fail the whole scrape
+ }
+ },
+ });
+ }
+ }
+}
diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.ts
index 3a9e77ac7..0ef918778 100644
--- a/packages/core/src/jobs/schedulers.ts
+++ b/packages/core/src/jobs/schedulers.ts
@@ -43,9 +43,11 @@ export interface SchedulerQueue {
jobTemplate?: { data?: unknown }
): Promise<unknown>;
removeJobScheduler(id: string): Promise<boolean>;
+ // Mutable, not `readonly`: BullMQ's own `getJobs` declares `JobType[]`, and
+ // a readonly parameter here would make the real `Queue` unassignable.
getJobs(
- types: readonly ('delayed' | 'waiting' | 'completed' | 'failed')[]
- ): Promise<{ id?: string | null; remove(): Promise<void> }[]>;
+ types: ('delayed' | 'waiting' | 'completed' | 'failed')[]
+ ): Promise<{ id?: string | null; remove(): Promise<unknown> }[]>;
}
const CONFLICT_JOB_STATES = [
diff --git a/packages/core/src/jobs/workers.ts b/packages/core/src/jobs/workers.ts
index 96ecd9ac9..135ed46a8 100644
--- a/packages/core/src/jobs/workers.ts
+++ b/packages/core/src/jobs/workers.ts
@@ -9,6 +9,7 @@ import { generateId } from '../shared/id';
import { resolveJob } from './compat';
import type { QueueDefinition, QueueMap } from './define';
import { isEnvelope } from './envelope';
+import { jobDurationMs } from './jobs.metrics';
import { queueKey } from './naming';
export interface TerminalFailure {
@@ -91,6 +92,16 @@ export function startWorkers<TQueues extends QueueMap>({
return;
}
+ // FAILURE-ONLY, and the label is the Redis key, both V1's
+ // (apps/worker/src/boot-workers.ts's `failed` listener). See
+ // jobs.metrics.ts for why a success timing may not join this series.
+ if (job.processedOn) {
+ jobDurationMs.observe(
+ { name: key, status: 'failed' },
+ job.finishedOn - job.processedOn
+ );
+ }
+
logger.error(
{
err: error,
diff --git a/packages/core/src/metrics.ts b/packages/core/src/metrics.ts
index 09531311d..e469991a3 100644
--- a/packages/core/src/metrics.ts
+++ b/packages/core/src/metrics.ts
@@ -10,3 +10,24 @@
import client from 'prom-client';
export const registry = new client.Registry();
+
+// `collectDefaultMetrics` registers a fixed set of series names; prom-client
+// throws on a duplicate, so a second call would take the process down at boot.
+let defaultMetricsRegistered = false;
+
+/**
+ * prom-client's Node-derived process/GC collectors, on the one registry.
+ *
+ * V1 registered these twice — `fastify-metrics` did it for the api
+ * (`defaultMetrics.enabled` defaults true) and `apps/worker/src/index.ts`
+ * called `collectDefaultMetrics` itself. V2 has one registry per process and
+ * calls this once, in every role (TARGET_ARCHITECTURE §18: "HTTP and default
+ * metrics register everywhere").
+ */
+export function registerDefaultMetrics(): void {
+ if (defaultMetricsRegistered) {
+ return;
+ }
+ defaultMetricsRegistered = true;
+ client.collectDefaultMetrics({ register: registry });
+}
diff --git a/packages/core/src/modules/health/health.routes.ts b/packages/core/src/modules/health/health.routes.ts
index 7c5fd215e..cf6eeddb2 100644
--- a/packages/core/src/modules/health/health.routes.ts
+++ b/packages/core/src/modules/health/health.routes.ts
@@ -1,11 +1,17 @@
-// Kubernetes liveness — shallow, event loop only. Ported from V1's
-// controllers/healthcheck.controller.ts `liveness`; `/healthcheck` and
-// `/healthz/ready`'s dependency/shutdown checks return once db, ch, redis and
-// a shutdown flag are real (P3+) — porting them against `unknown` deps now
-// would be a fake check, not a deferred one.
+// Kubernetes liveness and readiness. Ported from V1's
+// controllers/healthcheck.controller.ts `liveness`/`readiness` and
+// apps/worker/src/index.ts's two probes, which agreed on both bodies.
+//
+// `/healthcheck` — the deep db/ch/redis probe — still waits for real clients
+// on `AppDeps` (P3+); porting it against `unknown` deps would be a fake check,
+// not a deferred one. `/healthz/ready` does NOT wait for them: its two inputs
+// are the shutdown flag and the events-consumer heartbeat, both of which are
+// real from P9 (M9-002), and a readiness probe that ignores a draining process
+// is the one that actually loses requests.
import { z } from 'zod';
import { defineRoutes } from '../../http/define';
+import { currentReadiness } from './src/readiness';
// The first zod schema through the OpenAPI plugin (ADR-003): response schemas
// generate with `io: 'output'`, which is what zod 4's native Standard Schema
@@ -14,14 +20,40 @@ import { defineRoutes } from '../../http/define';
// no `io` argument and would collapse the input/output split the ADR requires.
const healthLiveResponseSchema = z.object({ live: z.literal(true) });
+// V1's two readiness bodies, verbatim: `{ready:true}` at 200, and
+// `{ready:false, reason, idleMs?, thresholdMs?}` at 503.
+const healthReadyResponseSchema = z.object({
+ ready: z.boolean(),
+ reason: z.string().optional(),
+ idleMs: z.number().optional(),
+ thresholdMs: z.number().optional(),
+});
+
+const SERVICE_UNAVAILABLE = 503;
+
// Hidden from the OpenAPI document — ported from V1's `schema: { hide: true }`
// on `/healthz/live`, `/healthz/ready` and `/healthcheck`
// (healthcheck.controller.ts): liveness probes are not part of the public API
// contract. See verification/golden/openapi's migrated-route gate, which pins
// `/healthz` to an empty `paths` object against V1's golden document.
export const healthRoutes = defineRoutes((app) =>
- app.get('/healthz/live', () => ({ live: true as const }), {
- response: healthLiveResponseSchema,
- detail: { hide: true },
- })
+ app
+ .get('/healthz/live', () => ({ live: true as const }), {
+ response: healthLiveResponseSchema,
+ detail: { hide: true },
+ })
+ .get(
+ '/healthz/ready',
+ ({ set }) => {
+ const readiness = currentReadiness();
+ if (!readiness.ready) {
+ set.status = SERVICE_UNAVAILABLE;
+ }
+ return readiness;
+ },
+ {
+ response: healthReadyResponseSchema,
+ detail: { hide: true },
+ }
+ )
);
diff --git a/packages/core/src/modules/health/src/readiness.test.ts b/packages/core/src/modules/health/src/readiness.test.ts
new file mode 100644
index 000000000..4f9a4e049
--- /dev/null
+++ b/packages/core/src/modules/health/src/readiness.test.ts
@@ -0,0 +1,58 @@
+import { expect, test } from 'bun:test';
+import { EVENTS_HEARTBEAT_STALE_MS, evaluateReadiness } from './readiness';
+
+const NOW = 1_000_000;
+const fresh = { enabled: true, lastActivityAt: NOW };
+
+test('shutting down wins over everything else', () => {
+ expect(
+ evaluateReadiness({ shuttingDown: true, heartbeat: fresh, now: NOW })
+ ).toEqual({ ready: false, reason: 'shutting down' });
+});
+
+test('no events consumer on this instance skips the heartbeat check', () => {
+ expect(
+ evaluateReadiness({
+ shuttingDown: false,
+ // V1 left the heartbeat disabled on instances with no events worker,
+ // whose lastActivityAt then never moves off boot time.
+ heartbeat: { enabled: false, lastActivityAt: 0 },
+ now: NOW,
+ })
+ ).toEqual({ ready: true });
+});
+
+test('a fresh consumer heartbeat is ready', () => {
+ expect(
+ evaluateReadiness({ shuttingDown: false, heartbeat: fresh, now: NOW })
+ ).toEqual({ ready: true });
+});
+
+test('a stale consumer heartbeat is 503 with the numbers that explain it', () => {
+ const idleMs = EVENTS_HEARTBEAT_STALE_MS + 1;
+ expect(
+ evaluateReadiness({
+ shuttingDown: false,
+ heartbeat: { enabled: true, lastActivityAt: NOW - idleMs },
+ now: NOW,
+ })
+ ).toEqual({
+ ready: false,
+ reason: 'events consumer heartbeat stale',
+ idleMs,
+ thresholdMs: EVENTS_HEARTBEAT_STALE_MS,
+ });
+});
+
+test('exactly at the threshold is still ready — V1 compared with >', () => {
+ expect(
+ evaluateReadiness({
+ shuttingDown: false,
+ heartbeat: {
+ enabled: true,
+ lastActivityAt: NOW - EVENTS_HEARTBEAT_STALE_MS,
+ },
+ now: NOW,
+ })
+ ).toEqual({ ready: true });
+});
diff --git a/packages/core/src/modules/health/src/readiness.ts b/packages/core/src/modules/health/src/readiness.ts
new file mode 100644
index 000000000..aba6ef422
--- /dev/null
+++ b/packages/core/src/modules/health/src/readiness.ts
@@ -0,0 +1,58 @@
+// V1's `/healthz/ready` decision, lifted out of the express handler
+// (apps/worker/src/index.ts) so it can be asserted without a server.
+//
+// Shallow + shutdown-aware, and — only where this process runs the events
+// consumer — heartbeat-aware. V1 skipped the heartbeat check on instances
+// with no events worker; V2 keeps that by leaving `enabled` false unless the
+// role actually started the consumer.
+
+import { getEventsHeartbeat } from '../../ingest/src/heartbeat';
+import { isShuttingDown } from './shutdown';
+
+/** V1's threshold (apps/worker/src/index.ts `EVENTS_HEARTBEAT_STALE_MS`). */
+export const EVENTS_HEARTBEAT_STALE_MS = 60_000;
+
+export type ReadinessResult =
+ | { ready: true }
+ | { ready: false; reason: string; idleMs?: number; thresholdMs?: number };
+
+export interface ReadinessInputs {
+ shuttingDown: boolean;
+ heartbeat: { enabled: boolean; lastActivityAt: number };
+ now: number;
+}
+
+/** Pure: every branch is a value, so the probe's rules are unit-testable. */
+export function evaluateReadiness({
+ shuttingDown,
+ heartbeat,
+ now,
+}: ReadinessInputs): ReadinessResult {
+ if (shuttingDown) {
+ return { ready: false, reason: 'shutting down' };
+ }
+
+ if (!heartbeat.enabled) {
+ return { ready: true };
+ }
+
+ const idleMs = now - heartbeat.lastActivityAt;
+ if (idleMs > EVENTS_HEARTBEAT_STALE_MS) {
+ return {
+ ready: false,
+ reason: 'events consumer heartbeat stale',
+ idleMs,
+ thresholdMs: EVENTS_HEARTBEAT_STALE_MS,
+ };
+ }
+
+ return { ready: true };
+}
+
+export function currentReadiness(): ReadinessResult {
+ return evaluateReadiness({
+ shuttingDown: isShuttingDown(),
+ heartbeat: getEventsHeartbeat(),
+ now: Date.now(),
+ });
+}
diff --git a/packages/core/src/modules/health/src/shutdown.ts b/packages/core/src/modules/health/src/shutdown.ts
new file mode 100644
index 000000000..8ca9850bd
--- /dev/null
+++ b/packages/core/src/modules/health/src/shutdown.ts
@@ -0,0 +1,14 @@
+// Ported from apps/worker/src/utils/graceful-shutdown.ts (P9). One flag, set
+// by main.ts's signal handler, read by the readiness probe — so a draining
+// process is taken out of the load balancer before its in-flight jobs finish
+// rather than after.
+
+let shuttingDown = false;
+
+export function setShuttingDown(value: boolean): void {
+ shuttingDown = value;
+}
+
+export function isShuttingDown(): boolean {
+ return shuttingDown;
+}
diff --git a/packages/core/src/modules/ingest/src/heartbeat.ts b/packages/core/src/modules/ingest/src/heartbeat.ts
new file mode 100644
index 000000000..ae3c11842
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/heartbeat.ts
@@ -0,0 +1,30 @@
+// The events consumer-loop heartbeat, ported verbatim from
+// apps/worker/src/utils/worker-heartbeat.ts (P9). It is what turns "the
+// process is up" into "the Kafka consumer is still turning": the timestamp is
+// refreshed on every kafkajs HEARTBEAT and after every handled batch
+// (consumer.ts's `onActivity`), so a healthy loop refreshes it even with no
+// traffic, and a wedged one goes stale.
+//
+// Module state rather than a field on `AppDeps` because the readiness probe
+// and the consumer are on opposite sides of the process and neither owns the
+// other; there is exactly one consumer per process.
+
+let enabled = false;
+let lastActivityAt = Date.now();
+
+/** Called only by a role that actually starts the consumer. */
+export function enableEventsHeartbeat(): void {
+ enabled = true;
+ lastActivityAt = Date.now();
+}
+
+export function markEventsActivity(): void {
+ lastActivityAt = Date.now();
+}
+
+export function getEventsHeartbeat(): {
+ enabled: boolean;
+ lastActivityAt: number;
+} {
+ return { enabled, lastActivityAt };
+}
diff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.ts
index 2c4762133..366996c3d 100644
--- a/packages/core/src/modules/ingest/src/ingest.metrics.ts
+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts
@@ -7,6 +7,7 @@
import client from 'prom-client';
import { registry } from '../../../metrics';
+import type { ConsumerMetrics, DeadLetterReason } from './consumer';
// Kafka event messages reprocessed (same offset redelivered outside a
// rebalance). Should stay ~0. A sustained non-zero rate means the consumer is
@@ -52,3 +53,18 @@ export const sessionsStarted = new client.Counter({
labelNames: ['kind'], // 'new' | 'boundary'
registers: [registry],
});
+
+/**
+ * The consumer's `ConsumerMetrics`, bound to the counters above. Exists so
+ * `main.ts` wires the consumer without reaching for four individual counters
+ * — a metric is a module's business, not the entrypoint's.
+ */
+export const ingestConsumerMetrics = {
+ reprocessed: (partition: string) => kafkaReprocessedTotal.inc({ partition }),
+ handlerFailed: (partition: string) =>
+ kafkaHandlerFailuresTotal.inc({ partition }),
+ deadLettered: (partition: string, reason: DeadLetterReason) =>
+ kafkaDeadLetteredTotal.inc({ partition, reason }),
+ deadLetterFailed: (partition: string) =>
+ kafkaDeadLetterFailedTotal.inc({ partition }),
+} satisfies ConsumerMetrics;
diff --git a/packages/core/src/modules/session/src/session.metrics.scrape.test.ts b/packages/core/src/modules/session/src/session.metrics.scrape.test.ts
new file mode 100644
index 000000000..136b4aa8f
--- /dev/null
+++ b/packages/core/src/modules/session/src/session.metrics.scrape.test.ts
@@ -0,0 +1,78 @@
+// The three at-scrape gauges do arithmetic over a Redis pipeline result, which
+// is the half a name-only assertion would not catch.
+import { expect, test } from 'bun:test';
+import client from 'prom-client';
+import {
+ registerSessionScrapeMetrics,
+ type SessionMetricsRedis,
+} from './session.metrics';
+
+type Reply = [Error | null, unknown][] | null;
+
+function fakeRedis(projects: string[], replies: Reply): SessionMetricsRedis {
+ return {
+ smembers: () => Promise.resolve(projects),
+ scard: () => Promise.resolve(projects.length),
+ multi: () => ({
+ zcard: () => undefined,
+ get: () => undefined,
+ exec: () => Promise.resolve(replies),
+ }),
+ };
+}
+
+test('sessions_active_total sums the per-project ZCARDs', async () => {
+ const register = new client.Registry();
+ registerSessionScrapeMetrics(
+ () =>
+ fakeRedis(
+ ['proj_a', 'proj_b'],
+ [
+ [null, 3],
+ [null, 4],
+ ]
+ ),
+ register
+ );
+
+ const text = await register.metrics();
+ expect(text).toContain('sessions_active_total 7');
+ expect(text).toContain('sessions_projects_active 2');
+});
+
+test('no active projects reads zero rather than skipping the sample', async () => {
+ const register = new client.Registry();
+ registerSessionScrapeMetrics(() => fakeRedis([], null), register);
+
+ const text = await register.metrics();
+ expect(text).toContain('sessions_active_total 0');
+ expect(text).toContain('sessions_hwm_lag_ms 0');
+});
+
+test('sessions_hwm_lag_ms takes the MAX lag and ignores unset HWMs', async () => {
+ const register = new client.Registry();
+ const now = Date.now();
+ registerSessionScrapeMetrics(
+ () =>
+ fakeRedis(
+ ['proj_a', 'proj_b', 'proj_c'],
+ [
+ [null, String(now - 5000)],
+ // 0 / null means the project has no HWM yet — not a lag of `now`.
+ [null, '0'],
+ [null, String(now - 60_000)],
+ ]
+ ),
+ register
+ );
+
+ const text = await register.metrics();
+ const lag = Number(
+ text
+ .split('\n')
+ .find((line) => line.startsWith('sessions_hwm_lag_ms '))
+ ?.split(' ')[1]
+ );
+ expect(lag).toBeGreaterThanOrEqual(60_000);
+ expect(lag).toBeLessThan(70_000);
+});
diff --git a/packages/core/src/modules/session/src/session.metrics.ts b/packages/core/src/modules/session/src/session.metrics.ts
index 38b8050b4..ca6ce7ff4 100644
--- a/packages/core/src/modules/session/src/session.metrics.ts
+++ b/packages/core/src/modules/session/src/session.metrics.ts
@@ -1,12 +1,31 @@
// The session lifecycle collectors, moved from apps/worker/src/metrics.ts
// (M7-001) onto core's one registry. Names, labels and buckets are V1's; the
-// worker-side copies die with apps/worker (P9). The at-scrape gauges
-// (`sessions_active_total`, `sessions_projects_active`, `sessions_hwm_lag_ms`)
-// and the ingest-side `sessions_started_total` stay with the ingest path.
+// worker-side copies die with apps/worker (P9). The ingest-side
+// `sessions_started_total` lives with the ingest path
+// (modules/ingest/src/ingest.metrics.ts); the three at-scrape gauges joined
+// this file at M9-002 — see `registerSessionScrapeMetrics` at the bottom.
import client from 'prom-client';
import { registry } from '../../../metrics';
+// V1's key names, unchanged (apps/worker/src/metrics.ts).
+const SESSION_PROJECTS_KEY = 'session:projects';
+
+/**
+ * The slice of the cache client the gauges use. `multi()` is V1's, and it is
+ * batching rather than a transaction — the ADR-006 swap replaces it with
+ * `Promise.all` over Bun's auto-pipelining when packages/redis moves.
+ */
+export interface SessionMetricsRedis {
+ smembers(key: string): Promise<string[]>;
+ scard(key: string): Promise<number>;
+ multi(): {
+ zcard(key: string): unknown;
+ get(key: string): unknown;
+ exec(): Promise<[Error | null, unknown][] | null>;
+ };
+}
+
const SECOND_MS = 1000;
const MINUTE_MS = 60 * SECOND_MS;
const HOUR_MS = 60 * MINUTE_MS;
@@ -75,3 +94,88 @@ export const sessionsVacuumed = new client.Counter({
labelNames: ['reason'], // 'stale_blob' | 'missing_blob'
registers: [registry],
});
+
+/**
+ * The three scrape-time session gauges, moved from apps/worker/src/metrics.ts.
+ *
+ * Registered by main.ts and ONLY where the role consumes queues: each scrape
+ * costs one `ZCARD` and one `GET` per project, and ten api replicas exposing
+ * them would multiply that Redis load for no new information
+ * (TARGET_ARCHITECTURE §18).
+ *
+ * The client is injected rather than imported so core does not open a Redis
+ * connection at import time — `main.ts` hands in `getRedisCache`.
+ */
+export function registerSessionScrapeMetrics(
+ getRedis: () => SessionMetricsRedis,
+ register: client.Registry = registry
+): void {
+ // `registers` explicitly, never prom-client's global default — see
+ // jobs/jobs.metrics.ts.
+ const registerOn = [register];
+
+ new client.Gauge({
+ name: 'sessions_active_total',
+ help: 'Active sessions in Redis across all projects',
+ registers: registerOn,
+ async collect() {
+ const redis = getRedis();
+ const projectIds = await redis.smembers(SESSION_PROJECTS_KEY);
+ if (projectIds.length === 0) {
+ this.set(0);
+ return;
+ }
+ const multi = redis.multi();
+ for (const projectId of projectIds) {
+ multi.zcard(`session:wallclock:${projectId}`);
+ }
+ const results = await multi.exec();
+ let total = 0;
+ for (const entry of results ?? []) {
+ const count = Number(entry?.[1] ?? 0);
+ if (Number.isFinite(count)) {
+ total += count;
+ }
+ }
+ this.set(total);
+ },
+ });
+
+ new client.Gauge({
+ name: 'sessions_projects_active',
+ help: 'Projects with at least one active session',
+ registers: registerOn,
+ async collect() {
+ this.set(await getRedis().scard(SESSION_PROJECTS_KEY));
+ },
+ });
+
+ new client.Gauge({
+ name: 'sessions_hwm_lag_ms',
+ help: 'Max lag (ms) between wall-clock now and project event-time HWM, across all projects. Big number → queue lag or imports.',
+ registers: registerOn,
+ async collect() {
+ const redis = getRedis();
+ const projectIds = await redis.smembers(SESSION_PROJECTS_KEY);
+ if (projectIds.length === 0) {
+ this.set(0);
+ return;
+ }
+ const multi = redis.multi();
+ for (const projectId of projectIds) {
+ multi.get(`session:hwm:${projectId}`);
+ }
+ const results = await multi.exec();
+ const now = Date.now();
+ let maxLag = 0;
+ for (const entry of results ?? []) {
+ const hwm = Number(entry?.[1] ?? 0);
+ if (!Number.isFinite(hwm) || hwm <= 0) {
+ continue;
+ }
+ maxLag = Math.max(maxLag, now - hwm);
+ }
+ this.set(maxLag);
+ },
+ });
+}
diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.ts
index 80563b023..e6496dba3 100644
--- a/packages/core/src/rest.routes.ts
+++ b/packages/core/src/rest.routes.ts
@@ -7,6 +7,7 @@
import { openapi } from '@elysiajs/openapi';
import { Elysia } from 'elysia';
import type { AppDeps } from './context';
+import { httpMetrics } from './http/http.metrics';
import { registry } from './metrics';
import { assistantRoutes } from './modules/assistant/assistant.routes';
import { authRoutes } from './modules/auth/auth.routes';
@@ -90,6 +91,11 @@ export const dashboardRoutes = (deps: AppDeps) =>
.use(realtimeRoutes(deps))
.use(miscRoutes(deps));
+// `httpMetrics` is on the ops surface because that is where `/metrics` is
+// served and every role mounts it (TARGET_ARCHITECTURE §18: "HTTP and default
+// metrics register everywhere"). Its hook is `{ as: 'global' }`, so it covers
+// the dashboard and public-API surfaces too once they mount.
+//
// healthz/metrics/misc share V1's ops surface (http/context.ts's
// UNLOGGED_PATH_PREFIXES) — unauthenticated, uncorsed, unlogged. /metrics and
// /openapi.json are the M3 gate's ops surface: the one core registry's
@@ -97,6 +103,7 @@ export const dashboardRoutes = (deps: AppDeps) =>
// schemas the routes validate against.
export const opsRoutes = (deps: AppDeps) =>
new Elysia({ name: 'core/ops-routes' })
+ .use(httpMetrics())
.use(healthRoutes(deps))
.use(
openapi({
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 406021958..37b478f1d 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1079,6 +1079,12 @@ importers:
'@better-agent/providers':
specifier: ^0.1.0-beta.3
version: 0.1.0-beta.3(@better-agent/[email protected])
+ '@bull-board/api':
+ specifier: 9.6.0
+ '@bull-board/elysia':
+ specifier: 9.6.0
'@elysiajs/cors':
specifier: 1.4.2
@@ -3273,12 +3279,32 @@ packages:
'@bull-board/ui': 6.14.0
bullmq: '*'
+ '@bull-board/[email protected]':
+ resolution: {integrity: sha512-RJcXIaFDO2l76t1MNxhIiKNaDIKIm4F8vsQq32grV9Rw4DiclKFlXSht2SN/+VIVyHJ7NzlCskMd8HgMpb/aEg==}
+ peerDependencies:
+ '@bull-board/ui': 9.6.0
+ bull: ^4.16.5
+ bullmq: ^5.56.0 || ^6.0.0
+ peerDependenciesMeta:
+ bull:
+ optional: true
+ bullmq:
+ optional: true
+
+ '@bull-board/[email protected]':
+ resolution: {integrity: sha512-rWfOfleHGwIgXfkznO2MMwSGEMOP78cw0jmQINL946w66J9z532JZoRvgmV8FPZM+NZghvtzVT7AsB6/M881Uw==}
+ peerDependencies:
+ elysia: ^1.1.0
+
'@bull-board/[email protected]':
resolution: {integrity: sha512-3H1ame2G1+eVnqqSsw6KfzTGYAWSpVsIx6EPwg9vPSP2eKfNAm12Cm4zvL6ZkwAvTCkAByt5PPDRWbbwWB6HHQ==}
'@bull-board/[email protected]':
resolution: {integrity: sha512-5yqfS9CwWR8DBxpReIbqv/VSPFM/zT4KZ75keyApMiejasRC2joaHqEzYWlMCjkMycbNNCvlQNlTbl+C3dE/dg==}
+ '@bull-board/[email protected]':
+ resolution: {integrity: sha512-rJDD+CvxI43Avq2gZpHunPT67HToYXyHSRoXsiVXDh/6MB93YtU+ofkgrrYBY25FswDSvtfnQN/ID3Ypf9OyOg==}
+
'@capsizecss/[email protected]':
resolution: {integrity: sha512-VERIM64vtTP1C4mxQ5thVT9fK0apjPFobqybMtA1UdUujWka24ERHbRHFGmpbbhp73MhV+KSsHQH9C6uOTdEQA==}
engines: {node: '>=18'}
@@ -11908,6 +11934,11 @@ packages:
engines: {node: '>=0.10.0'}
hasBin: true
+ resolution: {integrity: sha512-UaaM14yby8U3k02ihS1Bmj5Kz2d7CCQM1scxpgs4Mhkq8F1wR2gl3+Ts4h5Ne4Mnt7M9m4Dw7jsuMr3+xO4vZA==}
+ engines: {node: '>=0.12.18'}
+ hasBin: true
+
resolution: {integrity: sha512-NlAsMteRHek05jRUxUR0a5jpjYq9ykk6+kO0yRaMi5moe7u0fVIOeQ3Y30A8dIiWFBNUoQGi1ljb1i5VtS9WQQ==}
@@ -22476,6 +22507,24 @@ snapshots:
bullmq: 5.63.0([email protected])
redis-info: 3.1.0
+ dependencies:
+ redis-info: 3.1.0
+ optionalDependencies:
+
+ dependencies:
+ ejs: 6.0.1
+ mime: 4.1.0
+ transitivePeerDependencies:
+ - bull
+ - bullmq
+
dependencies:
@@ -22492,6 +22541,13 @@ snapshots:
transitivePeerDependencies:
- bullmq
+ dependencies:
+ transitivePeerDependencies:
+ - bull
+ - bullmq
+
'@capsizecss/[email protected]':
dependencies:
fontkitten: 1.0.3
@@ -32269,6 +32325,8 @@ snapshots:
dependencies:
jake: 10.8.7
+