OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M9-002): main.ts by ROLE: startWorkers + startSchedulers + ingest consumer + bull-board + one metrics registry
ralph6fc8851f2dd3
diff --git a/apps/api/e2e/boot-proof.sh b/apps/api/e2e/boot-proof.shindex 1ef2c44eb..b999ae51c 100755--- a/apps/api/e2e/boot-proof.sh+++ b/apps/api/e2e/boot-proof.sh@@ -1,12 +1,35 @@#!/usr/bin/env bash-# M3-CLOSE-002 verification: proves `bun run apps/api/src/main.ts` actually-# boots, rather than merely typechecking. Three assertions:+# Proves `bun run apps/api/src/main.ts` actually boots — every ROLE the ROLE+# table declares — rather than merely typechecking (playbook rule 1).#-# 1. ROLE=api serves healthz 200 and /metrics 200 on a configurable port.-# 2. An unknown ROLE fails boot loudly (non-zero exit, named value in the-# message) — same doctrine as ENABLED_QUEUES.+# M3-CLOSE-002 wrote parts 1-3; M9-002 added parts 4-5.+#+# 1. An unknown ROLE fails boot loudly (non-zero exit, named value in the+# message) — and so does an unknown ENABLED_QUEUES token.+# 2. ROLE=api serves /healthz/live, /healthz/ready and /metrics, runs NO+# workers/schedulers/consumer, and mounts neither bull-board nor the+# debug routes.# 3. SIGTERM shuts the process down within 5s, exit code 0, no orphan left# behind.+# 4. ROLE=worker starts all seven BullMQ workers, the cron schedulers and+# the Kafka ingest consumer; exposes the consuming-role scrape gauges;+# serves bull-board behind the session guard (401, not 200 and not 404).+# 5. ROLE=all does everything ROLE=worker does AND sets redis+# keyspace-notify, which ROLE=worker must not.+# 6. The two opt-outs: DISABLE_BULLBOARD unmounts the UI, DISABLE_WORKERS+# leaves the ops surface up with nothing consuming.+#+# Parts 4-5 need the real backing services, so they source the repo `.env`+# and are ISOLATED from whatever else uses this box:+#+# * QUEUE_NAMESPACE suffixes every BullMQ key, so the proof cannot consume+# or prune the live `cron`/`sessions`/... keys. The un-namespaced keys+# byte-identity is pinned separately by+# packages/core/src/jobs/naming.test.ts and the controllers queue-keys+# golden.+# * KAFKA_CONSUMER_GROUP is a proof-only group, so joining does not+# rebalance the real `openpanel-events` consumers. The topic is the real+# one and is only read from `latest`.## Run from anywhere: `bash apps/api/e2e/boot-proof.sh`.@@ -25,12 +48,28 @@ fail() {FAILURES=$((FAILURES + 1))}+assert_contains() {+ if printf '%s' "$1" | grep -qF -- "$2"; then+ pass "$3"+ else+ fail "$3 (missing: $2)"+ fi+}++assert_absent() {+ if printf '%s' "$1" | grep -qF -- "$2"; then+ fail "$3 (unexpectedly present: $2)"+ else+ pass "$3"+ fi+}+# A real free port, not a guessed constant — this box runs other processes.find_free_port() {bun -e 'const s = Bun.listen({ port: 0, hostname: "127.0.0.1", socket: { data() {} } }); console.log(s.port); s.stop();'}-# --- Part 1: unknown ROLE fails boot loudly ------------------------------+# --- Part 1: an unknown ROLE / ENABLED_QUEUES fails boot loudly ----------UNKNOWN_LOG="$(mktemp)"ROLE=bogus API_PORT=0 bun run "$MAIN_TS" >"$UNKNOWN_LOG" 2>&1@@ -50,7 +89,20 @@ elsefirm -f "$UNKNOWN_LOG"-# --- Part 2: ROLE=api boots and serves healthz + /metrics ----------------+QUEUES_LOG="$(mktemp)"+ROLE=worker ENABLED_QUEUES=events_kafka API_PORT=0 bun run "$MAIN_TS" >"$QUEUES_LOG" 2>&1+QUEUES_EXIT=$?++if [ "$QUEUES_EXIT" -ne 0 ] && grep -q 'events_kafka' "$QUEUES_LOG" \+ && grep -q 'was renamed to' "$QUEUES_LOG"; then+ pass "ENABLED_QUEUES=events_kafka exits non-zero and names the rename"+else+ fail "ENABLED_QUEUES=events_kafka should exit non-zero naming the rename (exit $QUEUES_EXIT) — log follows:"+ cat "$QUEUES_LOG"+fi+rm -f "$QUEUES_LOG"++# --- Part 2: ROLE=api boots and serves the ops surface -------------------PORT="$(find_free_port)"BOOT_LOG="$(mktemp)"@@ -87,6 +139,13 @@ elsefail "GET /healthz/live -> $HEALTHZ_CODE, expected 200"fi+READY_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/healthz/ready")+if [ "$READY_CODE" = "200" ]; then+ pass "GET /healthz/ready -> 200 (ROLE=api: no consumer, so no heartbeat gate)"+else+ fail "GET /healthz/ready -> $READY_CODE, expected 200"+fi+METRICS_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/metrics")if [ "$METRICS_CODE" = "200" ]; thenpass "GET /metrics -> 200"@@ -94,6 +153,44 @@ elsefail "GET /metrics -> $METRICS_CODE, expected 200"fi+API_METRICS="$(curl -s "http://127.0.0.1:$PORT/metrics")"+assert_contains "$API_METRICS" 'process_cpu_seconds_total' \+ "ROLE=api /metrics carries the default process metrics"+assert_contains "$API_METRICS" 'http_request_duration_seconds' \+ "ROLE=api /metrics carries the route histogram"+assert_contains "$API_METRICS" 'http_request_summary_seconds' \+ "ROLE=api /metrics carries the route summary"+# One registry: the queue/buffer/session scrape collectors register ONLY where+# the role consumes (TARGET_ARCHITECTURE 18).+assert_absent "$API_METRICS" 'sessions_waiting_count' \+ "ROLE=api does NOT register the queue gauges"+assert_absent "$API_METRICS" 'sessions_active_total' \+ "ROLE=api does NOT register the session scrape gauges"+assert_absent "$API_METRICS" 'buffer_event_count' \+ "ROLE=api does NOT register the buffer gauges"++BOARD_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/bullboard")+if [ "$BOARD_CODE" = "404" ]; then+ pass "ROLE=api does NOT mount bull-board (GET /bullboard -> 404)"+else+ fail "ROLE=api should not mount bull-board, got $BOARD_CODE"+fi++if grep -q '"msg":"worker started"' "$BOOT_LOG"; then+ fail "ROLE=api must not start workers — log follows:"+ cat "$BOOT_LOG"+else+ pass "ROLE=api starts no workers, schedulers or ingest consumer"+fi++# The debug routes lived on the worker in V1; ROLE=api never had them.+DEBUG_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/debug/cron")+if [ "$DEBUG_CODE" = "404" ]; then+ pass "ROLE=api does NOT mount the debug routes (GET /debug/cron -> 404)"+else+ fail "ROLE=api should not mount the debug routes, got $DEBUG_CODE"+fi+# --- Part 3: SIGTERM shuts down within 5s, exit 0, no orphan -------------kill -TERM "$BOOT_PID"@@ -135,6 +232,289 @@ firm -f "$BOOT_LOG"+# --- Parts 4-5: the consuming roles --------------------------------------+#+# These need the real Redis/Kafka/Postgres, so they source `.env`. Isolated+# by QUEUE_NAMESPACE + a proof-only Kafka consumer group (see the header).++if [ ! -f "$REPO_ROOT/.env" ]; then+ fail "$REPO_ROOT/.env is missing — parts 4-5 need REDIS_URL/KAFKA_BROKERS/DATABASE_URL"+ echo+ echo "boot-proof.sh: $FAILURES failure(s)"+ exit 1+fi++PROOF_NAMESPACE="bootproof$$"+PROOF_KAFKA_GROUP="openpanel-events-bootproof-$$"+CONSUMING_READY_TIMEOUT_S=90++# Boots one consuming role, waits for it to listen, and leaves $ROLE_PID and+# $ROLE_LOG set for the caller's assertions.+boot_consuming_role() { boot_consuming_role_with_env "$1"; }++# `boot_consuming_role_with_env <role> [NAME=VALUE ...]`+boot_consuming_role_with_env() {+ local role="$1"+ shift+ ROLE_PORT="$(find_free_port)"+ ROLE_LOG="$(mktemp)"++ (+ set -a+ # shellcheck disable=SC1091+ . "$REPO_ROOT/.env"+ set +a+ export ROLE="$role"+ export API_PORT="$ROLE_PORT"+ export QUEUE_NAMESPACE="$PROOF_NAMESPACE"+ export KAFKA_CONSUMER_GROUP="$PROOF_KAFKA_GROUP"+ for assignment in "$@"; do+ export "${assignment?}"+ done+ exec bun run "$MAIN_TS"+ ) >"$ROLE_LOG" 2>&1 &+ ROLE_PID=$!++ local waited=0+ until curl -s -o /dev/null "http://127.0.0.1:$ROLE_PORT/healthz/live"; do+ if ! kill -0 "$ROLE_PID" 2>/dev/null; then+ fail "ROLE=$role exited before listening — log follows:"+ cat "$ROLE_LOG"+ return 1+ fi+ sleep 1+ waited=$((waited + 1))+ if [ "$waited" -ge "$CONSUMING_READY_TIMEOUT_S" ]; then+ fail "ROLE=$role did not listen within ${CONSUMING_READY_TIMEOUT_S}s — log follows:"+ cat "$ROLE_LOG"+ kill -9 "$ROLE_PID" 2>/dev/null+ return 1+ fi+ done+ pass "ROLE=$role booted and is listening on :$ROLE_PORT (${waited}s)"+ return 0+}++stop_consuming_role() {+ local role="$1"+ kill -TERM "$ROLE_PID" 2>/dev/null+ local waited=0+ while kill -0 "$ROLE_PID" 2>/dev/null; do+ sleep 1+ waited=$((waited + 1))+ if [ "$waited" -ge 30 ]; then+ break+ fi+ done+ if kill -0 "$ROLE_PID" 2>/dev/null; then+ fail "ROLE=$role still alive 30s after SIGTERM — killing it"+ kill -9 "$ROLE_PID" 2>/dev/null+ wait "$ROLE_PID" 2>/dev/null+ else+ wait "$ROLE_PID" 2>/dev/null+ local exit_code=$?+ if [ "$exit_code" -eq 0 ]; then+ pass "ROLE=$role SIGTERM shutdown exit code 0 (${waited}s)"+ else+ fail "ROLE=$role SIGTERM shutdown exit code $exit_code, expected 0"+ fi+ fi+}++# The seven registry queues (jobs.registry.ts) — the same names as the Redis+# keys, `cohortCompute` included.+REGISTRY_QUEUES="sessions cron notification import insights gsc cohortCompute"++assert_consuming_role() {+ local role="$1" log="$2" port="$3"++ local started+ started=$(grep -c '"msg":"worker started"' "$log")+ if [ "$started" -eq 7 ]; then+ pass "ROLE=$role started all 7 BullMQ workers"+ else+ fail "ROLE=$role started $started workers, expected 7"+ fi++ for queue in $REGISTRY_QUEUES; do+ if grep -q "\"key\":\"${queue}-${PROOF_NAMESPACE}\"" "$log"; then+ pass "ROLE=$role worker on queue key ${queue}-${PROOF_NAMESPACE}"+ else+ fail "ROLE=$role has no worker for queue $queue"+ fi+ done++ if grep -q '"msg":"updating cron jobs"' "$log"; then+ pass "ROLE=$role upserted the cron schedulers"+ else+ fail "ROLE=$role did not run startSchedulers"+ fi++ if grep -q '"msg":"kafka events consumer running"' "$log"; then+ pass "ROLE=$role started the Kafka ingest consumer"+ else+ fail "ROLE=$role did not start the Kafka ingest consumer"+ fi++ # M9-003 lands dataHealth/windDown/flushExports. Until then main.ts must SAY+ # they are missing at boot rather than let each tick fail silently.+ if grep -q 'cron schedulers have no handler' "$log"; then+ pass "ROLE=$role names the three scheduler ids with no handler yet (M9-003)"+ else+ fail "ROLE=$role did not report the unhandled schedulers"+ fi++ local ready_code+ ready_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/healthz/ready")+ if [ "$ready_code" = "200" ]; then+ pass "ROLE=$role GET /healthz/ready -> 200 (consumer heartbeat fresh)"+ else+ fail "ROLE=$role GET /healthz/ready -> $ready_code, expected 200"+ fi++ local metrics+ metrics="$(curl -s "http://127.0.0.1:$port/metrics")"+ assert_contains "$metrics" 'process_cpu_seconds_total' \+ "ROLE=$role /metrics carries the default process metrics"+ assert_contains "$metrics" 'http_request_duration_seconds' \+ "ROLE=$role /metrics carries the route histogram"+ assert_contains "$metrics" 'job_duration_ms' \+ "ROLE=$role /metrics carries job_duration_ms"+ assert_contains "$metrics" 'kafka_events_reprocessed_total' \+ "ROLE=$role /metrics carries the kafka counters"+ assert_contains "$metrics" 'sessions_active_total' \+ "ROLE=$role /metrics carries the session scrape gauges"+ assert_contains "$metrics" 'buffer_event_count' \+ "ROLE=$role /metrics carries the buffer gauges"+ for queue in $REGISTRY_QUEUES; do+ assert_contains "$metrics" "${queue}_${PROOF_NAMESPACE}_waiting_count" \+ "ROLE=$role /metrics carries the $queue queue gauges"+ done++ local board_code+ board_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/bullboard")+ if [ "$board_code" = "401" ]; then+ pass "ROLE=$role mounts bull-board BEHIND the session guard (401, not 200)"+ else+ fail "ROLE=$role GET /bullboard -> $board_code, expected 401"+ fi++ local debug_code+ debug_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/debug/cron")+ if [ "$debug_code" = "200" ]; then+ pass "ROLE=$role serves the dev-only debug routes (NODE_ENV is not production)"+ else+ fail "ROLE=$role GET /debug/cron -> $debug_code, expected 200"+ fi+}++echo+echo "--- Part 4: ROLE=worker ---"+if boot_consuming_role worker; then+ WORKER_LOG="$ROLE_LOG"+ assert_consuming_role worker "$WORKER_LOG" "$ROLE_PORT"++ if grep -q 'keyspace notifications configured' "$WORKER_LOG"; then+ fail "ROLE=worker must NOT set redis keyspace notifications"+ else+ pass "ROLE=worker does NOT set redis keyspace notifications"+ fi++ stop_consuming_role worker+ rm -f "$WORKER_LOG"+fi++echo+echo "--- Part 5: ROLE=all ---"+if boot_consuming_role all; then+ ALL_LOG="$ROLE_LOG"+ assert_consuming_role all "$ALL_LOG" "$ROLE_PORT"++ if grep -q 'keyspace notifications configured' "$ALL_LOG"; then+ pass "ROLE=all sets redis keyspace notifications (ROLE != worker)"+ else+ fail "ROLE=all did not set redis keyspace notifications"+ fi++ stop_consuming_role all+ rm -f "$ALL_LOG"+fi++echo+echo "--- Part 6: the DISABLE_* opt-outs ---"+if boot_consuming_role_with_env worker DISABLE_BULLBOARD=1 DISABLE_WORKERS=1; then+ OPTOUT_LOG="$ROLE_LOG"++ board_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$ROLE_PORT/bullboard")+ if [ "$board_code" = "404" ]; then+ pass "DISABLE_BULLBOARD=1 unmounts bull-board (GET /bullboard -> 404)"+ else+ fail "DISABLE_BULLBOARD=1 should unmount bull-board, got $board_code"+ fi++ if grep -q '"msg":"worker started"' "$OPTOUT_LOG"; then+ fail "DISABLE_WORKERS should start no workers"+ else+ pass "DISABLE_WORKERS starts no workers, schedulers or ingest consumer"+ fi++ if grep -q '"msg":"Workers are disabled"' "$OPTOUT_LOG"; then+ pass "DISABLE_WORKERS says so at boot"+ else+ fail "DISABLE_WORKERS did not log why nothing is consuming"+ fi++ live_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$ROLE_PORT/healthz/live")+ if [ "$live_code" = "200" ]; then+ pass "DISABLE_WORKERS still serves the ops surface"+ else+ fail "DISABLE_WORKERS ops surface -> $live_code, expected 200"+ fi++ stop_consuming_role worker+ rm -f "$OPTOUT_LOG"+fi++# The namespaced BullMQ keys and their job schedulers are this proof's litter;+# an `every` scheduler left in Redis would keep firing forever.+echo+(+ # From apps/api, whose node_modules resolves @openpanel/redis.+ cd "$REPO_ROOT/apps/api" || exit 1+ set -a+ # shellcheck disable=SC1091+ . "$REPO_ROOT/.env"+ set +a+ PROOF_NAMESPACE="$PROOF_NAMESPACE" bun -e '+ const { Redis } = await import("@openpanel/redis");+ const redis = new Redis(process.env.REDIS_URL ?? "redis://127.0.0.1:6379");+ const pattern = `bull:*-${process.env.PROOF_NAMESPACE}:*`;+ let cursor = "0";+ let removed = 0;+ do {+ const [next, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000);+ cursor = next;+ if (keys.length > 0) {+ removed += await redis.del(...keys);+ }+ } while (cursor !== "0");+ console.log(`cleaned ${removed} proof queue keys`);+ await redis.quit();+ '+) && pass "cleaned up the namespaced proof queue keys" \+ || fail "could not clean up the namespaced proof queue keys"++# Nothing this script started may outlive it.+sleep 0.3+LEFTOVER="$(pgrep -f "$MAIN_TS" || true)"+if [ -z "$LEFTOVER" ]; then+ pass "no boot-proof process left behind"+else+ fail "process(es) still running: $LEFTOVER"+ # shellcheck disable=SC2086+ kill -9 $LEFTOVER 2>/dev/null+fi+echoif [ "$FAILURES" -eq 0 ]; thenecho "boot-proof.sh: all checks passed"diff --git a/apps/api/src/config/env.test.ts b/apps/api/src/config/env.test.tsnew file mode 100644index 000000000..cbdab607b--- /dev/null+++ b/apps/api/src/config/env.test.ts@@ -0,0 +1,128 @@+import { describe, expect, it } from 'vitest';+import {+ concurrencyEnvKey,+ concurrencyOverride,+ ENABLED_QUEUE_VALUES,+ loadConfig,+} from './env';++// `loadConfig` takes its source, so nothing here touches process.env.+const base = { ROLE: 'worker' } as NodeJS.ProcessEnv;++describe('ROLE', () => {+ it('defaults to api and accepts the three roles', () => {+ expect(loadConfig({}).ROLE).toBe('api');+ for (const role of ['api', 'worker', 'all']) {+ expect(loadConfig({ ROLE: role }).ROLE).toBe(role);+ }+ });++ it('fails boot loudly on an unknown value, naming it', () => {+ expect(() => loadConfig({ ROLE: 'wroker' })).toThrow(+ /unknown value "wroker"/+ );+ });+});++describe('ENABLED_QUEUES', () => {+ it('is every known consumer when unset', () => {+ expect(loadConfig(base).ENABLED_QUEUES).toEqual([...ENABLED_QUEUE_VALUES]);+ });++ it('covers the Kafka token plus the seven registry queues', () => {+ expect(ENABLED_QUEUE_VALUES).toEqual([+ 'events',+ 'sessions',+ 'cron',+ 'notification',+ 'import',+ 'insights',+ 'gsc',+ 'cohortCompute',+ ]);+ });++ it('parses the cloud split (docs/ANSWERS.md 1.3)', () => {+ expect(+ loadConfig({ ...base, ENABLED_QUEUES: 'events' }).ENABLED_QUEUES+ ).toEqual(['events']);+ expect(+ loadConfig({ ...base, ENABLED_QUEUES: ' sessions , cron ' })+ .ENABLED_QUEUES+ ).toEqual(['sessions', 'cron']);+ });++ it('rejects an unknown token instead of silently idling (V1 ignored it)', () => {+ expect(() =>+ loadConfig({ ...base, ENABLED_QUEUES: 'sessions,nope' })+ ).toThrow(/unknown queue "nope"/);+ });++ it('names the rename when it sees the old events_kafka spelling', () => {+ expect(() =>+ loadConfig({ ...base, ENABLED_QUEUES: 'events_kafka' })+ ).toThrow(/"events_kafka" was renamed to "events"/);+ });+});++describe('<QUEUE>_CONCURRENCY', () => {+ it('keeps V1 key derivation — cohortCompute reads COHORTCOMPUTE_CONCURRENCY', () => {+ expect(concurrencyEnvKey('cohortCompute')).toBe(+ 'COHORTCOMPUTE_CONCURRENCY'+ );+ const config = loadConfig({ ...base, COHORTCOMPUTE_CONCURRENCY: '9' });+ expect(concurrencyOverride(config, 'cohortCompute')).toBe(9);+ });++ it('is undefined when unset, so the registry default stands', () => {+ expect(concurrencyOverride(loadConfig(base), 'sessions')).toBeUndefined();+ });++ it('ignores a non-numeric or non-positive value, as V1 did', () => {+ expect(+ concurrencyOverride(+ loadConfig({ ...base, SESSIONS_CONCURRENCY: 'lots' }),+ 'sessions'+ )+ ).toBeUndefined();+ expect(+ concurrencyOverride(+ loadConfig({ ...base, SESSIONS_CONCURRENCY: '0' }),+ 'sessions'+ )+ ).toBeUndefined();+ });+});++describe('the boot flags main.ts branches on', () => {+ it('DISABLE_WORKERS is any defined value; DISABLE_BULLBOARD is 1 or true', () => {+ expect(loadConfig(base).DISABLE_WORKERS).toBe(false);+ expect(loadConfig({ ...base, DISABLE_WORKERS: '0' }).DISABLE_WORKERS).toBe(+ true+ );++ expect(loadConfig(base).DISABLE_BULLBOARD).toBe(false);+ expect(+ loadConfig({ ...base, DISABLE_BULLBOARD: '1' }).DISABLE_BULLBOARD+ ).toBe(true);+ expect(+ loadConfig({ ...base, DISABLE_BULLBOARD: 'true' }).DISABLE_BULLBOARD+ ).toBe(true);+ // V1 only ever accepted those two spellings.+ expect(+ loadConfig({ ...base, DISABLE_BULLBOARD: 'yes' }).DISABLE_BULLBOARD+ ).toBe(false);+ });++ it('QUEUE_NAMESPACE and QUEUE_CLUSTER default to off, so queue keys are V1s', () => {+ const config = loadConfig(base);+ expect(config.QUEUE_CLUSTER).toBe(false);+ expect(config.QUEUE_NAMESPACE).toBeUndefined();+ });++ it('reports every problem at once, never just the first', () => {+ expect(() =>+ loadConfig({ ROLE: 'nope', ENABLED_QUEUES: 'alsonope' })+ ).toThrow(/unknown value "nope"[\s\S]*unknown queue "alsonope"/);+ });+});diff --git a/apps/api/src/config/env.ts b/apps/api/src/config/env.tsindex 744ffbe6e..34e95c560 100644--- a/apps/api/src/config/env.ts+++ b/apps/api/src/config/env.ts@@ -2,12 +2,15 @@// per TARGET_ARCHITECTURE §7 "apps/api — three files". `packages/db` and// `packages/redis` keep reading their own env until they grow factories —// an accepted pragmatic deviation (ADR-007 §7) — so this catalogue covers-// only what `main.ts` itself reads today: TZ, ROLE, LOG_LEVEL, API_PORT,-// SELF_HOSTED and QUEUE_CLUSTER.+// only what `main.ts` itself reads today: TZ, ROLE, NODE_ENV, LOG_LEVEL,+// API_PORT, API_HOST, SELF_HOSTED, QUEUE_CLUSTER, QUEUE_NAMESPACE,+// ENABLED_QUEUES, the seven `<QUEUE>_CONCURRENCY` overrides, DISABLE_WORKERS,+// DISABLE_BULLBOARD and SHUTDOWN_FORCE_EXIT_MS.//// Invalid config fails boot loudly, with every issue reported at once —// never a fail-fast on the first bad var.+import { queues } from '@openpanel/core';import { z } from 'zod';export const ROLE_VALUES = ['api', 'worker', 'all'] as const;@@ -16,6 +19,38 @@ export type Role = (typeof ROLE_VALUES)[number];const DEFAULT_API_PORT = 3000;const DEFAULT_LOG_LEVEL = 'info';const TRUE_STRING = 'true';+const PRODUCTION = 'production';+/** V1's worker deadline (apps/worker/src/boot-workers.ts). */+const DEFAULT_SHUTDOWN_FORCE_EXIT_MS = 20_000;++/**+ * The Kafka events consumer's token. Renamed from `events_kafka` — there is+ * one events transport now, so there is one token for it (ADR-004 rec 5/6,+ * docs/ANSWERS.md §1.3). Carl updates the cloud env at cutover, which is why+ * the old spelling gets its own message below rather than being aliased.+ */+export const KAFKA_QUEUE_TOKEN = 'events';++/** `events` plus the seven registry keys, read from the registry itself so+ * the accepted set cannot drift from what actually exists. */+export const ENABLED_QUEUE_VALUES = [+ KAFKA_QUEUE_TOKEN,+ ...Object.keys(queues),+] as const;++export type EnabledQueue = (typeof ENABLED_QUEUE_VALUES)[number];++const RENAMED_QUEUE_TOKENS: Record<string, string> = {+ events_kafka: KAFKA_QUEUE_TOKEN,+};++/** V1's `getConcurrencyFor` key derivation, verbatim — `cohortCompute` keeps+ * reading `COHORTCOMPUTE_CONCURRENCY` (ADR-005 acceptance note: no rename). */+const NON_ENV_KEY_CHARS = /[^A-Z0-9]/g;++export function concurrencyEnvKey(queueName: string): string {+ return `${queueName.toUpperCase().replace(NON_ENV_KEY_CHARS, '_')}_CONCURRENCY`;+}// An unset or blank var should fall through to the default, matching the// `value?.trim() || fallback` idiom the ported reads used before this file@@ -52,8 +87,83 @@ const roleSchema = z.preprocess(.transform((value) => value as Role));+/**+ * Same doctrine as ROLE: an unknown token fails boot loudly rather than+ * leaving a worker silently idle (V1 ignored it — boot-workers.ts:78-87 —+ * which docs/ANSWERS.md §1.3 rules must change). Unset means all of them.+ */+const enabledQueuesSchema = z.preprocess(+ blankToUndefined,+ z+ .string()+ .optional()+ .superRefine((value, ctx) => {+ if (value === undefined) {+ return;+ }+ for (const token of splitTokens(value)) {+ if ((ENABLED_QUEUE_VALUES as readonly string[]).includes(token)) {+ continue;+ }+ const renamedTo = RENAMED_QUEUE_TOKENS[token];+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message: renamedTo+ ? `ENABLED_QUEUES: "${token}" was renamed to "${renamedTo}" — update ENABLED_QUEUES to use the new name.`+ : `ENABLED_QUEUES: unknown queue "${token}". Accepted values: ${ENABLED_QUEUE_VALUES.join(', ')}.`,+ });+ }+ })+ .transform((value) =>+ value === undefined+ ? [...ENABLED_QUEUE_VALUES]+ : (splitTokens(value) as EnabledQueue[])+ )+);++function splitTokens(value: string): string[] {+ return value+ .split(',')+ .map((token) => token.trim())+ .filter(Boolean);+}++/** V1 treated any defined value as "off" for workers and only `'1'`/`'true'`+ * as "off" for bull-board. Both spellings port as-is. */+const definedIsTrueSchema = z.preprocess(+ blankToUndefined,+ z+ .string()+ .optional()+ .transform((value) => value !== undefined)+);++const oneOrTrueSchema = z.preprocess(+ blankToUndefined,+ z+ .string()+ .optional()+ .transform((value) => value === '1' || value === TRUE_STRING)+);++/** One `<QUEUE>_CONCURRENCY` slot per registry queue. V1 ignored a+ * non-numeric or non-positive value and kept the default; so does this. */+const concurrencySchema = z.preprocess(+ blankToUndefined,+ z.coerce.number().int().positive().optional().catch(undefined)+);++const concurrencyShape = Object.fromEntries(+ Object.keys(queues).map((name) => [+ concurrencyEnvKey(name),+ concurrencySchema,+ ])+) as Record<string, typeof concurrencySchema>;+const envSchema = z.object({+ ...concurrencyShape,ROLE: roleSchema,+ NODE_ENV: z.preprocess(blankToUndefined, z.string().optional()),LOG_LEVEL: z.preprocess(blankToUndefined,z.string().default(DEFAULT_LOG_LEVEL)@@ -79,9 +189,55 @@ const envSchema = z.object({.optional().transform((value) => value !== undefined)),+ /**+ * Isolates two deployments — or a proof run and the dev box — sharing one+ * Redis (`queueKey`'s `namespace`). UNSET in every real deployment: setting+ * it renames every queue key, which orphans the jobs already in Redis.+ */+ QUEUE_NAMESPACE: z.preprocess(blankToUndefined, z.string().optional()),+ ENABLED_QUEUES: enabledQueuesSchema,+ DISABLE_WORKERS: definedIsTrueSchema,+ DISABLE_BULLBOARD: oneOrTrueSchema,+ /**+ * Left UNSET means Bun's own default, `0.0.0.0`.+ *+ * Deliberate deviation from V1, which defaulted to `localhost` outside+ * production (apps/api/src/index.ts): `Bun.serve({hostname:'localhost'})`+ * binds `::1` ONLY, so a dev box's `127.0.0.1` request is refused. Making+ * the loopback-only bind explicit rather than the accidental default is+ * what keeps `bun run src/main.ts` reachable the way `node dist/index.js`+ * was.+ */+ API_HOST: z.preprocess(blankToUndefined, z.string().optional()),+ SHUTDOWN_FORCE_EXIT_MS: z.coerce+ .number()+ .int()+ .positive()+ .default(DEFAULT_SHUTDOWN_FORCE_EXIT_MS),});-export type Config = z.infer<typeof envSchema>;+export type Config = z.infer<typeof envSchema> & {+ [key: `${string}_CONCURRENCY`]: number | undefined;+};++/** V1's default host rule (apps/api/src/index.ts). */+export function apiHost(config: Config): string {+ return config.API_HOST ?? (isProduction(config) ? '0.0.0.0' : 'localhost');+}++export function isProduction(config: Config): boolean {+ return config.NODE_ENV === PRODUCTION;+}++/** The `<QUEUE>_CONCURRENCY` override for one registry queue, if any. */+export function concurrencyOverride(+ config: Config,+ queueName: string+): number | undefined {+ return (config as Record<string, unknown>)[concurrencyEnvKey(queueName)] as+ | number+ | undefined;+}function formatIssues(issues: z.ZodIssue[]): string {return issuesdiff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex f2a20c96d..236965740 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -1,43 +1,99 @@// V2 boot entrypoint (ADR-007 §7 "apps/api — three files"; TARGET_ARCHITECTURE// §7 main.ts boot order). Runs on Bun; the V1 Fastify boot at `./index.ts`-// keeps serving the golden harness untouched until `apps/worker` is deleted-// at P9.+// keeps serving the golden harness untouched until it is deleted at M9-004.//-// This is the M3 gate boot only: db/ch/clients are not wired into `AppDeps`-// yet — their types stay the `unknown` stubs `context.ts` declares until-// P3/P4 land the real clients — so only the ops surface (healthz + metrics)-// is mounted here. `dashboardRoutes`/`publicApiRoutes` land with their first-// module (see `rest.routes.ts`). The buffers ARE real from M8-001:-// `createBuffers` builds them once, here, and nowhere else. The producers are-// real from M9-001: all seven queues, envelope on, byte-identical Redis keys.-// Workers, schedulers and the ingest consumer are M9-002.+// THE ROLE TABLE (TARGET_ARCHITECTURE §7). One process, three shapes:+//+// | | api | worker | all |+// | ops HTTP + /metrics | x | x | x |+// | producers | x | x | x |+// | BullMQ workers | | x | x |+// | schedulers (cron) | | x | x |+// | Kafka ingest | | x | x |+// | initial salts | | x | x |+// | bull-board | | x | x |+// | queue/buffer/session scrape gauges | | x | x |+// | redis keyspace-notify | x | | x |+//+// An unknown ROLE fails boot loudly, naming the value — `config/env.ts`.+// Inside a consuming role, ENABLED_QUEUES narrows *which* of the eight+// consumers start (cloud runs 4 replicas on `events` and 6 on the rest,+// docs/ANSWERS.md §1.3), and an unknown token there fails boot the same way.+//+// db/ch/clients are not on `AppDeps` yet — their types stay the `unknown`+// stubs `context.ts` declares until P3/P4 land the real clients — so the+// mounted HTTP surface is still ops-only; `dashboardRoutes`/`publicApiRoutes`+// land at M9-004. The buffers are real from M8-001, the producers from+// M9-001, and the workers/schedulers/consumer from here.process.env.TZ = 'UTC';import {type AppDeps,+ BULL_BOARD_BASE_PATH,type BufferDeps,+ bullBoardRoutes,createBuffers,+ createInitialSalts,createProducers,+ debugRoutes,+ enableEventsHeartbeat,+ incomingEvent,+ ingestConsumerMetrics,+ type KafkaConsumerHandle,+ loadIncomingEventDeps,+ markEventsActivity,opsRoutes,+ type QueueDefinition,type QueueProducerHandle,queueKey,queues,+ rawStderrWrite,registerBufferMetrics,+ registerDefaultMetrics,+ registerQueueMetrics,+ registerSessionScrapeMetrics,+ type SessionMetricsRedis,+ sessionEndEnqueueOptions,+ sessionEndJobPayload,+ setShuttingDown,+ startKafkaEventsConsumer,+ startSchedulers,+ startWorkers,+ type WorkerHandle,} from '@openpanel/core';-import { produceIncomingEvent } from '@openpanel/queue';-import { getRedisQueue } from '@openpanel/redis';+import {+ assertKafkaConfigured,+ createKafkaEventsConsumer,+ KAFKA_EVENTS_TOPIC,+ KAFKA_HANDLER_MAX_ATTEMPTS,+ KAFKA_HANDLER_RETRY_INITIAL_MS,+ KAFKA_HANDLER_RETRY_MAX_MS,+ KAFKA_PARTITIONS_CONCURRENT,+ kafkaLogger,+ produceDeadLetterEvent,+ produceIncomingEvent,+} from '@openpanel/queue';+import { getRedisCache, getRedisPub, getRedisQueue } from '@openpanel/redis';import pino from 'pino';-import { type Config, loadConfig } from './config/env';+import {+ type Config,+ concurrencyOverride,+ isProduction,+ KAFKA_QUEUE_TOKEN,+ loadConfig,+} from './config/env';-const SHUTDOWN_FORCE_EXIT_MS = 5000;// The registry key and the Redis name are the same string for all seven queues-// (jobs.registry.ts); `queueKey` only braces it under `QUEUE_CLUSTER`, which no-// deployment sets.+// (jobs.registry.ts); `queueKey` only braces it under `QUEUE_CLUSTER`.const CRON_QUEUE_NAME = 'cron';// Used only to report a config error itself — the real level isn't known// until `config/env.ts` has validated `LOG_LEVEL`.const BOOTSTRAP_LOG_LEVEL = 'info';+// V1's cron drain bound (apps/worker/src/boot-workers.ts `waitForQueueToEmpty`).+const CRON_DRAIN_TIMEOUT_MS = 60_000;+const CRON_DRAIN_POLL_MS = 500;+const FATAL_EXIT_DELAY_MS = 1000;/*** `config/env.ts` is the sole `process.env` reader on this path; an invalid@@ -59,6 +115,10 @@ function loadConfigOrExit(): Config {const config = loadConfigOrExit();const logger = pino({ name: 'api', level: config.LOG_LEVEL });+/** ROLE=api produces and serves; it never consumes. */+const roleConsumes = config.ROLE !== 'api';+const workersEnabled = roleConsumes && !config.DISABLE_WORKERS;+/*** All seven queues, envelope on, on the Redis keys V1 already uses — so a job* V1 enqueued before the cutover is read back by V2's `resolveJob` through@@ -74,6 +134,7 @@ function buildProducerHandle(): QueueProducerHandle {return createProducers(queues, {connection: getRedisQueue(),cluster: config.QUEUE_CLUSTER,+ namespace: config.QUEUE_NAMESPACE,logger,});}@@ -85,18 +146,23 @@ function buildProducerHandle(): QueueProducerHandle {* deliberately (docs/ANSWERS.md §3: "known!").*/function bufferDeps(producers: QueueProducerHandle): BufferDeps {- // `bullQueues` are keyed by the Redis name, which `QUEUE_CLUSTER` braces.- const cronKey = queueKey(CRON_QUEUE_NAME, { cluster: config.QUEUE_CLUSTER });+ const cron = findBullQueue(producers, CRON_QUEUE_NAME);return {createLogger: (name) => logger.child({ name }),- isCronPaused: async () => {- const cron = producers.bullQueues.find((queue) => queue.name === cronKey);- return cron ? await cron.isPaused() : false;- },+ isCronPaused: async () => (cron ? await cron.isPaused() : false),};}+/** `bullQueues` are keyed by the Redis name, which cluster/namespace decide. */+function findBullQueue(producers: QueueProducerHandle, name: string) {+ const key = queueKey(name, {+ cluster: config.QUEUE_CLUSTER,+ namespace: config.QUEUE_NAMESPACE,+ });+ return producers.bullQueues.find((queue) => queue.name === key);+}+function buildDeps(): AppDeps {const producers = buildProducerHandle();return {@@ -110,39 +176,237 @@ function buildDeps(): AppDeps {buffers: createBuffers(bufferDeps(producers)),producers,// @openpanel/queue's Kafka producer, injected because core cannot import- // it back (M8-002). M8-003 replaces it with core's own.+ // it back (M8-002). It dies with that package at M9-003.produceIncomingEvent,logger,config: { selfHosted: config.SELF_HOSTED },};}+/**+ * The queues this process consumes, `<QUEUE>_CONCURRENCY` applied.+ *+ * ENABLED_QUEUES is the same allowlist V1 read, and its `events` token selects+ * the Kafka consumer rather than a BullMQ queue — hence it is filtered out+ * here and asked about separately.+ */+function consumedQueues(): Record<string, QueueDefinition> {+ const enabled = new Set(config.ENABLED_QUEUES);+ const selected: Record<string, QueueDefinition> = {};++ for (const [name, definition] of Object.entries(queues)) {+ if (!enabled.has(name)) {+ continue;+ }+ const concurrency = concurrencyOverride(config, name);+ selected[name] =+ concurrency === undefined+ ? definition+ : { ...definition, worker: { ...definition.worker, concurrency } };+ }++ return selected;+}++/**+ * Every declared scheduler must have a handler on the `cron` queue, or its+ * tick enqueues a job the worker can only throw on. Reported at boot, once,+ * rather than discovered N minutes later in a failed-job list.+ */+function warnOnUnhandledSchedulers(schedulerIds: string[]): void {+ const handled = new Set(Object.keys(queues.cron.jobs));+ const missing = schedulerIds.filter((id) => !handled.has(id));+ if (missing.length > 0) {+ logger.error(+ { schedulers: missing },+ 'cron schedulers have no handler on the cron queue — every tick will fail'+ );+ }+}++/**+ * The Kafka events consumer. The kafkajs client, the topic, the consumer+ * group, the DLQ producer and the retry bounds all come from+ * `@openpanel/queue`, which is what keeps those names byte-identical across+ * the cutover — core never spells one (see modules/ingest/src/consumer.ts).+ */+async function startIngestConsumer(+ deps: AppDeps+): Promise<KafkaConsumerHandle> {+ assertKafkaConfigured();+ enableEventsHeartbeat();++ const eventDeps = await loadIncomingEventDeps(logger, async (input) => {+ await deps.producers.queues.sessions.session.add(+ sessionEndJobPayload(input),+ sessionEndEnqueueOptions(input.closedSession.id)+ );+ });++ return await startKafkaEventsConsumer({+ createConsumer: createKafkaEventsConsumer,+ logger,+ kafkaLogger,+ topic: KAFKA_EVENTS_TOPIC,+ partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,+ batch: {+ handleEvent: (payload, meta) => incomingEvent(payload, eventDeps, meta),+ sendToDeadLetter: produceDeadLetterEvent,+ logger,+ metrics: ingestConsumerMetrics,+ onActivity: markEventsActivity,+ maxAttempts: KAFKA_HANDLER_MAX_ATTEMPTS,+ initialRetryMs: KAFKA_HANDLER_RETRY_INITIAL_MS,+ maxRetryMs: KAFKA_HANDLER_RETRY_MAX_MS,+ },+ });+}++/**+ * V1's shutdown drain (apps/worker/src/boot-workers.ts): let the cron queue+ * finish what it started before closing the workers, because a cron job cut+ * in half comes back as a stalled retry. One of ADR-005's four sanctioned+ * `bullQueues` call sites.+ */+async function waitForCronToDrain(producers: QueueProducerHandle) {+ const cron = findBullQueue(producers, CRON_QUEUE_NAME);+ if (!cron) {+ return;+ }++ const startedAt = performance.now();+ while (true) {+ const active = await cron.getActiveCount();+ if (active === 0) {+ return;+ }+ if (performance.now() - startedAt > CRON_DRAIN_TIMEOUT_MS) {+ logger.warn(+ { queue: cron.name, remainingCount: active },+ 'Timeout reached while waiting for queue to empty'+ );+ return;+ }+ logger.info({ queue: cron.name, count: active }, 'Waiting for queue');+ await new Promise((resolve) => setTimeout(resolve, CRON_DRAIN_POLL_MS));+ }+}++function registerConsumerMetrics(deps: AppDeps): void {+ // Scrape-time collectors register only where a role consumes: each is a+ // Redis round trip per scrape, and ten API replicas exposing them would+ // multiply that for no new information (TARGET_ARCHITECTURE §18).+ registerBufferMetrics(deps.buffers);+ registerQueueMetrics(deps.producers.bullQueues);+ registerSessionScrapeMetrics(+ () => getRedisCache() as unknown as SessionMetricsRedis+ );+}++async function buildHttpApp(deps: AppDeps) {+ const app = opsRoutes(deps);++ // Local-only: trigger a cron job on demand instead of waiting for its+ // schedule. Two conditions, both V1's: `NODE_ENV != production`, because it+ // is an unauthenticated "run anything now" surface, and a consuming role,+ // because in V1 these routes only ever existed on the worker.+ if (roleConsumes && !isProduction(config)) {+ app.use(debugRoutes(deps));+ logger.info('Debug routes enabled at /debug/cron');+ }++ if (roleConsumes && !config.DISABLE_BULLBOARD) {+ app.use(await bullBoardRoutes(deps, deps.producers.bullQueues));+ logger.info({ path: BULL_BOARD_BASE_PATH }, 'bull-board mounted');+ }++ return app;+}++/**+ * uncaughtException / unhandledRejection: the process state is corrupt, so log+ * and exit fast rather than draining through a poisoned process.+ *+ * Fatals are mirrored to the REAL stderr, bypassing the output interceptor:+ * the OTLP flush window is often lost on the way down, and `docker logs` must+ * always show why we died (V1 did this in both apps).+ *+ * Installed in every role. V1's api only did so in production and its worker+ * did so always; the merged process takes the worker's.+ */+function installFatalHandlers(): void {+ process.on('uncaughtException', (error) => {+ logger.fatal({ err: error }, 'Uncaught exception — exiting');+ rawStderrWrite(`Uncaught exception — exiting: ${error?.stack ?? error}\n`);+ setShuttingDown(true);+ setTimeout(() => process.exit(1), FATAL_EXIT_DELAY_MS).unref();+ });+ process.on('unhandledRejection', (reason) => {+ logger.fatal({ reason }, 'Unhandled rejection — exiting');+ rawStderrWrite(+ `Unhandled rejection — exiting: ${+ reason instanceof Error ? reason.stack : String(reason)+ }\n`+ );+ setShuttingDown(true);+ setTimeout(() => process.exit(1), FATAL_EXIT_DELAY_MS).unref();+ });+}+async function main() {const role = config.ROLE;- const port = config.API_PORT;const deps = buildDeps();- // Scrape-time collectors register only where a role consumes queues: the- // LLEN gauges cost one Redis round trip per buffer per scrape, and ten API- // replicas exposing them would multiply that for no new information- // (TARGET_ARCHITECTURE §18).- if (role !== 'api') {- registerBufferMetrics(deps.buffers);+ // HTTP and default metrics register everywhere (TARGET_ARCHITECTURE §18).+ registerDefaultMetrics();+ if (roleConsumes) {+ registerConsumerMetrics(deps);}- const app = opsRoutes(deps);+ const app = await buildHttpApp(deps);- if (role !== 'api') {- logger.info(- { role },- 'ROLE=%s: worker/ingest bootstrap (workers, schedulers, Kafka consumer) is not implemented yet — running the HTTP ops surface only',- role- );+ let workers: WorkerHandle | undefined;+ let consumer: KafkaConsumerHandle | undefined;++ if (workersEnabled) {+ const definitions = consumedQueues();+ workers = startWorkers({+ definitions,+ deps,+ // BullMQ duplicates this for each worker's blocking fetch; the queue+ // client itself stays packages/redis's singleton, as in V1.+ createConnection: () => getRedisQueue(),+ cluster: config.QUEUE_CLUSTER,+ namespace: config.QUEUE_NAMESPACE,+ });++ if (definitions[CRON_QUEUE_NAME]) {+ const cron = findBullQueue(deps.producers, CRON_QUEUE_NAME);+ if (cron) {+ await startSchedulers({+ queue: cron,+ flags: {+ selfHosted: config.SELF_HOSTED,+ production: isProduction(config),+ },+ logger,+ });+ warnOnUnhandledSchedulers(+ (await cron.getJobSchedulers()).map((scheduler) => scheduler.key)+ );+ }+ }++ if (config.ENABLED_QUEUES.includes(KAFKA_QUEUE_TOKEN)) {+ consumer = await startIngestConsumer(deps);+ }++ await createInitialSalts();+ } else if (roleConsumes) {+ logger.warn('Workers are disabled');}- app.listen(port, () => {- logger.info({ role, port }, 'API listening');- });+ installFatalHandlers();let shuttingDown = false;const shutdown = async (signal: string) => {@@ -150,16 +414,25 @@ async function main() {return;}shuttingDown = true;+ // Readiness answers 503 from here on, so the load balancer stops sending+ // work before the drain below finishes rather than after.+ setShuttingDown(true);logger.info({ signal }, 'Starting graceful shutdown');const forceExit = setTimeout(() => {logger.error({ signal }, 'Graceful shutdown timed out — forcing exit');process.exit(1);- }, SHUTDOWN_FORCE_EXIT_MS);+ }, config.SHUTDOWN_FORCE_EXIT_MS);forceExit.unref();try {await app.stop();+ // Only if THIS process consumes cron: waiting on a queue another replica+ // owns would stall the whole shutdown budget on someone else's jobs.+ if (workers && config.ENABLED_QUEUES.includes(CRON_QUEUE_NAME)) {+ await waitForCronToDrain(deps.producers);+ }+ await Promise.all([workers?.close(), consumer?.stop()]);await deps.producers.close();logger.info('Graceful shutdown completed');clearTimeout(forceExit);@@ -177,6 +450,32 @@ async function main() {process.on('SIGINT', () => {shutdown('SIGINT');});++ // No `hostname` unless API_HOST says one: Bun's default is `0.0.0.0`, and+ // its `localhost` binds IPv6-only (see config/env.ts's API_HOST).+ const listen = config.API_HOST+ ? { port: config.API_PORT, hostname: config.API_HOST }+ : { port: config.API_PORT };+ app.listen(listen, () => {+ logger.info(+ { role, port: config.API_PORT, hostname: config.API_HOST ?? '0.0.0.0' },+ 'API listening'+ );+ });++ // V1's api did this and its worker did not — the expiry notifications only+ // matter where something subscribes (apps/api/src/index.ts).+ if (role !== 'worker') {+ try {+ await getRedisPub().config('SET', 'notify-keyspace-events', 'Ex');+ logger.info({ role }, 'redis keyspace notifications configured');+ } catch (error) {+ logger.warn({ err: error }, 'Failed to set redis notify-keyspace-events');+ logger.warn(+ 'If you use a managed Redis service, you may need to set this manually.'+ );+ }+ }}main();diff --git a/packages/core/docs/OPS_GRAFANA_MIGRATION.md b/packages/core/docs/OPS_GRAFANA_MIGRATION.mdnew file mode 100644index 000000000..b617f743c--- /dev/null+++ b/packages/core/docs/OPS_GRAFANA_MIGRATION.md@@ -0,0 +1,225 @@+# OPS: Grafana / alert migration for the V1 → V2 metrics cutover++**Audience: whoever owns the OpenPanel dashboards and alerts. This is a written+spec, not a change. No dashboard, alert or datasource was edited by the work+that produced it — deliberately (M9 cutover discipline).**++Written for M9-002. Sources: `packages/core/src/metrics.ts` (the one registry),+`packages/core/src/http/http.metrics.ts`, `packages/core/src/jobs/jobs.metrics.ts`,+`apps/api/src/main.ts` (the ROLE table), and — for the V1 side —+`[email protected]/dist/fastify-metrics.js` and+`apps/worker/src/metrics.ts`.++> Where to put this file: it lives here because the task's scope globs cover+> `packages/**` and not a repo-root `docs/`. Move it to `docs/OPS_GRAFANA_MIGRATION.md`+> with one `git mv` if that is where you want it.++---++## 0. TL;DR — what a dashboard owner must do++1. **No metric name changes.** Every panel and alert keeps working by name.+2. **`route` label VALUES change** for a handful of routes (§2). Any panel or+ alert that pins `route="…"` for one of those needs its literal updated.+3. **Ten new `<queue>_*_count` series appear** (§3.1). Nothing to fix; you may+ want new panels.+4. **`buffer_add_*` for the replay, group and profile buffers steps up from+ near-zero** (§3.3). It looks like a regression on a graph. It is a fix.+5. **Bull-board moved** from `http://<worker>:3040/` to+ `http://<host>:<port>/bullboard`, and is now behind the dashboard session+ cookie (§4).+6. **`/metrics` is unchanged**: same path, same container port 3000, served in+ every ROLE.++---++## 1. Metric names and labels: unchanged, on purpose++`fastify-metrics` is deleted with Fastify (ADR-002). Its two route series are+re-created by one Elysia hook, reproducing the plugin's own definitions:++| Series | Type | Labels | V1 source | V2 source |+|---|---|---|---|---|+| `http_request_duration_seconds` | histogram | `method`, `route`, `status_code` | `fastify-metrics` default | `core/src/http/http.metrics.ts` |+| `http_request_summary_seconds` | summary | `method`, `route`, `status_code` | `fastify-metrics` default | same file |++Both still observe **seconds**, not milliseconds — a millisecond observation on+the same series would multiply every existing percentile by 1000, so it is+pinned by a test (`http.metrics.test.ts`).++Also unchanged in name and labels: the default process/GC metrics,+`job_duration_ms{name,status}`, `kafka_events_*_total`, the twelve `buffer_*`+series, the twelve session-lifecycle series, and the five+`<queue>_{active,delayed,failed,completed,waiting}_count` gauges.++`job_duration_ms` keeps its **failure-only** semantics: V1 observed it+exclusively in the workers' `failed` listener, so every percentile ever computed+from it is a percentile of failures. Success timings, if you ever want them,+must get a **new** series name.++---++## 2. The `route` label: same name, different values++This is the one thing that needs action.++**V1's rule:** the label was Fastify's `routeOptions.url` — the declared route+pattern *including* the plugin prefix, with Fastify's `:param` syntax.++**V2's rule:** the label is Elysia's `route` — the matched route pattern, with+Elysia's `:param` syntax.++The two syntaxes agree, so **most values are byte-identical**: every+`/insights/:projectId/...`, `/manage/...`, `/export/...`, `/import/events`,+`/track/device-id`, `/healthz/live` and so on carries the same literal it does+today. The differences are structural, and they are these:++### 2.1 Routes whose label changes++| What | V1 `route` label | V2 `route` label | Why |+|---|---|---|---|+| Track | `/track/` | `/track` | V1's label was `prefix` + the route `'/'`, giving a trailing slash. V2 declares the full path. |+| Profile | `/profile/` | `/profile` | same |+| Legacy event | `/event/` | `/event` | same (the route itself is KEPT — ADR-004/ADR-015 acceptance notes) |+| tRPC | `/trpc/:path` | `/trpc/*` | the fetch adapter derives the path by slicing the prefix, and a **batched** request puts commas in that segment, which `:path` cannot match (ADR-009) |+| OpenAPI UI | `/documentation` | `/openapi` | `@fastify/swagger-ui` → `@elysiajs/openapi` |+| OpenAPI document | `/documentation/json` | `/openapi.json` | same |+| Bull-board | *(separate worker port, `/`)* | `/bullboard`, and its sub-routes | §4 |+| Debug cron | *(separate worker port, `/debug/cron/:type`)* | `/debug/cron/:type` on the main port | §4 |++### 2.2 Routes whose label disappears++| V1 `route` label | Why it goes |+|---|---|+| `/misc/og/clear`, `/misc/favicon/clear` | deleted (removal register entry 6) |+| `/mcp` GET and DELETE | deleted (removal register entry 2); `POST /mcp` stays |+| the whole worker-side `/` bull-board tree on :3040/:3050 | merged onto the main port |++### 2.3 Behaviour at the edges, reproduced deliberately++| Case | V1 (`fastify-metrics`) | V2 |+|---|---|---|+| `HEAD` / `OPTIONS` | not measured (`methodBlacklist` default) | not measured |+| a request matching no route (404) | not measured (`registeredRoutesOnly` default true) | not measured |+| a handler that threw and was mapped to 5xx | measured — `onResponse` fires | measured — the hook is `onAfterResponse`, which likewise fires |+| status code grouping | ungrouped (`groupStatusCodes` default false) | ungrouped |++### 2.4 One thing this table cannot promise yet++At the time of writing, `main.ts` mounts only the **ops** surface; the dashboard+and public-API surfaces mount at M9-004, when the V1 Fastify boot is deleted.+The rules above are final, and §2.1 lists every difference in the route set that+exists today — but **re-read §2.1 against the M9-004 commit before you edit a+dashboard**, because that is the commit in which the values actually start being+produced.++Two route-set observations that belong to that commit, not to this one, and are+recorded here so they are not discovered from a graph:++* `packages/core/src/modules/tools/tools.routes.ts` declares `/site-checker`+ and `/ip-lookup` at the root, where V1 served them under the router prefix+ `/tools`. Whoever mounts the public-API surface has to restore the prefix;+ until then both the URL and the metric label would differ.+* `/manage/references*`, `/manage/projects/:projectId/dashboards*`,+ `POST /event`, `/healthcheck` and `/trpc/*` are still V1-only routes.++---++## 3. Series that appear, disappear, or change level++### 3.1 New: the queue gauges cover all seven queues++V1 registered `<queue>_{active,delayed,failed,completed,waiting}_count` for+**two** queues — `sessions` and `cron`. V2 registers them for all seven+(ADR-018's continuity register). **Ten new series**, none of the existing ten+changed:++```+notification_{active,delayed,failed,completed,waiting}_count+import_{...}_count+insights_{...}_count+gsc_{...}_count+cohortCompute_{...}_count+```++The `cohortCompute` spelling is intentional and permanent — the registry key+was NOT renamed (ADR-005 acceptance note), so `COHORTCOMPUTE_CONCURRENCY` keeps+working and the series name matches the queue.++### 3.2 Gone: the GroupMQ shard series++Removed with GroupMQ itself (ADR-004). Any panel or alert still selecting them+will read "No data" rather than error. Delete those panels.++### 3.3 Level change: `buffer_add_*` steps up++V1 installed the buffer *add* observers only inside the worker process, so every+API-side add — the replay, group and profile buffers on the `/track` path — was+invisible. V2 has one registry per process and every role builds the buffers, so+those adds now count.++**On a graph this looks like a regression at the cutover. It is a fix.** Expect+`buffer_add_*` for `replay`, `group` and `profile` to step up from near-zero;+`event` and `session` are already counted in V1 and should not move.++If you have an alert with an absolute threshold on any `buffer_add_*` series,+re-baseline it after the cutover.++### 3.4 Where each collector registers now++`/metrics` is served in **every** role, but not every role registers every+collector — a scrape-time gauge costs a Redis round trip per project, and ten+API replicas exposing the session gauges would multiply that Redis load for no+new information.++| Collector | `ROLE=api` | `ROLE=worker` | `ROLE=all` |+|---|---|---|---|+| default process metrics | yes | yes | yes |+| `http_request_*` | yes | yes | yes |+| `<queue>_*_count` | **no** | yes | yes |+| `buffer_*` | **no** | yes | yes |+| `sessions_active_total`, `sessions_projects_active`, `sessions_hwm_lag_ms` | **no** | yes | yes |+| `job_duration_ms`, `kafka_events_*` | declared, never observed | observed | observed |++**Consequence for a dashboard:** a panel that today sums a queue or buffer gauge+across every scrape target keeps working, because only the consuming replicas+ever exposed a non-trivial value. A panel that *asserts presence* on an API+target (e.g. `absent(buffer_event_count)`) will fire. Scope such alerts to the+consuming job.++---++## 4. Two URLs moved (deploy note, not a dashboard note)++The worker's second HTTP server is gone; one process serves one port in every+role (ADR-007).++| What | V1 | V2 |+|---|---|---|+| Bull-board | `http://<worker>:3040/` (and `:3050`), **unauthenticated on that port** | `http://<host>:<port>/bullboard`, **behind the dashboard session cookie** |+| Debug cron trigger | `http://<worker>:3040/debug/cron/:type` | `/debug/cron/:type` on the main port, still `NODE_ENV != production` only |+| `/metrics` | `:3030` (api), `:3040`/`:3050` (workers) | unchanged — same path, container port still 3000 |++Bull-board is mounted only when the role consumes **and** `DISABLE_BULLBOARD` is+unset (`'1'` and `'true'` are both still accepted as "disable"). It now returns+**401** rather than the UI to an unauthenticated caller.++Unchanged and deliberate: **pausing the `cron` queue from bull-board halts all+buffer flushing** (docs/ANSWERS.md §3 — "known!").++---++## 5. Checklist++- [ ] Search dashboards and alert rules for `route="/track/"`, `route="/profile/"`,+ `route="/event/"`, `route="/trpc/:path"`, `route="/documentation"` and+ update the literals per §2.1.+- [ ] Delete panels selecting the GroupMQ shard series (§3.2).+- [ ] Re-baseline absolute-threshold alerts on `buffer_add_*` for the replay,+ group and profile buffers (§3.3).+- [ ] Scope any `absent()`/`up`-style alert on a queue, buffer or session gauge+ to the consuming job only (§3.4).+- [ ] Optionally add panels for the five newly-covered queues (§3.1).+- [ ] Update the bull-board bookmark and tell operators it now needs a dashboard+ login (§4).+- [ ] Re-read §2.1 against the M9-004 commit before editing anything (§2.4).diff --git a/packages/core/package.json b/packages/core/package.jsonindex 528b482fd..39387f0ca 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -24,6 +24,8 @@"@babel/parser": "^7.26.0","@better-agent/core": "^0.1.0-beta.3","@better-agent/providers": "^0.1.0-beta.3",+ "@bull-board/api": "9.6.0",+ "@bull-board/elysia": "9.6.0","@elysiajs/cors": "1.4.2","@elysiajs/openapi": "1.4.15","@google-cloud/storage": "^7.18.0",diff --git a/packages/core/src/http/bull-board.ts b/packages/core/src/http/bull-board.tsnew file mode 100644index 000000000..3a75ee50a--- /dev/null+++ b/packages/core/src/http/bull-board.ts@@ -0,0 +1,79 @@+// Bull-board, on the main port, behind the dashboard session guard+// (ADR-005 recommendation 6). What dies with it is `@bull-board/express` and+// the worker's second express server — nothing named `@bull-board/fastify`+// ever existed. `@bull-board/api` + `@bull-board/elysia` are both pinned+// exact 9.6.0 (ADR-016; ADR-017 register row 4 grants the bump+swap at P9).+//+// TWO things a reader must not mistake for accidents:+//+// 1. **The base path moved.** V1 served bull-board at `/` on the worker's own+// express server (apps/worker/src/index.ts, `setBasePath('/')`). V2 has+// one port for everything, so `/` would shadow every other route. It is+// `/bullboard`; that is a deploy-note, written up in+// `packages/core/docs/OPS_GRAFANA_MIGRATION.md`.+// 2. **Pausing `cron` from this UI halts ALL buffer flushing.** Preserved+// deliberately — docs/ANSWERS.md §3: "known!".++import type { Queue as BullQueue } from 'bullmq';+import { Elysia } from 'elysia';+import type { AppDeps } from '../context';+import { requestContext } from './context';++export const BULL_BOARD_BASE_PATH = '/bullboard';++const UNAUTHORIZED = 401;++/**+ * Mounted only where the role consumes and `DISABLE_BULLBOARD` is unset+ * (main.ts owns both conditions — core reads no environment).+ *+ * The guard is `{ as: 'global' }` and re-checks the path itself rather than+ * leaning on Elysia's hook scoping: `.use()`-ing a NAMED plugin+ * (`@bull-board/elysia`) puts its routes in their own scope, where a `local`+ * hook of the mounting instance would not run at all — and a guard that+ * silently does not run is precisely the failure this must not have. The cost+ * on every other route is one `startsWith`.+ */+export async function bullBoardRoutes(+ deps: AppDeps,+ queues: BullQueue[]+): Promise<Elysia> {+ // Imported here, not at module scope. `@bull-board/elysia` is CommonJS, so+ // it `require()`s Elysia's CJS build, which `require()`s memoirist — and+ // under Bun 1.4.0 that throws "require() async module ... is unsupported"+ // if it happens while this package's module graph is still evaluating.+ // Deferring it to the one call site that needs it puts the require after+ // evaluation, where it resolves cleanly. Proven by+ // `bun run apps/api/e2e/boot-proof.sh`'s ROLE=worker/all legs, which 401+ // on /bullboard rather than failing to boot.+ const [{ createBullBoard }, { BullMQAdapter }, { ElysiaAdapter }] =+ await Promise.all([+ import('@bull-board/api'),+ import('@bull-board/api/bullMQAdapter'),+ import('@bull-board/elysia'),+ ]);++ const serverAdapter = new ElysiaAdapter({+ prefix: BULL_BOARD_BASE_PATH,+ basePath: BULL_BOARD_BASE_PATH,+ });++ createBullBoard({+ queues: queues.map((queue) => new BullMQAdapter(queue)),+ serverAdapter,+ });++ const board = await serverAdapter.registerPlugin();++ return new Elysia({ name: 'core/http/bull-board' })+ .use(requestContext(deps))+ .onBeforeHandle({ as: 'global' }, async ({ path, ctx, status }) => {+ if (!path.startsWith(BULL_BOARD_BASE_PATH)) {+ return;+ }+ if (!(await ctx.session())) {+ return status(UNAUTHORIZED);+ }+ })+ .use(board) as unknown as Elysia;+}diff --git a/packages/core/src/http/debug.routes.ts b/packages/core/src/http/debug.routes.tsnew file mode 100644index 000000000..686c4907d--- /dev/null+++ b/packages/core/src/http/debug.routes.ts@@ -0,0 +1,229 @@+// V1's `apps/worker/src/boot-debug.ts`, ported onto Elysia and onto the ONE+// port. Local-only: `main.ts` mounts this surface only outside production AND+// only in a consuming role — the same two facts that were true of V1, where+// the routes lived in the worker behind `NODE_ENV !== 'production'`. They are+// what keeps an unauthenticated "run any cron job now" endpoint off a+// production box.+//+// Three things changed on the way over, all deliberate:+//+// * The job list is `queues.cron.jobs`, not a hand-written array of the 20+// `CronQueueType` strings. The array could drift from the registry; the+// registry cannot drift from itself.+// * A trigger runs through `runJob` with V1's `{ type }` data — the same+// entry point the real worker uses, compat hook included — so what this+// exercises is the production path and not a second dispatcher.+// * `/debug/insights/:projectId?inline=1` summarises with the insight+// service's own `listAllInsights` instead of V1's ad-hoc+// `projectInsight.groupBy`, because core reaches Postgres through its+// services, not through a Prisma client of its own.++import { z } from 'zod';+import { runJob } from '../jobs/workers';+import { queues } from '../jobs.registry';+import { defineRoutes } from './define';++const BAD_REQUEST = 400;+const INTERNAL_ERROR = 500;+const DIGEST_INSIGHT_LIMIT = 15;++const cronTypes = Object.keys(queues.cron.jobs);++const HTML_ESCAPES: Record<string, string> = {+ '&': '&',+ '<': '<',+ '>': '>',+ '"': '"',+};+const HTML_ESCAPABLE = /[&<>"]/g;++function escapeHtml(value: string): string {+ return value.replace(HTML_ESCAPABLE, (char) => HTML_ESCAPES[char] ?? char);+}++// Browsers send an explicit `text/html` accept header; curl sends the+// wildcard, which falls through to JSON — V1's `req.accepts(['json','html'])`+// ordering, kept.+function wantsHtml(headers: Headers): boolean {+ return (headers.get('accept') ?? '').includes('text/html');+}++function buildJobs(request: Request) {+ const base = new URL(request.url).origin;+ return cronTypes.map((type) => ({+ type,+ url: `${base}/debug/cron/${type}`,+ }));+}++function renderHtml(+ jobs: { type: string; url: string }[],+ message?: string+): string {+ const links = jobs+ .map((job) => `<li><a href="${job.url}">${escapeHtml(job.type)}</a></li>`)+ .join('');+ const note = message+ ? `<pre>${escapeHtml(message)}</pre>`+ : '<p>Click a job to run it now.</p>';+ return `<!doctype html>+<html lang="en">+<head><meta charset="utf-8"><title>Debug cron</title></head>+<body style="font-family: ui-monospace, monospace; max-width: 40rem; margin: 2rem auto;">+<h1>Trigger a cron job</h1>+${note}+<ul>${links}</ul>+</body>+</html>`;+}++const errorMessage = (error: unknown): string =>+ error instanceof Error ? error.message : String(error);++const projectIdParams = z.object({ projectId: z.string() });+const HIDDEN = { hide: true } as const;++export const debugRoutes = defineRoutes((app, deps) =>+ app+ .get(+ '/debug/cron',+ ({ request, set }) => {+ const jobs = buildJobs(request);+ if (wantsHtml(request.headers)) {+ set.headers['content-type'] = 'text/html; charset=utf-8';+ return renderHtml(jobs);+ }+ return {+ message: 'Run a cron job now with GET or POST /debug/cron/:type',+ jobs,+ };+ },+ // Hidden from the OpenAPI document, like every other ops-surface route:+ // in V1 these lived on the worker's own server and were never in it.+ { detail: HIDDEN }+ )+ .all(+ '/debug/cron/:type',+ async ({ params, request, set, ctx }) => {+ const { type } = params as { type: string };+ const jobs = buildJobs(request);+ const html = wantsHtml(request.headers);++ if (!cronTypes.includes(type)) {+ set.status = BAD_REQUEST;+ const message = `Unknown cron type "${type}"`;+ if (html) {+ set.headers['content-type'] = 'text/html; charset=utf-8';+ return renderHtml(jobs, message);+ }+ return { ok: false, error: message, jobs };+ }++ ctx.logger.info({ type }, 'Manually triggering cron job');++ try {+ // V1's wire data, so this goes through the `cron` compat hook.+ await runJob(+ queues.cron,+ { name: type, data: { type }, attemptsMade: 0 },+ deps+ );+ } catch (error) {+ ctx.logger.error({ err: error, type }, 'Manual cron trigger failed');+ set.status = INTERNAL_ERROR;+ if (html) {+ set.headers['content-type'] = 'text/html; charset=utf-8';+ return renderHtml(+ jobs,+ `Error running ${type}: ${errorMessage(error)}`+ );+ }+ return { ok: false, type, error: errorMessage(error) };+ }++ if (html) {+ set.headers['content-type'] = 'text/html; charset=utf-8';+ return renderHtml(jobs, `${type} ran. Check the logs for details.`);+ }+ return { ok: true, type };+ },+ { detail: HIDDEN }+ )+ // The insights pipeline for ONE project, bypassing the daily cron's+ // eligibility filter. Default enqueues a real job so it shows up in+ // bull-board; `?inline=1` runs it here and returns a summary.+ .get(+ '/debug/insights/:projectId',+ async ({ params, query, set, ctx }) => {+ const { projectId } = params;+ const inline = query.inline === '1' || query.inline === 'true';+ const date = new Date().toISOString().slice(0, 10);++ try {+ if (inline) {+ ctx.logger.info({ projectId }, 'Manual insights run (inline)');+ await ctx.services.insight.runProjectInsights({+ projectId,+ date,+ });+ return {+ ok: true,+ projectId,+ date,+ insights: await ctx.services.insight.listAllInsights({+ projectId,+ limit: DIGEST_INSIGHT_LIMIT,+ }),+ };+ }++ const jobId = await ctx.queues.insights.insightsProject.add(+ { projectId, date },+ // Unique id so repeated runs are not deduped by BullMQ.+ { jobId: `debug:${projectId}:${Date.now()}` }+ );+ ctx.logger.info({ projectId, jobId }, 'Enqueued insights job');+ return {+ ok: true,+ projectId,+ jobId,+ message:+ 'Enqueued on the "insights" queue — watch it in bull-board. Add ?inline=1 to run synchronously.',+ };+ } catch (error) {+ ctx.logger.error({ err: error, projectId }, 'Manual insights failed');+ set.status = INTERNAL_ERROR;+ return { ok: false, projectId, error: errorMessage(error) };+ }+ },+ { params: projectIdParams, detail: HIDDEN }+ )+ // Weekly digest for ONE project, bypassing eligibility.+ // [email protected] → send only to that address+ // (no ?to) → return the assembled payload without sending+ // ?force=1 → build even with 0 visitors this week+ .get(+ '/debug/weekly-digest/:projectId',+ async ({ params, query, set, ctx }) => {+ const { projectId } = params;+ const to = typeof query.to === 'string' ? query.to : undefined;+ const force = query.force === '1' || query.force === 'true';+ ctx.logger.info({ projectId, to, force }, 'Manual weekly digest');++ try {+ return {+ ok: true,+ ...(await ctx.services.insight.previewWeeklyDigest(projectId, {+ to,+ force,+ })),+ };+ } catch (error) {+ ctx.logger.error({ err: error, projectId }, 'Manual digest failed');+ set.status = INTERNAL_ERROR;+ return { ok: false, projectId, error: errorMessage(error) };+ }+ },+ { params: projectIdParams, detail: HIDDEN }+ )+);diff --git a/packages/core/src/http/http.metrics.test.ts b/packages/core/src/http/http.metrics.test.tsnew file mode 100644index 000000000..fa222c51f--- /dev/null+++ b/packages/core/src/http/http.metrics.test.ts@@ -0,0 +1,81 @@+import { beforeEach, expect, test } from 'bun:test';+import client from 'prom-client';+import { registry } from '../metrics';+import { observeHttpRequest } from './http.metrics';++const HISTOGRAM = 'http_request_duration_seconds';+const SUMMARY = 'http_request_summary_seconds';++async function samples(name: string) {+ const metric = await registry.getSingleMetricAsString(name);+ return metric;+}++beforeEach(() => {+ registry.resetMetrics();+});++test('both series exist on the ONE registry, with V1s names and labels', () => {+ const registered = registry.getMetricsAsArray().map((metric) => metric.name);+ expect(registered).toContain(HISTOGRAM);+ expect(registered).toContain(SUMMARY);++ // A second registry anywhere would be the defect: prom-client's own global+ // default must stay empty of these.+ expect(client.register.getMetricsAsArray().map((m) => m.name)).not.toContain(+ HISTOGRAM+ );+});++test('observes in SECONDS, under method/route/status_code', async () => {+ observeHttpRequest({+ method: 'POST',+ route: '/track',+ statusCode: 202,+ durationMs: 1500,+ });++ const text = await samples(HISTOGRAM);+ expect(text).toContain('method="POST"');+ expect(text).toContain('route="/track"');+ expect(text).toContain('status_code="202"');+ // 1500ms is 1.5s — a milliseconds observation would read 1500 here and+ // multiply every existing percentile by 1000.+ expect(text).toContain(+ `${HISTOGRAM}_sum{method="POST",route="/track",status_code="202"} 1.5`+ );+});++test('HEAD and OPTIONS are not measured (fastify-metrics methodBlacklist)', async () => {+ observeHttpRequest({+ method: 'OPTIONS',+ route: '/track',+ statusCode: 204,+ durationMs: 1,+ });+ observeHttpRequest({+ method: 'HEAD',+ route: '/track',+ statusCode: 200,+ durationMs: 1,+ });++ expect(await samples(HISTOGRAM)).not.toContain('route="/track"');+});++test('an unmatched request produces no series (registeredRoutesOnly)', async () => {+ observeHttpRequest({+ method: 'GET',+ route: '/*',+ statusCode: 404,+ durationMs: 1,+ });+ observeHttpRequest({+ method: 'GET',+ route: '',+ statusCode: 404,+ durationMs: 1,+ });++ expect(await samples(SUMMARY)).not.toContain('status_code="404"');+});diff --git a/packages/core/src/http/http.metrics.ts b/packages/core/src/http/http.metrics.tsnew file mode 100644index 000000000..13e38d99e--- /dev/null+++ b/packages/core/src/http/http.metrics.ts@@ -0,0 +1,109 @@+// `fastify-metrics`'s two route series, re-created on Elysia+// (TARGET_ARCHITECTURE §18 / ADR-018's metrics-continuity register: "kept,+// name and labels reproduced exactly"). The plugin itself dies with Fastify+// (ADR-002's deleted-plugin list), so these are ours now.+//+// What "reproduced exactly" pins, from fastify-metrics 12.1.0's own source+// (dist/fastify-metrics.js:181-210, 258-290):+//+// * names `http_request_duration_seconds` (histogram) and+// `http_request_summary_seconds` (summary)+// * help "request duration in seconds" / "... summary"+// * labels `method`, `route`, `status_code`, in that order+// * unit SECONDS. prom-client's `startTimer()` observes seconds; a+// milliseconds observation on the same series silently multiplies+// every existing percentile by 1000.+// * scope HEAD and OPTIONS are excluded (fastify-metrics' default+// `methodBlacklist`), and only a MATCHED route is observed+// (`registeredRoutesOnly` defaults true) — so a 404 produces no+// series, exactly as today.+//+// The `route` label VALUE moves, because it is the framework's own route+// pattern and the framework changed. That is the one thing a dashboard owner+// has to act on; the old->new mapping is+// `packages/core/docs/OPS_GRAFANA_MIGRATION.md`.++import { Elysia } from 'elysia';+import client from 'prom-client';+import { registry } from '../metrics';++const DURATION_HELP = 'request duration in seconds';+const SUMMARY_HELP = 'request duration in seconds summary';+const LABEL_NAMES = ['method', 'route', 'status_code'] as const;+// fastify-metrics' `methodBlacklist` default (dist/fastify-metrics.js:63-66).+const UNMEASURED_METHODS = new Set(['HEAD', 'OPTIONS']);+// Elysia's `route` for a request that matched no route. Observing it would+// create the unbounded-cardinality series `registeredRoutesOnly` avoids.+const UNMATCHED_ROUTES = new Set(['', '/*']);+const MS_PER_SECOND = 1000;+// Elysia leaves `set.status` unset on a plain successful return.+const DEFAULT_STATUS = 200;++export const httpRequestDuration = new client.Histogram({+ name: 'http_request_duration_seconds',+ help: DURATION_HELP,+ labelNames: LABEL_NAMES,+ registers: [registry],+});++export const httpRequestSummary = new client.Summary({+ name: 'http_request_summary_seconds',+ help: SUMMARY_HELP,+ labelNames: LABEL_NAMES,+ registers: [registry],+});++/**+ * Observes a single request. Exported so a test can assert the labels and the+ * unit without standing up a server.+ */+export function observeHttpRequest(sample: {+ method: string;+ route: string;+ statusCode: number;+ durationMs: number;+}): void {+ if (UNMEASURED_METHODS.has(sample.method)) {+ return;+ }+ if (UNMATCHED_ROUTES.has(sample.route)) {+ return;+ }++ const labels = {+ method: sample.method,+ route: sample.route,+ status_code: sample.statusCode,+ };+ const seconds = sample.durationMs / MS_PER_SECOND;++ httpRequestDuration.observe(labels, seconds);+ httpRequestSummary.observe(labels, seconds);+}++/**+ * The one hook. `onAfterResponse` rather than `onAfterHandle` because+ * fastify-metrics observed in `onResponse`: an error mapped to a 4xx/5xx by+ * the error handler never reaches `onAfterHandle`, and dropping those samples+ * would quietly remove every error-status bucket the existing panels group by.+ *+ * The start time rides on a `derive`, not on `store` — `store` is app-level+ * state shared by every in-flight request, so a second concurrent request+ * would overwrite the first one's clock.+ */+export function httpMetrics() {+ return new Elysia({ name: 'core/http/metrics' })+ .derive({ as: 'global' }, () => ({ metricsStartedAt: performance.now() }))+ .onAfterResponse(+ { as: 'global' },+ ({ request, route, set, metricsStartedAt }) => {+ observeHttpRequest({+ method: request.method,+ route,+ statusCode:+ typeof set.status === 'number' ? set.status : DEFAULT_STATUS,+ durationMs: performance.now() - metricsStartedAt,+ });+ }+ );+}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 9b9c9580d..8c963ae3f 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -141,7 +141,12 @@ export type {Session,} from './context';export { createCtx, extendCtx } from './context';+// The role-conditional HTTP surfaces main.ts mounts over the three in+// rest.routes.ts (M9-002). `bullBoardRoutes` is async because the adapter+// reads its own UI assets off disk before it can register.+export { BULL_BOARD_BASE_PATH, bullBoardRoutes } from './http/bull-board';export { requestContext, requestLogging } from './http/context';+export { debugRoutes } from './http/debug.routes';// The two runtime seams over the registry (ADR-005). `main.ts` builds// producers in every role and workers only where the role consumes; both take// the registry as data, so neither opens a connection until it is called.@@ -151,6 +156,8 @@ export type {QueueDefinition,} from './jobs/define';export type { JobEnvelope, JobMeta } from './jobs/envelope';+export type { CountableQueue } from './jobs/jobs.metrics';+export { registerQueueMetrics } from './jobs/jobs.metrics';export { queueKey } from './jobs/naming';export type { CreateProducersOptions } from './jobs/producers';export { createProducers } from './jobs/producers';@@ -168,7 +175,7 @@ export type {TerminalFailure,WorkerHandle,} from './jobs/workers';-export { startWorkers } from './jobs/workers';+export { runJob, startWorkers } from './jobs/workers';export type {QueueProducerHandle,QueueProducers,@@ -181,6 +188,10 @@ export {REQUEST_ID_LENGTH,REQUEST_ID_LOG_FIELD,} from './logger';+// The one registry's own boot-time registrars. `registry` itself is NOT+// exported: a caller that can reach it can start a second collector graph+// somewhere other than a module, which is the thing §18 forbids.+export { registerDefaultMetrics } from './metrics';// Moved from apps/api/src/agents/* + packages/trpc/src/agents/filter-command.ts// (M5-005) — apps/api's `/ai/agents/*` Fastify wrapper and packages/trpc's// overview router call these.@@ -1151,6 +1162,7 @@ export async function handleMcpRequest(const token = extractToken(query, authHeader);return handleStatelessMcpRequest(token, body);}+export { isShuttingDown, setShuttingDown } from './modules/health/src/shutdown';// Dissolved from @openpanel/db's services/import.service.ts +// apps/worker's job file + apps/api's /import controller (M5-004) —// apps/worker's import job file and apps/api's import controller call these@@ -1233,6 +1245,13 @@ export {createEventsBatchHandler,startKafkaEventsConsumer,} from './modules/ingest/src/consumer';+// The readiness probe's two inputs (M9-002). `markEventsActivity` is the+// consumer's `onActivity`; `setShuttingDown` is main.ts's signal handler.+export {+ enableEventsHeartbeat,+ getEventsHeartbeat,+ markEventsActivity,+} from './modules/ingest/src/heartbeat';export type {IncomingEventDelivery,IncomingEventDeps,@@ -1243,10 +1262,7 @@ export {incomingEvent,loadIncomingEventDeps,} from './modules/ingest/src/incoming-event-handler';-// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service-// (M5-001) — apps/worker's insight job files and packages/trpc's insight-// router call these directly, the same way V1 reaches every other dissolved-// service here.+export { ingestConsumerMetrics } from './modules/ingest/src/ingest.metrics';export type {DailyInsightCandidate,GetReferrerSpikesInput,@@ -1295,6 +1311,12 @@ export {getSalts,rotateSalt,} from './modules/salt/salt.service';+// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service+// (M5-001) — apps/worker's insight job files and packages/trpc's insight+// router call these directly, the same way V1 reaches every other dissolved+// service here.+export type { SessionMetricsRedis } from './modules/session/src/session.metrics';+export { registerSessionScrapeMetrics } from './modules/session/src/session.metrics';// New module (M7-008) — apps/api's tools controller calls these directly,// the same way V1 reaches every other dissolved service here.export type {diff --git a/packages/core/src/jobs/jobs.metrics.test.ts b/packages/core/src/jobs/jobs.metrics.test.tsnew file mode 100644index 000000000..cc0d4f06e--- /dev/null+++ b/packages/core/src/jobs/jobs.metrics.test.ts@@ -0,0 +1,76 @@+import { expect, test } from 'bun:test';+import client from 'prom-client';+import { queues } from '../jobs.registry';+import { type CountableQueue, registerQueueMetrics } from './jobs.metrics';+import { queueKey } from './naming';++const COUNT_SUFFIXES = ['active', 'delayed', 'failed', 'completed', 'waiting'];++function fakeQueue(name: string, value: number): CountableQueue {+ return {+ name,+ getActiveCount: () => Promise.resolve(value),+ getDelayedCount: () => Promise.resolve(value),+ getFailedCount: () => Promise.resolve(value),+ getCompletedCount: () => Promise.resolve(value),+ getWaitingCount: () => Promise.resolve(value),+ };+}++test('five gauges per queue, named from the Redis key exactly as V1 named them', () => {+ const register = new client.Registry();+ registerQueueMetrics(+ Object.values(queues).map((definition) => fakeQueue(definition.name, 0)),+ register+ );++ const names = register.getMetricsAsArray().map((metric) => metric.name);+ expect(names).toHaveLength(+ Object.keys(queues).length * COUNT_SUFFIXES.length+ );+ for (const suffix of COUNT_SUFFIXES) {+ expect(names).toContain(`sessions_${suffix}_count`);+ expect(names).toContain(`cron_${suffix}_count`);+ // ADR-005 acceptance note: the registry key stays `cohortCompute`, so the+ // series name does too.+ expect(names).toContain(`cohortCompute_${suffix}_count`);+ }+});++test('cluster braces are stripped from the series name, not from the queue key', () => {+ const register = new client.Registry();+ const key = queueKey('cron', { cluster: true });+ expect(key).toBe('{cron}');++ registerQueueMetrics([fakeQueue(key, 0)], register);++ expect(register.getMetricsAsArray().map((metric) => metric.name)).toContain(+ 'cron_active_count'+ );+});++test('a QUEUE_NAMESPACE key stays a legal metric name', () => {+ const register = new client.Registry();+ const key = queueKey('cron', { namespace: 'proof-1' });+ expect(key).toBe('cron-proof-1');++ registerQueueMetrics([fakeQueue(key, 0)], register);++ // `-` is not legal in a prom-client name; an unsanitised key throws at+ // construction and takes boot down.+ expect(register.getMetricsAsArray().map((metric) => metric.name)).toContain(+ 'cron_proof_1_active_count'+ );+});++test('a gauge whose Redis read throws does not fail the scrape', async () => {+ const register = new client.Registry();+ const broken: CountableQueue = {+ ...fakeQueue('sessions', 7),+ getActiveCount: () => Promise.reject(new Error('redis down')),+ };+ registerQueueMetrics([broken], register);++ const text = await register.metrics();+ expect(text).toContain('sessions_waiting_count 7');+});diff --git a/packages/core/src/jobs/jobs.metrics.ts b/packages/core/src/jobs/jobs.metrics.tsnew file mode 100644index 000000000..ad56c1317--- /dev/null+++ b/packages/core/src/jobs/jobs.metrics.ts@@ -0,0 +1,130 @@+// The queue-side collectors, moved from apps/worker/src/metrics.ts onto the+// one core registry (TARGET_ARCHITECTURE §18). Names, labels and buckets are+// V1's; the worker-side copies die with apps/worker (P9).+//+// Three deliberate differences from V1, all listed in+// `packages/core/docs/OPS_GRAFANA_MIGRATION.md`:+//+// 1. V1 registered the five `<queue>_*_count` gauges for TWO queues+// (`sessionsQueue`, `cronQueue`). V2 registers them for all seven, which+// ADR-018's continuity register asks for ("kept for the 7 BullMQ+// queues"). Ten new series appear; none of the existing ten changes.+// 2. They register only where the role consumes (main.ts), because each+// gauge is one Redis round trip per scrape and ten api replicas exposing+// them would multiply that for no new information.+// 3. A gauge whose Redis read throws is skipped rather than failing the whole+// scrape. V1 let it reject `registry.metrics()`, which turns one Redis+// blip into a 500 on /metrics and a gap in every panel; core's buffer+// gauges already swallow the same way (buffers/buffer.metrics.ts).++import type { Queue as BullQueue } from 'bullmq';+import client from 'prom-client';+import { registry } from '../metrics';++const JOB_DURATION_BUCKETS_MS = [+ 10, 25, 50, 100, 250, 500, 750, 1000, 2000, 5000, 10_000, 30_000,+];++// A Redis queue key is not a legal prom-client metric name: `{cron}`'s braces+// and a `QUEUE_NAMESPACE`'s `-` are both rejected by prom-client, which throws+// at construction and takes boot down. V1 stripped the braces the same way+// (apps/worker/src/metrics.ts); the `_` substitution is the namespace half,+// and it changes nothing for a deployment that sets no namespace.+const QUEUE_BRACES = /[{}]/g;+const NON_METRIC_NAME_CHARS = /[^a-zA-Z0-9_]/g;++function metricPrefix(queueName: string): string {+ return queueName+ .replace(QUEUE_BRACES, '')+ .replace(NON_METRIC_NAME_CHARS, '_');+}++/**+ * FAILURE-ONLY, deliberately. V1 observed this exclusively in the workers'+ * `failed` listener (apps/worker/src/boot-workers.ts), so every percentile+ * ever computed from it is a percentile of failures. Success timings, if+ * wanted, get a NEW series name — emitting them here silently rewrites every+ * existing panel (ADR-018 risk 7).+ */+export const jobDurationMs = new client.Histogram({+ name: 'job_duration_ms',+ help: 'Duration of job processing (in ms)',+ labelNames: ['name', 'status'],+ buckets: JOB_DURATION_BUCKETS_MS,+ registers: [registry],+});++/** The `Queue` surface these gauges need — narrowed so a test can fake it. */+export interface CountableQueue {+ name: string;+ getActiveCount(): Promise<number>;+ getDelayedCount(): Promise<number>;+ getFailedCount(): Promise<number>;+ getCompletedCount(): Promise<number>;+ getWaitingCount(): Promise<number>;+}++const COUNTS: {+ suffix: string;+ help: string;+ read: (queue: CountableQueue) => Promise<number>;+}[] = [+ {+ suffix: 'active',+ help: 'Active count',+ read: (queue) => queue.getActiveCount(),+ },+ {+ suffix: 'delayed',+ help: 'Delayed count',+ read: (queue) => queue.getDelayedCount(),+ },+ {+ suffix: 'failed',+ help: 'Failed count',+ read: (queue) => queue.getFailedCount(),+ },+ {+ suffix: 'completed',+ help: 'Completed count',+ read: (queue) => queue.getCompletedCount(),+ },+ {+ suffix: 'waiting',+ help: 'Waiting count',+ read: (queue) => queue.getWaitingCount(),+ },+];++/**+ * Five scrape-time gauges per queue, named from the queue's Redis key exactly+ * as V1 named them. Call once per process, only where the role consumes.+ *+ * `register` defaults to the one core registry; it is a parameter so a test+ * can assert on an isolated one.+ */+export function registerQueueMetrics(+ queues: (CountableQueue | BullQueue)[],+ register: client.Registry = registry+): void {+ for (const queue of queues as CountableQueue[]) {+ const prefix = metricPrefix(queue.name);+ for (const count of COUNTS) {+ // `registers` explicitly, never prom-client's global default: a gauge+ // constructed without it lands on `client.register` too, which is a+ // second registry by any other name.+ new client.Gauge({+ name: `${prefix}_${count.suffix}_count`,+ help: count.help,+ registers: [register],+ async collect() {+ try {+ this.set(await count.read(queue));+ } catch {+ // ignore — a Redis blip must not fail the whole scrape+ }+ },+ });+ }+ }+}diff --git a/packages/core/src/jobs/schedulers.ts b/packages/core/src/jobs/schedulers.tsindex 3a9e77ac7..0ef918778 100644--- a/packages/core/src/jobs/schedulers.ts+++ b/packages/core/src/jobs/schedulers.ts@@ -43,9 +43,11 @@ export interface SchedulerQueue {jobTemplate?: { data?: unknown }): Promise<unknown>;removeJobScheduler(id: string): Promise<boolean>;+ // Mutable, not `readonly`: BullMQ's own `getJobs` declares `JobType[]`, and+ // a readonly parameter here would make the real `Queue` unassignable.getJobs(- types: readonly ('delayed' | 'waiting' | 'completed' | 'failed')[]- ): Promise<{ id?: string | null; remove(): Promise<void> }[]>;+ types: ('delayed' | 'waiting' | 'completed' | 'failed')[]+ ): Promise<{ id?: string | null; remove(): Promise<unknown> }[]>;}const CONFLICT_JOB_STATES = [diff --git a/packages/core/src/jobs/workers.ts b/packages/core/src/jobs/workers.tsindex 96ecd9ac9..135ed46a8 100644--- a/packages/core/src/jobs/workers.ts+++ b/packages/core/src/jobs/workers.ts@@ -9,6 +9,7 @@ import { generateId } from '../shared/id';import { resolveJob } from './compat';import type { QueueDefinition, QueueMap } from './define';import { isEnvelope } from './envelope';+import { jobDurationMs } from './jobs.metrics';import { queueKey } from './naming';export interface TerminalFailure {@@ -91,6 +92,16 @@ export function startWorkers<TQueues extends QueueMap>({return;}+ // FAILURE-ONLY, and the label is the Redis key, both V1's+ // (apps/worker/src/boot-workers.ts's `failed` listener). See+ // jobs.metrics.ts for why a success timing may not join this series.+ if (job.processedOn) {+ jobDurationMs.observe(+ { name: key, status: 'failed' },+ job.finishedOn - job.processedOn+ );+ }+logger.error({err: error,diff --git a/packages/core/src/metrics.ts b/packages/core/src/metrics.tsindex 09531311d..e469991a3 100644--- a/packages/core/src/metrics.ts+++ b/packages/core/src/metrics.ts@@ -10,3 +10,24 @@import client from 'prom-client';export const registry = new client.Registry();++// `collectDefaultMetrics` registers a fixed set of series names; prom-client+// throws on a duplicate, so a second call would take the process down at boot.+let defaultMetricsRegistered = false;++/**+ * prom-client's Node-derived process/GC collectors, on the one registry.+ *+ * V1 registered these twice — `fastify-metrics` did it for the api+ * (`defaultMetrics.enabled` defaults true) and `apps/worker/src/index.ts`+ * called `collectDefaultMetrics` itself. V2 has one registry per process and+ * calls this once, in every role (TARGET_ARCHITECTURE §18: "HTTP and default+ * metrics register everywhere").+ */+export function registerDefaultMetrics(): void {+ if (defaultMetricsRegistered) {+ return;+ }+ defaultMetricsRegistered = true;+ client.collectDefaultMetrics({ register: registry });+}diff --git a/packages/core/src/modules/health/health.routes.ts b/packages/core/src/modules/health/health.routes.tsindex 7c5fd215e..cf6eeddb2 100644--- a/packages/core/src/modules/health/health.routes.ts+++ b/packages/core/src/modules/health/health.routes.ts@@ -1,11 +1,17 @@-// Kubernetes liveness — shallow, event loop only. Ported from V1's-// controllers/healthcheck.controller.ts `liveness`; `/healthcheck` and-// `/healthz/ready`'s dependency/shutdown checks return once db, ch, redis and-// a shutdown flag are real (P3+) — porting them against `unknown` deps now-// would be a fake check, not a deferred one.+// Kubernetes liveness and readiness. Ported from V1's+// controllers/healthcheck.controller.ts `liveness`/`readiness` and+// apps/worker/src/index.ts's two probes, which agreed on both bodies.+//+// `/healthcheck` — the deep db/ch/redis probe — still waits for real clients+// on `AppDeps` (P3+); porting it against `unknown` deps would be a fake check,+// not a deferred one. `/healthz/ready` does NOT wait for them: its two inputs+// are the shutdown flag and the events-consumer heartbeat, both of which are+// real from P9 (M9-002), and a readiness probe that ignores a draining process+// is the one that actually loses requests.import { z } from 'zod';import { defineRoutes } from '../../http/define';+import { currentReadiness } from './src/readiness';// The first zod schema through the OpenAPI plugin (ADR-003): response schemas// generate with `io: 'output'`, which is what zod 4's native Standard Schema@@ -14,14 +20,40 @@ import { defineRoutes } from '../../http/define';// no `io` argument and would collapse the input/output split the ADR requires.const healthLiveResponseSchema = z.object({ live: z.literal(true) });+// V1's two readiness bodies, verbatim: `{ready:true}` at 200, and+// `{ready:false, reason, idleMs?, thresholdMs?}` at 503.+const healthReadyResponseSchema = z.object({+ ready: z.boolean(),+ reason: z.string().optional(),+ idleMs: z.number().optional(),+ thresholdMs: z.number().optional(),+});++const SERVICE_UNAVAILABLE = 503;+// Hidden from the OpenAPI document — ported from V1's `schema: { hide: true }`// on `/healthz/live`, `/healthz/ready` and `/healthcheck`// (healthcheck.controller.ts): liveness probes are not part of the public API// contract. See verification/golden/openapi's migrated-route gate, which pins// `/healthz` to an empty `paths` object against V1's golden document.export const healthRoutes = defineRoutes((app) =>- app.get('/healthz/live', () => ({ live: true as const }), {- response: healthLiveResponseSchema,- detail: { hide: true },- })+ app+ .get('/healthz/live', () => ({ live: true as const }), {+ response: healthLiveResponseSchema,+ detail: { hide: true },+ })+ .get(+ '/healthz/ready',+ ({ set }) => {+ const readiness = currentReadiness();+ if (!readiness.ready) {+ set.status = SERVICE_UNAVAILABLE;+ }+ return readiness;+ },+ {+ response: healthReadyResponseSchema,+ detail: { hide: true },+ }+ ));diff --git a/packages/core/src/modules/health/src/readiness.test.ts b/packages/core/src/modules/health/src/readiness.test.tsnew file mode 100644index 000000000..4f9a4e049--- /dev/null+++ b/packages/core/src/modules/health/src/readiness.test.ts@@ -0,0 +1,58 @@+import { expect, test } from 'bun:test';+import { EVENTS_HEARTBEAT_STALE_MS, evaluateReadiness } from './readiness';++const NOW = 1_000_000;+const fresh = { enabled: true, lastActivityAt: NOW };++test('shutting down wins over everything else', () => {+ expect(+ evaluateReadiness({ shuttingDown: true, heartbeat: fresh, now: NOW })+ ).toEqual({ ready: false, reason: 'shutting down' });+});++test('no events consumer on this instance skips the heartbeat check', () => {+ expect(+ evaluateReadiness({+ shuttingDown: false,+ // V1 left the heartbeat disabled on instances with no events worker,+ // whose lastActivityAt then never moves off boot time.+ heartbeat: { enabled: false, lastActivityAt: 0 },+ now: NOW,+ })+ ).toEqual({ ready: true });+});++test('a fresh consumer heartbeat is ready', () => {+ expect(+ evaluateReadiness({ shuttingDown: false, heartbeat: fresh, now: NOW })+ ).toEqual({ ready: true });+});++test('a stale consumer heartbeat is 503 with the numbers that explain it', () => {+ const idleMs = EVENTS_HEARTBEAT_STALE_MS + 1;+ expect(+ evaluateReadiness({+ shuttingDown: false,+ heartbeat: { enabled: true, lastActivityAt: NOW - idleMs },+ now: NOW,+ })+ ).toEqual({+ ready: false,+ reason: 'events consumer heartbeat stale',+ idleMs,+ thresholdMs: EVENTS_HEARTBEAT_STALE_MS,+ });+});++test('exactly at the threshold is still ready — V1 compared with >', () => {+ expect(+ evaluateReadiness({+ shuttingDown: false,+ heartbeat: {+ enabled: true,+ lastActivityAt: NOW - EVENTS_HEARTBEAT_STALE_MS,+ },+ now: NOW,+ })+ ).toEqual({ ready: true });+});diff --git a/packages/core/src/modules/health/src/readiness.ts b/packages/core/src/modules/health/src/readiness.tsnew file mode 100644index 000000000..aba6ef422--- /dev/null+++ b/packages/core/src/modules/health/src/readiness.ts@@ -0,0 +1,58 @@+// V1's `/healthz/ready` decision, lifted out of the express handler+// (apps/worker/src/index.ts) so it can be asserted without a server.+//+// Shallow + shutdown-aware, and — only where this process runs the events+// consumer — heartbeat-aware. V1 skipped the heartbeat check on instances+// with no events worker; V2 keeps that by leaving `enabled` false unless the+// role actually started the consumer.++import { getEventsHeartbeat } from '../../ingest/src/heartbeat';+import { isShuttingDown } from './shutdown';++/** V1's threshold (apps/worker/src/index.ts `EVENTS_HEARTBEAT_STALE_MS`). */+export const EVENTS_HEARTBEAT_STALE_MS = 60_000;++export type ReadinessResult =+ | { ready: true }+ | { ready: false; reason: string; idleMs?: number; thresholdMs?: number };++export interface ReadinessInputs {+ shuttingDown: boolean;+ heartbeat: { enabled: boolean; lastActivityAt: number };+ now: number;+}++/** Pure: every branch is a value, so the probe's rules are unit-testable. */+export function evaluateReadiness({+ shuttingDown,+ heartbeat,+ now,+}: ReadinessInputs): ReadinessResult {+ if (shuttingDown) {+ return { ready: false, reason: 'shutting down' };+ }++ if (!heartbeat.enabled) {+ return { ready: true };+ }++ const idleMs = now - heartbeat.lastActivityAt;+ if (idleMs > EVENTS_HEARTBEAT_STALE_MS) {+ return {+ ready: false,+ reason: 'events consumer heartbeat stale',+ idleMs,+ thresholdMs: EVENTS_HEARTBEAT_STALE_MS,+ };+ }++ return { ready: true };+}++export function currentReadiness(): ReadinessResult {+ return evaluateReadiness({+ shuttingDown: isShuttingDown(),+ heartbeat: getEventsHeartbeat(),+ now: Date.now(),+ });+}diff --git a/packages/core/src/modules/health/src/shutdown.ts b/packages/core/src/modules/health/src/shutdown.tsnew file mode 100644index 000000000..8ca9850bd--- /dev/null+++ b/packages/core/src/modules/health/src/shutdown.ts@@ -0,0 +1,14 @@+// Ported from apps/worker/src/utils/graceful-shutdown.ts (P9). One flag, set+// by main.ts's signal handler, read by the readiness probe — so a draining+// process is taken out of the load balancer before its in-flight jobs finish+// rather than after.++let shuttingDown = false;++export function setShuttingDown(value: boolean): void {+ shuttingDown = value;+}++export function isShuttingDown(): boolean {+ return shuttingDown;+}diff --git a/packages/core/src/modules/ingest/src/heartbeat.ts b/packages/core/src/modules/ingest/src/heartbeat.tsnew file mode 100644index 000000000..ae3c11842--- /dev/null+++ b/packages/core/src/modules/ingest/src/heartbeat.ts@@ -0,0 +1,30 @@+// The events consumer-loop heartbeat, ported verbatim from+// apps/worker/src/utils/worker-heartbeat.ts (P9). It is what turns "the+// process is up" into "the Kafka consumer is still turning": the timestamp is+// refreshed on every kafkajs HEARTBEAT and after every handled batch+// (consumer.ts's `onActivity`), so a healthy loop refreshes it even with no+// traffic, and a wedged one goes stale.+//+// Module state rather than a field on `AppDeps` because the readiness probe+// and the consumer are on opposite sides of the process and neither owns the+// other; there is exactly one consumer per process.++let enabled = false;+let lastActivityAt = Date.now();++/** Called only by a role that actually starts the consumer. */+export function enableEventsHeartbeat(): void {+ enabled = true;+ lastActivityAt = Date.now();+}++export function markEventsActivity(): void {+ lastActivityAt = Date.now();+}++export function getEventsHeartbeat(): {+ enabled: boolean;+ lastActivityAt: number;+} {+ return { enabled, lastActivityAt };+}diff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.tsindex 2c4762133..366996c3d 100644--- a/packages/core/src/modules/ingest/src/ingest.metrics.ts+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts@@ -7,6 +7,7 @@import client from 'prom-client';import { registry } from '../../../metrics';+import type { ConsumerMetrics, DeadLetterReason } from './consumer';// Kafka event messages reprocessed (same offset redelivered outside a// rebalance). Should stay ~0. A sustained non-zero rate means the consumer is@@ -52,3 +53,18 @@ export const sessionsStarted = new client.Counter({labelNames: ['kind'], // 'new' | 'boundary'registers: [registry],});++/**+ * The consumer's `ConsumerMetrics`, bound to the counters above. Exists so+ * `main.ts` wires the consumer without reaching for four individual counters+ * — a metric is a module's business, not the entrypoint's.+ */+export const ingestConsumerMetrics = {+ reprocessed: (partition: string) => kafkaReprocessedTotal.inc({ partition }),+ handlerFailed: (partition: string) =>+ kafkaHandlerFailuresTotal.inc({ partition }),+ deadLettered: (partition: string, reason: DeadLetterReason) =>+ kafkaDeadLetteredTotal.inc({ partition, reason }),+ deadLetterFailed: (partition: string) =>+ kafkaDeadLetterFailedTotal.inc({ partition }),+} satisfies ConsumerMetrics;diff --git a/packages/core/src/modules/session/src/session.metrics.scrape.test.ts b/packages/core/src/modules/session/src/session.metrics.scrape.test.tsnew file mode 100644index 000000000..136b4aa8f--- /dev/null+++ b/packages/core/src/modules/session/src/session.metrics.scrape.test.ts@@ -0,0 +1,78 @@+// The three at-scrape gauges do arithmetic over a Redis pipeline result, which+// is the half a name-only assertion would not catch.+import { expect, test } from 'bun:test';+import client from 'prom-client';+import {+ registerSessionScrapeMetrics,+ type SessionMetricsRedis,+} from './session.metrics';++type Reply = [Error | null, unknown][] | null;++function fakeRedis(projects: string[], replies: Reply): SessionMetricsRedis {+ return {+ smembers: () => Promise.resolve(projects),+ scard: () => Promise.resolve(projects.length),+ multi: () => ({+ zcard: () => undefined,+ get: () => undefined,+ exec: () => Promise.resolve(replies),+ }),+ };+}++test('sessions_active_total sums the per-project ZCARDs', async () => {+ const register = new client.Registry();+ registerSessionScrapeMetrics(+ () =>+ fakeRedis(+ ['proj_a', 'proj_b'],+ [+ [null, 3],+ [null, 4],+ ]+ ),+ register+ );++ const text = await register.metrics();+ expect(text).toContain('sessions_active_total 7');+ expect(text).toContain('sessions_projects_active 2');+});++test('no active projects reads zero rather than skipping the sample', async () => {+ const register = new client.Registry();+ registerSessionScrapeMetrics(() => fakeRedis([], null), register);++ const text = await register.metrics();+ expect(text).toContain('sessions_active_total 0');+ expect(text).toContain('sessions_hwm_lag_ms 0');+});++test('sessions_hwm_lag_ms takes the MAX lag and ignores unset HWMs', async () => {+ const register = new client.Registry();+ const now = Date.now();+ registerSessionScrapeMetrics(+ () =>+ fakeRedis(+ ['proj_a', 'proj_b', 'proj_c'],+ [+ [null, String(now - 5000)],+ // 0 / null means the project has no HWM yet — not a lag of `now`.+ [null, '0'],+ [null, String(now - 60_000)],+ ]+ ),+ register+ );++ const text = await register.metrics();+ const lag = Number(+ text+ .split('\n')+ .find((line) => line.startsWith('sessions_hwm_lag_ms '))+ ?.split(' ')[1]+ );+ expect(lag).toBeGreaterThanOrEqual(60_000);+ expect(lag).toBeLessThan(70_000);+});diff --git a/packages/core/src/modules/session/src/session.metrics.ts b/packages/core/src/modules/session/src/session.metrics.tsindex 38b8050b4..ca6ce7ff4 100644--- a/packages/core/src/modules/session/src/session.metrics.ts+++ b/packages/core/src/modules/session/src/session.metrics.ts@@ -1,12 +1,31 @@// The session lifecycle collectors, moved from apps/worker/src/metrics.ts// (M7-001) onto core's one registry. Names, labels and buckets are V1's; the-// worker-side copies die with apps/worker (P9). The at-scrape gauges-// (`sessions_active_total`, `sessions_projects_active`, `sessions_hwm_lag_ms`)-// and the ingest-side `sessions_started_total` stay with the ingest path.+// worker-side copies die with apps/worker (P9). The ingest-side+// `sessions_started_total` lives with the ingest path+// (modules/ingest/src/ingest.metrics.ts); the three at-scrape gauges joined+// this file at M9-002 — see `registerSessionScrapeMetrics` at the bottom.import client from 'prom-client';import { registry } from '../../../metrics';+// V1's key names, unchanged (apps/worker/src/metrics.ts).+const SESSION_PROJECTS_KEY = 'session:projects';++/**+ * The slice of the cache client the gauges use. `multi()` is V1's, and it is+ * batching rather than a transaction — the ADR-006 swap replaces it with+ * `Promise.all` over Bun's auto-pipelining when packages/redis moves.+ */+export interface SessionMetricsRedis {+ smembers(key: string): Promise<string[]>;+ scard(key: string): Promise<number>;+ multi(): {+ zcard(key: string): unknown;+ get(key: string): unknown;+ exec(): Promise<[Error | null, unknown][] | null>;+ };+}+const SECOND_MS = 1000;const MINUTE_MS = 60 * SECOND_MS;const HOUR_MS = 60 * MINUTE_MS;@@ -75,3 +94,88 @@ export const sessionsVacuumed = new client.Counter({labelNames: ['reason'], // 'stale_blob' | 'missing_blob'registers: [registry],});++/**+ * The three scrape-time session gauges, moved from apps/worker/src/metrics.ts.+ *+ * Registered by main.ts and ONLY where the role consumes queues: each scrape+ * costs one `ZCARD` and one `GET` per project, and ten api replicas exposing+ * them would multiply that Redis load for no new information+ * (TARGET_ARCHITECTURE §18).+ *+ * The client is injected rather than imported so core does not open a Redis+ * connection at import time — `main.ts` hands in `getRedisCache`.+ */+export function registerSessionScrapeMetrics(+ getRedis: () => SessionMetricsRedis,+ register: client.Registry = registry+): void {+ // `registers` explicitly, never prom-client's global default — see+ // jobs/jobs.metrics.ts.+ const registerOn = [register];++ new client.Gauge({+ name: 'sessions_active_total',+ help: 'Active sessions in Redis across all projects',+ registers: registerOn,+ async collect() {+ const redis = getRedis();+ const projectIds = await redis.smembers(SESSION_PROJECTS_KEY);+ if (projectIds.length === 0) {+ this.set(0);+ return;+ }+ const multi = redis.multi();+ for (const projectId of projectIds) {+ multi.zcard(`session:wallclock:${projectId}`);+ }+ const results = await multi.exec();+ let total = 0;+ for (const entry of results ?? []) {+ const count = Number(entry?.[1] ?? 0);+ if (Number.isFinite(count)) {+ total += count;+ }+ }+ this.set(total);+ },+ });++ new client.Gauge({+ name: 'sessions_projects_active',+ help: 'Projects with at least one active session',+ registers: registerOn,+ async collect() {+ this.set(await getRedis().scard(SESSION_PROJECTS_KEY));+ },+ });++ new client.Gauge({+ name: 'sessions_hwm_lag_ms',+ help: 'Max lag (ms) between wall-clock now and project event-time HWM, across all projects. Big number → queue lag or imports.',+ registers: registerOn,+ async collect() {+ const redis = getRedis();+ const projectIds = await redis.smembers(SESSION_PROJECTS_KEY);+ if (projectIds.length === 0) {+ this.set(0);+ return;+ }+ const multi = redis.multi();+ for (const projectId of projectIds) {+ multi.get(`session:hwm:${projectId}`);+ }+ const results = await multi.exec();+ const now = Date.now();+ let maxLag = 0;+ for (const entry of results ?? []) {+ const hwm = Number(entry?.[1] ?? 0);+ if (!Number.isFinite(hwm) || hwm <= 0) {+ continue;+ }+ maxLag = Math.max(maxLag, now - hwm);+ }+ this.set(maxLag);+ },+ });+}diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.tsindex 80563b023..e6496dba3 100644--- a/packages/core/src/rest.routes.ts+++ b/packages/core/src/rest.routes.ts@@ -7,6 +7,7 @@import { openapi } from '@elysiajs/openapi';import { Elysia } from 'elysia';import type { AppDeps } from './context';+import { httpMetrics } from './http/http.metrics';import { registry } from './metrics';import { assistantRoutes } from './modules/assistant/assistant.routes';import { authRoutes } from './modules/auth/auth.routes';@@ -90,6 +91,11 @@ export const dashboardRoutes = (deps: AppDeps) =>.use(realtimeRoutes(deps)).use(miscRoutes(deps));+// `httpMetrics` is on the ops surface because that is where `/metrics` is+// served and every role mounts it (TARGET_ARCHITECTURE §18: "HTTP and default+// metrics register everywhere"). Its hook is `{ as: 'global' }`, so it covers+// the dashboard and public-API surfaces too once they mount.+//// healthz/metrics/misc share V1's ops surface (http/context.ts's// UNLOGGED_PATH_PREFIXES) — unauthenticated, uncorsed, unlogged. /metrics and// /openapi.json are the M3 gate's ops surface: the one core registry's@@ -97,6 +103,7 @@ export const dashboardRoutes = (deps: AppDeps) =>// schemas the routes validate against.export const opsRoutes = (deps: AppDeps) =>new Elysia({ name: 'core/ops-routes' })+ .use(httpMetrics()).use(healthRoutes(deps)).use(openapi({diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 406021958..37b478f1d 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1079,6 +1079,12 @@ importers:'@better-agent/providers':specifier: ^0.1.0-beta.3version: 0.1.0-beta.3(@better-agent/[email protected])+ '@bull-board/api':+ specifier: 9.6.0+ version: 9.6.0(@bull-board/[email protected]([email protected]([email protected]([email protected]))))([email protected]([email protected]([email protected])))+ '@bull-board/elysia':+ specifier: 9.6.0+ version: 9.6.0([email protected]([email protected]([email protected])))([email protected](@sinclair/[email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]))'@elysiajs/cors':specifier: 1.4.2version: 1.4.2([email protected](@sinclair/[email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]))@@ -3273,12 +3279,32 @@ packages:'@bull-board/ui': 6.14.0bullmq: '*'+ '@bull-board/[email protected]':+ resolution: {integrity: sha512-RJcXIaFDO2l76t1MNxhIiKNaDIKIm4F8vsQq32grV9Rw4DiclKFlXSht2SN/+VIVyHJ7NzlCskMd8HgMpb/aEg==}+ peerDependencies:+ '@bull-board/ui': 9.6.0+ bull: ^4.16.5+ bullmq: ^5.56.0 || ^6.0.0+ peerDependenciesMeta:+ bull:+ optional: true+ bullmq:+ optional: true++ '@bull-board/[email protected]':+ resolution: {integrity: sha512-rWfOfleHGwIgXfkznO2MMwSGEMOP78cw0jmQINL946w66J9z532JZoRvgmV8FPZM+NZghvtzVT7AsB6/M881Uw==}+ peerDependencies:+ elysia: ^1.1.0+'@bull-board/[email protected]':resolution: {integrity: sha512-3H1ame2G1+eVnqqSsw6KfzTGYAWSpVsIx6EPwg9vPSP2eKfNAm12Cm4zvL6ZkwAvTCkAByt5PPDRWbbwWB6HHQ==}'@bull-board/[email protected]':resolution: {integrity: sha512-5yqfS9CwWR8DBxpReIbqv/VSPFM/zT4KZ75keyApMiejasRC2joaHqEzYWlMCjkMycbNNCvlQNlTbl+C3dE/dg==}+ '@bull-board/[email protected]':+ resolution: {integrity: sha512-rJDD+CvxI43Avq2gZpHunPT67HToYXyHSRoXsiVXDh/6MB93YtU+ofkgrrYBY25FswDSvtfnQN/ID3Ypf9OyOg==}+'@capsizecss/[email protected]':resolution: {integrity: sha512-VERIM64vtTP1C4mxQ5thVT9fK0apjPFobqybMtA1UdUujWka24ERHbRHFGmpbbhp73MhV+KSsHQH9C6uOTdEQA==}engines: {node: '>=18'}@@ -11908,6 +11934,11 @@ packages:engines: {node: '>=0.10.0'}hasBin: true+ resolution: {integrity: sha512-UaaM14yby8U3k02ihS1Bmj5Kz2d7CCQM1scxpgs4Mhkq8F1wR2gl3+Ts4h5Ne4Mnt7M9m4Dw7jsuMr3+xO4vZA==}+ engines: {node: '>=0.12.18'}+ hasBin: true+resolution: {integrity: sha512-NlAsMteRHek05jRUxUR0a5jpjYq9ykk6+kO0yRaMi5moe7u0fVIOeQ3Y30A8dIiWFBNUoQGi1ljb1i5VtS9WQQ==}@@ -22476,6 +22507,24 @@ snapshots:bullmq: 5.63.0([email protected])redis-info: 3.1.0+ '@bull-board/[email protected](@bull-board/[email protected]([email protected]([email protected]([email protected]))))([email protected]([email protected]([email protected])))':+ dependencies:+ redis-info: 3.1.0+ optionalDependencies:+ bullmq: 6.3.3([email protected]([email protected]))++ '@bull-board/[email protected]([email protected]([email protected]([email protected])))([email protected](@sinclair/[email protected])([email protected])([email protected]([email protected]))([email protected])([email protected]))':+ dependencies:+ '@bull-board/api': 9.6.0(@bull-board/[email protected]([email protected]([email protected]([email protected]))))([email protected]([email protected]([email protected])))+ ejs: 6.0.1+ elysia: 1.4.30(@sinclair/[email protected])([email protected])([email protected]([email protected]))([email protected])([email protected])+ mime: 4.1.0+ transitivePeerDependencies:+ - bull+ - bullmq+dependencies:'@bull-board/api': 6.14.0(@bull-board/[email protected]([email protected]([email protected])))([email protected]([email protected]))@@ -22492,6 +22541,13 @@ snapshots:transitivePeerDependencies:- bullmq+ dependencies:+ '@bull-board/api': 9.6.0(@bull-board/[email protected]([email protected]([email protected]([email protected]))))([email protected]([email protected]([email protected])))+ transitivePeerDependencies:+ - bull+ - bullmq+'@capsizecss/[email protected]':dependencies:fontkitten: 1.0.3@@ -32269,6 +32325,8 @@ snapshots:dependencies:jake: 10.8.7+ [email protected]: {}+