OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
chore: remove dead code and leftover rewrite scripts
Carl-Gerhard Lindesvärd9ddd37f4a683
diff --git a/.cursorrules b/.cursorrulesdeleted file mode 100644index 32e08004b..000000000--- a/.cursorrules+++ /dev/null@@ -1,3 +0,0 @@-- When we write clickhouse queries you should always use the custom query builder we have in- - `./packages/db/src/clickhouse/query-builder.ts`- - `./packages/db/src/clickhouse/query-functions.ts`\ No newline at end of filediff --git a/.dev-stack-supervisor.sh b/.dev-stack-supervisor.shdeleted file mode 100755index 6f91003c9..000000000--- a/.dev-stack-supervisor.sh+++ /dev/null@@ -1,11 +0,0 @@-#!/usr/bin/env bash-# Operator-run: keeps the api/worker dev stack up for the M15 wave's e2e gates.-# The stack currently exits on an unhandled rejection when http.metrics.ts-# observes a NaN duration (a 404 can trigger it), so a bare `bun run dev` dies-# and every task with an e2e gate then fails at preflight.-cd /home/deploy/openpanel/apps/api || exit 1-while true; do- API_PORT=3333 bun run dev >> /tmp/api-dev.log 2>&1- echo "[$(date -u +%FT%TZ)] dev stack exited, restarting in 3s" >> /tmp/api-dev.log- sleep 3-donediff --git a/.m39-004-full.log b/.m39-004-full.logdeleted file mode 100644index b2af8a96a..000000000--- a/.m39-004-full.log+++ /dev/null@@ -1,11 +0,0 @@-OK: no forbidden patterns-== bun install --frozen-lockfile (/home/deploy/openpanel) ==-[0.05ms] ".env"-bun install v1.4.0 (34cbb9a40)--Checked 3036 installs across 3405 packages (no changes) [126.00ms]-OK: bun install-== build api: skipped (V1 fastify boot deleted; apps/api runs from source) ==-== typecheck api (/home/deploy/openpanel/apps/api) ==-$ tsc --noEmit-error: script "typecheck" was terminated by signal SIGTERM (Polite quit request)diff --git a/.m39-full.log b/.m39-full.logdeleted file mode 100644index 29b544556..000000000--- a/.m39-full.log+++ /dev/null@@ -1,14 +0,0 @@-OK: no forbidden patterns-== bun install --frozen-lockfile (/home/deploy/openpanel) ==-[0.05ms] ".env"-bun install v1.4.0 (34cbb9a40)--Checked 3036 installs across 3405 packages (no changes) [133.00ms]-OK: bun install-== build api: skipped (V1 fastify boot deleted; apps/api runs from source) ==-== typecheck api (/home/deploy/openpanel/apps/api) ==-$ tsc --noEmit-OK: typecheck api-== typecheck start (/home/deploy/openpanel/apps/start) ==-$ tsc --noEmit-error: script "typecheck" was terminated by signal SIGTERM (Polite quit request)diff --git a/apps/api/e2e/boot-proof.sh b/apps/api/e2e/boot-proof.shdeleted file mode 100755index b600debeb..000000000--- a/apps/api/e2e/boot-proof.sh+++ /dev/null@@ -1,530 +0,0 @@-#!/usr/bin/env bash-# Proves `bun run apps/api/src/main.ts` actually boots — every ROLE the ROLE-# table declares — rather than merely typechecking (playbook rule 1).-#-# M3-CLOSE-002 wrote parts 1-3; M9-002 added parts 4-6; M9-003 (which deleted-# apps/worker) flipped the unhandled-scheduler check in part 4 from "names the-# three missing handlers" to "there are none".-#-# 1. An unknown ROLE fails boot loudly (non-zero exit, named value in the-# message) — and so does an unknown ENABLED_QUEUES token.-# 2. ROLE=api serves /healthz/live, /healthz/ready and /metrics, runs NO-# workers/schedulers/consumer, and mounts neither bull-board nor the-# debug routes.-# 3. SIGTERM shuts the process down within 5s, exit code 0, no orphan left-# behind.-# 4. ROLE=worker starts all seven BullMQ workers, the cron schedulers and-# the Kafka ingest consumer; exposes the consuming-role scrape gauges;-# serves bull-board behind the session guard (401, not 200 and not 404).-# 5. ROLE=all does everything ROLE=worker does AND sets redis-# keyspace-notify, which ROLE=worker must not.-# 6. The two opt-outs: DISABLE_BULLBOARD unmounts the UI, DISABLE_WORKERS-# leaves the ops surface up with nothing consuming.-#-# Parts 4-5 need the real backing services, so they source the repo `.env`-# and are ISOLATED from whatever else uses this box:-#-# * QUEUE_NAMESPACE suffixes every BullMQ key, so the proof cannot consume-# or prune the live `cron`/`sessions`/... keys. The un-namespaced keys-# byte-identity is pinned separately by-# packages/core/src/jobs/naming.test.ts and the controllers queue-keys-# golden.-# * KAFKA_CONSUMER_GROUP is a proof-only group, so joining does not-# rebalance the real `openpanel-events` consumers. The topic is the real-# one and is only read from `latest`.-#-# Run from anywhere: `bash apps/api/e2e/boot-proof.sh`.--set -u--REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"-cd "$REPO_ROOT" || exit 1--MAIN_TS="apps/api/src/main.ts"-SHUTDOWN_TIMEOUT_S=5-FAILURES=0--pass() { echo "PASS: $1"; }-fail() {- echo "FAIL: $1"- FAILURES=$((FAILURES + 1))-}--assert_contains() {- if printf '%s' "$1" | grep -qF -- "$2"; then- pass "$3"- else- fail "$3 (missing: $2)"- fi-}--assert_absent() {- if printf '%s' "$1" | grep -qF -- "$2"; then- fail "$3 (unexpectedly present: $2)"- else- pass "$3"- fi-}--# A real free port, not a guessed constant — this box runs other processes.-find_free_port() {- bun -e 'const s = Bun.listen({ port: 0, hostname: "127.0.0.1", socket: { data() {} } }); console.log(s.port); s.stop();'-}--# --- Part 1: an unknown ROLE / ENABLED_QUEUES fails boot loudly ------------UNKNOWN_LOG="$(mktemp)"-ROLE=bogus API_PORT=0 bun run "$MAIN_TS" >"$UNKNOWN_LOG" 2>&1-UNKNOWN_EXIT=$?--if [ "$UNKNOWN_EXIT" -ne 0 ]; then- pass "unknown ROLE exits non-zero (got $UNKNOWN_EXIT)"-else- fail "unknown ROLE should exit non-zero, got $UNKNOWN_EXIT"-fi--if grep -qi 'unknown value' "$UNKNOWN_LOG" && grep -q 'bogus' "$UNKNOWN_LOG"; then- pass "unknown ROLE names the offending value in a clear message"-else- fail "unknown ROLE message doesn't name the value — log follows:"- cat "$UNKNOWN_LOG"-fi-rm -f "$UNKNOWN_LOG"--QUEUES_LOG="$(mktemp)"-ROLE=worker ENABLED_QUEUES=events_kafka API_PORT=0 bun run "$MAIN_TS" >"$QUEUES_LOG" 2>&1-QUEUES_EXIT=$?--if [ "$QUEUES_EXIT" -ne 0 ] && grep -q 'events_kafka' "$QUEUES_LOG" \- && grep -q 'was renamed to' "$QUEUES_LOG"; then- pass "ENABLED_QUEUES=events_kafka exits non-zero and names the rename"-else- fail "ENABLED_QUEUES=events_kafka should exit non-zero naming the rename (exit $QUEUES_EXIT) — log follows:"- cat "$QUEUES_LOG"-fi-rm -f "$QUEUES_LOG"--# --- Part 2: ROLE=api boots and serves the ops surface ---------------------PORT="$(find_free_port)"-BOOT_LOG="$(mktemp)"--ROLE=api API_PORT="$PORT" bun run "$MAIN_TS" >"$BOOT_LOG" 2>&1 &-BOOT_PID=$!--READY=0-for _ in $(seq 1 50); do- if curl -s -o /dev/null "http://127.0.0.1:$PORT/healthz/live"; then- READY=1- break- fi- # A dead boot process will never become ready — stop waiting for it.- if ! kill -0 "$BOOT_PID" 2>/dev/null; then- break- fi- sleep 0.1-done--if [ "$READY" -ne 1 ]; then- fail "server on port $PORT never became ready — log follows:"- cat "$BOOT_LOG"- kill -9 "$BOOT_PID" 2>/dev/null- rm -f "$BOOT_LOG"- echo "boot-proof.sh: $FAILURES failure(s)"- exit 1-fi--HEALTHZ_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/healthz/live")-if [ "$HEALTHZ_CODE" = "200" ]; then- pass "GET /healthz/live -> 200"-else- fail "GET /healthz/live -> $HEALTHZ_CODE, expected 200"-fi--READY_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/healthz/ready")-if [ "$READY_CODE" = "200" ]; then- pass "GET /healthz/ready -> 200 (ROLE=api: no consumer, so no heartbeat gate)"-else- fail "GET /healthz/ready -> $READY_CODE, expected 200"-fi--METRICS_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/metrics")-if [ "$METRICS_CODE" = "200" ]; then- pass "GET /metrics -> 200"-else- fail "GET /metrics -> $METRICS_CODE, expected 200"-fi--API_METRICS="$(curl -s "http://127.0.0.1:$PORT/metrics")"-assert_contains "$API_METRICS" 'process_cpu_seconds_total' \- "ROLE=api /metrics carries the default process metrics"-assert_contains "$API_METRICS" 'http_request_duration_seconds' \- "ROLE=api /metrics carries the route histogram"-assert_contains "$API_METRICS" 'http_request_summary_seconds' \- "ROLE=api /metrics carries the route summary"-# One registry: the queue/buffer/session scrape collectors register ONLY where-# the role consumes (TARGET_ARCHITECTURE 18).-assert_absent "$API_METRICS" 'sessions_waiting_count' \- "ROLE=api does NOT register the queue gauges"-assert_absent "$API_METRICS" 'sessions_active_total' \- "ROLE=api does NOT register the session scrape gauges"-assert_absent "$API_METRICS" 'buffer_event_count' \- "ROLE=api does NOT register the buffer gauges"--BOARD_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/bullboard")-if [ "$BOARD_CODE" = "404" ]; then- pass "ROLE=api does NOT mount bull-board (GET /bullboard -> 404)"-else- fail "ROLE=api should not mount bull-board, got $BOARD_CODE"-fi--if grep -q '"msg":"worker started"' "$BOOT_LOG"; then- fail "ROLE=api must not start workers — log follows:"- cat "$BOOT_LOG"-else- pass "ROLE=api starts no workers, schedulers or ingest consumer"-fi--# The debug routes lived on the worker in V1; ROLE=api never had them.-DEBUG_CODE=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/debug/cron")-if [ "$DEBUG_CODE" = "404" ]; then- pass "ROLE=api does NOT mount the debug routes (GET /debug/cron -> 404)"-else- fail "ROLE=api should not mount the debug routes, got $DEBUG_CODE"-fi--# --- Part 3: SIGTERM shuts down within 5s, exit 0, no orphan ---------------kill -TERM "$BOOT_PID"--WAITED=0-while kill -0 "$BOOT_PID" 2>/dev/null; do- sleep 0.1- WAITED=$((WAITED + 1))- if [ "$WAITED" -ge $((SHUTDOWN_TIMEOUT_S * 10)) ]; then- break- fi-done--if kill -0 "$BOOT_PID" 2>/dev/null; then- fail "process still alive ${SHUTDOWN_TIMEOUT_S}s after SIGTERM — killing it"- kill -9 "$BOOT_PID" 2>/dev/null- wait "$BOOT_PID" 2>/dev/null-else- wait "$BOOT_PID" 2>/dev/null- SHUTDOWN_EXIT=$?- pass "process exited within ${SHUTDOWN_TIMEOUT_S}s of SIGTERM"- if [ "$SHUTDOWN_EXIT" -eq 0 ]; then- pass "SIGTERM shutdown exit code 0 (got $SHUTDOWN_EXIT)"- else- fail "SIGTERM shutdown exit code $SHUTDOWN_EXIT, expected 0"- fi-fi--# The process itself is gone by now; confirm nothing it spawned lingers.-sleep 0.3-ORPHANS="$(pgrep -f "$MAIN_TS" || true)"-if [ -z "$ORPHANS" ]; then- pass "no orphan process left behind"-else- fail "orphan process(es) still running: $ORPHANS"- # shellcheck disable=SC2086- kill -9 $ORPHANS 2>/dev/null-fi--rm -f "$BOOT_LOG"--# --- Parts 4-5: the consuming roles ---------------------------------------#-# These need the real Redis/Kafka/Postgres, so they source `.env`. Isolated-# by QUEUE_NAMESPACE + a proof-only Kafka consumer group (see the header).--if [ ! -f "$REPO_ROOT/.env" ]; then- fail "$REPO_ROOT/.env is missing — parts 4-5 need REDIS_URL/KAFKA_BROKERS/DATABASE_URL"- echo- echo "boot-proof.sh: $FAILURES failure(s)"- exit 1-fi--PROOF_NAMESPACE="bootproof$$"-PROOF_KAFKA_GROUP="openpanel-events-bootproof-$$"-CONSUMING_READY_TIMEOUT_S=90--# Boots one consuming role, waits for it to listen, and leaves $ROLE_PID and-# $ROLE_LOG set for the caller's assertions.-boot_consuming_role() { boot_consuming_role_with_env "$1"; }--# `boot_consuming_role_with_env <role> [NAME=VALUE ...]`-boot_consuming_role_with_env() {- local role="$1"- shift- ROLE_PORT="$(find_free_port)"- ROLE_LOG="$(mktemp)"-- (- set -a- # shellcheck disable=SC1091- . "$REPO_ROOT/.env"- set +a- export ROLE="$role"- export API_PORT="$ROLE_PORT"- export QUEUE_NAMESPACE="$PROOF_NAMESPACE"- export KAFKA_CONSUMER_GROUP="$PROOF_KAFKA_GROUP"- for assignment in "$@"; do- export "${assignment?}"- done- exec bun run "$MAIN_TS"- ) >"$ROLE_LOG" 2>&1 &- ROLE_PID=$!-- local waited=0- until curl -s -o /dev/null "http://127.0.0.1:$ROLE_PORT/healthz/live"; do- if ! kill -0 "$ROLE_PID" 2>/dev/null; then- fail "ROLE=$role exited before listening — log follows:"- cat "$ROLE_LOG"- return 1- fi- sleep 1- waited=$((waited + 1))- if [ "$waited" -ge "$CONSUMING_READY_TIMEOUT_S" ]; then- fail "ROLE=$role did not listen within ${CONSUMING_READY_TIMEOUT_S}s — log follows:"- cat "$ROLE_LOG"- kill -9 "$ROLE_PID" 2>/dev/null- return 1- fi- done- pass "ROLE=$role booted and is listening on :$ROLE_PORT (${waited}s)"- return 0-}--stop_consuming_role() {- local role="$1"- kill -TERM "$ROLE_PID" 2>/dev/null- local waited=0- while kill -0 "$ROLE_PID" 2>/dev/null; do- sleep 1- waited=$((waited + 1))- if [ "$waited" -ge 30 ]; then- break- fi- done- if kill -0 "$ROLE_PID" 2>/dev/null; then- fail "ROLE=$role still alive 30s after SIGTERM — killing it"- kill -9 "$ROLE_PID" 2>/dev/null- wait "$ROLE_PID" 2>/dev/null- else- wait "$ROLE_PID" 2>/dev/null- local exit_code=$?- if [ "$exit_code" -eq 0 ]; then- pass "ROLE=$role SIGTERM shutdown exit code 0 (${waited}s)"- else- fail "ROLE=$role SIGTERM shutdown exit code $exit_code, expected 0"- fi- fi-}--# The seven registry queues (jobs.registry.ts) — the same names as the Redis-# keys, `cohortCompute` included.-REGISTRY_QUEUES="sessions cron notification import insights gsc cohortCompute"--assert_consuming_role() {- local role="$1" log="$2" port="$3"-- local started- started=$(grep -c '"msg":"worker started"' "$log")- if [ "$started" -eq 7 ]; then- pass "ROLE=$role started all 7 BullMQ workers"- else- fail "ROLE=$role started $started workers, expected 7"- fi-- for queue in $REGISTRY_QUEUES; do- if grep -q "\"key\":\"${queue}-${PROOF_NAMESPACE}\"" "$log"; then- pass "ROLE=$role worker on queue key ${queue}-${PROOF_NAMESPACE}"- else- fail "ROLE=$role has no worker for queue $queue"- fi- done-- if grep -q '"msg":"updating cron jobs"' "$log"; then- pass "ROLE=$role upserted the cron schedulers"- else- fail "ROLE=$role did not run startSchedulers"- fi-- if grep -q '"msg":"kafka events consumer running"' "$log"; then- pass "ROLE=$role started the Kafka ingest consumer"- else- fail "ROLE=$role did not start the Kafka ingest consumer"- fi-- # M9-003 landed dataHealth/windDown/flushExports, so every one of the 20- # scheduler ids now has a handler on the `cron` queue and this line must NOT- # appear. It is the runtime half of "every job the old worker ran is served- # by the merged app under ROLE=worker" — the static half is- # jobs.registry.test.ts's both-ways scheduler/handler match.- if grep -q 'cron schedulers have no handler' "$log"; then- fail "ROLE=$role reported cron schedulers with no handler: $(grep -o '"schedulers":\[[^]]*\]' "$log" | head -1)"- else- pass "ROLE=$role: every declared cron scheduler has a handler"- fi-- local ready_code- ready_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/healthz/ready")- if [ "$ready_code" = "200" ]; then- pass "ROLE=$role GET /healthz/ready -> 200 (consumer heartbeat fresh)"- else- fail "ROLE=$role GET /healthz/ready -> $ready_code, expected 200"- fi-- local metrics- metrics="$(curl -s "http://127.0.0.1:$port/metrics")"- assert_contains "$metrics" 'process_cpu_seconds_total' \- "ROLE=$role /metrics carries the default process metrics"- assert_contains "$metrics" 'http_request_duration_seconds' \- "ROLE=$role /metrics carries the route histogram"- assert_contains "$metrics" 'job_duration_ms' \- "ROLE=$role /metrics carries job_duration_ms"- assert_contains "$metrics" 'kafka_events_reprocessed_total' \- "ROLE=$role /metrics carries the kafka counters"- assert_contains "$metrics" 'sessions_active_total' \- "ROLE=$role /metrics carries the session scrape gauges"- assert_contains "$metrics" 'buffer_event_count' \- "ROLE=$role /metrics carries the buffer gauges"- for queue in $REGISTRY_QUEUES; do- assert_contains "$metrics" "${queue}_${PROOF_NAMESPACE}_waiting_count" \- "ROLE=$role /metrics carries the $queue queue gauges"- done-- local board_code- board_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/bullboard")- if [ "$board_code" = "401" ]; then- pass "ROLE=$role mounts bull-board BEHIND the session guard (401, not 200)"- else- fail "ROLE=$role GET /bullboard -> $board_code, expected 401"- fi-- local debug_code- debug_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$port/debug/cron")- if [ "$debug_code" = "200" ]; then- pass "ROLE=$role serves the dev-only debug routes (NODE_ENV is not production)"- else- fail "ROLE=$role GET /debug/cron -> $debug_code, expected 200"- fi-}--echo-echo "--- Part 4: ROLE=worker ---"-if boot_consuming_role worker; then- WORKER_LOG="$ROLE_LOG"- assert_consuming_role worker "$WORKER_LOG" "$ROLE_PORT"-- if grep -q 'keyspace notifications configured' "$WORKER_LOG"; then- fail "ROLE=worker must NOT set redis keyspace notifications"- else- pass "ROLE=worker does NOT set redis keyspace notifications"- fi-- stop_consuming_role worker- rm -f "$WORKER_LOG"-fi--echo-echo "--- Part 5: ROLE=all ---"-if boot_consuming_role all; then- ALL_LOG="$ROLE_LOG"- assert_consuming_role all "$ALL_LOG" "$ROLE_PORT"-- if grep -q 'keyspace notifications configured' "$ALL_LOG"; then- pass "ROLE=all sets redis keyspace notifications (ROLE != worker)"- else- fail "ROLE=all did not set redis keyspace notifications"- fi-- stop_consuming_role all- rm -f "$ALL_LOG"-fi--echo-echo "--- Part 6: the DISABLE_* opt-outs ---"-if boot_consuming_role_with_env worker DISABLE_BULLBOARD=1 DISABLE_WORKERS=1; then- OPTOUT_LOG="$ROLE_LOG"-- board_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$ROLE_PORT/bullboard")- if [ "$board_code" = "404" ]; then- pass "DISABLE_BULLBOARD=1 unmounts bull-board (GET /bullboard -> 404)"- else- fail "DISABLE_BULLBOARD=1 should unmount bull-board, got $board_code"- fi-- if grep -q '"msg":"worker started"' "$OPTOUT_LOG"; then- fail "DISABLE_WORKERS should start no workers"- else- pass "DISABLE_WORKERS starts no workers, schedulers or ingest consumer"- fi-- if grep -q '"msg":"Workers are disabled"' "$OPTOUT_LOG"; then- pass "DISABLE_WORKERS says so at boot"- else- fail "DISABLE_WORKERS did not log why nothing is consuming"- fi-- live_code=$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$ROLE_PORT/healthz/live")- if [ "$live_code" = "200" ]; then- pass "DISABLE_WORKERS still serves the ops surface"- else- fail "DISABLE_WORKERS ops surface -> $live_code, expected 200"- fi-- stop_consuming_role worker- rm -f "$OPTOUT_LOG"-fi--# The namespaced BullMQ keys and their job schedulers are this proof's litter;-# an `every` scheduler left in Redis would keep firing forever.-echo-(- # From apps/api, whose node_modules resolves @openpanel/redis.- cd "$REPO_ROOT/apps/api" || exit 1- set -a- # shellcheck disable=SC1091- . "$REPO_ROOT/.env"- set +a- PROOF_NAMESPACE="$PROOF_NAMESPACE" bun -e '- const { Redis } = await import("@openpanel/redis");- const redis = new Redis(process.env.REDIS_URL ?? "redis://127.0.0.1:23379");- const pattern = `bull:*-${process.env.PROOF_NAMESPACE}:*`;- let cursor = "0";- let removed = 0;- do {- const [next, keys] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1000);- cursor = next;- if (keys.length > 0) {- removed += await redis.del(...keys);- }- } while (cursor !== "0");- console.log(`cleaned ${removed} proof queue keys`);- await redis.quit();- '-) && pass "cleaned up the namespaced proof queue keys" \- || fail "could not clean up the namespaced proof queue keys"--# Nothing this script started may outlive it.-sleep 0.3-LEFTOVER="$(pgrep -f "$MAIN_TS" || true)"-if [ -z "$LEFTOVER" ]; then- pass "no boot-proof process left behind"-else- fail "process(es) still running: $LEFTOVER"- # shellcheck disable=SC2086- kill -9 $LEFTOVER 2>/dev/null-fi--echo-if [ "$FAILURES" -eq 0 ]; then- echo "boot-proof.sh: all checks passed"- exit 0-else- echo "boot-proof.sh: $FAILURES failure(s)"- exit 1-fidiff --git a/apps/api/e2e/trpc-routing-golden.ts b/apps/api/e2e/trpc-routing-golden.tsdeleted file mode 100644index 959e7e781..000000000--- a/apps/api/e2e/trpc-routing-golden.ts+++ /dev/null@@ -1,203 +0,0 @@-#!/usr/bin/env bun-/**- * TRPC routing golden — proves every procedure the mounted router declares is- * reachable through the real HTTP lifecycle, and that the auth layer answers- * identically before and after swapping the mount.- *- * Only 11 of 217 procedures have any end-to-end coverage and 22 of the 29- * routers have none, so swapping `apps/api/src/main.ts`'s `appRouter` import- * needs a proof of its own. This is it: with the harness up, run it against the- * OLD mount, swap, run it against the NEW mount, and diff the two maps.- *- * It asserts REACHABILITY, not behaviour. Every request is unauthenticated with- * an empty input, so 401/403/400 are the expected answers — they prove the- * procedure was resolved and its auth/validation middleware ran. Only a 404 or- * a tRPC `NOT_FOUND` means the router did not resolve the path, and that is the- * single failure this script exists to catch.- *- * Usage (needs `verification/harness start`): cd apps/api && bun run- * e2e/trpc-routing-golden.ts /tmp/trpc-routing-after.json- *- * The router is imported from whatever specifier `main.ts` mounts, read out of- * main.ts itself, so the same committed script measures both sides of the swap- * with no flag to forget. `@openpanel/trpc` opens Redis connections at import- * time and never releases the event loop, hence the explicit process.exit.- */-import { readFileSync, writeFileSync } from 'node:fs';-import { dirname, join } from 'node:path';-import { fileURLToPath } from 'node:url';--// `import.meta.dir` is the Bun idiom, but apps/api's tsconfig carries no bun-// type declarations, so this file uses the portable form.-const REPO_ROOT = join(- dirname(fileURLToPath(import.meta.url)),- '..',- '..',- '..'-);-const MAIN_TS = join(REPO_ROOT, 'apps', 'api', 'src', 'main.ts');-const DEFAULT_API_PORT = '3333';-const TRPC_ENDPOINT = '/trpc';-const REQUEST_TIMEOUT_MS = 30_000;-const CONCURRENCY = 8;--/**- * Every `import { … } from '…'` in main.ts. The mounted router is the one- * whose specifier list contains `appRouter` — matched as a whole binding- * rather than as a line, because biome's organizeImports merges it into the- * package's existing import block.- */-const IMPORT_STATEMENT = /import\s*\{([^}]*)\}\s*from\s*'([^']+)'/g;--function readEnvVar(key: string): string | undefined {- const line = readFileSync(join(REPO_ROOT, '.env'), 'utf8')- .split('\n')- .filter((l) => l.startsWith(`${key}=`))- .pop();- return (- line- ?.slice(key.length + 1)- .replaceAll('"', '')- .trim() || undefined- );-}--const API_BASE_URL =- process.env.E2E_API_URL ??- `http://127.0.0.1:${readEnvVar('API_PORT') ?? DEFAULT_API_PORT}`;--function mountedRouterSpecifier(): string {- const source = readFileSync(MAIN_TS, 'utf8');- for (const [, bindings, specifier] of source.matchAll(IMPORT_STATEMENT)) {- const names = (bindings as string).split(',').map(- (binding) =>- binding- .trim()- .replace(/^type\s+/, '')- .split(/\s+as\s+/)[0]- );- if (names.includes('appRouter')) {- return specifier as string;- }- }- throw new Error(`no import of \`appRouter\` in ${MAIN_TS}`);-}--interface ProcedureProbe {- type: string;- status: number;- code: string | null;-}--/**- * tRPC 11.17 keys `_def.procedures` by dotted path, so the keys are the flat- * procedure set and each value carries its own `_def.type`.- */-async function enumerateProcedures(): Promise<Map<string, string>> {- const specifier = mountedRouterSpecifier();- const module = (await import(specifier)) as {- appRouter: {- _def: { procedures: Record<string, { _def: { type: string } }> };- };- };- const entries = Object.entries(module.appRouter._def.procedures)- .map(([path, procedure]) => [path, procedure._def.type] as const)- .sort(([a], [b]) => a.localeCompare(b));- console.log(`mounted router: ${specifier}`);- return new Map(entries);-}--/** One unauthenticated request, shaped exactly as a superjson tRPC client sends it. */-async function probe(path: string, type: string): Promise<ProcedureProbe> {- const body = JSON.stringify({ json: {} });- const isMutation = type === 'mutation';- const url = isMutation- ? `${API_BASE_URL}${TRPC_ENDPOINT}/${path}`- : `${API_BASE_URL}${TRPC_ENDPOINT}/${path}?input=${encodeURIComponent(body)}`;-- try {- const response = await fetch(url, {- method: isMutation ? 'POST' : 'GET',- headers: { 'content-type': 'application/json' },- body: isMutation ? body : undefined,- signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),- });- const raw = (await response.json().catch(() => null)) as {- error?: { json?: { data?: { code?: string } } };- } | null;- return {- type,- status: response.status,- code: raw?.error?.json?.data?.code ?? null,- };- } catch (error) {- return {- type,- status: -1,- code: `TRANSPORT_ERROR: ${(error as Error).message}`,- };- }-}--/** A path the router did not resolve — the only outcome this golden fails on. */-function isRoutingFailure(result: ProcedureProbe): boolean {- return result.status === 404 || result.code === 'NOT_FOUND';-}--async function main(): Promise<number> {- const outputPath = process.argv[2];- if (!outputPath) {- console.error('usage: bun run e2e/trpc-routing-golden.ts <output.json>');- return 2;- }-- const procedures = await enumerateProcedures();- const paths = [...procedures.keys()];- console.log(- `probing ${paths.length} procedures across ` +- `${new Set(paths.map((p) => p.split('.')[0])).size} routers at ${API_BASE_URL}`- );-- const results: Record<string, ProcedureProbe> = {};- let cursor = 0;- await Promise.all(- Array.from({ length: CONCURRENCY }, async () => {- while (cursor < paths.length) {- const path = paths[cursor++] as string;- results[path] = await probe(path, procedures.get(path) as string);- }- })- );-- const sorted: Record<string, ProcedureProbe> = {};- for (const path of paths) {- sorted[path] = results[path] as ProcedureProbe;- }- writeFileSync(outputPath, `${JSON.stringify(sorted, null, 2)}\n`);-- const tally = new Map<string, number>();- for (const result of Object.values(sorted)) {- const key = `${result.status} ${result.code ?? 'ok'}`;- tally.set(key, (tally.get(key) ?? 0) + 1);- }- console.log(`\nwrote ${outputPath}`);- for (const [key, count] of [...tally].sort()) {- console.log(` ${String(count).padStart(4)} ${key}`);- }-- const failures = Object.entries(sorted).filter(([, r]) =>- isRoutingFailure(r)- );- console.log(- `\ntrpc-routing-golden: ${paths.length - failures.length}/${paths.length} reachable, ` +- `${failures.length} routing failure(s)`- );- for (const [path, result] of failures) {- console.log(- ` UNREACHABLE ${path} -> ${result.status} ${result.code ?? ''}`- );- }- return failures.length === 0 ? 0 : 1;-}--process.exit(await main());diff --git a/apps/start/src/app/global-middleware.ts b/apps/start/src/app/global-middleware.tsdeleted file mode 100644index 53fe07265..000000000--- a/apps/start/src/app/global-middleware.ts+++ /dev/null@@ -1,10 +0,0 @@-// import * as Sentry from '@sentry/tanstackstart-react';-// import { createMiddleware } from '@tanstack/react-start';--// registerGlobalMiddleware({-// middleware: [-// createMiddleware({-// type: 'function',-// }).server(Sentry.sentryGlobalServerMiddlewareHandler()),-// ],-// });diff --git a/bun.lock b/bun.lockindex 19b413dc9..23abbffd7 100644--- a/bun.lock+++ b/bun.lock@@ -356,6 +356,7 @@"@openpanel/core": "workspace:*","@openpanel/payments": "workspace:*","@openpanel/redis": "workspace:*",+ "@openpanel/shared": "workspace:*","@prisma/client": "6.14.0","pino": "catalog:","pino-pretty": "catalog:",@@ -610,7 +611,7 @@"name": "@openpanel/release","version": "0.1.0","dependencies": {- "openai": "^7.25.0",+ "openai": "7.25.0","semver": "7.6.0",},"devDependencies": {diff --git a/docs/migrations-fresh-install/ch-scratch-proxy.ts b/docs/migrations-fresh-install/ch-scratch-proxy.tsdeleted file mode 100644index 0de73b6f2..000000000--- a/docs/migrations-fresh-install/ch-scratch-proxy.ts+++ /dev/null@@ -1,153 +0,0 @@-/**- * A ClickHouse HTTP proxy that lets a fresh-install migration drill run on a- * box whose `openpanel` database already holds the prod-copy dataset.- *- * Why it has to exist: `getExistingTables()`- * (`packages/db/src/clickhouse/migration.ts`) asks- * `SELECT name FROM system.tables WHERE database = 'openpanel'` — the database- * name is a literal, not the one in `CLICKHOUSE_URL` — and `4-add-sessions.ts`- * writes `INSERT INTO openpanel.sessions ... FROM openpanel.events`. Pointing- * `CLICKHOUSE_URL` at a scratch database is therefore NOT isolation: the run- * still reads, and would still write, the real `openpanel`.- *- * Modes:- * rewrite every `openpanel` token (URL and body) becomes SCRATCH_DATABASE,- * so the migration's own hardcoded name resolves to the scratch- * database. This is what a genuinely fresh install looks like.- * guard pass through untouched — used to reproduce a run that points only- * CLICKHOUSE_URL at a scratch database — but refuse any statement- * that is not a read and still names `openpanel` / `openpanel_test`.- *- * Both modes refuse anything naming `openpanel_test`, and every statement is- * appended to LOG_FILE.- */--const UPSTREAM_URL =- process.env.UPSTREAM_CLICKHOUSE_URL ?? 'http://127.0.0.1:8123';-const LISTEN_PORT = Number(process.env.PROXY_PORT ?? 8199);-const SCRATCH_DATABASE = process.env.SCRATCH_DATABASE ?? '';-const MODE = process.env.PROXY_MODE ?? 'rewrite';-const LOG_FILE = process.env.PROXY_LOG ?? '/tmp/ch-scratch-proxy.log';--// Two spellings of the same token: `.test()` on a /g regex carries lastIndex-// between calls, so the guard gets its own non-global copy.-const PROTECTED_DATABASE_TOKEN_ALL = /\bopenpanel\b/g;-const PROTECTED_DATABASE_TOKEN = /\bopenpanel\b/;-const TEST_DATABASE_TOKEN = /\bopenpanel_test\b/;-const READ_STATEMENT = /^\s*(SELECT|WITH|SHOW|DESC|DESCRIBE|EXISTS|EXPLAIN)\b/i;-const REFUSED_STATUS = 400;--if (MODE === 'rewrite' && !SCRATCH_DATABASE) {- throw new Error('SCRATCH_DATABASE is required in rewrite mode');-}-if (SCRATCH_DATABASE === 'openpanel' || SCRATCH_DATABASE === 'openpanel_test') {- throw new Error(`refusing to use ${SCRATCH_DATABASE} as a scratch database`);-}--await Bun.write(LOG_FILE, '');-const logHandle = Bun.file(LOG_FILE).writer();-const append = (line: string) => {- logHandle.write(`${line}\n`);- logHandle.flush();-};--/** ClickHouse clients may gzip the request body; rewriting needs the plain text. */-async function readBody(request: Request): Promise<string> {- const raw = new Uint8Array(await request.arrayBuffer());- if (request.headers.get('content-encoding') === 'gzip') {- return new TextDecoder().decode(Bun.gunzipSync(raw));- }- return new TextDecoder().decode(raw);-}--const rewrite = (text: string) =>- MODE === 'rewrite'- ? text.replace(PROTECTED_DATABASE_TOKEN_ALL, SCRATCH_DATABASE)- : text;--/**- * `statement` is the SQL alone — the read/write classification has to look at- * the query text, not at the URL it arrived on.- */-const isRefused = (target: string, statement: string) => {- if (TEST_DATABASE_TOKEN.test(target) || TEST_DATABASE_TOKEN.test(statement)) {- return 'names openpanel_test';- }- if (MODE === 'rewrite') {- // Nothing may reach ClickHouse still naming the real database.- return PROTECTED_DATABASE_TOKEN.test(target) ||- PROTECTED_DATABASE_TOKEN.test(statement)- ? 'still names openpanel'- : null;- }- if (- PROTECTED_DATABASE_TOKEN.test(statement) &&- !READ_STATEMENT.test(statement)- ) {- return 'non-read statement naming openpanel';- }- if (PROTECTED_DATABASE_TOKEN.test(target)) {- return 'request routed at the openpanel database';- }- return null;-};--const server = Bun.serve({- port: LISTEN_PORT,- hostname: '127.0.0.1',- idleTimeout: 255,- async fetch(request) {- const incoming = new URL(request.url);- const body = request.method === 'GET' ? '' : await readBody(request);-- const outgoing = new URL(- rewrite(incoming.pathname + incoming.search),- UPSTREAM_URL- );- const outgoingBody = rewrite(body);- const target = `${outgoing.pathname}?${outgoing.searchParams.get('database') ?? ''}`;- const statement =- outgoingBody || (outgoing.searchParams.get('query') ?? '');- const inspected = `${outgoing.pathname}${outgoing.search}\n${outgoingBody}`;-- const refusal = isRefused(target, statement);- if (refusal) {- append(`[REFUSED: ${refusal}] ${inspected}`);- return new Response(`proxy refused request: ${refusal}\n`, {- status: REFUSED_STATUS,- });- }-- append(- `[${request.method}] ${outgoing.pathname}${outgoing.search}\n${outgoingBody}`- );-- const headers = new Headers(request.headers);- // The body is rewritten, so any length/encoding the client set is stale.- headers.delete('content-length');- headers.delete('content-encoding');-- const response = await fetch(outgoing, {- method: request.method,- headers,- body: request.method === 'GET' ? undefined : outgoingBody,- });-- // `fetch` already decompressed the upstream body; forwarding ClickHouse's- // `Content-Encoding: gzip` would make the client decompress it twice.- const responseHeaders = new Headers(response.headers);- responseHeaders.delete('content-encoding');- responseHeaders.delete('content-length');-- return new Response(response.body, {- status: response.status,- statusText: response.statusText,- headers: responseHeaders,- });- },-});--append(`--- proxy up mode=${MODE} scratch=${SCRATCH_DATABASE || '(none)'} ---`);-console.log(- `ch-scratch-proxy listening on http://127.0.0.1:${server.port} mode=${MODE} scratch=${SCRATCH_DATABASE || '(none)'}`-);diff --git a/docs/migrations-fresh-install/fresh-install-drill.sh b/docs/migrations-fresh-install/fresh-install-drill.shdeleted file mode 100755index 65b8e5111..000000000--- a/docs/migrations-fresh-install/fresh-install-drill.sh+++ /dev/null@@ -1,117 +0,0 @@-#!/usr/bin/env bash-# Fresh-install migration drill (M16-004).-#-# Runs the shipped container start command's migration half — 153 Prisma-# migrations then the 24 code-migrations — against scratch databases, twice,-# and prints what each step did.-#-# bash docs/migrations-fresh-install/fresh-install-drill.sh rewrite-# bash docs/migrations-fresh-install/fresh-install-drill.sh guard-#-# `rewrite` is the real drill: ch-scratch-proxy.ts maps the `openpanel`-# database name the migrations hardcode onto the scratch database, so the run-# sees a genuinely empty install. `guard` reproduces M15-203's setup, where-# only CLICKHOUSE_URL pointed at a scratch database.-#-# Teardown is `fresh-install-teardown.sh`. Nothing here touches the `openpanel`-# or `openpanel_test` databases; the proxy refuses any request that would.-set -euo pipefail--MODE="${1:-rewrite}"-SCRATCH_DB="${SCRATCH_DB:-openpanel_m16_004}"-PROXY_PORT="${PROXY_PORT:-8199}"-PROXY_LOG="${PROXY_LOG:-/tmp/ch-scratch-proxy-${MODE}.log}"-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-PG_BASE_URL="${PG_BASE_URL:-postgresql://postgres:postgres@localhost:5432}"-PG_ADMIN_URL="$PG_BASE_URL/postgres"-CH_ADMIN_URL="${CH_ADMIN_URL:-http://127.0.0.1:8123}"--case "$SCRATCH_DB" in- openpanel|openpanel_test|postgres)- echo "refusing to use '$SCRATCH_DB' as a scratch database" >&2- exit 1- ;;-esac--pg_query() { PG_QUERY_URL="$1" bun -e "-const sql = new Bun.SQL(process.env.PG_QUERY_URL);-const rows = await sql.unsafe(process.argv[1]);-if (Array.isArray(rows) && rows.length) console.log(JSON.stringify(rows));-await sql.end();-" "$2"; }--# The maintenance connection (creates/drops databases) and the scratch database-# itself are different connections — reporting through the wrong one reports the-# dev database's state.-pg_admin() { pg_query "$PG_ADMIN_URL" "$1"; }-pg_scratch() { pg_query "$PG_BASE_URL/$SCRATCH_DB" "$1"; }--ch_admin() { curl -sS --fail-with-body "$CH_ADMIN_URL/" --data-binary "$1"; }--banner() { printf '\n========== %s ==========\n' "$1"; }--banner "0. scratch databases"-pg_admin "DROP DATABASE IF EXISTS \"$SCRATCH_DB\""-pg_admin "CREATE DATABASE \"$SCRATCH_DB\""-ch_admin "DROP DATABASE IF EXISTS $SCRATCH_DB"-ch_admin "CREATE DATABASE $SCRATCH_DB"-echo "postgres: $SCRATCH_DB created"-echo "clickhouse: $SCRATCH_DB created, tables: $(ch_admin "SELECT count() FROM system.tables WHERE database='$SCRATCH_DB'")"--banner "0b. protected databases before the run"-ch_admin "SELECT database, count() FROM system.tables WHERE database IN ('openpanel','openpanel_test') GROUP BY database ORDER BY database FORMAT TSV"--if curl -sS "http://127.0.0.1:$PROXY_PORT/" --data-binary "SELECT 1" >/dev/null 2>&1; then- echo "port $PROXY_PORT already answers — a stale proxy would silently serve this run" >&2- exit 1-fi--if [ "$MODE" = "rewrite" ]; then- PROXY_MODE=rewrite SCRATCH_DATABASE="$SCRATCH_DB" PROXY_PORT="$PROXY_PORT" PROXY_LOG="$PROXY_LOG" \- bun "$REPO_ROOT/docs/migrations-fresh-install/ch-scratch-proxy.ts" &-else- PROXY_MODE=guard PROXY_PORT="$PROXY_PORT" PROXY_LOG="$PROXY_LOG" \- bun "$REPO_ROOT/docs/migrations-fresh-install/ch-scratch-proxy.ts" &-fi-PROXY_PID=$!-trap 'kill $PROXY_PID 2>/dev/null || true' EXIT-until curl -sS "http://127.0.0.1:$PROXY_PORT/?query=SELECT%201" >/dev/null 2>&1; do sleep 0.2; done--export DATABASE_URL="$PG_BASE_URL/$SCRATCH_DB?schema=public"-export DATABASE_URL_DIRECT="$DATABASE_URL"-export CLICKHOUSE_URL="http://127.0.0.1:$PROXY_PORT/$SCRATCH_DB"-export SELF_HOSTED=true--run_once() {- banner "$1 — prisma migrate deploy"- local started=$SECONDS- (cd "$REPO_ROOT/packages/db" && bunx [email protected] migrate deploy) || return 1- echo "[timing] $1 prisma migrate deploy: $((SECONDS - started))s"-- banner "$1 — code migrations"- started=$SECONDS- (cd "$REPO_ROOT/packages/core" && bun scripts/migrate-code.ts) || return 1- echo "[timing] $1 code migrations: $((SECONDS - started))s"-}--FIRST_RESULT=ok-run_once "RUN 1" || FIRST_RESULT=failed--banner "state after run 1 ($FIRST_RESULT)"-pg_scratch "SELECT count(*)::int AS applied FROM \"_prisma_migrations\" WHERE finished_at IS NOT NULL" || true-pg_scratch "SELECT count(*)::int AS code_migrations FROM \"__code_migrations\"" || true-ch_admin "SELECT count() FROM system.tables WHERE database='$SCRATCH_DB'"--SECOND_RESULT=ok-run_once "RUN 2 (idempotency)" || SECOND_RESULT=failed--banner "state after run 2 ($SECOND_RESULT)"-pg_scratch "SELECT count(*)::int AS applied FROM \"_prisma_migrations\" WHERE finished_at IS NOT NULL" || true-pg_scratch "SELECT count(*)::int AS code_migrations FROM \"__code_migrations\"" || true-ch_admin "SELECT count() FROM system.tables WHERE database='$SCRATCH_DB'"--banner "protected databases after the run"-ch_admin "SELECT database, count() FROM system.tables WHERE database IN ('openpanel','openpanel_test') GROUP BY database ORDER BY database FORMAT TSV"--banner "verdict"-echo "mode=$MODE run1=$FIRST_RESULT run2=$SECOND_RESULT proxy log: $PROXY_LOG"diff --git a/docs/migrations-fresh-install/fresh-install-teardown.sh b/docs/migrations-fresh-install/fresh-install-teardown.shdeleted file mode 100755index 8554588c9..000000000--- a/docs/migrations-fresh-install/fresh-install-teardown.sh+++ /dev/null@@ -1,26 +0,0 @@-#!/usr/bin/env bash-# Drops ONLY the scratch databases the drill created, and removes the `.sql`-# dumps the migrations write next to themselves (untracked build artifacts).-set -euo pipefail--SCRATCH_DB="${SCRATCH_DB:-openpanel_m16_004}"-PG_ADMIN_URL="${PG_ADMIN_URL:-postgresql://postgres:postgres@localhost:5432/postgres}"-CH_ADMIN_URL="${CH_ADMIN_URL:-http://127.0.0.1:8123}"--case "$SCRATCH_DB" in- openpanel|openpanel_test|postgres)- echo "refusing to drop '$SCRATCH_DB'" >&2- exit 1- ;;-esac--PG_ADMIN_URL="$PG_ADMIN_URL" bun -e "-const sql = new Bun.SQL(process.env.PG_ADMIN_URL);-await sql.unsafe(\`DROP DATABASE IF EXISTS \"\${process.argv[1]}\"\`);-await sql.end();-" "$SCRATCH_DB"-curl -sS --fail-with-body "$CH_ADMIN_URL/" --data-binary "DROP DATABASE IF EXISTS $SCRATCH_DB"-REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"-rm -f "$REPO_ROOT"/packages/db/src/code-migrations/*.sql--echo "dropped scratch databases named $SCRATCH_DB and removed the .sql dumps"diff --git a/packages/core/scripts/get-datacenter-asns.ts b/packages/core/scripts/get-datacenter-asns.tsindex ff0988ed2..eaa442efd 100644--- a/packages/core/scripts/get-datacenter-asns.ts+++ b/packages/core/scripts/get-datacenter-asns.ts@@ -8,7 +8,7 @@ const __dirname = dirname(__filename);// Upstream list of ASNs belonging to datacenters / hosting providers. Traffic// from them is almost never a real end user, so an IP's ASN (MaxMind GeoLite2-ASN)// is checked for membership in this set. The list is regenerated, never-// hand-edited, so upstream additions flow in on every `pnpm codegen`.+// hand-edited, so upstream additions flow in on every `bun run codegen`.const SOURCE_URL ='https://raw.githubusercontent.com/X4BNet/lists_vpn/main/input/datacenter/ASN.txt';diff --git a/packages/core/src/buffers/profile-buffer.ts b/packages/core/src/buffers/profile-buffer.tsindex 28efec9cb..d4f447d8b 100644--- a/packages/core/src/buffers/profile-buffer.ts+++ b/packages/core/src/buffers/profile-buffer.ts@@ -6,12 +6,6 @@ import type { IClickhouseProfile } from '../modules/profile/profile.service';import { TABLE_NAMES } from '../shared/ch-tables';import { BaseBuffer, type BufferDeps } from './base-buffer';-// Inlined to avoid a circular value-import with `profile.service.ts`-// (which imports `profileBuffer` from `../buffers`). Keep this in sync-// with the `PROFILE_COLUMNS` constant there.-const PROFILE_COLUMNS =- 'id, first_name, last_name, email, avatar, properties, project_id, is_external, created_at, last_seen_at, groups';-// Equivalent of `SELECT ... FROM profiles FINAL` but expressed via GROUP BY +// argMax. ReplicatedReplacingMergeTree uses `last_seen_at` as the version// column, so argMax(col, last_seen_at) yields the same value FINAL woulddiff --git a/packages/core/src/clients/geo.ts b/packages/core/src/clients/geo.tsindex 8dd72e3a6..cf309045d 100644--- a/packages/core/src/clients/geo.ts+++ b/packages/core/src/clients/geo.ts@@ -23,12 +23,10 @@ async function loadDatabase(filename: string): Promise<ReaderModel | null> {const dbPathLocal = path.join(__dirname, `../../${filename}`);try {const dbBuffer = await readFile(dbPath);- console.log(`${filename} loaded (dist)`, dbPath);return Reader.openBuffer(dbBuffer);} catch {try {const dbBuffer = await readFile(dbPathLocal);- console.log(`${filename} loaded (local)`, dbPathLocal);return Reader.openBuffer(dbBuffer);} catch {console.error(`${filename} not found`, { dbPath, dbPathLocal });diff --git a/packages/core/src/modules/insight/insight.service.test.ts b/packages/core/src/modules/insight/insight.service.test.tsindex dcf81fc71..cd49d2112 100644--- a/packages/core/src/modules/insight/insight.service.test.ts+++ b/packages/core/src/modules/insight/insight.service.test.ts@@ -1,6 +1,6 @@// The subject's functions take `ServiceDeps`, so `deps.db` is the fake below and// no Prisma module mock is needed. The intra-package modules (store,-// referrer-spikes, legacy-scan) are `mock.module`'d, registered before the+// referrer-spikes, explain) are `mock.module`'d, registered before the// subject's first call.import { beforeAll, expect, mock, test } from 'bun:test';@@ -49,25 +49,6 @@ mock.module('./src/referrer-spikes', () => ({getReferrerSpikes: spikesQuery,}));-const LEGACY_SPIKE = {- type: 'traffic_spike',- message: 'test',- // A `TrafficSpikeResult` — `Insight.data` is the union of the ten detector- // result shapes, not `any`.- data: {- referrer_name: 'instagram.com',- date: '2026-09-01',- visitor_count: 120,- avg_previous_7_days: 30,- },-};-const legacyGenerateInsights = mock(async () => [LEGACY_SPIKE]);-mock.module('./src/legacy-scan', () => ({- createLegacyInsightsScanner: () => ({- generateInsights: legacyGenerateInsights,- }),-}));-const generateInsightExplanation = mock(async () => ({summary: 'x',drivers: [],@@ -131,13 +112,6 @@ test('getReferrerSpikes delegates to the ClickHouse query', async () => {expect(result).toEqual([{ anchorDate: '2026-09-01', spikes: [] }]);});-test('scanLegacyInsights delegates to the pre-engine detector, kept for parity', async () => {- const result = await subject.scanLegacyInsights(deps, 'p1');-- expect(legacyGenerateInsights).toHaveBeenCalledWith('p1');- expect(result).toEqual([LEGACY_SPIKE]);-});-test('explainInsight is a cache-aside over the AI call, keyed by the caller-supplied cacheKey', async () => {const input = {insight: { title: 't', dimension: 'd', window: 'yesterday' },@@ -204,7 +178,6 @@ test('createInsightService binds every InsightService method', () => {'listAllInsights','explainInsight','getReferrerSpikes',- 'scanLegacyInsights',] as const) {expect(typeof service[method]).toBe('function');}diff --git a/packages/core/src/modules/insight/insight.service.ts b/packages/core/src/modules/insight/insight.service.tsindex 55adcc4c2..84d07f2a4 100644--- a/packages/core/src/modules/insight/insight.service.ts+++ b/packages/core/src/modules/insight/insight.service.ts@@ -17,10 +17,6 @@ import {generateInsightExplanation,type InsightExplanation,} from './src/explain';-import {- createLegacyInsightsScanner,- type Insight as LegacyInsight,-} from './src/legacy-scan';import {devicesModule,entryPagesModule,@@ -647,14 +643,6 @@ export async function getReferrerSpikes(return getReferrerSpikesQuery(deps, input);}-/** Legacy pre-engine detector — no live callers today. */-export async function scanLegacyInsights(- deps: ServiceDeps,- projectId: string-): Promise<LegacyInsight[]> {- return createLegacyInsightsScanner(deps).generateInsights(projectId);-}-// Re-exported at each type's own import site (noExportedImports): a type// that is also used locally still gets `export type {} from '<module>'`// rather than a second `import`-then-`export` pair gathered here.@@ -708,7 +696,5 @@ export function createInsightService(getReferrerSpikes: (input: GetReferrerSpikesInput): Promise<ReferrerSpikeCluster[]> => getReferrerSpikes(deps, input),- scanLegacyInsights: (projectId: string): Promise<LegacyInsight[]> =>- scanLegacyInsights(deps, projectId),};}diff --git a/packages/core/src/modules/insight/src/legacy-scan.ts b/packages/core/src/modules/insight/src/legacy-scan.tsdeleted file mode 100644index b9abdad40..000000000--- a/packages/core/src/modules/insight/src/legacy-scan.ts+++ /dev/null@@ -1,332 +0,0 @@-// The pre-engine insight detector — no live callers today, kept only so the-// defects below stay visible rather than silently dropped.-//-// Nine of the ten statements are BROKEN AND WERE ALWAYS BROKEN: most compare-// a column against a quoted string literal threshold, so the comparison-// fails with TYPE_MISMATCH; three more read columns (`is_new`,-// `is_returning`, `event_name`, `status`) that no OpenPanel table has, and-// one puts a window function in HAVING. The defects are reproduced verbatim,-// not fixed — fixing behaviour is a product decision, and there is no-// caller to serve it.-//-// Every statement sends `session_timezone`, defaulted to `'UTC'`.--import {- type SqlFragment,- type SqlParam,- sql,-} from '@openpanel/db/src/clickhouse/sql';-import { type ChScope, chQuery } from '../../../ch-query';-import { formatClickhouseDate } from '../../../shared/ch-dates';--/** Sent on every query; defaults to `'UTC'`. */-const CLIX_SESSION_TIMEZONE = { session_timezone: 'UTC' } as const;--const MS_PER_DAY = 24 * 60 * 60 * 1000;--const TRAFFIC_SPIKE_WINDOW_DAYS = 30;-const EVENT_SURGE_WINDOW_DAYS = 30;-const NEW_VISITOR_WINDOW_DAYS = 60;-const REFERRAL_SOURCE_WINDOW_DAYS = 7;-const SESSION_DURATION_WINDOW_DAYS = 14;-const TOP_CONTENT_WINDOW_DAYS = 30;-const BOUNCE_RATE_WINDOW_DAYS = 60;-const RETURNING_VISITOR_WINDOW_DAYS = 180;-const GEOGRAPHIC_SHIFT_WINDOW_DAYS = 14;-const EVENT_COMPLETION_WINDOW_DAYS = 60;--const TOP_CONTENT_LIMIT = 1;--// Rendered as a quoted STRING comparand — reaches ClickHouse as text, not a-// numeric threshold. Kept exactly, defect included.-const TRAFFIC_SPIKE_THRESHOLD = 'avg_previous_7_days * 2';-const EVENT_SURGE_THRESHOLD = 'avg_previous_7_days * 1.3';-const NEW_VISITOR_THRESHOLD = 'prev_month_visitors * 1.2';-const SESSION_DURATION_THRESHOLD = 'prev_week_duration * 1.25';-const BOUNCE_RATE_THRESHOLD = 'prev_month_bounce_rate * 0.85';-const RETURNING_VISITOR_THRESHOLD = 'prev_quarter_visitors * 1.1';-const GEOGRAPHIC_SHIFT_THRESHOLD = 'prev_week_count * 1.5';-const EVENT_COMPLETION_THRESHOLD = 'prev_month_count * 1.05';-const REFERRAL_SOURCE_MIN_SHARE = 0.5;--const COMPLETED_EVENT_STATUS = 'completed';--/** Whatever the detector that produced the insight selected. Three of the ten- * shapes carry no date-ish column at all — see `insightPeriod`. */-export type LegacyInsightData =- | TrafficSpikeResult- | EventSurgeResult- | NewVisitorTrendResult- | ReferralSourceResult- | SessionDurationResult- | TopContentResult- | BounceRateResult- | ReturningVisitorResult- | GeographicShiftResult- | EventCompletionResult;--export interface Insight {- type: string;- message: string;- data: LegacyInsightData;-}--/** The one period column a result shape carries, or '' for the shapes that- * carry none — those sort last, as an Invalid Date always has. */-function insightPeriod(data: LegacyInsightData): string {- const periods = data as Partial<- Record<'date' | 'month' | 'week' | 'quarter', string>- >;- return periods.date || periods.month || periods.week || periods.quarter || '';-}--interface TrafficSpikeResult {- referrer_name: string;- date: string;- visitor_count: number;- avg_previous_7_days: number;-}--interface EventSurgeResult {- date: string;- event_count: number;- avg_previous_7_days: number;-}--interface NewVisitorTrendResult {- month: string;- new_visitors: number;- prev_month_visitors: number;-}--interface ReferralSourceResult {- referrer_name: string;- count: number;- percentage: number;-}--interface SessionDurationResult {- week: string;- avg_duration: number;- prev_week_duration: number;-}--interface TopContentResult {- path: string;- view_count: number;- unique_viewers: number;-}--interface BounceRateResult {- month: string;- bounce_rate: number;- prev_month_bounce_rate: number;-}--interface ReturningVisitorResult {- quarter: string;- returning_visitors: number;- prev_quarter_visitors: number;-}--interface GeographicShiftResult {- country: string;- visitor_count: number;- prev_week_count: number;-}--interface EventCompletionResult {- event_name: string;- month: string;- completion_count: number;- prev_month_count: number;-}--/** Formatted as `'YYYY-MM-DD HH:mm:ss'`. */-function since(days: number): SqlParam {- return sql.string(- formatClickhouseDate(new Date(Date.now() - days * MS_PER_DAY))- );-}--/** No live callers; kept only so the defects above stay visible. */-class LegacyInsightsScanner {- constructor(private readonly deps: ChScope) {}-- private run<T extends object>(statement: SqlFragment): Promise<T[]> {- return chQuery<T>(this.deps, statement, CLIX_SESSION_TIMEZONE);- }-- private async getTrafficSpikes(projectId: string): Promise<Insight[]> {- const results = await this.run<TrafficSpikeResult>(- sql`SELECT referrer_name, toDate(created_at) as date, COUNT(*) as visitor_count, avg(COUNT(*)) OVER (ORDER BY date ROWS BETWEEN 7 PRECEDING AND 1 PRECEDING) as avg_previous_7_days FROM events WHERE created_at >= ${since(TRAFFIC_SPIKE_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} GROUP BY referrer_name, date HAVING visitor_count > ${sql.string(TRAFFIC_SPIKE_THRESHOLD)} ORDER BY visitor_count DESC`- );- return results.map((result) => ({- type: 'traffic_spike',- message: `Your website experienced a significant increase in visitors from ${result.referrer_name} on ${result.date}.`,- data: result,- }));- }-- private async getEventSurges(projectId: string): Promise<Insight[]> {- const results = await this.run<EventSurgeResult>(- sql`SELECT toDate(created_at) as date, COUNT(*) as event_count, avg(COUNT(*)) OVER (ORDER BY date ROWS BETWEEN 7 PRECEDING AND 1 PRECEDING) as avg_previous_7_days FROM events WHERE created_at >= ${since(EVENT_SURGE_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} GROUP BY date HAVING event_count > ${sql.string(EVENT_SURGE_THRESHOLD)} ORDER BY event_count DESC`- );- return results.map((result) => ({- type: 'event_surge',- message: `There was a surge in events recorded on ${result.date}, marking a ${Math.round((result.event_count / result.avg_previous_7_days - 1) * 100)}% increase from the previous average.`,- data: result,- }));- }-- private async getNewVisitorTrends(projectId: string): Promise<Insight[]> {- const results = await this.run<NewVisitorTrendResult>(- sql`SELECT toMonth(created_at) as month, COUNT(DISTINCT device_id) as new_visitors, lag(COUNT(DISTINCT device_id)) OVER (ORDER BY month) as prev_month_visitors FROM sessions WHERE created_at >= ${since(NEW_VISITOR_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} AND is_new = true GROUP BY month HAVING new_visitors > ${sql.string(NEW_VISITOR_THRESHOLD)} ORDER BY month DESC`- );- return results.map((result) => ({- type: 'new_visitor_trend',- message: `This month, you saw a ${Math.round((result.new_visitors / result.prev_month_visitors - 1) * 100)}% increase in new visitors compared to last month.`,- data: result,- }));- }-- private async getReferralSourceHighlights(- projectId: string- ): Promise<Insight[]> {- const results = await this.run<ReferralSourceResult>(- sql`SELECT referrer_name, COUNT(*) as count, COUNT(*) / sum(COUNT(*)) OVER () as percentage FROM sessions WHERE created_at >= ${since(REFERRAL_SOURCE_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} GROUP BY referrer_name HAVING percentage >= ${sql.float64(REFERRAL_SOURCE_MIN_SHARE)} ORDER BY count DESC`- );- return results.map((result) => ({- type: 'referral_source',- message: `${result.referrer_name} was your top referral source this week, contributing to ${Math.round(result.percentage * 100)}% of the total traffic.`,- data: result,- }));- }-- private async getSessionDurationChanges(- projectId: string- ): Promise<Insight[]> {- const results = await this.run<SessionDurationResult>(- sql`SELECT toWeek(created_at) as week, avg(duration) as avg_duration, lag(avg(duration)) OVER (ORDER BY week) as prev_week_duration FROM sessions WHERE created_at >= ${since(SESSION_DURATION_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} GROUP BY week HAVING avg_duration > ${sql.string(SESSION_DURATION_THRESHOLD)} ORDER BY week DESC`- );- return results.map((result) => ({- type: 'session_duration',- message: `Users spent ${Math.round((result.avg_duration / result.prev_week_duration - 1) * 100)}% more time on average per session this week compared to last week.`,- data: result,- }));- }-- private async getTopPerformingContent(projectId: string): Promise<Insight[]> {- const results = await this.run<TopContentResult>(- sql`SELECT path, COUNT(*) as view_count, COUNT(DISTINCT device_id) as unique_viewers FROM events WHERE created_at >= ${since(TOP_CONTENT_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} GROUP BY path ORDER BY view_count DESC LIMIT ${sql.uint64(TOP_CONTENT_LIMIT)}`- );- return results.map((result) => ({- type: 'top_content',- message: `Your content at "${result.path}" was the most viewed content this month with ${result.view_count} views from ${result.unique_viewers} unique viewers.`,- data: result,- }));- }-- private async getBounceRateImprovements(- projectId: string- ): Promise<Insight[]> {- const results = await this.run<BounceRateResult>(- sql`SELECT toMonth(created_at) as month, sum(is_bounce) / COUNT(*) as bounce_rate, lag(sum(is_bounce) / COUNT(*)) OVER (ORDER BY month) as prev_month_bounce_rate FROM sessions WHERE created_at >= ${since(BOUNCE_RATE_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} GROUP BY month HAVING bounce_rate < ${sql.string(BOUNCE_RATE_THRESHOLD)} ORDER BY month DESC`- );- return results.map((result) => ({- type: 'bounce_rate',- message: `The bounce rate decreased by ${Math.round((1 - result.bounce_rate / result.prev_month_bounce_rate) * 100)}% this month, indicating more engaging content.`,- data: result,- }));- }-- private async getReturningVisitorTrends(- projectId: string- ): Promise<Insight[]> {- const results = await this.run<ReturningVisitorResult>(- sql`SELECT toQuarter(created_at) as quarter, COUNT(DISTINCT device_id) as returning_visitors, lag(COUNT(DISTINCT device_id)) OVER (ORDER BY quarter) as prev_quarter_visitors FROM sessions WHERE created_at >= ${since(RETURNING_VISITOR_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} AND is_returning = true GROUP BY quarter HAVING returning_visitors > ${sql.string(RETURNING_VISITOR_THRESHOLD)} ORDER BY quarter DESC`- );- return results.map((result) => ({- type: 'returning_visitors',- message: `Returning visitors increased by ${Math.round((result.returning_visitors / result.prev_quarter_visitors - 1) * 100)}% this quarter, showing growing user loyalty.`,- data: result,- }));- }-- private async getGeographicInterestShifts(- projectId: string- ): Promise<Insight[]> {- const results = await this.run<GeographicShiftResult>(- sql`SELECT country, COUNT(*) as visitor_count, lag(COUNT(*)) OVER (ORDER BY toWeek(created_at)) as prev_week_count FROM sessions WHERE created_at >= ${since(GEOGRAPHIC_SHIFT_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} GROUP BY country, toWeek(created_at) HAVING visitor_count > ${sql.string(GEOGRAPHIC_SHIFT_THRESHOLD)} ORDER BY visitor_count DESC`- );- return results.map((result) => ({- type: 'geographic_shift',- message: `There was a noticeable increase in traffic from ${result.country} this week.`,- data: result,- }));- }-- private async getEventCompletionChanges(- projectId: string- ): Promise<Insight[]> {- const results = await this.run<EventCompletionResult>(- sql`SELECT event_name, toMonth(created_at) as month, COUNT(*) as completion_count, lag(COUNT(*)) OVER (ORDER BY month) as prev_month_count FROM events WHERE created_at >= ${since(EVENT_COMPLETION_WINDOW_DAYS)} AND project_id = ${sql.string(projectId)} AND status = ${sql.string(COMPLETED_EVENT_STATUS)} GROUP BY event_name, month HAVING completion_count > ${sql.string(EVENT_COMPLETION_THRESHOLD)} ORDER BY month DESC`- );- return results.map((result) => ({- type: 'event_completion',- message: `The completion rate for your "${result.event_name}" event increased by ${Math.round((result.completion_count / result.prev_month_count - 1) * 100)}% this month.`,- data: result,- }));- }-- async generateInsights(projectId: string): Promise<Insight[]> {- const [- trafficSpikes,- eventSurges,- newVisitorTrends,- referralSources,- sessionDurations,- topContent,- bounceRates,- returningVisitors,- geographicShifts,- eventCompletions,- ] = await Promise.all([- this.getTrafficSpikes(projectId),- this.getEventSurges(projectId),- this.getNewVisitorTrends(projectId),- this.getReferralSourceHighlights(projectId),- this.getSessionDurationChanges(projectId),- this.getTopPerformingContent(projectId),- this.getBounceRateImprovements(projectId),- this.getReturningVisitorTrends(projectId),- this.getGeographicInterestShifts(projectId),- this.getEventCompletionChanges(projectId),- ]);-- return [- ...trafficSpikes,- ...eventSurges,- ...newVisitorTrends,- ...referralSources,- ...sessionDurations,- ...topContent,- ...bounceRates,- ...returningVisitors,- ...geographicShifts,- ...eventCompletions,- ].sort((a, b) => {- const dateA = new Date(insightPeriod(a.data));- const dateB = new Date(insightPeriod(b.data));- return dateB.getTime() - dateA.getTime();- });- }-}--export function createLegacyInsightsScanner(deps: ChScope) {- const scanner = new LegacyInsightsScanner(deps);- return {- generateInsights: (projectId: string): Promise<Insight[]> =>- scanner.generateInsights(projectId),- };-}diff --git a/packages/core/src/modules/report/report.constants.ts b/packages/core/src/modules/report/report.constants.tsindex 6676d0858..2f9b461d6 100644--- a/packages/core/src/modules/report/report.constants.ts+++ b/packages/core/src/modules/report/report.constants.ts@@ -931,9 +931,6 @@ export const zReport = zReportInput.extend({.describe('The visual style of the line in the chart'),});-// Alias for backward compatibility-export const zChartInput = zReportInput;-// `UnionOmit` has no owning module; it lives with the report types it is applied to.export type UnionOmit<T, K extends keyof any> = T extends any@@ -986,7 +983,7 @@ export type PreviousValue =}| undefined;-export type Metrics = {+export interface Metrics {sum: number;average: number;min: number;@@ -999,9 +996,9 @@ export type Metrics = {max: PreviousValue;count: PreviousValue;};-};+}-export type IChartSerie = {+export interface IChartSerie {id: string;names: string[];event: {@@ -1021,9 +1018,9 @@ export type IChartSerie = {*/previous?: PreviousValue;}[];-};+}-export type FinalChart = {+export interface FinalChart {series: IChartSerie[];metrics: Metrics;-};+}diff --git a/packages/core/src/shared/json.ts b/packages/core/src/shared/json.tsdeleted file mode 100644index 662f8f26e..000000000--- a/packages/core/src/shared/json.ts+++ /dev/null@@ -1,5 +0,0 @@-// A re-export seam kept for ONE importer: packages/db/src/clickhouse/client.ts-// reaches `getSafeJson` by relative path (`../../../core/src/shared/json`)-// because importing the core barrel there would invert the core -> db edge.-// That import should point at @openpanel/shared; delete this file with it.-export { getSafeJson, getSuperJson, setSuperJson } from '@openpanel/shared';diff --git a/packages/db/package.json b/packages/db/package.jsonindex 69661dd43..31badef46 100644--- a/packages/db/package.json+++ b/packages/db/package.json@@ -21,6 +21,7 @@"@openpanel/core": "workspace:*","@openpanel/payments": "workspace:*","@openpanel/redis": "workspace:*",+ "@openpanel/shared": "workspace:*","@prisma/client": "6.14.0","pino": "catalog:","pino-pretty": "catalog:",diff --git a/packages/db/scripts/ch-copy-from-remote.ts b/packages/db/scripts/ch-copy-from-remote.tsdeleted file mode 100644index 0c84a952f..000000000--- a/packages/db/scripts/ch-copy-from-remote.ts+++ /dev/null@@ -1,112 +0,0 @@-import { stdin as input, stdout as output } from 'node:process';-import { createInterface } from 'node:readline/promises';-import { parseArgs } from 'node:util';-import { ch } from '../src/clickhouse/client';-import { sql } from '../src/clickhouse/sql';--async function main() {- const rl = createInterface({ input, output });-- try {- const { values } = parseArgs({- args: process.argv.slice(2),- options: {- host: { type: 'string' },- user: { type: 'string' },- password: { type: 'string' },- db: { type: 'string' },- start: { type: 'string' },- end: { type: 'string' },- projects: { type: 'string' },- },- strict: false,- });-- const getArg = (val: unknown): string | undefined =>- typeof val === 'string' ? val : undefined;-- console.log('Copy data from remote ClickHouse to local');- console.log('---------------------------------------');-- const host =- getArg(values.host) || (await rl.question('Remote Host (IP/Domain): '));- if (!host) {- throw new Error('Host is required');- }-- const user = getArg(values.user) || (await rl.question('Remote User: '));- if (!user) {- throw new Error('User is required');- }-- const password =- getArg(values.password) || (await rl.question('Remote Password: '));- if (!password) {- throw new Error('Password is required');- }-- const dbName =- getArg(values.db) ||- (await rl.question('Remote DB Name (default: openpanel): ')) ||- 'openpanel';-- const startDate =- getArg(values.start) ||- (await rl.question('Start Date (YYYY-MM-DD HH:mm:ss): '));- if (!startDate) {- throw new Error('Start date is required');- }-- const endDate =- getArg(values.end) ||- (await rl.question('End Date (YYYY-MM-DD HH:mm:ss): '));- if (!endDate) {- throw new Error('End date is required');- }-- const projectIdsInput =- getArg(values.projects) ||- (await rl.question(- 'Project IDs (comma separated, leave empty for all): '- ));- const projectIds = projectIdsInput- ? projectIdsInput.split(',').map((s: string) => s.trim())- : [];-- console.log('\nStarting copy process...');-- const tables = ['sessions', 'events'];-- for (const table of tables) {- console.log(`Processing table: ${table}`);-- // Every remote() argument is a bound param, so credentials never enter- // the query text and the log line below is safe. `dbName` and `table` are- // bound as {x:Identifier} inside remote() and validated by sql.id() in- // the INSERT target, where a param is not accepted.- const remoteTable = sql`remote(${sql.string(host)}, ${sql.identifier(dbName)}, ${sql.identifier(table)}, ${sql.string(user)}, ${sql.string(password)})`;-- const projectFilter =- projectIds.length > 0- ? sql` AND project_id IN ${sql.array('String', projectIds)}`- : sql.empty;-- const selectQuery = sql`SELECT * FROM ${remoteTable} WHERE created_at BETWEEN ${sql.string(startDate)} AND ${sql.string(endDate)}${projectFilter}`;-- const insertQuery =- sql`INSERT INTO ${sql.id(dbName)}.${sql.id(table)} ${selectQuery}`.toStatement();-- console.log(`Executing: ${insertQuery.query}`);- }-- console.log('\nDone!');- } catch (error) {- console.error('\nError:', error);- } finally {- rl.close();- await ch.close();- process.exit(0);- }-}--main();diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex 5eb4a8451..9b7a0df43 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -5,6 +5,7 @@ import type {} from '@clickhouse/client';import { ClickHouseLogLevel, createClient } from '@clickhouse/client';import type { NodeClickHouseClientConfigOptions } from '@clickhouse/client/dist/config';+import { getSafeJson } from '@openpanel/shared';import { createLogger } from '../logger';import { RoundRobinPicker, withRoundRobinRetry } from './round-robin';import { type SqlFragment, toStatement } from './sql';@@ -14,9 +15,6 @@ export { createClient } from '@clickhouse/client';const logger = createLogger({ name: 'clickhouse' });import type { Logger } from '@clickhouse/client';-// Relative path, not the `@openpanel/core` barrel: core depends on this-// package, so the barrel would pull core's module graph in for one helper.-import { getSafeJson } from '../../../core/src/shared/json';interface LogParams {module: string;diff --git a/packages/db/src/services/access.service.ts b/packages/db/src/services/access.service.tsdeleted file mode 100644index 449257a49..000000000--- a/packages/db/src/services/access.service.ts+++ /dev/null@@ -1,8 +0,0 @@-// Deep-path-only re-export of the lookups in packages/core/src/shared/access-lookups.ts.-export {- canWriteProject,- getClientAccess,- getOrganizationAccess,- getProjectAccess,- type IProjectAccess,-} from '@openpanel/core';diff --git a/scripts/seed-events.mjs b/scripts/seed-events.mjsdeleted file mode 100644index 47bb2a86e..000000000--- a/scripts/seed-events.mjs+++ /dev/null@@ -1,1289 +0,0 @@-#!/usr/bin/env node-/**- * Seed script for generating realistic analytics events.- *- * Usage:- * node scripts/seed-events.mjs [--timeline=30] [--sessions=500] [--url=http://localhost:3333]- *- * Options:- * --timeline=N Duration in minutes to spread events over (default: 30)- * --sessions=N Number of sessions to generate (default: 500)- * --url=URL API base URL (default: http://localhost:3333)- * --clientId=ID Client ID to use (required or set CLIENT_ID env var)- */--const args = Object.fromEntries(- process.argv.slice(2).map((a) => {- const [k, v] = a.replace(/^--/, '').split('=');- return [k, v ?? true];- })-);--const TIMELINE_MINUTES = Number(args.timeline ?? 30);-const SESSION_COUNT = Number(args.sessions ?? 500);-const BASE_URL = args.url ?? 'http://localhost:3333';-const CLIENT_ID = args.clientId ?? process.env.CLIENT_ID ?? '';-const ORIGIN = args.origin ?? process.env.ORIGIN ?? 'https://shop.example.com';-const CONCURRENCY = 20; // max parallel requests--if (!CLIENT_ID) {- console.error('ERROR: provide --clientId=<id> or set CLIENT_ID env var');- process.exit(1);-}--const TRACK_URL = `${BASE_URL}/track`;--function mulberry32(seed) {- return () => {- seed |= 0;- seed = (seed + 0x6d_2b_79_f5) | 0;- let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);- t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;- return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296;- };-}--// Non-deterministic random for events (differs on each run)-const eventRng = Math.random.bind(Math);--function pick(arr, rng = eventRng) {- return arr[Math.floor(rng() * arr.length)];-}--function randInt(min, max, rng = eventRng) {- return Math.floor(rng() * (max - min + 1)) + min;-}--function randFloat(min, max, rng = eventRng) {- return rng() * (max - min) + min;-}--const FIRST_NAMES = [- 'Alice',- 'Bob',- 'Charlie',- 'Diana',- 'Eve',- 'Frank',- 'Grace',- 'Hank',- 'Iris',- 'Jack',- 'Karen',- 'Leo',- 'Mia',- 'Noah',- 'Olivia',- 'Pete',- 'Quinn',- 'Rachel',- 'Sam',- 'Tina',- 'Uma',- 'Victor',- 'Wendy',- 'Xavier',- 'Yara',- 'Zoe',- 'Aaron',- 'Bella',- 'Carlos',- 'Dani',- 'Ethan',- 'Fiona',-];--const LAST_NAMES = [- 'Smith',- 'Johnson',- 'Williams',- 'Brown',- 'Jones',- 'Garcia',- 'Miller',- 'Davis',- 'Wilson',- 'Taylor',- 'Anderson',- 'Thomas',- 'Jackson',- 'White',- 'Harris',- 'Martin',- 'Thompson',- 'Moore',- 'Young',- 'Allen',-];--const EMAIL_DOMAINS = [- 'gmail.com',- 'yahoo.com',- 'outlook.com',- 'icloud.com',- 'proton.me',-];--const USER_AGENTS = [- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',- 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',- 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1',- 'Mozilla/5.0 (iPad; CPU OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1',- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0',- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14.3; rv:122.0) Gecko/20100101 Firefox/122.0',- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0',- 'Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36',- 'Mozilla/5.0 (Linux; Android 13; Samsung Galaxy S23) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/23.0 Chrome/115.0.0.0 Mobile Safari/537.36',- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 OPR/107.0.0.0',-];--// Ensure each session has a unique UA by appending a suffix-function makeUniqueUA(base, index) {- return `${base} Session/${index}`;-}--// Generate a plausible IP address (avoiding private ranges)-function makeIP(index) {- // Use a spread across several /8 public ranges- const ranges = [- [34, 0, 0, 0],- [52, 0, 0, 0],- [104, 0, 0, 0],- [185, 0, 0, 0],- [213, 0, 0, 0],- ];- const base = ranges[index % ranges.length];- const a = base[0];- const b = Math.floor(index / 65_025) % 256;- const c = Math.floor(index / 255) % 256;- const d = (index % 255) + 1;- return `${a}.${b}.${c}.${d}`;-}--const PRODUCTS = [- {- id: 'prod_001',- name: 'Wireless Headphones',- category: 'Electronics',- price: 8999,- },- { id: 'prod_002', name: 'Running Shoes', category: 'Sports', price: 12_999 },- { id: 'prod_003', name: 'Coffee Maker', category: 'Kitchen', price: 5499 },- { id: 'prod_004', name: 'Yoga Mat', category: 'Sports', price: 2999 },- {- id: 'prod_005',- name: 'Smart Watch',- category: 'Electronics',- price: 29_999,- },- { id: 'prod_006', name: 'Blender', category: 'Kitchen', price: 7999 },- { id: 'prod_007', name: 'Backpack', category: 'Travel', price: 4999 },- { id: 'prod_008', name: 'Sunglasses', category: 'Accessories', price: 3499 },- {- id: 'prod_009',- name: 'Novel: The Last Algorithm',- category: 'Books',- price: 1499,- },- {- id: 'prod_010',- name: 'Standing Desk',- category: 'Furniture',- price: 45_999,- },-];--const CATEGORIES = [- 'Electronics',- 'Sports',- 'Kitchen',- 'Travel',- 'Accessories',- 'Books',- 'Furniture',-];--const GROUPS = [- {- id: 'org_acme',- type: 'company',- name: 'Acme Corp',- properties: { plan: 'enterprise', industry: 'Technology', employees: 500 },- },- {- id: 'org_globex',- type: 'company',- name: 'Globex Inc',- properties: { plan: 'pro', industry: 'Finance', employees: 120 },- },- {- id: 'org_initech',- type: 'company',- name: 'Initech LLC',- properties: { plan: 'starter', industry: 'Consulting', employees: 45 },- },-];--/**- * Each scenario returns a list of event descriptors.- * screen_view events use a `path` property (origin + pathname).- */--const SCENARIOS = [- // 1. Full e-commerce checkout success- (product) => [- {- name: 'screen_view',- props: {- path: `${ORIGIN}/`,- title: 'Home',- referrer: 'https://google.com',- },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- {- name: 'add_to_cart',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- quantity: 1,- },- },- { name: 'screen_view', props: { path: `${ORIGIN}/cart`, title: 'Cart' } },- {- name: 'checkout_started',- props: { cart_total: product.price, item_count: 1 },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/checkout/shipping`, title: 'Shipping Info' },- },- {- name: 'shipping_info_submitted',- props: { shipping_method: 'standard', estimated_days: 5 },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/checkout/payment`, title: 'Payment' },- },- {- name: 'payment_info_submitted',- props: { payment_method: 'credit_card' },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/checkout/review`, title: 'Order Review' },- },- {- name: 'purchase',- props: {- order_id: `ord_${Date.now()}`,- revenue: product.price,- product_id: product.id,- product_name: product.name,- quantity: 1,- },- revenue: true,- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/checkout/success`, title: 'Order Confirmed' },- },- {- name: 'checkout_success',- props: { order_id: `ord_${Date.now()}`, revenue: product.price },- },- ],-- // 2. Checkout failed (payment declined)- (product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- {- name: 'add_to_cart',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- quantity: 1,- },- },- {- name: 'checkout_started',- props: { cart_total: product.price, item_count: 1 },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/checkout/shipping`, title: 'Shipping Info' },- },- {- name: 'shipping_info_submitted',- props: { shipping_method: 'express', estimated_days: 2 },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/checkout/payment`, title: 'Payment' },- },- {- name: 'payment_info_submitted',- props: { payment_method: 'credit_card' },- },- {- name: 'checkout_failed',- props: { reason: 'payment_declined', error_code: 'insufficient_funds' },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/checkout/payment`, title: 'Payment' },- },- ],-- // 3. Browse only — no purchase- (product) => [- {- name: 'screen_view',- props: {- path: `${ORIGIN}/`,- title: 'Home',- referrer: 'https://facebook.com',- },- },- {- name: 'screen_view',- props: {- path: `${ORIGIN}/categories/${product.category.toLowerCase()}`,- title: product.category,- },- },- { name: 'category_viewed', props: { category: product.category } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- {- name: 'screen_view',- props: {- path: `${ORIGIN}/products/${PRODUCTS[1].id}`,- title: PRODUCTS[1].name,- },- },- {- name: 'product_viewed',- props: {- product_id: PRODUCTS[1].id,- product_name: PRODUCTS[1].name,- price: PRODUCTS[1].price,- category: PRODUCTS[1].category,- },- },- ],-- // 4. Add to cart then abandon- (product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- {- name: 'add_to_cart',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- quantity: 1,- },- },- { name: 'screen_view', props: { path: `${ORIGIN}/cart`, title: 'Cart' } },- {- name: 'cart_abandoned',- props: { cart_total: product.price, item_count: 1 },- },- ],-- // 5. Search → product → purchase- (product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'search',- props: {- query: product.name.split(' ')[0],- result_count: randInt(3, 20),- },- },- {- name: 'screen_view',- props: {- path: `${ORIGIN}/search?q=${encodeURIComponent(product.name)}`,- title: 'Search Results',- },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- {- name: 'add_to_cart',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- quantity: 1,- },- },- {- name: 'checkout_started',- props: { cart_total: product.price, item_count: 1 },- },- {- name: 'shipping_info_submitted',- props: { shipping_method: 'standard', estimated_days: 5 },- },- { name: 'payment_info_submitted', props: { payment_method: 'paypal' } },- {- name: 'purchase',- props: {- order_id: `ord_${Date.now()}`,- revenue: product.price,- product_id: product.id,- product_name: product.name,- quantity: 1,- },- revenue: true,- },- {- name: 'checkout_success',- props: { order_id: `ord_${Date.now()}`, revenue: product.price },- },- ],-- // 6. Sign up flow- (_product) => [- {- name: 'screen_view',- props: {- path: `${ORIGIN}/`,- title: 'Home',- referrer: 'https://twitter.com',- },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/signup`, title: 'Sign Up' },- },- { name: 'signup_started', props: {} },- { name: 'signup_step_completed', props: { step: 'email', step_number: 1 } },- {- name: 'signup_step_completed',- props: { step: 'password', step_number: 2 },- },- {- name: 'signup_step_completed',- props: { step: 'profile', step_number: 3 },- },- { name: 'signup_completed', props: { method: 'email' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/dashboard`, title: 'Dashboard' },- },- ],-- // 7. Login → browse → wishlist- (product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/login`, title: 'Login' } },- { name: 'login', props: { method: 'email' } },- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- {- name: 'add_to_wishlist',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/wishlist`, title: 'Wishlist' },- },- ],-- // 8. Promo code → purchase- (product) => [- {- name: 'screen_view',- props: {- path: `${ORIGIN}/`,- title: 'Home',- referrer: 'https://newsletter.example.com',- },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- {- name: 'add_to_cart',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- quantity: 1,- },- },- {- name: 'checkout_started',- props: { cart_total: product.price, item_count: 1 },- },- {- name: 'promo_code_applied',- props: {- code: 'SAVE20',- discount_percent: 20,- discount_amount: Math.round(product.price * 0.2),- },- },- {- name: 'shipping_info_submitted',- props: { shipping_method: 'standard', estimated_days: 5 },- },- {- name: 'payment_info_submitted',- props: { payment_method: 'credit_card' },- },- {- name: 'purchase',- props: {- order_id: `ord_${Date.now()}`,- revenue: Math.round(product.price * 0.8),- product_id: product.id,- product_name: product.name,- quantity: 1,- promo_code: 'SAVE20',- },- revenue: true,- },- {- name: 'checkout_success',- props: {- order_id: `ord_${Date.now()}`,- revenue: Math.round(product.price * 0.8),- },- },- ],-- // 9. Multi-item purchase- (_product) => {- const p1 = PRODUCTS[0];- const p2 = PRODUCTS[3];- const total = p1.price + p2.price;- return [- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${p1.id}`, title: p1.name },- },- {- name: 'product_viewed',- props: {- product_id: p1.id,- product_name: p1.name,- price: p1.price,- category: p1.category,- },- },- {- name: 'add_to_cart',- props: {- product_id: p1.id,- product_name: p1.name,- price: p1.price,- quantity: 1,- },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${p2.id}`, title: p2.name },- },- {- name: 'product_viewed',- props: {- product_id: p2.id,- product_name: p2.name,- price: p2.price,- category: p2.category,- },- },- {- name: 'add_to_cart',- props: {- product_id: p2.id,- product_name: p2.name,- price: p2.price,- quantity: 1,- },- },- { name: 'checkout_started', props: { cart_total: total, item_count: 2 } },- {- name: 'shipping_info_submitted',- props: { shipping_method: 'express', estimated_days: 2 },- },- {- name: 'payment_info_submitted',- props: { payment_method: 'credit_card' },- },- {- name: 'purchase',- props: { order_id: `ord_${Date.now()}`, revenue: total, item_count: 2 },- revenue: true,- },- {- name: 'checkout_success',- props: { order_id: `ord_${Date.now()}`, revenue: total },- },- ];- },-- // 10. Help center visit- (_product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/help`, title: 'Help Center' },- },- { name: 'help_search', props: { query: 'return policy', result_count: 4 } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/help/returns`, title: 'Return Policy' },- },- {- name: 'help_article_read',- props: { article: 'return_policy', time_on_page: randInt(60, 180) },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/help/shipping`, title: 'Shipping Info' },- },- {- name: 'help_article_read',- props: { article: 'shipping_times', time_on_page: randInt(30, 120) },- },- ],-- // 11. Product review submitted- (product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- { name: 'review_started', props: { product_id: product.id } },- {- name: 'review_submitted',- props: { product_id: product.id, rating: randInt(3, 5), has_text: true },- },- ],-- // 12. Newsletter signup only- (_product) => [- {- name: 'screen_view',- props: {- path: `${ORIGIN}/`,- title: 'Home',- referrer: 'https://instagram.com',- },- },- { name: 'screen_view', props: { path: `${ORIGIN}/blog`, title: 'Blog' } },- {- name: 'screen_view',- props: {- path: `${ORIGIN}/blog/top-10-gadgets-2024`,- title: 'Top 10 Gadgets 2024',- },- },- {- name: 'newsletter_signup',- props: { source: 'blog_article', campaign: 'gadgets_2024' },- },- ],-- // 13. Account settings update- (_product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/login`, title: 'Login' } },- { name: 'login', props: { method: 'google' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/account`, title: 'Account' },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/account/settings`, title: 'Settings' },- },- {- name: 'settings_updated',- props: { field: 'notification_preferences', value: 'email_only' },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/account/address`, title: 'Addresses' },- },- { name: 'address_added', props: { is_default: true } },- ],-- // 14. Referral program engagement- (product) => [- {- name: 'screen_view',- props: {- path: `${ORIGIN}/`,- title: 'Home',- referrer: 'https://referral.example.com/?ref=abc123',- },- },- {- name: 'referral_link_clicked',- props: { referrer_id: 'usr_ref123', campaign: 'summer_referral' },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- {- name: 'add_to_cart',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- quantity: 1,- },- },- {- name: 'checkout_started',- props: { cart_total: product.price, item_count: 1 },- },- {- name: 'shipping_info_submitted',- props: { shipping_method: 'standard', estimated_days: 5 },- },- {- name: 'payment_info_submitted',- props: { payment_method: 'credit_card' },- },- {- name: 'purchase',- props: {- order_id: `ord_${Date.now()}`,- revenue: product.price,- referral_code: 'abc123',- },- revenue: true,- },- {- name: 'checkout_success',- props: { order_id: `ord_${Date.now()}`, revenue: product.price },- },- ],-- // 15. Mobile quick browse — short session- (product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'screen_view',- props: {- path: `${ORIGIN}/categories/${product.category.toLowerCase()}`,- title: product.category,- },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- ],-- // 16. Compare products- (product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- { name: 'compare_added', props: { product_id: product.id } },- {- name: 'screen_view',- props: {- path: `${ORIGIN}/products/${PRODUCTS[4].id}`,- title: PRODUCTS[4].name,- },- },- {- name: 'product_viewed',- props: {- product_id: PRODUCTS[4].id,- product_name: PRODUCTS[4].name,- price: PRODUCTS[4].price,- category: PRODUCTS[4].category,- },- },- { name: 'compare_added', props: { product_id: PRODUCTS[4].id } },- {- name: 'screen_view',- props: {- path: `${ORIGIN}/compare?ids=${product.id},${PRODUCTS[4].id}`,- title: 'Compare Products',- },- },- {- name: 'compare_viewed',- props: { product_ids: [product.id, PRODUCTS[4].id] },- },- ],-- // 17. Shipping failure retry → success- (product) => [- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- {- name: 'screen_view',- props: { path: `${ORIGIN}/products/${product.id}`, title: product.name },- },- {- name: 'product_viewed',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- category: product.category,- },- },- {- name: 'add_to_cart',- props: {- product_id: product.id,- product_name: product.name,- price: product.price,- quantity: 1,- },- },- {- name: 'checkout_started',- props: { cart_total: product.price, item_count: 1 },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/checkout/shipping`, title: 'Shipping Info' },- },- {- name: 'shipping_info_error',- props: { error: 'invalid_address', attempt: 1 },- },- {- name: 'shipping_info_submitted',- props: { shipping_method: 'standard', estimated_days: 5, attempt: 2 },- },- {- name: 'payment_info_submitted',- props: { payment_method: 'credit_card' },- },- {- name: 'purchase',- props: {- order_id: `ord_${Date.now()}`,- revenue: product.price,- product_id: product.id,- },- revenue: true,- },- {- name: 'checkout_success',- props: { order_id: `ord_${Date.now()}`, revenue: product.price },- },- ],-- // 18. Subscription / SaaS upgrade- (_product) => [- {- name: 'screen_view',- props: { path: `${ORIGIN}/pricing`, title: 'Pricing' },- },- { name: 'pricing_viewed', props: {} },- {- name: 'plan_selected',- props: { plan: 'pro', billing: 'annual', price: 9900 },- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/checkout/subscription`, title: 'Subscribe' },- },- {- name: 'payment_info_submitted',- props: { payment_method: 'credit_card' },- },- {- name: 'subscription_started',- props: { plan: 'pro', billing: 'annual', revenue: 9900 },- revenue: true,- },- {- name: 'screen_view',- props: { path: `${ORIGIN}/dashboard`, title: 'Dashboard' },- },- ],-- // 19. Deep content engagement (blog)- (_product) => [- {- name: 'screen_view',- props: {- path: `${ORIGIN}/blog`,- title: 'Blog',- referrer: 'https://google.com',- },- },- {- name: 'screen_view',- props: {- path: `${ORIGIN}/blog/buying-guide-headphones`,- title: 'Headphones Buying Guide',- },- },- {- name: 'content_read',- props: {- article: 'headphones_buying_guide',- reading_time: randInt(120, 480),- scroll_depth: randFloat(0.6, 1.0),- },- },- {- name: 'screen_view',- props: {- path: `${ORIGIN}/blog/best-running-shoes-2024`,- title: 'Best Running Shoes 2024',- },- },- {- name: 'content_read',- props: {- article: 'best_running_shoes_2024',- reading_time: randInt(90, 300),- scroll_depth: randFloat(0.5, 1.0),- },- },- ],-- // 20. Error / 404 bounce- (_product) => [- {- name: 'screen_view',- props: {- path: `${ORIGIN}/products/old-discontinued-product`,- title: 'Product Not Found',- },- },- {- name: 'page_error',- props: { error_code: 404, path: '/products/old-discontinued-product' },- },- { name: 'screen_view', props: { path: `${ORIGIN}/`, title: 'Home' } },- ],-];--function generateIdentity(sessionIndex, sessionRng) {- const firstName = pick(FIRST_NAMES, sessionRng);- const lastName = pick(LAST_NAMES, sessionRng);- const emailDomain = pick(EMAIL_DOMAINS, sessionRng);- const profileId = `user_${String(sessionIndex + 1).padStart(4, '0')}`;- return {- profileId,- firstName,- lastName,- email: `${firstName.toLowerCase()}.${lastName.toLowerCase()}${sessionIndex}@${emailDomain}`,- };-}--// Which sessions belong to which group (roughly 1/6 each)-function getGroupForSession(sessionIndex) {- if (sessionIndex % 6 === 0) {- return GROUPS[0];- }- if (sessionIndex % 6 === 1) {- return GROUPS[1];- }- if (sessionIndex % 6 === 2) {- return GROUPS[2];- }- return null;-}--async function sendEvent(payload, ua, ip) {- const headers = {- 'Content-Type': 'application/json',- 'user-agent': ua,- 'openpanel-client-id': CLIENT_ID,- 'x-forwarded-for': ip,- origin: ORIGIN,- };-- const res = await fetch(TRACK_URL, {- method: 'POST',- headers,- body: JSON.stringify(payload),- });-- if (!res.ok) {- const text = await res.text().catch(() => '');- console.warn(- ` [WARN] ${res.status} ${payload.type}/${payload.payload?.name ?? ''}: ${text.slice(0, 120)}`- );- }- return res;-}--function buildSession(sessionIndex) {- const sessionRng = mulberry32(sessionIndex * 9973 + 1337); // deterministic per session-- const identity = generateIdentity(sessionIndex, sessionRng);- const group = getGroupForSession(sessionIndex);- const ua = makeUniqueUA(pick(USER_AGENTS, sessionRng), sessionIndex);- const ip = makeIP(sessionIndex);- const product = pick(PRODUCTS, sessionRng);- const scenarioFn = SCENARIOS[sessionIndex % SCENARIOS.length];- const events = scenarioFn(product);-- return { identity, group, ua, ip, events };-}--function scheduleSession(session, sessionIndex, totalSessions) {- const timelineMs = TIMELINE_MINUTES * 60 * 1000;- const now = Date.now();-- // Sessions are spread across the timeline- const sessionStartOffset = (sessionIndex / totalSessions) * timelineMs;- const sessionStart = now - timelineMs + sessionStartOffset;-- // Events within session: spread over 2-10 minutes- const sessionDurationMs = randInt(2, 10) * 60 * 1000;- const eventCount = session.events.length;-- return session.events.map((event, i) => {- const eventOffset =- eventCount > 1 ? (i / (eventCount - 1)) * sessionDurationMs : 0;- return {- ...event,- timestamp: Math.round(sessionStart + eventOffset),- };- });-}--async function withConcurrency(tasks, limit) {- const results = [];- const executing = [];- for (const task of tasks) {- const p = Promise.resolve().then(task);- results.push(p);- const e = p.then(() => executing.splice(executing.indexOf(e), 1));- executing.push(e);- if (executing.length >= limit) {- await Promise.race(executing);- }- }- return Promise.all(results);-}--async function main() {- console.log(- `\nSeeding ${SESSION_COUNT} sessions over ${TIMELINE_MINUTES} minutes`- );- console.log(`API: ${TRACK_URL}`);- console.log(`Client ID: ${CLIENT_ID}\n`);-- let totalEvents = 0;- let errors = 0;-- const sessionTasks = Array.from(- { length: SESSION_COUNT },- (_, i) => async () => {- const session = buildSession(i);- const scheduledEvents = scheduleSession(session, i, SESSION_COUNT);- const { identity, group, ua, ip } = session;-- // 1. Identify- try {- await sendEvent({ type: 'identify', payload: identity }, ua, ip);- } catch (e) {- errors++;- console.error(` [ERROR] identify session ${i}:`, e.message);- }-- // 2. Group (if applicable)- if (group) {- try {- await sendEvent(- {- type: 'group',- payload: { ...group, profileId: identity.profileId },- },- ua,- ip- );- } catch (e) {- errors++;- console.error(` [ERROR] group session ${i}:`, e.message);- }- }-- // 3. Track events in order- for (const ev of scheduledEvents) {- const trackPayload = {- name: ev.name,- profileId: identity.profileId,- properties: {- ...ev.props,- __timestamp: new Date(ev.timestamp).toISOString(),- ...(group ? { __group: group.id } : {}),- },- groups: group ? [group.id] : [],- };-- if (ev.revenue) {- trackPayload.properties.__revenue = ev.props.revenue;- }-- try {- await sendEvent({ type: 'track', payload: trackPayload }, ua, ip);- totalEvents++;- } catch (e) {- errors++;- console.error(` [ERROR] track ${ev.name} session ${i}:`, e.message);- }- }-- if ((i + 1) % 50 === 0 || i + 1 === SESSION_COUNT) {- console.log(` Progress: ${i + 1}/${SESSION_COUNT} sessions`);- }- }- );-- await withConcurrency(sessionTasks, CONCURRENCY);-- console.log('\nDone!');- console.log(` Sessions: ${SESSION_COUNT}`);- console.log(` Events sent: ${totalEvents}`);- console.log(` Errors: ${errors}`);-}--main().catch((err) => {- console.error('Fatal:', err);- process.exit(1);-});