OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
self-hosting: add the doctor core for the upcoming openpanel CLI
Carl-Gerhard Lindesvärd120ef4491514
diff --git a/self-hosting/.gitignore b/self-hosting/.gitignoreindex ab97a22ac..8669d3924 100644--- a/self-hosting/.gitignore+++ b/self-hosting/.gitignore@@ -2,3 +2,6 @@docker-compose.ymlcaddy/Caddyfileredpanda/bootstrap.yaml++# Test fixtures are real install layouts, so they need the ignored names.+!cli/test/fixtures/**diff --git a/self-hosting/cli/bun.lock b/self-hosting/cli/bun.locknew file mode 100644index 000000000..8c5cf1751--- /dev/null+++ b/self-hosting/cli/bun.lock@@ -0,0 +1,35 @@+{+ "lockfileVersion": 2,+ "configVersion": 1,+ "workspaces": {+ "": {+ "name": "@openpanel/cli",+ "dependencies": {+ "@clack/prompts": "^0.11.0",+ "yaml": "^2.8.0",+ },+ "devDependencies": {+ "@types/bun": "latest",+ },+ },+ },+ "packages": {+ "@clack/core": ["@clack/[email protected]", "", { "dependencies": { "picocolors": "^1.0.0", "sisteransi": "^1.0.5" } }, "sha512-p3y0FIOwaYRUPRcMO7+dlmLh8PSRcrjuTndsiA0WAFbWES0mLZlrjVoBRZ9DzkPFJZG6KGkJmoEAY0ZcVWTkow=="],++ "@clack/prompts": ["@clack/[email protected]", "", { "dependencies": { "@clack/core": "0.5.0", "picocolors": "^1.0.0", "sisteransi": "^1.0.5" } }, "sha512-pMN5FcrEw9hUkZA4f+zLlzivQSeQf5dRGJjSUbvVYDLvpKCdQx5OaknvKzgbtXOizhP+SJJJjqEbOe55uKKfAw=="],++ "@types/bun": ["@types/[email protected]", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="],++ "@types/node": ["@types/[email protected]", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-dsqMQQoeTLqu9wynDD00q573mNzso3IdQOAfHRJqLCcmCFPoGo9A1bDpUcv/9tnKpErQWv9uKeGfl37EIS02Yg=="],++ "bun-types": ["[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="],++ "picocolors": ["[email protected]", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],++ "sisteransi": ["[email protected]", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="],++ "undici-types": ["[email protected]", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="],++ "yaml": ["[email protected]", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="],+ }+}diff --git a/self-hosting/cli/package.json b/self-hosting/cli/package.jsonnew file mode 100644index 000000000..0c873e417--- /dev/null+++ b/self-hosting/cli/package.json@@ -0,0 +1,19 @@+{+ "name": "@openpanel/cli",+ "version": "0.1.0",+ "private": true,+ "type": "module",+ "description": "Set up, upgrade and diagnose a self-hosted OpenPanel install",+ "scripts": {+ "dev": "bun src/main.ts",+ "test": "bun test",+ "build": "bun build --compile src/main.ts --outfile dist/openpanel"+ },+ "dependencies": {+ "@clack/prompts": "^0.11.0",+ "yaml": "^2.8.0"+ },+ "devDependencies": {+ "@types/bun": "latest"+ }+}diff --git a/self-hosting/cli/src/doctor/checks.ts b/self-hosting/cli/src/doctor/checks.tsnew file mode 100644index 000000000..9ebd1d121--- /dev/null+++ b/self-hosting/cli/src/doctor/checks.ts@@ -0,0 +1,458 @@+import { randomBytes } from 'node:crypto';+import { isMap, isSeq, parseDocument } from 'yaml';+import {+ DEFAULT_EVENTS_TOPIC_PARTITIONS,+ renderRedpandaBootstrap,+ templates,+} from '../templates';+import type { Check, Install } from './types';++const ENCRYPTION_KEY_PATTERN = /^[0-9a-f]{64}$/i;+const BOOTSTRAP_PATH = 'redpanda/bootstrap.yaml';+const WORKER_READY_PATH = '/healthz/ready';+const REDIS_DRAIN_NOTE =+ 'Events still queued in Redis by the old worker are not moved to Redpanda. Stop sending traffic and let the old worker drain (watch `/bullboard`) before you switch.';++// Env vars v3 no longer reads; leaving them is harmless but misleading.+const REMOVED_ENV_VARS = [+ 'WORKER_PORT',+ 'EVENT_JOB_CONCURRENCY',+ 'EVENTS_GROUP_QUEUES_SHARDS',+ 'EVENT_BLOCKING_TIMEOUT_SEC',+ 'ORDERING_DELAY_MS',+ 'AUTO_BATCH_SIZE',+ 'AUTO_BATCH_MAX_WAIT_MS',+ 'SHUTDOWN_GRACE_PERIOD_MS',+ 'FUNNEL_NON_STRICT_ORDERING',+ 'CHART_VALUES_LOOKBACK_DAYS',+ 'NEXT_PUBLIC_API_URL',+] as const;++const templateCompose = parseDocument(templates.compose);++const randomHex = (bytes: number) => randomBytes(bytes).toString('hex');++const hasService = (install: Install, name: string) =>+ install.compose.hasIn(['services', name]);++const serviceImage = (install: Install, name: string) => {+ const image = install.compose.getIn(['services', name, 'image']);+ return typeof image === 'string' ? image : undefined;+};++const envEntries = (install: Install, service: string): string[] => {+ const environment = install.compose.getIn([+ 'services',+ service,+ 'environment',+ ]);+ if (isSeq(environment)) {+ return environment.items.map((item) =>+ String((item as { value?: unknown }).value ?? item)+ );+ }+ if (isMap(environment)) {+ return environment.items.map((pair) => {+ const key = String((pair.key as { value?: unknown }).value ?? pair.key);+ const value = (pair.value as { value?: unknown } | null)?.value;+ return `${key}=${value ?? ''}`;+ });+ }+ return [];+};++const hasWorkerRole = (install: Install) =>+ envEntries(install, 'op-worker').includes('ROLE=worker');++const hasHealthyDependency = (+ install: Install,+ service: string,+ dependency: string+) => install.compose.hasIn(['services', service, 'depends_on', dependency]);++const isLegacyWorkerImage = (install: Install) =>+ (serviceImage(install, 'op-worker') ?? '').includes('openpanel-worker');++export const redpandaService: Check = (install) => {+ if (hasService(install, 'op-rp')) {+ return null;+ }+ return {+ id: 'compose/redpanda-service',+ severity: 'error',+ title: 'Redpanda (op-rp) is missing from docker-compose.yml',+ detail:+ 'Events are always produced to Kafka now; without op-rp the api refuses to start.',+ fix: (target) => {+ const service = templateCompose.getIn(['services', 'op-rp'], true);+ const volume = templateCompose.getIn(['volumes', 'op-rp-data'], true);+ target.compose.setIn(['services', 'op-rp'], service);+ target.compose.setIn(['volumes', 'op-rp-data'], volume);+ for (const name of ['op-api', 'op-worker']) {+ if (+ hasService(target, name) &&+ !hasHealthyDependency(target, name, 'op-rp')+ ) {+ target.compose.setIn(+ ['services', name, 'depends_on', 'op-rp'],+ target.compose.createNode({ condition: 'service_healthy' })+ );+ }+ }+ },+ };+};++export const redpandaBootstrapFile: Check = (install) => {+ if (install.hasFile(BOOTSTRAP_PATH)) {+ return null;+ }+ return {+ id: 'files/redpanda-bootstrap',+ severity: 'error',+ title: `${BOOTSTRAP_PATH} is missing`,+ detail:+ 'op-rp bind-mounts this file; docker compose fails to start without it.',+ fix: (target) => {+ const configured = Number(+ target.env.get('KAFKA_EVENTS_TOPIC_PARTITIONS')+ );+ const partitions =+ Number.isInteger(configured) && configured > 0+ ? configured+ : DEFAULT_EVENTS_TOPIC_PARTITIONS;+ target.files.set(BOOTSTRAP_PATH, renderRedpandaBootstrap(partitions));+ },+ };+};++export const workerImage: Check = (install) => {+ if (!isLegacyWorkerImage(install)) {+ return null;+ }+ return {+ id: 'compose/worker-image',+ severity: 'error',+ title: 'op-worker still uses the separate openpanel-worker image',+ detail:+ 'That image is no longer built. The worker is the api image started with ROLE=worker.',+ fix: (target) => {+ const apiImage = serviceImage(target, 'op-api');+ const fallback = templateCompose.getIn([+ 'services',+ 'op-worker',+ 'image',+ ]);+ target.compose.setIn(+ ['services', 'op-worker', 'image'],+ apiImage ?? fallback+ );+ },+ };+};++export const workerRole: Check = (install) => {+ if (!hasService(install, 'op-worker') || hasWorkerRole(install)) {+ return null;+ }+ return {+ id: 'compose/worker-role',+ severity: 'error',+ title: 'op-worker has no ROLE=worker',+ detail: 'Without it the api image starts as an api server, not a worker.',+ fix: (target) => {+ const path = ['services', 'op-worker', 'environment'];+ const environment = target.compose.getIn(path);+ if (isMap(environment)) {+ target.compose.setIn([...path, 'ROLE'], 'worker');+ return;+ }+ if (isSeq(environment)) {+ environment.add('ROLE=worker');+ return;+ }+ target.compose.setIn(path, target.compose.createNode(['ROLE=worker']));+ },+ };+};++export const workerHealthcheck: Check = (install) => {+ if (!hasService(install, 'op-worker')) {+ return null;+ }+ const test = install.compose.getIn([+ 'services',+ 'op-worker',+ 'healthcheck',+ 'test',+ ]);+ const hasReadyProbe = JSON.stringify(test ?? '').includes(WORKER_READY_PATH);+ if (hasReadyProbe) {+ return null;+ }+ return {+ id: 'compose/worker-healthcheck',+ severity: 'warn',+ title: `op-worker healthcheck does not probe ${WORKER_READY_PATH}`,+ detail:+ "The worker process serves its own readiness probe; /healthcheck is the api's.",+ fix: (target) => {+ const probe = templateCompose.getIn([+ 'services',+ 'op-worker',+ 'healthcheck',+ 'test',+ ]);+ target.compose.setIn(+ ['services', 'op-worker', 'healthcheck', 'test'],+ probe+ );+ },+ };+};++export const apiMigrations: Check = (install) => {+ const command = install.compose.getIn(['services', 'op-api', 'command']);+ if (+ typeof command !== 'string' ||+ command.includes('scripts/migrate-code.ts')+ ) {+ return null;+ }+ return {+ id: 'compose/api-migrations',+ severity: 'error',+ title: 'op-api start command runs the old migration steps',+ detail:+ 'v3 migrates with prisma + scripts/migrate-code.ts under bun; pnpm is gone from the image.',+ fix: (target) => {+ const next = templateCompose.getIn(['services', 'op-api', 'command']);+ target.compose.setIn(['services', 'op-api', 'command'], next);+ },+ };+};++export const kafkaBrokers: Check = (install) => {+ if (install.env.get('KAFKA_BROKERS')) {+ return null;+ }+ return {+ id: 'env/kafka-brokers',+ severity: 'error',+ title: 'KAFKA_BROKERS is not set',+ detail: 'Required to boot. The bundled Redpanda listens on op-rp:9092.',+ fix: (target) => target.env.set('KAFKA_BROKERS', 'op-rp:9092'),+ };+};++export const kafkaPartitions: Check = (install) => {+ if (install.env.get('KAFKA_EVENTS_TOPIC_PARTITIONS')) {+ return null;+ }+ return {+ id: 'env/kafka-partitions',+ severity: 'warn',+ title: 'KAFKA_EVENTS_TOPIC_PARTITIONS is not set',+ detail:+ 'Read once when the events topic is first created; changing it later reshuffles key ordering.',+ fix: (target) =>+ target.env.set(+ 'KAFKA_EVENTS_TOPIC_PARTITIONS',+ String(DEFAULT_EVENTS_TOPIC_PARTITIONS)+ ),+ };+};++export const encryptionKey: Check = (install) => {+ const value = install.env.get('ENCRYPTION_KEY');+ if (value && ENCRYPTION_KEY_PATTERN.test(value)) {+ return null;+ }+ if (value) {+ return {+ id: 'env/encryption-key',+ severity: 'error',+ title: 'ENCRYPTION_KEY is not 64 hex characters',+ detail:+ 'It guards stored credentials (2FA, Search Console, exports), so doctor will not replace an existing value.',+ manual:+ 'Generate a replacement with `openssl rand -hex 32` only if you have nothing encrypted yet.',+ };+ }+ return {+ id: 'env/encryption-key',+ severity: 'error',+ title: 'ENCRYPTION_KEY is not set',+ detail:+ 'Needed for 2FA, Google Search Console and export credentials; must match on api and worker.',+ fix: (target) => target.env.set('ENCRYPTION_KEY', randomHex(32)),+ };+};++export const cookieSecret: Check = (install) => {+ if (install.env.get('COOKIE_SECRET')) {+ return null;+ }+ return {+ id: 'env/cookie-secret',+ severity: 'error',+ title: 'COOKIE_SECRET is not set',+ detail: 'Signs unsubscribe links and unlocks password-protected shares.',+ fix: (target) => target.env.set('COOKIE_SECRET', randomHex(16)),+ };+};++export const dashboardUrl: Check = (install) => {+ if (install.env.get('DASHBOARD_URL')) {+ return null;+ }+ return {+ id: 'env/dashboard-url',+ severity: 'error',+ title: 'DASHBOARD_URL is not set',+ detail:+ 'It seeds the CORS allowlist for the dashboard; without it every dashboard request is rejected.',+ manual:+ 'Set DASHBOARD_URL to the exact origin you open in the browser, e.g. https://analytics.example.com.',+ };+};++export const adminCredentials: Check = (install) => {+ const hasUsername = Boolean(install.env.get('ADMIN_USERNAME'));+ const hasPassword = Boolean(install.env.get('ADMIN_PASSWORD'));+ if (hasUsername === hasPassword) {+ return null;+ }+ return {+ id: 'env/admin-credentials',+ severity: 'error',+ title: 'ADMIN_USERNAME and ADMIN_PASSWORD must be set together',+ detail: 'Setting only one fails boot.',+ manual: 'Set both, or remove the one that is set.',+ };+};++export const selfHosted: Check = (install) => {+ const value = install.env.get('SELF_HOSTED');+ if (value === 'true' || value === '1') {+ return null;+ }+ return {+ id: 'env/self-hosted',+ severity: 'warn',+ title: `SELF_HOSTED is ${value === undefined ? 'not set' : `"${value}"`}`,+ detail:+ 'Only `true` or `1` turns self-hosted mode on in the api and worker.',+ fix: (target) => target.env.set('SELF_HOSTED', 'true'),+ };+};++export const enabledQueues: Check = (install) => {+ const value = install.env.get('ENABLED_QUEUES');+ if (!value) {+ return null;+ }+ const tokens = value.split(',').map((token) => token.trim());+ const isStale = tokens.includes('events_kafka') || tokens.includes('misc');+ if (!isStale) {+ return null;+ }+ return {+ id: 'env/enabled-queues',+ severity: 'error',+ title: 'ENABLED_QUEUES lists queues that no longer exist',+ detail:+ '`events_kafka` was renamed `events` and `misc` was removed; unknown names fail boot.',+ fix: (target) => {+ const next = tokens+ .filter((token) => token !== 'misc')+ .map((token) => (token === 'events_kafka' ? 'events' : token));+ target.env.set('ENABLED_QUEUES', [...new Set(next)].join(','));+ },+ };+};++export const removedEnvVars: Check = (install) => {+ const present = REMOVED_ENV_VARS.filter((key) => install.env.has(key));+ if (present.length === 0) {+ return null;+ }+ return {+ id: 'env/removed-vars',+ severity: 'info',+ title: `Unused variables in .env: ${present.join(', ')}`,+ detail: 'v3 no longer reads these.',+ fix: (target) => {+ for (const key of present) {+ target.env.remove(key);+ }+ },+ };+};++const WORKER_BLOCK_PROXY = 'reverse_proxy op-worker:3000';+const BULLBOARD_REDIRECT = 'redir / /bullboard/';++export const caddyBullboardRedirect: Check = (install) => {+ const caddyfile = install.caddyfile;+ if (caddyfile === null) {+ return null;+ }+ const hasWorkerBlock = caddyfile.includes(WORKER_BLOCK_PROXY);+ const hasRedirect = caddyfile.includes(BULLBOARD_REDIRECT);+ if (!hasWorkerBlock || hasRedirect) {+ return null;+ }+ return {+ id: 'caddy/bullboard-redirect',+ severity: 'warn',+ title: 'worker.<domain> has no redirect to /bullboard/',+ detail:+ "Bull-board moved from `/` to `/bullboard` on the worker's single port.",+ fix: (target) => {+ target.caddyfile = (target.caddyfile ?? '').replace(+ /^(\s*)reverse_proxy op-worker:3000/m,+ `$1${BULLBOARD_REDIRECT}\n\n$1${WORKER_BLOCK_PROXY}`+ );+ },+ };+};++// Reminder, not a repair: surfaced only while the install still looks like v1.+export const drainOldQueue: Check = (install) => {+ const looksLikeV1 =+ isLegacyWorkerImage(install) || !hasService(install, 'op-rp');+ if (!looksLikeV1) {+ return null;+ }+ return {+ id: 'upgrade/drain-old-queue',+ severity: 'warn',+ title: 'Drain the old event queue before switching',+ detail: REDIS_DRAIN_NOTE,+ manual:+ 'Stop tracking traffic, wait until the events queue is empty, then run `openpanel upgrade`.',+ };+};++// Order matters for fixes: the worker image is swapped before its role is+// checked, and op-rp is added before anything asks for its bootstrap file.+export const checks: Check[] = [+ drainOldQueue,+ redpandaService,+ redpandaBootstrapFile,+ workerImage,+ workerRole,+ workerHealthcheck,+ apiMigrations,+ kafkaBrokers,+ kafkaPartitions,+ encryptionKey,+ cookieSecret,+ dashboardUrl,+ adminCredentials,+ selfHosted,+ enabledQueues,+ removedEnvVars,+ caddyBullboardRedirect,+];diff --git a/self-hosting/cli/src/doctor/index.ts b/self-hosting/cli/src/doctor/index.tsnew file mode 100644index 000000000..5185aa9b1--- /dev/null+++ b/self-hosting/cli/src/doctor/index.ts@@ -0,0 +1,22 @@+import { checks } from './checks';+import type { Finding, Install } from './types';++export type { Finding, Install, Severity } from './types';++export const runDoctor = (install: Install): Finding[] =>+ checks.flatMap((check) => check(install) ?? []);++// Fixes run one at a time against live state, in check order, so a later fix+// sees the result of an earlier one (the worker image before its role).+export const applyFixes = (install: Install): Finding[] => {+ const applied: Finding[] = [];+ for (const check of checks) {+ const finding = check(install);+ if (!finding?.fix) {+ continue;+ }+ finding.fix(install);+ applied.push(finding);+ }+ return applied;+};diff --git a/self-hosting/cli/src/doctor/types.ts b/self-hosting/cli/src/doctor/types.tsnew file mode 100644index 000000000..df71cda37--- /dev/null+++ b/self-hosting/cli/src/doctor/types.ts@@ -0,0 +1,27 @@+import type { Document } from 'yaml';+import type { EnvFile } from '../env-file';++export type Severity = 'error' | 'warn' | 'info';++// Everything doctor looks at, in memory. Fixes mutate this; the caller decides+// whether to write it back, so `--check` and tests never touch the disk.+export interface Install {+ compose: Document;+ env: EnvFile;+ caddyfile: string | null;+ // Files doctor may create, keyed by path relative to the install dir.+ files: Map<string, string>;+ hasFile: (relativePath: string) => boolean;+}++export interface Finding {+ id: string;+ severity: Severity;+ title: string;+ detail: string;+ // Present when doctor can repair it; absent means the user has to act.+ fix?: (install: Install) => void;+ manual?: string;+}++export type Check = (install: Install) => Finding | null;diff --git a/self-hosting/cli/src/env-file.ts b/self-hosting/cli/src/env-file.tsnew file mode 100644index 000000000..1088bf5ad--- /dev/null+++ b/self-hosting/cli/src/env-file.ts@@ -0,0 +1,60 @@+// A .env model that round-trips: comments, blank lines and ordering survive+// edits, so `doctor --fix` never rewrites a file the user has annotated.+const ASSIGNMENT = /^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/;++const unquote = (raw: string): string => {+ const isQuoted =+ raw.length >= 2 &&+ (raw.startsWith('"') || raw.startsWith("'")) &&+ raw.endsWith(raw[0] as string);+ return isQuoted ? raw.slice(1, -1) : raw;+};++export class EnvFile {+ private readonly lines: string[];++ constructor(text = '') {+ this.lines = text.length === 0 ? [] : text.replace(/\n$/, '').split('\n');+ }++ private indexOf(key: string): number {+ return this.lines.findIndex((line) => ASSIGNMENT.exec(line)?.[1] === key);+ }++ has(key: string): boolean {+ return this.indexOf(key) !== -1;+ }++ // A blank `KEY=` counts as unset, matching how the api reads its env.+ get(key: string): string | undefined {+ const index = this.indexOf(key);+ if (index === -1) {+ return undefined;+ }+ const value = unquote(+ ASSIGNMENT.exec(this.lines[index] as string)?.[2] ?? ''+ );+ return value === '' ? undefined : value;+ }++ set(key: string, value: string): void {+ const line = `${key}="${value}"`;+ const index = this.indexOf(key);+ if (index === -1) {+ this.lines.push(line);+ return;+ }+ this.lines[index] = line;+ }++ remove(key: string): void {+ const index = this.indexOf(key);+ if (index !== -1) {+ this.lines.splice(index, 1);+ }+ }++ toString(): string {+ return `${this.lines.join('\n')}\n`;+ }+}diff --git a/self-hosting/cli/src/install.ts b/self-hosting/cli/src/install.tsnew file mode 100644index 000000000..9fe5caa9a--- /dev/null+++ b/self-hosting/cli/src/install.ts@@ -0,0 +1,68 @@+import { existsSync, readFileSync } from 'node:fs';+import { copyFile, mkdir, writeFile } from 'node:fs/promises';+import { dirname, join } from 'node:path';+import { parseDocument } from 'yaml';+import type { Install } from './doctor/types';+import { EnvFile } from './env-file';++const COMPOSE_FILE = 'docker-compose.yml';+const ENV_FILE = '.env';+const CADDYFILE = 'caddy/Caddyfile';+// Keep the user's formatting: no padding inside [..], no re-folding of long lines.+const YAML_OUTPUT = { flowCollectionPadding: false, lineWidth: 0 } as const;++const readIfExists = (path: string): string | null =>+ existsSync(path) ? readFileSync(path, 'utf8') : null;++export const isInstallDir = (dir: string): boolean =>+ existsSync(join(dir, COMPOSE_FILE));++export const loadInstall = (dir: string): Install => {+ const composeText = readIfExists(join(dir, COMPOSE_FILE));+ if (composeText === null) {+ throw new Error(+ `No ${COMPOSE_FILE} in ${dir}. Run \`openpanel init\` first.`+ );+ }+ const files = new Map<string, string>();+ return {+ compose: parseDocument(composeText),+ env: new EnvFile(readIfExists(join(dir, ENV_FILE)) ?? ''),+ caddyfile: readIfExists(join(dir, CADDYFILE)),+ files,+ hasFile: (relativePath) =>+ files.has(relativePath) || existsSync(join(dir, relativePath)),+ };+};++// Originals are kept next to the file as `<name>.bak` so a bad fix is one `mv`+// away from undone. Only files that actually change are touched.+export const writeInstall = async (+ dir: string,+ install: Install+): Promise<string[]> => {+ const outputs = new Map<string, string>([+ [COMPOSE_FILE, install.compose.toString(YAML_OUTPUT)],+ [ENV_FILE, install.env.toString()],+ ...(install.caddyfile === null+ ? []+ : ([[CADDYFILE, install.caddyfile]] as const)),+ ...install.files,+ ]);++ const written: string[] = [];+ for (const [relativePath, content] of outputs) {+ const target = join(dir, relativePath);+ const previous = readIfExists(target);+ if (previous === content) {+ continue;+ }+ await mkdir(dirname(target), { recursive: true });+ if (previous !== null) {+ await copyFile(target, `${target}.bak`);+ }+ await writeFile(target, content);+ written.push(relativePath);+ }+ return written;+};diff --git a/self-hosting/cli/src/templates.ts b/self-hosting/cli/src/templates.tsnew file mode 100644index 000000000..224d5d633--- /dev/null+++ b/self-hosting/cli/src/templates.ts@@ -0,0 +1,25 @@+import envTemplate from '../../.env.template' with { type: 'text' };+import caddyTemplate from '../../caddy/Caddyfile.template' with {+ type: 'text',+};+import composeTemplate from '../../docker-compose.template.yml' with {+ type: 'text',+};+import redpandaTemplate from '../../redpanda/bootstrap.template.yaml' with {+ type: 'text',+};++export const templates = {+ caddy: caddyTemplate,+ compose: composeTemplate,+ env: envTemplate,+ redpanda: redpandaTemplate,+} as const;++export const DEFAULT_EVENTS_TOPIC_PARTITIONS = 24;++export const renderRedpandaBootstrap = (partitions: number): string =>+ templates.redpanda.replace(+ '$KAFKA_EVENTS_TOPIC_PARTITIONS',+ String(partitions)+ );diff --git a/self-hosting/cli/src/text-imports.d.ts b/self-hosting/cli/src/text-imports.d.tsnew file mode 100644index 000000000..ca0e0565d--- /dev/null+++ b/self-hosting/cli/src/text-imports.d.ts@@ -0,0 +1,4 @@+declare module '*.template' {+ const text: string;+ export default text;+}diff --git a/self-hosting/cli/test/doctor.test.ts b/self-hosting/cli/test/doctor.test.tsnew file mode 100644index 000000000..6edac6edc--- /dev/null+++ b/self-hosting/cli/test/doctor.test.ts@@ -0,0 +1,113 @@+import { afterEach, describe, expect, test } from 'bun:test';+import { cpSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { parseDocument } from 'yaml';+import { applyFixes, runDoctor } from '../src/doctor';+import { loadInstall, writeInstall } from '../src/install';++const V1_FIXTURE = join(import.meta.dir, 'fixtures/v1');+const tempDirs: string[] = [];++const copyFixture = () => {+ const dir = mkdtempSync(join(tmpdir(), 'op-doctor-'));+ tempDirs.push(dir);+ cpSync(V1_FIXTURE, dir, { recursive: true });+ return dir;+};++afterEach(() => {+ for (const dir of tempDirs.splice(0)) {+ rmSync(dir, { recursive: true, force: true });+ }+});++describe('doctor on a v1 install', () => {+ test('reports every breaking change', () => {+ const ids = runDoctor(loadInstall(V1_FIXTURE)).map((finding) => finding.id);+ expect(ids).toEqual([+ 'upgrade/drain-old-queue',+ 'compose/redpanda-service',+ 'files/redpanda-bootstrap',+ 'compose/worker-image',+ 'compose/worker-role',+ 'compose/worker-healthcheck',+ 'compose/api-migrations',+ 'env/kafka-brokers',+ 'env/kafka-partitions',+ 'env/enabled-queues',+ 'env/removed-vars',+ 'caddy/bullboard-redirect',+ ]);+ });++ test('fixes leave nothing fixable behind', () => {+ const install = loadInstall(V1_FIXTURE);+ applyFixes(install);+ const remaining = runDoctor(install).filter((finding) => finding.fix);+ expect(remaining).toEqual([]);+ });++ test('the repaired compose file matches the v3 shape', () => {+ const install = loadInstall(V1_FIXTURE);+ applyFixes(install);+ const compose = parseDocument(String(install.compose)).toJS();++ expect(compose.services['op-rp'].image).toContain('redpanda');+ expect(compose.volumes).toHaveProperty('op-rp-data');+ expect(compose.services['op-worker'].image).toBe(+ compose.services['op-api'].image+ );+ expect(compose.services['op-worker'].environment).toContain('ROLE=worker');+ expect(compose.services['op-api'].command).toContain(+ 'scripts/migrate-code.ts'+ );+ expect(compose.services['op-api'].depends_on).toHaveProperty('op-rp');+ expect(compose.services['op-worker'].depends_on).toHaveProperty('op-rp');+ expect(JSON.stringify(compose.services['op-worker'].healthcheck)).toContain(+ '/healthz/ready'+ );+ });++ test('env keeps user values and only touches what is stale', () => {+ const install = loadInstall(V1_FIXTURE);+ applyFixes(install);+ const { env } = install;++ expect(env.get('ENCRYPTION_KEY')).toBe('0123456789abcdef'.repeat(4));+ expect(env.get('COOKIE_SECRET')).toBe('0123456789abcdef'.repeat(2));+ expect(env.get('KAFKA_BROKERS')).toBe('op-rp:9092');+ expect(env.get('ENABLED_QUEUES')).toBe('events,sessions,cron');+ expect(env.has('WORKER_PORT')).toBe(false);+ expect(env.has('EVENT_JOB_CONCURRENCY')).toBe(false);+ });++ test('the worker block in the Caddyfile gains the bullboard redirect', () => {+ const install = loadInstall(V1_FIXTURE);+ applyFixes(install);+ expect(install.caddyfile).toContain('redir / /bullboard/');+ expect(install.caddyfile).toContain('reverse_proxy op-worker:3000');+ });++ test('writing backs up originals and is idempotent', async () => {+ const dir = copyFixture();+ const before = readFileSync(join(dir, 'docker-compose.yml'), 'utf8');++ const install = loadInstall(dir);+ applyFixes(install);+ const written = await writeInstall(dir, install);++ expect(written).toContain('docker-compose.yml');+ expect(written).toContain('redpanda/bootstrap.yaml');+ expect(readFileSync(join(dir, 'docker-compose.yml.bak'), 'utf8')).toBe(+ before+ );+ expect(+ readFileSync(join(dir, 'redpanda/bootstrap.yaml'), 'utf8')+ ).toContain('default_topic_partitions: 24');++ const second = loadInstall(dir);+ expect(runDoctor(second).filter((finding) => finding.fix)).toEqual([]);+ expect(await writeInstall(dir, second)).toEqual([]);+ });+});diff --git a/self-hosting/cli/test/fixtures/v1/.env b/self-hosting/cli/test/fixtures/v1/.envnew file mode 100644index 000000000..cfb897122--- /dev/null+++ b/self-hosting/cli/test/fixtures/v1/.env@@ -0,0 +1,18 @@+NODE_ENV="production"+SELF_HOSTED="true"+BATCH_SIZE="5000"+BATCH_INTERVAL="10000"+ALLOW_REGISTRATION="false"+ALLOW_INVITATION="true"+REDIS_URL="redis://op-kv:6379"+CLICKHOUSE_URL="http://op-ch:8123/openpanel"+DATABASE_URL="postgresql://postgres:postgres@op-db:5432/postgres?schema=public"+DATABASE_URL_DIRECT="postgresql://postgres:postgres@op-db:5432/postgres?schema=public"+DASHBOARD_URL="https://analytics.example.com"+API_URL="https://analytics.example.com/api"+COOKIE_SECRET="0123456789abcdef0123456789abcdef"+EMAIL_SENDER="[email protected]"+ENCRYPTION_KEY="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"+ENABLED_QUEUES="events_kafka,sessions,cron"+EVENT_JOB_CONCURRENCY="4"+WORKER_PORT="9999"diff --git a/self-hosting/cli/test/fixtures/v1/caddy/Caddyfile b/self-hosting/cli/test/fixtures/v1/caddy/Caddyfilenew file mode 100644index 000000000..d2a4cb7b7--- /dev/null+++ b/self-hosting/cli/test/fixtures/v1/caddy/Caddyfile@@ -0,0 +1,19 @@+analytics.example.com {+ encode gzip++ handle_path /api* {+ reverse_proxy op-api:3000+ }++ reverse_proxy /* op-dashboard:3000+}++worker.analytics.example.com {+ encode gzip++ basic_auth {+ admin $2b$10$abcdefghijklmnopqrstuuABCDEFGHIJKLMNOPQRSTUVWXYZ01234+ }++ reverse_proxy op-worker:3000+}diff --git a/self-hosting/cli/test/fixtures/v1/docker-compose.yml b/self-hosting/cli/test/fixtures/v1/docker-compose.ymlnew file mode 100644index 000000000..42a6141a5--- /dev/null+++ b/self-hosting/cli/test/fixtures/v1/docker-compose.yml@@ -0,0 +1,174 @@+version: "3"++services:+ op-proxy:+ image: caddy:2-alpine+ restart: always+ ports:+ - "80:80"+ - "443:443"+ volumes:+ - op-proxy-data:/data+ - op-proxy-config:/config+ - ./caddy/Caddyfile:/etc/caddy/Caddyfile+ depends_on:+ op-dashboard:+ condition: service_healthy+ op-api:+ condition: service_healthy+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"++ op-db:+ image: postgres:14-alpine+ restart: always+ volumes:+ - op-db-data:/var/lib/postgresql/data+ healthcheck:+ test: ["CMD-SHELL", "pg_isready -U postgres"]+ interval: 10s+ timeout: 5s+ retries: 5+ environment:+ - POSTGRES_USER=postgres+ - POSTGRES_PASSWORD=postgres+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"+ # Uncomment to expose ports+ # ports:+ # - 5432:5432++ op-kv:+ image: redis:7.2.5-alpine+ restart: always+ volumes:+ - op-kv-data:/data+ command: ["redis-server", "--maxmemory-policy", "noeviction"]+ healthcheck:+ test: ["CMD-SHELL", "redis-cli ping"]+ interval: 10s+ timeout: 5s+ retries: 5+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"+ # Uncomment to expose ports+ # ports:+ # - 6379:6379++ op-ch:+ image: clickhouse/clickhouse-server:25.10.2.65+ restart: always+ volumes:+ - op-ch-data:/var/lib/clickhouse+ - ./clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/op-config.xml:ro+ - ./clickhouse/clickhouse-user-config.xml:/etc/clickhouse-server/users.d/op-user-config.xml:ro+ - ./clickhouse/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro+ environment:+ - CLICKHOUSE_SKIP_USER_SETUP=1+ healthcheck:+ test: ["CMD-SHELL", 'clickhouse-client --query "SELECT 1"']+ interval: 10s+ timeout: 5s+ retries: 5+ ulimits:+ nofile:+ soft: 262144+ hard: 262144+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"++ op-api:+ image: lindesvard/openpanel-api:2+ restart: always+ command: >+ sh -c "+ echo 'Running migrations...'+ CI=true pnpm -r run migrate:deploy++ pnpm start+ "+ healthcheck:+ test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]+ interval: 10s+ timeout: 5s+ retries: 5+ depends_on:+ op-db:+ condition: service_healthy+ op-ch:+ condition: service_healthy+ op-kv:+ condition: service_healthy+ env_file:+ - .env+ logging:+ driver: "json-file"+ options:+ max-size: "50m"+ max-file: "3"++ op-dashboard:+ image: lindesvard/openpanel-dashboard:2+ restart: always+ depends_on:+ op-api:+ condition: service_healthy+ env_file:+ - .env+ healthcheck:+ test:+ ["CMD-SHELL", "curl -f http://localhost:3000/api/healthcheck || exit 1"]+ interval: 10s+ timeout: 5s+ retries: 5+ logging:+ driver: "json-file"+ options:+ max-size: "20m"+ max-file: "3"++ op-worker:+ image: lindesvard/openpanel-worker:2+ restart: always+ depends_on:+ op-api:+ condition: service_healthy+ env_file:+ - .env+ healthcheck:+ test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]+ interval: 10s+ timeout: 5s+ retries: 5+ deploy:+ mode: replicated+ replicas: 1+ logging:+ driver: "json-file"+ options:+ max-size: "30m"+ max-file: "3"++volumes:+ op-db-data:+ driver: local+ op-kv-data:+ driver: local+ op-ch-data:+ driver: local+ op-proxy-data:+ driver: local+ op-proxy-config:+ driver: localdiff --git a/self-hosting/cli/tsconfig.json b/self-hosting/cli/tsconfig.jsonnew file mode 100644index 000000000..adbe77958--- /dev/null+++ b/self-hosting/cli/tsconfig.json@@ -0,0 +1,12 @@+{+ "compilerOptions": {+ "target": "ESNext",+ "module": "ESNext",+ "moduleResolution": "bundler",+ "strict": true,+ "noEmit": true,+ "skipLibCheck": true,+ "types": ["bun"]+ },+ "include": ["src", "test"]+}