OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
self-hosting: pin image tags to the CLI release and announce new versions
Carl-Gerhard Lindesvärd747c300b993b
diff --git a/self-hosting/cli/src/doctor/checks.ts b/self-hosting/cli/src/doctor/checks.tsindex 9ebd1d121..255bc5783 100644--- a/self-hosting/cli/src/doctor/checks.ts+++ b/self-hosting/cli/src/doctor/checks.ts@@ -126,6 +126,63 @@ export const redpandaBootstrapFile: Check = (install) => {};};+const OPENPANEL_IMAGE = /^(lindesvard\/openpanel-(?:api|dashboard)):(.+)$/;+// A floating tag follows a release line; anything else is a version the user chose.+const FLOATING_TAG = /^(\d+|latest)$/;+const IMAGE_SERVICES = ['op-api', 'op-dashboard', 'op-worker'] as const;++// The tag this CLI release is built for, read from the template so there is one+// place to change it.+export const STACK_IMAGE_TAG = String(+ templateCompose.getIn(['services', 'op-api', 'image'])+)+ .split(':')+ .pop() as string;++const staleImages = (install: Install) =>+ IMAGE_SERVICES.flatMap((service) => {+ const image = serviceImage(install, service);+ const match = image ? OPENPANEL_IMAGE.exec(image) : null;+ const [, repository, tag] = match ?? [];+ return repository && tag && tag !== STACK_IMAGE_TAG+ ? [{ service, repository, tag }]+ : [];+ });++export const imageTag: Check = (install) => {+ const stale = staleImages(install);+ if (stale.length === 0) {+ return null;+ }+ const floating = stale.filter(({ tag }) => FLOATING_TAG.test(tag));+ const pinned = stale.filter(({ tag }) => !FLOATING_TAG.test(tag));+ const pinnedNote =+ pinned.length > 0+ ? ` Pinned versions (${pinned.map(({ service, tag }) => `${service}:${tag}`).join(', ')}) are left for you to change.`+ : '';+ return {+ id: 'compose/image-tag',+ severity: floating.length > 0 ? 'error' : 'warn',+ title: `OpenPanel images are not on :${STACK_IMAGE_TAG}`,+ detail: `This CLI release targets :${STACK_IMAGE_TAG}; the compose changes here only work with those images.${pinnedNote}`,+ fix:+ floating.length > 0+ ? (target) => {+ for (const { service, repository } of floating) {+ target.compose.setIn(+ ['services', service, 'image'],+ `${repository}:${STACK_IMAGE_TAG}`+ );+ }+ }+ : undefined,+ manual:+ floating.length === 0+ ? `Change the pinned tags to ${STACK_IMAGE_TAG} (or a ${STACK_IMAGE_TAG}.x.y version) in docker-compose.yml.`+ : undefined,+ };+};+export const workerImage: Check = (install) => {if (!isLegacyWorkerImage(install)) {return null;@@ -441,6 +498,7 @@ export const checks: Check[] = [drainOldQueue,redpandaService,redpandaBootstrapFile,+ imageTag,workerImage,workerRole,workerHealthcheck,diff --git a/self-hosting/cli/src/main.ts b/self-hosting/cli/src/main.tsindex c2beb054d..e866498af 100644--- a/self-hosting/cli/src/main.ts+++ b/self-hosting/cli/src/main.ts@@ -9,6 +9,7 @@ import { reset } from './commands/reset';import { upgrade } from './commands/upgrade';import { isInstallDir } from './install';import { log, red } from './ui';+import { CHECK_TIMEOUT_MS, startUpdateCheck } from './update-notice';import { VERSION } from './version';const DEFAULT_DIR = join(homedir(), 'openpanel');@@ -45,8 +46,8 @@ const resolveDir = (flag: string | undefined): string => {return isInstallDir(process.cwd()) ? process.cwd() : DEFAULT_DIR;};-const main = async (): Promise<number> => {- const { values, positionals } = parseArgs({+const parseFlags = () =>+ parseArgs({args: process.argv.slice(2),allowPositionals: true,options: {@@ -79,6 +80,11 @@ const main = async (): Promise<number> => {version: { type: 'boolean', short: 'v', default: false },},});++type Flags = ReturnType<typeof parseFlags>['values'];++const main = async (): Promise<number> => {+ const { values, positionals } = parseFlags();const [command, ...rest] = positionals;if (values.version) {@@ -91,6 +97,21 @@ const main = async (): Promise<number> => {}const dir = resolveDir(values.dir);+ // `upgrade` does its own (authoritative) check.+ const updateCheck =+ command === 'upgrade' ? Promise.resolve() : startUpdateCheck(VERSION);+ const code = await route(command, rest, values, dir);+ // A slow or offline network must not hold up the command's own exit.+ await Promise.race([updateCheck, Bun.sleep(CHECK_TIMEOUT_MS)]);+ return code;+};++const route = async (+ command: string,+ rest: string[],+ values: Flags,+ dir: string+): Promise<number> => {if (command === 'init') {return init({dir,diff --git a/self-hosting/cli/src/update-notice.ts b/self-hosting/cli/src/update-notice.tsnew file mode 100644index 000000000..385ca778b--- /dev/null+++ b/self-hosting/cli/src/update-notice.ts@@ -0,0 +1,90 @@+import { existsSync, readFileSync } from 'node:fs';+import { mkdir, writeFile } from 'node:fs/promises';+import { homedir } from 'node:os';+import { dirname, join } from 'node:path';+import { type Fetcher, fetchLatestRelease, isNewer } from './self-update';+import { yellow } from './ui';++const CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;+export const CHECK_TIMEOUT_MS = 1500;++const DEFAULT_CACHE_PATH = join(+ process.env.XDG_CACHE_HOME ?? join(homedir(), '.cache'),+ 'openpanel',+ 'update-check.json'+);++interface Cache {+ checkedAt: number;+ latest?: string;+}++const readCache = (cachePath: string): Cache | null => {+ try {+ return existsSync(cachePath)+ ? (JSON.parse(readFileSync(cachePath, 'utf8')) as Cache)+ : null;+ } catch {+ // A corrupt cache is just a stale one.+ return null;+ }+};++const optedOut = () =>+ Boolean(process.env.OPENPANEL_NO_UPDATE_CHECK || process.env.CI);++// Instant: reads what a previous run learned, so the notice never waits on the network.+export const cachedNewerVersion = (+ currentVersion: string,+ cachePath = DEFAULT_CACHE_PATH+): string | null => {+ const latest = readCache(cachePath)?.latest;+ return latest && isNewer(latest, currentVersion) ? latest : null;+};++interface RefreshOptions {+ fetcher: Fetcher;+ cachePath?: string;+ now?: number;+}++// At most one lookup a day. The attempt is recorded even when it fails, so being+// offline costs one short timeout a day instead of one per command.+export const refreshUpdateCache = async ({+ fetcher,+ cachePath = DEFAULT_CACHE_PATH,+ now = Date.now(),+}: RefreshOptions): Promise<void> => {+ const cache = readCache(cachePath);+ if (cache && now - cache.checkedAt < CHECK_INTERVAL_MS) {+ return;+ }+ let latest = cache?.latest;+ try {+ latest = (await fetchLatestRelease(fetcher))?.version ?? latest;+ } catch {+ // keep the previous answer+ }+ await mkdir(dirname(cachePath), { recursive: true });+ await writeFile(cachePath, JSON.stringify({ checkedAt: now, latest }));+};++export const updateNotice = (current: string, latest: string): string =>+ `A new openpanel CLI is available: ${current} → ${latest}. Run \`openpanel upgrade\`.`;++// Prints any known update right away, and returns the background refresh for the+// caller to await (briefly) once its own work is done.+export const startUpdateCheck = (+ currentVersion: string,+ fetcher: Fetcher = (url) =>+ fetch(url, { signal: AbortSignal.timeout(CHECK_TIMEOUT_MS) })+): Promise<void> => {+ if (optedOut() || !process.stderr.isTTY) {+ return Promise.resolve();+ }+ const known = cachedNewerVersion(currentVersion);+ if (known) {+ process.stderr.write(`${yellow(updateNotice(currentVersion, known))}\n`);+ }+ return refreshUpdateCache({ fetcher }).catch(() => undefined);+};diff --git a/self-hosting/cli/test/image-tag.test.ts b/self-hosting/cli/test/image-tag.test.tsnew file mode 100644index 000000000..7d53e56ad--- /dev/null+++ b/self-hosting/cli/test/image-tag.test.ts@@ -0,0 +1,113 @@+import { describe, expect, test } from 'bun:test';+import { parseDocument } from 'yaml';+import { applyFixes, runDoctor } from '../src/doctor';+import { STACK_IMAGE_TAG } from '../src/doctor/checks';+import type { Install } from '../src/doctor/types';+import { EnvFile } from '../src/env-file';+import { templates } from '../src/templates';++// A current install whose image lines are replaced, so only the tag check can fire.+const installWithImages = (images: Record<string, string>): Install => {+ const compose = parseDocument(templates.compose);+ for (const [service, image] of Object.entries(images)) {+ compose.setIn(['services', service, 'image'], image);+ }+ const env = new EnvFile(+ [+ 'SELF_HOSTED="true"',+ 'DASHBOARD_URL="https://a.example.com"',+ 'KAFKA_BROKERS="op-rp:9092"',+ 'KAFKA_EVENTS_TOPIC_PARTITIONS="24"',+ `ENCRYPTION_KEY="${'ab'.repeat(32)}"`,+ 'COOKIE_SECRET="secret"',+ ].join('\n')+ );+ return {+ compose,+ env,+ caddyfile: null,+ files: new Map(),+ hasFile: () => true,+ };+};++const tagFinding = (install: Install) =>+ runDoctor(install).find((finding) => finding.id === 'compose/image-tag');++describe('compose/image-tag', () => {+ test('the expected tag is read from the template', () => {+ expect(STACK_IMAGE_TAG).toMatch(/^[\w.-]+$/);+ expect(+ String(+ parseDocument(templates.compose).getIn(['services', 'op-api', 'image'])+ )+ ).toEndWith(`:${STACK_IMAGE_TAG}`);+ });++ test('an install already on the expected tag is left alone', () => {+ expect(tagFinding(installWithImages({}))).toBeUndefined();+ });++ test('floating tags from an older line are rewritten', () => {+ const install = installWithImages({+ 'op-api': 'lindesvard/openpanel-api:1',+ 'op-dashboard': 'lindesvard/openpanel-dashboard:latest',+ });+ expect(tagFinding(install)?.fix).toBeDefined();++ applyFixes(install);+ const compose = install.compose.toJS();+ expect(compose.services['op-api'].image).toBe(+ `lindesvard/openpanel-api:${STACK_IMAGE_TAG}`+ );+ expect(compose.services['op-dashboard'].image).toBe(+ `lindesvard/openpanel-dashboard:${STACK_IMAGE_TAG}`+ );+ expect(tagFinding(install)).toBeUndefined();+ });++ test('an exact pin is reported but never rewritten', () => {+ const install = installWithImages({+ 'op-api': 'lindesvard/openpanel-api:1.2.3',+ });+ const finding = tagFinding(install);++ expect(finding?.fix).toBeUndefined();+ expect(finding?.manual).toContain(STACK_IMAGE_TAG);+ expect(finding?.detail).toContain('op-api:1.2.3');++ applyFixes(install);+ expect(install.compose.getIn(['services', 'op-api', 'image'])).toBe(+ 'lindesvard/openpanel-api:1.2.3'+ );+ });++ test('a mix fixes the floating tags and leaves the pin', () => {+ const install = installWithImages({+ 'op-api': 'lindesvard/openpanel-api:1.2.3',+ 'op-dashboard': 'lindesvard/openpanel-dashboard:1',+ });+ applyFixes(install);+ const compose = install.compose.toJS();+ expect(compose.services['op-api'].image).toBe(+ 'lindesvard/openpanel-api:1.2.3'+ );+ expect(compose.services['op-dashboard'].image).toBe(+ `lindesvard/openpanel-dashboard:${STACK_IMAGE_TAG}`+ );+ });++ test('supporter and third-party registries are never touched', () => {+ const supporter = 'docker.openpanel.dev/openpanel-dev/api:main-ab12';+ const install = installWithImages({+ 'op-api': supporter,+ 'op-db': 'postgres:14-alpine',+ });++ expect(tagFinding(install)).toBeUndefined();+ applyFixes(install);+ expect(install.compose.getIn(['services', 'op-api', 'image'])).toBe(+ supporter+ );+ });+});diff --git a/self-hosting/cli/test/update-notice.test.ts b/self-hosting/cli/test/update-notice.test.tsnew file mode 100644index 000000000..6510ff283--- /dev/null+++ b/self-hosting/cli/test/update-notice.test.ts@@ -0,0 +1,126 @@+import { afterEach, describe, expect, test } from 'bun:test';+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import type { Fetcher } from '../src/self-update';+import {+ cachedNewerVersion,+ refreshUpdateCache,+ updateNotice,+} from '../src/update-notice';++const tempDirs: string[] = [];+afterEach(() => {+ for (const dir of tempDirs.splice(0)) {+ rmSync(dir, { recursive: true, force: true });+ }+});++const cachePath = () => {+ const dir = mkdtempSync(join(tmpdir(), 'op-notice-'));+ tempDirs.push(dir);+ return join(dir, 'nested', 'update-check.json');+};++const DAY_MS = 24 * 60 * 60 * 1000;+const NOW = 1_800_000_000_000;++const feed = (version: string): { fetcher: Fetcher; calls: () => number } => {+ let calls = 0;+ return {+ calls: () => calls,+ fetcher: async () => {+ calls++;+ return Response.json([+ {+ tag_name: `cli-v${version}`,+ draft: false,+ prerelease: false,+ assets: [],+ },+ ]);+ },+ };+};++describe('update notice', () => {+ test('records the latest release, and reports it once it is newer', async () => {+ const path = cachePath();+ await refreshUpdateCache({+ fetcher: feed('0.3.0').fetcher,+ cachePath: path,+ now: NOW,+ });++ expect(cachedNewerVersion('0.1.0', path)).toBe('0.3.0');+ expect(cachedNewerVersion('0.3.0', path)).toBeNull();+ expect(cachedNewerVersion('0.4.0', path)).toBeNull();+ });++ test('looks up at most once a day', async () => {+ const path = cachePath();+ const { fetcher, calls } = feed('0.3.0');++ await refreshUpdateCache({ fetcher, cachePath: path, now: NOW });+ await refreshUpdateCache({+ fetcher,+ cachePath: path,+ now: NOW + DAY_MS - 1,+ });+ expect(calls()).toBe(1);++ await refreshUpdateCache({ fetcher, cachePath: path, now: NOW + DAY_MS });+ expect(calls()).toBe(2);+ });++ test('offline: the attempt is recorded and the last answer is kept', async () => {+ const path = cachePath();+ await refreshUpdateCache({+ fetcher: feed('0.3.0').fetcher,+ cachePath: path,+ now: NOW,+ });++ const offline: Fetcher = async () => {+ throw new Error('network down');+ };+ await refreshUpdateCache({+ fetcher: offline,+ cachePath: path,+ now: NOW + DAY_MS,+ });++ const cache = JSON.parse(readFileSync(path, 'utf8'));+ expect(cache).toEqual({ checkedAt: NOW + DAY_MS, latest: '0.3.0' });+ });++ test('a first-ever failure is also recorded, so it is not retried per command', async () => {+ const path = cachePath();+ const offline: Fetcher = async () => {+ throw new Error('network down');+ };+ await refreshUpdateCache({ fetcher: offline, cachePath: path, now: NOW });++ expect(cachedNewerVersion('0.1.0', path)).toBeNull();+ expect(JSON.parse(readFileSync(path, 'utf8')).checkedAt).toBe(NOW);+ });++ test('a missing or corrupt cache is treated as empty', async () => {+ const path = cachePath();+ expect(cachedNewerVersion('0.1.0', path)).toBeNull();++ await refreshUpdateCache({+ fetcher: feed('0.3.0').fetcher,+ cachePath: path,+ now: NOW,+ });+ writeFileSync(path, '{not json');+ expect(cachedNewerVersion('0.1.0', path)).toBeNull();+ });++ test('the notice names both versions and the command', () => {+ expect(updateNotice('0.1.0', '0.3.0')).toBe(+ 'A new openpanel CLI is available: 0.1.0 → 0.3.0. Run `openpanel upgrade`.'+ );+ });+});