Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
chore: add cli e2e tests (#2464)
Kyle Mendell31dc1736a931
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 5a29cf5b9..bfb91599e 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -284,7 +284,7 @@ jobs:run: just deps install tests- name: Run Playwright E2E tests- run: just test e2e+ run: pnpm -C tests exec playwright test --project=chromiumenv:COMPOSE_FILE: ${{ matrix.database.compose_file }}BASE_URL: http://localhost:${{ matrix.database.port }}@@ -297,3 +297,57 @@ jobs:path: tests/.reportinclude-hidden-files: trueretention-days: 15++ cli-e2e-tests:+ name: CLI E2E Tests (proxy)+ if: ${{ github.head_ref != 'l10n_crowdin' }}+ runs-on: depot-ubuntu-latest+ permissions:+ contents: read+ actions: write+ id-token: write+ steps:+ - name: Checkout code+ uses: actions/checkout@v6++ - name: Setup pnpm+ uses: pnpm/action-setup@v6+ with:+ run_install: false++ - name: Setup Node.js+ uses: actions/setup-node@v6+ with:+ node-version: 25+ cache: pnpm++ - name: Setup Go+ uses: actions/setup-go@v6+ with:+ go-version-file: cli/go.mod+ cache-dependency-path: cli/go.sum++ - name: Set up Depot CLI+ uses: depot/setup-action@v1++ - name: Setup depot buildx driver+ run: depot configure-docker++ - name: Install Playwright test dependencies+ run: pnpm -C tests install++ - name: Run CLI E2E tests against socket proxy+ run: pnpm -C tests exec playwright test --project=cli+ env:+ COMPOSE_FILE: setup/compose-proxy.yaml+ BASE_URL: http://localhost:3002+ E2E_ADMIN_STATIC_API_KEY: ${{ secrets.E2E_ADMIN_STATIC_API_KEY }}++ - name: Upload Playwright HTML report+ if: always()+ uses: actions/upload-artifact@v7+ with:+ name: playwright-report-proxy-cli+ path: tests/.report+ include-hidden-files: true+ retention-days: 15diff --git a/.gitignore b/.gitignoreindex 85af2db75..fb85863cd 100644--- a/.gitignore+++ b/.gitignore@@ -35,6 +35,7 @@ vite.config.ts.timestamp-*tests/test-results/*tests/.report/*tests/.auth/+tests/.bin/backend/.binbackend/build-errors.logfrontend/bun.lock@@ -54,4 +55,4 @@ arcane-devcli.btcacli/arcane-cli-completions\ No newline at end of file+completionsdiff --git a/cli/pkg/admin/notifications/cmd.go b/cli/pkg/admin/notifications/cmd.goindex affcd6466..78f8df9c5 100644--- a/cli/pkg/admin/notifications/cmd.go+++ b/cli/pkg/admin/notifications/cmd.go@@ -91,13 +91,13 @@ var settingsGetCmd = &cobra.Command{}defer func() { _ = resp.Body.Close() }()- var result base.ApiResponse[[]notification.Response]+ var result []notification.Responseif err := json.NewDecoder(resp.Body).Decode(&result); err != nil {return fmt.Errorf("failed to parse response: %w", err)}if jsonOutput {- resultBytes, err := json.MarshalIndent(result.Data, "", " ")+ resultBytes, err := json.MarshalIndent(result, "", " ")if err != nil {return fmt.Errorf("failed to marshal JSON: %w", err)}@@ -106,8 +106,8 @@ var settingsGetCmd = &cobra.Command{}headers := []string{"ID", "PROVIDER", "ENABLED"}- rows := make([][]string, len(result.Data))- for i, setting := range result.Data {+ rows := make([][]string, len(result))+ for i, setting := range result {rows[i] = []string{fmt.Sprintf("%d", setting.ID),string(setting.Provider),@@ -116,7 +116,7 @@ var settingsGetCmd = &cobra.Command{}output.Table(headers, rows)- fmt.Printf("\nTotal: %d notification settings\n", len(result.Data))+ fmt.Printf("\nTotal: %d notification settings\n", len(result))return nil},}diff --git a/cli/pkg/version/cmd.go b/cli/pkg/version/cmd.goindex c123e2892..b39345720 100644--- a/cli/pkg/version/cmd.go+++ b/cli/pkg/version/cmd.go@@ -6,6 +6,7 @@ import ("io""github.com/getarcaneapp/arcane/cli/internal/client"+ "github.com/getarcaneapp/arcane/cli/internal/cmdutil""github.com/getarcaneapp/arcane/cli/internal/logger""github.com/getarcaneapp/arcane/cli/internal/output"clitypes "github.com/getarcaneapp/arcane/cli/internal/types"@@ -50,6 +51,15 @@ var VersionCmd = &cobra.Command{logger.GetLogger().Debugf("Parsed version data: %+v", result)+ if cmdutil.JSONOutputEnabled(cmd) {+ resultBytes, err := json.MarshalIndent(result, "", " ")+ if err != nil {+ return fmt.Errorf("failed to marshal JSON: %w", err)+ }+ fmt.Println(string(resultBytes))+ return nil+ }+output.Header("Arcane Environment Details: \n")output.KeyValue("Version", result.DisplayVersion)diff --git a/tests/playwright.config.ts b/tests/playwright.config.tsindex 5fac5d790..95029b210 100644--- a/tests/playwright.config.ts+++ b/tests/playwright.config.ts@@ -33,7 +33,11 @@ export default defineConfig({name: 'chromium',use: { ...devices['Desktop Chrome'], storageState: '.auth/login.json' },dependencies: ['auth-setup', 'gitops-setup'],- testMatch: /spec\/.*\.spec\.ts/+ testMatch: /spec\/(?!cli\.).*\.spec\.ts/+ },+ {+ name: 'cli',+ testMatch: /spec\/cli\.spec\.ts/}]});diff --git a/tests/setup/compose-postgres.yaml b/tests/setup/compose-postgres.yamlindex b8628f977..a5c7503d1 100644--- a/tests/setup/compose-postgres.yaml+++ b/tests/setup/compose-postgres.yaml@@ -24,6 +24,7 @@ services:- DATABASE_URL=postgres://arcane:arcane_test_password@postgres:5432/arcane_test?sslmode=disable- ENCRYPTION_KEY=3JDIgolks2tJ9ymm1AdqzlYMWu0DUWyt- JWT_SECRET=your-super-secret-jwt-key-change-this-in-production+ - ADMIN_STATIC_API_KEY=${E2E_ADMIN_STATIC_API_KEY:-}volumes:- /var/run/docker.sock:/var/run/docker.sock- arcane_data:/app/datadiff --git a/tests/setup/compose-proxy.yaml b/tests/setup/compose-proxy.yamlindex 99e2620dc..dfdcfba1e 100644--- a/tests/setup/compose-proxy.yaml+++ b/tests/setup/compose-proxy.yaml@@ -40,6 +40,7 @@ services:- APP_ENV=test- ENCRYPTION_KEY=3JDIgolks2tJ9ymm1AdqzlYMWu0DUWyt- JWT_SECRET=your-super-secret-jwt-key-change-this-in-production+ - ADMIN_STATIC_API_KEY=${E2E_ADMIN_STATIC_API_KEY:-}- DOCKER_HOST=tcp://docker-socket-proxy:2375volumes:- arcane_data:/app/datadiff --git a/tests/setup/compose.yaml b/tests/setup/compose.yamlindex c282acb44..53e193c6a 100644--- a/tests/setup/compose.yaml+++ b/tests/setup/compose.yaml@@ -9,6 +9,7 @@ services:- APP_ENV=test- ENCRYPTION_KEY=3JDIgolks2tJ9ymm1AdqzlYMWu0DUWyt- JWT_SECRET=your-super-secret-jwt-key-change-this-in-production+ - ADMIN_STATIC_API_KEY=${E2E_ADMIN_STATIC_API_KEY:-}volumes:- /var/run/docker.sock:/var/run/docker.sock- arcane_data:/app/datadiff --git a/tests/spec/cli.spec.ts b/tests/spec/cli.spec.tsnew file mode 100644index 000000000..2e244bf90--- /dev/null+++ b/tests/spec/cli.spec.ts@@ -0,0 +1,408 @@+import { expect, test } from '@playwright/test';+import playwrightConfig from '../playwright.config';+import { createCLIConfig, runCLI, runCLIJSON, type CLIConfig } from '../utils/cli.util';+import { createTestApiKeys, deleteTestApiKeys } from '../utils/playwright.util';++type CreatedApiKey = {+ id: string;+ name: string;+ description?: string;+ key: string;+};++type PaginatedResponse<T> = {+ data: T[];+ pagination?: { totalItems?: number };+};++type JsonSmokeCommand = {+ name: string;+ args: string[];+ expectation: (value: unknown) => void;+};++const baseURL = String(playwrightConfig.use!.baseURL);+const staticApiKey = process.env.E2E_ADMIN_STATIC_API_KEY;+let apiKey = staticApiKey ?? '';++async function withConfig<T>(fn: (config: CLIConfig) => Promise<T>): Promise<T> {+ const config = await createCLIConfig(baseURL, apiKey);+ try {+ return await fn(config);+ } finally {+ await config.cleanup();+ }+}++async function runCommandJSON<T>(configPath: string, args: string[]): Promise<T> {+ const result = await runCLI(configPath, args);++ try {+ return JSON.parse(result.stdout) as T;+ } catch (error: unknown) {+ const message = error instanceof Error ? error.message : String(error);+ throw new Error(`failed to parse arcane-cli JSON output: ${message}\n\n${result.stdout}`);+ }+}++function expectPaginated(value: unknown): void {+ expect(value).toEqual(+ expect.objectContaining({+ data: expect.any(Array)+ })+ );+}++const readOnlyJsonSmokeCommands: JsonSmokeCommand[] = [+ {+ name: 'alerts',+ args: ['alerts', '--debug-all-good', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.objectContaining({ items: expect.any(Array) }));+ }+ },+ {+ name: 'images list',+ args: ['--output', 'json', 'images', 'list', '--limit', '5'],+ expectation: expectPaginated+ },+ {+ name: 'images counts',+ args: ['--output', 'json', 'images', 'counts'],+ expectation: (value) => {+ expect(value).toEqual(+ expect.objectContaining({+ data: expect.objectContaining({ totalImages: expect.any(Number) })+ })+ );+ }+ },+ {+ name: 'volumes list',+ args: ['volumes', 'list', '--limit', '5', '--json'],+ expectation: expectPaginated+ },+ {+ name: 'volumes counts',+ args: ['volumes', 'counts', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.objectContaining({ total: expect.any(Number) }));+ }+ },+ {+ name: 'networks list',+ args: ['networks', 'list', '--limit', '5', '--json'],+ expectation: expectPaginated+ },+ {+ name: 'networks counts',+ args: ['networks', 'counts', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.objectContaining({ total: expect.any(Number) }));+ }+ },+ {+ name: 'projects list',+ args: ['projects', 'list', '--limit', '5', '--json'],+ expectation: expectPaginated+ },+ {+ name: 'projects counts',+ args: ['projects', 'counts', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.any(Object));+ }+ },+ {+ name: 'settings list',+ args: ['settings', 'list', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.any(Array));+ }+ },+ {+ name: 'settings public',+ args: ['settings', 'public', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.any(Array));+ }+ },+ {+ name: 'jobs get',+ args: ['jobs', 'get', '--json'],+ expectation: (value) => {+ expect(value).toEqual(+ expect.objectContaining({ environmentHealthInterval: expect.any(String) })+ );+ }+ },+ {+ name: 'updater status',+ args: ['updater', 'status', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.objectContaining({ updatingContainers: expect.any(Number) }));+ }+ },+ {+ name: 'updater history',+ args: ['updater', 'history', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.any(Array));+ }+ },+ {+ name: 'registries list',+ args: ['registries', 'list', '--json'],+ expectation: expectPaginated+ },+ {+ name: 'repos list',+ args: ['repos', 'list', '--limit', '5', '--json'],+ expectation: expectPaginated+ },+ {+ name: 'templates registries',+ args: ['templates', 'registries', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.any(Array));+ }+ },+ {+ name: 'templates variables',+ args: ['templates', 'variables', '--json'],+ expectation: (value) => {+ expect(value === null || Array.isArray(value)).toBe(true);+ }+ },+ {+ name: 'admin users list',+ args: ['admin', 'users', 'list', '--limit', '5', '--json'],+ expectation: expectPaginated+ },+ {+ name: 'admin events list',+ args: ['admin', 'events', 'list', '--limit', '5', '--json'],+ expectation: expectPaginated+ },+ {+ name: 'admin events list-env',+ args: ['admin', 'events', 'list-env', '--limit', '5', '--json'],+ expectation: expectPaginated+ },+ {+ name: 'admin notifications apprise get',+ args: ['admin', 'notifications', 'apprise', 'get', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.objectContaining({ enabled: expect.any(Boolean) }));+ }+ },+ {+ name: 'admin notifications settings get',+ args: ['admin', 'notifications', 'settings', 'get', '--json'],+ expectation: (value) => {+ expect(value).toEqual(expect.any(Array));+ }+ }+];++test.describe('arcane-cli e2e', () => {+ test.beforeAll(async () => {+ if (staticApiKey) return;++ const response = await createTestApiKeys(1);+ apiKey = response.apiKeys[0].key;+ });++ test.afterAll(async () => {+ if (staticApiKey) return;++ await deleteTestApiKeys();+ });++ test('config set and show use an isolated config file', async () => {+ const config = await createCLIConfig('', '');+ try {+ await runCLI(config.configPath, [+ 'config',+ 'set',+ 'server-url',+ baseURL,+ 'api-key',+ apiKey,+ 'environment',+ '0'+ ]);++ const show = await runCLI(config.configPath, ['config', 'show']);+ expect(show.stdout).toContain(`Server URL: ${baseURL}`);+ expect(show.stdout).toContain('API Key: arc_');+ expect(show.stdout).toContain('Default Environment: 0');+ } finally {+ await config.cleanup();+ }+ });++ test('generated config quotes YAML metacharacters', async () => {+ const serverURL = `${baseURL}/path#fragment`;+ const config = await createCLIConfig(serverURL, `arc_test:'#{value}`);+ try {+ const show = await runCLI(config.configPath, ['config', 'show']);+ expect(show.stdout).toContain(`Server URL: ${serverURL}`);+ expect(show.stdout).toContain('Default Environment: 0');+ } finally {+ await config.cleanup();+ }+ });++ test('doctor reports a healthy live connection as JSON', async () => {+ await withConfig(async (config) => {+ const report = await runCLIJSON<{+ healthy: boolean;+ checks: { name: string; status: string; details?: string }[];+ }>(config.configPath, ['doctor']);++ expect(report.healthy).toBe(true);+ expect(report.checks).toEqual(+ expect.arrayContaining([+ expect.objectContaining({ name: 'server_url', status: 'ok', details: baseURL }),+ expect.objectContaining({ name: 'auth', status: 'ok' }),+ expect.objectContaining({ name: 'api_connection', status: 'ok' })+ ])+ );+ });+ });++ test('version returns server details as JSON', async () => {+ await withConfig(async (config) => {+ const version = await runCLIJSON<{+ currentVersion: string;+ displayVersion: string;+ updateAvailable: boolean;+ }>(config.configPath, ['version']);++ expect(version).toEqual(+ expect.objectContaining({+ currentVersion: expect.any(String),+ displayVersion: expect.any(String),+ updateAvailable: expect.any(Boolean)+ })+ );+ expect(version.currentVersion).toMatch(/^(v?\d+\.\d+\.\d+|dev)$/);+ });+ });++ test('environments list and get return local environment JSON', async () => {+ await withConfig(async (config) => {+ const environments = await runCLIJSON<PaginatedResponse<{ id: string; name: string }>>(+ config.configPath,+ ['environments', 'list']+ );+ expect(environments.data).toEqual(+ expect.arrayContaining([expect.objectContaining({ id: '0' })])+ );++ const local = await runCLIJSON<{ id: string; name: string }>(config.configPath, [+ 'environments',+ 'get',+ '0'+ ]);+ expect(local.id).toBe('0');+ expect(local.name).toBeTruthy();+ });+ });++ test('containers list uses the configured environment', async () => {+ await withConfig(async (config) => {+ const containers = await runCLIJSON<PaginatedResponse<{ id: string; name?: string }>>(+ config.configPath,+ ['containers', 'list', '--limit', '5']+ );++ expect(Array.isArray(containers.data)).toBe(true);+ expect(containers.data.length).toBeLessThanOrEqual(5);+ expect(containers.pagination?.totalItems ?? containers.data.length).toBeGreaterThanOrEqual(+ containers.data.length+ );+ });+ });++ for (const command of readOnlyJsonSmokeCommands) {+ test(`${command.name} returns JSON`, async () => {+ await withConfig(async (config) => {+ const result = await runCommandJSON<unknown>(config.configPath, command.args);+ command.expectation(result);+ });+ });+ }++ test('admin api-keys create, get, update, and delete mutate through the CLI', async () => {+ await withConfig(async (config) => {+ const name = `cli-e2e-${Date.now()}`;+ const updatedName = `${name}-updated`;+ let created: CreatedApiKey | undefined;++ try {+ created = await runCLIJSON<CreatedApiKey>(config.configPath, [+ 'admin',+ 'api-keys',+ 'create',+ name,+ '--description',+ 'Created by CLI e2e'+ ]);+ expect(created.id).toBeTruthy();+ expect(created.key).toMatch(/^arc_/);+ expect(created.name).toBe(name);++ const fetched = await runCLIJSON<{ id: string; name: string }>(config.configPath, [+ 'admin',+ 'api-keys',+ 'get',+ created.id+ ]);+ expect(fetched).toEqual(expect.objectContaining({ id: created.id, name }));++ await runCLIJSON(config.configPath, [+ 'admin',+ 'api-keys',+ 'update',+ created.id,+ '--name',+ updatedName,+ '--description',+ 'Updated by CLI e2e'+ ]);++ const updated = await runCLIJSON<{ id: string; name: string }>(config.configPath, [+ 'admin',+ 'api-keys',+ 'get',+ created.id+ ]);+ expect(updated).toEqual(expect.objectContaining({ id: created.id, name: updatedName }));++ await runCLIJSON(config.configPath, ['--yes', 'admin', 'api-keys', 'delete', created.id]);++ const list = await runCLIJSON<PaginatedResponse<{ id: string }>>(config.configPath, [+ 'admin',+ 'api-keys',+ 'list',+ '--limit',+ '100'+ ]);+ expect(list.data.some((item) => item.id === created!.id)).toBe(false);+ created = undefined;+ } finally {+ if (created) {+ await runCLIJSON(config.configPath, [+ '--yes',+ 'admin',+ 'api-keys',+ 'delete',+ created.id+ ]).catch(() => undefined);+ }+ }+ });+ });+});diff --git a/tests/utils/cli.util.ts b/tests/utils/cli.util.tsnew file mode 100644index 000000000..3e313aaf7--- /dev/null+++ b/tests/utils/cli.util.ts@@ -0,0 +1,127 @@+import { execFile } from 'node:child_process';+import fs from 'node:fs/promises';+import os from 'node:os';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { promisify } from 'node:util';++const execFileAsync = promisify(execFile);++const __filename = fileURLToPath(import.meta.url);+const __dirname = path.dirname(__filename);+const repoRoot = path.resolve(__dirname, '..', '..');+const cliDir = path.join(repoRoot, 'cli');+const cliBinDir = path.join(repoRoot, 'tests', '.bin');+const cliBinName = process.platform === 'win32' ? 'arcane-cli.exe' : 'arcane-cli';+const cliBinPath = path.join(cliBinDir, cliBinName);++let buildPromise: Promise<string> | undefined;++export type CLIConfig = {+ configPath: string;+ cleanup: () => Promise<void>;+};++export type CLICommandResult = {+ stdout: string;+ stderr: string;+};++function quoteYAMLString(value: string): string {+ return `'${value.replaceAll("'", "''")}'`;+}++function getErrorField(error: unknown, field: 'code' | 'signal' | 'stdout' | 'stderr'): unknown {+ if (typeof error !== 'object' || error === null || !(field in error)) {+ return undefined;+ }++ return (error as Record<typeof field, unknown>)[field];+}++export async function buildCLI(): Promise<string> {+ if (!buildPromise) {+ buildPromise = (async () => {+ await fs.mkdir(cliBinDir, { recursive: true });+ await execFileAsync('go', ['build', '-o', cliBinPath, '.'], {+ cwd: cliDir,+ maxBuffer: 1024 * 1024 * 10+ });+ return cliBinPath;+ })();+ }++ return buildPromise;+}++export async function createCLIConfig(+ serverURL: string,+ apiKey: string,+ environment = '0'+): Promise<CLIConfig> {+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'arcane-cli-e2e-'));+ const configPath = path.join(dir, 'arcanecli.yml');+ const content = [+ `server_url: ${quoteYAMLString(serverURL)}`,+ `api_key: ${quoteYAMLString(apiKey)}`,+ `default_environment: "${environment}"`,+ 'log_level: info',+ ''+ ].join('\n');++ await fs.writeFile(configPath, content, { mode: 0o600 });++ return {+ configPath,+ cleanup: () => fs.rm(dir, { recursive: true, force: true })+ };+}++export async function runCLI(+ configPath: string,+ args: string[],+ options: { timeoutMs?: number } = {}+): Promise<CLICommandResult> {+ const binary = await buildCLI();+ const commandArgs = ['--config', configPath, '--no-color', ...args];++ try {+ const result = await execFileAsync(binary, commandArgs, {+ cwd: repoRoot,+ env: { ...process.env, NO_COLOR: '1' },+ timeout: options.timeoutMs ?? 30000,+ maxBuffer: 1024 * 1024 * 10+ });+ return { stdout: result.stdout, stderr: result.stderr };+ } catch (error: unknown) {+ const rawStdout = getErrorField(error, 'stdout');+ const rawStderr = getErrorField(error, 'stderr');+ const rawCode = getErrorField(error, 'code') ?? getErrorField(error, 'signal') ?? 'unknown';+ const stdout = typeof rawStdout === 'string' ? rawStdout : '';+ const stderr = typeof rawStderr === 'string' ? rawStderr : '';+ const code = typeof rawCode === 'string' || typeof rawCode === 'number' ? rawCode : 'unknown';+ throw new Error(+ [+ `arcane-cli failed with exit ${code}`,+ `command: ${binary} ${commandArgs.join(' ')}`,+ `stdout:\n${stdout}`,+ `stderr:\n${stderr}`+ ].join('\n\n')+ );+ }+}++export async function runCLIJSON<T>(+ configPath: string,+ args: string[],+ options?: { timeoutMs?: number }+): Promise<T> {+ const result = await runCLI(configPath, ['--output', 'json', ...args], options);++ try {+ return JSON.parse(result.stdout) as T;+ } catch (error: unknown) {+ const message = error instanceof Error ? error.message : String(error);+ throw new Error(`failed to parse arcane-cli JSON output: ${message}\n\n${result.stdout}`);+ }+}