Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep organization browser fixtures within server operation contracts
rcourtman3b566f59d14b
diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.mdindex c1933dfc44..ffc1d7f93a 100644--- a/docs/release-control/v6/internal/subsystems/deployment-installability.md+++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md@@ -1997,6 +1997,7 @@ Windows lifecycle controls or macOS installer and logging checks on native CI.104. `.github/workflows/dependency-advisory-watch.yml`105. `scripts/dependency_advisory_watch.py`106. `tests/integration/scripts/settings-scroll-audit.mjs`+107. `tests/integration/scripts/managed-local-backend.mjs``scripts/installtests/run_unix_contracts.py` is the canonical exhaustive native Unix installer execution runner. Its dedicated verification files are `scripts/installtests/native_unix_contract_runner_test.go` and `scripts/tests/test_native_installer_contract_runner.py`. Script-smoke discovery and the native Go wrapper execute that same Python control suite, so phase inventory, failure propagation and real compiled-Go exit behavior retain one verification source.@@ -3057,6 +3058,23 @@ artifact-selection behaviour.overflow assertions. Infrastructure onboarding uses this same content-owneraudit, including its bottom-position assertion. Its SPA route assertionsobserve the requested URL rather than wait for the document's load event.+ Organization audits use the worker's authenticated cookie context to create+ and activate their API fixtures before entering the workspace. The initial+ load observes the complete organization list without an intervening reload.+ The audit owns created organization IDs before preparation starts, so a+ preparation failure still reaches the same explicit cleanup boundary.+ License activation has a 15-second server request deadline and tenant+ deletion permits 30 seconds of graceful shutdown. Their fixture requests+ use explicit 20-second and 35-second bounds so the default 10-second browser+ action timeout cannot discard the server verdict. The overall journey+ deadlines and ordinary API request timeout remain unchanged. Shared creation+ cleans up an ID when activation fails before returning it, and batch cleanup+ attempts every owned ID while retaining failures. The controlled organization+ fixture tests preserve rejection, timeout and incomplete-shutdown failures.+ Managed native backend teardown copies its final server log to the existing+ test-results directory before removing its runtime root. Missing pre-launch+ logs still permit cleanup; a failed diagnostic copy cannot silently discard+ the runtime containing that evidence.Bottom position and horizontal width are judged from the same browser snapshot.If content reflows after traversal, the audit finishes the changed extent beforeaccepting that snapshot. A scroll reset with an unchanged extent remains andiff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 7d2dc99934..fada460c70 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -4745,6 +4745,7 @@"tests/integration/scripts/hosted-mobile-token-runtime.mjs","tests/integration/scripts/hosted-tenant-runtime.mjs","tests/integration/scripts/managed-dev-runtime.mjs",+ "tests/integration/scripts/managed-local-backend.mjs","tests/integration/scripts/relay-mobile-token-helper.go","tests/integration/scripts/settings-scroll-audit.mjs","tests/integration/tests/helpers.ts",@@ -5267,6 +5268,7 @@"test_prefixes": [],"exact_files": ["tests/integration/scripts/fixture-auth.test.mjs",+ "tests/integration/scripts/organization-fixtures.test.mjs","tests/integration/tests/62-runtime-home-onboarding-contract.spec.ts"]},diff --git a/tests/integration/scripts/managed-local-backend.mjs b/tests/integration/scripts/managed-local-backend.mjsindex 67f91d1993..bfbd39c946 100644--- a/tests/integration/scripts/managed-local-backend.mjs+++ b/tests/integration/scripts/managed-local-backend.mjs@@ -789,6 +789,20 @@ export async function stopManagedLocalBackend({}}+ // Keep the final server diagnostics with the existing browser results.+ // Native/offline teardown removes the runtime root, including pulse.log.+ if (trim(runtimeState.logPath) !== '') {+ const resultsDir = path.resolve(+ getRepoRoot(), 'tests/integration', trim(env.PULSE_E2E_RESULTS_DIR) || 'test-results',+ );+ try {+ await fs.mkdir(resultsDir, { recursive: true, mode: 0o700 });+ await fs.copyFile(runtimeState.logPath, path.join(resultsDir, 'managed-backend.log'));+ } catch (error) {+ if (error.code !== 'ENOENT') throw error;+ }+ }+if (!preserveData && trim(runtimeState.dataDir) !== '') {await fs.rm(path.dirname(runtimeState.dataDir), {recursive: true,diff --git a/tests/integration/scripts/managed-local-backend.test.mjs b/tests/integration/scripts/managed-local-backend.test.mjsindex 7a02adbb2f..8cc2afbcec 100644--- a/tests/integration/scripts/managed-local-backend.test.mjs+++ b/tests/integration/scripts/managed-local-backend.test.mjs@@ -13,11 +13,43 @@ import {buildManagedLocalBackendState,shouldBuildManagedLocalBackendFrontend,shouldBuildManagedLocalBackendBinary,+ stopManagedLocalBackend,} from './managed-local-backend.mjs';const scriptsDir = path.dirname(fileURLToPath(import.meta.url));const integrationRoot = path.resolve(scriptsDir, '..');+test('native teardown retains backend diagnostics before removing its runtime root', async (t) => {+ const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'pulse-backend-log-'));+ t.after(() => fs.rm(directory, { recursive: true, force: true }));+ const root = path.join(directory, 'owned-runtime');+ const results = path.join(directory, 'results');+ await fs.mkdir(path.join(root, 'data'), { recursive: true });+ await fs.writeFile(path.join(root, 'pulse.log'), 'started\nfinal shutdown diagnostic\n');+ await stopManagedLocalBackend({+ env: { PULSE_E2E_RESULTS_DIR: results },+ state: { managedLocalBackend: true, pid: null, dataDir: path.join(root, 'data'), logPath: path.join(root, 'pulse.log') },+ preserveRuntimeState: true,+ logger: { log() {} },+ });+ assert.equal(await fs.readFile(path.join(results, 'managed-backend.log'), 'utf8'), 'started\nfinal shutdown diagnostic\n');+ await assert.rejects(fs.stat(root), { code: 'ENOENT' });+});++test('native teardown still removes a runtime that never produced a backend log', async (t) => {+ const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'pulse-backend-no-log-'));+ t.after(() => fs.rm(directory, { recursive: true, force: true }));+ const root = path.join(directory, 'owned-runtime');+ await fs.mkdir(path.join(root, 'data'), { recursive: true });+ await stopManagedLocalBackend({+ env: { PULSE_E2E_RESULTS_DIR: path.join(directory, 'results') },+ state: { managedLocalBackend: true, pid: null, dataDir: path.join(root, 'data'), logPath: path.join(root, 'pulse.log') },+ preserveRuntimeState: true,+ logger: { log() {} },+ });+ await assert.rejects(fs.stat(root), { code: 'ENOENT' });+});+test('integration setup pins the governed Node.js major', async () => {const [quickStart, setupScript] = await Promise.all([fs.readFile(path.join(integrationRoot, 'QUICK_START.md'), 'utf8'),diff --git a/tests/integration/scripts/organization-fixtures.test.mjs b/tests/integration/scripts/organization-fixtures.test.mjsnew file mode 100644index 0000000000..634e36c6e7--- /dev/null+++ b/tests/integration/scripts/organization-fixtures.test.mjs@@ -0,0 +1,134 @@+import assert from 'node:assert/strict';+import fs from 'node:fs';+import { stripTypeScriptTypes } from 'node:module';+import path from 'node:path';+import test from 'node:test';+import { fileURLToPath } from 'node:url';+import { runInNewContext } from 'node:vm';++const integration = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');+const source = fs.readFileSync(process.env.PULSE_ORG_FIXTURE_SOURCE || path.join(integration, 'tests/helpers.ts'), 'utf8');+const start = source.indexOf('export async function activateOfflineOrganization(');+const end = source.indexOf('export async function switchOrg(', start);+assert.ok(start >= 0 && end > start, 'missing public organization fixture helpers');+const helpers = stripTypeScriptTypes(`${source.slice(start, end).replace(/^export /gm, '')}\n({ activateOfflineOrganization, createOrg, deleteOrg, deleteOrgs });`);+const response = (status, data = {}) => ({+ ok: () => status >= 200 && status < 300,+ status: () => status,+ json: async () => data,+ text: async () => JSON.stringify(data),+});++function fixture(request) {+ const calls = [];+ const api = runInNewContext(helpers, {+ process: { env: { PULSE_E2E_OFFLINE_ACTIVATION_KEY: 'ephemeral-fixture-key' } },+ toOrgID: () => 'fixture-org',+ expect: value => ({+ toBe: expected => assert.equal(value, expected),+ toContain: expected => assert.ok(value.includes(expected)),+ not: { toContain: expected => assert.ok(!value.includes(expected)) },+ }),+ apiRequest: async (page, endpoint, options = {}) => {+ calls.push({ endpoint, options });+ if (endpoint === '/api/license/runtime-capabilities') return response(200, { capabilities: ['multi_tenant'] });+ if (endpoint === '/api/security/status') return response(200, { sessionCapabilities: { demoMode: false } });+ return request(endpoint, options);+ },+ });+ return { ...api, calls };+}++// A logical clock keeps this a fast boundary test. These operations are valid+// at the server's existing 15s activation and 30s graceful-shutdown deadlines.+function completeWithinRequestBudget(options, elapsed, status, payload) {+ const timeout = options.timeout ?? 10_000;+ assert.ok(timeout > 0 && timeout <= 35_000, 'operation must retain a bounded request timeout');+ if (elapsed >= timeout) throw new Error('request timed out before the server contract completed');+ return response(status, payload);+}++test('activation receives the server verdict beyond the browser action timeout', async () => {+ const f = fixture((endpoint, options) => {+ assert.equal(endpoint, '/api/license/activate');+ return completeWithinRequestBudget(options, 15_000, 200);+ });+ await f.activateOfflineOrganization({}, 'owned-org');+ assert.equal(f.calls[0].options.headers['X-Pulse-Org-ID'], 'owned-org');+ assert.equal(f.calls.filter(call => call.endpoint === '/api/license/activate').length, 1);+});++test('activation rejection remains a failure without another attempt', async () => {+ const f = fixture(() => response(403, { error: 'rejected' }));+ await assert.rejects(f.activateOfflineOrganization({}), /403/);+ assert.equal(f.calls.length, 1);+});++test('deletion waits for graceful shutdown and uses the default admin scope', async () => {+ const f = fixture((endpoint, options) => {+ assert.equal(endpoint, '/api/orgs/owned%2Forg');+ assert.equal(options.method, 'DELETE');+ assert.equal(options.headers['X-Org-ID'], 'default');+ assert.equal(options.headers['X-Pulse-Org-ID'], 'default');+ return completeWithinRequestBudget(options, 30_000, 204);+ });+ await f.deleteOrg({}, 'owned/org');+ assert.equal(f.calls.length, 1);+});++test('deletion still rejects incomplete shutdown, unexpected success and an exhausted request budget', async () => {+ for (const status of [200, 403, 503]) {+ const f = fixture(() => response(status));+ await assert.rejects(f.deleteOrg({}, 'owned'), new RegExp(String(status)));+ assert.equal(f.calls.length, 1);+ }+ const missing = fixture(() => response(404));+ await missing.deleteOrg({}, 'already-removed');+ const expired = fixture((endpoint, options) => completeWithinRequestBudget(options, 35_001, 204));+ await assert.rejects(expired.deleteOrg({}, 'stuck'), /request timed out/);+});++test('creation owns the returned ID before activation can fail', async () => {+ const activationError = new Error('issuer unavailable');+ const f = fixture((endpoint, options) => {+ if (endpoint === '/api/orgs') return response(201, { id: 'created-org' });+ if (endpoint === '/api/license/activate') throw activationError;+ assert.equal(endpoint, '/api/orgs/created-org');+ assert.equal(options.method, 'DELETE');+ return response(204);+ });+ await assert.rejects(f.createOrg({}, 'Fixture'), error => error === activationError);+ assert.deepEqual(f.calls.map(call => call.endpoint), ['/api/orgs', '/api/license/activate', '/api/orgs/created-org']);+});++test('creation preserves activation and cleanup failures together', async () => {+ const activationError = new Error('issuer unavailable');+ const cleanupError = new Error('shutdown incomplete');+ const f = fixture(endpoint => {+ if (endpoint === '/api/orgs') return response(201, { id: 'created-org' });+ if (endpoint === '/api/license/activate') throw activationError;+ throw cleanupError;+ });+ await assert.rejects(f.createOrg({}, 'Fixture'), error => {+ assert.equal(error.name, 'AggregateError');+ assert.deepEqual([...error.errors], [activationError, cleanupError]);+ return true;+ });+});++test('successful creation returns ownership without deleting the fixture', async () => {+ const f = fixture(endpoint => endpoint === '/api/orgs' ? response(201, { id: 'created-org' }) : response(200));+ assert.equal((await f.createOrg({}, 'Fixture')).id, 'created-org');+ assert.ok(!f.calls.some(call => call.options.method === 'DELETE'));+});++test('cleanup attempts every owned ID in reverse order and reports every failure', async () => {+ const f = fixture(endpoint => endpoint.endsWith('/middle') ? response(503) : response(204));+ await assert.rejects(f.deleteOrgs({}, ['first', 'middle', 'last']), error => {+ assert.equal(error.name, 'AggregateError');+ assert.equal(error.errors.length, 1);+ assert.match(error.errors[0].message, /middle.*503/);+ return true;+ });+ assert.deepEqual(f.calls.map(call => call.endpoint), ['/api/orgs/last', '/api/orgs/middle', '/api/orgs/first']);+});diff --git a/tests/integration/tests/05-settings-mobile-audit.spec.ts b/tests/integration/tests/05-settings-mobile-audit.spec.tsindex 3e1f2e38de..0374d77cdf 100644--- a/tests/integration/tests/05-settings-mobile-audit.spec.ts+++ b/tests/integration/tests/05-settings-mobile-audit.spec.ts@@ -6,9 +6,9 @@ import { test as base, expect } from "@playwright/test";import {apiRequest,activateOfflineOrganization,- ensureSessionAuthenticated,createAuthenticatedStorageState,createOrg,+ deleteOrgs,} from "./helpers";const __dirname = path.dirname(fileURLToPath(import.meta.url));@@ -96,19 +96,15 @@ const MOBILE_VIEWPORTS = [const prepareOrganizationAuditFixture = async (page: import("@playwright/test").Page,route: string,-): Promise<string[]> => {+ createdOrgIDs: string[],+): Promise<void> => {if (!(ORGANIZATION_SETTINGS_ROUTES as readonly string[]).includes(route))- return [];+ return;if (process.env.PULSE_E2E_OFFLINE_ACTIVATION_KEY) {- await ensureSessionAuthenticated(page);await activateOfflineOrganization(page);}- await page.goto("/settings/infrastructure", {- waitUntil: "domcontentloaded",- });- const createdOrgIDs: string[] = [];let targetOrgID = "";if (route === "/settings/organization/sharing") {@@ -119,7 +115,12 @@ const prepareOrganizationAuditFixture = async (const source = await createOrg(page, "Mobile Audit Organization");createdOrgIDs.push(source.id);- await page.reload({ waitUntil: "domcontentloaded" });+ // The worker fixture already supplies a cookie session. Arrange the API+ // fixtures before the first workspace load so admission sees the complete+ // organization list without a second bootstrap or an intervening reload.+ await page.goto("/settings/infrastructure", {+ waitUntil: "domcontentloaded",+ });const organizationSwitcher = page.getByRole("combobox", {name: "Organization",});@@ -175,7 +176,6 @@ const prepareOrganizationAuditFixture = async ().toBeTruthy();}- return createdOrgIDs;};const test = base.extend<{}, WorkerFixtures>({@@ -211,12 +211,13 @@ test.describe("Settings mobile optimization audit", () => {test(`no horizontal overflow after full scroll on ${route}`, async ({page,}) => {- const createdOrgIDs = await prepareOrganizationAuditFixture(page, route);+ const createdOrgIDs: string[] = [];try {+ await prepareOrganizationAuditFixture(page, route, createdOrgIDs);for (const viewport of MOBILE_VIEWPORTS) {await page.setViewportSize(viewport);await page.goto(route, { waitUntil: "domcontentloaded" });- await page.waitForURL((url) => url.pathname === route, {+ await expect(page).toHaveURL((url) => url.pathname === route, {timeout: 15000,});expect(@@ -293,35 +294,20 @@ test.describe("Settings mobile optimization audit", () => {}}} finally {- if (createdOrgIDs.length > 0) {- const organizationSwitcher = page.getByRole("combobox", {- name: "Organization",- });- if (await organizationSwitcher.isVisible().catch(() => false)) {- await organizationSwitcher.selectOption("default");- await expect(organizationSwitcher).toHaveValue("default", {- timeout: 20000,+ try {+ if (createdOrgIDs.length > 0) {+ const organizationSwitcher = page.getByRole("combobox", {+ name: "Organization",});+ if (await organizationSwitcher.isVisible().catch(() => false)) {+ await organizationSwitcher.selectOption("default");+ await expect(organizationSwitcher).toHaveValue("default", {+ timeout: 20000,+ });+ }}- }- for (const orgID of [...createdOrgIDs].reverse()) {- const response = await apiRequest(- page,- `/api/orgs/${encodeURIComponent(orgID)}`,- {- method: "DELETE",- headers: {- "X-Org-ID": "default",- "X-Pulse-Org-ID": "default",- },- },- );- const responseBody =- response.status() === 204 ? "" : await response.text();- expect(- [204, 404].includes(response.status()),- `cleanup organization ${orgID}: ${response.status()} ${responseBody}`,- ).toBeTruthy();+ } finally {+ await deleteOrgs(page, createdOrgIDs);}}});@@ -332,7 +318,7 @@ test.describe("Settings mobile optimization audit", () => {page,}) => {await page.goto(route, { waitUntil: "domcontentloaded" });- await page.waitForURL((url) => url.pathname === expectedPath, {+ await expect(page).toHaveURL((url) => url.pathname === expectedPath, {timeout: 15000,});expect(new URL(page.url()).pathname).toBe(expectedPath);@@ -346,7 +332,7 @@ test.describe("Settings mobile optimization audit", () => {await page.goto("/settings/system-general", {waitUntil: "domcontentloaded",});- await page.waitForURL(+ await expect(page).toHaveURL((url) => url.pathname === "/settings/system-general",{timeout: 15000,diff --git a/tests/integration/tests/helpers.ts b/tests/integration/tests/helpers.tsindex 9e203b3343..d65689a020 100644--- a/tests/integration/tests/helpers.ts+++ b/tests/integration/tests/helpers.ts@@ -1733,7 +1733,9 @@ export async function activateOfflineOrganization(page: Page, orgId = "default")if (!key) return;const headers = { "X-Pulse-Org-ID": orgId, "X-Org-ID": orgId };const activation = await apiRequest(page, "/api/license/activate", {- method: "POST", headers, data: { license_key: key },+ // The issuer request alone has a 15s server deadline. The browser's+ // 10s action timeout must not pre-empt a valid activation response.+ method: "POST", headers, data: { license_key: key }, timeout: 20_000,});if (!activation.ok()) throw new Error(`Offline org activation failed: ${activation.status()}`);const response = await apiRequest(page, "/api/license/runtime-capabilities", { headers });@@ -1765,21 +1767,50 @@ export async function createOrg(throw new Error("Failed to create org: response missing org id");}- await activateOfflineOrganization(page, payload.id);+ try {+ await activateOfflineOrganization(page, payload.id);+ } catch (error) {+ // The API creation already succeeded. Keep ownership if subsequent+ // activation fails before the caller can receive and track the ID.+ try {+ await deleteOrg(page, payload.id);+ } catch (cleanupError) {+ throw new AggregateError([error, cleanupError], `Organization ${payload.id} activation and cleanup failed`);+ }+ throw error;+ }return { id: payload.id };}export async function deleteOrg(page: Page, orgId: string): Promise<void> {const res = await apiRequest(page, `/api/orgs/${encodeURIComponent(orgId)}`, {method: "DELETE",+ // Tenant deletion allows 30s for graceful shutdown before reporting an+ // incomplete teardown. Retain that verdict instead of abandoning at 10s.+ timeout: 35_000,+ headers: { "X-Org-ID": "default", "X-Pulse-Org-ID": "default" },});- if (!res.ok() && res.status() !== 404) {+ if (![204, 404].includes(res.status())) {throw new Error(- `Failed to delete org: ${res.status()} ${await res.text()}`,+ `Failed to delete org ${orgId}: ${res.status()} ${await res.text()}`,);}}+export async function deleteOrgs(page: Page, orgIds: readonly string[]): Promise<void> {+ const failures: unknown[] = [];+ for (const orgId of [...orgIds].reverse()) {+ try {+ await deleteOrg(page, orgId);+ } catch (error) {+ failures.push(error);+ }+ }+ if (failures.length) {+ throw new AggregateError(failures, "Organization fixture cleanup failed");+ }+}+export async function switchOrg(page: Page, orgId: string): Promise<void> {await page.evaluate((id) => {window.sessionStorage.setItem("pulse_org_id", id);