Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

test: protect fixture recovery dry-run and transactional restore (#11042)

Adam Eivyedba8bd1e8dd
diff --git a/scripts/lib/dbTestFiles.js b/scripts/lib/dbTestFiles.js
index 6fad201cc5..4ff2b95cca 100644
--- a/scripts/lib/dbTestFiles.js
+++ b/scripts/lib/dbTestFiles.js
@@ -13,6 +13,7 @@ export const DB_TEST_INCLUDE = [
'services/voice/profiles.db.test.js',
'services/voice/studio.db.test.js',
'../scripts/perf/collectionFixture.db.test.js',
+ '../scripts/recover-synced-test-fixtures.db.test.js',
'services/appQuality.db.test.js',
'services/deepAudit.db.test.js',
'**/db.test.js',
diff --git a/scripts/recover-synced-test-fixtures.db.test.js b/scripts/recover-synced-test-fixtures.db.test.js
new file mode 100644
index 0000000000..87750db532
--- /dev/null
+++ b/scripts/recover-synced-test-fixtures.db.test.js
@@ -0,0 +1,184 @@
+import { afterAll, afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { createRequire } from 'node:module';
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, existsSync, rmSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { randomUUID } from 'node:crypto';
+import { requireDbOrSkip } from '../server/lib/dbTestGate.js';
+
+const { Client } = createRequire(new URL('../server/package.json', import.meta.url))('pg');
+const script = fileURLToPath(new URL('./recover-synced-test-fixtures.mjs', import.meta.url));
+const connection = {
+ database: 'portos_test', host: process.env.PGHOST || 'localhost',
+ port: Number(process.env.PGPORT || 5432), user: process.env.PGUSER || 'portos',
+ password: process.env.PGPASSWORD || 'portos', options: '', connectionTimeoutMillis: 3000,
+};
+let client;
+let ready = false;
+let reason = 'requires PGDATABASE=portos_test';
+if (process.env.PGDATABASE === 'portos_test') {
+ client = new Client(connection);
+ try {
+ await client.connect();
+ const psql = spawnSync('psql', ['--version'], { encoding: 'utf8', timeout: 5000 });
+ ready = psql.status === 0;
+ reason = ready ? '' : 'psql executable unavailable';
+ } catch {
+ reason = 'portos_test is unavailable';
+ }
+}
+const runDb = requireDbOrSkip('fixture recovery CLI', ready, reason);
+afterAll(async () => { await client?.end(); });
+
+describe.skipIf(!runDb)('fixture recovery CLI against isolated PostgreSQL tables', () => {
+ let schema;
+ let root;
+ let subsPath;
+ let originalBytes;
+ const threshold = 1000;
+ const tables = ['universes', 'pipeline_series', 'writers_room_works'];
+
+ function run(args = []) {
+ return spawnSync(process.execPath, [script, '--data-root=' + root,
+ '--uni-threshold=' + threshold, '--ser-threshold=' + threshold, ...args], {
+ encoding: 'utf8', timeout: 15000,
+ env: {
+ ...process.env, PGDATABASE: 'portos_test', PGHOST: connection.host,
+ PGPORT: String(connection.port), PGUSER: connection.user, PGPASSWORD: connection.password,
+ // No public fallback: every unqualified child query must use only our schema.
+ PGOPTIONS: '-c search_path=' + schema + ' -c statement_timeout=5000',
+ PGCONNECT_TIMEOUT: '3', PSQLRC: join(root, 'empty-psqlrc'),
+ },
+ });
+ }
+
+ async function snapshot() {
+ const result = {};
+ for (const table of tables) {
+ result[table] = (await client.query('SELECT * FROM ' + schema + '.' + table + ' ORDER BY id')).rows;
+ }
+ return result;
+ }
+
+ beforeEach(async () => {
+ schema = 'recovery_test_' + randomUUID().replaceAll('-', '');
+ root = mkdtempSync(join(tmpdir(), 'portos-recovery-db-'));
+ mkdirSync(join(root, 'sharing'));
+ writeFileSync(join(root, 'empty-psqlrc'), '');
+ subsPath = join(root, 'sharing', 'peer_subscriptions.json');
+ originalBytes = JSON.stringify({ version: 1, subscriptions: [
+ { recordKind: 'universe', recordId: 'restore', peerId: 'example-peer' },
+ { recordKind: 'universe', recordId: 'fixture' },
+ { recordKind: 'universe', recordId: 'missing' },
+ { recordKind: 'series', recordId: 'inconsistent' },
+ { recordKind: 'series', recordId: 'fixture' },
+ { recordKind: 'series', recordId: 'missing' },
+ { recordKind: 'mediaCollection', recordId: 'unrelated' },
+ { recordKind: 'future-kind', recordId: 'unrelated' },
+ ] }) + '\n';
+ writeFileSync(subsPath, originalBytes);
+ await client.query('CREATE SCHEMA ' + schema);
+ for (const table of tables) {
+ await client.query('CREATE TABLE ' + schema + '.' + table + ` (
+ id text PRIMARY KEY, name text, title text, data jsonb NOT NULL,
+ deleted boolean NOT NULL, deleted_at timestamptz, updated_at timestamptz NOT NULL
+ )`);
+ const records = [
+ ['fixture', false, false, threshold - 1],
+ ['boundary', false, false, threshold],
+ ['above', false, false, threshold + 1],
+ ['restore', true, true, threshold + 100],
+ ['inconsistent', true, false, threshold + 100],
+ ];
+ for (const [id, jsonDeleted, columnDeleted, size] of records) {
+ const data = { id, deleted: jsonDeleted, updatedAt: '2020-01-01T00:00:00.000Z',
+ authored: { text: 'Synthetic authored content', tags: ['retained'] }, padding: '' };
+ if (jsonDeleted) data.deletedAt = '2020-01-02T00:00:00.000Z';
+ // Measure PostgreSQL's JSONB text representation, which is the CLI's policy.
+ const { rows } = await client.query('SELECT length($1::jsonb::text) AS size', [JSON.stringify(data)]);
+ data.padding = 'x'.repeat(size - rows[0].size);
+ await client.query('INSERT INTO ' + schema + '.' + table +
+ ' (id, name, title, data, deleted, deleted_at, updated_at) VALUES ($1,$1,$1,$2,$3,$4,$5)',
+ [id, JSON.stringify(data), columnDeleted, jsonDeleted ? '2020-01-02T00:00:00.000Z' : null,
+ '2020-01-01T00:00:00.000Z']);
+ }
+ }
+ });
+
+ afterEach(async () => {
+ try {
+ if (schema) await client.query('DROP SCHEMA IF EXISTS ' + schema + ' CASCADE');
+ } finally {
+ if (root) rmSync(root, { recursive: true, force: true });
+ }
+ });
+
+ it('default invocation leaves all three tables and subscription bytes unchanged', async () => {
+ const before = await snapshot();
+ const result = run();
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.stdout).toContain('DRY-RUN');
+ expect(await snapshot()).toEqual(before);
+ expect(readFileSync(subsPath, 'utf8')).toBe(originalBytes);
+ expect(existsSync(subsPath + '.bak')).toBe(false);
+ });
+
+ it('restores JSONB and mirrors, respects the size boundary, prunes subscriptions, and is idempotent', async () => {
+ const before = await snapshot();
+ // Pin the policy boundary independently of the fixture-size construction.
+ for (const table of ['universes', 'pipeline_series']) {
+ const sizes = (await client.query('SELECT id, length(data::text) AS size FROM ' + schema + '.' + table)).rows;
+ expect(sizes.find(r => r.id === 'fixture').size).toBe(threshold - 1);
+ expect(sizes.find(r => r.id === 'boundary').size).toBe(threshold);
+ expect(sizes.find(r => r.id === 'above').size).toBe(threshold + 1);
+ }
+ const result = run(['--apply']);
+ expect(result.status, result.stderr).toBe(0);
+ const after = await snapshot();
+ expect(after.writers_room_works).toEqual(before.writers_room_works);
+ for (const table of ['universes', 'pipeline_series']) {
+ expect(after[table].map(r => r.id)).toEqual(['above', 'boundary', 'inconsistent', 'restore']);
+ for (const id of ['above', 'boundary']) {
+ expect(after[table].find(r => r.id === id)).toEqual(before[table].find(r => r.id === id));
+ }
+ for (const id of ['restore', 'inconsistent']) {
+ const restored = after[table].find(r => r.id === id);
+ const original = before[table].find(r => r.id === id);
+ const { deletedAt: _deletedAt, ...originalData } = original.data;
+ expect(restored.data).toEqual({ ...originalData, deleted: false, updatedAt: restored.data.updatedAt });
+ expect(restored.data).not.toHaveProperty('deletedAt');
+ expect(restored.deleted).toBe(false);
+ expect(restored.deleted_at).toBeNull();
+ expect(restored.updated_at.getTime()).toBeGreaterThan(original.updated_at.getTime());
+ expect(restored.data.updatedAt).toBe(restored.updated_at.toISOString());
+ }
+ }
+ const prunedBytes = readFileSync(subsPath, 'utf8');
+ const original = JSON.parse(originalBytes);
+ expect(JSON.parse(prunedBytes)).toEqual({ ...original,
+ subscriptions: original.subscriptions.filter(s => !['fixture', 'missing'].includes(s.recordId)) });
+ expect(readFileSync(subsPath + '.bak', 'utf8')).toBe(originalBytes);
+ const second = run(['--apply']);
+ expect(second.status, second.stderr).toBe(0);
+ expect(await snapshot()).toEqual(after);
+ expect(readFileSync(subsPath, 'utf8')).toBe(prunedBytes);
+ expect(readFileSync(subsPath + '.bak', 'utf8')).toBe(prunedBytes);
+ });
+
+ it('rolls back earlier mutations in both tables and leaves subscriptions untouched after a later SQL failure', async () => {
+ // The final series purge fails after both restores and the universe purge.
+ await client.query('CREATE FUNCTION ' + schema + `.reject_purge() RETURNS trigger LANGUAGE plpgsql AS $$
+ BEGIN RAISE EXCEPTION 'synthetic purge failure'; END $$`);
+ await client.query('CREATE TRIGGER reject_purge BEFORE DELETE ON ' + schema +
+ '.pipeline_series FOR EACH ROW EXECUTE FUNCTION ' + schema + '.reject_purge()');
+ const before = await snapshot();
+ const result = run(['--apply']);
+ expect(result.status).not.toBe(0);
+ expect(result.stderr).toContain('synthetic purge failure');
+ expect(await snapshot()).toEqual(before);
+ expect(readFileSync(subsPath, 'utf8')).toBe(originalBytes);
+ expect(existsSync(subsPath + '.bak')).toBe(false);
+ });
+});
diff --git a/scripts/recover-synced-test-fixtures.test.js b/scripts/recover-synced-test-fixtures.test.js
new file mode 100644
index 0000000000..bcb8cfa3fe
--- /dev/null
+++ b/scripts/recover-synced-test-fixtures.test.js
@@ -0,0 +1,106 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, existsSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+const { execFileSync } = vi.hoisted(() => ({ execFileSync: vi.fn() }));
+vi.mock('child_process', () => ({ execFileSync }));
+
+let root;
+let subsPath;
+let originalBytes;
+const originalArgv = process.argv;
+const dryRunExit = new Error('dry-run exit');
+
+async function run(args = []) {
+ process.argv = ['node', 'recover-synced-test-fixtures.mjs', '--data-root=' + root, ...args];
+ await import('./recover-synced-test-fixtures.mjs');
+}
+
+beforeEach(() => {
+ vi.resetModules();
+ root = mkdtempSync(join(tmpdir(), 'portos-recovery-cli-'));
+ mkdirSync(join(root, 'sharing'));
+ subsPath = join(root, 'sharing', 'peer_subscriptions.json');
+ originalBytes = JSON.stringify({
+ version: 1,
+ subscriptions: [
+ { recordKind: 'universe', recordId: 'live-universe', peerId: 'example-peer' },
+ { recordKind: 'universe', recordId: 'missing-universe' },
+ { recordKind: 'series', recordId: 'live-series' },
+ { recordKind: 'series', recordId: 'missing-series' },
+ { recordKind: 'mediaCollection', recordId: 'unrelated-media' },
+ { recordKind: 'future-kind', recordId: 'unrelated-record' },
+ ],
+ }) + '\n';
+ writeFileSync(subsPath, originalBytes);
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ vi.spyOn(process, 'exit').mockImplementation(code => {
+ expect(code).toBe(0);
+ throw dryRunExit;
+ });
+ execFileSync.mockReset().mockImplementation((_command, args, options) => {
+ if (options.input) return 'COMMIT\n';
+ const sql = args.at(-1);
+ if (sql.startsWith('SELECT id FROM universes')) return 'live-universe\n';
+ if (sql.startsWith('SELECT id FROM pipeline_series')) return 'live-series\n';
+ return sql.startsWith('SELECT count(*)') ? '1\n' : '';
+ });
+});
+
+afterEach(() => {
+ process.argv = originalArgv;
+ vi.restoreAllMocks();
+ rmSync(root, { recursive: true, force: true });
+});
+
+describe('fixture recovery CLI process and filesystem boundary', () => {
+ it('defaults to inspection without issuing writes or touching subscription bytes', async () => {
+ await expect(run()).rejects.toBe(dryRunExit);
+ expect(execFileSync).toHaveBeenCalledTimes(7);
+ for (const [command, args, options] of execFileSync.mock.calls) {
+ expect(command).toBe('psql');
+ expect(args.at(-1)).toMatch(/^SELECT /);
+ expect(options.input).toBeUndefined();
+ }
+ expect(readFileSync(subsPath, 'utf8')).toBe(originalBytes);
+ expect(existsSync(subsPath + '.bak')).toBe(false);
+ });
+
+ it('requires explicit apply, forwards thresholds/connection, and prunes only missing universes and series', async () => {
+ vi.stubEnv('PGDATABASE', 'portos_test');
+ vi.stubEnv('PGHOST', 'db.example.invalid');
+ try {
+ await run(['--apply', '--uni-threshold=1500', '--ser-threshold=900', '--port=6543']);
+ } finally {
+ vi.unstubAllEnvs();
+ }
+ const batches = execFileSync.mock.calls.filter(([, , options]) => options.input);
+ expect(batches).toHaveLength(1);
+ expect(batches[0][1]).toEqual([
+ '-h', 'db.example.invalid', '-p', '6543', '-d', 'portos_test', '-v', 'ON_ERROR_STOP=1',
+ ]);
+ expect(batches[0][2].input).toMatch(/^\s*BEGIN;/);
+ expect(batches[0][2].input).toMatch(/COMMIT;\s*$/);
+ expect(batches[0][2].input).toContain('length(data::text) < 1500');
+ expect(batches[0][2].input).toContain('length(data::text) < 900');
+ const original = JSON.parse(originalBytes);
+ expect(JSON.parse(readFileSync(subsPath, 'utf8'))).toEqual({
+ ...original, subscriptions: original.subscriptions.filter(s => !s.recordId.startsWith('missing-')),
+ });
+ expect(readFileSync(subsPath + '.bak', 'utf8')).toBe(originalBytes);
+ });
+
+ it('propagates mutation-command failure before rewriting or backing up subscriptions', async () => {
+ const failure = new Error('synthetic psql failure');
+ const reads = execFileSync.getMockImplementation();
+ execFileSync.mockImplementation((command, args, options) => {
+ if (options.input) throw failure;
+ return reads(command, args, options);
+ });
+ await expect(run(['--apply'])).rejects.toBe(failure);
+ expect(execFileSync).toHaveBeenCalledTimes(8);
+ expect(readFileSync(subsPath, 'utf8')).toBe(originalBytes);
+ expect(existsSync(subsPath + '.bak')).toBe(false);
+ });
+});