Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve every health recovery archive when requests share a timestamp (#10697)
Adam Eivy6f6b918b75a9
diff --git a/server/services/dataManager.archive.test.js b/server/services/dataManager.archive.test.jsnew file mode 100644index 0000000000..37054232a9--- /dev/null+++ b/server/services/dataManager.archive.test.js@@ -0,0 +1,141 @@+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { mkdir, readFile, readdir, rm, writeFile } from 'fs/promises';+import { basename, dirname, join, resolve } from 'path';+import { execFileSync } from '../lib/childProcess.js';+import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';++const controls = vi.hoisted(() => ({ open: null, tar: null, id: null, tarArgs: [] }));+vi.mock('../lib/fileUtils.js', async (importOriginal) => {+ const actual = await importOriginal();+ return makePathsProxy(actual, { dataRoot: () => root });+});+vi.mock('fs/promises', async (importOriginal) => {+ const actual = await importOriginal();+ return {+ ...actual,+ open: (...args) => controls.open ? controls.open(...args) : actual.open(...args),+ };+});+vi.mock('crypto', async (importOriginal) => {+ const actual = await importOriginal();+ return { ...actual, randomUUID: () => controls.id ?? actual.randomUUID() };+});+vi.mock('../lib/childProcess.js', async (importOriginal) => {+ const actual = await importOriginal();+ return { ...actual, execFile: (command, args, options, callback) => {+ controls.tarArgs.push(args);+ return controls.tar ? controls.tar(command, args, options, callback) : actual.execFile(command, args, options, callback);+ } };+});++const root = createTempDataRoot('portos-data-manager-archive-');+const { archiveCategory, getBackups, deleteBackup } = await import('./dataManager.js');+const health = join(root, 'health');+const backup = join(root, 'backup');+const archiveBytes = (result, file) => execFileSync('tar', ['-xOzf', resolve(result.archivePath), file], { encoding: 'utf8' });+const seedHealth = async () => {+ await mkdir(health, { recursive: true });+ await writeFile(join(health, '2024-01-01.json'), '{"example":"older"}');+ await writeFile(join(health, '2026-10-01.json'), '{"example":"recent"}');+};++beforeEach(async () => {+ await rm(root, { recursive: true, force: true });+ await mkdir(root, { recursive: true });+ Object.assign(controls, { open: null, tar: null, id: null, tarArgs: [] });+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime(new Date('2026-10-08T12:00:00Z'));+});+afterEach(() => vi.useRealTimers());+afterAll(async () => { await rm(root, { recursive: true, force: true }); });++describe('Data Manager recovery archives (#10697)', () => {+ it('keeps both same-second health selections recoverable and supports new and legacy archive names', async () => {+ await seedHealth();+ const first = await archiveCategory('health', { daysToKeep: 365 });+ const second = await archiveCategory('health', { daysToKeep: 1 });+ expect(first.archivePath).not.toBe(second.archivePath);+ expect(await archiveBytes(first, '2024-01-01.json')).toBe('{"example":"older"}');+ expect(await archiveBytes(second, '2026-10-01.json')).toBe('{"example":"recent"}');+ expect(await readdir(health)).toEqual([]);+ const legacy = 'health-2026-01-01T00-00-00.tar.gz';+ await writeFile(join(backup, legacy), 'legacy fixture');+ expect((await getBackups()).map(row => row.name).sort()).toEqual([basename(first.archivePath), basename(second.archivePath), legacy].sort());+ for (const name of [basename(first.archivePath), basename(second.archivePath), legacy]) await deleteBackup(name);+ expect(await readdir(backup)).toEqual([]);+ });++ it('reserves separate input lists and staging archives for concurrent health requests', async () => {+ await seedHealth();+ const actual = await vi.importActual('../lib/childProcess.js');+ const completions = [];+ // Hold successful tar callbacks until both have read their selected inputs:+ // source-import/deletion coordination is a separate lifecycle issue.+ controls.tar = (command, args, options, callback) => actual.execFile(command, args, options, (error, ...output) => {+ completions.push(() => callback(error, ...output));+ if (completions.length === 2) completions.forEach(finish => finish());+ });+ const results = await Promise.all([archiveCategory('health'), archiveCategory('health', { daysToKeep: 1 })]);+ expect(new Set(controls.tarArgs.map(args => dirname(args[1]))).size).toBe(2);+ expect(new Set(controls.tarArgs.map(args => args[args.indexOf('-T') + 1])).size).toBe(2);+ expect(await archiveBytes(results[0], '2024-01-01.json')).toBe('{"example":"older"}');+ expect(await archiveBytes(results[1], '2026-10-01.json')).toBe('{"example":"recent"}');+ expect((await getBackups()).map(row => row.name).sort()).toEqual(results.map(result => basename(result.archivePath)).sort());+ expect((await readdir(backup)).every(name => name.endsWith('.tar.gz'))).toBe(true);+ });++ it('also preserves successive generic category archives in the same second', async () => {+ await mkdir(join(root, 'calendar'));+ await writeFile(join(root, 'calendar', 'example.json'), 'first example');+ const first = await archiveCategory('calendar');+ await writeFile(join(root, 'calendar', 'example.json'), 'second example');+ const second = await archiveCategory('calendar');+ expect(await archiveBytes(first, 'calendar/example.json')).toBe('first example');+ expect(await archiveBytes(second, 'calendar/example.json')).toBe('second example');+ });++ it('cleans only its scratch and keeps health sources when tar fails', async () => {+ await seedHealth();+ await mkdir(backup);+ await writeFile(join(backup, 'earlier.tar.gz'), 'earlier bytes');+ controls.tar = (_command, _args, _options, callback) => callback(new Error('synthetic tar failure'));+ await expect(archiveCategory('health')).rejects.toThrow('synthetic tar failure');+ expect(await readdir(health)).toHaveLength(2);+ expect(await readdir(backup)).toEqual(['earlier.tar.gz']);+ expect(await readFile(join(backup, 'earlier.tar.gz'), 'utf8')).toBe('earlier bytes');+ });++ it('refuses even a UUID collision without replacing an earlier archive or removing sources', async () => {+ await seedHealth();+ controls.id = '00000000-0000-4000-8000-000000000000';+ const first = await archiveCategory('health');+ const original = await readFile(resolve(first.archivePath));+ await expect(archiveCategory('health', { daysToKeep: 1 })).rejects.toMatchObject({ code: 'EEXIST' });+ expect(await readFile(resolve(first.archivePath))).toEqual(original);+ expect(await readdir(health)).toEqual(['2026-10-01.json']);+ expect(await readdir(backup)).toEqual([basename(first.archivePath)]);+ });++ it('keeps health sources when syncing archive bytes fails', async () => {+ await seedHealth();+ const close = vi.fn();+ controls.open = async () => ({ sync: async () => { throw new Error('synthetic sync failure'); }, close });+ await expect(archiveCategory('health')).rejects.toThrow('synthetic sync failure');+ expect(close).toHaveBeenCalledOnce();+ expect(await readdir(health)).toHaveLength(2);+ expect(await readdir(backup)).toEqual([]);+ });+ it.skipIf(process.platform === 'win32')('keeps health sources if the published directory cannot be synced', async () => {+ await seedHealth();+ const actual = await vi.importActual('fs/promises');+ controls.open = async (path, ...args) => path === backup+ ? { sync: async () => { throw new Error('synthetic directory sync failure'); }, close: async () => {} }+ : actual.open(path, ...args);+ await expect(archiveCategory('health')).rejects.toThrow('synthetic directory sync failure');+ expect(await readdir(health)).toHaveLength(2);+ const archives = await getBackups();+ expect(archives).toHaveLength(1);+ expect(await archiveBytes({ archivePath: join(backup, archives[0].name) }, '2024-01-01.json')).toBe('{"example":"older"}');+ });++});diff --git a/server/services/dataManager.js b/server/services/dataManager.jsindex a63b601b54..1798bb5347 100644--- a/server/services/dataManager.js+++ b/server/services/dataManager.js@@ -1,6 +1,8 @@-import { readdir, opendir, stat, lstat, statfs } from 'fs/promises';+import { readdir, opendir, stat, lstat, statfs, mkdtemp, open, link } from 'fs/promises';import { join, relative, resolve, isAbsolute } from 'path';import { existsSync } from 'fs';+import { randomUUID } from 'crypto';+import { assertNotRealDataWrite } from '../lib/testDataIsolation.js';import { execFile } from '../lib/childProcess.js';import { promisify } from 'util';import { parseFilesystemStats } from '../lib/fileCore.js';@@ -407,44 +409,56 @@ export async function archiveCategory(categoryKey, options = {}) {const backupDir = join(DATA_DIR, 'backup');await ensureDir(backupDir);+ // Reserve scratch before selecting inputs: even a fixed clock cannot make+ // separate requests share their file list or partially written archive.+ assertNotRealDataWrite(backupDir, 'archiveCategory');+ const operationDir = await mkdtemp(join(backupDir, '.archive-'));+ const stagedArchive = join(operationDir, 'archive.tar.gz');const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);- const archiveName = `${categoryKey}-${timestamp}.tar.gz`;+ const archiveName = `${categoryKey}-${timestamp}-${randomUUID()}.tar.gz`;const archivePath = join(backupDir, archiveName);- // Date-based archiving for daily-file categories (health)- if (categoryKey === 'health') {- const daysToKeep = options.daysToKeep ?? 365;- const cutoff = new Date();- cutoff.setDate(cutoff.getDate() - daysToKeep);- const cutoffStr = cutoff.toISOString().slice(0, 10);-- const files = await readdir(dirPath).catch(() => []);- const oldFiles = files.filter(f => f.endsWith('.json') && f.slice(0, 10) < cutoffStr);- if (oldFiles.length === 0) return { archived: 0, archivePath: null, message: 'No old files to archive' };+ try {+ let oldFiles = [];+ if (categoryKey === 'health') {+ const daysToKeep = options.daysToKeep ?? 365;+ const cutoff = new Date();+ cutoff.setDate(cutoff.getDate() - daysToKeep);+ const cutoffStr = cutoff.toISOString().slice(0, 10);++ const files = await readdir(dirPath);+ oldFiles = files.filter(f => f.endsWith('.json') && f.slice(0, 10) < cutoffStr);+ if (oldFiles.length === 0) return { archived: 0, archivePath: null, message: 'No old files to archive' };++ const listPath = join(operationDir, 'files.txt');+ await writeFileGuarded(listPath, oldFiles.join('\n'));+ await execFileAsync('tar', ['-czf', stagedArchive, '-C', dirPath, '-T', listPath], { timeout: 120000 });+ } else {+ await execFileAsync('tar', ['-czf', stagedArchive, '-C', DATA_DIR, categoryKey], { timeout: 120000 });+ }- // Write file list to temp file to avoid shell argument limits- const listPath = join(backupDir, `.filelist-${Date.now()}.txt`);- await writeFileGuarded(listPath, oldFiles.join('\n'));- await execFileAsync('tar', ['-czf', archivePath, '-C', dirPath, '-T', listPath], { timeout: 120000 });- await rmGuarded(listPath).catch(() => {});+ // A hard link publishes complete bytes atomically and refuses an existing+ // destination. Sync bytes first, then the published directory entry before+ // deleting health sources. Both paths live on the same filesystem.+ const archiveFile = await open(stagedArchive, 'r');+ try { await archiveFile.sync(); } finally { await archiveFile.close(); }+ await link(stagedArchive, archivePath);+ // Node does not expose directory fsync on Windows.+ if (process.platform !== 'win32') {+ const directory = await open(backupDir, 'r');+ try { await directory.sync(); } finally { await directory.close(); }+ }+ const archiveStat = await stat(archivePath);for (const f of oldFiles) {await rmGuarded(join(dirPath, f)).catch(() => {});}-- const archiveStat = await stat(archivePath).catch(() => null);- return { archived: oldFiles.length, archivePath: relative(process.cwd(), archivePath), size: archiveStat?.size || 0 };+ return categoryKey === 'health'+ ? { archived: oldFiles.length, archivePath: relative(process.cwd(), archivePath), size: archiveStat.size }+ : { archived: 0, archivePath: relative(process.cwd(), archivePath), archiveSize: archiveStat.size };+ } finally {+ await rmGuarded(operationDir, { recursive: true, force: true });}-- // Generic: archive entire category contents- await execFileAsync('tar', ['-czf', archivePath, '-C', DATA_DIR, categoryKey], { timeout: 120000 });- const archiveStat = await stat(archivePath).catch(() => null);-- return {- archived: 0,- archivePath: relative(process.cwd(), archivePath),- archiveSize: archiveStat?.size || 0- };}export async function purgeCategory(categoryKey, options = {}) {