Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve every health recovery archive when requests share a timestamp (#10697)

Adam Eivy6f6b918b75a9
diff --git a/server/services/dataManager.archive.test.js b/server/services/dataManager.archive.test.js
new file mode 100644
index 0000000000..37054232a9
--- /dev/null
+++ b/server/services/dataManager.archive.test.js
@@ -0,0 +1,141 @@
+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdir, readFile, readdir, rm, writeFile } from 'fs/promises';
+import { basename, dirname, join, resolve } from 'path';
+import { execFileSync } from '../lib/childProcess.js';
+import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';
+
+const controls = vi.hoisted(() => ({ open: null, tar: null, id: null, tarArgs: [] }));
+vi.mock('../lib/fileUtils.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ return makePathsProxy(actual, { dataRoot: () => root });
+});
+vi.mock('fs/promises', async (importOriginal) => {
+ const actual = await importOriginal();
+ return {
+ ...actual,
+ open: (...args) => controls.open ? controls.open(...args) : actual.open(...args),
+ };
+});
+vi.mock('crypto', async (importOriginal) => {
+ const actual = await importOriginal();
+ return { ...actual, randomUUID: () => controls.id ?? actual.randomUUID() };
+});
+vi.mock('../lib/childProcess.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ return { ...actual, execFile: (command, args, options, callback) => {
+ controls.tarArgs.push(args);
+ return controls.tar ? controls.tar(command, args, options, callback) : actual.execFile(command, args, options, callback);
+ } };
+});
+
+const root = createTempDataRoot('portos-data-manager-archive-');
+const { archiveCategory, getBackups, deleteBackup } = await import('./dataManager.js');
+const health = join(root, 'health');
+const backup = join(root, 'backup');
+const archiveBytes = (result, file) => execFileSync('tar', ['-xOzf', resolve(result.archivePath), file], { encoding: 'utf8' });
+const seedHealth = async () => {
+ await mkdir(health, { recursive: true });
+ await writeFile(join(health, '2024-01-01.json'), '{"example":"older"}');
+ await writeFile(join(health, '2026-10-01.json'), '{"example":"recent"}');
+};
+
+beforeEach(async () => {
+ await rm(root, { recursive: true, force: true });
+ await mkdir(root, { recursive: true });
+ Object.assign(controls, { open: null, tar: null, id: null, tarArgs: [] });
+ vi.useFakeTimers({ toFake: ['Date'] });
+ vi.setSystemTime(new Date('2026-10-08T12:00:00Z'));
+});
+afterEach(() => vi.useRealTimers());
+afterAll(async () => { await rm(root, { recursive: true, force: true }); });
+
+describe('Data Manager recovery archives (#10697)', () => {
+ it('keeps both same-second health selections recoverable and supports new and legacy archive names', async () => {
+ await seedHealth();
+ const first = await archiveCategory('health', { daysToKeep: 365 });
+ const second = await archiveCategory('health', { daysToKeep: 1 });
+ expect(first.archivePath).not.toBe(second.archivePath);
+ expect(await archiveBytes(first, '2024-01-01.json')).toBe('{"example":"older"}');
+ expect(await archiveBytes(second, '2026-10-01.json')).toBe('{"example":"recent"}');
+ expect(await readdir(health)).toEqual([]);
+ const legacy = 'health-2026-01-01T00-00-00.tar.gz';
+ await writeFile(join(backup, legacy), 'legacy fixture');
+ expect((await getBackups()).map(row => row.name).sort()).toEqual([basename(first.archivePath), basename(second.archivePath), legacy].sort());
+ for (const name of [basename(first.archivePath), basename(second.archivePath), legacy]) await deleteBackup(name);
+ expect(await readdir(backup)).toEqual([]);
+ });
+
+ it('reserves separate input lists and staging archives for concurrent health requests', async () => {
+ await seedHealth();
+ const actual = await vi.importActual('../lib/childProcess.js');
+ const completions = [];
+ // Hold successful tar callbacks until both have read their selected inputs:
+ // source-import/deletion coordination is a separate lifecycle issue.
+ controls.tar = (command, args, options, callback) => actual.execFile(command, args, options, (error, ...output) => {
+ completions.push(() => callback(error, ...output));
+ if (completions.length === 2) completions.forEach(finish => finish());
+ });
+ const results = await Promise.all([archiveCategory('health'), archiveCategory('health', { daysToKeep: 1 })]);
+ expect(new Set(controls.tarArgs.map(args => dirname(args[1]))).size).toBe(2);
+ expect(new Set(controls.tarArgs.map(args => args[args.indexOf('-T') + 1])).size).toBe(2);
+ expect(await archiveBytes(results[0], '2024-01-01.json')).toBe('{"example":"older"}');
+ expect(await archiveBytes(results[1], '2026-10-01.json')).toBe('{"example":"recent"}');
+ expect((await getBackups()).map(row => row.name).sort()).toEqual(results.map(result => basename(result.archivePath)).sort());
+ expect((await readdir(backup)).every(name => name.endsWith('.tar.gz'))).toBe(true);
+ });
+
+ it('also preserves successive generic category archives in the same second', async () => {
+ await mkdir(join(root, 'calendar'));
+ await writeFile(join(root, 'calendar', 'example.json'), 'first example');
+ const first = await archiveCategory('calendar');
+ await writeFile(join(root, 'calendar', 'example.json'), 'second example');
+ const second = await archiveCategory('calendar');
+ expect(await archiveBytes(first, 'calendar/example.json')).toBe('first example');
+ expect(await archiveBytes(second, 'calendar/example.json')).toBe('second example');
+ });
+
+ it('cleans only its scratch and keeps health sources when tar fails', async () => {
+ await seedHealth();
+ await mkdir(backup);
+ await writeFile(join(backup, 'earlier.tar.gz'), 'earlier bytes');
+ controls.tar = (_command, _args, _options, callback) => callback(new Error('synthetic tar failure'));
+ await expect(archiveCategory('health')).rejects.toThrow('synthetic tar failure');
+ expect(await readdir(health)).toHaveLength(2);
+ expect(await readdir(backup)).toEqual(['earlier.tar.gz']);
+ expect(await readFile(join(backup, 'earlier.tar.gz'), 'utf8')).toBe('earlier bytes');
+ });
+
+ it('refuses even a UUID collision without replacing an earlier archive or removing sources', async () => {
+ await seedHealth();
+ controls.id = '00000000-0000-4000-8000-000000000000';
+ const first = await archiveCategory('health');
+ const original = await readFile(resolve(first.archivePath));
+ await expect(archiveCategory('health', { daysToKeep: 1 })).rejects.toMatchObject({ code: 'EEXIST' });
+ expect(await readFile(resolve(first.archivePath))).toEqual(original);
+ expect(await readdir(health)).toEqual(['2026-10-01.json']);
+ expect(await readdir(backup)).toEqual([basename(first.archivePath)]);
+ });
+
+ it('keeps health sources when syncing archive bytes fails', async () => {
+ await seedHealth();
+ const close = vi.fn();
+ controls.open = async () => ({ sync: async () => { throw new Error('synthetic sync failure'); }, close });
+ await expect(archiveCategory('health')).rejects.toThrow('synthetic sync failure');
+ expect(close).toHaveBeenCalledOnce();
+ expect(await readdir(health)).toHaveLength(2);
+ expect(await readdir(backup)).toEqual([]);
+ });
+ it.skipIf(process.platform === 'win32')('keeps health sources if the published directory cannot be synced', async () => {
+ await seedHealth();
+ const actual = await vi.importActual('fs/promises');
+ controls.open = async (path, ...args) => path === backup
+ ? { sync: async () => { throw new Error('synthetic directory sync failure'); }, close: async () => {} }
+ : actual.open(path, ...args);
+ await expect(archiveCategory('health')).rejects.toThrow('synthetic directory sync failure');
+ expect(await readdir(health)).toHaveLength(2);
+ const archives = await getBackups();
+ expect(archives).toHaveLength(1);
+ expect(await archiveBytes({ archivePath: join(backup, archives[0].name) }, '2024-01-01.json')).toBe('{"example":"older"}');
+ });
+
+});
diff --git a/server/services/dataManager.js b/server/services/dataManager.js
index a63b601b54..1798bb5347 100644
--- a/server/services/dataManager.js
+++ b/server/services/dataManager.js
@@ -1,6 +1,8 @@
-import { readdir, opendir, stat, lstat, statfs } from 'fs/promises';
+import { readdir, opendir, stat, lstat, statfs, mkdtemp, open, link } from 'fs/promises';
import { join, relative, resolve, isAbsolute } from 'path';
import { existsSync } from 'fs';
+import { randomUUID } from 'crypto';
+import { assertNotRealDataWrite } from '../lib/testDataIsolation.js';
import { execFile } from '../lib/childProcess.js';
import { promisify } from 'util';
import { parseFilesystemStats } from '../lib/fileCore.js';
@@ -407,44 +409,56 @@ export async function archiveCategory(categoryKey, options = {}) {
const backupDir = join(DATA_DIR, 'backup');
await ensureDir(backupDir);
+ // Reserve scratch before selecting inputs: even a fixed clock cannot make
+ // separate requests share their file list or partially written archive.
+ assertNotRealDataWrite(backupDir, 'archiveCategory');
+ const operationDir = await mkdtemp(join(backupDir, '.archive-'));
+ const stagedArchive = join(operationDir, 'archive.tar.gz');
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
- const archiveName = `${categoryKey}-${timestamp}.tar.gz`;
+ const archiveName = `${categoryKey}-${timestamp}-${randomUUID()}.tar.gz`;
const archivePath = join(backupDir, archiveName);
- // Date-based archiving for daily-file categories (health)
- if (categoryKey === 'health') {
- const daysToKeep = options.daysToKeep ?? 365;
- const cutoff = new Date();
- cutoff.setDate(cutoff.getDate() - daysToKeep);
- const cutoffStr = cutoff.toISOString().slice(0, 10);
-
- const files = await readdir(dirPath).catch(() => []);
- const oldFiles = files.filter(f => f.endsWith('.json') && f.slice(0, 10) < cutoffStr);
- if (oldFiles.length === 0) return { archived: 0, archivePath: null, message: 'No old files to archive' };
+ try {
+ let oldFiles = [];
+ if (categoryKey === 'health') {
+ const daysToKeep = options.daysToKeep ?? 365;
+ const cutoff = new Date();
+ cutoff.setDate(cutoff.getDate() - daysToKeep);
+ const cutoffStr = cutoff.toISOString().slice(0, 10);
+
+ const files = await readdir(dirPath);
+ oldFiles = files.filter(f => f.endsWith('.json') && f.slice(0, 10) < cutoffStr);
+ if (oldFiles.length === 0) return { archived: 0, archivePath: null, message: 'No old files to archive' };
+
+ const listPath = join(operationDir, 'files.txt');
+ await writeFileGuarded(listPath, oldFiles.join('\n'));
+ await execFileAsync('tar', ['-czf', stagedArchive, '-C', dirPath, '-T', listPath], { timeout: 120000 });
+ } else {
+ await execFileAsync('tar', ['-czf', stagedArchive, '-C', DATA_DIR, categoryKey], { timeout: 120000 });
+ }
- // Write file list to temp file to avoid shell argument limits
- const listPath = join(backupDir, `.filelist-${Date.now()}.txt`);
- await writeFileGuarded(listPath, oldFiles.join('\n'));
- await execFileAsync('tar', ['-czf', archivePath, '-C', dirPath, '-T', listPath], { timeout: 120000 });
- await rmGuarded(listPath).catch(() => {});
+ // A hard link publishes complete bytes atomically and refuses an existing
+ // destination. Sync bytes first, then the published directory entry before
+ // deleting health sources. Both paths live on the same filesystem.
+ const archiveFile = await open(stagedArchive, 'r');
+ try { await archiveFile.sync(); } finally { await archiveFile.close(); }
+ await link(stagedArchive, archivePath);
+ // Node does not expose directory fsync on Windows.
+ if (process.platform !== 'win32') {
+ const directory = await open(backupDir, 'r');
+ try { await directory.sync(); } finally { await directory.close(); }
+ }
+ const archiveStat = await stat(archivePath);
for (const f of oldFiles) {
await rmGuarded(join(dirPath, f)).catch(() => {});
}
-
- const archiveStat = await stat(archivePath).catch(() => null);
- return { archived: oldFiles.length, archivePath: relative(process.cwd(), archivePath), size: archiveStat?.size || 0 };
+ return categoryKey === 'health'
+ ? { archived: oldFiles.length, archivePath: relative(process.cwd(), archivePath), size: archiveStat.size }
+ : { archived: 0, archivePath: relative(process.cwd(), archivePath), archiveSize: archiveStat.size };
+ } finally {
+ await rmGuarded(operationDir, { recursive: true, force: true });
}
-
- // Generic: archive entire category contents
- await execFileAsync('tar', ['-czf', archivePath, '-C', DATA_DIR, categoryKey], { timeout: 120000 });
- const archiveStat = await stat(archivePath).catch(() => null);
-
- return {
- archived: 0,
- archivePath: relative(process.cwd(), archivePath),
- archiveSize: archiveStat?.size || 0
- };
}
export async function purgeCategory(categoryKey, options = {}) {