Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve health imports during archival (#10698)

Adam Eivyae10562799d3
diff --git a/server/services/appleHealthIngest.js b/server/services/appleHealthIngest.js
index 2af41715be..03e8de234d 100644
--- a/server/services/appleHealthIngest.js
+++ b/server/services/appleHealthIngest.js
@@ -15,8 +15,8 @@ import { createKeyedFileWriteQueue } from '../lib/fileWriteQueue.js';
/**
* Per-date write queue to serialize read-modify-write cycles.
* Keyed by date string (YYYY-MM-DD) so different days fan out in parallel
- * while writes to the same day serialize. Shared with the XML importer so a
- * JSON ingest and an XML import can never interleave on the same day file.
+ * while writes to the same day serialize. Shared with XML import and archival
+ * capture/removal so every day-file mutation sees the latest committed bytes.
*/
export const queueDayWrite = createKeyedFileWriteQueue();
diff --git a/server/services/dataManager.archive.test.js b/server/services/dataManager.archive.test.js
index 6e8fbe5c1a..e26f6660af 100644
--- a/server/services/dataManager.archive.test.js
+++ b/server/services/dataManager.archive.test.js
@@ -4,7 +4,7 @@ import { basename, dirname, join, resolve } from 'path';
import { execFileSync } from '../lib/childProcess.js';
import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';
-const controls = vi.hoisted(() => ({ open: null, tar: null, id: null, tarArgs: [] }));
+const controls = vi.hoisted(() => ({ open: null, tar: null, id: null, tarArgs: [], readFile: null, rm: null, queued: null }));
vi.mock('../lib/fileUtils.js', async (importOriginal) => {
const actual = await importOriginal();
return makePathsProxy(actual, { dataRoot: () => root });
@@ -13,9 +13,18 @@ vi.mock('fs/promises', async (importOriginal) => {
const actual = await importOriginal();
return {
...actual,
+ readFile: (...args) => controls.readFile ? controls.readFile(...args) : actual.readFile(...args),
+ rm: (...args) => controls.rm ? controls.rm(...args) : actual.rm(...args),
open: (...args) => controls.open ? controls.open(...args) : actual.open(...args),
};
});
+vi.mock('./appleHealthIngest.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ return { ...actual, queueDayWrite: (...args) => {
+ controls.queued?.(...args);
+ return actual.queueDayWrite(...args);
+ } };
+});
vi.mock('crypto', async (importOriginal) => {
const actual = await importOriginal();
return { ...actual, randomUUID: () => controls.id ?? actual.randomUUID() };
@@ -30,6 +39,8 @@ vi.mock('../lib/childProcess.js', async (importOriginal) => {
const root = createTempDataRoot('portos-data-manager-archive-');
const { archiveCategory, getBackups, deleteBackup } = await import('./dataManager.js');
+const { ingestHealthData } = await import('./appleHealthIngest.js');
+const { importAppleHealthXml } = await import('./appleHealthXml.js');
const health = join(root, 'health');
const backup = join(root, 'backup');
const archiveBytes = (result, file) => execFileSync('tar', ['-xOzf', resolve(result.archivePath), file], { encoding: 'utf8' });
@@ -42,7 +53,7 @@ const seedHealth = async () => {
beforeEach(async () => {
await rm(root, { recursive: true, force: true });
await mkdir(root, { recursive: true });
- Object.assign(controls, { open: null, tar: null, id: null, tarArgs: [] });
+ Object.assign(controls, { open: null, tar: null, id: null, tarArgs: [], readFile: null, rm: null, queued: null });
vi.useFakeTimers({ toFake: ['Date'] });
vi.setSystemTime(new Date('2026-10-08T12:00:00Z'));
});
@@ -57,6 +68,7 @@ describe('Data Manager recovery archives (#10697)', () => {
expect(first.archivePath).not.toBe(second.archivePath);
expect(await archiveBytes(first, '2024-01-01.json')).toBe('{"example":"older"}');
expect(await archiveBytes(second, '2026-10-01.json')).toBe('{"example":"recent"}');
+ expect(first).toMatchObject({ archived: 1, removed: 1, retained: 0, alreadyAbsent: 0 });
expect(await readdir(health)).toEqual([]);
const legacy = 'health-2026-01-01T00-00-00.tar.gz';
await writeFile(join(backup, legacy), 'legacy fixture');
@@ -70,7 +82,7 @@ describe('Data Manager recovery archives (#10697)', () => {
const actual = await vi.importActual('../lib/childProcess.js');
const completions = [];
// Hold successful tar callbacks until both have read their selected inputs:
- // source-import/deletion coordination is a separate lifecycle issue.
+ // each request must preserve its own captured bytes and count its own removals.
controls.tar = (command, args, options, callback) => actual.execFile(command, args, options, (error, ...output) => {
completions.push(() => callback(error, ...output));
if (completions.length === 2) completions.forEach(finish => finish());
@@ -80,6 +92,8 @@ describe('Data Manager recovery archives (#10697)', () => {
expect(new Set(controls.tarArgs.map(args => args[args.indexOf('-T') + 1])).size).toBe(2);
expect(await archiveBytes(results[0], '2024-01-01.json')).toBe('{"example":"older"}');
expect(await archiveBytes(results[1], '2026-10-01.json')).toBe('{"example":"recent"}');
+ expect(results.reduce((total, result) => total + result.removed, 0)).toBe(2);
+ expect(results.reduce((total, result) => total + result.alreadyAbsent, 0)).toBe(1);
expect((await getBackups()).map(row => row.name).sort()).toEqual(results.map(result => basename(result.archivePath)).sort());
expect((await readdir(backup)).every(name => name.endsWith('.tar.gz'))).toBe(true);
});
@@ -160,3 +174,140 @@ describe('Data Manager recovery archives (#10697)', () => {
});
});
+
+const day = '2024-01-01';
+const liveDay = () => join(health, `${day}.json`);
+const originalPoint = { date: `${day} 08:00:00 +0000`, qty: 60 };
+const newPoint = { date: `${day} 09:00:00 +0000`, qty: 72 };
+const seedDay = async () => {
+ await mkdir(health, { recursive: true });
+ await writeFile(liveDay(), JSON.stringify({ date: day, metrics: { heart_rate: [originalPoint] } }));
+};
+const writers = {
+ JSON: () => ingestHealthData({ data: { metrics: [{ name: 'heart_rate', data: [newPoint] }] } }),
+ XML: async () => {
+ const xmlPath = join(root, 'export.xml');
+ await writeFile(xmlPath, `<HealthData><Record type="HKQuantityTypeIdentifierHeartRate" sourceName="Watch" unit="count/min" startDate="${newPoint.date}" endDate="${newPoint.date}" value="72"/></HealthData>`);
+ return importAppleHealthXml(xmlPath);
+ },
+};
+const deferred = () => {
+ let resolve;
+ const promise = new Promise(done => { resolve = done; });
+ return { promise, resolve };
+};
+
+describe('health archive/import coordination (#10698)', () => {
+ for (const [kind, writeDay] of Object.entries(writers)) {
+ it(`retains acknowledged ${kind} updates committed while tar runs`, async () => {
+ await seedDay();
+ const actual = await vi.importActual('../lib/childProcess.js');
+ const captured = deferred();
+ const release = deferred();
+ controls.tar = (command, args, options, callback) => actual.execFile(command, args, options, (error, ...output) => {
+ captured.resolve();
+ release.promise.then(() => callback(error, ...output));
+ });
+ const archive = archiveCategory('health');
+ try {
+ await captured.promise;
+ await writeDay();
+ } finally {
+ release.resolve();
+ }
+ const result = await archive;
+ expect(result).toMatchObject({ archived: 1, removed: 0, retained: 1, alreadyAbsent: 0 });
+ expect(JSON.parse(archiveBytes(result, `${day}.json`)).metrics.heart_rate).toEqual([originalPoint]);
+ expect(JSON.parse(await readFile(liveDay(), 'utf8')).metrics.heart_rate.map(point => point.qty)).toEqual([60, 72]);
+ });
+
+ it(`captures a ${kind} writer's acknowledged pre-image before removing the day`, async () => {
+ await seedDay();
+ const actual = await vi.importActual('fs/promises');
+ const reading = deferred();
+ const release = deferred();
+ const archiveQueued = deferred();
+ let held = false;
+ controls.readFile = async (path, ...args) => {
+ const bytes = await actual.readFile(path, ...args);
+ if (path === liveDay() && !held) {
+ held = true;
+ reading.resolve();
+ await release.promise;
+ }
+ return bytes;
+ };
+ const writer = writeDay();
+ await reading.promise;
+ controls.queued = () => archiveQueued.resolve();
+ const archive = archiveCategory('health');
+ try {
+ await Promise.race([archiveQueued.promise, archive.then(() => { throw new Error('Archive skipped the day queue'); })]);
+ } finally {
+ release.resolve();
+ }
+ await writer;
+ const result = await archive;
+ expect(result).toMatchObject({ removed: 1, retained: 0 });
+ expect(JSON.parse(archiveBytes(result, `${day}.json`)).metrics.heart_rate.map(point => point.qty)).toEqual([60, 72]);
+ expect(await readdir(health)).toEqual([]);
+ });
+
+ it(`makes a ${kind} writer queued during deletion read the post-archive day`, async () => {
+ await seedDay();
+ const actual = await vi.importActual('fs/promises');
+ const deleting = deferred();
+ const release = deferred();
+ controls.rm = async (path, ...args) => {
+ if (path === liveDay()) {
+ deleting.resolve();
+ await release.promise;
+ }
+ return actual.rm(path, ...args);
+ };
+ const archive = archiveCategory('health');
+ await deleting.promise;
+ const writerQueued = deferred();
+ controls.queued = () => writerQueued.resolve();
+ const writer = writeDay();
+ try {
+ // JSON ingest enqueues synchronously. XML first streams its input.
+ if (kind === 'XML') await writerQueued.promise;
+ } finally {
+ release.resolve();
+ }
+ const [result] = await Promise.all([archive, writer]);
+ expect(result).toMatchObject({ removed: 1, retained: 0 });
+ expect(JSON.parse(archiveBytes(result, `${day}.json`)).metrics.heart_rate).toEqual([originalPoint]);
+ expect(JSON.parse(await readFile(liveDay(), 'utf8')).metrics.heart_rate.map(point => point.qty)).toEqual([72]);
+ });
+ }
+
+ for (const phase of ['capture', 'comparison', 'removal']) {
+ it(`propagates ${phase} failure without reporting successful removal`, async () => {
+ await seedDay();
+ const actual = await vi.importActual('fs/promises');
+ let reads = 0;
+ controls.readFile = async (path, ...args) => {
+ if (path === liveDay()) {
+ reads++;
+ if ((phase === 'capture' && reads === 1) || (phase === 'comparison' && reads === 2)) {
+ throw Object.assign(new Error(`synthetic ${phase} failure`), { code: 'EACCES' });
+ }
+ }
+ return actual.readFile(path, ...args);
+ };
+ controls.rm = async (path, ...args) => {
+ if (path === liveDay() && phase === 'removal') throw new Error('synthetic removal failure');
+ return actual.rm(path, ...args);
+ };
+ await expect(archiveCategory('health')).rejects.toThrow(`synthetic ${phase} failure`);
+ expect(await readdir(health)).toEqual([`${day}.json`]);
+ const archives = await readdir(backup);
+ expect(archives).toHaveLength(phase === 'capture' ? 0 : 1);
+ if (archives.length) {
+ expect(JSON.parse(archiveBytes({ archivePath: join(backup, archives[0]) }, `${day}.json`)).metrics.heart_rate).toEqual([originalPoint]);
+ }
+ });
+ }
+});
diff --git a/server/services/dataManager.js b/server/services/dataManager.js
index 4bf53919b1..2b0486ac05 100644
--- a/server/services/dataManager.js
+++ b/server/services/dataManager.js
@@ -1,11 +1,11 @@
-import { readdir, opendir, stat, lstat, statfs, mkdtemp, open, link } from 'fs/promises';
+import { readdir, opendir, stat, lstat, statfs, mkdtemp, open, link, readFile } from 'fs/promises';
import { join, relative, resolve, isAbsolute } from 'path';
import { existsSync } from 'fs';
import { randomUUID } from 'crypto';
import { assertNotRealDataWrite } from '../lib/testDataIsolation.js';
import { execFile } from '../lib/childProcess.js';
import { promisify } from 'util';
-import { parseFilesystemStats } from '../lib/fileCore.js';
+import { parseFilesystemStats, sha256Text } from '../lib/fileCore.js';
import { PATHS, ensureDir, isTopLevelEntryName, rmGuarded, writeFileGuarded } from '../lib/fileUtils.js';
import { ServerError } from '../lib/errorHandler.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
@@ -419,7 +419,8 @@ export async function archiveCategory(categoryKey, options = {}) {
const archivePath = join(backupDir, archiveName);
try {
- let oldFiles = [];
+ const capturedDays = [];
+ let queueDayWrite;
if (categoryKey === 'health') {
const daysToKeep = options.daysToKeep ?? 365;
const cutoff = new Date();
@@ -427,12 +428,24 @@ export async function archiveCategory(categoryKey, options = {}) {
const cutoffStr = cutoff.toISOString().slice(0, 10);
const files = await readdir(dirPath);
- oldFiles = files.filter(f => f.endsWith('.json') && f.slice(0, 10) < cutoffStr);
- if (oldFiles.length === 0) return { archived: 0, archivePath: null, message: 'No old files to archive' };
-
+ const oldFiles = files.filter(f => /^\d{4}-\d{2}-\d{2}\.json$/.test(f) && f.slice(0, 10) < cutoffStr);
+ if (oldFiles.length === 0) return { archived: 0, removed: 0, retained: 0, alreadyAbsent: 0, archivePath: null, message: 'No old files to archive' };
+
+ // Share the importers' queue for capture and removal, but release it
+ // during tar so a large archive does not block a historical backfill.
+ ({ queueDayWrite } = await import('./appleHealthIngest.js'));
+ const stagedDays = join(operationDir, 'days');
+ await ensureDir(stagedDays);
+ for (const file of oldFiles) {
+ await queueDayWrite(file.slice(0, 10), async () => {
+ const bytes = await readFile(join(dirPath, file));
+ await writeFileGuarded(join(stagedDays, file), bytes, { flag: 'wx' });
+ capturedDays.push({ file, hash: sha256Text(bytes) });
+ });
+ }
const listPath = join(operationDir, 'files.txt');
await writeFileGuarded(listPath, oldFiles.join('\n'));
- await execFileAsync('tar', ['-czf', stagedArchive, '-C', dirPath, '-T', listPath], { timeout: 120000 });
+ await execFileAsync('tar', ['-czf', stagedArchive, '-C', stagedDays, '-T', listPath], { timeout: 120000 });
} else {
await execFileAsync('tar', ['-czf', stagedArchive, '-C', DATA_DIR, categoryKey], { timeout: 120000 });
}
@@ -451,11 +464,29 @@ export async function archiveCategory(categoryKey, options = {}) {
}
const archiveStat = await stat(archivePath);
- for (const f of oldFiles) {
- await rmGuarded(join(dirPath, f)).catch(() => {});
+ let removed = 0;
+ let retained = 0;
+ let alreadyAbsent = 0;
+ for (const { file, hash } of capturedDays) {
+ await queueDayWrite(file.slice(0, 10), async () => {
+ const livePath = join(dirPath, file);
+ const bytes = await readFile(livePath).catch(error => {
+ // Another archive may already have removed this exact day.
+ if (error.code === 'ENOENT') return null;
+ throw error;
+ });
+ if (bytes === null) {
+ alreadyAbsent++;
+ } else if (sha256Text(bytes) !== hash) {
+ retained++;
+ } else {
+ await rmGuarded(livePath);
+ removed++;
+ }
+ });
}
return categoryKey === 'health'
- ? { archived: oldFiles.length, archivePath: relative(process.cwd(), archivePath), size: archiveStat.size }
+ ? { archived: capturedDays.length, removed, retained, alreadyAbsent, archivePath: relative(process.cwd(), archivePath), size: archiveStat.size }
: { archived: 0, archivePath: relative(process.cwd(), archivePath), archiveSize: archiveStat.size };
} finally {
await rmGuarded(operationDir, { recursive: true, force: true });