Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve health imports during archival (#10698)
Adam Eivyae10562799d3
diff --git a/server/services/appleHealthIngest.js b/server/services/appleHealthIngest.jsindex 2af41715be..03e8de234d 100644--- a/server/services/appleHealthIngest.js+++ b/server/services/appleHealthIngest.js@@ -15,8 +15,8 @@ import { createKeyedFileWriteQueue } from '../lib/fileWriteQueue.js';/*** Per-date write queue to serialize read-modify-write cycles.* Keyed by date string (YYYY-MM-DD) so different days fan out in parallel- * while writes to the same day serialize. Shared with the XML importer so a- * JSON ingest and an XML import can never interleave on the same day file.+ * while writes to the same day serialize. Shared with XML import and archival+ * capture/removal so every day-file mutation sees the latest committed bytes.*/export const queueDayWrite = createKeyedFileWriteQueue();diff --git a/server/services/dataManager.archive.test.js b/server/services/dataManager.archive.test.jsindex 6e8fbe5c1a..e26f6660af 100644--- a/server/services/dataManager.archive.test.js+++ b/server/services/dataManager.archive.test.js@@ -4,7 +4,7 @@ import { basename, dirname, join, resolve } from 'path';import { execFileSync } from '../lib/childProcess.js';import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';-const controls = vi.hoisted(() => ({ open: null, tar: null, id: null, tarArgs: [] }));+const controls = vi.hoisted(() => ({ open: null, tar: null, id: null, tarArgs: [], readFile: null, rm: null, queued: null }));vi.mock('../lib/fileUtils.js', async (importOriginal) => {const actual = await importOriginal();return makePathsProxy(actual, { dataRoot: () => root });@@ -13,9 +13,18 @@ vi.mock('fs/promises', async (importOriginal) => {const actual = await importOriginal();return {...actual,+ readFile: (...args) => controls.readFile ? controls.readFile(...args) : actual.readFile(...args),+ rm: (...args) => controls.rm ? controls.rm(...args) : actual.rm(...args),open: (...args) => controls.open ? controls.open(...args) : actual.open(...args),};});+vi.mock('./appleHealthIngest.js', async (importOriginal) => {+ const actual = await importOriginal();+ return { ...actual, queueDayWrite: (...args) => {+ controls.queued?.(...args);+ return actual.queueDayWrite(...args);+ } };+});vi.mock('crypto', async (importOriginal) => {const actual = await importOriginal();return { ...actual, randomUUID: () => controls.id ?? actual.randomUUID() };@@ -30,6 +39,8 @@ vi.mock('../lib/childProcess.js', async (importOriginal) => {const root = createTempDataRoot('portos-data-manager-archive-');const { archiveCategory, getBackups, deleteBackup } = await import('./dataManager.js');+const { ingestHealthData } = await import('./appleHealthIngest.js');+const { importAppleHealthXml } = await import('./appleHealthXml.js');const health = join(root, 'health');const backup = join(root, 'backup');const archiveBytes = (result, file) => execFileSync('tar', ['-xOzf', resolve(result.archivePath), file], { encoding: 'utf8' });@@ -42,7 +53,7 @@ const seedHealth = async () => {beforeEach(async () => {await rm(root, { recursive: true, force: true });await mkdir(root, { recursive: true });- Object.assign(controls, { open: null, tar: null, id: null, tarArgs: [] });+ Object.assign(controls, { open: null, tar: null, id: null, tarArgs: [], readFile: null, rm: null, queued: null });vi.useFakeTimers({ toFake: ['Date'] });vi.setSystemTime(new Date('2026-10-08T12:00:00Z'));});@@ -57,6 +68,7 @@ describe('Data Manager recovery archives (#10697)', () => {expect(first.archivePath).not.toBe(second.archivePath);expect(await archiveBytes(first, '2024-01-01.json')).toBe('{"example":"older"}');expect(await archiveBytes(second, '2026-10-01.json')).toBe('{"example":"recent"}');+ expect(first).toMatchObject({ archived: 1, removed: 1, retained: 0, alreadyAbsent: 0 });expect(await readdir(health)).toEqual([]);const legacy = 'health-2026-01-01T00-00-00.tar.gz';await writeFile(join(backup, legacy), 'legacy fixture');@@ -70,7 +82,7 @@ describe('Data Manager recovery archives (#10697)', () => {const actual = await vi.importActual('../lib/childProcess.js');const completions = [];// Hold successful tar callbacks until both have read their selected inputs:- // source-import/deletion coordination is a separate lifecycle issue.+ // each request must preserve its own captured bytes and count its own removals.controls.tar = (command, args, options, callback) => actual.execFile(command, args, options, (error, ...output) => {completions.push(() => callback(error, ...output));if (completions.length === 2) completions.forEach(finish => finish());@@ -80,6 +92,8 @@ describe('Data Manager recovery archives (#10697)', () => {expect(new Set(controls.tarArgs.map(args => args[args.indexOf('-T') + 1])).size).toBe(2);expect(await archiveBytes(results[0], '2024-01-01.json')).toBe('{"example":"older"}');expect(await archiveBytes(results[1], '2026-10-01.json')).toBe('{"example":"recent"}');+ expect(results.reduce((total, result) => total + result.removed, 0)).toBe(2);+ expect(results.reduce((total, result) => total + result.alreadyAbsent, 0)).toBe(1);expect((await getBackups()).map(row => row.name).sort()).toEqual(results.map(result => basename(result.archivePath)).sort());expect((await readdir(backup)).every(name => name.endsWith('.tar.gz'))).toBe(true);});@@ -160,3 +174,140 @@ describe('Data Manager recovery archives (#10697)', () => {});});++const day = '2024-01-01';+const liveDay = () => join(health, `${day}.json`);+const originalPoint = { date: `${day} 08:00:00 +0000`, qty: 60 };+const newPoint = { date: `${day} 09:00:00 +0000`, qty: 72 };+const seedDay = async () => {+ await mkdir(health, { recursive: true });+ await writeFile(liveDay(), JSON.stringify({ date: day, metrics: { heart_rate: [originalPoint] } }));+};+const writers = {+ JSON: () => ingestHealthData({ data: { metrics: [{ name: 'heart_rate', data: [newPoint] }] } }),+ XML: async () => {+ const xmlPath = join(root, 'export.xml');+ await writeFile(xmlPath, `<HealthData><Record type="HKQuantityTypeIdentifierHeartRate" sourceName="Watch" unit="count/min" startDate="${newPoint.date}" endDate="${newPoint.date}" value="72"/></HealthData>`);+ return importAppleHealthXml(xmlPath);+ },+};+const deferred = () => {+ let resolve;+ const promise = new Promise(done => { resolve = done; });+ return { promise, resolve };+};++describe('health archive/import coordination (#10698)', () => {+ for (const [kind, writeDay] of Object.entries(writers)) {+ it(`retains acknowledged ${kind} updates committed while tar runs`, async () => {+ await seedDay();+ const actual = await vi.importActual('../lib/childProcess.js');+ const captured = deferred();+ const release = deferred();+ controls.tar = (command, args, options, callback) => actual.execFile(command, args, options, (error, ...output) => {+ captured.resolve();+ release.promise.then(() => callback(error, ...output));+ });+ const archive = archiveCategory('health');+ try {+ await captured.promise;+ await writeDay();+ } finally {+ release.resolve();+ }+ const result = await archive;+ expect(result).toMatchObject({ archived: 1, removed: 0, retained: 1, alreadyAbsent: 0 });+ expect(JSON.parse(archiveBytes(result, `${day}.json`)).metrics.heart_rate).toEqual([originalPoint]);+ expect(JSON.parse(await readFile(liveDay(), 'utf8')).metrics.heart_rate.map(point => point.qty)).toEqual([60, 72]);+ });++ it(`captures a ${kind} writer's acknowledged pre-image before removing the day`, async () => {+ await seedDay();+ const actual = await vi.importActual('fs/promises');+ const reading = deferred();+ const release = deferred();+ const archiveQueued = deferred();+ let held = false;+ controls.readFile = async (path, ...args) => {+ const bytes = await actual.readFile(path, ...args);+ if (path === liveDay() && !held) {+ held = true;+ reading.resolve();+ await release.promise;+ }+ return bytes;+ };+ const writer = writeDay();+ await reading.promise;+ controls.queued = () => archiveQueued.resolve();+ const archive = archiveCategory('health');+ try {+ await Promise.race([archiveQueued.promise, archive.then(() => { throw new Error('Archive skipped the day queue'); })]);+ } finally {+ release.resolve();+ }+ await writer;+ const result = await archive;+ expect(result).toMatchObject({ removed: 1, retained: 0 });+ expect(JSON.parse(archiveBytes(result, `${day}.json`)).metrics.heart_rate.map(point => point.qty)).toEqual([60, 72]);+ expect(await readdir(health)).toEqual([]);+ });++ it(`makes a ${kind} writer queued during deletion read the post-archive day`, async () => {+ await seedDay();+ const actual = await vi.importActual('fs/promises');+ const deleting = deferred();+ const release = deferred();+ controls.rm = async (path, ...args) => {+ if (path === liveDay()) {+ deleting.resolve();+ await release.promise;+ }+ return actual.rm(path, ...args);+ };+ const archive = archiveCategory('health');+ await deleting.promise;+ const writerQueued = deferred();+ controls.queued = () => writerQueued.resolve();+ const writer = writeDay();+ try {+ // JSON ingest enqueues synchronously. XML first streams its input.+ if (kind === 'XML') await writerQueued.promise;+ } finally {+ release.resolve();+ }+ const [result] = await Promise.all([archive, writer]);+ expect(result).toMatchObject({ removed: 1, retained: 0 });+ expect(JSON.parse(archiveBytes(result, `${day}.json`)).metrics.heart_rate).toEqual([originalPoint]);+ expect(JSON.parse(await readFile(liveDay(), 'utf8')).metrics.heart_rate.map(point => point.qty)).toEqual([72]);+ });+ }++ for (const phase of ['capture', 'comparison', 'removal']) {+ it(`propagates ${phase} failure without reporting successful removal`, async () => {+ await seedDay();+ const actual = await vi.importActual('fs/promises');+ let reads = 0;+ controls.readFile = async (path, ...args) => {+ if (path === liveDay()) {+ reads++;+ if ((phase === 'capture' && reads === 1) || (phase === 'comparison' && reads === 2)) {+ throw Object.assign(new Error(`synthetic ${phase} failure`), { code: 'EACCES' });+ }+ }+ return actual.readFile(path, ...args);+ };+ controls.rm = async (path, ...args) => {+ if (path === liveDay() && phase === 'removal') throw new Error('synthetic removal failure');+ return actual.rm(path, ...args);+ };+ await expect(archiveCategory('health')).rejects.toThrow(`synthetic ${phase} failure`);+ expect(await readdir(health)).toEqual([`${day}.json`]);+ const archives = await readdir(backup);+ expect(archives).toHaveLength(phase === 'capture' ? 0 : 1);+ if (archives.length) {+ expect(JSON.parse(archiveBytes({ archivePath: join(backup, archives[0]) }, `${day}.json`)).metrics.heart_rate).toEqual([originalPoint]);+ }+ });+ }+});diff --git a/server/services/dataManager.js b/server/services/dataManager.jsindex 4bf53919b1..2b0486ac05 100644--- a/server/services/dataManager.js+++ b/server/services/dataManager.js@@ -1,11 +1,11 @@-import { readdir, opendir, stat, lstat, statfs, mkdtemp, open, link } from 'fs/promises';+import { readdir, opendir, stat, lstat, statfs, mkdtemp, open, link, readFile } from 'fs/promises';import { join, relative, resolve, isAbsolute } from 'path';import { existsSync } from 'fs';import { randomUUID } from 'crypto';import { assertNotRealDataWrite } from '../lib/testDataIsolation.js';import { execFile } from '../lib/childProcess.js';import { promisify } from 'util';-import { parseFilesystemStats } from '../lib/fileCore.js';+import { parseFilesystemStats, sha256Text } from '../lib/fileCore.js';import { PATHS, ensureDir, isTopLevelEntryName, rmGuarded, writeFileGuarded } from '../lib/fileUtils.js';import { ServerError } from '../lib/errorHandler.js';import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';@@ -419,7 +419,8 @@ export async function archiveCategory(categoryKey, options = {}) {const archivePath = join(backupDir, archiveName);try {- let oldFiles = [];+ const capturedDays = [];+ let queueDayWrite;if (categoryKey === 'health') {const daysToKeep = options.daysToKeep ?? 365;const cutoff = new Date();@@ -427,12 +428,24 @@ export async function archiveCategory(categoryKey, options = {}) {const cutoffStr = cutoff.toISOString().slice(0, 10);const files = await readdir(dirPath);- oldFiles = files.filter(f => f.endsWith('.json') && f.slice(0, 10) < cutoffStr);- if (oldFiles.length === 0) return { archived: 0, archivePath: null, message: 'No old files to archive' };-+ const oldFiles = files.filter(f => /^\d{4}-\d{2}-\d{2}\.json$/.test(f) && f.slice(0, 10) < cutoffStr);+ if (oldFiles.length === 0) return { archived: 0, removed: 0, retained: 0, alreadyAbsent: 0, archivePath: null, message: 'No old files to archive' };++ // Share the importers' queue for capture and removal, but release it+ // during tar so a large archive does not block a historical backfill.+ ({ queueDayWrite } = await import('./appleHealthIngest.js'));+ const stagedDays = join(operationDir, 'days');+ await ensureDir(stagedDays);+ for (const file of oldFiles) {+ await queueDayWrite(file.slice(0, 10), async () => {+ const bytes = await readFile(join(dirPath, file));+ await writeFileGuarded(join(stagedDays, file), bytes, { flag: 'wx' });+ capturedDays.push({ file, hash: sha256Text(bytes) });+ });+ }const listPath = join(operationDir, 'files.txt');await writeFileGuarded(listPath, oldFiles.join('\n'));- await execFileAsync('tar', ['-czf', stagedArchive, '-C', dirPath, '-T', listPath], { timeout: 120000 });+ await execFileAsync('tar', ['-czf', stagedArchive, '-C', stagedDays, '-T', listPath], { timeout: 120000 });} else {await execFileAsync('tar', ['-czf', stagedArchive, '-C', DATA_DIR, categoryKey], { timeout: 120000 });}@@ -451,11 +464,29 @@ export async function archiveCategory(categoryKey, options = {}) {}const archiveStat = await stat(archivePath);- for (const f of oldFiles) {- await rmGuarded(join(dirPath, f)).catch(() => {});+ let removed = 0;+ let retained = 0;+ let alreadyAbsent = 0;+ for (const { file, hash } of capturedDays) {+ await queueDayWrite(file.slice(0, 10), async () => {+ const livePath = join(dirPath, file);+ const bytes = await readFile(livePath).catch(error => {+ // Another archive may already have removed this exact day.+ if (error.code === 'ENOENT') return null;+ throw error;+ });+ if (bytes === null) {+ alreadyAbsent++;+ } else if (sha256Text(bytes) !== hash) {+ retained++;+ } else {+ await rmGuarded(livePath);+ removed++;+ }+ });}return categoryKey === 'health'- ? { archived: oldFiles.length, archivePath: relative(process.cwd(), archivePath), size: archiveStat.size }+ ? { archived: capturedDays.length, removed, retained, alreadyAbsent, archivePath: relative(process.cwd(), archivePath), size: archiveStat.size }: { archived: 0, archivePath: relative(process.cwd(), archivePath), archiveSize: archiveStat.size };} finally {await rmGuarded(operationDir, { recursive: true, force: true });