Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

perf: data overview scans each category once and shares the result (#12994)

[._.]/ Adam Eivydf8040bb4954
diff --git a/server/services/dataManager.js b/server/services/dataManager.js
index ac6daf4f0b..0ba19780b8 100644
--- a/server/services/dataManager.js
+++ b/server/services/dataManager.js
@@ -357,23 +357,75 @@ async function getDirSizeAndCount(dirPath, { strict = false } = {}) {
return { size: kb * 1024, fileCount };
}
-export async function getDataOverview({ strict = false } = {}) {
- // Capacity of the volume holding data/ so the page can show how much room is
- // left, not just what PortOS occupies. Null when the platform can't say.
- // Independent of the scans below, so it runs alongside them.
- const diskPromise = statfs(DATA_DIR).then(parseFilesystemStats).catch(() => null);
+// The scan is the multi-second part of the overview (du + a recursive walk per
+// category), and the Data page, the resources report and the data-introspection
+// harbor all request it. Share one result per `strict` mode for a short window
+// and coalesce concurrent callers; mutations here invalidate it. Rejections are
+// never cached. Busy state and disk capacity are read fresh on every call.
+const OVERVIEW_SCAN_TTL_MS = 30_000;
+const overviewScans = new Map();
+let overviewScanGeneration = 0;
+
+export function invalidateOverviewScan() {
+ overviewScanGeneration++;
+ overviewScans.clear();
+}
+
+async function scanDataTree(strict) {
const entries = await readdir(DATA_DIR, { withFileTypes: true }).catch((err) => {
if (strict) throw err;
return [];
});
const dirs = entries.filter(e => e.isDirectory());
-
- // Parallel: get total size + per-directory sizes in one batch
- const [totalResult, ...dirResults] = await Promise.all([
- getDirSizeAndCount(DATA_DIR, { strict }),
- ...dirs.map(d => getDirSizeAndCount(join(DATA_DIR, d.name), { strict }))
+ const topFiles = entries.filter(e => e.isFile());
+
+ // The root is the sum of its parts: scanning DATA_DIR itself would re-walk
+ // every category. Top-level files are sized by allocated blocks to match du.
+ const [dirResults, fileSizes] = await Promise.all([
+ Promise.all(dirs.map(d => getDirSizeAndCount(join(DATA_DIR, d.name), { strict }))),
+ Promise.all(topFiles.map(f => stat(join(DATA_DIR, f.name)).then(
+ st => (typeof st.blocks === 'number' ? st.blocks * 512 : st.size),
+ (err) => {
+ if (strict) throw err;
+ return 0;
+ },
+ ))),
]);
+ let totalSize = fileSizes.reduce((sum, n) => sum + n, 0);
+ let totalFileCount = topFiles.length;
+ for (const r of dirResults) {
+ totalSize += r.size;
+ // A failed non-strict walk reports a null count; keep the total unknown too.
+ totalFileCount = totalFileCount === null || r.fileCount === null ? null : totalFileCount + r.fileCount;
+ }
+ return { dirs, dirResults, totalSize, totalFileCount };
+}
+
+function getCachedOverviewScan(strict) {
+ const cached = overviewScans.get(strict);
+ if (cached && Date.now() - cached.at < OVERVIEW_SCAN_TTL_MS) return cached.promise;
+ const generation = overviewScanGeneration;
+ const promise = scanDataTree(strict);
+ const entry = { at: Date.now(), promise };
+ overviewScans.set(strict, entry);
+ promise.catch(() => {
+ if (overviewScans.get(strict) === entry) overviewScans.delete(strict);
+ });
+ // A mutation that landed mid-scan makes this result stale for later callers.
+ promise.then(() => {
+ if (generation !== overviewScanGeneration && overviewScans.get(strict) === entry) overviewScans.delete(strict);
+ }, () => {});
+ return promise;
+}
+
+export async function getDataOverview({ strict = false } = {}) {
+ // Capacity of the volume holding data/ so the page can show how much room is
+ // left, not just what PortOS occupies. Null when the platform can't say.
+ // Independent of the scan below, so it runs alongside it.
+ const diskPromise = statfs(DATA_DIR).then(parseFilesystemStats).catch(() => null);
+ const { dirs, dirResults, totalSize, totalFileCount } = await getCachedOverviewScan(strict);
+
// Busy state ships with the overview so the Data Manager can drop the Purge
// button before the user commits, rather than failing on click (#3342). Only
// the handful of categories carrying a `busyCheck` cost anything here.
@@ -392,8 +444,8 @@ export async function getDataOverview({ strict = false } = {}) {
const disk = await diskPromise;
return {
- totalSize: totalResult.size,
- totalFileCount: totalResult.fileCount,
+ totalSize,
+ totalFileCount,
categories,
disk,
dataDir: 'data'
@@ -461,6 +513,14 @@ export async function getCategoryDetail(categoryKey, { measure = false } = {}) {
}
export async function archiveCategory(categoryKey, options = {}) {
+ try {
+ return await archiveCategoryUncached(categoryKey, options);
+ } finally {
+ invalidateOverviewScan();
+ }
+}
+
+async function archiveCategoryUncached(categoryKey, options) {
if (!SAFE_NAME.test(categoryKey)) throw new ServerError('Invalid category name', { status: 400, code: 'VALIDATION_ERROR' });
const meta = CATEGORIES[categoryKey];
if (!meta?.archivable) {
@@ -564,6 +624,14 @@ export async function archiveCategory(categoryKey, options = {}) {
}
export async function purgeCategory(categoryKey, options = {}) {
+ try {
+ return await purgeCategoryUncached(categoryKey, options);
+ } finally {
+ invalidateOverviewScan();
+ }
+}
+
+async function purgeCategoryUncached(categoryKey, options) {
// A purge may retain external record references by explicit operator choice,
// but its deletions must never interleave with a snapshot's copy and dump.
return withBackupAssetPublication(async () => {
@@ -697,6 +765,7 @@ export async function deleteBackup(filename) {
throw new ServerError('Path traversal not allowed', { status: 400, code: 'VALIDATION_ERROR' });
}
await rmGuarded(fullPath);
+ invalidateOverviewScan();
console.log(`🗑️ Deleted backup archive ${filename}`);
return { deleted: filename };
}
diff --git a/server/services/dataManager.strict.test.js b/server/services/dataManager.strict.test.js
index 242c8ec012..d4969d9a55 100644
--- a/server/services/dataManager.strict.test.js
+++ b/server/services/dataManager.strict.test.js
@@ -1,5 +1,5 @@
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';
-import { mkdirSync, mkdtempSync, rmSync } from 'fs';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { makePathsProxy } from '../lib/mockPathsDataRoot.js';
@@ -42,11 +42,14 @@ vi.mock('./dataManagerBusy.js', () => ({
updateDetachedBusy: vi.fn(async () => ({ busy: false })),
}));
-const { getDataOverview } = await import('./dataManager.js');
+const { getDataOverview, deleteBackup, invalidateOverviewScan } = await import('./dataManager.js');
afterAll(() => rmSync(testDataRoot, { recursive: true, force: true }));
-beforeEach(() => {
+beforeEach(async () => {
+ // A strict scan rejects before its sibling category scans finish; let those settle so they don't spill into the next case's call log.
+ await new Promise(resolve => setTimeout(resolve, 10));
+ invalidateOverviewScan();
mocks.statErrorPath = null; mocks.countErrorPath = null; mocks.virtualCount = 0;
rmSync(testDataRoot, { recursive: true, force: true });
mkdirSync(join(testDataRoot, 'cache'), { recursive: true });
@@ -105,3 +108,39 @@ it('keeps byte totals with unavailable counts, and rejects enumeration errors in
expect(overview.totalFileCount).toBeNull();
await expect(getDataOverview({ strict: true })).rejects.toThrow('enumeration denied');
});
+
+describe('dataManager overview scan sharing (#12994)', () => {
+ const duTargets = () => mocks.execFile.mock.calls.filter(([command]) => command === 'du').map(([, args]) => args[args.length - 1]);
+
+ it('sums categories and top-level files without re-walking the data root', async () => {
+ writeFileSync(join(testDataRoot, 'settings.json'), '{}');
+ mocks.execFile.mockImplementation((command, args, _options, callback) => callback(null, { stdout: `4\t${args[args.length - 1]}\n`, stderr: '' }));
+
+ const overview = await getDataOverview();
+
+ expect(duTargets().sort()).toEqual([join(testDataRoot, 'cache'), join(testDataRoot, 'runs')]);
+ expect(overview.totalFileCount).toBe(1);
+ expect(overview.totalSize).toBeGreaterThanOrEqual(2 * 4 * 1024);
+ });
+
+ it('coalesces concurrent calls into one scan and drops it after a backup deletion', async () => {
+ mocks.execFile.mockImplementation((command, args, _options, callback) => callback(null, { stdout: `1\t${args[args.length - 1]}\n`, stderr: '' }));
+ mkdirSync(join(testDataRoot, 'backup'), { recursive: true });
+ const archive = 'cache-2026-01-01T00-00-00.tar.gz';
+ writeFileSync(join(testDataRoot, 'backup', archive), 'x');
+
+ await Promise.all([getDataOverview(), getDataOverview()]);
+ await getDataOverview();
+ expect(duTargets()).toHaveLength(3);
+
+ await deleteBackup(archive);
+ await getDataOverview();
+ expect(duTargets()).toHaveLength(6);
+ });
+
+ it('does not cache a rejected strict scan', async () => {
+ await expect(getDataOverview({ strict: true })).rejects.toThrow('permission denied');
+ mocks.execFile.mockImplementation((command, args, _options, callback) => callback(null, { stdout: `1\t${args[args.length - 1]}\n`, stderr: '' }));
+ await expect(getDataOverview({ strict: true })).resolves.toMatchObject({ dataDir: 'data' });
+ });
+});