Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
perf: data overview scans each category once and shares the result (#12994)
[._.]/ Adam Eivydf8040bb4954
diff --git a/server/services/dataManager.js b/server/services/dataManager.jsindex ac6daf4f0b..0ba19780b8 100644--- a/server/services/dataManager.js+++ b/server/services/dataManager.js@@ -357,23 +357,75 @@ async function getDirSizeAndCount(dirPath, { strict = false } = {}) {return { size: kb * 1024, fileCount };}-export async function getDataOverview({ strict = false } = {}) {- // Capacity of the volume holding data/ so the page can show how much room is- // left, not just what PortOS occupies. Null when the platform can't say.- // Independent of the scans below, so it runs alongside them.- const diskPromise = statfs(DATA_DIR).then(parseFilesystemStats).catch(() => null);+// The scan is the multi-second part of the overview (du + a recursive walk per+// category), and the Data page, the resources report and the data-introspection+// harbor all request it. Share one result per `strict` mode for a short window+// and coalesce concurrent callers; mutations here invalidate it. Rejections are+// never cached. Busy state and disk capacity are read fresh on every call.+const OVERVIEW_SCAN_TTL_MS = 30_000;+const overviewScans = new Map();+let overviewScanGeneration = 0;++export function invalidateOverviewScan() {+ overviewScanGeneration++;+ overviewScans.clear();+}++async function scanDataTree(strict) {const entries = await readdir(DATA_DIR, { withFileTypes: true }).catch((err) => {if (strict) throw err;return [];});const dirs = entries.filter(e => e.isDirectory());-- // Parallel: get total size + per-directory sizes in one batch- const [totalResult, ...dirResults] = await Promise.all([- getDirSizeAndCount(DATA_DIR, { strict }),- ...dirs.map(d => getDirSizeAndCount(join(DATA_DIR, d.name), { strict }))+ const topFiles = entries.filter(e => e.isFile());++ // The root is the sum of its parts: scanning DATA_DIR itself would re-walk+ // every category. Top-level files are sized by allocated blocks to match du.+ const [dirResults, fileSizes] = await Promise.all([+ Promise.all(dirs.map(d => getDirSizeAndCount(join(DATA_DIR, d.name), { strict }))),+ Promise.all(topFiles.map(f => stat(join(DATA_DIR, f.name)).then(+ st => (typeof st.blocks === 'number' ? st.blocks * 512 : st.size),+ (err) => {+ if (strict) throw err;+ return 0;+ },+ ))),]);+ let totalSize = fileSizes.reduce((sum, n) => sum + n, 0);+ let totalFileCount = topFiles.length;+ for (const r of dirResults) {+ totalSize += r.size;+ // A failed non-strict walk reports a null count; keep the total unknown too.+ totalFileCount = totalFileCount === null || r.fileCount === null ? null : totalFileCount + r.fileCount;+ }+ return { dirs, dirResults, totalSize, totalFileCount };+}++function getCachedOverviewScan(strict) {+ const cached = overviewScans.get(strict);+ if (cached && Date.now() - cached.at < OVERVIEW_SCAN_TTL_MS) return cached.promise;+ const generation = overviewScanGeneration;+ const promise = scanDataTree(strict);+ const entry = { at: Date.now(), promise };+ overviewScans.set(strict, entry);+ promise.catch(() => {+ if (overviewScans.get(strict) === entry) overviewScans.delete(strict);+ });+ // A mutation that landed mid-scan makes this result stale for later callers.+ promise.then(() => {+ if (generation !== overviewScanGeneration && overviewScans.get(strict) === entry) overviewScans.delete(strict);+ }, () => {});+ return promise;+}++export async function getDataOverview({ strict = false } = {}) {+ // Capacity of the volume holding data/ so the page can show how much room is+ // left, not just what PortOS occupies. Null when the platform can't say.+ // Independent of the scan below, so it runs alongside it.+ const diskPromise = statfs(DATA_DIR).then(parseFilesystemStats).catch(() => null);+ const { dirs, dirResults, totalSize, totalFileCount } = await getCachedOverviewScan(strict);+// Busy state ships with the overview so the Data Manager can drop the Purge// button before the user commits, rather than failing on click (#3342). Only// the handful of categories carrying a `busyCheck` cost anything here.@@ -392,8 +444,8 @@ export async function getDataOverview({ strict = false } = {}) {const disk = await diskPromise;return {- totalSize: totalResult.size,- totalFileCount: totalResult.fileCount,+ totalSize,+ totalFileCount,categories,disk,dataDir: 'data'@@ -461,6 +513,14 @@ export async function getCategoryDetail(categoryKey, { measure = false } = {}) {}export async function archiveCategory(categoryKey, options = {}) {+ try {+ return await archiveCategoryUncached(categoryKey, options);+ } finally {+ invalidateOverviewScan();+ }+}++async function archiveCategoryUncached(categoryKey, options) {if (!SAFE_NAME.test(categoryKey)) throw new ServerError('Invalid category name', { status: 400, code: 'VALIDATION_ERROR' });const meta = CATEGORIES[categoryKey];if (!meta?.archivable) {@@ -564,6 +624,14 @@ export async function archiveCategory(categoryKey, options = {}) {}export async function purgeCategory(categoryKey, options = {}) {+ try {+ return await purgeCategoryUncached(categoryKey, options);+ } finally {+ invalidateOverviewScan();+ }+}++async function purgeCategoryUncached(categoryKey, options) {// A purge may retain external record references by explicit operator choice,// but its deletions must never interleave with a snapshot's copy and dump.return withBackupAssetPublication(async () => {@@ -697,6 +765,7 @@ export async function deleteBackup(filename) {throw new ServerError('Path traversal not allowed', { status: 400, code: 'VALIDATION_ERROR' });}await rmGuarded(fullPath);+ invalidateOverviewScan();console.log(`🗑️ Deleted backup archive ${filename}`);return { deleted: filename };}diff --git a/server/services/dataManager.strict.test.js b/server/services/dataManager.strict.test.jsindex 242c8ec012..d4969d9a55 100644--- a/server/services/dataManager.strict.test.js+++ b/server/services/dataManager.strict.test.js@@ -1,5 +1,5 @@import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';-import { mkdirSync, mkdtempSync, rmSync } from 'fs';+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs';import { tmpdir } from 'os';import { join } from 'path';import { makePathsProxy } from '../lib/mockPathsDataRoot.js';@@ -42,11 +42,14 @@ vi.mock('./dataManagerBusy.js', () => ({updateDetachedBusy: vi.fn(async () => ({ busy: false })),}));-const { getDataOverview } = await import('./dataManager.js');+const { getDataOverview, deleteBackup, invalidateOverviewScan } = await import('./dataManager.js');afterAll(() => rmSync(testDataRoot, { recursive: true, force: true }));-beforeEach(() => {+beforeEach(async () => {+ // A strict scan rejects before its sibling category scans finish; let those settle so they don't spill into the next case's call log.+ await new Promise(resolve => setTimeout(resolve, 10));+ invalidateOverviewScan();mocks.statErrorPath = null; mocks.countErrorPath = null; mocks.virtualCount = 0;rmSync(testDataRoot, { recursive: true, force: true });mkdirSync(join(testDataRoot, 'cache'), { recursive: true });@@ -105,3 +108,39 @@ it('keeps byte totals with unavailable counts, and rejects enumeration errors inexpect(overview.totalFileCount).toBeNull();await expect(getDataOverview({ strict: true })).rejects.toThrow('enumeration denied');});++describe('dataManager overview scan sharing (#12994)', () => {+ const duTargets = () => mocks.execFile.mock.calls.filter(([command]) => command === 'du').map(([, args]) => args[args.length - 1]);++ it('sums categories and top-level files without re-walking the data root', async () => {+ writeFileSync(join(testDataRoot, 'settings.json'), '{}');+ mocks.execFile.mockImplementation((command, args, _options, callback) => callback(null, { stdout: `4\t${args[args.length - 1]}\n`, stderr: '' }));++ const overview = await getDataOverview();++ expect(duTargets().sort()).toEqual([join(testDataRoot, 'cache'), join(testDataRoot, 'runs')]);+ expect(overview.totalFileCount).toBe(1);+ expect(overview.totalSize).toBeGreaterThanOrEqual(2 * 4 * 1024);+ });++ it('coalesces concurrent calls into one scan and drops it after a backup deletion', async () => {+ mocks.execFile.mockImplementation((command, args, _options, callback) => callback(null, { stdout: `1\t${args[args.length - 1]}\n`, stderr: '' }));+ mkdirSync(join(testDataRoot, 'backup'), { recursive: true });+ const archive = 'cache-2026-01-01T00-00-00.tar.gz';+ writeFileSync(join(testDataRoot, 'backup', archive), 'x');++ await Promise.all([getDataOverview(), getDataOverview()]);+ await getDataOverview();+ expect(duTargets()).toHaveLength(3);++ await deleteBackup(archive);+ await getDataOverview();+ expect(duTargets()).toHaveLength(6);+ });++ it('does not cache a rejected strict scan', async () => {+ await expect(getDataOverview({ strict: true })).rejects.toThrow('permission denied');+ mocks.execFile.mockImplementation((command, args, _options, callback) => callback(null, { stdout: `1\t${args[args.length - 1]}\n`, stderr: '' }));+ await expect(getDataOverview({ strict: true })).resolves.toMatchObject({ dataDir: 'data' });+ });+});