Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Coverage guard for the Data Manager's CATEGORIES map (issue #3285).
*
* An unclassified `data/` directory doesn't merely render as "Unknown category"
* — `categoryMeta()` also falls back to `archivable: false, deletable: false`,
* so the row loses its Archive/Purge affordances entirely. That is how the
* single largest directory on disk ended up being the one thing the cleanup
* page refused to act on.
*
* This test enumerates the top-level `data/` directories the codebase can
* create and fails when one of them has no CATEGORIES entry. Discovery is
* derived, never hand-maintained, so a new data dir can't regress silently:
*
* 1. `PATHS` in lib/fileUtils.js — every value under `PATHS.data`.
* 2. `DEFAULT_EXCLUDES` in services/backup.js — anchored rsync filter paths.
* 3. Source scan of server/ + scripts/ for the ways a data dir is spelled
* outside PATHS: `dataPath('name')`, `join(PATHS.data, 'name')`,
* `` `${PATHS.data}/name` ``, `join(root, 'data', 'name')`, and the
* collection-store idiom `join(PATHS.data, TYPE)` with a const name.
* 4. `data.reference/` — dirs seeded into `data/` on first install.
*/
import { describe, it, expect } from 'vitest';
import { readdirSync, readFileSync } from 'fs';
import { join, relative, sep } from 'path';
import { PATHS } from '../lib/fileUtils.js';
import { DEFAULT_EXCLUDES } from './backup.js';
import { CATEGORIES, UNKNOWN_CATEGORY_DESCRIPTION, purgeCategory, archiveCategory } from './dataManager.js';
// A path segment that is a file, not a directory (`settings.json`, `TASKS.md`).
const isFileSegment = (segment) => segment.includes('.');
// Every spelling of "a path under data/" that appears in the codebase. Missing
// one means a whole family of directories escapes the guard — `dataPath('x')`
// was the gap that let `data/spotify` and `data/youtube` slip through.
const DATA_DIR_PATTERNS = [
/\bdataPath\(\s*'([a-z0-9._-]+)'/g,
/PATHS\.data,\s*'([a-z0-9._-]+)'/g,
/\$\{PATHS\.data\}\/([a-z0-9._-]+)/g,
/\b(?:join|resolve)\(\s*[^,;]+,\s*'data',\s*'([a-z0-9._-]+)'/g
];
// `join(PATHS.data, TYPE)` — the collection-store idiom, where the directory
// name is a module constant rather than an inline literal. Capture the
// identifier, then resolve it against its declaration in the same file.
const INDIRECT_DATA_DIR_PATTERN = /PATHS\.data,\s*([A-Z][A-Z0-9_]*)\s*\)/g;
function resolveConstant(source, identifier) {
const declaration = new RegExp(`\\bconst\\s+${identifier}\\s*=\\s*'([a-z0-9._-]+)'`);
return source.match(declaration)?.[1] ?? null;
}
function walkJsFiles(dir, out = []) {
for (const entry of readdirSync(dir, { withFileTypes: true })) {
if (entry.name === 'node_modules' || entry.name.startsWith('.')) continue;
const full = join(dir, entry.name);
if (entry.isDirectory()) walkJsFiles(full, out);
else if (entry.name.endsWith('.js') && !entry.name.includes('.test.')) out.push(full);
}
return out;
}
/** @returns {Map<string, string>} data dir name → where it was discovered */
function discoverDataDirs() {
const found = new Map();
const add = (name, source) => {
if (!name || isFileSegment(name) || found.has(name)) return;
found.set(name, source);
};
const dataRoot = PATHS.data + sep;
for (const [key, value] of Object.entries(PATHS)) {
if (typeof value !== 'string' || !value.startsWith(dataRoot)) continue;
add(value.slice(dataRoot.length).split(sep)[0], `PATHS.${key}`);
}
for (const { path } of DEFAULT_EXCLUDES) {
add(path.replace(/^\//, '').split('/')[0], 'backup.js DEFAULT_EXCLUDES');
}
for (const dir of ['server', 'scripts']) {
for (const file of walkJsFiles(join(PATHS.root, dir))) {
const source = readFileSync(file, 'utf8');
for (const pattern of DATA_DIR_PATTERNS) {
for (const match of source.matchAll(pattern)) {
add(match[1], relative(PATHS.root, file));
}
}
for (const match of source.matchAll(INDIRECT_DATA_DIR_PATTERN)) {
add(resolveConstant(source, match[1]), relative(PATHS.root, file));
}
}
}
for (const entry of readdirSync(join(PATHS.root, 'data.reference'), { withFileTypes: true })) {
if (entry.isDirectory()) add(entry.name, 'data.reference/');
}
return found;
}
describe('dataManager CATEGORIES coverage (#3285)', () => {
it('discovers literal data paths without treating domain-name arrays as paths', () => {
const source = `
const a = join(rootDir, 'data', 'example-one');
const b = path.resolve(
tmpdir(),
'data', 'example-two'
);
const districts = ['agents', 'data', 'federation'];
`;
expect(DATA_DIR_PATTERNS.flatMap((pattern) => [...source.matchAll(pattern)].map((match) => match[1])))
.toEqual(['example-one', 'example-two']);
});
it('classifies every top-level data/ directory the codebase can create', () => {
const discovered = discoverDataDirs();
expect(discovered.size).toBeGreaterThan(20); // discovery itself must not silently break
const unclassified = [...discovered]
.filter(([name]) => !CATEGORIES[name])
.map(([name, source]) => `${name} (from ${source})`);
expect(unclassified, 'Add a CATEGORIES entry in server/services/dataManager.js for each of these').toEqual([]);
});
it('gives every category a label, a description, and explicit permission flags', () => {
for (const [key, meta] of Object.entries(CATEGORIES)) {
expect(meta.label, `${key} label`).toBeTruthy();
expect(meta.description, `${key} description`).toBeTruthy();
expect(meta.description, `${key} description`).not.toBe(UNKNOWN_CATEGORY_DESCRIPTION);
expect(typeof meta.archivable, `${key} archivable`).toBe('boolean');
expect(typeof meta.deletable, `${key} deletable`).toBe('boolean');
}
});
});
describe('dataManager purge scope (#3327)', () => {
it('gives every purgeable category an explicit purgeScope and none to the protected ones', () => {
for (const [key, meta] of Object.entries(CATEGORIES)) {
if (meta.deletable) {
expect(['items', 'category'], `${key} purgeScope`).toContain(meta.purgeScope);
} else {
expect(meta.purgeScope, `${key} is protected and must not carry a purgeScope`).toBeUndefined();
}
}
});
it('keeps the irreplaceable media categories out of the all-or-nothing purge', () => {
// LoRA adapters are excluded from backup snapshots and cost hours of GPU
// time to retrain — no purge affordance at all.
expect(CATEGORIES.loras.deletable).toBe(false);
// Uploads and renders that other records point at: reclaimable one at a time.
expect(CATEGORIES.images.purgeScope).toBe('items');
expect(CATEGORIES.videos.purgeScope).toBe('items');
// Screenshots are task scratch — still wipeable whole.
expect(CATEGORIES.screenshots.purgeScope).toBe('category');
});
// Issue #10912: these directories hold user-authored state with no other copy
// (drafts and the send-attempt audit, todos and assessment reports). Each entry
// records why it is protected; the table is the audit trail, not an inference.
const USER_STATE_CATEGORIES = {
messages: 'drafts, the delivery_unknown send audit, triage rules and account config',
review: 'Review Hub todos, pending CoS action requests and security assessment reports',
'image-render-usage': 'historical image accounting and deduplication identities',
'image-render-usage-backfills': 'import plans, preservation fingerprints and recovery journals',
};
it.each(Object.entries(USER_STATE_CATEGORIES))(
'never offers a purge for %s, which holds %s',
(key) => {
expect(CATEGORIES[key]).toMatchObject({ archivable: true, deletable: false });
expect(CATEGORIES[key].purgeScope).toBeUndefined();
}
);
it('protects durable backup admission authority from archive and both purge forms', async () => {
expect(CATEGORIES['backup-admission']).toMatchObject({ archivable: false, deletable: false });
await expect(archiveCategory('backup-admission')).rejects.toThrow(/is not archivable/);
await expect(purgeCategory('backup-admission')).rejects.toThrow(/is not purgeable/);
await expect(purgeCategory('backup-admission', { subPath: 'cut' })).rejects.toThrow(/is not purgeable/);
});
// Audit cluster 04, decision 2: `data/beeper/` holds one subdirectory
// (`attachments/`), so neither purge form was ever reachable — a category
// wipe needs `purgeScope: 'category'` and a per-item purge refuses a
// directory entry outright. The visible purge control was dead either way;
// this category now reports size only and reclamation happens through the
// attachment budget sweep and the per-conversation purge instead.
it('reports the Beeper attachment mirror as size-only — no purge control that was never reachable', () => {
expect(CATEGORIES.beeper.deletable).toBe(false);
expect(CATEGORIES.beeper.purgeScope).toBeUndefined();
});
// These only exercise the refusal paths, which all return before purgeCategory
// touches the filesystem — no test here may reach `rm` against the real data/.
it('refuses a category-wide purge for item-scoped and protected categories', async () => {
await expect(purgeCategory('images')).rejects.toThrow(/only supports per-item purge/);
await expect(purgeCategory('videos')).rejects.toThrow(/only supports per-item purge/);
await expect(purgeCategory('loras')).rejects.toThrow(/is not purgeable/);
await expect(purgeCategory('loras', { subPath: 'anything' })).rejects.toThrow(/is not purgeable/);
await expect(purgeCategory('beeper')).rejects.toThrow(/is not purgeable/);
await expect(purgeCategory('beeper', { subPath: 'attachments' })).rejects.toThrow(/is not purgeable/);
await expect(purgeCategory('peer-execution')).rejects.toThrow(/is not purgeable/);
await expect(purgeCategory('peer-execution', { subPath: 'fixture-operation/evidence.json' })).rejects.toThrow(/is not purgeable/);
for (const category of Object.keys(USER_STATE_CATEGORIES)) {
for (const options of [undefined, { subPath: 'account' }]) {
await expect(purgeCategory(category, options)).rejects.toMatchObject({ status: 403, code: 'CATEGORY_NOT_PURGEABLE' });
}
}
});
it('fails closed when a deletable category has no recognized purgeScope', async () => {
const original = CATEGORIES.screenshots.purgeScope;
CATEGORIES.screenshots.purgeScope = 'typo';
await expect(purgeCategory('screenshots')).rejects.toThrow(/only supports per-item purge/);
CATEGORIES.screenshots.purgeScope = original;
});
});