Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { readdir, opendir, stat, lstat, statfs, mkdtemp, open, link, readFile } from 'fs/promises';
import { join, relative, resolve, isAbsolute } from 'path';
import { existsSync } from 'fs';
import { randomUUID } from 'crypto';
import { assertNotRealDataWrite } from '../lib/testDataIsolation.js';
import { execFile } from '../lib/childProcess.js';
import { promisify } from 'util';
import { parseFilesystemStats, sha256Text } from '../lib/fileCore.js';
import { PATHS, ensureDir, isTopLevelEntryName, rmGuarded, writeFileGuarded } from '../lib/fileUtils.js';
import { isPersistentMindScreenshotEntry } from '../lib/persistentMindAttachmentOwnership.js';
import { ServerError } from '../lib/errorHandler.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import {
federatedMediaInboxBusy, imageCleanTmpBusy, trainingRunsBusy, updateDetachedBusy,
} from './dataManagerBusy.js';
import { getEidoversePaths } from './eidoverse.js';
import { getAllApps } from './apps.js';
const execFileAsync = promisify(execFile);
const DATA_DIR = PATHS.data;
// Every top-level directory PortOS can create under `data/` needs an entry here.
// A missing entry doesn't just show "Unknown category" on the Data Manager — it
// also strips the Archive/Purge affordances, so the biggest thing on a cleanup
// page becomes the one thing the cleanup page refuses to act on (issue #3285).
// `dataManager.categories.test.js` enumerates the directories the codebase can
// emit and fails when one of them is unclassified.
//
// Flag semantics — be conservative, the user cannot undo a purge:
// archivable — worth tarring into `data/backup/`. False for large binary trees
// (the tarball lands *inside* data/, so archiving them grows the very number
// the user came here to shrink) and for anything holding secrets.
// deletable — the bytes are genuinely reproducible (caches, ephemeral working
// dirs, re-downloadable assets) or are already duplicated elsewhere. False
// whenever purging would destroy the only copy of generated or uploaded work.
// purgeScope — how much a purge may take at once. Only meaningful when
// `deletable`; omit it (it resolves to null) on protected categories.
// 'category' — the whole directory can go in one action.
// 'items' — only a per-item delete (`subPath`) is offered; the
// category-wide wipe is refused server-side. For a mixed directory of
// uploads and renders that other records point at, "reclaim space" is a
// legitimate need but an all-or-nothing wipe is not (issue #3327).
// `purgeCategory` fails closed: anything other than an explicit 'category'
// refuses the directory-wide purge, so a missing or misspelled scope can
// never widen a delete.
// busyCheck — optional `() => Promise<{ busy, reason }>`. Some directories are
// genuinely reproducible scratch, yet only while nothing is USING them: a
// category-wide purge mid-job destroys the working state of a render, a
// trainer, or a self-update. A category carrying a `busyCheck` refuses the
// purge with 409 CATEGORY_BUSY while its probe says busy, including
// per-item deletion: naming a directory does not make a live job safe.
// Categories with no `busyCheck` are unaffected. Probes live in
// `dataManagerBusy.js`.
// purgeFilter — optional `name => boolean`. A category-wide purge removes only
// the entries it accepts. Use it when the directory also holds files the
// category's own service owns (backup's state.json and in-flight staging).
export const CATEGORIES = {
'agents': { label: 'Agents', description: 'Agent personality data', archivable: false, deletable: false },
'ask-conversations': { label: 'Ask Conversations', description: 'Saved Ask chat transcripts', archivable: true, deletable: false },
'audio': { label: 'Audio', description: 'Rendered voice-over lines referenced by pipeline issues', archivable: true, deletable: false },
'autofixer': { label: 'Autofixer', description: 'Autofixer run data', archivable: true, deletable: true, purgeScope: 'category' },
'avatar': { label: 'Avatar', description: 'Uploaded avatar images', archivable: true, deletable: false },
// Machine-local ownership authority must survive every generic cleanup action.
'backup-admission': { label: 'Backup Admission', description: 'Active publication and snapshot ownership — reconcile through backup recovery', archivable: false, deletable: false },
// data/backup/ also holds the backup service's state.json and .archive-* staging; the purge is archives-only.
'backup': { label: 'Backups', description: 'Data backup archives', archivable: false, deletable: true, purgeScope: 'category', purgeFilter: name => ARCHIVE_FILENAME.test(name) },
'brain': { label: 'Brain', description: 'Brain items and sync log', archivable: true, deletable: false },
// The Beeper attachment byte mirror (#37) — the only thing the Beeper feature
// writes to `data/`; every other Beeper record is Postgres. Not archivable:
// the tarball would land inside `data/` and grow the number the user came here
// to shrink, and the bytes are re-fetchable from Beeper anyway. NOT deletable
// from here either (audit cluster 04, decision 2): the directory holds a
// single subdirectory, `attachments/`, which the per-item purge form refuses
// outright (item-scoped purges remove a single FILE and never recurse — see
// `purgeCategory` below) and the category-wide form was never reachable
// without `purgeScope: 'category'`. The visible purge control was dead on
// arrival either way. Reclamation is two paths that both reference-check
// before touching a byte: the attachment budget sweep (least-recently-viewed
// eviction, bounded by `settings.beeper.attachmentBudgetGb`) and the
// per-conversation purge (`DELETE /api/beeper/conversations/:id`, content-
// addressed so a photo forwarded into three chats survives one being
// purged). This category reports size only.
'beeper': { label: 'Beeper Attachments', description: 'Mirrored Beeper attachment bytes — reclaimed by the attachment budget sweep and per-conversation purge, not from here', archivable: false, deletable: false },
// Legacy location — current installs download to ~/Downloads (PATHS.browserDownloads),
// but installs that predate that move still carry the dir, and backup still excludes it.
'browser-downloads': { label: 'Browser Downloads', description: 'Files the agent browser downloaded — re-downloadable, safe to purge', archivable: false, deletable: true, purgeScope: 'category' },
'browser-profile': { label: 'Browser Profile', description: 'Chrome/Chromium browser data', archivable: false, deletable: true, purgeScope: 'category' },
cache: { label: 'Remote API and Reading Cache', description: 'Cached remote API metadata and the author-hosted Accelerando reading source — refetched on demand, safe to purge', archivable: false, deletable: true, purgeScope: 'category' },
'calendar': { label: 'Calendar', description: 'Calendar sync data', archivable: true, deletable: false },
'cover-fonts': { label: 'Cover Fonts', description: 'Typefaces uploaded for music video cover lettering; covers made with one fall back to a built-in typeface if it is removed', archivable: true, deletable: false },
'private': { label: 'Private Keys', description: 'Machine-local integration credentials — managed in Settings > Credentials', archivable: false, deletable: false },
'certs': { label: 'TLS Certificates', description: 'HTTPS certificate and private key — purging drops the install back to HTTP', archivable: false, deletable: false },
'launch-videos': { label: 'Launch Videos', description: 'App launch-video compositions, plans, captions and rendered deliverables', archivable: true, deletable: false },
'composition-proofs': { label: 'Composition Proofs', description: 'Contact-sheet review stills from HTML-composition proof renders — re-rendered from source, safe to purge', archivable: false, deletable: true, purgeScope: 'category' },
// Development artifacts ("ingredients") of Music Video projects — every
// version of a Cast & Sets sheet, animatic, treatment or storyboard, one
// immutable file each, referenced by the project record. Not deletable from
// here: a purge would strand those references (delete the artifact instead).
'music-video': { label: 'Music Video Development Files', description: 'Versioned development artifacts (check-in sheets, animatics, treatments) and composition documents referenced by Music Video projects', archivable: true, deletable: false },
'music-video-compositions': { label: 'Music Video Overlay Scratch', description: 'Typography overlay frames for an in-flight composed music-video render — removed when the render ends and swept at boot', archivable: false, deletable: false },
'music-video-film-look': { label: 'Music Video Film Look Scratch', description: 'A staged link to a finished render for its film look pass — removed when the pass ends', archivable: false, deletable: false },
'music-video-stats-outro': { label: 'Music Video Outro Scratch', description: 'The cutting room floor outro page (fonts and floor stills) for an in-flight music-video render — removed when the render ends and swept at boot', archivable: false, deletable: false },
'music-video-segments': { label: 'Music Video Shot Renders', description: 'Cached per-shot segments of composed music-video final renders — pruned after each final render and removed with the project; the final render re-renders any that are missing', archivable: false, deletable: false },
'music-video-song-renders': { label: 'Music Video Composition Render Scratch', description: 'A staged copy of a composition document plus its scene media for an in-flight music-video render — removed when the render ends and swept at boot', archivable: false, deletable: false },
'code-animation-workspaces': { label: 'Code Animation Contained Workers', description: 'Per-run staged input, output and scratch for a sandboxed Code Animation worker — removed when the run ends and swept on the next run', archivable: false, deletable: false },
'code-animations': { label: 'Code Animations', description: 'Generated animation HTML retained with its job history — the only copy of provider-generated work', archivable: true, deletable: false },
'code-animation-exports': { label: 'Code Animation Export Staging', description: 'Animation HTML plus the frame-exact render shim, staged once per MP4 export — safe to purge', archivable: false, deletable: true, purgeScope: 'category' },
'commission-feedback': { label: 'Commission Feedback', description: 'Reactions on creative commissions (file mirror of the Postgres store)', archivable: true, deletable: false },
'conflict-journal': { label: 'Conflict Journal', description: 'Peer-sync conflict history — diagnostics only, safe to purge', archivable: true, deletable: true, purgeScope: 'category' },
'cos': { label: 'Chief of Staff', description: 'Agent data, reports, memories', archivable: true, deletable: false },
'creative': { label: 'Creative Ledger', description: 'Append-only ledger of creative generation runs', archivable: true, deletable: false },
'creative-commissions': { label: 'Creative Commissions', description: 'Commission records (file mirror of the Postgres store)', archivable: true, deletable: false },
'db-dumps': { label: 'DB Dumps', description: 'PostgreSQL database backups', archivable: true, deletable: true, purgeScope: 'category' },
'digital-twin': { label: 'Digital Twin', description: 'Identity, goals, character data', archivable: true, deletable: false },
'eidoverse': { label: 'Eidoverse Worlds', description: 'Machine-local world history owned by the optional Eidoverse runtime — backed up and not federated', archivable: true, deletable: false },
'fableloom-shot-runs': { label: 'FableLoom Shot Runs', description: 'Machine-local shot-planning drafts, reviews and resume checkpoints — regenerating them costs provider calls', archivable: true, deletable: false },
'fableloom': { label: 'FableLoom', description: 'Branching-narrative records (file mirror of the Postgres store)', archivable: true, deletable: false },
// Another machine's conditioning images, staged for one federated render and
// swept on a TTL (#4348). Not archivable for the same reason it is excluded
// from backups: none of it is this install's data, and a restored entry is
// either already expired or already rendered. Purgeable as a category —
// deleting a staged asset only forces the peer that owns it to re-upload,
// which is one transfer, not lost work.
'federated-media-inbox': { label: 'Federated Media Inbox', description: 'Source images peers uploaded for federated renders — expire automatically, safe to purge when no federated render is in flight', archivable: false, deletable: true, purgeScope: 'category', busyCheck: federatedMediaInboxBusy },
'games': { label: 'Games', description: 'Game project records and assets', archivable: true, deletable: false },
'health': { label: 'Apple Health', description: 'Daily health JSON snapshots', archivable: true, deletable: false },
'image-clean-tmp': { label: 'Image Cleaner Scratch', description: 'Ephemeral working files for Image Cleaner renders — swept automatically, safe to purge when no render is in flight', archivable: false, deletable: true, purgeScope: 'category', busyCheck: imageCleanTmpBusy },
'image-render-usage': { label: 'Image Render Usage', description: 'Durable image accounting receipts — retain historical usage and deduplication identities', archivable: true, deletable: false },
'image-render-usage-backfills': { label: 'Image Usage Recovery', description: 'Private historical import plans, preservation fingerprints and recovery journals', archivable: true, deletable: false },
'image-refs': { label: 'Image References', description: 'Reference images uploaded for multi-reference edits — still served to existing renders', archivable: true, deletable: false },
'image-to-3d': { label: 'Image to 3D', description: 'Generated GLB meshes — the only copy; records live in Postgres', archivable: false, deletable: false },
'images': { label: 'Images', description: 'Uploaded and generated images — delete individually; gallery, pipeline, and collection records point at these files', archivable: true, deletable: true, purgeScope: 'items' },
'insights': { label: 'Insights', description: 'Derived goal scorecards and insights — rebuilt on the next insights run', archivable: true, deletable: true, purgeScope: 'category' },
// Project-specific jev heads plus the corpora and cached embeddings behind
// them.
//
// NOT deletable: the corpus and the embedding cache regenerate, but a trained
// head does not — the forge queries that built its corpus return different
// rows a month later, and it is the one artifact here an operator made an
// adoption decision about. Purging the category would silently return scope
// adherence to the stock scorer with nothing to restore.
//
// NOT archivable either, and for the opposite reason: this tree is mostly the
// embedding cache, which `backup.js` excludes precisely because it is
// gigabytes of pure cache. Archiving gzips exactly those gigabytes INTO
// `data/backup/` — growing the directory an operator opened this page to
// shrink, and blowing the shared tar timeout on the way. The few kilobytes
// worth keeping (the heads) are already in every snapshot.
'jev': { label: 'jev Project Heads', description: 'Trained project-specific scorer heads, their training corpora, and cached frozen-encoder outputs — machine-local; a head is not regenerable once its corpus is stale, and the cache beside it is bulk that rebuilds', archivable: false, deletable: false },
'jira-reports': { label: 'Jira Reports', description: 'Generated Jira reports — regenerable from Jira', archivable: true, deletable: true, purgeScope: 'category' },
'loops': { label: 'Loops', description: 'Output history from scheduled loop runs', archivable: true, deletable: false },
'lora-datasets': { label: 'LoRA Datasets', description: 'Training images and captions for LoRA runs — uploaded source material, not regenerable', archivable: false, deletable: false },
// Measured local-model assessments. Deletable: every record is reproducible by
// re-running the assessment, and a stale one (taken before a RAM upgrade or a
// backend update) actively misleads the picker.
'local-llm': { label: 'Local Model Assessments', description: 'Measured fit and performance results for installed local models — re-runnable, and stale after a hardware or backend change', archivable: true, deletable: true, purgeScope: 'category' },
'loras': { label: 'LoRAs', description: 'Trained LoRA adapters — excluded from backups and not regenerable without hours of GPU retraining', archivable: false, deletable: false },
'meatspace': { label: 'MeatSpace', description: 'Body metrics, blood tests, eyes', archivable: true, deletable: false },
'media-collections': { label: 'Media Collections', description: 'Media collection records', archivable: true, deletable: false },
'media-sketches': { label: 'Media Sketches', description: 'Saved sketch canvases used as render inputs', archivable: true, deletable: false },
// Not deletable: drafts, the send-attempt audit (`delivery_unknown` records
// that stop a possibly-sent message being resent), triage rules and account
// configuration are user-authored and not regenerable. Only the mail cache is
// reproducible, and it is bounded per account, so no cache-only purge is offered.
'messages': { label: 'Messages', description: 'Message accounts, drafts, triage rules and cached mail — drafts and rules are not regenerable', archivable: true, deletable: false },
'migrations': { label: 'Legacy Migration Ledger', description: 'Pre-2026-05 location of the migration applied-list — the next migration run moves it to migrations.applied.json, and setup removes the empty folder', archivable: false, deletable: false },
'model-personality': { label: 'Model Personality', description: 'Model personality probe results and settings', archivable: true, deletable: false },
'model-tests': { label: 'Model Capability Tests', description: 'Throwaway agent sandboxes from the capability test suite — recreated per run, safe to purge', archivable: false, deletable: true, purgeScope: 'category' },
'music': { label: 'Music', description: 'Uploaded and generated background tracks', archivable: true, deletable: false },
'openclaw': { label: 'OpenClaw', description: 'OpenClaw integration config', archivable: true, deletable: false },
'peer-execution': { label: 'Peer Execution Recovery', description: 'Machine-local launch evidence and completion receipts required to reconcile authorized peer maintenance — preserve unresolved operations', archivable: false, deletable: false },
'pipeline-comparative-rank': { label: 'Comparative Rank', description: 'Cached comparative issue rankings — re-running them costs LLM calls', archivable: true, deletable: false },
'pipeline-editorial': { label: 'Editorial Analysis', description: 'Cached editorial analyses — re-running them costs LLM calls', archivable: true, deletable: false },
'pipeline-editorial-health': { label: 'Editorial Health', description: 'Cached editorial health scores — re-running them costs LLM calls', archivable: true, deletable: false },
'pipeline-foundation-judge': { label: 'Foundation Judge', description: 'Cached foundation-judge verdicts — re-running them costs LLM calls', archivable: true, deletable: false },
'pipeline-issues': { label: 'Pipeline Issues (legacy)', description: 'Pre-Postgres issue files kept as the recovery source until the migration is confirmed', archivable: true, deletable: false },
'pipeline-judge': { label: 'Pipeline Judge', description: 'Cached pipeline-judge verdicts — re-running them costs LLM calls', archivable: true, deletable: false },
'pipeline-pov-rewrites': { label: 'POV Rewrites', description: 'Cached perspective rewrites — re-running them costs LLM calls', archivable: true, deletable: false },
'pipeline-reader-panel': { label: 'Reader Panel', description: 'Cached reader-panel reactions — re-running them costs LLM calls', archivable: true, deletable: false },
'pipeline-series': { label: 'Pipeline Series (legacy)', description: 'Pre-Postgres series files kept as the recovery source until the migration is confirmed', archivable: true, deletable: false },
'pipeline-series-review': { label: 'Series Review', description: 'Cached series reviews — re-running them costs LLM calls', archivable: true, deletable: false },
'privacy': { label: 'Privacy', description: 'Data-broker opt-out records and request history', archivable: true, deletable: false },
'prompts': { label: 'Prompts', description: 'AI prompt templates', archivable: false, deletable: false },
'venvs': { label: 'Optional SDK Runtimes', description: 'Explicitly installed Python environments for media SDKs; reinstall with the corresponding setup command', archivable: false, deletable: false },
'python': { label: 'Python Runtime', description: 'Managed virtualenv for local ML tooling — rebuilt on the next Python-backed run (re-downloads several GB of wheels)', archivable: false, deletable: true, purgeScope: 'category' },
// A pasted or URL-fetched book is the user's only copy — the shelf is not a
// cache of anything re-fetchable, so it is neither archivable nor deletable.
'rapid-reader-library': { label: 'Rapid Reader Shelf', description: 'Saved books for Rapid Reader — pasted and URL-imported prose kept on this machine, not regenerable', archivable: false, deletable: false },
'repos': { label: 'Cloned Repos', description: 'Git repositories cloned by agents. Managed app checkouts (Eidoverse Worlds and its video repo) are kept on purge',archivable: false, deletable: true, purgeScope: 'category' },
// Not deletable: user todos, pending CoS action requests and private security
// assessment reports live here and have no other copy.
'review': { label: 'Review', description: 'Review Hub todos, action requests and assessment reports', archivable: true, deletable: false },
'rigging': { label: 'Rigging Clip Library', description: 'User-dropped animation-bearing GLB source files for retargeting — the only copy of assets you supplied', archivable: false, deletable: false },
'runs': { label: 'AI Runs', description: 'Agent run logs and outputs', archivable: true, deletable: true, purgeScope: 'category' },
'screenshots': { label: 'Screenshots', description: 'Task screenshots and images dropped into shell sessions — shell drops auto-delete after 7 days, others after 30 days unless an open task still uses them; Persistent Mind attachments are retained by Mind', archivable: true, deletable: true, purgeScope: 'category', purgeFilter: name => !isPersistentMindScreenshotEntry(name) },
'settings': { label: 'Settings', description: 'Per-feature settings files', archivable: true, deletable: false },
'sharing': { label: 'Peer Sync State', description: 'Peer-sync bookkeeping — purging forces a full resync and can resurrect deleted records', archivable: true, deletable: false },
'spotify': { label: 'Spotify Sync', description: 'Machine-local Spotify sync cursor and cache — purging resets the cursor and can leave a gap in imported history', archivable: true, deletable: false },
'sprites': { label: 'Sprites', description: 'Sprite reference art, walk frames, and runtime atlases — the only copy of the generated art; records live in Postgres', archivable: false, deletable: false },
'story-builder': { label: 'Story Builder', description: 'Story Builder project records', archivable: true, deletable: false },
'storyboards': { label: 'Storyboards', description: 'Storyboard beats and panels (file mirror of the Postgres store)', archivable: true, deletable: false },
'storyboard-recipes': { label: 'Storyboard recipes', description: 'Saved panel maker recipes (file mirror of the Postgres store)', archivable: true, deletable: false },
'supercollider': { label: 'SuperCollider Runtime', description: 'Machine-local readiness evidence for the managed SuperCollider image, in-flight render scratch and 24-hour render previews — re-created by npm run setup:supercollider, safe to purge', archivable: false, deletable: true, purgeScope: 'category' },
'telegram': { label: 'Telegram', description: 'Telegram bot data', archivable: true, deletable: true, purgeScope: 'category' },
'templates': { label: 'Visual Templates', description: 'Shipped layout assets used as render anchors', archivable: false, deletable: false },
'tools': { label: 'Tools', description: 'Tool execution data', archivable: true, deletable: true, purgeScope: 'category' },
'training-runs': { label: 'LoRA Training Runs', description: 'Retired training checkpoints and previews; use Settings → Data Maintenance for a fresh preview and confirmed removal. May contain unique adapters.', archivable: false, deletable: false, busyCheck: trainingRunsBusy },
'universes': { label: 'Universes', description: 'Universe Builder records — bibles, canon, and style references', archivable: true, deletable: false },
'update-detached': { label: 'Update Control', description: 'Control files for a detached self-update run — safe to purge when no update is running', archivable: false, deletable: true, purgeScope: 'category', busyCheck: updateDetachedBusy },
'uploads': { label: 'Uploads', description: 'Files uploaded through the UI and referenced by records', archivable: true, deletable: false },
'video-thumbnails': { label: 'Video Thumbnails', description: 'JPEG thumbnails for generated videos', archivable: false, deletable: true, purgeScope: 'category' },
'image-thumbnails': { label: 'Image Thumbnails', description: 'Regenerable WebP previews for generated images', archivable: false, deletable: true, purgeScope: 'category' },
'videos': { label: 'Videos', description: 'Locally generated videos — delete individually; the render is the only copy and re-rendering costs provider spend', archivable: true, deletable: true, purgeScope: 'items' },
'voice-profiles': { label: 'Voice Profiles', description: 'Machine-local voice profile source recordings, benchmarks, and local engine artifacts', archivable: true, deletable: false },
'writers-room': { label: 'Writers Room', description: 'Writers Room works and story bibles', archivable: true, deletable: false },
'youtube': { label: 'YouTube Sync', description: 'Machine-local YouTube sync state — purging resets the cursor and can leave a gap in imported history', archivable: true, deletable: false }
};
// Shown for a directory with no CATEGORIES entry. Phrased as an outcome ("we
// don't know whether removing this is safe") rather than a mechanism ("unknown
// category") so the absent Archive/Purge buttons read as a deliberate safety
// stance instead of a broken row.
export const UNKNOWN_CATEGORY_DESCRIPTION = "Not classified — PortOS doesn't know if this is safe to remove";
/**
* Resolve the display/permission metadata for a `data/` directory name.
* Adds `classified: false` for the unknown fallback so the client can style
* the row as "deliberately unactionable" instead of guessing from the copy.
* `purgeScope` is always present in the payload (null on protected categories)
* so the client never has to distinguish "absent" from "not purgeable".
* `busyCheck` is stripped — it is server-side behavior, not payload.
*/
function categoryMeta(name) {
const known = CATEGORIES[name];
if (known) {
const { busyCheck: _busyCheck, ...serializable } = known;
return { purgeScope: null, ...serializable, classified: true };
}
return { label: name, description: UNKNOWN_CATEGORY_DESCRIPTION, archivable: false, deletable: false, purgeScope: null, classified: false };
}
/**
* Run a category's `busyCheck`, if it has one, and shape the answer for both
* the API payload (`busy` / `busyReason`) and the purge gate (issue #3342).
* A category with no probe is always `{ busy: false }` — today's behavior.
*
* Fails CLOSED. A probe that throws, or hands back a shape we can't read,
* reports busy: "we could not verify" and "nothing is running" must not
* collapse into the same answer on a page whose buttons are irreversible.
*/
export async function resolveCategoryBusy(categoryKey) {
const check = CATEGORIES[categoryKey]?.busyCheck;
if (typeof check !== 'function') return { busy: false, busyReason: null };
const result = await check().catch((err) => {
console.error(`❌ Busy check failed for data/${categoryKey}: ${err.message}`);
return null;
});
if (typeof result?.busy !== 'boolean') {
return { busy: true, busyReason: `PortOS could not confirm whether anything is using data/${categoryKey} right now, so the purge is withheld.` };
}
if (!result.busy) return { busy: false, busyReason: null };
return { busy: true, busyReason: result.reason || `Something is using data/${categoryKey} right now — purge once it finishes.` };
}
// Managed app checkouts (Eidoverse worlds, its video companion, any registered
// app repo) live under data/repos but hold installed code, .env.portos and local
// commits, so a "Cloned Repos" purge must keep them. Fails closed: if the app
// registry cannot be read, the purge is refused rather than run unprotected.
async function protectedRepoPaths() {
const reposDir = resolve(PATHS.repos);
const eidoverse = getEidoversePaths();
const candidates = [eidoverse.worlds, eidoverse.video];
for (const app of await getAllApps()) {
if (typeof app.repoPath === 'string') candidates.push(app.repoPath);
for (const companion of app.companionRepoPaths || []) {
if (typeof companion === 'string') candidates.push(companion);
}
}
return candidates.map(p => resolve(p)).filter(p => isInsideDir(reposDir, p));
}
function isInsideDir(parent, child) {
const rel = relative(parent, child);
return Boolean(rel) && !rel.startsWith('..') && !isAbsolute(rel);
}
// Removes every entry under `dir` that is not protected. An ancestor of a
// protected checkout (e.g. the owner directory holding it) is descended into,
// never removed whole.
async function purgeUnprotectedEntries(dir, protectedPaths) {
const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
let removed = 0;
let kept = 0;
for (const entry of entries) {
const full = resolve(join(dir, entry.name));
if (protectedPaths.includes(full)) {
kept++;
continue;
}
if (protectedPaths.some(p => isInsideDir(full, p))) {
const nested = await purgeUnprotectedEntries(full, protectedPaths);
removed += nested.removed;
kept += nested.kept;
continue;
}
await rmGuarded(full, { recursive: true, force: true });
removed++;
}
return { removed, kept };
}
// Validate category key contains only safe characters
const SAFE_NAME = /^[a-z0-9_-]+$/;
// Walk one directory at a time: never buffer an entire tree's path list in a
// child-process stdout pipe. Failed scans stay unknown, distinct from empty.
async function countFiles(dirPath, deadline) {
if (Date.now() >= deadline) throw new Error('File count timed out');
let count = 0;
const directory = await opendir(dirPath);
for await (const entry of directory) {
if (Date.now() >= deadline) throw new Error('File count timed out');
if (entry.isDirectory()) count += await countFiles(join(dirPath, entry.name), deadline);
else if (entry.isFile()) count++;
}
return count;
}
async function getDirSizeAndCount(dirPath, { strict = false } = {}) {
if (strict) {
const present = await stat(dirPath).then(
() => true,
(err) => {
if (err?.code === 'ENOENT') return false;
throw err;
},
);
if (!present) return { size: 0, fileCount: 0 };
} else if (!existsSync(dirPath)) {
return { size: 0, fileCount: 0 };
}
const [duOut, findOut] = await Promise.all([
execFileAsync('du', ['-sk', dirPath], { timeout: 30000 })
.then(r => r.stdout.trim())
.catch((err) => {
if (strict) throw err;
return '0';
}),
countFiles(dirPath, Date.now() + 30000).catch((err) => {
if (strict) throw err;
return null;
})
]);
const kb = typeof duOut === 'string' ? parseInt(duOut.split('\t')[0], 10) : NaN;
if (!Number.isFinite(kb) || kb < 0) {
if (strict) throw new Error(`Could not parse directory size for ${dirPath}`);
return { size: 0, fileCount: findOut };
}
const fileCount = findOut;
return { size: kb * 1024, fileCount };
}
// The scan is the multi-second part of the overview (du + a recursive walk per
// category), and the Data page, the resources report and the data-introspection
// harbor all request it. Share one result per `strict` mode for a short window
// after it completes and coalesce concurrent callers while it is pending;
// mutations here invalidate it. Rejections are never
// cached. Busy state and disk capacity are read fresh on every call.
const OVERVIEW_SCAN_TTL_MS = 30_000;
const overviewScans = new Map();
function invalidateOverviewScan() {
overviewScans.clear();
}
async function scanDataTree(strict) {
let rootUnreadable = false;
const entries = await readdir(DATA_DIR, { withFileTypes: true }).catch((err) => {
if (strict) throw err;
rootUnreadable = true;
return [];
});
const dirs = entries.filter(e => e.isDirectory());
const topFiles = entries.filter(e => e.isFile());
// The root is the sum of its parts: scanning DATA_DIR itself would re-walk
// every category. Top-level files are sized by allocated blocks to match du.
// allSettled: a strict failure waits for its sibling scans rather than
// leaving them running unobserved after the rejection.
const settled = await Promise.allSettled([
...dirs.map(d => getDirSizeAndCount(join(DATA_DIR, d.name), { strict })),
...topFiles.map(f => stat(join(DATA_DIR, f.name)).then(
st => (typeof st.blocks === 'number' ? st.blocks * 512 : st.size),
(err) => {
if (strict) throw err;
return 0;
},
)),
]);
const failed = settled.find(r => r.status === 'rejected');
if (failed) throw failed.reason;
const values = settled.map(r => r.value);
const dirResults = values.slice(0, dirs.length);
const fileSizes = values.slice(dirs.length);
let totalSize = fileSizes.reduce((sum, n) => sum + n, 0);
let totalFileCount = rootUnreadable ? null : topFiles.length;
for (const r of dirResults) {
totalSize += r.size;
// A failed non-strict walk or root listing reports a null count; keep the total unknown too.
totalFileCount = totalFileCount === null || r.fileCount === null ? null : totalFileCount + r.fileCount;
}
return { dirs, dirResults, totalSize, totalFileCount };
}
function getCachedOverviewScan(strict) {
const cached = overviewScans.get(strict);
// A pending scan is always shared; the TTL starts when it fulfills.
if (cached && (cached.at === null || Date.now() - cached.at < OVERVIEW_SCAN_TTL_MS)) return cached.promise;
const promise = scanDataTree(strict);
const entry = { at: null, promise };
overviewScans.set(strict, entry);
promise.then(
() => { entry.at = Date.now(); },
() => {
if (overviewScans.get(strict) === entry) overviewScans.delete(strict);
},
);
return promise;
}
export async function getDataOverview({ strict = false } = {}) {
// Capacity of the volume holding data/ so the page can show how much room is
// left, not just what PortOS occupies. Null when the platform can't say.
// Independent of the scan below, so it runs alongside it.
const diskPromise = statfs(DATA_DIR).then(parseFilesystemStats).catch(() => null);
const { dirs, dirResults, totalSize, totalFileCount } = await getCachedOverviewScan(strict);
// Busy state ships with the overview so the Data Manager can drop the Purge
// button before the user commits, rather than failing on click (#3342). Only
// the handful of categories carrying a `busyCheck` cost anything here.
const busyStates = await Promise.all(dirs.map(d => resolveCategoryBusy(d.name)));
const categories = dirs.map((d, i) => ({
key: d.name,
path: `data/${d.name}`,
...categoryMeta(d.name),
...busyStates[i],
...dirResults[i]
}));
categories.sort((a, b) => b.size - a.size);
const disk = await diskPromise;
return {
totalSize,
totalFileCount,
categories,
disk,
dataDir: 'data'
};
}
export async function getCategoryDetail(categoryKey, { measure = false } = {}) {
if (!SAFE_NAME.test(categoryKey)) return null;
const dirPath = join(DATA_DIR, categoryKey);
if (!existsSync(dirPath)) return null;
const entries = await readdir(dirPath, { withFileTypes: true }).catch(() => []);
// Parallel: stat files + getDirSizeAndCount for subdirs
const itemPromises = entries.map(async (entry) => {
const fullPath = join(dirPath, entry.name);
if (entry.isDirectory()) {
const { size, fileCount } = await getDirSizeAndCount(fullPath);
return { name: entry.name, type: 'directory', size, fileCount };
}
const fileStat = await stat(fullPath).catch(() => null);
return {
name: entry.name,
type: 'file',
size: fileStat?.size || 0,
modified: fileStat?.mtime?.toISOString() || null
};
});
const items = await Promise.all(itemPromises);
// Resolve opaque training-run folders in one bounded-to-the-list
// lookup, including old runs beyond the training page's recent-run limit.
if (categoryKey === 'training-runs' && items.length) {
const { query } = await import('../lib/db.js');
const result = await query(
'SELECT data FROM lora_training_runs WHERE id = ANY($1::text[])',
[items.filter(item => item.type === 'directory').map(item => item.name)],
).catch(() => null);
const runs = new Map((result?.rows || []).map(({ data }) => [data.id, data]));
for (const item of items) {
const run = runs.get(item.name);
if (run) {
item.label = run.name || run.character?.name || run.baseModelId || item.name;
item.description = [run.baseModelId, run.status].filter(Boolean).join(' · ');
if (run.datasetId) item.sourceHref = `/models/training/${encodeURIComponent(run.datasetId)}`;
}
item.sourceUnavailable = result === null ? 'Source lookup unavailable' : !run ? 'Run record no longer available' : null;
}
}
if (categoryKey === 'images' || categoryKey === 'videos') {
for (const item of items) {
if (item.type === 'file') item.sourceHref = `/data/${categoryKey}/${encodeURIComponent(item.name)}`;
}
}
items.sort((a, b) => b.size - a.size);
const totalSize = items.reduce((sum, item) => sum + item.size, 0);
const busy = await resolveCategoryBusy(categoryKey);
const measurement = measure ? {
...await getDirSizeAndCount(dirPath, { strict: true }),
disk: await statfs(DATA_DIR).then(parseFilesystemStats).catch(() => null),
} : undefined;
return { key: categoryKey, ...categoryMeta(categoryKey), ...busy, totalSize, items, ...(measurement && { measurement }) };
}
export async function archiveCategory(categoryKey, options = {}) {
try {
return await archiveCategoryUncached(categoryKey, options);
} finally {
invalidateOverviewScan();
}
}
async function archiveCategoryUncached(categoryKey, options) {
if (!SAFE_NAME.test(categoryKey)) throw new ServerError('Invalid category name', { status: 400, code: 'VALIDATION_ERROR' });
const meta = CATEGORIES[categoryKey];
if (!meta?.archivable) {
throw new ServerError(`Category "${categoryKey}" is not archivable`, { status: 403, code: 'CATEGORY_NOT_ARCHIVABLE' });
}
const dirPath = join(DATA_DIR, categoryKey);
if (!existsSync(dirPath)) {
throw new ServerError(`Category directory not found: ${categoryKey}`, { status: 404, code: 'NOT_FOUND' });
}
const backupDir = join(DATA_DIR, 'backup');
await ensureDir(backupDir);
// Reserve scratch before selecting inputs: even a fixed clock cannot make
// separate requests share their file list or partially written archive.
assertNotRealDataWrite(backupDir, 'archiveCategory');
const operationDir = await mkdtemp(join(backupDir, '.archive-'));
const stagedArchive = join(operationDir, 'archive.tar.gz');
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
const archiveName = `${categoryKey}-${timestamp}-${randomUUID()}.tar.gz`;
const archivePath = join(backupDir, archiveName);
try {
const capturedDays = [];
let queueHealthDayMutation;
if (categoryKey === 'health') {
const daysToKeep = options.daysToKeep ?? 365;
const cutoff = new Date();
cutoff.setDate(cutoff.getDate() - daysToKeep);
const cutoffStr = cutoff.toISOString().slice(0, 10);
const files = await readdir(dirPath);
const oldFiles = files.filter(f => /^\d{4}-\d{2}-\d{2}\.json$/.test(f) && f.slice(0, 10) < cutoffStr);
if (oldFiles.length === 0) return { archived: 0, removed: 0, retained: 0, alreadyAbsent: 0, archivePath: null, message: 'No old files to archive' };
// Share the importers' admission and day queue for capture and removal,
// but release both during tar so a large archive blocks neither a
// historical backfill nor a live health restore (#10899).
({ queueHealthDayMutation } = await import('./appleHealthIngest.js'));
const stagedDays = join(operationDir, 'days');
await ensureDir(stagedDays);
for (const file of oldFiles) {
await queueHealthDayMutation(file.slice(0, 10), async () => {
const bytes = await readFile(join(dirPath, file));
await writeFileGuarded(join(stagedDays, file), bytes, { flag: 'wx' });
capturedDays.push({ file, hash: sha256Text(bytes) });
});
}
const listPath = join(operationDir, 'files.txt');
await writeFileGuarded(listPath, oldFiles.join('\n'));
await execFileAsync('tar', ['-czf', stagedArchive, '-C', stagedDays, '-T', listPath], { timeout: 120000 });
} else {
await execFileAsync('tar', ['-czf', stagedArchive, '-C', DATA_DIR, categoryKey], { timeout: 120000 });
}
// A hard link publishes complete bytes atomically and refuses an existing
// destination. Sync bytes first, then the published directory entry before
// deleting health sources. Both paths live on the same filesystem.
// Windows requires write access for FlushFileBuffers; r+ preserves existing bytes.
const archiveFile = await open(stagedArchive, 'r+');
try { await archiveFile.sync(); } finally { await archiveFile.close(); }
await link(stagedArchive, archivePath);
// Node does not expose directory fsync on Windows.
if (process.platform !== 'win32') {
const directory = await open(backupDir, 'r');
try { await directory.sync(); } finally { await directory.close(); }
}
const archiveStat = await stat(archivePath);
let removed = 0;
let retained = 0;
let alreadyAbsent = 0;
for (const { file, hash } of capturedDays) {
await queueHealthDayMutation(file.slice(0, 10), async () => {
const livePath = join(dirPath, file);
const bytes = await readFile(livePath).catch(error => {
// Another archive may already have removed this exact day.
if (error.code === 'ENOENT') return null;
throw error;
});
if (bytes === null) {
alreadyAbsent++;
} else if (sha256Text(bytes) !== hash) {
retained++;
} else {
await rmGuarded(livePath);
removed++;
}
});
}
if (categoryKey === 'health') {
console.log(`📦 Archived data/health to ${archiveName}: ${capturedDays.length} days, ${removed} removed, ${retained} retained, ${alreadyAbsent} already absent`);
return { archived: capturedDays.length, removed, retained, alreadyAbsent, archivePath: relative(process.cwd(), archivePath), size: archiveStat.size };
}
console.log(`📦 Archived data/${categoryKey} to ${archiveName}`);
return { archived: 0, archivePath: relative(process.cwd(), archivePath), archiveSize: archiveStat.size };
} finally {
await rmGuarded(operationDir, { recursive: true, force: true });
}
}
export async function purgeCategory(categoryKey, options = {}) {
try {
return await purgeCategoryUncached(categoryKey, options);
} finally {
invalidateOverviewScan();
}
}
async function purgeCategoryUncached(categoryKey, options) {
// A purge may retain external record references by explicit operator choice,
// but its deletions must never interleave with a snapshot's copy and dump.
return withBackupAssetPublication(async () => {
if (!SAFE_NAME.test(categoryKey)) throw new ServerError('Invalid category name', { status: 400, code: 'VALIDATION_ERROR' });
const meta = CATEGORIES[categoryKey];
if (!meta?.deletable) {
throw new ServerError(`Category "${categoryKey}" is not purgeable`, { status: 403, code: 'CATEGORY_NOT_PURGEABLE' });
}
// "Caller asked for one entry" is keyed on the option being PRESENT, not on
// it being truthy: `{ subPath: '' }` is a caller that meant to name something
// and produced nothing, and must 400 — not fall through to the branch that
// empties the whole directory (#3327).
const wantsItem = options.subPath !== undefined && options.subPath !== null;
if (wantsItem && !isTopLevelEntryName(options.subPath)) {
// Refusing separators outright means no traversal segment can form and, more
// importantly, no *intermediate* component can be a symlink for `rm` to
// follow out of the category — the lexical containment check below never
// touches the filesystem, so it cannot see that.
throw new ServerError('subPath must name a single entry in the category', { status: 400, code: 'VALIDATION_ERROR' });
}
// Item-scoped categories only ever lose one entry at a time. Written as
// "must be exactly 'category'" so an absent or misspelled scope refuses the
// wipe instead of inheriting the old all-or-nothing behavior (#3327).
if (!wantsItem && meta.purgeScope !== 'category') {
throw new ServerError(
`Category "${categoryKey}" only supports per-item purge — pass a subPath`,
{ status: 400, code: 'CATEGORY_ITEM_PURGE_ONLY' }
);
}
const dirPath = join(DATA_DIR, categoryKey);
if (!existsSync(dirPath)) {
throw new ServerError(`Category directory not found: ${categoryKey}`, { status: 404, code: 'NOT_FOUND' });
}
// A named directory can still contain a live job's working state. Apply
// the same fail-closed busy probe to both deletion scopes.
const { busy, busyReason } = await resolveCategoryBusy(categoryKey);
if (busy) throw new ServerError(busyReason, { status: 409, code: 'CATEGORY_BUSY' });
const protectedPaths = categoryKey === 'repos' ? await protectedRepoPaths() : [];
if (wantsItem) {
if (categoryKey === 'screenshots' && isPersistentMindScreenshotEntry(options.subPath)) {
throw new ServerError('Persistent Mind owns this attachment or upload recovery marker; manage it through Mind', {
status: 409, code: 'SCREENSHOT_OWNED_BY_MIND',
});
}
const resolvedRoot = resolve(dirPath);
const resolvedTarget = resolve(join(dirPath, options.subPath));
// Boundary-aware containment check: use path.relative so a prefix like
// `/data/cat` cannot satisfy containment for `/data/cat2`.
const rel = relative(resolvedRoot, resolvedTarget);
if (!rel || rel.startsWith('..') || isAbsolute(rel)) {
throw new ServerError('Invalid subPath', { status: 400, code: 'VALIDATION_ERROR' });
}
// Ancestor (holds a checkout) or descendant (lies inside one) — both would destroy it.
if (protectedPaths.some(p => p === resolvedTarget || isInsideDir(resolvedTarget, p) || isInsideDir(p, resolvedTarget))) {
throw new ServerError(`"${options.subPath}" is a managed app checkout and is kept`, { status: 409, code: 'PROTECTED_CHECKOUT' });
}
// Item-scoped categories are flat directories of assets; their only
// subdirectories are working state for other features (`data/videos/.detached`
// holds the control files of in-flight renders). A one-click recursive
// delete next to those on a cleanup page is a foot-gun, so an item purge
// removes a single file and never recurses. `lstat`, not `stat`, so a
// symlinked entry is judged as the link it is and simply unlinked.
const itemScoped = meta.purgeScope === 'items';
if (itemScoped) {
const entryStat = await lstat(resolvedTarget).catch(() => null);
if (!entryStat) {
throw new ServerError(`Item not found in "${categoryKey}"`, { status: 404, code: 'NOT_FOUND' });
}
if (entryStat.isDirectory()) {
throw new ServerError(
`"${options.subPath}" is a directory — per-item purge in "${categoryKey}" only removes files`,
{ status: 400, code: 'ITEM_PURGE_FILE_ONLY' }
);
}
}
await rmGuarded(resolvedTarget, { recursive: !itemScoped, force: true });
console.log(`🗑️ Purged item from data/${categoryKey}`);
} else if (categoryKey === 'repos') {
const { removed, kept } = await purgeUnprotectedEntries(dirPath, protectedPaths);
console.log(`🗑️ Purged ${removed} entries from data/repos, kept ${kept} managed app checkout(s)`);
} else {
const allEntries = await readdir(dirPath).catch(() => []);
const entries = meta.purgeFilter ? allEntries.filter(meta.purgeFilter) : allEntries;
const removals = await Promise.allSettled(entries.map(entry => rmGuarded(join(dirPath, entry), { recursive: true, force: true })));
const failed = removals.find(result => result.status === 'rejected');
if (failed) throw failed.reason;
console.log(`🗑️ Purged ${entries.length} of ${allEntries.length} entries from data/${categoryKey}`);
}
return { category: categoryKey, subPath: wantsItem ? options.subPath : null };
});
}
// Names archiveCategory produces: `<category>-<timestamp>[-<uuid>].tar.gz`
// (the uuid is absent on archives written before same-second collisions were
// handled). data/backup/ is shared with the backup service's state.json, so
// listing and deletion are limited to this shape. The pattern excludes `/`,
// `\\` and bare dots, so no traversal segment can form (issue #1822).
const ARCHIVE_FILENAME = /^[a-z0-9_-]+-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(?:-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})?\.tar\.gz$/i;
export async function getBackups() {
const backupDir = join(DATA_DIR, 'backup');
if (!existsSync(backupDir)) return [];
const entries = await readdir(backupDir, { withFileTypes: true }).catch(() => []);
const files = entries.filter(e => e.isFile() && ARCHIVE_FILENAME.test(e.name));
const backups = await Promise.all(files.map(async (entry) => {
const fileStat = await stat(join(backupDir, entry.name)).catch(() => null);
return {
name: entry.name,
size: fileStat?.size || 0,
created: fileStat?.birthtime?.toISOString() || fileStat?.mtime?.toISOString() || null
};
}));
backups.sort((a, b) => (b.created || '').localeCompare(a.created || ''));
return backups;
}
export async function deleteBackup(filename) {
if (!ARCHIVE_FILENAME.test(filename)) {
throw new ServerError('Invalid filename', { status: 400, code: 'INVALID_FILENAME' });
}
const backupDir = join(DATA_DIR, 'backup');
const fullPath = join(backupDir, filename);
if (!fullPath.startsWith(backupDir)) {
throw new ServerError('Path traversal not allowed', { status: 400, code: 'VALIDATION_ERROR' });
}
await rmGuarded(fullPath);
invalidateOverviewScan();
console.log(`🗑️ Deleted backup archive ${filename}`);
return { deleted: filename };
}