Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/** Machine-local maintenance of raw terminal recordings. History is never deleted. */
import { createReadStream } from 'node:fs';
import { lstat, readdir, readFile, rename } from 'node:fs/promises';
import { join } from 'node:path';
import { randomUUID, createHash } from 'node:crypto';
import { createGzip, createGunzip } from 'node:zlib';
import { pipeline } from 'node:stream/promises';
import { Writable, Transform } from 'node:stream';
import { z } from 'zod';
import { atomicWrite, createWriteStreamGuarded, unlinkGuarded } from '../lib/fileUtils.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { ServerError } from '../lib/errorHandler.js';
import { AGENTS_DIR, loadConfig, saveConfig, withConfigLock, withStateLock, readAgentsStateForSafetyCheck } from './cosState.js';
import { cosEvents } from './cosEvents.js';
const DAY = 86400000;
const SAFE_ID = /^[a-zA-Z0-9_-]+$/;
const DATE = /^\d{4}-\d{2}-\d{2}$/;
const TEMP_RECORDING = /^raw\.txt\.gz\.[0-9a-f-]{36}\.tmp$/;
const MANIFEST = 'raw-storage.json';
export const agentStoragePolicySchema = z.object({
autoCompress: z.boolean().default(true),
compressAfterDays: z.number().int().min(1).max(36500).default(7),
autoPurge: z.boolean().default(false),
purgeAfterDays: z.number().int().min(7).max(36500).default(90),
}).strict();
export const agentStorageFilterSchema = z.object({
action: z.enum(['compress', 'purge']).default('compress'),
olderThanDays: z.number().int().min(1).max(36500).default(7),
model: z.string().max(300).default(''),
outcome: z.enum(['all', 'success', 'failure']).default('all'),
}).strict();
const previews = new Map();
let job = null;
let controller = null;
const changed = () => cosEvents.emit('storage:changed', {});
const missing = err => { if (err.code === 'ENOENT') return null; throw err; };
const info = path => lstat(path).catch(missing);
const fingerprint = st => st ? `${st.dev}:${st.ino}:${st.size}:${st.mtimeMs}:${st.ctimeMs}` : null;
const json = async path => {
const st = await lstat(path);
if (!st.isFile()) throw new Error('Non-regular metadata file');
return JSON.parse(await readFile(path, 'utf8'));
};
async function manifest(dir) {
const value = await json(join(dir, MANIFEST)).catch(missing);
if (value === null) return { version: 1, pinned: false };
if (value.version !== 1 || typeof value.pinned !== 'boolean') throw new Error('Recording manifest is invalid');
return value;
}
function location(date, id) {
if (!DATE.test(date) || !SAFE_ID.test(id)) throw new ServerError('Invalid run locator', { status: 400, code: 'VALIDATION_ERROR' });
return join(AGENTS_DIR, date, id);
}
async function safeDirectory(date, id) {
const dir = location(date, id);
for (const path of [AGENTS_DIR, join(AGENTS_DIR, date), dir]) {
if (!(await info(path))?.isDirectory()) throw new Error('Recording directory unavailable');
}
return dir;
}
async function inspect(date, id, filter, now, liveState) {
const dir = await safeDirectory(date, id);
const record = await json(join(dir, 'metadata.json'));
const storage = await manifest(dir);
const raw = await info(join(dir, 'raw.txt'));
const gzip = await info(join(dir, 'raw.txt.gz'));
const output = await info(join(dir, 'output.txt'));
const prompt = await info(join(dir, 'prompt.txt'));
const metadataFile = await info(join(dir, 'metadata.json'));
const metadata = record.metadata || {};
const model = String(metadata.model || metadata.modelName || record.model || 'Unknown');
const success = record.result?.success === true;
const matches = (!filter.model || model === filter.model) && (filter.outcome === 'all' || (filter.outcome === 'success') === success);
const completed = Date.parse(record.completedAt);
const state = liveState || await readAgentsStateForSafetyCheck();
let reason = null;
if (!state.trusted || !state.agents) reason = 'Live state unavailable';
else if (state.agents[id]) reason = 'Still in live state';
else if (Object.values(state.agents).some(agent => JSON.stringify(agent.metadata || {}).includes(id))) reason = 'Referenced by a live run';
else if (record.id !== id || record.status !== 'completed' || !Number.isFinite(completed)) reason = 'Not a finalized run';
else if (storage.pinned) reason = 'Pinned';
else if (metadata.pipeline || metadata.pipelineId || metadata.resumeSessionId || metadata.resumeFromAgentId || metadata.resumedFromAgentId || record.result?.worktreePreserved) reason = 'Pipeline or resumable run';
else if (completed > now - Math.max(1, filter.olderThanDays) * DAY) reason = 'Too recent';
else if ([raw, gzip, output].some(st => st && !st.isFile())) reason = 'Non-regular artifact';
else if ([raw, gzip, output].some(st => st && st.mtimeMs > now - DAY)) reason = 'Recently modified';
else if (filter.action === 'compress' && raw && storage.compressionSkippedFingerprint === fingerprint(raw)) reason = 'Compression would not save space';
else if (filter.action === 'compress' && !raw) reason = 'No uncompressed recording';
else if (filter.action === 'purge' && !raw && !gzip) reason = 'No recording';
else if (filter.action === 'purge' && (!output?.size || typeof metadata.taskSummary !== 'string' || !metadata.taskSummary.trim())) reason = 'Missing retained output or summary';
// Preserved worktrees carry potential resumable work, even after state eviction.
const worktree = metadata.worktreePath || metadata.worktreeInfo?.path || (metadata.isWorktree && metadata.workspacePath);
if (!reason && worktree && await info(worktree)) reason = 'Preserved worktree';
return { dir, storage, raw, gzip, record, matches, row: {
date, id, model, outcome: success ? 'success' : 'failure', completedAt: record.completedAt,
retainedBytes: { metadata: metadataFile?.size || 0, output: output?.size || 0, prompt: prompt?.size || 0 },
bytes: (raw?.size || 0) + (gzip?.size || 0), pinned: storage.pinned,
state: raw ? 'plain' : gzip ? 'compressed' : storage.disposition === 'purged' ? 'purged' : 'absent',
eligible: matches && !reason, reason: !matches ? 'Outside filter' : reason,
fingerprint: `${fingerprint(raw)}|${fingerprint(gzip)}`,
} };
}
async function* locators() {
const dates = await readdir(AGENTS_DIR, { withFileTypes: true }).catch(err => { if (err.code === 'ENOENT') return []; throw err; });
for (const date of dates.sort((a, b) => a.name.localeCompare(b.name))) {
if (!date.isDirectory() || !DATE.test(date.name)) continue;
for (const entry of (await readdir(join(AGENTS_DIR, date.name), { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {
if (entry.isDirectory() && SAFE_ID.test(entry.name)) yield { date: date.name, id: entry.name };
}
}
}
export async function getAgentStorageStatus() {
const config = await loadConfig();
return { policy: agentStoragePolicySchema.parse(config.agentStorage || {}), job: job ? { ...job } : config.lastAgentStorageJob || null };
}
export async function updateAgentStoragePolicy(input) {
const policy = agentStoragePolicySchema.parse(input);
await withConfigLock(async () => {
const current = await loadConfig();
await saveConfig({ ...current, agentStorage: policy });
});
changed();
// Reconcile immediately; the same gates are read again between auto-job files.
void runAutomaticAgentStorage().catch(() => console.error('❌ CoS recording maintenance could not start'));
return getAgentStorageStatus();
}
export async function previewAgentStorage(input, { limit = 1000, offset = 0 } = {}) {
const filter = agentStorageFilterSchema.parse(input);
const now = Date.now();
const liveState = await readAgentsStateForSafetyCheck();
const candidates = [];
const models = new Set();
let matching = 0;
const rows = [];
const reasons = {};
const totals = { recordingBytes: 0, eligibleBytes: 0, eligibleRuns: 0, scanned: 0, unreadable: 0, metadataBytes: 0, outputBytes: 0, promptBytes: 0 };
for await (const { date, id } of locators()) {
const entry = await inspect(date, id, filter, now, liveState).catch(() => null);
totals.scanned++;
if (!entry) { totals.unreadable++; continue; }
const row = entry.row;
models.add(row.model);
if (!entry.matches) continue;
totals.recordingBytes += row.bytes;
totals.metadataBytes += row.retainedBytes.metadata;
totals.outputBytes += row.retainedBytes.output;
totals.promptBytes += row.retainedBytes.prompt;
if (row.eligible && candidates.length < limit) {
candidates.push(row);
totals.eligibleBytes += row.bytes;
totals.eligibleRuns++;
} else {
const reason = row.reason || 'Next batch';
reasons[reason] = (reasons[reason] || 0) + 1;
}
if (matching++ >= offset && rows.length < 25) rows.push(row);
}
for (const [key, value] of previews) if (now - value.createdAt > 15 * 60000) previews.delete(key);
while (previews.size >= 5) previews.delete(previews.keys().next().value);
const token = randomUUID();
previews.set(token, { createdAt: now, filter, candidates });
return { token, filter, totals, reasons, rows, models: [...models].sort(), matching, offset, batchLimit: limit };
}
async function hashStream(stream, signal) {
const hash = createHash('sha256');
await pipeline(stream, new Writable({ write(chunk, encoding, done) { hash.update(chunk); done(); } }), { signal });
return hash.digest('hex');
}
async function compress(entry, signal, revalidate) {
const source = join(entry.dir, 'raw.txt');
const target = `${source}.gz`;
const temporary = `${target}.${randomUUID()}.tmp`;
// Recover only our reserved, old temporary files after an interrupted process.
for (const name of await readdir(entry.dir)) {
if (!TEMP_RECORDING.test(name)) continue;
const path = join(entry.dir, name);
const st = await info(path);
if (st?.isFile() && st.mtimeMs < Date.now() - DAY) await unlinkGuarded(path);
}
const hash = createHash('sha256');
const tap = new Transform({ transform(chunk, encoding, done) { hash.update(chunk); done(null, chunk); } });
// The old plain file remains authoritative until a verified gzip is published.
return (async () => {
await pipeline(createReadStream(source), tap, createGzip(), await createWriteStreamGuarded(temporary, { flags: 'wx' }), { signal });
const expected = hash.digest('hex');
const gunzip = createGunzip();
const verifying = pipeline(createReadStream(temporary), gunzip, { signal });
const [actual] = await Promise.all([hashStream(gunzip, signal), verifying]);
if (actual !== expected) throw new Error('Recording verification failed');
return withBackupAssetPublication(() => withStateLock(async () => {
const fresh = await revalidate();
if (!fresh || signal.aborted) return 0;
const zipped = await info(temporary);
if (zipped.size >= fresh.raw.size) {
await atomicWrite(join(entry.dir, MANIFEST), { ...fresh.storage, compressionSkippedFingerprint: fingerprint(fresh.raw) });
return 0;
}
await rename(temporary, target);
await atomicWrite(join(entry.dir, MANIFEST), { ...fresh.storage, disposition: 'compressed', sha256: expected, originalBytes: fresh.raw.size, compressedAt: new Date().toISOString() });
await unlinkGuarded(source);
return Math.max(0, fresh.raw.size + (fresh.gzip?.size || 0) - zipped.size);
}));
})().finally(() => unlinkGuarded(temporary).catch(err => { if (err.code !== 'ENOENT') throw err; }));
}
async function purge(entry, signal, revalidate) {
return withBackupAssetPublication(() => withStateLock(async () => {
const fresh = await revalidate();
if (!fresh || signal.aborted) return 0;
// Persist intent before unlink. A crash leaves an explicit disposition and
// surviving bytes visible; retries remove only the same allowlisted files.
await atomicWrite(join(entry.dir, MANIFEST), { ...fresh.storage, disposition: 'purged', purgedAt: new Date().toISOString(), removedFiles: ['raw.txt', 'raw.txt.gz'] });
let reclaimed = 0;
return (async () => {
for (const [name, st] of [['raw.txt', fresh.raw], ['raw.txt.gz', fresh.gzip]]) {
if (st) { await unlinkGuarded(join(entry.dir, name)); reclaimed += st.size; }
}
return reclaimed;
})().catch(err => { err.reclaimedBytes = reclaimed; throw err; });
}));
}
export async function startAgentStorage({ token, confirmation }, { automatic = false } = {}) {
if (controller) throw new ServerError('Recording maintenance already running', { status: 409, code: 'CONFLICT' });
const preview = previews.get(token);
if (!preview || Date.now() - preview.createdAt > 15 * 60000) throw new ServerError('Preview expired; preview again', { status: 409, code: 'STALE_PREVIEW' });
if (preview.filter.action === 'purge' && !automatic && confirmation !== 'PURGE RAW RECORDINGS') throw new ServerError('Confirm raw recording deletion', { status: 400, code: 'VALIDATION_ERROR' });
previews.delete(token);
controller = new AbortController();
job = { id: randomUUID(), action: preview.filter.action, state: 'running', total: preview.candidates.length, processed: 0, skipped: 0, failed: 0, reclaimedBytes: 0, startedAt: new Date().toISOString() };
const signal = controller.signal;
changed();
void (async () => {
for (const candidate of preview.candidates) {
if (signal.aborted) break;
const revalidate = async () => {
if (automatic) {
const { policy } = await getAgentStorageStatus();
if (!(preview.filter.action === 'compress' ? policy.autoCompress : policy.autoPurge)) return null;
if ((preview.filter.action === 'compress' ? policy.compressAfterDays : policy.purgeAfterDays) !== preview.filter.olderThanDays) return null;
}
const fresh = await inspect(candidate.date, candidate.id, preview.filter, preview.createdAt);
return fresh.row.eligible && fresh.row.fingerprint === candidate.fingerprint ? fresh : null;
};
await (async () => {
const entry = await revalidate();
if (!entry) { job.skipped++; return; }
const reclaimed = await (preview.filter.action === 'compress' ? compress : purge)(entry, signal, revalidate);
job.reclaimedBytes += reclaimed;
if (!reclaimed) job.skipped++;
})().catch(err => {
job.reclaimedBytes += err.reclaimedBytes || 0;
if (!signal.aborted) job.failed++;
});
job.processed++;
changed();
}
job.state = signal.aborted ? 'cancelled' : job.failed ? 'completed-with-errors' : 'completed';
})().catch(() => { job.state = 'failed'; }).finally(async () => {
job.finishedAt = new Date().toISOString();
// Small latest-operation audit lives in the existing machine-local config.
await withConfigLock(async () => {
const current = await loadConfig();
await saveConfig({ ...current, lastAgentStorageJob: { ...job } });
}).catch(() => { job.state = 'audit-write-failed'; });
controller = null;
changed();
});
return getAgentStorageStatus();
}
export function cancelAgentStorage() {
controller?.abort();
return getAgentStorageStatus();
}
export async function pinAgentRecording({ date, id, pinned }) {
await withStateLock(async () => {
const dir = await safeDirectory(date, id);
const value = await manifest(dir);
await atomicWrite(join(dir, MANIFEST), { ...value, pinned });
});
changed();
return { date, id, pinned };
}
export async function runAutomaticAgentStorage() {
if (controller) return;
const { policy } = await getAgentStorageStatus();
const action = policy.autoPurge ? 'purge' : 'compress';
if (!policy.autoPurge && !policy.autoCompress) return;
const preview = await previewAgentStorage({ action, olderThanDays: action === 'purge' ? policy.purgeAfterDays : policy.compressAfterDays }, { limit: 25 });
if (preview.totals.eligibleRuns) return startAgentStorage({ token: preview.token }, { automatic: true });
if (policy.autoPurge && policy.autoCompress) {
const compression = await previewAgentStorage({ action: 'compress', olderThanDays: policy.compressAfterDays }, { limit: 25 });
if (compression.totals.eligibleRuns) return startAgentStorage({ token: compression.token }, { automatic: true });
}
}
export async function getAgentRecordingDownload(date, id) {
const dir = await safeDirectory(date, id);
for (const name of ['raw.txt', 'raw.txt.gz']) {
if ((await info(join(dir, name)))?.isFile()) return { path: join(dir, name), name };
}
throw new ServerError('Raw recording was purged or was never captured; parsed output remains in run history', { status: 404, code: 'NOT_FOUND' });
}