Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { readFile, writeFile, mkdir, access, rename } from 'node:fs/promises';
import { join } from 'node:path';
// Dependency-light shared module (node builtins + pure arg builder only), so
// importing it from this standalone process doesn't pull in the AI toolkit.
// Lets the autofixer honor the user's configured CLI provider/model instead
// of hardcoding `claude -p`.
import { runCliProviderPrompt } from '../server/lib/cliProviderRun.js';
import { signalDetachedGroups } from '../server/lib/credentialBootstrap.js';
import { agentGuardEnv } from '../server/lib/agentGuard/index.js';
import {
sanitizeChildEnv,
collectSecretEnvValues,
buildFixPrompt,
pickRepairProvider,
repairWorkspaceHold,
validateProposedDiff,
isGitRepo,
createDisposableWorktree,
collectWorktreeDiff,
removeWorktree,
sweepStaleWorktrees,
applyDiffToLive,
revertDiffFromLive,
runVerifyCommand,
} from './sandbox.js';
import { execPm2, listProcessesStrict, DATA_DIR, AUTOFIXER_DIR, INDEX_FILE, loadApps } from './shared.js';
// Prepend the guarded pm2 shim to this process's PATH as defense-in-depth. The
// fix agent runs restricted (claude without Bash, or codex's sandbox) in an
// isolated worktree with a sanitized env, so it should never invoke pm2 at all. Our own
// execPm2 calls use an absolute PM2_BIN, so they bypass the shim. sanitizeChildEnv
// preserves this guarded PATH into the agent's env.
Object.assign(process.env, agentGuardEnv());
// Paths not shared with ui.js
const PROVIDERS_FILE = join(DATA_DIR, 'providers.json');
const SETTINGS_FILE = join(DATA_DIR, 'settings.json');
const SESSIONS_DIR = join(AUTOFIXER_DIR, 'sessions');
// Disposable worktrees for isolated repair runs (gitignored under data/).
const WORKTREES_DIR = join(AUTOFIXER_DIR, 'worktrees');
// Bound the agent-proposed patch before it can reach the live checkout.
const MAX_DIFF_BYTES = 200 * 1024;
// Track fixed processes to avoid repeated fixes
const recentlyFixed = new Map();
const FIX_COOLDOWN = 30 * 60 * 1000; // 30 minutes
const CHECK_INTERVAL = 15 * 60 * 1000; // 15 minutes
let checkTimer = null;
let shuttingDown = false;
// The repair in flight, so a shutdown can record it as interrupted.
let activeRepair = null;
// Parse JSON, returning `fallback` on read OR parse failure. A corrupt config
// file (partial write, hand-edit) must not throw inside fixProcess — this runs
// in the autofixer's interval loop, outside any request lifecycle, where an
// uncaught throw would take the process down.
async function readJsonSafe(file, fallback) {
const data = await readFile(file, 'utf8').catch(() => null);
if (data == null) return fallback;
try {
return JSON.parse(data);
} catch (err) {
console.error(`❌ [Autofixer] Corrupt JSON in ${file}: ${err.message}`);
return fallback;
}
}
// Load PortOS's AI provider registry (shared data file) so the autofixer runs
// through whichever CLI provider the user configured rather than hardcoding
// claude. Returns the on-disk provider map keyed by id.
async function loadProviders() {
const parsed = await readJsonSafe(PROVIDERS_FILE, { providers: {} });
return parsed.providers || {};
}
// Load PortOS settings — `settings.autofixer = { providerId, model }` selects
// which CLI provider/model fixes crashed processes.
async function loadSettings() {
return readJsonSafe(SETTINGS_FILE, {});
}
async function getMonitoredProcesses() {
const apps = await loadApps();
const processes = new Set();
for (const app of apps) {
for (const procName of app.pm2ProcessNames || []) {
processes.add(procName);
}
}
return Array.from(processes);
}
// Find app by process name
async function findAppByProcess(processName) {
const apps = await loadApps();
return apps.find(app =>
(app.pm2ProcessNames || []).includes(processName)
);
}
// History management
// Record a repair that never reached finalize (daemon restarted mid-run).
async function recordInterrupted({ sessionId, processName, app, startTime, prompt = '', output = '' }) {
const endTime = new Date().toISOString();
const metadata = {
sessionId, startTime, endTime,
duration: new Date(endTime).getTime() - new Date(startTime).getTime(),
exitCode: -1, success: false, processName,
appName: app?.name, appId: app?.id, repoPath: app?.repoPath,
type: 'autofixer', isolated: true, interrupted: true,
error: 'interrupted: autofixer restarted during repair',
};
await saveSession(sessionId, prompt, output, metadata);
}
// Sweep worktrees leaked by repairs a restart interrupted, and record each.
async function sweepInterruptedRepairs() {
const swept = await sweepStaleWorktrees(WORKTREES_DIR);
for (const sessionId of swept) {
// Shutdown already recorded this repair with its real start time, prompt and
// partial output; a second empty write would destroy that evidence.
const alreadyRecorded = await access(join(SESSIONS_DIR, sessionId, 'metadata.json')).then(() => true, () => false);
if (alreadyRecorded) continue;
const processName = sessionId.match(/^autofixer_(.+)_\d+_[a-z0-9]+$/)?.[1] || 'unknown';
const now = new Date().toISOString();
await recordInterrupted({ sessionId, processName, startTime: now }).catch((err) => {
console.error(`❌ [Autofixer] Could not record interrupted session ${sessionId}: ${err.message}`);
});
}
if (swept.length) console.warn(`🧹 [Autofixer] Swept ${swept.length} worktree(s) left by interrupted repairs`);
}
async function ensureHistoryDir() {
await mkdir(SESSIONS_DIR, { recursive: true });
await access(INDEX_FILE).catch(async () => {
await writeFile(INDEX_FILE, JSON.stringify([], null, 2));
});
}
async function loadIndex() {
const index = await readJsonSafe(INDEX_FILE, null);
if (index == null) {
throw new Error('Autofixer history index is unreadable');
}
if (!Array.isArray(index)) {
throw new Error('Autofixer history index must be an array');
}
return index;
}
async function saveIndex(index) {
// Temp + rename: a signal mid-write must not truncate the live index.
const tmp = `${INDEX_FILE}.tmp`;
await writeFile(tmp, JSON.stringify(index, null, 2));
await rename(tmp, INDEX_FILE);
}
async function saveSession(sessionId, prompt, output, metadata, patch) {
const sessionDir = join(SESSIONS_DIR, sessionId);
await mkdir(sessionDir, { recursive: true });
await writeFile(join(sessionDir, 'prompt.txt'), prompt);
await writeFile(join(sessionDir, 'output.txt'), output);
await writeFile(join(sessionDir, 'metadata.json'), JSON.stringify(metadata, null, 2));
// The staged/applied patch is the human-reviewable artifact of the fix — keep
// it beside the session so a user can inspect (and, when auto-promote is off,
// manually apply) exactly what the agent proposed.
if (typeof patch === 'string' && patch.trim().length > 0) {
await writeFile(join(sessionDir, 'fix.patch'), patch);
}
const index = await loadIndex();
const indexEntry = {
sessionId: metadata.sessionId,
startTime: metadata.startTime,
endTime: metadata.endTime,
duration: metadata.duration,
success: metadata.success,
promoted: metadata.promoted || false,
staged: metadata.staged || false,
processName: metadata.processName,
appName: metadata.appName,
promptPreview: prompt.substring(0, 200),
outputSize: output.length
};
// One row per session: a re-save replaces the earlier row instead of duplicating it.
const existing = index.findIndex(entry => entry.sessionId === sessionId);
if (existing >= 0) index.splice(existing, 1);
index.unshift(indexEntry);
if (index.length > 100) {
index.splice(100);
}
await saveIndex(index);
}
async function getProcessLogs(processName) {
const { stdout: errLogs } = await execPm2(['logs', processName, '--lines', '100', '--nostream', '--err']).catch(() => {
throw new Error(`PM2 stderr logs unavailable for ${processName.replace(/[\r\n]/g, ' ').slice(0, 120)} — skipping repair`);
});
const { stdout: outLogs } = await execPm2(['logs', processName, '--lines', '50', '--nostream', '--out']).catch(() => {
throw new Error(`PM2 stdout logs unavailable for ${processName.replace(/[\r\n]/g, ' ').slice(0, 120)} — skipping repair`);
});
return { errLogs, outLogs };
}
// Cooldown management
function isOnCooldown(processName) {
const lastFix = recentlyFixed.get(processName);
if (!lastFix) return false;
return (Date.now() - lastFix) < FIX_COOLDOWN;
}
function markAsFixed(processName) {
recentlyFixed.set(processName, Date.now());
}
// Run an autonomous repair, ISOLATED from the live checkout.
//
// The crash logs handed to this agent are untrusted (a payload reflected into a
// process's stderr can carry injected instructions), so we never let the agent
// touch the live repository or inherit host credentials. The flow:
// 1. Require a git checkout and cut a disposable, detached worktree at HEAD.
// 2. Sanitize the child env down to system + AI-provider auth only.
// 3. Run the agent against the worktree with a fenced, untrusted-log prompt,
// only on a provider with an enforced restricted mode (pickRepairProvider).
// 4. Collect + validate the resulting diff (size/scope/forbidden paths).
// 5. Optionally verify it against a user-configured test command, contained.
// 6. Promotion gate: only when the user opted into `autoPromote` do we apply
// the diff to the live checkout and restart the process; otherwise the
// validated patch is staged beside the session for manual review.
async function fixProcess(processName, app, errorLogs, outputLogs) {
const sessionId = `autofixer_${processName}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const startTime = new Date().toISOString();
console.log(`🔧 [Autofixer] Starting isolated fix for ${processName}: ${sessionId}`);
await ensureHistoryDir();
const prompt = buildFixPrompt({ processName, app, errorLogs, outputLogs });
const outputBuffer = [];
// Resolve the configured CLI provider/model (settings.autofixer), falling
// back to claude-code when unset, restricted to CLIs with an enforced
// restricted mode. `autoPromote` and `verifyCommand` come from the same slice.
const providers = await loadProviders();
const settings = await loadSettings();
const autofixerCfg = settings.autofixer || {};
const picked = pickRepairProvider(providers, autofixerCfg);
const autoPromote = autofixerCfg.autoPromote === true; // default OFF — stage only
const verifyCommand = typeof autofixerCfg.verifyCommand === 'string' ? autofixerCfg.verifyCommand.trim() : '';
// Single completion path — write the session record + return the result.
const finalize = async ({ success, exitCode, error, extra = {}, patch, cooldown }) => {
const endTime = new Date().toISOString();
const duration = new Date(endTime).getTime() - new Date(startTime).getTime();
const output = outputBuffer.join('') + (error ? `\n[ERROR] ${error}` : '');
const log = success ? console.log : console.error;
log(`${success ? '✅ [Autofixer] Fix produced' : '❌ [Autofixer] Fix failed'} for ${processName} (exit code: ${exitCode})`);
const metadata = {
sessionId,
startTime,
endTime,
duration,
exitCode,
success,
processName,
appName: app.name,
appId: app.id,
repoPath: app.repoPath,
type: 'autofixer',
isolated: true,
autoPromote,
provider: picked.provider?.id || null,
model: picked.model || null,
restriction: picked.mode || null,
...extra,
...(error ? { error } : {}),
};
// Cool down whenever a usable fix was produced (promoted, staged, OR
// applied-but-restart-failed) — not just on full success — so a crash the
// agent already addressed doesn't re-trigger the LLM every 15-min cycle.
// Set before persisting: a failed history write must not disable the cooldown.
if (cooldown ?? success) markAsFixed(processName);
activeRepair = null;
await saveSession(sessionId, prompt, output, metadata, patch);
console.log(`💾 [Autofixer] Saved session: ${sessionId}`);
return { success, sessionId, output, ...(error ? { error } : {}) };
};
if (picked.error) {
outputBuffer.push(`[ERROR] ${picked.error}`);
console.warn(`🛑 [Autofixer] Not repairing ${processName}: ${picked.error}`);
return finalize({ success: false, exitCode: -1, error: picked.error });
}
// ISOLATION REQUIREMENT: no git checkout ⇒ no rollback boundary ⇒ we refuse
// to run an autonomous, file-editing agent driven by untrusted logs.
if (!(await isGitRepo(app.repoPath))) {
const reason = `refusing autonomous repair: ${app.repoPath} is not a git checkout (no rollback boundary)`;
console.error(`🛑 [Autofixer] ${reason}`);
return finalize({ success: false, exitCode: -1, error: reason, extra: { isolated: false } });
}
const wt = await createDisposableWorktree(app.repoPath, WORKTREES_DIR, sessionId);
if (wt.error) {
console.error(`🛑 [Autofixer] Could not isolate ${processName}: ${wt.error}`);
return finalize({ success: false, exitCode: -1, error: `worktree isolation failed: ${wt.error}`, extra: { isolated: false } });
}
const worktreePath = wt.path;
activeRepair = { sessionId, processName, app, startTime, prompt, outputBuffer, worktreePath };
const workspaceHold = await repairWorkspaceHold(picked.mode, app.repoPath, worktreePath);
if (workspaceHold) {
console.warn(`🛑 [Autofixer] ${workspaceHold}`);
await removeWorktree(app.repoPath, worktreePath);
return finalize({ success: false, exitCode: -1, error: workspaceHold, extra: { isolated: true } });
}
// Strip host credentials the agent has no business seeing — it keeps only
// system + AI-provider auth. provider.envVars still overlays inside the runner.
const childEnv = sanitizeChildEnv(process.env);
console.log(`🤖 [Autofixer] Repairing ${processName} in isolated worktree via ${picked.provider.id}${picked.model ? ` (${picked.model})` : ''} [${picked.mode}]`);
const result = await runCliProviderPrompt({
provider: picked.provider,
model: picked.model,
prompt,
cwd: worktreePath,
baseEnv: childEnv,
timeoutMs: 600000, // 10 min — a fix may need several read/edit cycles
onData: (chunk, stream) => {
if (stream === 'stderr') {
outputBuffer.push(`[STDERR] ${chunk}`);
process.stderr.write(chunk);
} else {
outputBuffer.push(chunk);
process.stdout.write(chunk);
}
},
});
// Collect the agent's edits as a diff BEFORE tearing down the worktree. A
// spawn failure / timeout can leave a partial edit — don't promote or stage
// work from an agent that didn't finish cleanly.
const diff = await collectWorktreeDiff(worktreePath);
const agentOk = !result.error && result.partial === false && result.exitCode === 0;
// The live env's provider-auth values (incl. runner-injected provider.envVars
// keys) — reject any diff that tries to write one into the repo (agent
// read-then-promote exfil backstop).
const secretValues = collectSecretEnvValues(childEnv, picked.provider?.envVars);
const validation = agentOk
? validateProposedDiff(diff, { maxBytes: MAX_DIFF_BYTES, secretValues })
: { ok: false, reason: `agent did not complete: ${result.error || 'repair process returned incomplete output or an unsuccessful exit'}`, files: [] };
// Optional verification against the user-configured test command, contained
// over the isolated worktree — before anything reaches live.
let verify = null;
if (validation.ok && verifyCommand) {
console.log(`🧪 [Autofixer] Verifying ${processName} fix: ${verifyCommand}`);
verify = await runVerifyCommand(verifyCommand, worktreePath, {
dataDir: DATA_DIR,
// Settings > Features > Docker containment: the verify container used when
// bubblewrap is unavailable (always on macOS). An explicit opt-in only.
docker: { enabled: settings.instanceFeatures?.['docker-containment']?.enabled === true, settings: settings.dockerContainment ?? {} },
});
if (!verify.contained) console.warn(`🛑 [Autofixer] ${verify.output}`);
outputBuffer.push(`\n[VERIFY exit ${verify.code}]\n${verify.output}`);
}
const verifyOk = !verifyCommand || verify?.ok;
// Promotion gate — only a validated (and, if configured, verified) diff may
// touch the live checkout, and only when the user explicitly opted in.
let promoted = false;
let staged = false;
let promotionError = null;
if (validation.ok && verifyOk) {
if (autoPromote) {
const applied = await applyDiffToLive(app.repoPath, diff, AUTOFIXER_DIR);
if (applied.ok) {
const restart = await execPm2(['restart', processName]).catch((e) => ({ error: e.message }));
promoted = !restart.error;
promotionError = restart.error || null;
if (promoted) {
console.log(`🚀 [Autofixer] Promoted + restarted ${processName}`);
} else {
// Applied cleanly but the restart failed — roll the patch back so the
// live tree isn't left mutated (which would make the next cycle
// double-apply or fail `git apply --check` forever).
const reverted = await revertDiffFromLive(app.repoPath, diff, AUTOFIXER_DIR);
console.error(`❌ [Autofixer] Restart failed, rolled back applied fix for ${processName}: ${restart.error}${reverted.error ? ` (rollback also failed: ${reverted.error})` : ''}`);
}
} else {
promotionError = applied.error;
console.error(`❌ [Autofixer] Promotion blocked: ${applied.error}`);
}
} else {
staged = true;
console.log(`📋 [Autofixer] Fix staged for review (auto-promote off): ${sessionId}`);
}
} else {
console.warn(`⚠️ [Autofixer] Fix not promotable: ${validation.reason || (verify && `verify exit ${verify.code}`) || 'unknown'}`);
}
// Always discard the disposable worktree.
await removeWorktree(app.repoPath, worktreePath);
const producedFix = validation.ok && verifyOk; // usable fix, regardless of restart outcome
const success = producedFix && (autoPromote ? promoted : true);
return finalize({
success,
cooldown: producedFix,
exitCode: result.exitCode ?? -1,
error: result.error || promotionError || (!validation.ok ? validation.reason : null) || null,
patch: validation.ok ? diff : undefined,
extra: {
promoted,
staged,
diffBytes: Buffer.byteLength(diff || '', 'utf8'),
changedFiles: validation.files,
diffValid: validation.ok,
...(validation.ok ? {} : { diffRejectedReason: validation.reason }),
...(verify ? { verify: { ran: true, ok: verify.ok, exitCode: verify.code, contained: verify.contained } } : { verify: { ran: false } }),
},
});
}
// Main check function
async function checkAndFixProcesses() {
console.log(`🔍 [Autofixer] Checking PM2 processes...`);
const monitoredProcesses = await getMonitoredProcesses();
if (monitoredProcesses.length === 0) {
console.log(`⚠️ [Autofixer] No apps registered in PortOS`);
return;
}
// `listProcessesStrict()` returns `null` when the read itself FAILED (vs `[]`
// for a successful read with no processes) — the absent-vs-empty contract
// from issue #968, now shared with `server/services/pm2.js` (#8164). A
// failed read must not be reported as "no PM2 processes found": that would
// silently skip repair on the exact cycle a crashed process most needs it.
const pm2List = await listProcessesStrict();
if (pm2List === null) {
console.error(`❌ [Autofixer] PM2 process read failed — skipping this check cycle`);
return;
}
if (pm2List.length === 0) {
console.log(`⚠️ [Autofixer] No PM2 processes found`);
return;
}
const crashedProcesses = pm2List.filter(proc => {
const status = proc.pm2_env?.status;
return status === 'errored' && monitoredProcesses.includes(proc.name);
});
if (crashedProcesses.length === 0) {
console.log(`✅ [Autofixer] All monitored processes healthy`);
return;
}
console.log(`🚨 [Autofixer] Found ${crashedProcesses.length} crashed process(es)`);
for (const proc of crashedProcesses) {
const processName = proc.name;
if (isOnCooldown(processName)) {
console.log(`⏳ [Autofixer] ${processName} is on cooldown, skipping`);
continue;
}
const app = await findAppByProcess(processName);
if (!app) {
console.log(`⚠️ [Autofixer] No app found for process ${processName}, skipping`);
continue;
}
console.log(`🔧 [Autofixer] Attempting to fix ${processName} (${app.name})...`);
const { errLogs, outLogs } = await getProcessLogs(processName);
await fixProcess(processName, app, errLogs, outLogs);
}
}
function scheduleNextCheck() {
if (shuttingDown) return;
checkTimer = setTimeout(() => {
checkTimer = null;
void runCheckCycle();
}, CHECK_INTERVAL);
}
// Schedule from completion rather than wall-clock time so a long-running fix
// cannot overlap the next process scan. Errors are contained here because this
// loop runs outside a request lifecycle and must keep retrying autonomously.
async function runCheckCycle() {
try {
await checkAndFixProcesses();
} catch (error) {
console.error(`❌ [Autofixer] Process check failed: ${error?.message || String(error)}`);
} finally {
scheduleNextCheck();
}
}
// Main loop
async function main() {
console.log(`🚀 [Autofixer] Starting PortOS Autofixer daemon`);
console.log(`⏱️ [Autofixer] Check interval: ${CHECK_INTERVAL / 60000} minutes`);
console.log(`⏳ [Autofixer] Fix cooldown: ${FIX_COOLDOWN / 60000} minutes per process`);
await ensureHistoryDir();
await sweepInterruptedRepairs().catch((err) => {
console.error(`❌ [Autofixer] Worktree sweep failed: ${err.message}`);
});
// Run immediately, then wait a full interval after each pass completes.
await runCheckCycle();
}
// Graceful shutdown
async function shutdown() {
if (shuttingDown) return;
shuttingDown = true;
if (checkTimer) {
clearTimeout(checkTimer);
checkTimer = null;
}
console.log(`\n🛑 [Autofixer] Shutting down...`);
// Stop the in-flight provider (its detached group) and record the attempt.
// Worktree removal is left to the next startup's sweep, the guarantee.
signalDetachedGroups('SIGTERM');
if (activeRepair) {
const { outputBuffer, ...repair } = activeRepair;
await recordInterrupted({ ...repair, output: outputBuffer.join('') }).catch((err) => {
console.error(`❌ [Autofixer] Could not record interrupted session: ${err.message}`);
});
}
process.exit(0);
}
process.on('SIGINT', shutdown);
process.on('SIGTERM', shutdown);
// Start
main().catch(error => {
console.error('💥 [Autofixer] Fatal error:', error?.message || String(error), error?.stack);
process.exit(1);
});