Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { publishRuntimeFiles } from '../lib/runtimeFilePublication.js';
/**
* Loops Service
*
* Manages recurring AI CLI loops — the web equivalent of Claude Code's `/loop`.
* Each loop spawns a headless AI run on a configurable interval via the in-tree aiToolkit,
* so any configured provider (Claude, Antigravity, Codex, etc.) can power loops.
* Output streams in real-time via EventEmitter → Socket.IO.
*/
import { EventEmitter } from 'events';
import { join } from 'path';
import { PATHS, ensureDir, atomicWrite, tryReadFile } from '../lib/fileUtils.js';
import { randomUUID } from 'crypto';
import { ServerError } from '../lib/errorHandler.js';
import { MIN_INTERVAL_MS, parseInterval } from '../lib/loopInterval.js';
import { createRun } from './runner.js';
import { resolveProviderAndModel, runPromptThroughProvider } from './promptRunner.js';
import { listSelectableProviders, getActiveProvider } from './providers.js';
export const loopEvents = new EventEmitter();
const LOOPS_FILE = join(PATHS.data, 'loops.json');
const LOOPS_OUTPUT_DIR = join(PATHS.data, 'loops');
const DEFAULT_TIMEOUT_MS = 300_000;
const activeLoops = new Map();
// Overdue loops restored at boot catch up once, spaced apart so they do not
// all start together.
const CATCHUP_BASE_DELAY_MS = 1000;
const CATCHUP_STAGGER_MS = 10_000;
const MAX_TIMER_DELAY_MS = 2 ** 31 - 1;
function formatInterval(ms) {
if (ms >= 86_400_000) return `${ms / 86_400_000}d`;
if (ms >= 3_600_000) return `${ms / 3_600_000}h`;
if (ms >= 60_000) return `${ms / 60_000}m`;
return `${ms / 1000}s`;
}
// Serializes load-modify-save cycles to prevent concurrent updates from
// racing each other. Every public mutator that does a read-modify-write must
// go through this tail so writes are always ordered.
let loopsTail = Promise.resolve();
async function loadLoops() {
const raw = await tryReadFile(LOOPS_FILE);
if (raw === null) return [];
try {
return JSON.parse(raw);
} catch {
console.error(`❌ [loops] Failed to parse loops file — resetting to empty state`);
return [];
}
}
async function saveLoops(loops) {
await atomicWrite(LOOPS_FILE, loops);
}
/**
* Queue a mutator that does load-modify-save on the loops file. The mutator
* receives the current loops array and should return the modified array (or
* undefined to skip saving). Serializes all writes through one promise tail.
*/
function withLoopsTail(mutatorFn) {
loopsTail = loopsTail.catch(() => {}).then(mutatorFn);
return loopsTail;
}
async function executeIteration(loopId) {
const id = loopId;
const active = activeLoops.get(id);
if (!active || active.running) return;
// Claim the slot before the disk read below so a slow read cannot admit a
// second overlapping iteration through the same guard.
active.running = true;
try {
// Re-read the record on every tick so an edit through updateLoop (prompt,
// provider, cwd, timeout, name) takes effect on the NEXT iteration instead of
// at the next restart. Previously the interval closed over the record object
// it was armed with, so edits persisted to disk but never reached the run.
const loops = await loadLoops();
const loop = loops.find(l => l.id === id);
if (!loop) {
active.running = false;
return;
}
active.iterationCount++;
const iterationNum = active.iterationCount;
loopEvents.emit('iteration:start', { id, iteration: iterationNum, timestamp: Date.now() });
let { provider } = await resolveProviderAndModel({ providerId: loop.providerId })
.catch(() => ({ provider: null }));
if (!provider) {
const msg = 'No AI provider available';
active.running = false;
loopEvents.emit('iteration:error', { id, iteration: iterationNum, error: msg, timestamp: Date.now() });
console.error(`❌ Loop ${id}: ${msg}`);
return;
}
const outputLines = [];
const onData = (text) => {
const lines = text.split('\n').filter(l => l.trim());
for (const line of lines) {
outputLines.push(line);
loopEvents.emit('output', { id, iteration: iterationNum, line, timestamp: Date.now() });
}
};
const onComplete = async (metadata) => {
const result = outputLines.join('\n');
const iterResult = {
iteration: iterationNum,
exitCode: metadata.exitCode,
success: metadata.success,
output: result,
lineCount: outputLines.length,
duration: metadata.duration,
provider: provider.name,
// The model the central handler actually ran (set by the .then below
// when the run resolved), with provider.defaultModel as the fallback
// for the failure path where executedModel is still null.
model: metadata.model || provider.defaultModel,
timestamp: Date.now()
};
active.lastResult = iterResult;
active.history.push({
iteration: iterationNum,
exitCode: metadata.exitCode,
success: metadata.success,
summary: result.slice(0, 500),
duration: metadata.duration,
provider: provider.name,
timestamp: Date.now()
});
if (active.history.length > 50) active.history.shift();
active.running = false;
active.runId = null;
const outputPath = join(LOOPS_OUTPUT_DIR, `${id}-${iterationNum}.txt`);
await publishRuntimeFiles([outputPath], async () => {
await atomicWrite(outputPath, result);
await updatePersistedLoop(id, {
lastRun: Date.now(),
iterationCount: active.iterationCount,
lastExitCode: metadata.exitCode
});
}).finally(() => {
// Notify after persistence settles, including failures: live execution
// has ended either way, and consumers must leave the running state.
loopEvents.emit('iteration:complete', { id, ...iterResult });
});
console.log(`🔄 Loop ${id} iteration ${iterationNum} complete (${provider.name}, exit ${metadata.exitCode}, ${outputLines.length} lines)`);
};
// Pre-create the run so `active.runId` can be set before generation
// starts (cancellation needs the id). The central handler reuses our
// runId when provided rather than creating a second one.
const runResult = await createRun({
providerId: provider.id,
prompt: loop.prompt,
workspacePath: loop.cwd || PATHS.root,
source: 'loop',
sourceId: id,
label: `Loop: ${loop.name} #${iterationNum}`
}).catch(err => {
active.running = false;
loopEvents.emit('iteration:error', { id, iteration: iterationNum, error: err.message, timestamp: Date.now() });
console.error(`❌ Loop ${id} createRun failed: ${err.message}`);
return null;
});
if (!runResult) return;
active.runId = runResult.metadata.id;
// The toolkit's createRun may switch to a fallback provider when the
// requested one is marked unavailable (providerStatusService). Reassign
// `provider` to the effective one so dispatch, onComplete's
// history/persistence side, and the `iteration:complete` event all see
// the provider that actually ran. Without this, fallback would only
// update the run record while the spawn still hit the dead provider.
if (runResult.provider && runResult.provider.id !== provider.id) {
provider = runResult.provider;
}
// Adapt the central handler's resolve/reject to the legacy onComplete
// metadata shape. We can't get the runner's full metadata back from
// runPromptThroughProvider today — it only resolves `{ text, runId,
// model }`. Reconstruct the bits onComplete inspects (exitCode + success
// + duration) from the promise resolution; pass through `model` so
// iterResult records what actually ran (not just the saved default).
const startedAt = Date.now();
runPromptThroughProvider({
provider,
// A proactive fallback may pin a model that belongs only to the selected
// fallback provider. Thread that pin through the pre-created run path so
// execution uses the same model that createRun recorded.
model: runResult.fallbackModel ?? null,
prompt: loop.prompt, source: 'loop', runId: runResult.metadata.id,
onData, timeout: loop.timeout || DEFAULT_TIMEOUT_MS,
// loop.cwd is a user-facing setting on each loop record — without this
// pass-through, every loop runs against PortOS's own cwd instead of
// the directory the user picked.
cwd: loop.cwd || PATHS.root,
}).then(({ model: executedModel }) => {
// Wrap `onComplete` so a throw inside the success branch doesn't fall
// through to the chained `.catch` below and get misclassified as
// `iteration:error`. History/persistence side is best-effort anyway.
return Promise.resolve()
.then(() => onComplete({ exitCode: 0, success: true, duration: Date.now() - startedAt, model: executedModel }))
.catch((err) => {
console.error(`❌ Loop ${id} onComplete (success branch) threw: ${err?.message || err}`);
});
}).catch(err => {
// Pre-migration `executeCliRun` always invoked `onComplete` — even on
// non-zero exit — so loop history / persistence / `iteration:complete`
// observers saw failed runs too. The .then/.catch split here would
// skip onComplete on failure and only emit `iteration:error`, breaking
// anything that subscribed to completes (history.push, lastExitCode
// persistence). Fire onComplete with a failure shape before the error
// event so the history + persistence side stays consistent.
const failureMetadata = {
exitCode: 1,
success: false,
duration: Date.now() - startedAt,
error: err.message,
model: provider.defaultModel,
};
onComplete(failureMetadata).catch(() => { /* history write is best-effort */ });
loopEvents.emit('iteration:error', { id, iteration: iterationNum, error: err.message, timestamp: Date.now() });
console.error(`❌ Loop ${id} run failed: ${err.message}`);
});
} catch (err) {
// Early rejection before the main run execution (e.g. loadLoops, provider
// resolution, or createRun) — ensure active.running is reset so the next
// scheduled tick can proceed without being permanently wedged.
active.running = false;
loopEvents.emit('iteration:error', {
id,
iteration: active.iterationCount,
error: err?.message || String(err),
timestamp: Date.now()
});
console.error(`❌ Loop ${id} iteration setup failed: ${err?.message || String(err)}`);
}
}
/**
* Fire-and-forget wrapper: iterations run outside the request lifecycle, so an
* escaping rejection would become an unhandled rejection.
*/
function runIteration(loopId) {
executeIteration(loopId).catch(err => {
console.error(`❌ [loops] iteration error for loop ${loopId}: ${err?.stack || err?.message || String(err)}`);
});
}
async function updatePersistedLoop(id, updates) {
await withLoopsTail(async () => {
const loops = await loadLoops();
const idx = loops.findIndex(l => l.id === id);
if (idx >= 0) {
Object.assign(loops[idx], updates);
await saveLoops(loops);
}
});
}
export async function createLoop({ prompt, interval, name, cwd, providerId, timeout, runImmediately = true }) {
const intervalMs = typeof interval === 'number' ? interval : parseInterval(interval);
if (!intervalMs || intervalMs < MIN_INTERVAL_MS) {
throw new ServerError('Interval must be at least 10 seconds', { status: 400, code: 'VALIDATION_ERROR' });
}
if (!prompt?.trim()) {
throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' });
}
await ensureDir(LOOPS_OUTPUT_DIR);
const id = randomUUID().slice(0, 8);
const loop = {
id,
name: name || prompt.slice(0, 60),
prompt: prompt.trim(),
intervalMs,
cwd: cwd || null,
providerId: providerId || null,
timeout: timeout || null,
status: 'running',
createdAt: Date.now(),
lastRun: null,
iterationCount: 0,
lastExitCode: null
};
await withLoopsTail(async () => {
const loops = await loadLoops();
loops.push(loop);
await saveLoops(loops);
});
startLoopTimer(loop, runImmediately);
loopEvents.emit('created', { loop });
console.log(`🔄 Loop ${id} created: "${loop.name}" every ${formatInterval(intervalMs)}`);
return loop;
}
// setTimeout/setInterval handles share one id space in Node, so a single
// clear covers whichever of the pending first-fire timeout or the steady
// interval is stored on the active entry.
function clearLoopTimer(active) {
clearTimeout(active.timer);
clearInterval(active.timer);
}
// firstDelayMs (restore path) delays the first fire past one full interval's
// worth of waiting: the first tick runs once, then the steady interval takes over.
function startLoopTimer(loop, runImmediately = false, firstDelayMs = null) {
const runWithLogging = () => runIteration(loop.id);
const active = {
timer: null,
runId: null,
running: false,
iterationCount: loop.iterationCount || 0,
lastResult: null,
history: []
};
if (firstDelayMs === null) {
active.timer = setInterval(runWithLogging, loop.intervalMs);
} else {
active.timer = setTimeout(() => {
active.timer = setInterval(runWithLogging, loop.intervalMs);
runWithLogging();
}, Math.min(Math.max(firstDelayMs, 0), MAX_TIMER_DELAY_MS));
}
activeLoops.set(loop.id, active);
if (runImmediately) {
setTimeout(runWithLogging, 100);
}
}
export async function stopLoop(id) {
const active = activeLoops.get(id);
if (!active) throw new ServerError(`Loop ${id} is not running`, { status: 409, code: 'INVALID_STATE' });
clearLoopTimer(active);
activeLoops.delete(id);
await withLoopsTail(async () => {
const loops = await loadLoops();
const idx = loops.findIndex(l => l.id === id);
if (idx >= 0) {
loops[idx].status = 'stopped';
loops[idx].stoppedAt = Date.now();
await saveLoops(loops);
}
});
loopEvents.emit('stopped', { id });
console.log(`⏹️ Loop ${id} stopped`);
}
export async function resumeLoop(id) {
if (activeLoops.has(id)) throw new ServerError(`Loop ${id} is already running`, { status: 409, code: 'INVALID_STATE' });
let loopRecord;
await withLoopsTail(async () => {
const loops = await loadLoops();
const loop = loops.find(l => l.id === id);
if (!loop) throw new ServerError(`Loop ${id} not found`, { status: 404, code: 'NOT_FOUND' });
loop.status = 'running';
loop.stoppedAt = null;
await saveLoops(loops);
loopRecord = loop;
});
await ensureDir(LOOPS_OUTPUT_DIR);
startLoopTimer(loopRecord, true);
loopEvents.emit('resumed', { id });
console.log(`▶️ Loop ${id} resumed`);
return loopRecord;
}
export async function deleteLoop(id) {
if (activeLoops.has(id)) {
const active = activeLoops.get(id);
clearLoopTimer(active);
activeLoops.delete(id);
}
await withLoopsTail(async () => {
const loops = await loadLoops();
const filtered = loops.filter(l => l.id !== id);
await saveLoops(filtered);
});
loopEvents.emit('deleted', { id });
console.log(`🗑️ Loop ${id} deleted`);
}
export async function getLoops() {
const loops = await loadLoops();
return loops.map(loop => {
const active = activeLoops.get(loop.id);
return {
...loop,
isRunning: !!active,
isExecuting: active?.running || false,
currentIteration: active?.iterationCount || loop.iterationCount || 0,
lastResult: active?.lastResult || null,
history: active?.history || []
};
});
}
export async function getLoop(id) {
const loops = await getLoops();
return loops.find(l => l.id === id) || null;
}
export async function triggerLoop(id) {
const loops = await loadLoops();
const loop = loops.find(l => l.id === id);
if (!loop) throw new ServerError(`Loop ${id} not found`, { status: 404, code: 'NOT_FOUND' });
if (!activeLoops.has(id)) throw new ServerError(`Loop ${id} is not running`, { status: 409, code: 'INVALID_STATE' });
runIteration(id);
return { triggered: true };
}
export async function updateLoop(id, updates) {
let updatedRecord;
await withLoopsTail(async () => {
const loops = await loadLoops();
const idx = loops.findIndex(l => l.id === id);
if (idx < 0) throw new ServerError(`Loop ${id} not found`, { status: 404, code: 'NOT_FOUND' });
const allowed = ['name', 'prompt', 'interval', 'cwd', 'providerId', 'timeout'];
for (const key of allowed) {
if (updates[key] !== undefined) {
if (key === 'interval') {
const ms = typeof updates[key] === 'number' ? updates[key] : parseInterval(updates[key]);
if (!ms || ms < MIN_INTERVAL_MS) throw new ServerError('Interval must be at least 10 seconds', { status: 400, code: 'VALIDATION_ERROR' });
loops[idx].intervalMs = ms;
} else {
loops[idx][key] = updates[key];
}
}
}
await saveLoops(loops);
updatedRecord = loops[idx];
});
if (activeLoops.has(id) && updates.interval) {
const active = activeLoops.get(id);
clearLoopTimer(active);
active.timer = setInterval(() => runIteration(id), updatedRecord.intervalMs);
}
loopEvents.emit('updated', { loop: updatedRecord });
return updatedRecord;
}
export async function getAvailableProviders() {
const providers = await listSelectableProviders();
const active = await getActiveProvider().catch(() => null);
return {
providers: providers
.filter(p => p?.enabled !== false || active?.id === p.id)
.map(p => ({
id: p.id,
name: p.name,
type: p.type,
command: p.command,
defaultModel: p.defaultModel,
enabled: p.enabled !== false,
models: Array.isArray(p.models) ? p.models : [],
harnessId: p.harnessId,
method: p.method,
serviceId: p.serviceId,
gatewayBacked: p.gatewayBacked,
ollamaBacked: p.ollamaBacked,
effortLevels: p.effortLevels,
effortLevelsByModel: p.effortLevelsByModel,
hardwareCompatibility: p.hardwareCompatibility,
modelHardwareCompatibility: p.modelHardwareCompatibility,
unavailableReason: p.unavailableReason,
isActive: active?.id === p.id
})),
activeProviderId: active?.id || null
};
}
export async function restoreLoops() {
await ensureDir(LOOPS_OUTPUT_DIR);
const loops = await loadLoops();
let restored = 0;
let overdue = 0;
const now = Date.now();
for (const loop of loops) {
if (loop.status !== 'running') continue;
// Honor the persisted lastRun so a restart does not push the next
// iteration out by a full interval. An overdue loop gets one staggered
// catch-up run, never one per missed interval.
let firstDelay = null;
if (Number.isFinite(loop.lastRun)) {
const remaining = loop.lastRun + loop.intervalMs - now;
firstDelay = remaining > 0 ? remaining : CATCHUP_BASE_DELAY_MS + overdue++ * CATCHUP_STAGGER_MS;
}
startLoopTimer(loop, false, firstDelay);
restored++;
}
if (restored > 0) {
console.log(`🔄 Restored ${restored} active loop(s)`);
}
}