Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

18.9 KB
Raw Blame
import { publishRuntimeFiles } from '../lib/runtimeFilePublication.js';
/**
 * Loops Service
 *
 * Manages recurring AI CLI loops — the web equivalent of Claude Code's `/loop`.
 * Each loop spawns a headless AI run on a configurable interval via the in-tree aiToolkit,
 * so any configured provider (Claude, Antigravity, Codex, etc.) can power loops.
 * Output streams in real-time via EventEmitter → Socket.IO.
 */

import { EventEmitter } from 'events';
import { join } from 'path';
import { PATHS, ensureDir, atomicWrite, tryReadFile } from '../lib/fileUtils.js';
import { randomUUID } from 'crypto';
import { ServerError } from '../lib/errorHandler.js';
import { MIN_INTERVAL_MS, parseInterval } from '../lib/loopInterval.js';
import { createRun } from './runner.js';
import { resolveProviderAndModel, runPromptThroughProvider } from './promptRunner.js';
import { listSelectableProviders, getActiveProvider } from './providers.js';

export const loopEvents = new EventEmitter();

const LOOPS_FILE = join(PATHS.data, 'loops.json');
const LOOPS_OUTPUT_DIR = join(PATHS.data, 'loops');
const DEFAULT_TIMEOUT_MS = 300_000;

const activeLoops = new Map();

// Overdue loops restored at boot catch up once, spaced apart so they do not
// all start together.
const CATCHUP_BASE_DELAY_MS = 1000;
const CATCHUP_STAGGER_MS = 10_000;
const MAX_TIMER_DELAY_MS = 2 ** 31 - 1;

function formatInterval(ms) {
  if (ms >= 86_400_000) return `${ms / 86_400_000}d`;
  if (ms >= 3_600_000) return `${ms / 3_600_000}h`;
  if (ms >= 60_000) return `${ms / 60_000}m`;
  return `${ms / 1000}s`;
}

// Serializes load-modify-save cycles to prevent concurrent updates from
// racing each other. Every public mutator that does a read-modify-write must
// go through this tail so writes are always ordered.
let loopsTail = Promise.resolve();

async function loadLoops() {
  const raw = await tryReadFile(LOOPS_FILE);
  if (raw === null) return [];
  try {
    return JSON.parse(raw);
  } catch {
    console.error(`❌ [loops] Failed to parse loops file — resetting to empty state`);
    return [];
  }
}

async function saveLoops(loops) {
  await atomicWrite(LOOPS_FILE, loops);
}

/**
 * Queue a mutator that does load-modify-save on the loops file. The mutator
 * receives the current loops array and should return the modified array (or
 * undefined to skip saving). Serializes all writes through one promise tail.
 */
function withLoopsTail(mutatorFn) {
  loopsTail = loopsTail.catch(() => {}).then(mutatorFn);
  return loopsTail;
}

async function executeIteration(loopId) {
  const id = loopId;
  const active = activeLoops.get(id);
  if (!active || active.running) return;

  // Claim the slot before the disk read below so a slow read cannot admit a
  // second overlapping iteration through the same guard.
  active.running = true;

  try {
    // Re-read the record on every tick so an edit through updateLoop (prompt,
    // provider, cwd, timeout, name) takes effect on the NEXT iteration instead of
    // at the next restart. Previously the interval closed over the record object
    // it was armed with, so edits persisted to disk but never reached the run.
    const loops = await loadLoops();
    const loop = loops.find(l => l.id === id);
    if (!loop) {
      active.running = false;
      return;
    }

    active.iterationCount++;
    const iterationNum = active.iterationCount;

    loopEvents.emit('iteration:start', { id, iteration: iterationNum, timestamp: Date.now() });

    let { provider } = await resolveProviderAndModel({ providerId: loop.providerId })
      .catch(() => ({ provider: null }));
    if (!provider) {
      const msg = 'No AI provider available';
      active.running = false;
      loopEvents.emit('iteration:error', { id, iteration: iterationNum, error: msg, timestamp: Date.now() });
      console.error(`❌ Loop ${id}: ${msg}`);
      return;
    }

    const outputLines = [];

    const onData = (text) => {
      const lines = text.split('\n').filter(l => l.trim());
      for (const line of lines) {
        outputLines.push(line);
        loopEvents.emit('output', { id, iteration: iterationNum, line, timestamp: Date.now() });
      }
    };

    const onComplete = async (metadata) => {
      const result = outputLines.join('\n');
      const iterResult = {
        iteration: iterationNum,
        exitCode: metadata.exitCode,
        success: metadata.success,
        output: result,
        lineCount: outputLines.length,
        duration: metadata.duration,
        provider: provider.name,
        // The model the central handler actually ran (set by the .then below
        // when the run resolved), with provider.defaultModel as the fallback
        // for the failure path where executedModel is still null.
        model: metadata.model || provider.defaultModel,
        timestamp: Date.now()
      };

      active.lastResult = iterResult;
      active.history.push({
        iteration: iterationNum,
        exitCode: metadata.exitCode,
        success: metadata.success,
        summary: result.slice(0, 500),
        duration: metadata.duration,
        provider: provider.name,
        timestamp: Date.now()
      });
      if (active.history.length > 50) active.history.shift();

      active.running = false;
      active.runId = null;

      const outputPath = join(LOOPS_OUTPUT_DIR, `${id}-${iterationNum}.txt`);
      await publishRuntimeFiles([outputPath], async () => {
        await atomicWrite(outputPath, result);
        await updatePersistedLoop(id, {
          lastRun: Date.now(),
          iterationCount: active.iterationCount,
          lastExitCode: metadata.exitCode
        });
      }).finally(() => {
        // Notify after persistence settles, including failures: live execution
        // has ended either way, and consumers must leave the running state.
        loopEvents.emit('iteration:complete', { id, ...iterResult });
      });

      console.log(`🔄 Loop ${id} iteration ${iterationNum} complete (${provider.name}, exit ${metadata.exitCode}, ${outputLines.length} lines)`);
    };

    // Pre-create the run so `active.runId` can be set before generation
    // starts (cancellation needs the id). The central handler reuses our
    // runId when provided rather than creating a second one.
    const runResult = await createRun({
      providerId: provider.id,
      prompt: loop.prompt,
      workspacePath: loop.cwd || PATHS.root,
      source: 'loop',
      sourceId: id,
      label: `Loop: ${loop.name} #${iterationNum}`
    }).catch(err => {
      active.running = false;
      loopEvents.emit('iteration:error', { id, iteration: iterationNum, error: err.message, timestamp: Date.now() });
      console.error(`❌ Loop ${id} createRun failed: ${err.message}`);
      return null;
    });

    if (!runResult) return;

    active.runId = runResult.metadata.id;

    // The toolkit's createRun may switch to a fallback provider when the
    // requested one is marked unavailable (providerStatusService). Reassign
    // `provider` to the effective one so dispatch, onComplete's
    // history/persistence side, and the `iteration:complete` event all see
    // the provider that actually ran. Without this, fallback would only
    // update the run record while the spawn still hit the dead provider.
    if (runResult.provider && runResult.provider.id !== provider.id) {
      provider = runResult.provider;
    }

    // Adapt the central handler's resolve/reject to the legacy onComplete
    // metadata shape. We can't get the runner's full metadata back from
    // runPromptThroughProvider today — it only resolves `{ text, runId,
    // model }`. Reconstruct the bits onComplete inspects (exitCode + success
    // + duration) from the promise resolution; pass through `model` so
    // iterResult records what actually ran (not just the saved default).
    const startedAt = Date.now();
    runPromptThroughProvider({
      provider,
      // A proactive fallback may pin a model that belongs only to the selected
      // fallback provider. Thread that pin through the pre-created run path so
      // execution uses the same model that createRun recorded.
      model: runResult.fallbackModel ?? null,
      prompt: loop.prompt, source: 'loop', runId: runResult.metadata.id,
      onData, timeout: loop.timeout || DEFAULT_TIMEOUT_MS,
      // loop.cwd is a user-facing setting on each loop record — without this
      // pass-through, every loop runs against PortOS's own cwd instead of
      // the directory the user picked.
      cwd: loop.cwd || PATHS.root,
    }).then(({ model: executedModel }) => {
      // Wrap `onComplete` so a throw inside the success branch doesn't fall
      // through to the chained `.catch` below and get misclassified as
      // `iteration:error`. History/persistence side is best-effort anyway.
      return Promise.resolve()
        .then(() => onComplete({ exitCode: 0, success: true, duration: Date.now() - startedAt, model: executedModel }))
        .catch((err) => {
          console.error(`❌ Loop ${id} onComplete (success branch) threw: ${err?.message || err}`);
        });
    }).catch(err => {
      // Pre-migration `executeCliRun` always invoked `onComplete` — even on
      // non-zero exit — so loop history / persistence / `iteration:complete`
      // observers saw failed runs too. The .then/.catch split here would
      // skip onComplete on failure and only emit `iteration:error`, breaking
      // anything that subscribed to completes (history.push, lastExitCode
      // persistence). Fire onComplete with a failure shape before the error
      // event so the history + persistence side stays consistent.
      const failureMetadata = {
        exitCode: 1,
        success: false,
        duration: Date.now() - startedAt,
        error: err.message,
        model: provider.defaultModel,
      };
      onComplete(failureMetadata).catch(() => { /* history write is best-effort */ });
      loopEvents.emit('iteration:error', { id, iteration: iterationNum, error: err.message, timestamp: Date.now() });
      console.error(`❌ Loop ${id} run failed: ${err.message}`);
    });
  } catch (err) {
    // Early rejection before the main run execution (e.g. loadLoops, provider
    // resolution, or createRun) — ensure active.running is reset so the next
    // scheduled tick can proceed without being permanently wedged.
    active.running = false;
    loopEvents.emit('iteration:error', {
      id,
      iteration: active.iterationCount,
      error: err?.message || String(err),
      timestamp: Date.now()
    });
    console.error(`❌ Loop ${id} iteration setup failed: ${err?.message || String(err)}`);
  }
}

/**
 * Fire-and-forget wrapper: iterations run outside the request lifecycle, so an
 * escaping rejection would become an unhandled rejection.
 */
function runIteration(loopId) {
  executeIteration(loopId).catch(err => {
    console.error(`❌ [loops] iteration error for loop ${loopId}: ${err?.stack || err?.message || String(err)}`);
  });
}

async function updatePersistedLoop(id, updates) {
  await withLoopsTail(async () => {
    const loops = await loadLoops();
    const idx = loops.findIndex(l => l.id === id);
    if (idx >= 0) {
      Object.assign(loops[idx], updates);
      await saveLoops(loops);
    }
  });
}

export async function createLoop({ prompt, interval, name, cwd, providerId, timeout, runImmediately = true }) {
  const intervalMs = typeof interval === 'number' ? interval : parseInterval(interval);
  if (!intervalMs || intervalMs < MIN_INTERVAL_MS) {
    throw new ServerError('Interval must be at least 10 seconds', { status: 400, code: 'VALIDATION_ERROR' });
  }
  if (!prompt?.trim()) {
    throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' });
  }

  await ensureDir(LOOPS_OUTPUT_DIR);

  const id = randomUUID().slice(0, 8);
  const loop = {
    id,
    name: name || prompt.slice(0, 60),
    prompt: prompt.trim(),
    intervalMs,
    cwd: cwd || null,
    providerId: providerId || null,
    timeout: timeout || null,
    status: 'running',
    createdAt: Date.now(),
    lastRun: null,
    iterationCount: 0,
    lastExitCode: null
  };

  await withLoopsTail(async () => {
    const loops = await loadLoops();
    loops.push(loop);
    await saveLoops(loops);
  });

  startLoopTimer(loop, runImmediately);

  loopEvents.emit('created', { loop });
  console.log(`🔄 Loop ${id} created: "${loop.name}" every ${formatInterval(intervalMs)}`);

  return loop;
}

// setTimeout/setInterval handles share one id space in Node, so a single
// clear covers whichever of the pending first-fire timeout or the steady
// interval is stored on the active entry.
function clearLoopTimer(active) {
  clearTimeout(active.timer);
  clearInterval(active.timer);
}

// firstDelayMs (restore path) delays the first fire past one full interval's
// worth of waiting: the first tick runs once, then the steady interval takes over.
function startLoopTimer(loop, runImmediately = false, firstDelayMs = null) {
  const runWithLogging = () => runIteration(loop.id);
  const active = {
    timer: null,
    runId: null,
    running: false,
    iterationCount: loop.iterationCount || 0,
    lastResult: null,
    history: []
  };
  if (firstDelayMs === null) {
    active.timer = setInterval(runWithLogging, loop.intervalMs);
  } else {
    active.timer = setTimeout(() => {
      active.timer = setInterval(runWithLogging, loop.intervalMs);
      runWithLogging();
    }, Math.min(Math.max(firstDelayMs, 0), MAX_TIMER_DELAY_MS));
  }
  activeLoops.set(loop.id, active);

  if (runImmediately) {
    setTimeout(runWithLogging, 100);
  }
}

export async function stopLoop(id) {
  const active = activeLoops.get(id);
  if (!active) throw new ServerError(`Loop ${id} is not running`, { status: 409, code: 'INVALID_STATE' });

  clearLoopTimer(active);
  activeLoops.delete(id);

  await withLoopsTail(async () => {
    const loops = await loadLoops();
    const idx = loops.findIndex(l => l.id === id);
    if (idx >= 0) {
      loops[idx].status = 'stopped';
      loops[idx].stoppedAt = Date.now();
      await saveLoops(loops);
    }
  });

  loopEvents.emit('stopped', { id });
  console.log(`⏹️ Loop ${id} stopped`);
}

export async function resumeLoop(id) {
  if (activeLoops.has(id)) throw new ServerError(`Loop ${id} is already running`, { status: 409, code: 'INVALID_STATE' });

  let loopRecord;
  await withLoopsTail(async () => {
    const loops = await loadLoops();
    const loop = loops.find(l => l.id === id);
    if (!loop) throw new ServerError(`Loop ${id} not found`, { status: 404, code: 'NOT_FOUND' });
    loop.status = 'running';
    loop.stoppedAt = null;
    await saveLoops(loops);
    loopRecord = loop;
  });

  await ensureDir(LOOPS_OUTPUT_DIR);
  startLoopTimer(loopRecord, true);

  loopEvents.emit('resumed', { id });
  console.log(`▶️ Loop ${id} resumed`);
  return loopRecord;
}

export async function deleteLoop(id) {
  if (activeLoops.has(id)) {
    const active = activeLoops.get(id);
    clearLoopTimer(active);
    activeLoops.delete(id);
  }

  await withLoopsTail(async () => {
    const loops = await loadLoops();
    const filtered = loops.filter(l => l.id !== id);
    await saveLoops(filtered);
  });

  loopEvents.emit('deleted', { id });
  console.log(`🗑️ Loop ${id} deleted`);
}

export async function getLoops() {
  const loops = await loadLoops();
  return loops.map(loop => {
    const active = activeLoops.get(loop.id);
    return {
      ...loop,
      isRunning: !!active,
      isExecuting: active?.running || false,
      currentIteration: active?.iterationCount || loop.iterationCount || 0,
      lastResult: active?.lastResult || null,
      history: active?.history || []
    };
  });
}

export async function getLoop(id) {
  const loops = await getLoops();
  return loops.find(l => l.id === id) || null;
}

export async function triggerLoop(id) {
  const loops = await loadLoops();
  const loop = loops.find(l => l.id === id);
  if (!loop) throw new ServerError(`Loop ${id} not found`, { status: 404, code: 'NOT_FOUND' });
  if (!activeLoops.has(id)) throw new ServerError(`Loop ${id} is not running`, { status: 409, code: 'INVALID_STATE' });

  runIteration(id);
  return { triggered: true };
}

export async function updateLoop(id, updates) {
  let updatedRecord;
  await withLoopsTail(async () => {
    const loops = await loadLoops();
    const idx = loops.findIndex(l => l.id === id);
    if (idx < 0) throw new ServerError(`Loop ${id} not found`, { status: 404, code: 'NOT_FOUND' });

    const allowed = ['name', 'prompt', 'interval', 'cwd', 'providerId', 'timeout'];
    for (const key of allowed) {
      if (updates[key] !== undefined) {
        if (key === 'interval') {
          const ms = typeof updates[key] === 'number' ? updates[key] : parseInterval(updates[key]);
          if (!ms || ms < MIN_INTERVAL_MS) throw new ServerError('Interval must be at least 10 seconds', { status: 400, code: 'VALIDATION_ERROR' });
          loops[idx].intervalMs = ms;
        } else {
          loops[idx][key] = updates[key];
        }
      }
    }

    await saveLoops(loops);
    updatedRecord = loops[idx];
  });

  if (activeLoops.has(id) && updates.interval) {
    const active = activeLoops.get(id);
    clearLoopTimer(active);
    active.timer = setInterval(() => runIteration(id), updatedRecord.intervalMs);
  }

  loopEvents.emit('updated', { loop: updatedRecord });
  return updatedRecord;
}

export async function getAvailableProviders() {
  const providers = await listSelectableProviders();
  const active = await getActiveProvider().catch(() => null);
  return {
    providers: providers
      .filter(p => p?.enabled !== false || active?.id === p.id)
      .map(p => ({
        id: p.id,
        name: p.name,
        type: p.type,
        command: p.command,
        defaultModel: p.defaultModel,
        enabled: p.enabled !== false,
        models: Array.isArray(p.models) ? p.models : [],
        harnessId: p.harnessId,
        method: p.method,
        serviceId: p.serviceId,
        gatewayBacked: p.gatewayBacked,
        ollamaBacked: p.ollamaBacked,
        effortLevels: p.effortLevels,
        effortLevelsByModel: p.effortLevelsByModel,
        hardwareCompatibility: p.hardwareCompatibility,
        modelHardwareCompatibility: p.modelHardwareCompatibility,
        unavailableReason: p.unavailableReason,
        isActive: active?.id === p.id
      })),
    activeProviderId: active?.id || null
  };
}

export async function restoreLoops() {
  await ensureDir(LOOPS_OUTPUT_DIR);
  const loops = await loadLoops();
  let restored = 0;
  let overdue = 0;
  const now = Date.now();
  for (const loop of loops) {
    if (loop.status !== 'running') continue;
    // Honor the persisted lastRun so a restart does not push the next
    // iteration out by a full interval. An overdue loop gets one staggered
    // catch-up run, never one per missed interval.
    let firstDelay = null;
    if (Number.isFinite(loop.lastRun)) {
      const remaining = loop.lastRun + loop.intervalMs - now;
      firstDelay = remaining > 0 ? remaining : CATCHUP_BASE_DELAY_MS + overdue++ * CATCHUP_STAGGER_MS;
    }
    startLoopTimer(loop, false, firstDelay);
    restored++;
  }
  if (restored > 0) {
    console.log(`🔄 Restored ${restored} active loop(s)`);
  }
}