Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

12.6 KB
Raw Blame
/**
 * Autonomous Jobs — script handlers.
 *
 * Jobs of `type: 'script'` run one of these handlers directly as a function
 * (or a child process) instead of spawning an AI agent. `SCRIPT_HANDLERS` maps
 * the job's `scriptHandler` name to its implementation.
 */

import { readdir, stat } from 'fs/promises'
import { join, dirname } from 'path'
import { fileURLToPath } from 'url'
import { existsSync } from 'fs'
import { spawn } from '../../lib/childProcess.js';import { DAY, PATHS, rmGuarded } from '../../lib/fileUtils.js'
import { checkAndPrompt as autobiographyCheckAndPrompt } from '../autobiography.js'
import { runGoalCheckIn } from '../goalCheckIn.js'
import { cleanupOrphanedWorktrees, reapMergedWorktrees } from '../worktreeManager.js'
import { getActiveAgentIds } from '../agentState.js'
import { sweepStaleDoneSentinels, checkoutDirsOf } from '../agentSentinelSweep.js'
import { runSelfDiagnostics } from './selfDiagnostics.js'
import { runBrainParitySweep } from './brainParitySweep.js'

// Loaded lazily to keep the built-in handler registry free of an import cycle:
// the projection reads CoS state, while this registry is loaded by the CoS job
// store itself. The job remains deterministic and makes no provider call.
async function eidoverseProjection(options = {}) {
  const { projectEidoverseWorld } = await import('../eidoverseWorld.js')
  return projectEidoverseWorld({ ...options, compact: true })
}

// Loaded lazily: the inbox pulls in Postgres and the gh client, which this
// registry must not load at import time. Deterministic, no provider call.
async function githubNotificationsCheck() {
  const { describeScheduledCheck, runScheduledNotificationCheck } = await import('../githubNotifications.js')
  const results = await runScheduledNotificationCheck()
  const summary = describeScheduledCheck(results)
  console.log(`📬 GitHub notifications check: ${summary}`)
  return { summary, accounts: results.length }
}

/**
 * Run the moltworld-explore.mjs script as a child process (no AI agent needed).
 * Returns a summary object when the script exits.
 */
async function runMoltworldExploration() {
  const scriptPath = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'scripts', 'moltworld-explore.mjs')
  const durationMinutes = process.env.MOLTWORLD_DURATION_MINUTES || '30'

  // Lazy: agentApiAuth pulls in the session/auth stack, which this registry must not load at import time.
  const { resolveAgentApiEnv } = await import('../agentApiAuth.js');
  const apiEnv = await resolveAgentApiEnv();

  return new Promise((resolve, reject) => {
    const child = spawn('node', [scriptPath, durationMinutes], {
      stdio: ['ignore', 'pipe', 'pipe'],
      env: { ...process.env, ...apiEnv },
    })

    const output = []
    child.stdout.on('data', (chunk) => {
      const line = chunk.toString().trim()
      if (line) {
        output.push(line)
        console.log(`🌍 ${line}`)
      }
    })
    child.stderr.on('data', (chunk) => {
      const line = chunk.toString().trim()
      if (line) console.error(`🌍 ${line}`)
    })

    child.on('close', (code) => {
      if (code === 0) {
        resolve({ success: true, lines: output.length })
      } else {
        reject(new Error(`moltworld-explore.mjs exited with code ${code}`))
      }
    })

    child.on('error', (err) => reject(err))
  })
}

/**
 * Remove completed agent data directories older than 7 days, and reap leaked
 * CoS agent worktrees (dirs whose agent is no longer live).
 *
 * The worktree sweep lives here, on the daily scheduled cadence, rather than on
 * the 15-min CoS health-check hot path — running it that often gave it a wide
 * window to remove a human's in-flight `/claim` worktree mid-review.
 * `cleanupOrphanedWorktrees` itself also skips `claim-*` worktrees outright, so
 * the daily cadence is purely about reaping abandoned `agent-*` trees.
 */
async function agentDataCleanup() {
  // Active agent IDs so we never delete data / worktrees for running agents.
  // `getActiveAgentIds` lives in the side-effect-free `agentState.js`, so this
  // script job doesn't pull in the heavier `subAgentSpawner.js` orchestrator
  // (which re-exports the whole agent-lifecycle module graph) just to read the maps.
  const activeIds = new Set(getActiveAgentIds())

  // Also protect PAUSED agents. Pausing intentionally preserves the agent's
  // worktree + data dir as resume context, but a paused agent is removed from
  // the in-memory active/runner maps (so `getActiveAgentIds()` omits it) and —
  // critically — those maps are empty after a server restart while the agent's
  // `status: 'paused'` survives in the stored record. Read the persisted status
  // (ids and statuses only, never the record bodies) so a paused agent's
  // worktree/transcript is never reaped or rm'd out from under a later resume.
  // Lazy, and the repository rather than cosState, to stay off the
  // subAgentSpawner import path.
  const storedStatuses = await import('../cosAgentRepository.js')
    .then(m => m.listStoredAgentStatuses())
    .catch((err) => {
      console.warn(`⚠️ Sentinel sweep could not read stored agent statuses: ${err.message}`)
      return null
    })
  // Every agent record that has not reached an outcome yet. A superset of
  // `activeIds`: it re-adds the paused agents the in-memory maps drop, and —
  // after a restart — the interrupted runs still recorded `running`, whose
  // `.agent-done` sentinel is exactly the resume state the abandon path
  // preserved on purpose (#3202). Only the sentinel sweep reads this; the
  // worktree reaps below protect only live + paused ids, since an unfinished
  // record left behind by a dead process is precisely what they exist to collect.
  const unfinishedIds = new Set(activeIds)
  const pausedIds = new Set()
  for (const { id, status } of storedStatuses ?? []) {
    if (status === 'paused') pausedIds.add(id)
    if (status !== 'completed' && status !== 'failed') unfinishedIds.add(id)
  }
  for (const id of pausedIds) activeIds.add(id)
  // The worktree reaps run tens of seconds after the snapshot above, after the
  // sentinel sweep and transcript pruning; an agent CoS dispatches meanwhile is
  // absent from it. They re-read live liveness per tree through this (#11858).
  const getLiveProtectedIds = () => new Set([...getActiveAgentIds(), ...pausedIds])

  // Worktree-less agents run in a REAL checkout — this repo, or a managed app's
  // own repo — so a sentinel their cleanup never reached sits there as untracked
  // dirt in the user's git status forever. Sweep every workspace they can land
  // in, and every linked worktree of those repos: a worktree-backed sentinel is
  // carried off with its worktree only once that worktree is removed, and the
  // sentinel itself is what keeps a preserved one from being removed.
  // Lazy, like `eidoverseProjection` above: `apps.js` drags the PM2/status
  // subtree into this registry's static graph, and this job only needs the list
  // of repo paths (see the import-budget guard in lib/importScoping.test.js).
  const apps = await import('../apps.js').then(m => m.getAllApps()).catch((err) => {
    console.warn(`⚠️ Sentinel sweep could not list apps: ${err.message}`)
    return []
  })
  // `null` when the stored statuses could not be read: without them `unfinishedIds` is just
  // the in-memory map, which is EMPTY after a restart — sweeping on that would
  // delete the resume signal of every paused and interrupted run.
  const sentinelsRemoved = await sweepStaleDoneSentinels(
    await checkoutDirsOf([PATHS.root, ...apps.map(app => app?.repoPath)]),
    storedStatuses ? unfinishedIds : null
  ).catch((err) => {
    console.warn(`⚠️ Stale sentinel sweep failed: ${err.message}`)
    return 0
  })

  // Without the stored statuses a paused agent is indistinguishable from an
  // orphan, so everything below that deletes on "not live" waits for a run that
  // can read them.
  if (!storedStatuses) {
    console.warn('⚠️ Agent data cleanup: stored agent statuses unavailable — skipping agent dir and worktree reaping')
    return { cleaned: 0, sentinelsRemoved, mergedReaped: 0, worktreesReaped: 0, removalFailures: 0 }
  }

  const agentsDir = join(PATHS.cos, 'agents')
  let cleaned = 0
  if (existsSync(agentsDir)) {
    const entries = await readdir(agentsDir)
    const cutoff = Date.now() - 7 * DAY

    for (const entry of entries) {
      if (activeIds.has(entry)) continue
      const entryPath = join(agentsDir, entry)
      const info = await stat(entryPath).catch(() => null)
      if (!info?.isDirectory()) continue
      if (info.mtimeMs < cutoff) {
        const removed = await rmGuarded(entryPath, { recursive: true, force: true }).then(() => true, (err) => {
          console.warn(`⚠️ Failed to clean agent dir ${entry}: ${err.message}`)
          return false
        })
        if (removed) cleaned++
      }
    }
  }

  const { resolveRetirementApp } = await import('../retirementAppScope.js')
  const scope = await resolveRetirementApp(PATHS.root)
  if (!scope.app) {
    console.log(`🚫 Agent data cleanup: retaining worktrees (${scope.reason})`)
    return { cleaned, sentinelsRemoved, mergedReaped: 0, worktreesReaped: 0, removalFailures: 0, worktreeHold: scope.reason }
  }

  // First pass — SAFE reap: remove any worktree (CoS or `.claude/worktrees/`) whose
  // branch is fully merged into the default branch AND whose working tree is clean.
  // This is the bulk of the cleanup and cannot lose work (merged+clean ⇒ nothing
  // pending). It guards active agents and human `claim-*` worktrees.
  const merged = await reapMergedWorktrees(scope.app.repoPath, { activeAgentIds: getLiveProtectedIds, appId: scope.app.id }).catch((err) => {
    console.warn(`⚠️ Merged-worktree reap failed: ${err.message}`)
    return null
  })

  // Second pass — orphan integration: for inactive CoS worktrees that were NOT
  // merged (e.g. an agent died after committing but before its PR landed), attempt
  // to merge their commits back so the work isn't stranded. The helper guards
  // `claim-*` worktrees, so this can't touch a human's in-flight claim.
  // A failed preservation/receipt must not fall through to the older orphan
  // remover. Only unmerged candidates belong to that second pass.
  const retainedPaths = new Set((merged?.skipped || []).filter(row => row.reason !== 'unmerged').map(row => row.path))
  const worktreesReaped = merged ? await cleanupOrphanedWorktrees(PATHS.root, getLiveProtectedIds, { retainedPaths }).catch((err) => {
    console.warn(`⚠️ Orphaned worktree cleanup failed: ${err.message}`)
    return 0
  }) : 0

  const mergedCount = merged?.reaped?.length || 0
  // Reaps whose checkout survived removal: not counted above, surfaced here so a
  // retryable failure is visible in the job log instead of reading as a clean day.
  const removalFailures = merged?.skipped?.filter(s => s.reason === 'remove-failed').length || 0
  const worktreeHolds = merged?.skipped || [{ reason: 'merged-worktree-reap-failed' }]
  console.log(`🧹 Agent data cleanup: removed ${cleaned} dir(s) older than 7 days, ${sentinelsRemoved} stale sentinel(s), reaped ${mergedCount} merged + ${worktreesReaped} orphaned worktree(s)${removalFailures ? `, ${removalFailures} worktree removal(s) failed (retry next run)` : ''}`)
  return { cleaned, sentinelsRemoved, mergedReaped: mergedCount, worktreesReaped, removalFailures, worktreeHolds }
}

/**
 * Registry of script handlers for jobs that execute functions directly
 * instead of spawning AI agents. Key is the scriptHandler name, value is the function.
 */
const SCRIPT_HANDLERS = {
  'autobiography-prompt': autobiographyCheckAndPrompt,
  'moltworld-exploration': runMoltworldExploration,
  'agent-data-cleanup': agentDataCleanup,
  'goal-check-in': runGoalCheckIn,
  // Daily self-diagnostics (#2464): reads task-learning per-category metrics and
  // files a single `monitoring`-labeled GitHub summary of failing self-healing
  // categories. Deterministic — no LLM calls (cold-bootstrap-safe).
  'self-diagnostics': runSelfDiagnostics,
  // Federation brain-parity sweep (#4519): audits every federating peer for
  // record-level divergence the cursor-based sync cycle cannot see. Default
  // job ships disabled — deterministic, no LLM calls.
  'brain-parity-sweep': runBrainParitySweep,
  'eidoverse-projection': eidoverseProjection,
  'github-notifications': githubNotificationsCheck
  // 'layered-intelligence' retired here (#2322): LI is now a per-app scheduled
  // task, not a global autonomous-job sweep. It runs as a normal reasoning agent
  // with programmatic-I/O hooks — see taskTypeHooks.js +
  // autonomousJobs/layeredIntelligenceHooks.js.
}

export { runMoltworldExploration, agentDataCleanup, eidoverseProjection, SCRIPT_HANDLERS }