Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

22.4 KB
Raw Blame
/**
 * CoS Agent Index Module
 *
 * The date-bucket index + on-disk archive layout for CoS agents. Extracted from
 * the former monolithic cosAgents.js (issue #2530) so the index/migration/archive
 * concerns live in one focused module. Owns the lazy `agentIndex` singleton and
 * the directory-resolution helper the lifecycle/feedback/archive modules share.
 *
 * The `cosAgents.js` barrel that used to re-export this module is retired
 * (#3450) — callers import from here directly.
 */

import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { readFile, writeFile, rename, readdir, rm, stat, cp } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
import { AGENTS_DIR, withStateLock } from './cosState.js';
import { atomicWrite, ensureDir, safeJSONParse, tryReadFile, tryReadFileStrict } from '../lib/fileUtils.js';
import { isPlainObject } from '../lib/objects.js';
import { repairCodexTaskSummary } from './codexSummaryRepair.js';
import {
  loadCompletionOrderIndex,
  projectArchivedAgent,
  pruneCompletionOrderIndex,
  recordArchivedCompletions,
} from './cosAgentCompletionIndex.js';

const INDEX_FILE = join(AGENTS_DIR, 'index.json');

// Lightweight index mapping agentId → YYYY-MM-DD date bucket (~50KB vs 16MB full cache)
// Lazy-loaded from data/cos/agents/index.json on first access
let agentIndex = null;
let agentIndexPromise = null;

// Load agent index from disk (lazy init, singleton promise prevents concurrent migrations)
export async function loadAgentIndex() {
  if (agentIndex) return agentIndex;
  // Acquire before sharing the lazy-init promise. Otherwise an already-admitted
  // publisher could await a cold reader queued behind the cut it must drain.
  return withBackupAssetPublication(loadAgentIndexLeased);
}

async function loadAgentIndexLeased() {
  if (agentIndex) return agentIndex;
  if (agentIndexPromise) return agentIndexPromise;

  agentIndexPromise = (async () => {
    if (!existsSync(AGENTS_DIR)) {
      await ensureDir(AGENTS_DIR);
    }

    const { ok, value: content } = existsSync(INDEX_FILE)
      ? await tryReadFileStrict(INDEX_FILE)
      : { ok: true, value: null };
    if (content !== null || !ok) {
      const parsed = ok ? safeJSONParse(content, null, { allowArray: false }) : null;
      if (isPlainObject(parsed)) {
        agentIndex = new Map(Object.entries(parsed));
        console.log(`📂 Loaded agent index: ${agentIndex.size} entries`);
      } else {
        // Present but unreadable or malformed is not an empty index: saving that
        // would orphan every archived run. The date buckets are authoritative, so
        // rebuild in memory; the next save persists the full map, never an empty one.
        agentIndex = await scanDateBucketIndex();
        console.error(`❌ Agent index is unreadable or malformed; rebuilt ${agentIndex.size} entries from date buckets: ${INDEX_FILE}`);
      }
    } else {
      // No index yet — run migration from flat dirs to date buckets
      agentIndex = await migrateAgentsToDateBuckets();
    }

    return agentIndex;
  })().catch(err => {
    agentIndexPromise = null;
    throw err;
  });

  return agentIndexPromise;
}

/**
 * Refresh the primary lookup after a live file restore. The caller holds the
 * snapshot cut, which has drained lazy migrations and archive publishers, and
 * the CoS restore queues. Never call loadAgentIndex here: missing-index
 * migration would wait for admission behind the cut this restore owns.
 * Clear both singleton slots before reading so a failed reload cannot retain
 * a mapping to files the transfer has already replaced.
 */
export async function reloadAgentIndexAfterRestore() {
  agentIndex = null;
  agentIndexPromise = null;
  const content = await readFile(INDEX_FILE, 'utf8').catch(error => {
    if (error.code === 'ENOENT') return null;
    throw error;
  });
  // Defer absent-index migration until the first reader after cut release.
  if (content === null) return;
  const parsed = JSON.parse(content);
  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
    throw new Error('Restored CoS agent index must be an object');
  }
  agentIndex = new Map(Object.entries(parsed));
}

// Persist agent index to disk via the shared atomicWrite helper (temp file + rename,
// with Windows backup-swap fallback). Without atomic semantics a mid-write crash
// truncates index.json and on next boot the date-bucket migration would silently
// re-run (or worse, drop already-archived agents from the lookup).
export async function saveAgentIndex() {
  if (!agentIndex) return;
  const obj = Object.fromEntries(agentIndex);
  await atomicWrite(INDEX_FILE, obj).catch(err => {
    console.error(`❌ Failed to save agent index: ${err.message}`);
    throw err;
  });
  // The completion-order projection is a subset of this keyspace, so every
  // delete / retention prune / clear-completed sweep drops its rows here rather
  // than in a call site of its own.
  await pruneCompletionOrderIndex(new Set(agentIndex.keys()));
}

/**
 * Record the completion-order projection for runs that just landed in the
 * archive. Archive sites call this right after `idx.set(...)` and before
 * `saveAgentIndex()`, which would otherwise prune an entry whose id the legacy
 * index does not yet own.
 */
export async function recordArchivedAgentOrder(agents) {
  const pairs = [];
  for (const agent of agents) {
    const agentId = agent?.id;
    if (typeof agentId === 'string' && agentId) pairs.push([agentId, projectArchivedAgent(agent)]);
  }
  return recordArchivedCompletions(pairs);
}

/**
 * Merge a batch of {agentId, date} pairs into the agent index. Used by the
 * peer-sync CoS-history receiver (#1650): after pulling a peer's completed-agent
 * archives onto local disk, the date-bucket index must learn the new agentIds so
 * the history UI lists them. Idempotent union — only valid YYYY-MM-DD-bucketed
 * pairs are accepted. Returns the number of NEW entries added.
 *
 * A pre-existing agentId is treated as already-owned and is NEVER overwritten,
 * even when the incoming `date` differs: agent ids are generated independently
 * per instance, so an id collision (or a restored local archive) must not be able
 * to repoint a local agent's bucket at a peer's date — that would make
 * `getAgent()` read the wrong (or non-existent) directory and hide the local
 * agent's own history. First write wins; the local entry is authoritative.
 */
export async function addAgentArchivesToIndex(pairs) {
  if (!Array.isArray(pairs) || pairs.length === 0) return 0;
  const idx = await loadAgentIndex();
  let added = 0;
  let valid = false;
  for (const pair of pairs) {
    const agentId = pair?.agentId;
    const date = pair?.date;
    if (typeof agentId !== 'string' || !agentId) continue;
    if (typeof date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(date)) continue;
    valid = true;
    if (idx.has(agentId)) continue; // already owned — never overwrite (id is authoritative)
    idx.set(agentId, date);
    added += 1;
  }
  // A failed prior save can leave a valid pair in memory only. Persist again
  // before the peer receiver may mark that manifest reconciled.
  if (valid) await saveAgentIndex();
  return added;
}

/**
 * Drop index rows whose `<date>/<id>/` archive directory no longer exists
 * (legacy rows left by retired whole-run age deletion). Absence is rechecked
 * under the state lock against the live mapping, because archive writers create
 * the directory before indexing it. Returns the number of rows removed.
 */
export async function removeDanglingAgentIndexEntries(ids) {
  if (!Array.isArray(ids) || ids.length === 0) return 0;
  return withBackupAssetPublication(() => withStateLock(async () => {
    const idx = await loadAgentIndex();
    const removed = [];
    for (const id of new Set(ids)) {
      const date = idx.get(id);
      if (typeof date !== 'string' || existsSync(join(AGENTS_DIR, date, id))) continue;
      idx.delete(id);
      removed.push([id, date]);
    }
    if (!removed.length) return 0;
    try { await saveAgentIndex(); }
    catch (err) { for (const [id, date] of removed) idx.set(id, date); throw err; }
    return removed.length;
  }));
}

// Resolve the correct directory for an agent (running = flat, completed = date bucket)
export function getAgentDir(agentId, dateString) {
  if (dateString) return join(AGENTS_DIR, dateString, agentId);
  // Check index for date bucket
  const date = agentIndex?.get(agentId);
  if (date) return join(AGENTS_DIR, date, agentId);
  // Fallback to flat dir (running agents or pre-migration)
  return join(AGENTS_DIR, agentId);
}

/**
 * Move an agent's run directory (output, prompt, metadata, raw.txt.gz) to `targetDir`.
 *
 * `rename` is atomic but can fail (EXDEV across mounts, EPERM/EBUSY on Windows
 * while a handle is open). The fallback copies BYTES (`cp`, so the gzip
 * recording is not re-encoded as UTF-8 text and subdirectories survive) and
 * removes the source only after the whole copy succeeded. A failed copy rolls
 * back the partial target that this call created — `existsSync(targetDir)` is
 * what every later archive attempt reads as "already archived", so leaving it
 * would strand the rest of the run's files — and rethrows with the source
 * intact. Callers guarantee `targetDir` did not exist before the call.
 */
export async function moveAgentDir(flatDir, targetDir) {
  try {
    await rename(flatDir, targetDir);
    return;
  } catch {
    // Fall through to the copy-then-remove path.
  }
  const targetExisted = existsSync(targetDir);
  try {
    await cp(flatDir, targetDir, { recursive: true, force: false, errorOnExist: true });
  } catch (err) {
    if (!targetExisted) {
      await rm(targetDir, { recursive: true, force: true })
        .catch(rmErr => console.error(`❌ Failed to roll back partial archive ${targetDir}: ${rmErr.message}`));
    }
    throw err;
  }
  await rm(flatDir, { recursive: true, force: true });
}

// Migrate flat agent-* directories into YYYY-MM-DD date buckets
// Runs once when index.json doesn't exist. Idempotent — no-op if already migrated.
async function migrateAgentsToDateBuckets() {
  return withBackupAssetPublication(migrateAgentsToDateBucketsLeased);
}

// Index every agent already archived under a YYYY-MM-DD bucket directory.
async function scanDateBucketIndex(entries) {
  const index = new Map();
  const dirEntries = entries ?? await readdir(AGENTS_DIR, { withFileTypes: true });
  const dateDirPattern = /^\d{4}-\d{2}-\d{2}$/;
  for (const entry of dirEntries) {
    if (!entry.isDirectory() || !dateDirPattern.test(entry.name)) continue;
    const dateStr = entry.name;
    const dateDir = join(AGENTS_DIR, dateStr);
    // A listing failure other than a vanished bucket must not read as an empty
    // bucket: the caller may persist this map as the authoritative index.
    const agentDirs = await readdir(dateDir, { withFileTypes: true }).catch(err => {
      if (err.code === 'ENOENT') return [];
      throw err;
    });
    for (const agentEntry of agentDirs) {
      if (agentEntry.isDirectory() && agentEntry.name.startsWith('agent-')) {
        index.set(agentEntry.name, dateStr);
      }
    }
  }
  return index;
}

async function migrateAgentsToDateBucketsLeased() {
  if (!existsSync(AGENTS_DIR)) {
    await ensureDir(AGENTS_DIR);
    await atomicWrite(INDEX_FILE, {});
    console.log('📂 Created empty agent index (no agents to migrate)');
    return new Map();
  }

  const entries = await readdir(AGENTS_DIR, { withFileTypes: true });

  // Also scan existing date-bucket dirs to include them in the index
  const index = await scanDateBucketIndex(entries);

  // Find flat agent-* dirs that need migration
  const flatAgentDirs = entries.filter(e => e.isDirectory() && e.name.startsWith('agent-'));

  if (flatAgentDirs.length === 0) {
    await atomicWrite(INDEX_FILE, Object.fromEntries(index));
    console.log(`📂 Agent index built: ${index.size} entries (no flat dirs to migrate)`);
    return index;
  }

  console.log(`📦 Migrating ${flatAgentDirs.length} agents into date buckets...`);
  let migrated = 0;
  let skipped = 0;

  for (const entry of flatAgentDirs) {
    const agentId = entry.name;
    const agentDir = join(AGENTS_DIR, agentId);
    const metaPath = join(agentDir, 'metadata.json');

    let dateStr = null;

    // Try to get date from metadata
    if (existsSync(metaPath)) {
      const content = await tryReadFile(metaPath);
      if (content) {
        const raw = safeJSONParse(content, null);
        if (raw?.completedAt) {
          dateStr = raw.completedAt.slice(0, 10); // YYYY-MM-DD
        }
      }
    }

    // Fallback: directory mtime
    if (!dateStr) {
      const dirStat = await stat(agentDir).catch(() => null);
      if (dirStat?.mtime) {
        dateStr = dirStat.mtime.toISOString().slice(0, 10);
      }
    }

    if (!dateStr) {
      console.log(`⚠️ Cannot determine date for ${agentId}, skipping`);
      skipped++;
      continue;
    }

    // Move into date bucket
    const bucketDir = join(AGENTS_DIR, dateStr);
    await ensureDir(bucketDir);
    const targetDir = join(bucketDir, agentId);

    // If target already exists (partial previous migration), skip
    if (existsSync(targetDir)) {
      index.set(agentId, dateStr);
      migrated++;
      continue;
    }

    await rename(agentDir, targetDir).catch(async (renameErr) => {
      // rename can fail across filesystems — fall back to copy+delete
      console.log(`⚠️ Rename failed for ${agentId}, using copy: ${renameErr.message}`);
      try {
        await ensureDir(targetDir);
        const files = await readdir(agentDir);
        for (const file of files) {
          const content = await readFile(join(agentDir, file));
          await writeFile(join(targetDir, file), content);
        }
        await rm(agentDir, { recursive: true });
      } catch (copyErr) {
        console.error(`❌ Copy fallback failed for ${agentId}: ${copyErr.message}`);
        // Clean up partially-created target to avoid skipping on next startup
        await rm(targetDir, { recursive: true, force: true }).catch(() => {});
        throw copyErr;
      }
    });

    index.set(agentId, dateStr);
    migrated++;
  }

  // Persist index
  await atomicWrite(INDEX_FILE, Object.fromEntries(index));
  const uniqueDates = new Set(index.values()).size;
  const parts = [`📦 Migrated ${migrated} agents into date buckets (${uniqueDates} unique dates)`];
  if (skipped > 0) parts.push(`skipped ${skipped} undatable`);
  console.log(parts.join(', '));

  return index;
}

// Compatibility entry point for older callers. Whole-run age deletion is retired:
// metadata, feedback and parsed output are permanent. Raw recording maintenance
// is handled separately by cosAgentStorage, without dropping history indexes.
export async function pruneOldAgentArchives() {
  return { pruned: 0 };
}

// Get available agent date buckets with counts, sorted descending
export async function getAgentDates() {
  const idx = await loadAgentIndex();
  const dateCounts = {};
  for (const date of idx.values()) {
    dateCounts[date] = (dateCounts[date] || 0) + 1;
  }
  return Object.entries(dateCounts)
    .map(([date, count]) => ({ date, count }))
    .sort((a, b) => b.date.localeCompare(a.date));
}

// Agent ids grouped by date bucket for the requested dates, read straight from
// the in-memory id→date index (strings only, no disk I/O) in a single pass no
// matter how many dates are asked for. Returns a Map keyed by every requested
// date, with `[]` for dates the index doesn't know.
//
// Callers that already hold live agent records use this to learn WHICH ids a
// date owns before deciding whether the on-disk bucket is worth reading — see
// `collectCompletedAgents` in cosReports.js.
export async function getAgentIdsForDates(dates) {
  const byDate = new Map(dates.map(date => [date, []]));
  const idx = await loadAgentIndex();
  for (const [agentId, bucket] of idx.entries()) {
    byDate.get(bucket)?.push(agentId);
  }
  return byDate;
}

// Read one archived record from its date bucket, normalized the way every
// archive reader expects: the transcript is dropped (it is hydrated on demand by
// GET /agents/:id), a missing id falls back to the directory name, and a record
// written before `status` was persisted reads as completed.
async function readArchivedAgentMetadata(date, agentDirName, { repairSummaries = true } = {}) {
  const agentDir = join(AGENTS_DIR, date, agentDirName);
  const content = await tryReadFile(join(agentDir, 'metadata.json'));
  if (!content) return null;
  const raw = safeJSONParse(content, null);
  if (!raw) return null;
  const { output, ...rest } = raw;
  const agent = { ...rest, id: raw.id || raw.agentId || agentDirName, status: raw.status || 'completed' };
  const repaired = repairSummaries && await repairCodexTaskSummary(agentDir, agent);
  if (repaired) agent.metadata = { ...agent.metadata, taskSummary: repaired };
  return agent;
}

// Get completed agents for a specific date bucket
export async function getAgentsByDate(date, { repairSummaries = true } = {}) {
  const dateDir = join(AGENTS_DIR, date);
  if (!existsSync(dateDir)) return [];

  const entries = await readdir(dateDir, { withFileTypes: true });
  const agentDirs = entries.filter(e => e.isDirectory() && e.name.startsWith('agent-'));
  const agents = [];

  // Batch reads in chunks of 50 to avoid fd exhaustion on large date buckets
  const BATCH_SIZE = 50;
  for (let i = 0; i < agentDirs.length; i += BATCH_SIZE) {
    const batch = agentDirs.slice(i, i + BATCH_SIZE);
    const reads = batch.map(async (entry) => {
      const agent = await readArchivedAgentMetadata(date, entry.name, { repairSummaries });
      if (agent) agents.push(agent);
    });
    await Promise.allSettled(reads);
  }

  return agents.sort((a, b) => new Date(b.completedAt || 0) - new Date(a.completedAt || 0));
}

/**
 * One bounded page of completed runs, newest first.
 *
 * The cursor is still `archive day | completion timestamp | id`, byte-identical
 * to what the route round-trips, so a page taken before this projection existed
 * still resolves after it. What changed is where the ordering comes from: the
 * completion-order projection holds every archived run's timestamp in memory, so
 * a warm index answers a page with at most `limit + 1` metadata reads however
 * many runs share the day — instead of re-reading the whole day for every page
 * inside it.
 *
 * A day that still owns an unprojected id (a fresh federation import, an install
 * that has not run the backfill) is read in full exactly as before, and only if
 * the walk actually reaches it; that read backfills the projection so the next
 * page skips it. The legacy id→date index stays authoritative for WHICH archived
 * runs exist — it already is for `total`, `getAgentDir`, and every feedback and
 * report reader — so an archive directory missing from it is repaired by the
 * date-bucket migration, not discovered here.
 */
export async function getCompletedAgentPage({ liveAgents = [], limit = 25, cursor } = {}) {
  const idx = await loadAgentIndex();
  const order = await loadCompletionOrderIndex();
  const live = liveAgents.filter(agent => agent.status === 'completed');
  const liveById = new Map(live.map(agent => [agent.id, agent]));
  const dayOf = agent => (agent.completedAt || agent.startedAt || '1970-01-01').slice(0, 10);
  const keyOf = (day, completedAt, id) => `${day}|${completedAt || ''}|${id}`;
  const cursorDay = cursor ? cursor.slice(0, 10) : null;

  // Group every candidate by archive day WITHOUT touching disk. A day owning an
  // id the projection does not know is FLAGGED, not read — the walk below pays
  // for it only if the requested page actually reaches that day.
  const byDay = new Map();
  const dayBucket = (day) => {
    let bucket = byDay.get(day);
    if (!bucket) byDay.set(day, bucket = { entries: new Map(), unprojected: false });
    return bucket;
  };
  for (const [agentId, day] of idx) {
    if (liveById.has(agentId)) continue; // the live record is fresher — added below
    const projected = order.get(agentId);
    if (!projected) { dayBucket(day).unprojected = true; continue; }
    if (!projected.completed) continue;
    dayBucket(day).entries.set(agentId, { id: agentId, day, key: keyOf(day, projected.completedAt, agentId) });
  }
  for (const agent of live) {
    const day = idx.get(agent.id) || dayOf(agent);
    dayBucket(day).entries.set(agent.id, { id: agent.id, day, agent, key: keyOf(day, agent.completedAt, agent.id) });
  }

  const total = new Set([...idx.keys(), ...live.map(agent => agent.id)]).size;
  const days = [...byDay.keys()].sort().reverse();
  const newestFirst = (a, b) => (a.key < b.key ? 1 : a.key > b.key ? -1 : 0);

  const hydrateDay = async (day, bucket) => {
    bucket.unprojected = false;
    const learned = [];
    for (const agent of await getAgentsByDate(day, { repairSummaries: false })) {
      learned.push([agent.id, projectArchivedAgent(agent)]);
      const existing = bucket.entries.get(agent.id);
      if (existing) { existing.agent ||= agent; continue; }
      if (agent.status !== 'completed' || !idx.has(agent.id)) continue;
      bucket.entries.set(agent.id, { id: agent.id, day, agent, key: keyOf(day, agent.completedAt, agent.id) });
    }
    await recordArchivedCompletions(learned);
  };

  let dayCursor = 0;
  let queued = [];
  const queueNextDay = async () => {
    while (queued.length === 0 && dayCursor < days.length) {
      const day = days[dayCursor++];
      if (cursorDay && day > cursorDay) continue;
      const bucket = byDay.get(day);
      if (bucket.unprojected) await hydrateDay(day, bucket);
      queued = [...bucket.entries.values()].filter(entry => !cursor || entry.key < cursor).sort(newestFirst);
    }
    return queued.length > 0;
  };

  // limit + 1 rows prove whether another page exists. A row whose metadata went
  // missing or unparseable between the index read and now is skipped and refilled
  // from the same ordered queue, so a corrupt archive shortens no page.
  const selected = [];
  while (selected.length <= limit && (queued.length > 0 || await queueNextDay())) {
    const batch = queued.splice(0, limit + 1 - selected.length);
    const hydrated = await Promise.all(batch.map(async (entry) => (entry.agent ? entry : {
      ...entry,
      agent: await readArchivedAgentMetadata(entry.day, entry.id, { repairSummaries: false }),
    })));
    for (const entry of hydrated) {
      if (entry.agent?.status === 'completed') selected.push(entry);
    }
  }

  return {
    items: selected.slice(0, limit).map(entry => entry.agent),
    total,
    nextCursor: selected.length > limit ? selected[limit - 1].key : null,
  };
}