Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* CoS Agent Index Module
*
* The date-bucket index + on-disk archive layout for CoS agents. Extracted from
* the former monolithic cosAgents.js (issue #2530) so the index/migration/archive
* concerns live in one focused module. Owns the lazy `agentIndex` singleton and
* the directory-resolution helper the lifecycle/feedback/archive modules share.
*
* The `cosAgents.js` barrel that used to re-export this module is retired
* (#3450) — callers import from here directly.
*/
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { readFile, writeFile, rename, readdir, rm, stat, cp } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
import { AGENTS_DIR, withStateLock } from './cosState.js';
import { atomicWrite, ensureDir, safeJSONParse, tryReadFile, tryReadFileStrict } from '../lib/fileUtils.js';
import { isPlainObject } from '../lib/objects.js';
import { repairCodexTaskSummary } from './codexSummaryRepair.js';
import {
loadCompletionOrderIndex,
projectArchivedAgent,
pruneCompletionOrderIndex,
recordArchivedCompletions,
} from './cosAgentCompletionIndex.js';
const INDEX_FILE = join(AGENTS_DIR, 'index.json');
// Lightweight index mapping agentId → YYYY-MM-DD date bucket (~50KB vs 16MB full cache)
// Lazy-loaded from data/cos/agents/index.json on first access
let agentIndex = null;
let agentIndexPromise = null;
// Load agent index from disk (lazy init, singleton promise prevents concurrent migrations)
export async function loadAgentIndex() {
if (agentIndex) return agentIndex;
// Acquire before sharing the lazy-init promise. Otherwise an already-admitted
// publisher could await a cold reader queued behind the cut it must drain.
return withBackupAssetPublication(loadAgentIndexLeased);
}
async function loadAgentIndexLeased() {
if (agentIndex) return agentIndex;
if (agentIndexPromise) return agentIndexPromise;
agentIndexPromise = (async () => {
if (!existsSync(AGENTS_DIR)) {
await ensureDir(AGENTS_DIR);
}
const { ok, value: content } = existsSync(INDEX_FILE)
? await tryReadFileStrict(INDEX_FILE)
: { ok: true, value: null };
if (content !== null || !ok) {
const parsed = ok ? safeJSONParse(content, null, { allowArray: false }) : null;
if (isPlainObject(parsed)) {
agentIndex = new Map(Object.entries(parsed));
console.log(`📂 Loaded agent index: ${agentIndex.size} entries`);
} else {
// Present but unreadable or malformed is not an empty index: saving that
// would orphan every archived run. The date buckets are authoritative, so
// rebuild in memory; the next save persists the full map, never an empty one.
agentIndex = await scanDateBucketIndex();
console.error(`❌ Agent index is unreadable or malformed; rebuilt ${agentIndex.size} entries from date buckets: ${INDEX_FILE}`);
}
} else {
// No index yet — run migration from flat dirs to date buckets
agentIndex = await migrateAgentsToDateBuckets();
}
return agentIndex;
})().catch(err => {
agentIndexPromise = null;
throw err;
});
return agentIndexPromise;
}
/**
* Refresh the primary lookup after a live file restore. The caller holds the
* snapshot cut, which has drained lazy migrations and archive publishers, and
* the CoS restore queues. Never call loadAgentIndex here: missing-index
* migration would wait for admission behind the cut this restore owns.
* Clear both singleton slots before reading so a failed reload cannot retain
* a mapping to files the transfer has already replaced.
*/
export async function reloadAgentIndexAfterRestore() {
agentIndex = null;
agentIndexPromise = null;
const content = await readFile(INDEX_FILE, 'utf8').catch(error => {
if (error.code === 'ENOENT') return null;
throw error;
});
// Defer absent-index migration until the first reader after cut release.
if (content === null) return;
const parsed = JSON.parse(content);
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error('Restored CoS agent index must be an object');
}
agentIndex = new Map(Object.entries(parsed));
}
// Persist agent index to disk via the shared atomicWrite helper (temp file + rename,
// with Windows backup-swap fallback). Without atomic semantics a mid-write crash
// truncates index.json and on next boot the date-bucket migration would silently
// re-run (or worse, drop already-archived agents from the lookup).
export async function saveAgentIndex() {
if (!agentIndex) return;
const obj = Object.fromEntries(agentIndex);
await atomicWrite(INDEX_FILE, obj).catch(err => {
console.error(`❌ Failed to save agent index: ${err.message}`);
throw err;
});
// The completion-order projection is a subset of this keyspace, so every
// delete / retention prune / clear-completed sweep drops its rows here rather
// than in a call site of its own.
await pruneCompletionOrderIndex(new Set(agentIndex.keys()));
}
/**
* Record the completion-order projection for runs that just landed in the
* archive. Archive sites call this right after `idx.set(...)` and before
* `saveAgentIndex()`, which would otherwise prune an entry whose id the legacy
* index does not yet own.
*/
export async function recordArchivedAgentOrder(agents) {
const pairs = [];
for (const agent of agents) {
const agentId = agent?.id;
if (typeof agentId === 'string' && agentId) pairs.push([agentId, projectArchivedAgent(agent)]);
}
return recordArchivedCompletions(pairs);
}
/**
* Merge a batch of {agentId, date} pairs into the agent index. Used by the
* peer-sync CoS-history receiver (#1650): after pulling a peer's completed-agent
* archives onto local disk, the date-bucket index must learn the new agentIds so
* the history UI lists them. Idempotent union — only valid YYYY-MM-DD-bucketed
* pairs are accepted. Returns the number of NEW entries added.
*
* A pre-existing agentId is treated as already-owned and is NEVER overwritten,
* even when the incoming `date` differs: agent ids are generated independently
* per instance, so an id collision (or a restored local archive) must not be able
* to repoint a local agent's bucket at a peer's date — that would make
* `getAgent()` read the wrong (or non-existent) directory and hide the local
* agent's own history. First write wins; the local entry is authoritative.
*/
export async function addAgentArchivesToIndex(pairs) {
if (!Array.isArray(pairs) || pairs.length === 0) return 0;
const idx = await loadAgentIndex();
let added = 0;
let valid = false;
for (const pair of pairs) {
const agentId = pair?.agentId;
const date = pair?.date;
if (typeof agentId !== 'string' || !agentId) continue;
if (typeof date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(date)) continue;
valid = true;
if (idx.has(agentId)) continue; // already owned — never overwrite (id is authoritative)
idx.set(agentId, date);
added += 1;
}
// A failed prior save can leave a valid pair in memory only. Persist again
// before the peer receiver may mark that manifest reconciled.
if (valid) await saveAgentIndex();
return added;
}
/**
* Drop index rows whose `<date>/<id>/` archive directory no longer exists
* (legacy rows left by retired whole-run age deletion). Absence is rechecked
* under the state lock against the live mapping, because archive writers create
* the directory before indexing it. Returns the number of rows removed.
*/
export async function removeDanglingAgentIndexEntries(ids) {
if (!Array.isArray(ids) || ids.length === 0) return 0;
return withBackupAssetPublication(() => withStateLock(async () => {
const idx = await loadAgentIndex();
const removed = [];
for (const id of new Set(ids)) {
const date = idx.get(id);
if (typeof date !== 'string' || existsSync(join(AGENTS_DIR, date, id))) continue;
idx.delete(id);
removed.push([id, date]);
}
if (!removed.length) return 0;
try { await saveAgentIndex(); }
catch (err) { for (const [id, date] of removed) idx.set(id, date); throw err; }
return removed.length;
}));
}
// Resolve the correct directory for an agent (running = flat, completed = date bucket)
export function getAgentDir(agentId, dateString) {
if (dateString) return join(AGENTS_DIR, dateString, agentId);
// Check index for date bucket
const date = agentIndex?.get(agentId);
if (date) return join(AGENTS_DIR, date, agentId);
// Fallback to flat dir (running agents or pre-migration)
return join(AGENTS_DIR, agentId);
}
/**
* Move an agent's run directory (output, prompt, metadata, raw.txt.gz) to `targetDir`.
*
* `rename` is atomic but can fail (EXDEV across mounts, EPERM/EBUSY on Windows
* while a handle is open). The fallback copies BYTES (`cp`, so the gzip
* recording is not re-encoded as UTF-8 text and subdirectories survive) and
* removes the source only after the whole copy succeeded. A failed copy rolls
* back the partial target that this call created — `existsSync(targetDir)` is
* what every later archive attempt reads as "already archived", so leaving it
* would strand the rest of the run's files — and rethrows with the source
* intact. Callers guarantee `targetDir` did not exist before the call.
*/
export async function moveAgentDir(flatDir, targetDir) {
try {
await rename(flatDir, targetDir);
return;
} catch {
// Fall through to the copy-then-remove path.
}
const targetExisted = existsSync(targetDir);
try {
await cp(flatDir, targetDir, { recursive: true, force: false, errorOnExist: true });
} catch (err) {
if (!targetExisted) {
await rm(targetDir, { recursive: true, force: true })
.catch(rmErr => console.error(`❌ Failed to roll back partial archive ${targetDir}: ${rmErr.message}`));
}
throw err;
}
await rm(flatDir, { recursive: true, force: true });
}
// Migrate flat agent-* directories into YYYY-MM-DD date buckets
// Runs once when index.json doesn't exist. Idempotent — no-op if already migrated.
async function migrateAgentsToDateBuckets() {
return withBackupAssetPublication(migrateAgentsToDateBucketsLeased);
}
// Index every agent already archived under a YYYY-MM-DD bucket directory.
async function scanDateBucketIndex(entries) {
const index = new Map();
const dirEntries = entries ?? await readdir(AGENTS_DIR, { withFileTypes: true });
const dateDirPattern = /^\d{4}-\d{2}-\d{2}$/;
for (const entry of dirEntries) {
if (!entry.isDirectory() || !dateDirPattern.test(entry.name)) continue;
const dateStr = entry.name;
const dateDir = join(AGENTS_DIR, dateStr);
// A listing failure other than a vanished bucket must not read as an empty
// bucket: the caller may persist this map as the authoritative index.
const agentDirs = await readdir(dateDir, { withFileTypes: true }).catch(err => {
if (err.code === 'ENOENT') return [];
throw err;
});
for (const agentEntry of agentDirs) {
if (agentEntry.isDirectory() && agentEntry.name.startsWith('agent-')) {
index.set(agentEntry.name, dateStr);
}
}
}
return index;
}
async function migrateAgentsToDateBucketsLeased() {
if (!existsSync(AGENTS_DIR)) {
await ensureDir(AGENTS_DIR);
await atomicWrite(INDEX_FILE, {});
console.log('📂 Created empty agent index (no agents to migrate)');
return new Map();
}
const entries = await readdir(AGENTS_DIR, { withFileTypes: true });
// Also scan existing date-bucket dirs to include them in the index
const index = await scanDateBucketIndex(entries);
// Find flat agent-* dirs that need migration
const flatAgentDirs = entries.filter(e => e.isDirectory() && e.name.startsWith('agent-'));
if (flatAgentDirs.length === 0) {
await atomicWrite(INDEX_FILE, Object.fromEntries(index));
console.log(`📂 Agent index built: ${index.size} entries (no flat dirs to migrate)`);
return index;
}
console.log(`📦 Migrating ${flatAgentDirs.length} agents into date buckets...`);
let migrated = 0;
let skipped = 0;
for (const entry of flatAgentDirs) {
const agentId = entry.name;
const agentDir = join(AGENTS_DIR, agentId);
const metaPath = join(agentDir, 'metadata.json');
let dateStr = null;
// Try to get date from metadata
if (existsSync(metaPath)) {
const content = await tryReadFile(metaPath);
if (content) {
const raw = safeJSONParse(content, null);
if (raw?.completedAt) {
dateStr = raw.completedAt.slice(0, 10); // YYYY-MM-DD
}
}
}
// Fallback: directory mtime
if (!dateStr) {
const dirStat = await stat(agentDir).catch(() => null);
if (dirStat?.mtime) {
dateStr = dirStat.mtime.toISOString().slice(0, 10);
}
}
if (!dateStr) {
console.log(`⚠️ Cannot determine date for ${agentId}, skipping`);
skipped++;
continue;
}
// Move into date bucket
const bucketDir = join(AGENTS_DIR, dateStr);
await ensureDir(bucketDir);
const targetDir = join(bucketDir, agentId);
// If target already exists (partial previous migration), skip
if (existsSync(targetDir)) {
index.set(agentId, dateStr);
migrated++;
continue;
}
await rename(agentDir, targetDir).catch(async (renameErr) => {
// rename can fail across filesystems — fall back to copy+delete
console.log(`⚠️ Rename failed for ${agentId}, using copy: ${renameErr.message}`);
try {
await ensureDir(targetDir);
const files = await readdir(agentDir);
for (const file of files) {
const content = await readFile(join(agentDir, file));
await writeFile(join(targetDir, file), content);
}
await rm(agentDir, { recursive: true });
} catch (copyErr) {
console.error(`❌ Copy fallback failed for ${agentId}: ${copyErr.message}`);
// Clean up partially-created target to avoid skipping on next startup
await rm(targetDir, { recursive: true, force: true }).catch(() => {});
throw copyErr;
}
});
index.set(agentId, dateStr);
migrated++;
}
// Persist index
await atomicWrite(INDEX_FILE, Object.fromEntries(index));
const uniqueDates = new Set(index.values()).size;
const parts = [`📦 Migrated ${migrated} agents into date buckets (${uniqueDates} unique dates)`];
if (skipped > 0) parts.push(`skipped ${skipped} undatable`);
console.log(parts.join(', '));
return index;
}
// Compatibility entry point for older callers. Whole-run age deletion is retired:
// metadata, feedback and parsed output are permanent. Raw recording maintenance
// is handled separately by cosAgentStorage, without dropping history indexes.
export async function pruneOldAgentArchives() {
return { pruned: 0 };
}
// Get available agent date buckets with counts, sorted descending
export async function getAgentDates() {
const idx = await loadAgentIndex();
const dateCounts = {};
for (const date of idx.values()) {
dateCounts[date] = (dateCounts[date] || 0) + 1;
}
return Object.entries(dateCounts)
.map(([date, count]) => ({ date, count }))
.sort((a, b) => b.date.localeCompare(a.date));
}
// Agent ids grouped by date bucket for the requested dates, read straight from
// the in-memory id→date index (strings only, no disk I/O) in a single pass no
// matter how many dates are asked for. Returns a Map keyed by every requested
// date, with `[]` for dates the index doesn't know.
//
// Callers that already hold live agent records use this to learn WHICH ids a
// date owns before deciding whether the on-disk bucket is worth reading — see
// `collectCompletedAgents` in cosReports.js.
export async function getAgentIdsForDates(dates) {
const byDate = new Map(dates.map(date => [date, []]));
const idx = await loadAgentIndex();
for (const [agentId, bucket] of idx.entries()) {
byDate.get(bucket)?.push(agentId);
}
return byDate;
}
// Read one archived record from its date bucket, normalized the way every
// archive reader expects: the transcript is dropped (it is hydrated on demand by
// GET /agents/:id), a missing id falls back to the directory name, and a record
// written before `status` was persisted reads as completed.
async function readArchivedAgentMetadata(date, agentDirName, { repairSummaries = true } = {}) {
const agentDir = join(AGENTS_DIR, date, agentDirName);
const content = await tryReadFile(join(agentDir, 'metadata.json'));
if (!content) return null;
const raw = safeJSONParse(content, null);
if (!raw) return null;
const { output, ...rest } = raw;
const agent = { ...rest, id: raw.id || raw.agentId || agentDirName, status: raw.status || 'completed' };
const repaired = repairSummaries && await repairCodexTaskSummary(agentDir, agent);
if (repaired) agent.metadata = { ...agent.metadata, taskSummary: repaired };
return agent;
}
// Get completed agents for a specific date bucket
export async function getAgentsByDate(date, { repairSummaries = true } = {}) {
const dateDir = join(AGENTS_DIR, date);
if (!existsSync(dateDir)) return [];
const entries = await readdir(dateDir, { withFileTypes: true });
const agentDirs = entries.filter(e => e.isDirectory() && e.name.startsWith('agent-'));
const agents = [];
// Batch reads in chunks of 50 to avoid fd exhaustion on large date buckets
const BATCH_SIZE = 50;
for (let i = 0; i < agentDirs.length; i += BATCH_SIZE) {
const batch = agentDirs.slice(i, i + BATCH_SIZE);
const reads = batch.map(async (entry) => {
const agent = await readArchivedAgentMetadata(date, entry.name, { repairSummaries });
if (agent) agents.push(agent);
});
await Promise.allSettled(reads);
}
return agents.sort((a, b) => new Date(b.completedAt || 0) - new Date(a.completedAt || 0));
}
/**
* One bounded page of completed runs, newest first.
*
* The cursor is still `archive day | completion timestamp | id`, byte-identical
* to what the route round-trips, so a page taken before this projection existed
* still resolves after it. What changed is where the ordering comes from: the
* completion-order projection holds every archived run's timestamp in memory, so
* a warm index answers a page with at most `limit + 1` metadata reads however
* many runs share the day — instead of re-reading the whole day for every page
* inside it.
*
* A day that still owns an unprojected id (a fresh federation import, an install
* that has not run the backfill) is read in full exactly as before, and only if
* the walk actually reaches it; that read backfills the projection so the next
* page skips it. The legacy id→date index stays authoritative for WHICH archived
* runs exist — it already is for `total`, `getAgentDir`, and every feedback and
* report reader — so an archive directory missing from it is repaired by the
* date-bucket migration, not discovered here.
*/
export async function getCompletedAgentPage({ liveAgents = [], limit = 25, cursor } = {}) {
const idx = await loadAgentIndex();
const order = await loadCompletionOrderIndex();
const live = liveAgents.filter(agent => agent.status === 'completed');
const liveById = new Map(live.map(agent => [agent.id, agent]));
const dayOf = agent => (agent.completedAt || agent.startedAt || '1970-01-01').slice(0, 10);
const keyOf = (day, completedAt, id) => `${day}|${completedAt || ''}|${id}`;
const cursorDay = cursor ? cursor.slice(0, 10) : null;
// Group every candidate by archive day WITHOUT touching disk. A day owning an
// id the projection does not know is FLAGGED, not read — the walk below pays
// for it only if the requested page actually reaches that day.
const byDay = new Map();
const dayBucket = (day) => {
let bucket = byDay.get(day);
if (!bucket) byDay.set(day, bucket = { entries: new Map(), unprojected: false });
return bucket;
};
for (const [agentId, day] of idx) {
if (liveById.has(agentId)) continue; // the live record is fresher — added below
const projected = order.get(agentId);
if (!projected) { dayBucket(day).unprojected = true; continue; }
if (!projected.completed) continue;
dayBucket(day).entries.set(agentId, { id: agentId, day, key: keyOf(day, projected.completedAt, agentId) });
}
for (const agent of live) {
const day = idx.get(agent.id) || dayOf(agent);
dayBucket(day).entries.set(agent.id, { id: agent.id, day, agent, key: keyOf(day, agent.completedAt, agent.id) });
}
const total = new Set([...idx.keys(), ...live.map(agent => agent.id)]).size;
const days = [...byDay.keys()].sort().reverse();
const newestFirst = (a, b) => (a.key < b.key ? 1 : a.key > b.key ? -1 : 0);
const hydrateDay = async (day, bucket) => {
bucket.unprojected = false;
const learned = [];
for (const agent of await getAgentsByDate(day, { repairSummaries: false })) {
learned.push([agent.id, projectArchivedAgent(agent)]);
const existing = bucket.entries.get(agent.id);
if (existing) { existing.agent ||= agent; continue; }
if (agent.status !== 'completed' || !idx.has(agent.id)) continue;
bucket.entries.set(agent.id, { id: agent.id, day, agent, key: keyOf(day, agent.completedAt, agent.id) });
}
await recordArchivedCompletions(learned);
};
let dayCursor = 0;
let queued = [];
const queueNextDay = async () => {
while (queued.length === 0 && dayCursor < days.length) {
const day = days[dayCursor++];
if (cursorDay && day > cursorDay) continue;
const bucket = byDay.get(day);
if (bucket.unprojected) await hydrateDay(day, bucket);
queued = [...bucket.entries.values()].filter(entry => !cursor || entry.key < cursor).sort(newestFirst);
}
return queued.length > 0;
};
// limit + 1 rows prove whether another page exists. A row whose metadata went
// missing or unparseable between the index read and now is skipped and refilled
// from the same ordered queue, so a corrupt archive shortens no page.
const selected = [];
while (selected.length <= limit && (queued.length > 0 || await queueNextDay())) {
const batch = queued.splice(0, limit + 1 - selected.length);
const hydrated = await Promise.all(batch.map(async (entry) => (entry.agent ? entry : {
...entry,
agent: await readArchivedAgentMetadata(entry.day, entry.id, { repairSummaries: false }),
})));
for (const entry of hydrated) {
if (entry.agent?.status === 'completed') selected.push(entry);
}
}
return {
items: selected.slice(0, limit).map(entry => entry.agent),
total,
nextCursor: selected.length > limit ? selected[limit - 1].key : null,
};
}