Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

10.1 KB
Raw Blame
/**
 * CoS run reconciliation service — the I/O half of `lib/agentRunReconcile.js`.
 *
 * Completes the loop #4540 opened: slices 1 and 2 made every lifecycle boundary
 * append to the ledger, and this reads that stream back against the durable run
 * records in `data/runs/{id}/metadata.json`.
 *
 * Two operations, deliberately separated:
 *
 * - `getRunReconciliation()` is a pure read. It is what the diagnostic view and
 *   the GET route call, and it can never change anything.
 * - `repairRunRecords()` closes the records the ledger proves are finished. It
 *   only ever runs from an explicit POST — never from boot, never from a timer.
 *   A repair rewrites a user-visible run record, and the situation that calls
 *   for one (a record left open by a crash) is also the situation where a
 *   background sweep is most likely to race whatever recovery is already in
 *   flight.
 *
 * The record load is bounded by the ledger, not by the filesystem: only run ids
 * the projections actually name are read, so this never becomes a scan of a
 * `data/runs` directory holding thousands of historical runs.
 */

import { join } from 'path';
import { PATHS, readJSONFile, atomicWrite, sha256Text } from '../lib/fileUtils.js';
import { RUN_EVENT_READ_LIMITS } from '../lib/agentRunEvents.js';
import { reconcileRunRecords, planRunRecordRepair, isAgentFallbackKey } from '../lib/agentRunReconcile.js';
import { getRunProjections, appendRunEvent } from './agentRunEventLog.js';

/**
 * A run id is about to become a filesystem path, and it arrives from a file on
 * disk rather than from this process. The ledger's envelope schema bounds ids
 * by LENGTH only, so a hand-edited or truncated-and-recovered line could carry
 * `../` and point the read — or the repair write — at a `metadata.json`
 * outside `data/runs`. Every id the server itself mints is a uuid, so
 * demanding a single plain path segment costs nothing and closes that.
 */
const SAFE_RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
const metadataPath = (runId) =>
  (SAFE_RUN_ID.test(runId) && runId !== '.' && runId !== '..' ? join(PATHS.runs, runId, 'metadata.json') : null);

/**
 * A parsed `metadata.json`, or null if the file held something that is not a
 * run record. `readJSONFile` returns whatever the file parsed to, and a
 * truncated or hand-edited file can parse to `[]` or `"…"` — both truthy, both
 * with no `endTime`, so an unguarded read would call a corrupt file an open run
 * and then spread an array into the repaired record.
 */
const asRecord = (value) =>
  (value && typeof value === 'object' && !Array.isArray(value) ? value : null);

const readRecord = async (runId) => {
  const path = metadataPath(runId);
  // Non-strict exception: unreadable metadata becomes null; repair planning
  // skips it (including the final re-read), so no default is written back.
  return path ? asRecord(await readJSONFile(path, null)) : null;
};

/**
 * Load the run records the given projections name.
 *
 * Absent records are stored as an explicit `null` rather than left out of the
 * map, so the diff can tell "no record on disk" from "not looked at" — the
 * sentinel-over-truthiness rule that runs through this whole feature. An
 * unusable id or a corrupt file lands on that same `null`, and so is reported
 * as `record-missing` — which is what it is from here: no readable run record.
 */
async function loadRecordsFor(projections) {
  const runIds = projections.map((p) => p.id).filter((id) => id && !isAgentFallbackKey(id));
  const records = await Promise.all(runIds.map(readRecord));
  return new Map(runIds.map((id, index) => [id, records[index]]));
}

/**
 * The idempotency key for one repair: this run, closed at this instant.
 *
 * Hashed rather than spelled out because `buildRunEvent` truncates `eventId` at
 * 128 characters, and a readable `reconcile:<runId>:<endTime>` key puts the
 * run id FIRST — so two ids sharing a long prefix would truncate to the same
 * key and the second repair would be silently suppressed as a duplicate of the
 * first. A fixed-width digest cannot be truncated into a collision. The
 * `reconcile:` prefix stays so the key is still recognizable in the ledger.
 */
const repairEventId = (runId, endTime) => `reconcile:${sha256Text(`${runId}\u0000${endTime}`).slice(0, 32)}`;

const cap = (limit) => Math.min(
  Number.isFinite(limit) && limit > 0 ? Math.floor(limit) : RUN_EVENT_READ_LIMITS.default,
  RUN_EVENT_READ_LIMITS.max
);

/**
 * Compare the ledger's projections against the run records. Read-only.
 *
 * @param {object} [options]
 * @param {string} [options.runId] - restrict to one run
 * @param {number} [options.limit] - max projections examined
 * @returns {Promise<{checkedAt: string, findings: object[], summary: object}>}
 */
export async function getRunReconciliation({ runId, limit } = {}) {
  const projections = await getRunProjections({ runId, limit: cap(limit) });
  const records = await loadRecordsFor(projections);
  return { checkedAt: new Date().toISOString(), ...reconcileRunRecords({ projections, records }) };
}

// Re-entrancy guard: one repair pass at a time. Not a concurrency defense (see
// the trust model) — a second pass launched while the first is mid-write would
// read records the first has not closed yet and plan the same repairs twice,
// producing a duplicate-looking result for work already done.
let inFlight = null;

/**
 * Close every run record the ledger proves is finished.
 *
 * Each repair is a read-modify-write against the record itself, and the record
 * is **re-read immediately before the write**: a run that finalized normally in
 * the window between the report and the repair must be left exactly as its own
 * completion path wrote it. `planRunRecordRepair` is re-run against that fresh
 * copy, so a record that closed itself in the meantime plans to nothing.
 *
 * The returned `findings`/`summary` describe the state the pass STARTED from,
 * not what is left afterwards — they are the evidence for `repaired`, and a
 * caller wanting the post-repair picture calls `getRunReconciliation()` again.
 *
 * @param {object} [options]
 * @param {string} [options.runId] - restrict to one run
 * @param {number} [options.limit] - max projections examined
 * @returns {Promise<{checkedAt: string, repaired: object[], skipped: number, findings: object[], summary: object}>}
 */
export function repairRunRecords(options = {}) {
  if (inFlight) return inFlight;
  inFlight = repairNow(options).finally(() => { inFlight = null; });
  return inFlight;
}

async function repairNow({ runId, limit } = {}) {
  const projections = await getRunProjections({ runId, limit: cap(limit) });
  const records = await loadRecordsFor(projections);
  const report = reconcileRunRecords({ projections, records });
  const byId = new Map(projections.map((p) => [p.id, p]));

  const repaired = [];
  let skipped = 0;

  for (const item of report.findings) {
    if (!item.repairable) continue;
    const projection = byId.get(item.runId);
    const path = metadataPath(item.runId);
    // Fresh read — the report above may be a few disk reads old, and the run's
    // own completion path is the one writer allowed to win that race.
    const record = await readRecord(item.runId);
    const patch = planRunRecordRepair(projection, record, new Date().toISOString());
    if (!patch) {
      skipped += 1;
      continue;
    }

    // The ledger entry goes FIRST, and a failed append aborts the repair.
    //
    // Order decides which half of a half-finished repair survives a crash, and
    // only one order is recoverable. Event-then-record: the next pass reads the
    // run as terminal, the record as still open, plans the same repair, and the
    // event's natural key dedupes the second append — self-healing. The reverse
    // leaves a record closed by an event nothing recorded, which for an orphan
    // repair reads as a permanent `ledger-open` finding no later pass can
    // resolve, because the record it would have to fix is already closed.
    //
    // `appendRunEvent` never rejects; it REPORTS failure, so the result has to
    // be read rather than awaited and ignored.
    const appended = await appendRunEvent({
      kind: 'run.reconciled',
      eventId: repairEventId(item.runId, patch.endTime),
      runId: item.runId,
      agentId: item.agentId,
      taskId: item.taskId,
      at: patch.reconciledAt,
      data: {
        fromStatus: patch.reconciledFromStatus,
        success: patch.success,
        exitCode: patch.exitCode,
        durationMs: patch.duration
      }
    });
    if (appended?.error) {
      console.error(`❌ Skipped reconciling run ${item.runId}: could not record the repair (${appended.error})`);
      skipped += 1;
      continue;
    }

    // The append did disk I/O of its own, so the record read before it is a
    // read old again. The LAST read before the write is the one that has to be
    // current — otherwise a run whose own completion path landed during the
    // append gets overwritten with a verdict this pass computed before it. A
    // re-plan that comes back null means the record closed itself; the already
    // appended event is harmless (a real `run.finalized` outranks it in the
    // fold, and its key dedupes a later retry).
    const fresh = await readRecord(item.runId);
    // Re-planned against the fresh record and the SAME instant, so the plan's
    // `endTime` — and therefore the event key already in the ledger — is
    // unchanged, while any field the plan reads off the record (a partial error
    // the run managed to write) comes from the current copy rather than the
    // stale one.
    const finalPatch = planRunRecordRepair(projection, fresh, patch.reconciledAt);
    if (!finalPatch) {
      skipped += 1;
      continue;
    }

    await atomicWrite(path, { ...fresh, ...finalPatch });
    console.log(`🩹 Closed run ${item.runId} from the event ledger: ledger read ${item.detail.ledgerStatus}, record now ${finalPatch.success ? 'success' : 'failure'}`);

    repaired.push({ runId: item.runId, from: item.detail.ledgerStatus, success: finalPatch.success, endTime: finalPatch.endTime });
  }

  return { checkedAt: new Date().toISOString(), repaired, skipped, ...report };
}