Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.5 KB
Raw Blame
/**
 * CoS run reconciliation — the ledger read BACK against the run record.
 *
 * Slices 1 and 2 of #4540 made the lifecycle observable: every boundary appends
 * to `data/cos/run-events.jsonl`, and `projectRunStates` folds that stream into
 * "what this run is now". But the fold has been write-only so far — the ledger
 * is appended *alongside* `data/runs/{id}/metadata.json` and nothing ever
 * compared the two. When they disagree, the mutable record wins by default and
 * the disagreement is invisible, which is precisely the class of failure the
 * ledger exists to explain.
 *
 * This module is the comparison. It is pure — no disk, no clock — so every
 * finding case is a table test rather than a fixture tree. The service half
 * (`services/agentRunReconciler.js`) supplies the projections, the records, and
 * the write.
 *
 * **Repairs run one direction only: ledger → record.** A run record left open
 * because the process died between the ledger append and the metadata write can
 * be closed from the stream, because the stream holds the verdict. The reverse
 * — a record that is closed while the ledger never saw the close — is reported
 * and never "fixed": inventing the missing lifecycle event would put a fact in
 * the append-only stream that nothing ever observed, and a ledger that
 * back-fills itself from the record it is supposed to audit can no longer audit
 * it.
 */

/**
 * The closed finding vocabulary. Closed for the same reason the event kinds are:
 * the summary below is exhaustive over it, and a typo becomes a test failure
 * rather than a finding nothing counts.
 */
export const RUN_RECONCILE_FINDINGS = Object.freeze([
  // Ledger reached a verdict; the run record still has no `endTime`. The one
  // repairable case — the stream holds everything the close needs.
  'record-open',
  // The ledger names a run that has no `metadata.json` on disk. Not repairable:
  // a run record is written by the run, and synthesizing one from redacted
  // telemetry would fabricate a run history rather than recover one.
  'record-missing',
  // Both sides are closed and disagree on the verdict. Reported, never
  // repaired: the record's verdict is the one every other reader already used
  // (usage billing, task status), so silently flipping it would rewrite history
  // downstream of this module.
  'verdict-mismatch',
  // The record is closed but the ledger still reads live. The close event never
  // landed — a ledger gap, not a record defect.
  'ledger-open'
]);

/** Projection statuses that mean "this run is over, and the ledger knows how". */
const TERMINAL_STATUSES = new Set(['completed', 'failed']);

/**
 * Projection statuses that mean the run is over *as far as the process goes*,
 * even without a verdict event. `orphaned` belongs here and not in
 * TERMINAL_STATUSES: the sweep observed the process gone, so the run is
 * certainly not still working, but no `run.finalized` recorded a verdict — the
 * repair has to supply "failed" itself rather than copy one.
 */
const PROCESS_ENDED_STATUSES = new Set([...TERMINAL_STATUSES, 'orphaned']);

/** Projection statuses that contradict a CLOSED record. */
const LIVE_STATUSES = new Set(['running', 'orphaned', 'paused', 'interrupted']);

/**
 * `unknown` is deliberately in none of the three sets above. A projection with
 * only, say, a `run.pr-verified` event says nothing about whether the run is
 * open, so pairing it with any record shape is not a finding — reporting it
 * would fill the panel with rows that have no defect behind them.
 */

/** Is this projection key a real run id, or the `agent:<id>` fallback? */
export const isAgentFallbackKey = (id) => typeof id === 'string' && id.startsWith('agent:');

/**
 * Compare one projection against one run record.
 *
 * @param {object} projection - a `projectRunStates` entry
 * @param {object|null} record - the run's `metadata.json`, or null when absent
 * @returns {{finding: string, repairable: boolean, detail: object}|null}
 */
export function diffRunRecord(projection, record) {
  if (!projection || isAgentFallbackKey(projection.id)) return null;

  if (!record) {
    // Reported for any projection that has an opinion about the run's state —
    // NOT only one that saw the spawn. Retention can age out a `run.spawned`
    // while the `run.finalized` that followed it is still in the ledger, and a
    // finalized run with no record on disk is exactly the drift worth naming. A
    // projection built solely from an annotation event (a PR verdict that
    // outlived its run) folds to `unknown` and is skipped — it makes no claim
    // that a record should exist.
    if (projection.status === 'unknown') return null;
    return finding('record-missing', false, { ledgerStatus: projection.status });
  }

  const recordClosed = Boolean(record.endTime);

  if (!recordClosed) {
    if (!PROCESS_ENDED_STATUSES.has(projection.status)) return null;
    return finding('record-open', true, {
      ledgerStatus: projection.status,
      ledgerEndedAt: projection.endedAt ?? projection.lastEventAt ?? null,
      ledgerSuccess: projection.status === 'orphaned' ? false : projection.success === true
    });
  }

  if (TERMINAL_STATUSES.has(projection.status)) {
    const ledgerSuccess = projection.success === true;
    const recordSuccess = record.success === true;
    if (ledgerSuccess === recordSuccess) return null;
    return finding('verdict-mismatch', false, {
      ledgerStatus: projection.status,
      ledgerSuccess,
      recordSuccess
    });
  }

  if (LIVE_STATUSES.has(projection.status)) {
    return finding('ledger-open', false, {
      ledgerStatus: projection.status,
      recordEndedAt: record.endTime
    });
  }

  return null;
}

const finding = (kind, repairable, detail) => ({ finding: kind, repairable, detail });

/**
 * Reconcile a batch of projections against the records supplied for them.
 *
 * @param {object} input
 * @param {object[]} input.projections - `projectRunStates` output
 * @param {Map<string, object|null>} input.records - runId → metadata (null = absent)
 * @returns {{findings: object[], summary: object}}
 */
export function reconcileRunRecords({ projections = [], records = new Map() } = {}) {
  const findings = [];
  let agentOnly = 0;
  let checked = 0;

  for (const projection of projections) {
    if (isAgentFallbackKey(projection?.id)) {
      agentOnly += 1;
      continue;
    }
    checked += 1;
    const record = records.get(projection.id) ?? null;
    const diff = diffRunRecord(projection, record);
    if (!diff) continue;
    findings.push({
      runId: projection.id,
      agentId: projection.agentId ?? null,
      taskId: projection.taskId ?? null,
      lastEventAt: projection.lastEventAt ?? null,
      ...diff
    });
  }

  return { findings, summary: summarize(findings, { checked, agentOnly }) };
}

/**
 * Counts, one per finding kind, always present even at zero.
 *
 * Zero-filled on purpose: a UI reading `summary['record-open']` must get `0`
 * for a healthy install rather than `undefined`, or "no drift" and "this build
 * doesn't know that finding" render identically.
 */
function summarize(findings, { checked, agentOnly }) {
  const byFinding = Object.fromEntries(RUN_RECONCILE_FINDINGS.map((kind) => [kind, 0]));
  for (const item of findings) byFinding[item.finding] += 1;
  return {
    checked,
    agentOnly,
    findings: findings.length,
    repairable: findings.filter((item) => item.repairable).length,
    byFinding
  };
}

/**
 * The metadata patch that closes an open record from its ledger projection.
 *
 * Returns null unless the pair is genuinely a `record-open` finding, so the
 * service can call this per candidate without re-deriving the diff and can
 * never write from a finding that was only informational.
 *
 * The patch carries `reconciledFromLedger` so a later reader — a human, or the
 * usage reconciler — can tell a verdict the run reported from one the ledger
 * supplied after the fact. `duration` is derived rather than copied because a
 * run whose `run.finalized` never landed has no recorded duration either.
 *
 * @param {object} projection
 * @param {object|null} record
 * @param {string} at - ISO timestamp for the repair itself
 * @returns {object|null} shallow patch to merge into metadata.json
 */
export function planRunRecordRepair(projection, record, at) {
  const diff = diffRunRecord(projection, record);
  if (!diff || diff.finding !== 'record-open') return null;

  // Prefer the ledger's own end stamp; fall back to the repair time so the
  // record is never closed with a null `endTime` (the field every other reader
  // uses as "is this run over").
  const endTime = diff.detail.ledgerEndedAt || at;
  const success = diff.detail.ledgerSuccess;
  const startedMs = Date.parse(record?.startTime ?? '');
  const endedMs = Date.parse(endTime);
  const duration = Number.isFinite(record?.duration) ? record.duration
    : (Number.isFinite(startedMs) && Number.isFinite(endedMs) && endedMs >= startedMs ? endedMs - startedMs : null);

  const patch = {
    endTime,
    duration,
    success,
    exitCode: Number.isFinite(projection.exitCode) ? projection.exitCode : (record?.exitCode ?? null),
    reconciledFromLedger: true,
    reconciledAt: at,
    reconciledFromStatus: projection.status
  };

  if (!success) {
    patch.error = record?.error || (projection.status === 'orphaned'
      ? 'Agent process terminated unexpectedly (closed by run-event reconciliation)'
      : 'Run failed (closed by run-event reconciliation)');
    patch.errorCategory = record?.errorCategory || projection.errorCategory || 'reconciled';
  }

  return patch;
}