Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.8 KB
Raw Blame
import { describe, it, expect, beforeEach, afterEach, afterAll, vi } from 'vitest';
import { rmSync, mkdirSync, writeFileSync, readFileSync, existsSync } from 'fs';
import { join } from 'path';

// Both the ledger and the run records are redirected into throwaway dirs. The
// service resolves `PATHS` at module load — the same reason a real restart is
// what re-reads them — so the mock has to be hoisted above the imports.
const { LEDGER_DIR, RUNS_DIR, hooks } = await vi.hoisted(async () => {
  const { mkdtempSync: mk } = await import('fs');
  const { tmpdir: tmp } = await import('os');
  const { join: j } = await import('path');
  return {
    LEDGER_DIR: mk(j(tmp(), 'portos-reconcile-ledger-')),
    RUNS_DIR: mk(j(tmp(), 'portos-reconcile-runs-')),
    // Seams for the two tests that need to interleave a write between the
    // service's two reads of the same record, or make the ledger append fail.
    // Nothing else installs a hook.
    hooks: { afterRecordRead: null, failAppend: false }
  };
});

vi.mock('../lib/fileUtils.js', async (importOriginal) => {
  const actual = await importOriginal();
  return {
    ...actual,
    PATHS: { ...actual.PATHS, cos: LEDGER_DIR, runs: RUNS_DIR },
    readJSONFile: async (path, fallback) => {
      const value = await actual.readJSONFile(path, fallback);
      if (hooks.afterRecordRead) await hooks.afterRecordRead(path);
      return value;
    },
    appendJSONLine: async (path, line) => {
      if (hooks.failAppend) throw new Error('ledger disk full');
      return actual.appendJSONLine(path, line);
    }
  };
});

const { getRunReconciliation, repairRunRecords } = await import('./agentRunReconciler.js');
const { appendRunEvent, readRunEvents, __resetRunEventLogCache } = await import('./agentRunEventLog.js');

const ACTIVE = join(LEDGER_DIR, 'run-events.jsonl');
const ARCHIVE = join(LEDGER_DIR, 'run-events.1.jsonl');

const AT = '2026-08-18T12:00:00.000Z';
const LATER = '2026-08-18T12:05:00.000Z';
const NOW = '2026-08-18T13:00:00.000Z';

const metaPath = (runId) => join(RUNS_DIR, runId, 'metadata.json');

function writeRecord(runId, record) {
  mkdirSync(join(RUNS_DIR, runId), { recursive: true });
  writeFileSync(metaPath(runId), JSON.stringify({ id: runId, startTime: AT, ...record }));
}

const readRecord = (runId) => JSON.parse(readFileSync(metaPath(runId), 'utf8'));

const spawn = (runId) => appendRunEvent({ kind: 'run.spawned', runId, agentId: 'a1', taskId: 't1', at: AT, data: {} });
const finalize = (runId, success) => appendRunEvent({
  kind: 'run.finalized', runId, agentId: 'a1', at: LATER,
  data: { success, exitCode: success ? 0 : 1, durationMs: 300000 }
});

beforeEach(() => {
  for (const path of [ACTIVE, ARCHIVE]) if (existsSync(path)) rmSync(path);
  rmSync(RUNS_DIR, { recursive: true, force: true });
  mkdirSync(RUNS_DIR, { recursive: true });
  __resetRunEventLogCache();
  // Pinned next to the fixture timestamps so the ledger's 30-day age bound
  // can't age the fixtures out once the calendar passes them.
  vi.useFakeTimers({ toFake: ['Date'] });
  vi.setSystemTime(new Date(NOW));
});

afterEach(() => {
  hooks.afterRecordRead = null;
  hooks.failAppend = false;
  vi.useRealTimers();
});
afterAll(() => {
  rmSync(LEDGER_DIR, { recursive: true, force: true });
  rmSync(RUNS_DIR, { recursive: true, force: true });
  // The path-traversal test below deliberately writes ONE FIXTURE outside
  // RUNS_DIR (`join(RUNS_DIR, '..', 'escape')`, i.e. a sibling directory
  // literally named "escape" in the shared temp root) to prove the service
  // refuses to follow it back in. That fixture is the point of the test, not
  // application output, but it still has to be swept here — nothing else
  // owns a path outside RUNS_DIR, and left behind it fails PortOS's own
  // run-scoped temp-root leak check (#9039).
  rmSync(join(RUNS_DIR, '..', 'escape'), { recursive: true, force: true });
});

describe('getRunReconciliation', () => {
  it('reports nothing when the record matches the stream that produced it', async () => {
    writeRecord('r1', { endTime: LATER, success: true });
    await spawn('r1');
    await finalize('r1', true);

    const report = await getRunReconciliation();
    expect(report.findings).toEqual([]);
    expect(report.summary).toMatchObject({ checked: 1, repairable: 0 });
    expect(report.checkedAt).toBe(NOW);
  });

  it('finds an open record the ledger already finalized', async () => {
    writeRecord('r1', { endTime: null, success: null });
    await spawn('r1');
    await finalize('r1', false);

    const report = await getRunReconciliation();
    expect(report.findings).toHaveLength(1);
    expect(report.findings[0]).toMatchObject({ runId: 'r1', finding: 'record-open', repairable: true });
  });

  it('never writes — the record is untouched by a report', async () => {
    writeRecord('r1', { endTime: null, success: null });
    await spawn('r1');
    await finalize('r1', false);

    const before = readFileSync(metaPath('r1'), 'utf8');
    await getRunReconciliation();
    expect(readFileSync(metaPath('r1'), 'utf8')).toBe(before);
  });

  it('narrows to one run when asked', async () => {
    writeRecord('r1', { endTime: null });
    writeRecord('r2', { endTime: null });
    await spawn('r1'); await finalize('r1', false);
    await spawn('r2'); await finalize('r2', false);

    const report = await getRunReconciliation({ runId: 'r2' });
    expect(report.summary.checked).toBe(1);
    expect(report.findings.map((f) => f.runId)).toEqual(['r2']);
  });

  it('reports a run the ledger knows but that has no record on disk', async () => {
    await spawn('gone');
    await finalize('gone', true);

    const report = await getRunReconciliation();
    expect(report.findings[0]).toMatchObject({ runId: 'gone', finding: 'record-missing', repairable: false });
  });
});

describe('repairRunRecords', () => {
  it('closes an open record with the ledger verdict and records the repair as an event', async () => {
    writeRecord('r1', { endTime: null, success: null });
    await spawn('r1');
    await finalize('r1', false);

    const result = await repairRunRecords();
    expect(result.repaired).toEqual([{ runId: 'r1', from: 'failed', success: false, endTime: LATER }]);

    const record = readRecord('r1');
    expect(record).toMatchObject({ endTime: LATER, success: false, exitCode: 1, reconciledFromLedger: true, reconciledAt: NOW });
    // The run's own fields survive the merge — a repair closes a record, it
    // does not replace it.
    expect(record.startTime).toBe(AT);

    const events = await readRunEvents({ runId: 'r1', kind: 'run.reconciled' });
    expect(events).toHaveLength(1);
    expect(events[0].data).toMatchObject({ fromStatus: 'failed', success: false });
  });

  it('is idempotent — a second pass finds the record already closed', async () => {
    writeRecord('r1', { endTime: null, success: null });
    await spawn('r1');
    await finalize('r1', false);

    await repairRunRecords();
    const after = readFileSync(metaPath('r1'), 'utf8');
    const second = await repairRunRecords();

    expect(second.repaired).toEqual([]);
    expect(second.findings).toEqual([]);
    expect(readFileSync(metaPath('r1'), 'utf8')).toBe(after);
    expect(await readRunEvents({ runId: 'r1', kind: 'run.reconciled' })).toHaveLength(1);
  });

  it('leaves a run that closed itself between the report and the write alone', async () => {
    writeRecord('r1', { endTime: null, success: null });
    await spawn('r1');
    await finalize('r1', false);

    // The run's own completion path landing mid-pass. It fires after the
    // reconciler's FIRST read of the record (which still sees it open, so the
    // finding is genuinely 'record-open' and repairable) and before the fresh
    // read the repair loop takes — which is the read that has to notice.
    let reads = 0;
    hooks.afterRecordRead = (path) => {
      if (!path.includes('r1')) return;
      reads += 1;
      if (reads === 1) writeRecord('r1', { endTime: LATER, success: true, duration: 1 });
    };

    const result = await repairRunRecords();
    hooks.afterRecordRead = null;
    expect(reads).toBeGreaterThan(1);
    expect(result.repaired).toEqual([]);
    expect(result.skipped).toBe(1);
    expect(readRecord('r1')).toMatchObject({ success: true, duration: 1 });
    expect(readRecord('r1').reconciledFromLedger).toBeUndefined();
  });

  it('does not repair a verdict mismatch — the record verdict stands', async () => {
    writeRecord('r1', { endTime: LATER, success: false });
    await spawn('r1');
    await finalize('r1', true);

    const result = await repairRunRecords();
    expect(result.repaired).toEqual([]);
    expect(result.findings[0].finding).toBe('verdict-mismatch');
    expect(readRecord('r1').success).toBe(false);
  });

  it('does not invent a ledger event for a close the ledger missed', async () => {
    writeRecord('r1', { endTime: LATER, success: true });
    await spawn('r1');

    const result = await repairRunRecords();
    expect(result.findings[0].finding).toBe('ledger-open');
    expect(result.repaired).toEqual([]);
    expect(await readRunEvents({ runId: 'r1', kind: 'run.finalized' })).toEqual([]);
  });

  it('closes an orphaned run as a failure even with no verdict event', async () => {
    writeRecord('r1', { endTime: null, success: null });
    await spawn('r1');
    await appendRunEvent({ kind: 'run.orphan-recovered', runId: 'r1', agentId: 'a1', at: LATER, data: { pid: 4242 } });

    const result = await repairRunRecords();
    expect(result.repaired[0]).toMatchObject({ runId: 'r1', from: 'orphaned', success: false });
    const record = readRecord('r1');
    expect(record.success).toBe(false);
    expect(record.error).toContain('reconciliation');
  });

  it('leaves the record open when the repair cannot be recorded in the ledger', async () => {
    writeRecord('r1', { endTime: null, success: null });
    await spawn('r1');
    await finalize('r1', false);

    // Only one order survives a crash (or a failed write) mid-repair. With the
    // event first, a repair that could not be recorded leaves an open record
    // the NEXT pass retries. With the record first, the record would be closed
    // by an event nothing recorded — and for an orphan repair that reads as a
    // permanent 'ledger-open' finding no later pass can resolve.
    hooks.failAppend = true;
    const result = await repairRunRecords();
    hooks.failAppend = false;

    expect(result.repaired).toEqual([]);
    expect(result.skipped).toBe(1);
    expect(readRecord('r1').endTime).toBeNull();
    expect(await readRunEvents({ runId: 'r1', kind: 'run.reconciled' })).toEqual([]);

    // ...and the next pass still finds the same repairable drift.
    const retry = await repairRunRecords();
    expect(retry.repaired).toEqual([{ runId: 'r1', from: 'failed', success: false, endTime: LATER }]);
  });

  it('skips a run whose record file is not a run record', async () => {
    mkdirSync(join(RUNS_DIR, 'r1'), { recursive: true });
    writeFileSync(metaPath('r1'), '["not", "a", "record"]');
    await spawn('r1');
    await finalize('r1', false);

    const result = await repairRunRecords();
    // A corrupt file is unreadable, not an open run — repairing it would have
    // spread an array into the "repaired" record.
    expect(result.findings[0].finding).toBe('record-missing');
    expect(result.repaired).toEqual([]);
    expect(readFileSync(metaPath('r1'), 'utf8')).toBe('["not", "a", "record"]');
  });

  it('refuses to follow a run id out of the runs directory', async () => {
    // A hand-edited or truncated-and-recovered ledger line is the only way this
    // id gets in — every id the server mints is a uuid — but it would otherwise
    // become a path.
    const escaping = '../escape';
    mkdirSync(join(RUNS_DIR, '..', 'escape'), { recursive: true });
    const outside = join(RUNS_DIR, '..', 'escape', 'metadata.json');
    writeFileSync(outside, JSON.stringify({ id: escaping, startTime: AT, endTime: null }));
    await appendRunEvent({ kind: 'run.spawned', runId: escaping, at: AT, data: {} });
    await appendRunEvent({ kind: 'run.finalized', runId: escaping, at: LATER, data: { success: false, exitCode: 1 } });

    const result = await repairRunRecords();
    expect(result.findings.map((f) => f.finding)).toEqual(['record-missing']);
    expect(result.repaired).toEqual([]);
    expect(JSON.parse(readFileSync(outside, 'utf8')).endTime).toBeNull();
  });

  it('does not overwrite a record that closed itself while the repair was being recorded', async () => {
    writeRecord('r1', { endTime: null, success: null });
    await spawn('r1');
    await finalize('r1', false);

    // Reads of r1 in one pass: the batch load, the pre-plan read, and the
    // re-read after the ledger append. The run's own completion path lands
    // between the second and the third — which is exactly what the third read
    // exists to notice.
    let reads = 0;
    hooks.afterRecordRead = (path) => {
      if (!path.includes('r1')) return;
      reads += 1;
      if (reads === 2) writeRecord('r1', { endTime: LATER, success: true, duration: 7 });
    };

    const result = await repairRunRecords();
    hooks.afterRecordRead = null;

    expect(reads).toBe(3);
    expect(result.repaired).toEqual([]);
    expect(result.skipped).toBe(1);
    expect(readRecord('r1')).toMatchObject({ success: true, duration: 7 });
    expect(readRecord('r1').reconciledFromLedger).toBeUndefined();
  });

  it('keys repairs by a fixed-width digest so long run ids cannot collide', async () => {
    // `eventId` is truncated at 128 chars by the envelope schema. A readable
    // `reconcile:<runId>:<endTime>` key would put the run id first, so two long
    // ids sharing a prefix would truncate to one key and the second repair
    // would be suppressed as a duplicate that never happened.
    const long = (suffix) => `${'a'.repeat(120)}${suffix}`;
    for (const suffix of ['1', '2']) {
      const runId = long(suffix);
      writeRecord(runId, { endTime: null, success: null });
      await appendRunEvent({ kind: 'run.spawned', runId, at: AT, data: {} });
      await appendRunEvent({ kind: 'run.finalized', runId, at: LATER, data: { success: false, exitCode: 1 } });
    }

    const result = await repairRunRecords();
    expect(result.repaired.map((r) => r.runId).sort()).toEqual([long('1'), long('2')].sort());
    expect(readRecord(long('1')).endTime).toBe(LATER);
    expect(readRecord(long('2')).endTime).toBe(LATER);
    const ids = (await readRunEvents({ kind: 'run.reconciled' })).map((e) => e.eventId);
    expect(new Set(ids).size).toBe(2);
  });

  it('runs one pass at a time', async () => {
    writeRecord('r1', { endTime: null, success: null });
    await spawn('r1');
    await finalize('r1', false);

    const [a, b] = await Promise.all([repairRunRecords(), repairRunRecords()]);
    // The second call joins the first rather than planning the same repair from
    // records the first has not written yet.
    expect(b).toBe(a);
    expect(a.repaired).toHaveLength(1);
  });
});