Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

13.9 KB
Raw Blame
import { describe, it, expect } from 'vitest';
import {
  GOAL_FIDELITY_VERDICTS,
  MAX_OBJECTIVE_CHARS,
  formatGoalFidelitySummary,
  goalFidelityHoldsRun,
  goalFidelityLogMarker,
  mergeOutcomeObjective,
  mergeOutcomeReview,
  normalizeGoalFidelityVerdict,
  productionAfterChangeHunks,
  productionAuditEvidence,
  resolveGoalFidelityConfig,
  taskObjective,
} from './goalFidelity.js';

describe('taskObjective', () => {
  it('frames the premature agent-summary complaint as a repair while preserving the report', () => {
    const description = 'Completed-agent cards leak summaries before Show, and opening them duplicates the summary.';
    const objective = taskObjective({ description });
    expect(objective).toContain('Repair the current summary-disclosure defects reported below.');
    expect(objective).toContain(description);
    expect(objective.length).toBeLessThanOrEqual(MAX_OBJECTIVE_CHARS);
  });

  it.each([
    'Intentionally show summaries on completed-agent cards even before Show.',
    'Document why completed-agent cards leak summaries before Show.',
    'Dashboard cards leak summaries before Show.',
    'Completed-agent cards leak summaries before Show.',
  ])('preserves requests outside that repair-report shape: %s', description => {
    expect(taskObjective({ description })).toBe(description);
  });

  it('keeps the report unchanged when additional task context supplies its intent', () => {
    const description = 'Completed-agent cards leak summaries before Show.';
    expect(taskObjective({ description, metadata: { prompt: 'Keep the existing preview; document its behavior.' } }))
      .toBe(`${description}\n\nKeep the existing preview; document its behavior.`);
  });

  it('composes the description with the task prompt block', () => {
    expect(taskObjective({
      description: 'Add a retry to the uploader',
      metadata: { prompt: 'Retry three times\nwith backoff', context: 'queued by hand' },
    })).toBe('Add a retry to the uploader\n\nRetry three times\nwith backoff\n\nqueued by hand');
  });

  it('marks task screenshots as objective context without exposing their local reference', () => {
    const objective = taskObjective({
      description: 'Another instance with the latest code is seeing this',
      metadata: { screenshots: ['/api/screenshots/superclass-error.png'] },
    });

    expect(objective).toContain('Another instance with the latest code is seeing this');
    expect(objective).toContain('Task-provided screenshots are part of this objective');
    expect(objective).toContain('visible application behavior and errors');
    expect(objective).not.toContain('/api/screenshots/');
  });

  it('returns null when the task states no objective, so the gate skips rather than judging against ""', () => {
    expect(taskObjective({ description: '   ', metadata: {} })).toBeNull();
    expect(taskObjective(null)).toBeNull();
  });

  it('truncates an oversized objective to WITHIN the cap, marker included', () => {
    const objective = taskObjective({ description: 'x'.repeat(MAX_OBJECTIVE_CHARS + 500), metadata: {} });
    expect(objective.length).toBeLessThanOrEqual(MAX_OBJECTIVE_CHARS);
    expect(objective).toContain('[objective truncated]');
  });
});

describe('resolveGoalFidelityConfig', () => {
  it('inherits the chain\'s local reviewer and its pinned model/effort when nothing is configured', () => {
    expect(resolveGoalFidelityConfig({ ollamaModel: 'qwen3:8b', ollamaEffort: 'low' }, ['copilot', 'ollama']))
      .toEqual({ enabled: true, backend: 'ollama', model: 'qwen3:8b', effort: 'low' });
  });

  it('prefers the gate\'s own pins over the chain\'s', () => {
    expect(resolveGoalFidelityConfig(
      { ollamaModel: 'qwen3:8b', goalFidelity: { backend: 'lmstudio', model: 'gpt-oss-20b', effort: 'high' } },
      ['ollama'],
    )).toEqual({ enabled: true, backend: 'lmstudio', model: 'gpt-oss-20b', effort: 'high' });
  });

  it('declines when disabled, when no local reviewer is available, and when the named backend is not server-callable', () => {
    expect(resolveGoalFidelityConfig({ goalFidelity: { enabled: false } }, ['ollama'])).toBeNull();
    expect(resolveGoalFidelityConfig({}, ['copilot', 'codex'])).toBeNull();
    // A CLI reviewer has no server-side entry point; silently substituting the
    // chain's ollama would run a review on a model the user never picked.
    expect(resolveGoalFidelityConfig({ goalFidelity: { backend: 'codex' } }, ['ollama'])).toBeNull();
  });

  it('drops an unusable pin instead of persisting it into the request', () => {
    const config = resolveGoalFidelityConfig({ goalFidelity: { backend: 'ollama', model: '  ', effort: 'ultra' } }, []);
    expect(config).toEqual({ enabled: true, backend: 'ollama', model: null, effort: null });
  });
});

describe('normalizeGoalFidelityVerdict', () => {
  it('accepts every declared verdict and caps the named lists', () => {
    for (const verdict of GOAL_FIDELITY_VERDICTS) {
      expect(normalizeGoalFidelityVerdict({ verdict, missing: [], unrequested: [], evidence: '' })?.verdict).toBe(verdict);
    }
    const many = normalizeGoalFidelityVerdict({
      verdict: 'fix-first',
      missing: Array.from({ length: 40 }, (_, i) => `item ${i}`),
      unrequested: ['  ', 'a real one', 42],
      evidence: 'tests were run',
    });
    expect(many.missing).toHaveLength(10);
    expect(many.unrequested).toEqual(['a real one']);
    expect(many.evidence).toBe('tests were run');
  });

  it('renders a markdown link in model-authored text as prose, not a clickable destination', () => {
    // These strings come from an untrusted diff and land in a Review Hub alert
    // that goes through PortOS's markdown renderer.
    const result = normalizeGoalFidelityVerdict({
      verdict: 'rethink',
      missing: ['[click here](https://example.com/attacker)'],
      unrequested: ['![banner](https://example.com/pixel.png)'],
    });
    expect(result.missing[0]).toBe('[click here] (https://example.com/attacker)');
    expect(result.unrequested[0]).toBe('![banner] (https://example.com/pixel.png)');
  });

  it('returns null for an unusable answer so "nothing judged this run" never collapses into ship or rethink', () => {
    expect(normalizeGoalFidelityVerdict(null)).toBeNull();
    expect(normalizeGoalFidelityVerdict(['ship'])).toBeNull();
    expect(normalizeGoalFidelityVerdict({ verdict: 'looks good to me' })).toBeNull();
    expect(normalizeGoalFidelityVerdict({ missing: [] })).toBeNull();
  });
});

describe('goalFidelityHoldsRun', () => {
  it('holds only on rethink', () => {
    expect(goalFidelityHoldsRun({ verdict: 'rethink' })).toBe(true);
    expect(goalFidelityHoldsRun({ verdict: 'fix-first' })).toBe(false);
    expect(goalFidelityHoldsRun({ verdict: 'ship' })).toBe(false);
    expect(goalFidelityHoldsRun(null)).toBe(false);
  });
});

describe('goalFidelityLogMarker', () => {
  it('reserves the checkmark for a clean ship', () => {
    expect(goalFidelityLogMarker({ verdict: 'ship' })).toBe('✅');
    expect(goalFidelityLogMarker({ verdict: 'fix-first', missing: ['a'] })).toBe('🧩');
    expect(goalFidelityLogMarker({ verdict: 'rethink', missing: ['a'] })).toBe('🎯');
  });
});

describe('formatGoalFidelitySummary', () => {
  it('reports counts rather than the untrusted item text', () => {
    expect(formatGoalFidelitySummary({ verdict: 'rethink', missing: ['a', 'b'], unrequested: ['c'] }))
      .toBe('Goal-fidelity verdict: rethink (2 missing, 1 unrequested)');
    expect(formatGoalFidelitySummary({ verdict: 'ship', missing: [], unrequested: [] }))
      .toBe('Goal-fidelity verdict: ship');
  });
});

describe('mergeOutcomeObjective', () => {
  const followUp = (metadata) => ({ description: 'Resolve and merge PR #7653', metadata });

  it('recognizes a review-loop follow-up, tolerating Markdown-stored string booleans', () => {
    expect(mergeOutcomeObjective(followUp({
      reviewLoopFollowUp: true, reviewLoopPRNumber: 7653, reviewLoopPRBranch: 'claim/issue-7625',
    }))).toEqual({ number: 7653, branch: 'claim/issue-7625' });
    expect(mergeOutcomeObjective(followUp({
      reviewLoopFollowUp: 'true', reviewLoopPRNumber: '7653', reviewLoopPRBranch: 'claim/issue-7625',
    }))).toEqual({ number: 7653, branch: 'claim/issue-7625' });
  });

  it.each([
    ['an ordinary task', {}],
    ['a leave-open follow-up, whose deliverable IS a diff', { reviewLoopFollowUp: true, reviewLoopLeaveOpen: 'true', reviewLoopPRNumber: 7653, reviewLoopPRBranch: 'b' }],
    ['a follow-up missing its number', { reviewLoopFollowUp: true, reviewLoopPRBranch: 'b' }],
    ['a follow-up missing its branch', { reviewLoopFollowUp: true, reviewLoopPRNumber: 7653 }],
  ])('declines %s, leaving it to the ordinary diff review', (_label, metadata) => {
    expect(mergeOutcomeObjective(followUp(metadata))).toBeNull();
  });

  it('declines a task with no metadata at all', () => {
    expect(mergeOutcomeObjective(null)).toBeNull();
    expect(mergeOutcomeObjective({ description: 'Merge it' })).toBeNull();
  });
});

describe('mergeOutcomeReview', () => {
  it('establishes ship from a merge, naming the forge as the source', () => {
    expect(mergeOutcomeReview({ number: 7653, prState: 'MERGED' })).toEqual({
      verdict: 'ship',
      missing: [],
      unrequested: [],
      evidence: expect.stringContaining('#7653 MERGED'),
      source: 'forge-outcome',
    });
    expect(mergeOutcomeReview({ number: 7653, prState: 'merged' })?.verdict).toBe('ship');
  });

  // Only a merge is proof. An open PR is what a blocked review legitimately
  // leaves behind, and a PR closed as superseded can be a correct resolution —
  // turning either into a finding would hold runs that did nothing wrong.
  it.each(['OPEN', 'CLOSED', null])('establishes nothing from state %s', prState => {
    expect(mergeOutcomeReview({ number: 7653, prState })).toBeNull();
  });
});


describe('productionAfterChangeHunks', () => {
  it('does not invent a summary root or join an outer JSX condition across a hunk gap', () => {
    const diff = [
      'diff --git a/Card.jsx b/Card.jsx', '--- a/Card.jsx', '+++ b/Card.jsx',
      '@@ -1,6 +1,6 @@',
      ' {expanded && (', '   <div>', '     {taskSummary && (',
      '+      <MarkdownOutput content={taskSummary} />', '     )}',
      '@@ -20,2 +20,2 @@', '   </div>', ' )}',
    ].join('\n');
    const rows = productionAfterChangeHunks(diff);
    expect(rows).toHaveLength(2);
    expect(rows[0].source).toContain('<MarkdownOutput');
    expect(rows.every(row => row.renderPredicates.length === 0)).toBe(true);
  });
});


describe('productionAuditEvidence', () => {
  it('traces actual increment guards and log-level expressions with hunk provenance, excluding tests and deleted lines', () => {
    const makeDiff = statement => [
      'diff --git a/lifecycle.js b/lifecycle.js', '+++ b/lifecycle.js',
      '@@ -10,2 +10,2 @@', '-backfilled++;', `+${statement}`,
      "+emitLog(failed ? 'warn' : 'info', 'summary', { failed });",
      'diff --git a/lifecycle.test.js b/lifecycle.test.js', '+++ b/lifecycle.test.js',
      '@@ -1 +1 @@', '+expect(backfilled).toBe(1);',
    ].join('\n');
    const [row] = productionAuditEvidence(makeDiff('if (recorded) backfilled++;'));
    expect(row.file).toBe('lifecycle.js');
    expect(row.hunk).toBe('@@ -10,2 +10,2 @@');
    expect(row.statements.flatMap(statement => statement.facts)).toContainEqual({
      kind: 'write', source: 'backfilled++', target: 'backfilled', conditions: ['recorded'], callbacks: [],
    });
    expect(row.statements.flatMap(statement => statement.facts)).toContainEqual(expect.objectContaining({
      kind: 'conditional', test: 'failed', whenTruthy: "'warn'", whenFalsy: "'info'",
    }));
    const mutated = productionAuditEvidence(makeDiff('backfilled++;'));
    expect(mutated[0].statements.flatMap(statement => statement.facts)).toContainEqual({
      kind: 'write', source: 'backfilled++', target: 'backfilled', conditions: [], callbacks: [],
    });
    expect(JSON.stringify(mutated)).not.toContain('expect(');
  });

  it('never repairs incomplete syntax or roots a nested write across an unavailable enclosing guard', () => {
    const rows = productionAuditEvidence([
      'diff --git a/lifecycle.js b/lifecycle.js', '+++ b/lifecycle.js',
      '@@ -1,2 +1,2 @@', '+if (recorded) {', '+  backfilled++;',
      '@@ -20 +20 @@', '+}',
    ].join('\n'));
    expect(rows).toHaveLength(2);
    expect(rows.every(row => row.statements.length === 0)).toBe(true);
    expect(rows[0].source).toContain('if (recorded)');
  });
});


describe('handled-rejection syntax scenarios', () => {
  it('keeps an else write and ignores code-shaped comment/string content and unsupported control paths', () => {
    const diff = source => [
      'diff --git a/lifecycle.js b/lifecycle.js', '+++ b/lifecycle.js',
      '@@ -1 +1 @@', ...source.split('\n').map(line => `+${line}`),
    ].join('\n');
    const binding = 'const recorded = await recordTaskCompletion().catch(err => { return false; });';
    const [row] = productionAuditEvidence(diff(`${binding}
if (recorded) backfilled++;
else backfilled++;`));
    expect(row.handledRejectionScenarios.map(scenario => scenario.writeReached)).toEqual([false, true]);
    for (const statement of [
      'while (pending) backfilled++;',
      'if (recorded) { while (pending) backfilled++; }',
      'const next = () => { backfilled++; };',
      'if (recorded) {} else { recorded = true; if (recorded) backfilled++; }',
      'if (!recorded) { throw new Error("stop"); backfilled++; }',
      'if (!recorded) { reportFailure(); backfilled++; }',
    ]) expect(productionAuditEvidence(diff(`${binding}\n${statement}`))[0].handledRejectionScenarios).toEqual([]);
    expect(productionAuditEvidence(diff(`let recorded = await recordTaskCompletion().catch(() => false);
if (recorded) {} else { recorded = true; if (recorded) backfilled++; }`))[0].handledRejectionScenarios).toEqual([]);
    const [commented] = productionAuditEvidence(diff(`/*\nif (recorded) backfilled++;\n*/\nconst label = "if (recorded) backfilled++;";`));
    expect(commented.statements.flatMap(statement => statement.facts).filter(fact => fact.kind === 'write')).toEqual([]);
  });
});