Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
16.8 KB
Raw Blame
/**
 * gbrain friction diff — cross-run comparison tests (Phase E).
 *
 * Zero-binary: exercises runFriction dispatch + the exported diff helpers
 * directly against a tmp GBRAIN_HOME, same conventions as friction-cli.test.ts.
 *
 * Spec anchors (plan Phase E + adversarial gate): identity is (kind, phase,
 * normalized message prefix — digits collapsed) with severity EXCLUDED (it is
 * the compared attribute, as a per-severity multiset); identities are
 * MULTISETS (count deltas are differences); only kind friction|delight is
 * diffable (markers/interruptions feed the compatibility banner); exit codes
 * 0 = ran, 1 = resolution/IO error, 2 = usage error.
 */

import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, rmSync, appendFileSync, utimesSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { runFriction, computeFrictionDiff, resolveRunSpec } from '../src/commands/friction.ts';
import { logFriction, frictionFile } from '../src/core/friction.ts';
import type { FrictionSeverity } from '../src/core/friction.ts';

const ORIG_HOME = process.env.GBRAIN_HOME;
const ORIG_RUN_ID = process.env.GBRAIN_FRICTION_RUN_ID;
let tmp: string;
let stdoutLines: string[];
let stderrLines: string[];
let origStdoutWrite: typeof process.stdout.write;
let origConsoleLog: typeof console.log;
let origConsoleError: typeof console.error;

beforeEach(() => {
  tmp = mkdtempSync(join(tmpdir(), 'friction-diff-'));
  process.env.GBRAIN_HOME = tmp;
  delete process.env.GBRAIN_FRICTION_RUN_ID;
  stdoutLines = [];
  stderrLines = [];
  origStdoutWrite = process.stdout.write.bind(process.stdout);
  origConsoleLog = console.log;
  origConsoleError = console.error;
  process.stdout.write = ((chunk: string) => { stdoutLines.push(String(chunk)); return true; }) as any;
  console.log = (...args: unknown[]) => { stdoutLines.push(args.join(' ') + '\n'); };
  console.error = (...args: unknown[]) => { stderrLines.push(args.join(' ') + '\n'); };
});

afterEach(() => {
  process.env.GBRAIN_HOME = ORIG_HOME;
  if (ORIG_RUN_ID !== undefined) process.env.GBRAIN_FRICTION_RUN_ID = ORIG_RUN_ID;
  rmSync(tmp, { recursive: true, force: true });
  process.stdout.write = origStdoutWrite;
  console.log = origConsoleLog;
  console.error = origConsoleError;
});

/** Stamp the run-start marker the harness writes on every run. */
function startMarker(runId: string, opts: { agent?: string; scenario?: string } = {}): void {
  logFriction({
    runId, phase: 'run', kind: 'phase-marker', marker: 'start',
    message: 'run start', source: 'harness',
    agent: opts.agent, scenario: opts.scenario,
  });
}

function friction(runId: string, phase: string, message: string, severity: FrictionSeverity = 'error'): void {
  logFriction({ runId, phase, message, severity, kind: 'friction', source: 'claw' });
}

function diffJson(base: string, compare: string): { code: number; parsed: any } {
  stdoutLines.length = 0;
  const code = runFriction(['diff', '--base', base, '--compare', compare, '--json']);
  const out = stdoutLines.join('').trim();
  return { code, parsed: out ? JSON.parse(out) : undefined };
}

describe('three sections', () => {
  test('unique-to-compare, unique-to-base, and severity change land in the right buckets', () => {
    startMarker('run-base', { agent: 'scripted', scenario: 'fresh-install' });
    friction('run-base', 'install', 'init did not say which engine', 'error');
    startMarker('run-cmp', { agent: 'scripted', scenario: 'fresh-install' });
    friction('run-cmp', 'install', 'init did not say which engine', 'blocker'); // X, severity bumped
    friction('run-cmp', 'query', 'query returned nothing for a seeded doc', 'confused'); // Y

    const { code, parsed } = diffJson('run-base', 'run-cmp');
    expect(code).toBe(0);
    expect(parsed.unique_to_base).toEqual([]);
    expect(parsed.unique_to_compare.length).toBe(1);
    expect(parsed.unique_to_compare[0].phase).toBe('query');
    expect(parsed.unique_to_compare[0].message).toContain('query returned nothing');
    expect(parsed.changed.length).toBe(1);
    expect(parsed.changed[0].phase).toBe('install');
    expect(parsed.changed[0].severity_changed).toBe(true);
    expect(parsed.changed[0].count_changed).toBe(false);
    expect(parsed.changed[0].base_severities).toEqual(['error']);
    expect(parsed.changed[0].compare_severities).toEqual(['blocker']);
  });

  test('human output labels sections by run-id (instrument, not judge) and groups by phase', () => {
    startMarker('run-base', { agent: 'scripted', scenario: 'fresh-install' });
    friction('run-base', 'install', 'only in base run');
    startMarker('run-cmp', { agent: 'scripted', scenario: 'fresh-install' });
    friction('run-cmp', 'query', 'only in compare run');

    stdoutLines.length = 0;
    const code = runFriction(['diff', '--base', 'run-base', '--compare', 'run-cmp']);
    expect(code).toBe(0);
    const out = stdoutLines.join('');
    expect(out).toContain('Unique to `run-cmp`');
    expect(out).toContain('Unique to `run-base`');
    expect(out).toContain('Shared but changed');
    expect(out).toContain('### `install`');
    expect(out).toContain('### `query`');
  });
});

describe('multiset counts', () => {
  test('1×error vs 10×error of the same identity is a count delta, not equality', () => {
    startMarker('run-a', { agent: 'scripted' });
    friction('run-a', 'embed', 'embed batch rejected by provider', 'error');
    startMarker('run-b', { agent: 'scripted' });
    for (let i = 0; i < 10; i++) friction('run-b', 'embed', 'embed batch rejected by provider', 'error');

    const { code, parsed } = diffJson('run-a', 'run-b');
    expect(code).toBe(0);
    expect(parsed.unique_to_base).toEqual([]);
    expect(parsed.unique_to_compare).toEqual([]);
    expect(parsed.changed.length).toBe(1);
    expect(parsed.changed[0].base_count).toBe(1);
    expect(parsed.changed[0].compare_count).toBe(10);
    expect(parsed.changed[0].count_changed).toBe(true);
    expect(parsed.changed[0].severity_changed).toBe(false);
  });
});

describe('kind filter', () => {
  test('phase-marker and interrupted entries never enter the three sections; they feed the banner', () => {
    startMarker('run-a', { agent: 'scripted' });
    logFriction({ runId: 'run-a', phase: 'run', kind: 'phase-marker', marker: 'end', message: 'MARKER SENTINEL A', source: 'harness' });
    logFriction({ runId: 'run-a', phase: 'run', kind: 'interrupted', message: 'INTERRUPT SENTINEL A', source: 'harness' });
    friction('run-a', 'install', 'real base friction');
    startMarker('run-b', { agent: 'scripted' });
    logFriction({ runId: 'run-b', phase: 'run', kind: 'phase-marker', marker: 'end', message: 'MARKER SENTINEL B', source: 'harness' });
    logFriction({ runId: 'run-b', phase: 'run', kind: 'interrupted', message: 'INTERRUPT SENTINEL B', source: 'harness' });
    friction('run-b', 'install', 'real compare friction');

    const { code, parsed } = diffJson('run-a', 'run-b');
    expect(code).toBe(0);
    const sections = JSON.stringify([parsed.unique_to_base, parsed.unique_to_compare, parsed.changed]);
    expect(sections).not.toContain('SENTINEL');
    expect(parsed.unique_to_base.length).toBe(1);
    expect(parsed.unique_to_compare.length).toBe(1);
    expect(parsed.banner.base.interrupted).toBe(true);
    expect(parsed.banner.compare.interrupted).toBe(true);
  });
});

describe('run resolution', () => {
  test('agent name resolves to the LATEST run carrying that agent', () => {
    startMarker('older-run', { agent: 'hermes' });
    friction('older-run', 'install', 'old friction');
    startMarker('newer-run', { agent: 'hermes' });
    friction('newer-run', 'install', 'new friction');
    // Deterministic mtimes: older-run well in the past, newer-run now.
    const past = new Date(Date.now() - 60_000);
    utimesSync(frictionFile('older-run'), past, past);

    expect(resolveRunSpec('hermes')).toBe('newer-run');

    const { code, parsed } = diffJson('hermes', 'older-run');
    expect(code).toBe(0);
    expect(parsed.base).toBe('newer-run');
    expect(parsed.compare).toBe('older-run');
  });

  test('an exact run-id wins over agent-name interpretation', () => {
    // A run whose run-id IS the string 'hermes', carrying a different agent...
    startMarker('hermes', { agent: 'scripted' });
    // ...and a newer run whose AGENT is 'hermes'.
    startMarker('agent-stamped-run', { agent: 'hermes' });
    const past = new Date(Date.now() - 60_000);
    utimesSync(frictionFile('hermes'), past, past);

    expect(resolveRunSpec('hermes')).toBe('hermes');
  });
});

describe('failure semantics', () => {
  test('unknown run or agent exits 1 and stderr lists available runs', () => {
    startMarker('run-known', { agent: 'scripted' });
    stderrLines.length = 0;
    const code = runFriction(['diff', '--base', 'no-such-run', '--compare', 'run-known']);
    expect(code).toBe(1);
    const err = stderrLines.join('');
    expect(err).toContain('no-such-run');
    expect(err).toContain('available runs');
    expect(err).toContain('run-known');
  });

  test('no runs at all: unknown spec exits 1 with an empty available list', () => {
    const code = runFriction(['diff', '--base', 'ghost-a', '--compare', 'ghost-b']);
    expect(code).toBe(1);
    expect(stderrLines.join('')).toContain('(none)');
  });
});

describe('empty and identical runs', () => {
  test('a run with only a start marker produces a valid empty diff, exit 0', () => {
    startMarker('empty-a', { agent: 'scripted', scenario: 'fresh-install' });
    startMarker('empty-b', { agent: 'scripted', scenario: 'fresh-install' });
    const { code, parsed } = diffJson('empty-a', 'empty-b');
    expect(code).toBe(0);
    expect(parsed.unique_to_base).toEqual([]);
    expect(parsed.unique_to_compare).toEqual([]);
    expect(parsed.changed).toEqual([]);
    expect(parsed.banner.warnings).toEqual([]);
  });

  test('base equals compare: valid no-difference output, exit 0', () => {
    startMarker('run-same', { agent: 'scripted' });
    friction('run-same', 'install', 'a friction entry');
    const { code, parsed } = diffJson('run-same', 'run-same');
    expect(code).toBe(0);
    expect(parsed.base).toBe('run-same');
    expect(parsed.compare).toBe('run-same');
    expect(parsed.unique_to_base).toEqual([]);
    expect(parsed.unique_to_compare).toEqual([]);
    expect(parsed.changed).toEqual([]);

    stdoutLines.length = 0;
    expect(runFriction(['diff', '--base', 'run-same', '--compare', 'run-same'])).toBe(0);
    expect(stdoutLines.join('')).toContain('No differences');
  });
});

describe('malformed JSONL tolerance', () => {
  test('a malformed line is skipped, its count surfaces, and the diff is still produced', () => {
    startMarker('mal-run', { agent: 'scripted' });
    appendFileSync(frictionFile('mal-run'), '{this is not json\n', 'utf-8');
    friction('mal-run', 'install', 'friction after the bad line');
    startMarker('clean-run', { agent: 'scripted' });

    const { code, parsed } = diffJson('mal-run', 'clean-run');
    expect(code).toBe(0);
    expect(parsed.banner.base.malformed).toBe(1);
    expect(parsed.banner.compare.malformed).toBe(0);
    expect(parsed.unique_to_base.length).toBe(1);

    stdoutLines.length = 0;
    runFriction(['diff', '--base', 'mal-run', '--compare', 'clean-run']);
    expect(stdoutLines.join('')).toContain('1 malformed line(s) skipped');
  });
});

describe('compatibility banner', () => {
  test('scenario mismatch warns loudly, naming both scenarios', () => {
    startMarker('scen-a', { agent: 'scripted', scenario: 'fresh-install' });
    startMarker('scen-b', { agent: 'scripted', scenario: 'upgrade-from-v0.18' });

    const { code, parsed } = diffJson('scen-a', 'scen-b');
    expect(code).toBe(0);
    expect(parsed.banner.warnings.length).toBeGreaterThanOrEqual(1);
    const joined = parsed.banner.warnings.join('\n');
    expect(joined).toContain('fresh-install');
    expect(joined).toContain('upgrade-from-v0.18');

    stdoutLines.length = 0;
    runFriction(['diff', '--base', 'scen-a', '--compare', 'scen-b']);
    const out = stdoutLines.join('');
    expect(out).toContain('WARN');
    expect(out).toContain('fresh-install');
    expect(out).toContain('upgrade-from-v0.18');
  });

  test('banner carries agent, scenario, and gbrain version from the start markers', () => {
    startMarker('meta-a', { agent: 'hermes', scenario: 'fresh-install' });
    startMarker('meta-b', { agent: 'openclaw', scenario: 'fresh-install' });
    const { parsed } = diffJson('meta-a', 'meta-b');
    expect(parsed.banner.base.agent).toBe('hermes');
    expect(parsed.banner.compare.agent).toBe('openclaw');
    expect(parsed.banner.base.scenario).toBe('fresh-install');
    expect(typeof parsed.banner.base.gbrain_version).toBe('string');
    // Same version on both sides (same process) → no version warning.
    expect(parsed.banner.warnings).toEqual([]);
  });
});

describe('identity normalization', () => {
  test('case, whitespace, and redaction differences collapse to the same identity', () => {
    startMarker('norm-a', { agent: 'scripted' });
    friction('norm-a', 'install', `Engine   Init FAILED at ${process.cwd()}/x.ts`, 'error');
    startMarker('norm-b', { agent: 'scripted' });
    friction('norm-b', 'install', 'engine init failed at <CWD>/x.ts', 'error');

    const diff = computeFrictionDiff('norm-a', 'norm-b');
    expect(diff.unique_to_base).toEqual([]);
    expect(diff.unique_to_compare).toEqual([]);
    expect(diff.changed).toEqual([]);
  });

  test('messages sharing the first 80 normalized chars are the same identity', () => {
    const prefix = 'p'.repeat(80);
    startMarker('pre-a', { agent: 'scripted' });
    friction('pre-a', 'query', prefix + ' tail one', 'error');
    startMarker('pre-b', { agent: 'scripted' });
    friction('pre-b', 'query', prefix + ' completely different tail', 'error');

    const diff = computeFrictionDiff('pre-a', 'pre-b');
    expect(diff.unique_to_base).toEqual([]);
    expect(diff.unique_to_compare).toEqual([]);
    expect(diff.changed).toEqual([]);
  });
});

describe('CLI dispatch', () => {
  test('diff with base and compare run-ids returns 0', () => {
    startMarker('cli-a', { agent: 'scripted' });
    startMarker('cli-b', { agent: 'scripted' });
    expect(runFriction(['diff', '--base', 'cli-a', '--compare', 'cli-b'])).toBe(0);
  });

  test('missing compare flag is a usage error, exit 2', () => {
    const code = runFriction(['diff', '--base', 'cli-a']);
    expect(code).toBe(2);
    expect(stderrLines.join('')).toContain('usage');
  });

  test('missing both flags is a usage error, exit 2', () => {
    expect(runFriction(['diff'])).toBe(2);
  });
});

describe('identity hardening (adversarial-gate pins)', () => {
  test('a delight→friction flip surfaces as unique-to-each, never equality', () => {
    startMarker('run-base', { agent: 'scripted' });
    logFriction({ runId: 'run-base', phase: 'query', message: 'results ranked well', kind: 'delight', source: 'claw' });
    startMarker('run-cmp', { agent: 'scripted' });
    logFriction({ runId: 'run-cmp', phase: 'query', message: 'results ranked well', kind: 'friction', severity: 'confused', source: 'claw' });

    const { code, parsed } = diffJson('run-base', 'run-cmp');
    expect(code).toBe(0);
    expect(parsed.changed).toEqual([]);
    expect(parsed.unique_to_base.length).toBe(1);
    expect(parsed.unique_to_base[0].kind).toBe('delight');
    expect(parsed.unique_to_compare.length).toBe(1);
    expect(parsed.unique_to_compare[0].kind).toBe('friction');
  });

  test('severity redistribution with equal totals and equal severity sets is a reported difference', () => {
    startMarker('run-base', { agent: 'scripted' });
    friction('run-base', 'import', 'import warned', 'error');
    friction('run-base', 'import', 'import warned', 'error');
    friction('run-base', 'import', 'import warned', 'nit');
    startMarker('run-cmp', { agent: 'scripted' });
    friction('run-cmp', 'import', 'import warned', 'error');
    friction('run-cmp', 'import', 'import warned', 'nit');
    friction('run-cmp', 'import', 'import warned', 'nit');

    const { code, parsed } = diffJson('run-base', 'run-cmp');
    expect(code).toBe(0);
    expect(parsed.changed.length).toBe(1);
    expect(parsed.changed[0].severity_changed).toBe(true);
    expect(parsed.changed[0].count_changed).toBe(false);
    expect(parsed.changed[0].base_severity_counts).toEqual({ error: 2, nit: 1 });
    expect(parsed.changed[0].compare_severity_counts).toEqual({ error: 1, nit: 2 });
  });

  test('volatile digits (durations, retry counts) do not split identities across runs', () => {
    startMarker('run-base', { agent: 'scripted' });
    friction('run-base', 'agent_invoke', 'agent exited with code 1 after 5123ms');
    startMarker('run-cmp', { agent: 'scripted' });
    friction('run-cmp', 'agent_invoke', 'agent exited with code 1 after 98764ms');

    const { code, parsed } = diffJson('run-base', 'run-cmp');
    expect(code).toBe(0);
    expect(parsed.unique_to_base).toEqual([]);
    expect(parsed.unique_to_compare).toEqual([]);
    expect(parsed.changed).toEqual([]);
  });
});