Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
23.1 KB
Raw Blame
/**
 * Facts extraction outcome honesty — the silent-[] bug-class regression suite.
 *
 * Pins the D4/D6 contract:
 *   - keyless-expected chat_unavailable: NO ingest_log row, ONE calm stderr
 *     note (keyless is an expected state, not a failure)
 *   - keyed-but-failing: one ingest_log row (doctor reads it) + fix hint
 *   - transport-class failures (provider_error/truncated_output) THROW a
 *     typed FactsExtractionError; the queue-mode classifier maps it to
 *     precise absorb codes
 *   - the pre-enqueue gate consults the RESOLVED extraction model (a servable
 *     DB-plane override must never be skipped — the CX1 false-skip class)
 *   - durable facts-absorb jobs carry the slow-retry policy (max_attempts 5,
 *     60s backoff) and the handler retry-decision converts execution-time
 *     unavailability into a throw, never a silent consume
 *   - extract_facts returns reason-specific envelopes; the keyless one pins
 *     visibility: "private" (remember defaults to world)
 *   - put_page NEVER fails because its facts backstop threw (E1 invariant)
 */
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { dispatchToolCall } from '../src/mcp/dispatch.ts';
import {
  configureGateway,
  resetGateway,
  __setChatTransportForTests,
} from '../src/core/ai/gateway.ts';
import { AIConfigError } from '../src/core/ai/errors.ts';
import {
  runFactsBackstop,
  runFactsPipeline,
  __resetBackstopWarningsForTests,
} from '../src/core/facts/backstop.ts';
import {
  FactsExtractionError,
  _resetExtractWrapperWarningsForTests,
  extractFactsFromTurn,
} from '../src/core/facts/extract.ts';
import {
  classifyFactsAbsorbError,
  _resetFactsAbsorbDisconnectedFlagForTests,
} from '../src/core/facts/absorb-log.ts';
import { factsAbsorbShouldRetry, registerBuiltinHandlers } from '../src/commands/jobs.ts';
import { markShortLivedCliProcess, __resetShortLivedCliForTests } from '../src/core/facts/cli-process-mode.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import type { MinionHandler } from '../src/core/minions/types.ts';
import { runPersistenceEffects } from '../src/core/persistence/effects.ts';
import { localHostId } from '../src/core/persistence/identity.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';

let engine: PGLiteEngine;
const PINNED = ['ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'GBRAIN_MODEL', 'GBRAIN_HOME'] as const;
let saved: Record<string, string | undefined>;
let tmpHome: string;
let stderrCapture: string;
const origWrite = process.stderr.write.bind(process.stderr);
// backstop's warnOnce writes via console.warn (not process.stderr.write) —
// capture both channels into one buffer.
const origConsoleWarn = console.warn.bind(console);

function unwrap(r: { isError?: boolean; content: Array<{ text?: string }> }): Record<string, unknown> {
  expect(r.isError).toBeFalsy();
  return JSON.parse(r.content[0].text ?? '{}') as Record<string, unknown>;
}

const ELIGIBLE_BODY =
  'An eligible substantive note body. ' +
  'Enough prose to clear the minimum body length gate for the backstop. '.repeat(3);

async function absorbRows(): Promise<Array<{ summary: string }>> {
  return engine.executeRaw<{ summary: string }>(
    `SELECT summary FROM ingest_log WHERE source_type = 'facts:absorb' ORDER BY created_at`,
  );
}

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
});

beforeEach(async () => {
  saved = {};
  for (const k of PINNED) { saved[k] = process.env[k]; delete process.env[k]; }
  tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-outcome-'));
  process.env.GBRAIN_HOME = tmpHome;
  resetGateway();
  __setChatTransportForTests(null);
  __resetBackstopWarningsForTests();
  __resetShortLivedCliForTests();
  _resetExtractWrapperWarningsForTests();
  _resetFactsAbsorbDisconnectedFlagForTests();
  await engine.executeRaw(`DELETE FROM ingest_log WHERE source_type = 'facts:absorb'`).catch(() => {});
  await engine.executeRaw(`DELETE FROM minion_jobs WHERE name = 'facts-absorb'`).catch(() => {});
  await engine.unsetConfig('facts.extraction_model').catch(() => {});
  stderrCapture = '';
  process.stderr.write = ((chunk: string | Uint8Array) => {
    stderrCapture += typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString();
    return true;
  }) as typeof process.stderr.write;
  console.warn = ((...args: unknown[]) => { stderrCapture += args.map(String).join(' ') + '\n'; }) as typeof console.warn;
});

const restoreEnv = () => {
  process.stderr.write = origWrite;
  console.warn = origConsoleWarn;
  rmSync(tmpHome, { recursive: true, force: true });
  for (const k of PINNED) {
    if (saved[k] === undefined) delete process.env[k];
    else process.env[k] = saved[k];
  }
};
afterEach(async () => { await disposePersistenceConsumer(engine); restoreEnv(); });

afterAll(async () => {
  __setChatTransportForTests(null);
  await engine.disconnect();
  // Shard hygiene: restore the legacy embedding pin (see
  // facts-extract-silent-no-op.test.ts for the poisoning story).
  configureGateway({
    embedding_model: 'openai:text-embedding-3-large',
    embedding_dimensions: 1536,
    env: { ...process.env },
  });
});

describe('keyless vs keyed chat_unavailable', () => {
  test('keyless: NO ingest_log row + one calm stderr note', async () => {
    configureGateway({ env: {} }); // keyless gateway
    const r = await runFactsBackstop(
      { slug: 'outcome-keyless', type: 'note', compiled_truth: ELIGIBLE_BODY, frontmatter: {} },
      { engine: engine as BrainEngine, sourceId: 'default', sessionId: null, source: 'mcp:put_page', mode: 'queue' },
    );
    expect(r.mode).toBe('queue');
    expect((r as { skipped?: string }).skipped).toBe('extraction_unavailable');
    expect(await absorbRows()).toHaveLength(0);
    expect(stderrCapture).toContain('keyless');
    // Calm: the note fires once per process, not per write.
    const before = stderrCapture.length;
    await runFactsBackstop(
      { slug: 'outcome-keyless-2', type: 'note', compiled_truth: ELIGIBLE_BODY, frontmatter: {} },
      { engine: engine as BrainEngine, sourceId: 'default', sessionId: null, source: 'mcp:put_page', mode: 'queue' },
    );
    expect(stderrCapture.length).toBe(before);
  });

  test('keyed-but-misrouted: one ingest_log row + fix hint', async () => {
    // An OpenAI key exists but the pinned extraction model is Anthropic with
    // no Anthropic key — a keyed problem the operator must see.
    process.env.OPENAI_API_KEY = 'sk-test';
    await engine.setConfig('facts.extraction_model', 'anthropic:claude-sonnet-4-6');
    configureGateway({ env: { OPENAI_API_KEY: 'sk-test' } });
    const r = await runFactsBackstop(
      { slug: 'outcome-misroute', type: 'note', compiled_truth: ELIGIBLE_BODY, frontmatter: {} },
      { engine: engine as BrainEngine, sourceId: 'default', sessionId: null, source: 'mcp:put_page', mode: 'queue' },
    );
    expect((r as { skipped?: string }).skipped).toBe('extraction_unavailable');
    const rows = await absorbRows();
    expect(rows).toHaveLength(1);
    expect(rows[0].summary).toContain('chat_unavailable');
    expect(rows[0].summary).toContain('anthropic:claude-sonnet-4-6');
    expect(stderrCapture).toContain('facts.extraction_model');
  });

  test('pre-enqueue gate does NOT skip a servable DB-plane override (CX1 false-skip class)', async () => {
    // Global default would be Anthropic (unservable), but the DB-plane
    // extraction override IS servable with the OpenAI key — the engine-aware
    // gate must admit the work. Short-lived mode routes to the durable minion
    // (no LLM call executes in-test) so we can also pin the retry policy and
    // the handler-default timeout stamped on the submitted row.
    process.env.OPENAI_API_KEY = 'sk-test';
    await engine.setConfig('facts.extraction_model', 'openai:gpt-4o-mini');
    configureGateway({ env: { OPENAI_API_KEY: 'sk-test' } });
    markShortLivedCliProcess();
    const r = await runFactsBackstop(
      { slug: 'outcome-servable-override', type: 'note', compiled_truth: ELIGIBLE_BODY, frontmatter: {} },
      { engine: engine as BrainEngine, sourceId: 'default', sessionId: null, source: 'mcp:put_page', mode: 'queue' },
    );
    expect(r.mode).toBe('queue');
    expect((r as { enqueued: boolean }).enqueued).toBe(true);
    expect((r as { skipped?: string }).skipped).toBeUndefined();
    const jobs = await engine.executeRaw<{ max_attempts: number; backoff_delay: number; timeout_ms: number | string }>(
      `SELECT max_attempts, backoff_delay, timeout_ms FROM minion_jobs WHERE name = 'facts-absorb'`,
    );
    expect(jobs).toHaveLength(1);
    // Slow-retry policy (R2-5): config drift is fixed on human timescales.
    expect(Number(jobs[0].max_attempts)).toBe(5);
    expect(Number(jobs[0].backoff_delay)).toBe(60_000);
    expect(Number(jobs[0].timeout_ms)).toBe(10 * 60 * 1000);
  });
});

describe('transport-class failures throw typed', () => {
  test('provider_error propagates as FactsExtractionError in inline mode', async () => {
    configureGateway({ chat_model: 'anthropic:claude-sonnet-4-6', env: { ANTHROPIC_API_KEY: 'sk-ant-test' } });
    __setChatTransportForTests(async () => { throw new Error('simulated 500 from provider'); });
    let thrown: unknown;
    try {
      await runFactsPipeline('alice-example founded acme-example in 2010 with charlie-example.', {
        engine: engine as BrainEngine, sourceId: 'default', sessionId: null,
        source: 'mcp:extract_facts', mode: 'inline',
      });
    } catch (e) { thrown = e; }
    expect(thrown).toBeInstanceOf(FactsExtractionError);
    expect((thrown as FactsExtractionError).reason).toBe('provider_error');
    expect((thrown as FactsExtractionError).model).toBeTruthy();
  });

  test('classifier maps typed reasons precisely (queue-mode absorb codes)', () => {
    expect(classifyFactsAbsorbError(new FactsExtractionError('provider_error', 'openai:gpt-5.2', new Error('401')))).toBe('gateway_error');
    expect(classifyFactsAbsorbError(new FactsExtractionError('truncated_output', 'openai:gpt-5.2'))).toBe('truncated_output');
    expect(classifyFactsAbsorbError(new FactsExtractionError('chat_unavailable'))).toBe('chat_unavailable');
  });

  test('durable-job retry decision: KEYED unavailable throws, keyless completes calm, real results return (CX9)', () => {
    const unavailable: import('../src/core/facts/backstop.ts').FactsBackstopResult =
      { mode: 'inline', inserted: 0, duplicate: 0, superseded: 0, fact_ids: [], skipped: 'extraction_unavailable' };
    // Keyed worker + execution-time unavailability = config drift → retry.
    expect(factsAbsorbShouldRetry(unavailable, 'keyed')).toBe(true);
    expect(factsAbsorbShouldRetry({ mode: 'inline', inserted: 0, duplicate: 0, superseded: 0, fact_ids: [], skipped_reason: 'chat_unavailable' }, 'keyed')).toBe(true);
    // Keyless worker = expected steady state → calm skip-success, never a
    // failed-retry loop on every page write.
    expect(factsAbsorbShouldRetry(unavailable, 'keyless')).toBe(false);
    expect(factsAbsorbShouldRetry({ mode: 'inline', inserted: 2, duplicate: 0, superseded: 0, fact_ids: [1, 2] }, 'keyed')).toBe(false);
    // refusal/malformed are NOT retryable drift — they logged and returned.
    expect(factsAbsorbShouldRetry({ mode: 'inline', inserted: 0, duplicate: 0, superseded: 0, fact_ids: [], skipped_reason: 'malformed_output' }, 'keyed')).toBe(false);
    expect(factsAbsorbShouldRetry({ mode: 'queue', enqueued: true, queueDepth: 0 }, 'keyed')).toBe(false);
  });

  test('classifyUnavailable: keyed when any chat key or servable model exists; keyless otherwise', async () => {
    const { classifyUnavailable } = await import('../src/core/facts/backstop.ts');
    // Pinned env (beforeEach) + tmp GBRAIN_HOME → no keys anywhere → keyless.
    expect(await classifyUnavailable('anthropic:claude-sonnet-4-6')).toBe('keyless');
    process.env.OPENAI_API_KEY = 'sk-test';
    try {
      // A chat-capable key exists → a failure is a KEYED problem (visible).
      expect(await classifyUnavailable('anthropic:claude-sonnet-4-6')).toBe('keyed');
    } finally {
      delete process.env.OPENAI_API_KEY;
    }
  });
});

describe('extract_facts reason-specific envelopes', () => {
  test('keyless → skipped: extraction_unavailable + agent_action pinning visibility private (CX4)', async () => {
    configureGateway({ env: {} });
    const res = unwrap(await dispatchToolCall(engine, 'extract_facts', { turn_text: ELIGIBLE_BODY }, { remote: false }));
    expect(res.skipped).toBe('extraction_unavailable');
    expect(res.inserted).toBe(0);
    const action = String(res.agent_action);
    expect(action).toContain('remember');
    expect(action).toContain('visibility: "private"');
    expect(action).toContain('kind');
    expect(action).not.toContain('session id'); // remember has no session_id param (R2-8)
  });

  test('malformed extractor output → skipped: extraction_failed + reason (CX5 — never "unavailable")', async () => {
    configureGateway({ chat_model: 'anthropic:claude-sonnet-4-6', env: { ANTHROPIC_API_KEY: 'sk-ant-test' } });
    __setChatTransportForTests(async () => ({
      text: 'this is not json at all {{{',
      blocks: [{ type: 'text', text: 'this is not json at all {{{' }],
      stopReason: 'end' as const,
      usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 },
      model: 'anthropic:claude-sonnet-4-6',
      providerId: 'anthropic',
    }));
    const res = unwrap(await dispatchToolCall(engine, 'extract_facts', { turn_text: ELIGIBLE_BODY }, { remote: false }));
    expect(res.skipped).toBe('extraction_failed');
    expect(res.reason).toBe('malformed_output');
    expect(String(res.agent_action)).toContain('remember');
    // keyed-but-failing → visible to doctor
    const rows = await absorbRows();
    expect(rows.some((r) => r.summary.startsWith('malformed_output'))).toBe(true);
  });
});

describe('put_page never fails on extraction errors (E1 invariant)', () => {
  test('page write succeeds while the extraction transport throws', async () => {
    process.env.ANTHROPIC_API_KEY = 'sk-ant-test';
    configureGateway({ chat_model: 'anthropic:claude-sonnet-4-6', env: { ANTHROPIC_API_KEY: 'sk-ant-test' } });
    await engine.setConfig('facts.extraction_model', 'anthropic:claude-sonnet-4-6');
    __setChatTransportForTests(async () => { throw new Error('simulated provider outage'); });
    const res = await dispatchToolCall(engine, 'put_page', {
      slug: 'e1-invariant-page',
      content: `---\ntype: note\ntitle: E1 invariant\n---\n${ELIGIBLE_BODY}`,
    }, { remote: false });
    expect(res.isError).toBeFalsy();
    const page = await engine.getPage('e1-invariant-page', { sourceId: 'default' });
    expect(page).toBeTruthy();
    expect(page?.slug).toBe('e1-invariant-page');
    // Publication accepts durable extraction debt. Run its actual outbox
    // handoff and registered job handler; the later failure remains visible
    // while the original canonical receipt and page stay committed.
    expect(unwrap(res).facts_backstop).toEqual({ queued: true });
    await runPersistenceEffects(engine, { engine: 'pglite', embedding_disabled: true }, { hostId: localHostId(), limit: 8 });
    const jobs = await engine.executeRaw<{ id: number; data: Record<string, unknown> }>("SELECT id,data FROM minion_jobs WHERE name='facts-absorb'");
    expect(jobs).toHaveLength(1);
    const handlers = new Map<string, MinionHandler>();
    await registerBuiltinHandlers({ register: (name: string, handler: MinionHandler) => handlers.set(name, handler) } as never, engine, { quiet: true });
    await expect(handlers.get('facts-absorb')!({
      ...jobs[0], name: 'facts-absorb', attempts_made: 0, idempotency_key: null, signal: new AbortController().signal,
      deadlineAtMs: null, shutdownSignal: new AbortController().signal, updateProgress: async () => {},
      updateTokens: async () => {}, log: async () => {}, isActive: async () => true, readInbox: async () => [],
    })).rejects.toBeInstanceOf(FactsExtractionError);
    const rows = await absorbRows();
    expect(rows.some((r) => r.summary.startsWith('gateway_error'))).toBe(true);
  });
});

describe('legacy wrapper visibility (eng F3)', () => {
  test('extractFactsFromTurn warns once per reason instead of silent []', async () => {
    configureGateway({ env: {} }); // keyless → chat_unavailable
    const facts = await extractFactsFromTurn({ turnText: 'Some turn text.', source: 'test:wrapper' });
    expect(facts).toEqual([]);
    expect(stderrCapture).toContain('extraction skipped (chat_unavailable');
    const before = stderrCapture.length;
    await extractFactsFromTurn({ turnText: 'Another turn.', source: 'test:wrapper' });
    expect(stderrCapture.length).toBe(before); // once per reason per process
  });
});

describe('caller-resolved visibility pin (review-army addition)', () => {
  test('keyless envelope pins the CALLER visibility (world), not hardcoded private', async () => {
    configureGateway({ env: {} });
    const res = unwrap(await dispatchToolCall(engine, 'extract_facts', {
      turn_text: ELIGIBLE_BODY, visibility: 'world',
    }, { remote: false }));
    expect(res.skipped).toBe('extraction_unavailable');
    const action = String(res.agent_action);
    expect(action).toContain('visibility: "world"');
    expect(action).not.toContain('omitting it would widen');
  });
});

describe('coverage-gap additions (ship review)', () => {
  test('inline-lane keyless gate returns the skip envelope (the worker-consumed shape)', async () => {
    configureGateway({ env: {} });
    const r = await runFactsBackstop(
      { slug: 'outcome-inline-keyless', type: 'note', compiled_truth: ELIGIBLE_BODY, frontmatter: {} },
      { engine: engine as BrainEngine, sourceId: 'default', sessionId: null, source: 'mcp:put_page', mode: 'inline' },
    );
    expect(r.mode).toBe('inline');
    expect((r as { skipped?: string }).skipped).toBe('extraction_unavailable');
    expect((r as { inserted: number }).inserted).toBe(0);
  });

  test('CRITICAL placement rule: keyless SUBMITTER still enqueues the durable job (worker may be keyed)', async () => {
    configureGateway({ env: {} }); // keyless in the submitting process
    markShortLivedCliProcess();    // durable-submit lane
    const r = await runFactsBackstop(
      { slug: 'outcome-keyless-submit', type: 'note', compiled_truth: ELIGIBLE_BODY, frontmatter: {} },
      { engine: engine as BrainEngine, sourceId: 'default', sessionId: null, source: 'mcp:put_page', mode: 'queue' },
    );
    expect(r.mode).toBe('queue');
    expect((r as { enqueued: boolean }).enqueued).toBe(true); // NOT gated at submit time
    const jobs = await engine.executeRaw<{ name: string }>(
      `SELECT name FROM minion_jobs WHERE name = 'facts-absorb'`,
    );
    expect(jobs).toHaveLength(1);
  });

  test('FactsExtractionError redaction: JSON.stringify and spread omit the provider body', () => {
    const err = new FactsExtractionError('provider_error', 'openai:gpt-5.6-terra',
      new Error('401 Incorrect API key provided: sk-proj-****xyz'));
    expect(JSON.stringify(err)).not.toContain('sk-proj');
    expect(JSON.stringify({ ...err })).not.toContain('sk-proj');
    expect(err.message).not.toContain('sk-proj');
    // The cause stays reachable for local debugging.
    expect(String((err as { cause?: unknown }).cause)).toContain('401');
  });

  test('FactsExtractionError surfaces the cause CONSTRUCTOR name, never its .name or .message', () => {
    class FakeApiCallError extends Error {
      constructor(message: string) {
        super(message);
        // Deliberately different from the class name, so a test that reads
        // `.name` instead of `.constructor.name` would fail here — pinning
        // WHICH property the breadcrumb reads, not just that some name shows up.
        this.name = 'org_abc123_over_quota';
      }
    }
    const err = new FactsExtractionError('provider_error', 'claude-cli:claude-sonnet-5',
      new FakeApiCallError('connection reset by peer (org_abc123)'));
    // The class name is a diagnostic breadcrumb — it survives into the
    // persisted message so a durable job's `error_text` (which is all that
    // remains once the in-process `cause` is gone) can still distinguish
    // "which kind of provider failure" across occurrences.
    expect(err.message).toContain('(cause=FakeApiCallError)');
    // The message-carried breadcrumb is the CONSTRUCTOR name, never `.name`
    // and never `.message` — no room for a provider body to leak through it.
    expect(err.message).not.toContain('org_abc123');
    expect(err.message).not.toContain('connection reset');
    expect(JSON.stringify(err)).not.toContain('org_abc123');
  });

  test('FactsExtractionError drops a cause-constructor name that is not a plain identifier', () => {
    // A dependency could in principle construct an Error whose
    // `constructor.name` is an attacker-chosen string (this needs the
    // dependency to already run arbitrary code — not a normal provider
    // error path). The breadcrumb is validated against a plain-identifier
    // shape and DROPPED (not substituted) rather than trusted verbatim.
    class WeirdError extends Error {}
    Object.defineProperty(WeirdError, 'name', { value: 'org_example_secret sk-proj-fake123' });
    const err = new FactsExtractionError('provider_error', 'openai:gpt-5.2', new WeirdError('x'));
    expect(err.message).not.toContain('cause=');
    expect(err.message).not.toContain('org_example_secret');
    expect(err.message).not.toContain('sk-proj');
  });

  test('FactsExtractionError appends the whole-run failure class the gateway already computes', () => {
    // normalizeAIError wraps every provider failure as AIConfigError or
    // AITransientError, so the class name alone can't tell a 401 from a 400.
    // classifyGlobalLlmError (the vocabulary ingest_log already uses) walks
    // the cause chain for the status, so a dead job's error_text reads
    // `(cause=AIConfigError auth)` while the provider body — and the key it
    // echoes — still never reaches the message.
    const err = new FactsExtractionError('provider_error', 'openai:gpt-5.2',
      new AIConfigError('401 Incorrect API key sk-proj-x', undefined,
        Object.assign(new Error('x'), { statusCode: 401 })));
    expect(err.message).toContain('(cause=AIConfigError auth)');
    expect(err.message).not.toContain('sk-proj');
    expect(err.message).not.toContain('Incorrect API key');
  });

  test('FactsExtractionError omits the cause breadcrumb when there is no cause', () => {
    const err = new FactsExtractionError('truncated_output', 'openai:gpt-5.2');
    expect(err.message).toBe('[facts-extract] truncated_output (model=openai:gpt-5.2)');
    expect(err.message).not.toContain('cause=');
  });

  test('FactsExtractionError omits the cause breadcrumb when cause is not an Error', () => {
    const err = new FactsExtractionError('provider_error', 'openai:gpt-5.2', 'a raw string cause');
    expect(err.message).not.toContain('cause=');
    // The raw cause still stays reachable for local debugging.
    expect((err as { cause?: unknown }).cause).toBe('a raw string cause');
  });
});