Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
22.6 KB
Raw Blame
/**
 * v0.31.2 — runFactsBackstop pipeline tests.
 *
 * Pins the helper's full contract: eligibility/kill-switch gates,
 * mode dispatch (queue vs inline), notability filter, dedup fast-path,
 * abort propagation, and skipped envelope shapes.
 *
 * Real PGLite engine (in-memory, no DATABASE_URL). LLM is stubbed via
 * __setChatTransportForTests so tests are deterministic + fast.
 */

import { describe, test, expect, beforeAll, afterAll, afterEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { runFactsBackstop } from '../src/core/facts/backstop.ts';
import type { FactsBackstopCtx } from '../src/core/facts/backstop.ts';
import {
  __setChatTransportForTests,
  __setEmbedTransportForTests,
  configureGateway,
  resetGateway,
  type ChatResult,
} from '../src/core/ai/gateway.ts';
import { __resetFactsQueueForTests } from '../src/core/facts/queue.ts';
import { MinionWorker } from '../src/core/minions/worker.ts';
import type { MinionJobContext } from '../src/core/minions/types.ts';
import { registerBuiltinHandlers } from '../src/commands/jobs.ts';

let engine: PGLiteEngine;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
});

afterAll(async () => {
  await engine.disconnect();
});

afterEach(() => {
  __setChatTransportForTests(null);
  resetGateway();
  __resetFactsQueueForTests();
});

const LONG_BODY = 'this is a real meeting note longer than 80 chars '.repeat(3);

function chatStub(facts: Array<{ fact: string; kind: string; notability?: string; entity?: string | null }>) {
  __setChatTransportForTests(async (): Promise<ChatResult> => ({
    text: JSON.stringify({
      facts: facts.map(f => ({
        fact: f.fact,
        kind: f.kind,
        entity: f.entity ?? null,
        confidence: 1.0,
        notability: f.notability,
      })),
    }),
    blocks: [],
    stopReason: 'end',
    usage: { input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_creation_tokens: 0 },
    model: 'test:stub',
    providerId: 'test',
  }));
}

function makeCtx(overrides: Partial<FactsBackstopCtx> = {}): FactsBackstopCtx {
  return {
    engine,
    sourceId: 'default',
    sessionId: null,
    source: 'mcp:put_page',
    ...overrides,
  };
}

const meetingPage = (slug = 'meetings/test-' + Math.random().toString(36).slice(2, 9)) => ({
  slug,
  type: 'meeting' as const,
  compiled_truth: LONG_BODY,
  frontmatter: {} as Record<string, unknown>,
});

async function factsForIds(ids: number[]): Promise<Array<{ fact: string }>> {
  return Promise.all(ids.map(async (id) => {
    // PGLite's test engine deliberately exposes its raw query client for
    // storage assertions where the public API only offers scoped listings.
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const rows = await (engine as any).db.query('SELECT fact FROM facts WHERE id = $1', [id]);
    return rows.rows[0];
  }));
}

describe('runFactsBackstop — eligibility + kill-switch gates', () => {
  test('skips with extraction_disabled when kill-switch off', async () => {
    await engine.setConfig('facts.extraction_enabled', 'false');
    const r = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline' }));
    expect(r.mode).toBe('inline');
    if (r.mode === 'inline') expect(r.skipped).toBe('extraction_disabled');
    await engine.setConfig('facts.extraction_enabled', 'true');
  });

  test('skips with eligibility_failed:<reason> when subagent namespace', async () => {
    chatStub([]);
    const r = await runFactsBackstop(
      { ...meetingPage('wiki/agents/scratch/foo'), type: 'meeting' },
      makeCtx({ mode: 'inline' }),
    );
    expect(r.mode).toBe('inline');
    if (r.mode === 'inline') expect(r.skipped).toBe('eligibility_failed:subagent_namespace');
  });

  test('skips with eligibility_failed:dream_generated when frontmatter set', async () => {
    chatStub([]);
    const page = meetingPage();
    page.frontmatter = { dream_generated: true };
    const r = await runFactsBackstop(page, makeCtx({ mode: 'inline' }));
    expect(r.mode).toBe('inline');
    if (r.mode === 'inline') expect(r.skipped).toBe('eligibility_failed:dream_generated');
  });
});

describe('runFactsBackstop — mode: inline', () => {
  test('inserts the LLM-extracted facts and returns counts', async () => {
    chatStub([
      { fact: 'mode-inline-event-1', kind: 'event', notability: 'high', entity: 'people/alice-example' },
      { fact: 'mode-inline-event-2', kind: 'event', notability: 'medium', entity: 'people/alice-example' },
    ]);
    const r = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline' }));
    expect(r.mode).toBe('inline');
    if (r.mode === 'inline') {
      expect(r.inserted).toBe(2);
      expect(r.duplicate).toBe(0);
      expect(r.fact_ids.length).toBe(2);
    }
  });

  test('notabilityFilter=high-only embeds and persists only HIGH facts', async () => {
    const embeddedTexts: string[] = [];
    configureGateway({
      embedding_model: 'openai:text-embedding-3-small',
      embedding_dimensions: 1536,
      env: { OPENAI_API_KEY: 'test' },
    });
    chatStub([
      { fact: 'high-only-1', kind: 'event', notability: 'high', entity: 'people/bob-test' },
      { fact: 'high-only-medium-skip', kind: 'event', notability: 'medium', entity: 'people/bob-test' },
      { fact: 'high-only-low-skip', kind: 'event', notability: 'low', entity: 'people/bob-test' },
      { fact: 'high-only-absent-skip', kind: 'event', entity: 'people/bob-test' },
      { fact: 'high-only-unknown-skip', kind: 'event', notability: 'unknown', entity: 'people/bob-test' },
    ]);
    __setEmbedTransportForTests((async ({ values }: { values: string[] }) => {
      embeddedTexts.push(...values);
      return { embeddings: values.map(() => Array.from({ length: 1536 }, () => 0.1)) };
    }) as never);
    const r = await runFactsBackstop(
      meetingPage(),
      makeCtx({ mode: 'inline', notabilityFilter: 'high-only' }),
    );
    expect(r.mode).toBe('inline');
    if (r.mode === 'inline') {
      expect(r.inserted).toBe(1);
      expect(r.fact_ids.length).toBe(1);
      const rows = await factsForIds(r.fact_ids);
      expect(embeddedTexts).toEqual(['high-only-1']);
      expect(rows.map(f => f.fact)).toEqual(['high-only-1']);
    }
  });

  test('notabilityFilter=all embeds and persists every tier', async () => {
    const embeddedTexts: string[] = [];
    configureGateway({
      embedding_model: 'openai:text-embedding-3-small',
      embedding_dimensions: 1536,
      env: { OPENAI_API_KEY: 'test' },
    });
    chatStub([
      { fact: 'all-high', kind: 'event', notability: 'high', entity: 'people/all-test' },
      { fact: 'all-medium', kind: 'event', notability: 'medium', entity: 'people/all-test' },
      { fact: 'all-low', kind: 'event', notability: 'low', entity: 'people/all-test' },
      { fact: 'all-absent', kind: 'event', entity: 'people/all-test' },
    ]);
    __setEmbedTransportForTests((async ({ values }: { values: string[] }) => {
      embeddedTexts.push(...values);
      return { embeddings: values.map(() => Array.from({ length: 1536 }, () => 0.1)) };
    }) as never);
    const r = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline', notabilityFilter: 'all' }));
    expect(r.mode).toBe('inline');
    if (r.mode === 'inline') {
      const rows = await factsForIds(r.fact_ids);
      expect(embeddedTexts).toEqual(['all-high', 'all-medium', 'all-low', 'all-absent']);
      expect(rows.map(f => f.fact)).toEqual(['all-high', 'all-medium', 'all-low', 'all-absent']);
    }
  });

  test('source string lands on the inserted row', async () => {
    const sessionId = 'source-pin-session-' + Math.random().toString(36).slice(2, 9);
    chatStub([{ fact: 'source-pin-fact', kind: 'fact', notability: 'medium', entity: null }]);
    const r = await runFactsBackstop(
      meetingPage(),
      makeCtx({ mode: 'inline', source: 'sync:import', sessionId }),
    );
    expect(r.mode).toBe('inline');
    if (r.mode === 'inline' && r.fact_ids.length > 0) {
      // Query by source_session (deterministic, no resolveEntitySlug rewrite).
      const rows = await engine.listFactsBySession('default', sessionId);
      const ours = rows.find(x => x.id === r.fact_ids[0]);
      expect(ours?.source).toBe('sync:import');
      expect(ours?.source_session).toBe(sessionId);
    }
  });

  test('empty extraction → zero counts', async () => {
    chatStub([]);
    const r = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline' }));
    expect(r.mode).toBe('inline');
    if (r.mode === 'inline') {
      expect(r.inserted).toBe(0);
      expect(r.duplicate).toBe(0);
      expect(r.fact_ids.length).toBe(0);
    }
  });

  test('aborted before LLM call → zero counts, no throw', async () => {
    chatStub([]);
    const ac = new AbortController();
    ac.abort();
    const r = await runFactsBackstop(
      meetingPage(),
      makeCtx({ mode: 'inline', abortSignal: ac.signal }),
    );
    expect(r.mode).toBe('inline');
    if (r.mode === 'inline') expect(r.inserted).toBe(0);
  });
});

describe('runFactsBackstop — mode: queue', () => {
  test('default mode is queue; returns enqueued: true', async () => {
    chatStub([{ fact: 'queue-default-1', kind: 'event', notability: 'high', entity: 'people/queue-test' }]);
    const r = await runFactsBackstop(meetingPage(), makeCtx());  // no mode → default 'queue'
    expect(r.mode).toBe('queue');
    if (r.mode === 'queue') {
      expect(r.enqueued).toBe(true);
      expect(r.queueDepth).toBeGreaterThanOrEqual(0);
    }
  });

  test('explicit mode=queue returns immediately with enqueued: true', async () => {
    chatStub([{ fact: 'queue-explicit', kind: 'event', notability: 'high', entity: 'people/queue-explicit' }]);
    const r = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'queue' }));
    expect(r.mode).toBe('queue');
    if (r.mode === 'queue') expect(r.enqueued).toBe(true);
  });

  test('queue mode with extraction_disabled returns enqueued: false + skipped reason', async () => {
    await engine.setConfig('facts.extraction_enabled', 'false');
    const r = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'queue' }));
    expect(r.mode).toBe('queue');
    if (r.mode === 'queue') {
      expect(r.enqueued).toBe(false);
      expect(r.skipped).toBe('extraction_disabled');
    }
    await engine.setConfig('facts.extraction_enabled', 'true');
  });
});

describe('runFactsBackstop — dedup fast-path', () => {
  test('two near-identical inserts: second comes back as duplicate', async () => {
    // Insert first via inline mode; we'll re-fire with the same fact text
    // and rely on cosine ≥ 0.95 when the embedding matches. The B1 smoke
    // path's stubbed transport means embedding stays null (gateway not
    // configured), so the dedup path needs candidates with embeddings.
    // Skip the embedding-match assertion here and pin the no-dedup path:
    chatStub([
      { fact: 'distinct-fact-A', kind: 'event', notability: 'high', entity: 'people/dedup-a' },
    ]);
    const r1 = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline' }));
    expect(r1.mode).toBe('inline');
    if (r1.mode === 'inline') expect(r1.inserted).toBe(1);

    // Without embeddings present, dedup short-circuits; the second call
    // inserts a new row (insertFact does no further dedup unless caller
    // passes supersedeId). That's the contract for queue+inline backstop.
    chatStub([
      { fact: 'distinct-fact-A', kind: 'event', notability: 'high', entity: 'people/dedup-a' },
    ]);
    const r2 = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline' }));
    expect(r2.mode).toBe('inline');
    if (r2.mode === 'inline') {
      // Without embeddings the dedup fast-path can't fire; second insert lands.
      // Real production has embeddings via gateway — covered by E2E in a
      // future test that points at a configured chat+embed gateway.
      expect(r2.inserted + r2.duplicate).toBe(1);
    }
  });
});

describe('runFactsBackstop — exact duplicates without a resolved entity (#5275)', () => {
  // The legacy DB-only path used to skip the exact-fingerprint check when the
  // entity was null or fell back to slugification, so every re-extraction
  // inserted an identical active row outside the fence-key unique index.
  const census = async (text: string) => (await engine.executeRaw<{ entity_slug: string | null; n: number }>(
    `SELECT entity_slug, count(*)::int AS n FROM facts WHERE fact = $1 AND expired_at IS NULL GROUP BY entity_slug ORDER BY entity_slug`, [text]));

  test('a NULL-entity fact re-extracted twice lands once and the repeat reports duplicate with the same id', async () => {
    chatStub([{ fact: 'unparented-claim-A', kind: 'event', notability: 'high', entity: null }]);
    const r1 = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline', source: 'sync:import' }));
    expect(r1.mode).toBe('inline');
    if (r1.mode !== 'inline') return;
    expect(r1.inserted).toBe(1);

    chatStub([{ fact: 'unparented-claim-A', kind: 'event', notability: 'high', entity: null }]);
    const r2 = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline', source: 'sync:import' }));
    if (r2.mode !== 'inline') return;
    expect({ inserted: r2.inserted, duplicate: r2.duplicate, ids: r2.fact_ids }).toEqual({ inserted: 0, duplicate: 1, ids: r1.fact_ids });
    expect(await census('unparented-claim-A')).toEqual([{ entity_slug: null, n: 1 }]);
  });

  test('a bare-name entity that falls back to slugification dedups the same way; a fingerprint variant too', async () => {
    chatStub([{ fact: 'Said hello, twice', kind: 'event', notability: 'high', entity: 'nobodyresolvable' }]);
    const r1 = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline', source: 'sync:import' }));
    if (r1.mode !== 'inline') return;
    expect(r1.inserted).toBe(1);
    // Same fingerprint (whitespace and case differences collapse), different bytes.
    chatStub([{ fact: 'said  hello, twice', kind: 'event', notability: 'high', entity: 'nobodyresolvable' }]);
    const r2 = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline', source: 'sync:import' }));
    if (r2.mode !== 'inline') return;
    expect({ inserted: r2.inserted, duplicate: r2.duplicate }).toEqual({ inserted: 0, duplicate: 1 });
    expect(await census('Said hello, twice')).toEqual([{ entity_slug: null, n: 1 }]);
  });

  test('visibility and attribution still separate rows; different text inserts', async () => {
    chatStub([{ fact: 'unparented-claim-B', kind: 'event', notability: 'high', entity: null }]);
    const a = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline', source: 'sync:import' }));
    if (a.mode !== 'inline') return;
    expect(a.inserted).toBe(1);
    chatStub([{ fact: 'unparented-claim-B', kind: 'event', notability: 'high', entity: null }]);
    const b = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline', source: 'sync:import', visibility: 'world' }));
    if (b.mode !== 'inline') return;
    expect(b.inserted).toBe(1);
    chatStub([{ fact: 'unparented-claim-C', kind: 'event', notability: 'high', entity: null }]);
    const c = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline', source: 'sync:import' }));
    if (c.mode !== 'inline') return;
    expect(c.inserted).toBe(1);
    const rows = await engine.executeRaw<{ fact: string; visibility: string }>(
      `SELECT fact, visibility FROM facts WHERE fact LIKE 'unparented-claim-%' AND fact <> 'unparented-claim-A' ORDER BY id`);
    expect(rows).toEqual([
      { fact: 'unparented-claim-B', visibility: 'private' },
      { fact: 'unparented-claim-B', visibility: 'world' },
      { fact: 'unparented-claim-C', visibility: 'private' },
    ]);
  });
});

describe('runFactsBackstop — stub guard routing (v0.34.5)', () => {
  test('bare-name entity routes to legacy DB-only path (no phantom page)', async () => {
    const { mkdtempSync, rmSync, existsSync } = await import('node:fs');
    const { tmpdir } = await import('node:os');
    const { join } = await import('node:path');

    const brainDir = mkdtempSync(join(tmpdir(), 'backstop-stub-guard-'));
    try {
      // Point the default source at the tempdir so localPath is non-null.
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
      await (engine as any).db.query(
        `UPDATE sources SET local_path = $1 WHERE id = 'default'`,
        [brainDir],
      );

      // Stub the chat to return a fact with a bare-name entity. The
      // resolver will:
      //   1. exact slug match → miss (no 'noresolvable' row)
      //   2. fuzzy match → miss (no title contains noresolvable)
      //   3. prefix expansion → miss (no people/noresolvable-* rows)
      //   4. slugify fallback → 'noresolvable' (bare)
      chatStub([
        { fact: 'said hello at the meeting', kind: 'event', notability: 'high', entity: 'noresolvable' },
      ]);

      const r = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline' }));

      expect(r.mode).toBe('inline');
      if (r.mode === 'inline') {
        // The fact MUST be persisted via the DB-only fallback, not dropped.
        expect(r.inserted).toBe(1);
        expect(r.fact_ids.length).toBe(1);

        // No phantom file at the brain root (this is the whole point of the guard).
        expect(existsSync(join(brainDir, 'noresolvable.md'))).toBe(false);

        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        const rows = await (engine as any).db.query(
          `SELECT entity_slug, fact, source_markdown_slug FROM facts WHERE id = $1`,
          [r.fact_ids[0]],
        );
        expect(rows.rows[0].entity_slug).toBeNull();
        expect(rows.rows[0].fact).toBe('said hello at the meeting');
        // source_markdown_slug is the fence-tracking column; under DB-only
        // fallback it stays null (no .md file backs the row).
        expect(rows.rows[0].source_markdown_slug).toBeNull();
      }
    } finally {
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
      await (engine as any).db.query(`UPDATE sources SET local_path = NULL WHERE id = 'default'`);
      rmSync(brainDir, { recursive: true, force: true });
    }
  });
});

describe('runFactsBackstop — sync.write_through opt-out', () => {
  test('flag disabled routes fence-eligible facts to DB-only (no fence file, fact still lands)', async () => {
    const { mkdtempSync, rmSync, existsSync } = await import('node:fs');
    const { tmpdir } = await import('node:os');
    const { join } = await import('node:path');
    const { _resetWriteThroughCacheForTest } = await import('../src/core/write-through.ts');

    const brainDir = mkdtempSync(join(tmpdir(), 'backstop-write-through-flag-'));
    try {
      await engine.putPage('people/flag-test', {
        type: 'person', title: 'Flag Test', compiled_truth: '# Existing entity', frontmatter: {},
      }, { sourceId: 'default' });
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
      await (engine as any).db.query(
        `UPDATE sources SET local_path = $1 WHERE id = 'default'`,
        [brainDir],
      );
      await engine.setConfig('sync.write_through', 'false');
      _resetWriteThroughCacheForTest();

      chatStub([
        { fact: 'joined widget-co as cto', kind: 'event', notability: 'high', entity: 'people/flag-test' },
      ]);

      const r = await runFactsBackstop(meetingPage(), makeCtx({ mode: 'inline' }));

      expect(r.mode).toBe('inline');
      if (r.mode === 'inline') {
        // The fact MUST persist via the DB-only route, not get dropped.
        expect(r.inserted).toBe(1);
        expect(r.fact_ids.length).toBe(1);

        // No fence file, no stub page, not even the directory.
        expect(existsSync(join(brainDir, 'people/flag-test.md'))).toBe(false);
        expect(existsSync(join(brainDir, 'people'))).toBe(false);

        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        const rows = await (engine as any).db.query(
          `SELECT entity_slug, source_markdown_slug FROM facts WHERE id = $1`,
          [r.fact_ids[0]],
        );
        expect(rows.rows[0].entity_slug).toBe('people/flag-test');
        // DB-only rows have no .md of record.
        expect(rows.rows[0].source_markdown_slug).toBeNull();
      }
    } finally {
      await engine.unsetConfig('sync.write_through');
      _resetWriteThroughCacheForTest();
      // eslint-disable-next-line @typescript-eslint/no-explicit-any
      await (engine as any).db.query(`UPDATE sources SET local_path = NULL WHERE id = 'default'`);
      rmSync(brainDir, { recursive: true, force: true });
    }
  });
});

// #4870: the durable facts-absorb payload carries the writer's notabilityFilter
// verbatim (backstop.ts), but the minion handler (commands/jobs.ts) is the only
// reader of that value — it must honor every value the ctx union permits, not
// collapse everything but 'high-only' to 'all'. Drive the registered handler
// directly with a queued 'medium-and-up' payload (reachable at stock master via
// `gbrain jobs submit facts-absorb --params ...`).
describe('facts-absorb minion handler honors the queued notabilityFilter (#4870)', () => {
  test("a queued 'medium-and-up' payload persists high+medium and drops low", async () => {
    const worker = new MinionWorker(engine, { queue: 'test' });
    await registerBuiltinHandlers(worker, engine, { quiet: true });
    const handler = worker.getHandler('facts-absorb');
    expect(handler).toBeDefined();

    const page = meetingPage();
    await engine.executeRaw(
      `INSERT INTO pages (slug, source_id, type, title, compiled_truth) VALUES ($1, 'default', 'meeting', $1, $2)`,
      [page.slug, LONG_BODY],
    );
    chatStub([
      { fact: 'queued-medium-up-high', kind: 'event', notability: 'high', entity: 'people/queue-test' },
      { fact: 'queued-medium-up-medium', kind: 'event', notability: 'medium', entity: 'people/queue-test' },
      { fact: 'queued-medium-up-low', kind: 'event', notability: 'low', entity: 'people/queue-test' },
    ]);
    const job: MinionJobContext = {
      id: 1,
      name: 'facts-absorb',
      data: { slug: page.slug, sourceId: 'default', source: 'mcp:put_page', notabilityFilter: 'medium-and-up' },
      attempts_made: 0,
      idempotency_key: null,
      signal: new AbortController().signal,
      deadlineAtMs: null,
      shutdownSignal: new AbortController().signal,
      updateProgress: async () => {},
      updateTokens: async () => {},
      log: async () => {},
      isActive: async () => true,
      readInbox: async () => [],
    };
    await handler!(job);

    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const rows = await (engine as any).db.query(
      `SELECT fact FROM facts WHERE fact LIKE 'queued-medium-up-%' ORDER BY fact`,
    );
    expect(rows.rows.map((r: { fact: string }) => r.fact)).toEqual([
      'queued-medium-up-high',
      'queued-medium-up-medium',
    ]);
  });
});