Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.7 KB
Raw Blame
/**
 * Regression guards for the incremental extract path (PR #417).
 *
 * Eng-review Step 5: 8 unit cases asserting `runExtractCore({ slugs })`
 * processes only the requested slugs in the cycle path while
 * `slugs: undefined` falls through to the existing full-walk behavior.
 *
 * All tests use PGLite/in-memory — no DB connection required.
 */
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { runExtractCore } from '../src/commands/extract.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';

// One PGLite per file (beforeAll), wipe data per test (beforeEach).
// PGLite cold-start dominates wall-time; sharing the engine across all tests
// in this file cuts ~22s × 8 tests = ~3 min on CI.
let engine: PGLiteEngine;
let tempDir: string;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({ engine: 'pglite' });
  await engine.initSchema();
});

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(engine);
  tempDir = mkdtempSync(join(tmpdir(), 'gbrain-extract-test-'));
  mkdirSync(join(tempDir, 'people'), { recursive: true });
  mkdirSync(join(tempDir, 'companies'), { recursive: true });
});

afterEach(() => {
  rmSync(tempDir, { recursive: true, force: true });
});

async function seedPage(slug: string, body: string): Promise<void> {
  const [type, name] = slug.split('/');
  await engine.putPage(slug, {
    type: type as 'person' | 'company',
    title: name,
    compiled_truth: body,
    timeline: '',
    frontmatter: {},
    content_hash: 'h',
  });
  // Also write to disk so walkMarkdownFiles can find it
  const filePath = join(tempDir, slug + '.md');
  mkdirSync(join(tempDir, type), { recursive: true });
  writeFileSync(filePath, body);
}

describe('runExtractCore — incremental cycle path (#417)', () => {
  test('Dream incremental all-mode stamps the source-scoped extraction watermark (#2636)', async () => {
    await engine.executeRaw(
      `INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3)`,
      ['repo-a', 'repo-a', tempDir],
    );
    await engine.putPage('people/alice-example', {
      type: 'person',
      title: 'alice-example',
      compiled_truth: '# alice',
      timeline: '',
      frontmatter: {},
      content_hash: 'h',
    }, { sourceId: 'repo-a' });
    writeFileSync(join(tempDir, 'people/alice-example.md'), '# alice');

    await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'all',
      dir: tempDir,
      slugs: ['people/alice-example'],
      sourceId: 'repo-a',
    });

    const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
      `SELECT links_extracted_at FROM pages WHERE slug = $1 AND source_id = $2`,
      ['people/alice-example', 'repo-a'],
    );
    expect(rows[0]?.links_extracted_at).not.toBeNull();
    expect(await engine.countStalePagesForExtraction({ sourceId: 'repo-a' })).toBe(0);
  });

  test('Dream incremental dry-run does NOT stamp the watermark', async () => {
    await seedPage('people/alice-example', '# alice');
    await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'all',
      dir: tempDir,
      slugs: ['people/alice-example'],
      dryRun: true,
    });
    const rows = await engine.executeRaw<{ links_extracted_at: string | null }>(
      `SELECT links_extracted_at FROM pages WHERE slug = $1`,
      ['people/alice-example'],
    );
    expect(rows[0]?.links_extracted_at ?? null).toBeNull();
  });

  test('1. slugs: [] returns immediately with zero counts (early-return path)', async () => {
    await seedPage('people/alice-example', '# alice');
    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'all',
      dir: tempDir,
      slugs: [],
    });
    expect(result.links_created).toBe(0);
    expect(result.timeline_entries_created).toBe(0);
    expect(result.pages_processed).toBe(0);
  });

  test('2. slugs: undefined falls through to full-walk path', async () => {
    await seedPage('people/alice-example', '# alice\n\n[bob](people/bob-example)');
    await seedPage('people/bob-example', '# bob');
    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'all',
      dir: tempDir,
    });
    // Full walk processes everything found on disk
    expect(result.pages_processed).toBeGreaterThan(0);
  });

  test('3. slugs: [a, b] reads only those two files (incremental processing)', async () => {
    await seedPage('people/alice-example', '# alice');
    await seedPage('people/bob-example', '# bob');
    await seedPage('people/charlie-example', '# charlie');
    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'all',
      dir: tempDir,
      slugs: ['people/alice-example', 'people/bob-example'],
    });
    // Only 2 files processed even though 3 exist on disk
    expect(result.pages_processed).toBe(2);
  });

  test('4. Slug whose file no longer exists is silently skipped', async () => {
    await seedPage('people/alice-example', '# alice');
    // people/ghost has no file on disk but is in the slugs list
    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'all',
      dir: tempDir,
      slugs: ['people/alice-example', 'people/ghost'],
    });
    // alice processed; ghost skipped (no file)
    expect(result.pages_processed).toBe(1);
  });

  test('5. mode: links skips timeline extraction in incremental', async () => {
    const body = '# alice\n\n## Timeline\n- 2026-01-01: started';
    await seedPage('people/alice-example', body);
    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'links',
      dir: tempDir,
      slugs: ['people/alice-example'],
    });
    // Timeline extraction skipped even though body contains a timeline
    expect(result.timeline_entries_created).toBe(0);
  });

  test('6. dryRun: true does not invoke addLinksBatch / addTimelineEntriesBatch', async () => {
    await seedPage('people/alice-example', '# alice\n\n[bob](people/bob-example)');
    await seedPage('people/bob-example', '# bob');

    let linksBatchCalled = false;
    let timelineBatchCalled = false;
    const originalAddLinks = engine.addLinksBatch.bind(engine);
    const originalAddTimeline = engine.addTimelineEntriesBatch.bind(engine);
    (engine as unknown as { addLinksBatch: typeof originalAddLinks }).addLinksBatch = async (...args) => {
      linksBatchCalled = true;
      return originalAddLinks(...args);
    };
    (engine as unknown as { addTimelineEntriesBatch: typeof originalAddTimeline }).addTimelineEntriesBatch = async (...args) => {
      timelineBatchCalled = true;
      return originalAddTimeline(...args);
    };

    try {
      await runExtractCore(engine as unknown as BrainEngine, {
        mode: 'all',
        dir: tempDir,
        slugs: ['people/alice-example'],
        dryRun: true,
      });
    } finally {
      // Restore the shared engine's prototype methods: a wrapper bound to
      // `engine` would run a later test's in-transaction batch insert outside
      // its transaction.
      delete (engine as unknown as { addLinksBatch?: unknown }).addLinksBatch;
      delete (engine as unknown as { addTimelineEntriesBatch?: unknown }).addTimelineEntriesBatch;
    }

    expect(linksBatchCalled).toBe(false);
    expect(timelineBatchCalled).toBe(false);
  });

  test('7. BATCH_SIZE flush — slugs producing >100 candidate links exercise the mid-iteration flush', async () => {
    // BATCH_SIZE in extract.ts is 100. Create one slug with 150 outbound links.
    const targets: string[] = [];
    for (let i = 0; i < 150; i++) {
      const target = `companies/co-${i}`;
      targets.push(target);
      await seedPage(target, `# co-${i}`);
    }
    const linkBlock = targets.map(t => `- [${t}](${t})`).join('\n');
    await seedPage('people/alice-example', `# alice\n\n${linkBlock}`);

    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'links',
      dir: tempDir,
      slugs: ['people/alice-example'],
    });
    // The flush happens mid-iteration when batch hits 100; the remaining 50 flush at end.
    // No exception means the flush path executed cleanly.
    expect(result.pages_processed).toBe(1);
    expect(result.links_created).toBeGreaterThanOrEqual(0); // Just confirms the flush path didn't blow up
  });

  test('8. Full-slug-set resolution — slug references file outside changed set', async () => {
    // alice references bob, but only alice is in the incremental slugs list.
    // The allSlugs set must still include bob (from walkMarkdownFiles) so
    // resolveSlug succeeds; otherwise the link would silently drop.
    // Markdown link pattern requires .md target.
    await seedPage('people/alice-example', '# alice\n\n[bob](bob-example.md)');
    await seedPage('people/bob-example', '# bob');

    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'links',
      dir: tempDir,
      slugs: ['people/alice-example'],
    });

    // Only alice's file was read, but the resulting link must reference bob
    // (resolved via the full allSlugs set built from walkMarkdownFiles).
    expect(result.pages_processed).toBe(1);
    // Link from alice to bob was extracted successfully via the full allSlugs set
    expect(result.links_created).toBeGreaterThan(0);
  });
});
describe('runExtractCore — incremental frontmatter gate (includeFrontmatter)', () => {
  // alice has a `source:` frontmatter edge but NO body links. The incremental
  // path extracts body links only by default, so the frontmatter edge is the
  // sole signal that distinguishes the gate off vs on.
  const aliceFm = '---\nsource: companies/acme-example\n---\n# alice';

  test('9. default (flag omitted) does NOT extract frontmatter links on the incremental path', async () => {
    await seedPage('companies/acme-example', '# acme');
    await seedPage('people/alice-example', aliceFm);
    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'all',
      dir: tempDir,
      slugs: ['people/alice-example'],
    });
    // alice's only potential edge is her frontmatter `source:`; with the gate off
    // it must not be extracted (preserves the body-only incremental behavior).
    expect(result.pages_processed).toBe(1);
    expect(result.links_created).toBe(0);
  });

  test('10. includeFrontmatter: true extracts the frontmatter link on the incremental path', async () => {
    await seedPage('companies/acme-example', '# acme');
    await seedPage('people/alice-example', aliceFm);
    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'all',
      dir: tempDir,
      slugs: ['people/alice-example'],
      includeFrontmatter: true,
    });
    // Same page, gate on → the `source:` frontmatter edge is now extracted.
    expect(result.pages_processed).toBe(1);
    expect(result.links_created).toBeGreaterThan(0);
  });
});

// ─── wave review: `quiet` is its own knob, not a side effect of jsonMode ──
describe('runExtractCore — quiet vs jsonMode on the incremental path', () => {
  function captureStdout(): { lines: string[]; restore: () => void } {
    const lines: string[] = [];
    const saved = console.log;
    console.log = (...a: unknown[]) => { lines.push(a.map(String).join(' ')); };
    return { lines, restore: () => { console.log = saved; } };
  }

  test('quiet: true suppresses the human summary line while jsonMode stays false (human stderr channel)', async () => {
    // The cycle used `jsonMode: true` as a stand-in for "quiet": it hid the
    // summary, but it ALSO flipped batch errors to JSON events on stderr in a
    // plain `gbrain dream`. Embedded callers own the report — they say so.
    await seedPage('people/alice', 'Alice knows [[companies/acme]].');
    await seedPage('companies/acme', 'Acme.');
    const out = captureStdout();
    try {
      await runExtractCore(engine, { mode: 'all', dir: tempDir, slugs: ['people/alice'], jsonMode: false, quiet: true });
    } finally {
      out.restore();
    }
    expect(out.lines.filter((l) => l.includes('Incremental extract'))).toEqual([]);
  });

  test('without quiet the human summary line still prints (CLI behaviour unchanged)', async () => {
    await seedPage('people/alice', 'Alice knows [[companies/acme]].');
    await seedPage('companies/acme', 'Acme.');
    const out = captureStdout();
    try {
      await runExtractCore(engine, { mode: 'all', dir: tempDir, slugs: ['people/alice'], jsonMode: false });
    } finally {
      out.restore();
    }
    expect(out.lines.some((l) => l.startsWith('Incremental extract: created'))).toBe(true);
  });

  test('a filename that does not round-trip through slugification is still read', async () => {
    // `Widget Co.md` slugs to `companies/widget-co` — the space becomes a
    // hyphen and the capitals drop. Rebuilding the disk path from the slug
    // looks for `companies/widget-co.md`, which is not there, so the page was
    // treated as deleted and skipped without a word.
    await engine.putPage('companies/widget-co', {
      type: 'company', title: 'Widget Co',
      compiled_truth: 'Widget Co knows [[companies/acme]].',
      timeline: '', frontmatter: {}, content_hash: 'h',
    });
    writeFileSync(join(tempDir, 'companies/Widget Co.md'), 'Widget Co knows [[companies/acme]].');
    await seedPage('companies/acme', '# acme');

    const result = await runExtractCore(engine as unknown as BrainEngine, {
      mode: 'all', dir: tempDir, slugs: ['companies/widget-co'],
    });
    expect(result.pages_processed).toBe(1);
    const links = await engine.getLinks('companies/widget-co');
    expect(links.some(l => l.to_slug === 'companies/acme')).toBe(true);
  });

  test('when two filenames slug the same, the file named exactly the slug wins', async () => {
    const { buildSlugPathIndex } = await import('../src/commands/extract.ts');
    // `Acme!.md` and `acme.md` both slug to `companies/acme` (the `!` is not a
    // legal slug char). The walker does not sort, so a plain last-one-wins map
    // would resolve the slug differently depending on directory-read order —
    // and thus differently across filesystems. The exact-slug spelling is what
    // the old reconstructed path found, so it stays the winner either way.
    const files = [
      { relPath: 'companies/acme.md' },
      { relPath: 'companies/Acme!.md' },
    ];
    expect(buildSlugPathIndex(files).get('companies/acme')).toBe('companies/acme.md');
    expect(buildSlugPathIndex([...files].reverse()).get('companies/acme')).toBe('companies/acme.md');
  });
});