Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #4062 — the cycle's extract phase drains the stale-extraction backlog.
*
* Pre-fix, runPhaseExtract only ran runExtractCore over the slugs sync
* reported (or an fs walk of brainDir). Pages left stale for any other
* reason — an extractor version bump, DB-only writes, a prior aborted
* sweep — never re-extracted on the cycle, so the links_extracted_at
* backlog grew unboundedly until someone hand-ran `gbrain extract --stale`.
* This pins that the extract phase follows the targeted pass with a
* bounded, source-scoped extractStaleFromDB drain and reports
* staleRemaining in the phase details.
*
* GBRAIN_HOME is isolated per test because the PGLite cycle path takes a
* file lock at ~/.gbrain/cycle.lock (see cycle-extract-source.test.ts).
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach, spyOn } from 'bun:test';
import { mkdtempSync, rmSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { withEnv } from './helpers/with-env.ts';
import { runCycle } from '../src/core/cycle.ts';
import { extractStaleFromDB, runExtractCore } from '../src/commands/extract.ts';
let engine: PGLiteEngine;
let brainDir: string;
let gbrainHome: string;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 30_000);
afterAll(async () => {
await engine.disconnect();
}, 30_000);
beforeEach(async () => {
await resetPgliteState(engine);
// Empty checkout: the targeted fs-walk pass finds nothing, so any links
// created can only come from the stale DB drain.
brainDir = mkdtempSync(join(tmpdir(), 'gbrain-cycle-stale-drain-'));
gbrainHome = mkdtempSync(join(tmpdir(), 'gbrain-cycle-stale-drain-home-'));
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path) VALUES ('wiki', 'wiki', $1)
ON CONFLICT (id) DO UPDATE SET local_path = EXCLUDED.local_path`,
[brainDir],
);
// DB-only pages (never present in the checkout) with links_extracted_at
// NULL → stale. alice's body links to bob.
await engine.executeRaw(
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline)
VALUES
('people/alice', 'wiki', 'person', 'Alice', 'Met [[people/bob]] today.', ''),
('people/bob', 'wiki', 'person', 'Bob', 'Quiet page.', '')`,
);
});
async function stamped(slug: string): Promise<boolean> {
const [row] = await engine.executeRaw<{ stamped: boolean }>(
'SELECT links_extracted_at IS NOT NULL AS stamped FROM pages WHERE source_id = $1 AND slug = $2', ['wiki', slug]);
return row!.stamped;
}
afterEach(() => {
rmSync(brainDir, { recursive: true, force: true });
rmSync(gbrainHome, { recursive: true, force: true });
});
describe('cycle extract phase stale drain (#4062)', () => {
test('extraction totals count scanned pages rather than links', async () => {
writeFileSync(join(brainDir, 'quiet.md'), '# Quiet page\nNo outgoing links.\n');
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
const report = await runCycle(engine, { brainDir, sourceId: 'wiki', phases: ['extract'] });
expect(report.phases[0]?.details?.linksCreated).toBe(0);
expect(report.phases[0]?.details?.pages_processed).toBe(1);
expect(report.phases[0]?.details?.stale_pages_drained).toBe(2);
expect(report.totals.pages_extracted).toBe(3);
});
});
test('working and no-op drains leave stdout available for the cycle JSON report', async () => {
writeFileSync(join(brainDir, 'quiet.md'), '# Quiet page\nNo outgoing links.\n');
// A checkout page whose [[link]] resolves to another checkout page (both
// present in the DB under the same source) so the fs-walk link-extraction
// branch actually runs — and writes an edge — under the stdout spy.
writeFileSync(join(brainDir, 'linker.md'), '# Linker\nSee [[quiet]].\n');
await engine.executeRaw(
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline)
VALUES ('quiet', 'wiki', 'concept', 'Quiet page', 'No outgoing links.', ''),
('linker', 'wiki', 'concept', 'Linker', 'See [[quiet]].', '')`,
);
const log = spyOn(console, 'log').mockImplementation(() => {});
const stdout = spyOn(process.stdout, 'write').mockReturnValue(true);
try {
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
const first = await runCycle(engine, { brainDir, sourceId: 'wiki', phases: ['extract'] });
const second = await runCycle(engine, { brainDir, sourceId: 'wiki', phases: ['extract'] });
expect(first.phases[0]?.details?.pages_processed).toBe(2);
expect(first.phases[0]?.details?.linksCreated).toBe(1);
expect(first.phases[0]?.details?.stale_pages_drained).toBe(2);
expect(second.phases[0]?.details?.stale_pages_drained).toBe(0);
});
expect(log).not.toHaveBeenCalled();
expect(stdout).not.toHaveBeenCalled();
const links = await engine.getLinks('linker', { sourceId: 'wiki' });
expect(links.some(l => l.to_slug === 'quiet')).toBe(true);
} finally {
log.mockRestore();
stdout.mockRestore();
}
});
for (const dryRun of [false, true]) {
for (const jsonMode of [false, true]) {
test(`quiet stale helper suppresses output (dryRun=${dryRun}, jsonMode=${jsonMode})`, async () => {
const log = spyOn(console, 'log').mockImplementation(() => {});
const stdout = spyOn(process.stdout, 'write').mockReturnValue(true);
try {
const result = await extractStaleFromDB(engine, {
dryRun, jsonMode, quiet: true, includeFrontmatter: false,
sourceIdFilter: 'wiki', catchUp: false,
});
expect(result.pagesProcessed).toBe(dryRun ? 0 : 2);
expect(log).not.toHaveBeenCalled();
expect(stdout).not.toHaveBeenCalled();
} finally {
log.mockRestore();
stdout.mockRestore();
}
});
}
}
test('standalone stale helper still emits its JSON result', async () => {
const stdout = spyOn(process.stdout, 'write').mockReturnValue(true);
try {
await extractStaleFromDB(engine, {
dryRun: false, jsonMode: true, includeFrontmatter: false,
sourceIdFilter: 'wiki', catchUp: false,
});
expect(stdout).toHaveBeenCalledTimes(1);
const result = JSON.parse(String(stdout.mock.calls[0]?.[0]));
expect(result.action).toBe('extract_stale_done');
expect(result.pages_processed).toBe(2);
} finally {
stdout.mockRestore();
}
});
test('standalone stale helper prints its human summary when neither quiet nor JSON', async () => {
// The quiet/JSON variants above pin silence; this pins that the default
// CLI path (`gbrain extract --stale`) still tells the operator what ran.
const log = spyOn(console, 'log').mockImplementation(() => {});
const stdout = spyOn(process.stdout, 'write').mockReturnValue(true);
try {
const result = await extractStaleFromDB(engine, {
dryRun: false, jsonMode: false, includeFrontmatter: false,
sourceIdFilter: 'wiki', catchUp: false,
});
expect(result.pagesProcessed).toBe(2);
const summary = log.mock.calls.map(c => String(c[0])).find(l => l.startsWith('Extract --stale:'));
expect(summary).toBeDefined();
expect(summary).toContain('from 2 page(s)');
// Human output goes through console.log; nothing is written raw to stdout.
expect(stdout).not.toHaveBeenCalled();
} finally {
log.mockRestore();
stdout.mockRestore();
}
});
test('DB-only extraction contributes to totals and a working status', async () => {
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
const report = await runCycle(engine, { brainDir, sourceId: 'wiki', phases: ['extract'] });
expect(report.phases[0]?.details?.pages_processed).toBe(0);
expect(report.phases[0]?.details?.stale_pages_drained).toBe(2);
expect(report.totals.pages_extracted).toBe(2);
expect(report.status).toBe('ok');
});
});
test('drains stale DB-only pages after the targeted pass and reports staleRemaining', async () => {
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
const report = await runCycle(engine, {
brainDir,
sourceId: 'wiki',
phases: ['extract'],
});
const extractPhase = report.phases.find(p => p.phase === 'extract');
expect(extractPhase?.status).toBe('ok');
// Both stale pages processed by the drain; none left stale.
expect(Number(extractPhase?.details?.stale_pages_drained ?? -1)).toBe(2);
expect(Number(extractPhase?.details?.staleRemaining ?? -1)).toBe(0);
expect(Number(extractPhase?.details?.stale_links_created ?? 0)).toBeGreaterThanOrEqual(1);
});
// The stale drain persisted alice → bob.
const links = await engine.executeRaw<{ from_slug: string; to_slug: string }>(
`SELECT pf.slug AS from_slug, pt.slug AS to_slug
FROM links l
JOIN pages pf ON pf.id = l.from_page_id
JOIN pages pt ON pt.id = l.to_page_id`,
);
expect(links.some(l => l.from_slug === 'people/alice' && l.to_slug === 'people/bob')).toBe(true);
// Stamped fresh: a second cycle's drain is a no-op.
const stale = await engine.executeRaw<{ n: string }>(
`SELECT COUNT(*)::text AS n FROM pages WHERE links_extracted_at IS NULL AND deleted_at IS NULL`,
);
expect(Number(stale[0]?.n ?? -1)).toBe(0);
});
test('second cycle is a no-op drain (nothing stale, staleRemaining 0)', async () => {
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
await runCycle(engine, { brainDir, sourceId: 'wiki', phases: ['extract'] });
const report = await runCycle(engine, { brainDir, sourceId: 'wiki', phases: ['extract'] });
const extractPhase = report.phases.find(p => p.phase === 'extract');
expect(extractPhase?.status).toBe('ok');
expect(Number(extractPhase?.details?.stale_pages_drained ?? -1)).toBe(0);
expect(Number(extractPhase?.details?.staleRemaining ?? -1)).toBe(0);
});
});
test('the cycle-shaped slug-scoped extract keeps batch-loss diagnostics on stderr', async () => {
// The cycle calls runExtractCore with jsonMode: true so the helper's human
// summary never lands ahead of the dream --json report. Quiet must not mean
// silent: a flush that drops rows still has to surface on stderr.
// Links are replaced per page (the put_page contract), so the dropped write
// is that page's replacement; both endpoints are live pages.
// A lost write also fails the run with a typed error and leaves ONLY the
// failed page due: bob's write succeeded and is stamped, alice is not, and
// a clean retry converges.
writeFileSync(join(brainDir, 'alice.md'), '# Alice\nMet [[bob]] today.\n');
writeFileSync(join(brainDir, 'bob.md'), '# Bob\nQuiet page.\n');
await engine.executeRaw(
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline)
VALUES ('alice', 'wiki', 'person', 'Alice', 'Met [[bob]] today.', ''),
('bob', 'wiki', 'person', 'Bob', 'Quiet page.', '')`,
);
const original = engine.replaceDerivedLinks.bind(engine);
const addLinks = spyOn(engine, 'replaceDerivedLinks').mockImplementation(async (origin, links, opts) => {
if (origin.slug === 'alice') throw new Error('pool exhausted');
return original(origin, links, opts);
});
const log = spyOn(console, 'log').mockImplementation(() => {});
const stdout = spyOn(process.stdout, 'write').mockReturnValue(true);
const err = spyOn(console, 'error').mockImplementation(() => {});
const stderr = spyOn(process.stderr, 'write').mockReturnValue(true);
try {
await expect(runExtractCore(engine, { mode: 'all', dir: brainDir, slugs: ['alice', 'bob'], jsonMode: true, sourceId: 'wiki' }))
.rejects.toMatchObject({ code: 'extract_write_lost' });
expect(addLinks.mock.calls.some(c => c[0].slug === 'alice')).toBe(true);
expect(log).not.toHaveBeenCalled();
expect(stdout).not.toHaveBeenCalled();
const diagnostics = [...err.mock.calls, ...stderr.mock.calls].map(c => String(c[0]));
expect(diagnostics.some(d => d.includes('batch_error') && d.includes('pool exhausted'))).toBe(true);
} finally {
addLinks.mockRestore();
log.mockRestore();
stdout.mockRestore();
err.mockRestore();
stderr.mockRestore();
}
expect(await stamped('alice')).toBe(false);
expect(await stamped('bob')).toBe(true);
await runExtractCore(engine, { mode: 'all', dir: brainDir, slugs: ['alice', 'bob'], quiet: true, sourceId: 'wiki' });
expect(await stamped('alice')).toBe(true);
});
test('the cycle-shaped extract keeps TIMELINE batch-loss diagnostics on stderr too (#4890 sibling branch)', async () => {
// Same two-channel contract as the links flush: under jsonMode a dropped
// timeline batch must land on stderr as a batch_error event, never on stdout.
// The lost batch also fails the run and leaves carol due; a retry writes
// the entry once and stamps her.
writeFileSync(join(brainDir, 'carol.md'), '# Carol\n\n## Timeline\n- **2026-01-02** | Signed the term sheet\n');
await engine.executeRaw(
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline)
VALUES ('carol', 'wiki', 'person', 'Carol', '', '')`,
);
const addTimeline = spyOn(engine, 'addTimelineEntriesBatch').mockRejectedValueOnce(new Error('pool exhausted'));
const log = spyOn(console, 'log').mockImplementation(() => {});
const stdout = spyOn(process.stdout, 'write').mockReturnValue(true);
const err = spyOn(console, 'error').mockImplementation(() => {});
const stderr = spyOn(process.stderr, 'write').mockReturnValue(true);
try {
await expect(runExtractCore(engine, { mode: 'all', dir: brainDir, slugs: ['carol'], jsonMode: true, sourceId: 'wiki' }))
.rejects.toMatchObject({ code: 'extract_write_lost' });
expect(addTimeline).toHaveBeenCalledTimes(1);
expect(log).not.toHaveBeenCalled();
expect(stdout).not.toHaveBeenCalled();
const diagnostics = [...err.mock.calls, ...stderr.mock.calls].map(c => String(c[0]));
expect(diagnostics.some(d => d.includes('"event":"batch_error"') && d.includes('pool exhausted'))).toBe(true);
} finally {
addTimeline.mockRestore();
log.mockRestore();
stdout.mockRestore();
err.mockRestore();
stderr.mockRestore();
}
expect(await stamped('carol')).toBe(false);
await runExtractCore(engine, { mode: 'all', dir: brainDir, slugs: ['carol'], quiet: true, sourceId: 'wiki' });
expect(await stamped('carol')).toBe(true);
const [row] = await engine.executeRaw<{ n: number | string }>(
`SELECT count(*) AS n FROM timeline_entries t JOIN pages p ON p.id = t.page_id WHERE p.source_id = 'wiki' AND p.slug = 'carol'`);
expect(Number(row!.n)).toBe(1);
});
test('a lost walk write makes the extract phase warn and the same cycle stale drain retries the unstamped page (#6954, GBRA-130 A-1)', async () => {
// The walk loses carol's timeline batch. Pre-fix the walk still stamped her,
// so the drain never saw her and the entry was lost with an 'ok' phase.
writeFileSync(join(brainDir, 'carol.md'), '# Carol\n\n## Timeline\n- **2026-01-02** | Signed the term sheet\n');
await engine.executeRaw(
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline)
VALUES ('carol', 'wiki', 'person', 'Carol', '', '- **2026-01-02** | Signed the term sheet')`,
);
const addTimeline = spyOn(engine, 'addTimelineEntriesBatch').mockRejectedValueOnce(new Error('pool exhausted'));
const err = spyOn(console, 'error').mockImplementation(() => {});
const stderr = spyOn(process.stderr, 'write').mockReturnValue(true);
try {
await withEnv({ GBRAIN_HOME: gbrainHome }, async () => {
const report = await runCycle(engine, { brainDir, sourceId: 'wiki', phases: ['extract'] });
const extractPhase = report.phases.find(p => p.phase === 'extract');
expect(extractPhase?.status).toBe('warn');
expect(extractPhase?.summary).toContain('remain due');
expect(extractPhase?.details?.stale_pages_drained).toBeGreaterThan(0);
});
} finally {
addTimeline.mockRestore();
err.mockRestore();
stderr.mockRestore();
}
expect(await stamped('carol')).toBe(true);
const [row] = await engine.executeRaw<{ n: number | string }>(
`SELECT count(*) AS n FROM timeline_entries t JOIN pages p ON p.id = t.page_id WHERE p.source_id = 'wiki' AND p.slug = 'carol'`);
expect(Number(row!.n)).toBeGreaterThan(0);
});
});