Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(facts): a failed legacy fence step no longer books a healthy round (#8568)
garrytanc9489f07bc52
diff --git a/src/core/cycle/extract-facts.ts b/src/core/cycle/extract-facts.tsindex c52f0ee742..a3f27cfb55 100644--- a/src/core/cycle/extract-facts.ts+++ b/src/core/cycle/extract-facts.ts@@ -81,7 +81,7 @@ import {type SupersessionChain,} from '../facts/supersede-resolve.ts';import { writeReceipt } from '../extract/receipt-writer.ts';-import { fenceUnfencedFacts, planUnfencedFacts } from '../facts/unfenced-facts.ts';+import { fenceUnfencedFacts, isFenceHold, planUnfencedFacts } from '../facts/unfenced-facts.ts';import { classifyRunStop, upsertExtractRollup } from '../extract/rollup-writer.ts';import { factsFenceView } from '../facts-fence.ts';import { fenceReadOf, publishFenceGeneration, readFenceBlocks, FACT_FENCE_KIND, type FenceRead } from '../generations/fence.ts';@@ -728,7 +728,7 @@ export interface ExtractFactsResult {/** Active fence-owned rows expired because their page was soft-deleted. */factsExpiredForDeletedPages: number;guardTriggered: boolean;- /** Pages whose reconcile threw and rolled back; later pages still ran. */+ /** Pages whose fence write or reconcile threw and rolled back; later pages still ran. */pagesFailed: number;warnings: string[];/** #5575 B3: new fence rows the write gate flagged, held or rejected; present only when it did any. */@@ -830,6 +830,7 @@ export async function runExtractFacts(// fenced pages join this run's reconcile below.let fencedSlugs: string[] = [];let unboundRefused = new Set<string>();+ let fenceStepFailed = false;if (!opts.dryRun) {try {const fence = await fenceUnfencedFacts(engine, await planUnfencedFacts(engine, { sourceId }), { pageLockRoot: opts.pageLockRoot });@@ -837,7 +838,9 @@ export async function runExtractFacts(fencedSlugs = fence.fenced_slugs;unboundRefused = new Set(fence.unbound_refused.map(page => page.slug));for (const page of [...fence.failed_pages, ...fence.missing_files]) result.warnings.push(`FACTS_FENCE_FAILED: ${page}`);+ result.pagesFailed += fence.write_errors;} catch (e) {+ fenceStepFailed = !isFenceHold(e);result.warnings.push(`FACTS_FENCE_FAILED: ${(e instanceof Error ? e.message : String(e)).slice(0, 300)}`);}}@@ -1149,7 +1152,7 @@ export async function runExtractFacts(kind: 'facts.fence',source_id: sourceId,cost_delta: 0,- ...classifyRunStop({ error: result.pagesFailed > 0 ? 'item_error' : false }),+ ...classifyRunStop({ error: fenceStepFailed || result.pagesFailed > 0 ? 'item_error' : false }),});}diff --git a/src/core/facts/unfenced-facts.ts b/src/core/facts/unfenced-facts.tsindex 231aae29d8..87be8cb903 100644--- a/src/core/facts/unfenced-facts.ts+++ b/src/core/facts/unfenced-facts.ts@@ -62,6 +62,7 @@ import { FENCE_SOURCE_DEFAULT } from './extract-from-fence.ts';import { parseMarkdown, serializePageToMarkdown } from '../markdown.ts';import { assertSourceFilesystemActive, hasSourceFilesystemLock, withSourceFilesystemLock } from '../minions/source-filesystem.ts';import { withPageLock } from '../page-lock.ts';+import { LockUnavailableError } from '../db-lock.ts';import { maintenanceTransaction } from '../persistence/attribution.ts';import { assertLifetimeIdHeadroom } from '../persistence/journal.ts';import { OperationError, type OperationContext } from '../ops/contract.ts';@@ -115,6 +116,8 @@ export interface UnfencedFactsOutcome {pages_touched: number;/** `<slug> (<reason>)` per page whose fence write failed; its rows stay unfenced. */failed_pages: string[];+ /** #8568: pages in `failed_pages` whose fence write threw (I/O, lock, database), as opposed to holds the next run or another host settles. */+ write_errors: number;/** Slugs whose fence this run wrote. */fenced_slugs: string[];/** Unmanaged: `<slug> (<reason>)` per live page whose recorded canonical file is not on this host (counted in skipped_no_page). */@@ -304,7 +307,7 @@ export async function planUnfencedFacts(engine: BrainEngine, opts: { sourceId?:);const outcome: UnfencedFactsOutcome = {scanned: rows.length, fenced: 0, skipped_no_entity: 0, skipped_no_local_path: 0, skipped_no_page: 0,- skipped_archived: 0, skipped_url_clone: 0, pages_touched: 0, failed_pages: [], fenced_slugs: [], missing_files: [], unbound_refused: [], unadoptable_rows: [],+ skipped_archived: 0, skipped_url_clone: 0, pages_touched: 0, failed_pages: [], write_errors: 0, fenced_slugs: [], missing_files: [], unbound_refused: [], unadoptable_rows: [],};const groups = new Map<string, UnfencedFactRow[]>();for (const row of rows) {@@ -444,6 +447,14 @@ function factsChangedSincePlanning(body: string, assignments: Array<{ id: string});}+/**+ * #8568: a typed refusal (a revision conflict, a stored fence defect, an owner or host check) or a busy source lock is a+ * hold the next run or the owning host settles; any other throw (I/O, database) is a fence write failure.+ */+export function isFenceHold(err: unknown): boolean {+ return err instanceof OperationError || err instanceof LockUnavailableError;+}+/** Keep the typed records and name the page once in `failed_pages` with the stable token; the rows stay as they are. */function recordUnadoptable(outcome: UnfencedFactsOutcome, slug: string, records: UnadoptableFactRecord[]): void {if (records.length === 0) return;@@ -530,6 +541,7 @@ async function fenceManaged(engine: BrainEngine, plan: UnfencedFactsPlan): Promioutcome.fenced_slugs.push(entitySlug);} catch (err) {outcome.failed_pages.push(`${entitySlug} (${err instanceof Error ? err.message : String(err)})`);+ if (!isFenceHold(err)) outcome.write_errors += 1;}}}@@ -561,6 +573,7 @@ async function fenceDatabaseOnlyPage(engine: BrainEngine, sourceId: string, entioutcome.fenced_slugs.push(entitySlug);} catch (err) {outcome.failed_pages.push(`${entitySlug} (${err instanceof Error ? err.message : String(err)})`);+ if (!isFenceHold(err)) outcome.write_errors += 1;}}@@ -644,6 +657,7 @@ async function fenceFilePage(engine: BrainEngine, sourceId: string, entitySlug:await (hasSourceFilesystemLock(writeRoot) ? write() : withSourceFilesystemLock(engine, writeRoot, write));} catch (err) {outcome.failed_pages.push(`${entitySlug} (${err instanceof Error ? err.message : String(err)})`);+ if (!isFenceHold(err)) outcome.write_errors += 1;}}diff --git a/test/extract-facts-fence-failure-health-8758.test.ts b/test/extract-facts-fence-failure-health-8758.test.tsnew file mode 100644index 0000000000..0d3739156d--- /dev/null+++ b/test/extract-facts-fence-failure-health-8758.test.ts@@ -0,0 +1,142 @@+/**+ * #8568 / #8758 — a legacy fact-fence step that fails must not book a healthy+ * extract_facts round. A fence write that errors (real I/O: the page's .tmp+ * path is a directory) counts the page in pagesFailed, and a fence step that+ * throws as a whole halts the round with item_error, so doctor extract_health+ * warns. Holds by design (a page whose file lives on another host) still book+ * a completed round. Real PGLite (and Postgres through testBackends).+ */+import { expect, test } from 'bun:test';+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join } from 'node:path';+import { runCycle } from '../src/core/cycle.ts';+import type { BrainEngine } from '../src/core/engine.ts';+import { serializePageToMarkdown } from '../src/core/markdown.ts';+import { computeExtractHealthCheck } from '../src/commands/doctor/checks/extraction-sync.ts';+import { isolatedSharedSkillsEngine } from './helpers/shared-skills-engine.ts';+import { managedBrain } from './helpers/managed-brain.ts';+import { opError } from '../src/core/ops/contract.ts';+import { LEGACY_FILE_SLUG, seedLegacyManagedContent } from './helpers/managed-legacy-fixture.ts';+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';+import { withEnv } from './helpers/with-env.ts';++const SLUG = 'people/alice-example';++async function unmanagedBrain(databaseUrl: string | undefined, run: (brain: { engine: BrainEngine; root: string; path: string }) => Promise<void>): Promise<void> {+ const home = mkdtempSync(join(tmpdir(), 'gbrain-fence-health-'));+ try {+ await withEnv({ GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, async () => {+ const { engine, close } = await isolatedSharedSkillsEngine(databaseUrl);+ try {+ const root = join(home, 'brain');+ mkdirSync(root, { recursive: true });+ await engine.executeRaw("UPDATE sources SET local_path = $1 WHERE id = 'default'", [root]);+ const page = await engine.putPage(SLUG, { type: 'person', title: 'Alice Example', compiled_truth: '# Alice Example\n\nProse.' });+ await engine.executeRaw('UPDATE pages SET source_path = $1 WHERE id = $2', [`${SLUG}.md`, page.id]);+ const snapshot = (await engine.readPageSnapshot(SLUG, { sourceId: 'default' }))!;+ const path = join(root, `${SLUG}.md`);+ mkdirSync(dirname(path), { recursive: true });+ writeFileSync(path, serializePageToMarkdown(snapshot.page, snapshot.tags));+ await engine.executeRaw(+ `INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability, valid_from, source, confidence)+ VALUES ('default', $1, 'Alice example moved to Lisbon', 'fact', 'private', 'medium', '2026-02-05', 'mcp:remember', 0.9)`, [SLUG]);+ await run({ engine, root, path });+ } finally { await close(); }+ });+ } finally { rmSync(home, { recursive: true, force: true }); }+}++async function extractFactsPhase(engine: BrainEngine, root: string) {+ const report = await runCycle(engine, { brainDir: root, sourceId: 'default', phases: ['extract_facts'] });+ return report.phases.find(p => p.phase === 'extract_facts')!;+}++async function fenceRollup(engine: BrainEngine) {+ const [row] = await engine.executeRaw<{ halts: number; rounds: number }>(+ `SELECT COALESCE(SUM(halt_count), 0)::int AS halts, COALESCE(SUM(round_completed_count), 0)::int AS rounds+ FROM extract_rollup_7d WHERE kind = 'facts.fence' AND source_id = 'default'`);+ return { halts: Number(row!.halts), rounds: Number(row!.rounds), doctor: (await computeExtractHealthCheck(engine)).status };+}++/** The engine with one method answering `fail` when `when` matches its first argument. */+function failing(engine: BrainEngine, method: 'executeRaw' | 'readPageSnapshot', when: (first: unknown) => boolean, message: string, typed = false): BrainEngine {+ return new Proxy(engine, {+ get(target, prop, receiver) {+ const value = Reflect.get(target, prop, receiver);+ if (typeof value !== 'function') return value;+ if (prop !== method) return value.bind(target);+ const error = () => typed ? opError('revision_conflict', message, 'Run maintenance again.') : new Error(message);+ return (first: unknown, ...rest: unknown[]) => when(first) ? Promise.reject(error()) : value.call(target, first, ...rest);+ },+ }) as BrainEngine;+}++/** The engine with the fence planner's source read failing, so the whole fence step throws. */+const failingFencePlanner = (engine: BrainEngine) =>+ failing(engine, 'executeRaw', sql => sql === 'SELECT id, local_path, archived, config FROM sources', 'synthetic fence planner failure');++for (const backend of testBackends()) {+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;++ test(`${backend}: a fence write that fails with an I/O error counts as a failed page and halts the round, then recovers`, async () => {+ await unmanagedBrain(databaseUrl, async ({ engine, root, path }) => {+ mkdirSync(`${path}.tmp`);+ const first = await extractFactsPhase(engine, root);+ expect((first.details as { warnings: string[] }).warnings.join('\n')).toContain('EISDIR');+ expect(first.details).toMatchObject({ pagesFailed: 1, unfencedRowsFenced: 0 });+ expect(await fenceRollup(engine)).toEqual({ halts: 1, rounds: 0, doctor: 'warn' });++ rmSync(`${path}.tmp`, { recursive: true });+ const second = await extractFactsPhase(engine, root);+ expect(second.details).toMatchObject({ pagesFailed: 0, unfencedRowsFenced: 1 });+ expect(await fenceRollup(engine)).toMatchObject({ halts: 1, rounds: 1 });+ });+ }, 120_000);++ test(`${backend}: a fence step that throws halts the round instead of booking it healthy`, async () => {+ await unmanagedBrain(databaseUrl, async ({ engine, root }) => {+ const result = await extractFactsPhase(failingFencePlanner(engine), root);+ expect((result.details as { warnings: string[] }).warnings.join('\n')).toContain('FACTS_FENCE_FAILED: synthetic fence planner failure');+ expect(await fenceRollup(engine)).toEqual({ halts: 1, rounds: 0, doctor: 'warn' });+ });+ }, 120_000);++ test(`${backend}: managed: one page whose fence adoption throws counts once; the other page still fences`, async () => {+ await managedBrain(async ({ engine, root }) => {+ const broken = failing(engine, 'readPageSnapshot', slug => slug === LEGACY_FILE_SLUG, 'synthetic snapshot failure');+ const result = await extractFactsPhase(broken, root);+ expect((result.details as { warnings: string[] }).warnings.join('\n')).toContain(`FACTS_FENCE_FAILED: ${LEGACY_FILE_SLUG} (synthetic snapshot failure)`);+ expect(result.details).toMatchObject({ pagesFailed: 1, unfencedRowsFenced: 1 });+ expect(await fenceRollup(engine)).toMatchObject({ halts: 1, rounds: 0 });+ }, { databaseUrl, setup: async ({ engine, root }) => { await seedLegacyManagedContent(engine, root); } });+ }, 120_000);++ test(`${backend}: managed: a typed refusal (a concurrent writer moved the page) is a hold, not a failure`, async () => {+ await managedBrain(async ({ engine, root }) => {+ const moved = failing(engine, 'readPageSnapshot', slug => slug === LEGACY_FILE_SLUG, 'synthetic revision conflict', true);+ const result = await extractFactsPhase(moved, root);+ expect((result.details as { warnings: string[] }).warnings.join('\n')).toContain('synthetic revision conflict');+ expect(result.details).toMatchObject({ pagesFailed: 0 });+ expect(await fenceRollup(engine)).toMatchObject({ halts: 0, rounds: 1 });+ }, { databaseUrl, setup: async ({ engine, root }) => { await seedLegacyManagedContent(engine, root); } });+ }, 120_000);++ test(`${backend}: a fence step refused with a typed error (owner checks) is a hold, not a halt`, async () => {+ await unmanagedBrain(databaseUrl, async ({ engine, root }) => {+ const refused = failing(engine, 'executeRaw', sql => sql === 'SELECT id, local_path, archived, config FROM sources', 'synthetic owner refusal', true);+ const result = await extractFactsPhase(refused, root);+ expect((result.details as { warnings: string[] }).warnings.join('\n')).toContain('FACTS_FENCE_FAILED: synthetic owner refusal');+ expect(await fenceRollup(engine)).toMatchObject({ halts: 0, rounds: 1 });+ });+ }, 120_000);++ test(`${backend}: a page whose file lives on another host is a hold, not a failure`, async () => {+ await unmanagedBrain(databaseUrl, async ({ engine, root, path }) => {+ rmSync(path);+ const result = await extractFactsPhase(engine, root);+ expect(result.details).toMatchObject({ pagesFailed: 0 });+ expect(await fenceRollup(engine)).toMatchObject({ halts: 0, rounds: 1 });+ });+ }, 120_000);+}diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txtindex 72ad5657dd..a9cee05bd1 100644--- a/test/postgres-unit-arms.txt+++ b/test/postgres-unit-arms.txt@@ -84,6 +84,7 @@ test/extract-atoms-input-truncated-6503.test.tstest/extract-atoms-raised-cap-drain-7391.test.tstest/extract-endpoint-revision.serial.test.tstest/extract-entities-managed.test.ts+test/extract-facts-fence-failure-health-8758.test.tstest/extract-timeline-from-meetings-managed.test.tstest/extraction-watermark-atoms.test.tstest/fact-embedding-oversized-7211.test.ts