Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(facts): a failed legacy fence step no longer books a healthy round (#8568)

garrytanc9489f07bc52
diff --git a/src/core/cycle/extract-facts.ts b/src/core/cycle/extract-facts.ts
index c52f0ee742..a3f27cfb55 100644
--- a/src/core/cycle/extract-facts.ts
+++ b/src/core/cycle/extract-facts.ts
@@ -81,7 +81,7 @@ import {
type SupersessionChain,
} from '../facts/supersede-resolve.ts';
import { writeReceipt } from '../extract/receipt-writer.ts';
-import { fenceUnfencedFacts, planUnfencedFacts } from '../facts/unfenced-facts.ts';
+import { fenceUnfencedFacts, isFenceHold, planUnfencedFacts } from '../facts/unfenced-facts.ts';
import { classifyRunStop, upsertExtractRollup } from '../extract/rollup-writer.ts';
import { factsFenceView } from '../facts-fence.ts';
import { fenceReadOf, publishFenceGeneration, readFenceBlocks, FACT_FENCE_KIND, type FenceRead } from '../generations/fence.ts';
@@ -728,7 +728,7 @@ export interface ExtractFactsResult {
/** Active fence-owned rows expired because their page was soft-deleted. */
factsExpiredForDeletedPages: number;
guardTriggered: boolean;
- /** Pages whose reconcile threw and rolled back; later pages still ran. */
+ /** Pages whose fence write or reconcile threw and rolled back; later pages still ran. */
pagesFailed: number;
warnings: string[];
/** #5575 B3: new fence rows the write gate flagged, held or rejected; present only when it did any. */
@@ -830,6 +830,7 @@ export async function runExtractFacts(
// fenced pages join this run's reconcile below.
let fencedSlugs: string[] = [];
let unboundRefused = new Set<string>();
+ let fenceStepFailed = false;
if (!opts.dryRun) {
try {
const fence = await fenceUnfencedFacts(engine, await planUnfencedFacts(engine, { sourceId }), { pageLockRoot: opts.pageLockRoot });
@@ -837,7 +838,9 @@ export async function runExtractFacts(
fencedSlugs = fence.fenced_slugs;
unboundRefused = new Set(fence.unbound_refused.map(page => page.slug));
for (const page of [...fence.failed_pages, ...fence.missing_files]) result.warnings.push(`FACTS_FENCE_FAILED: ${page}`);
+ result.pagesFailed += fence.write_errors;
} catch (e) {
+ fenceStepFailed = !isFenceHold(e);
result.warnings.push(`FACTS_FENCE_FAILED: ${(e instanceof Error ? e.message : String(e)).slice(0, 300)}`);
}
}
@@ -1149,7 +1152,7 @@ export async function runExtractFacts(
kind: 'facts.fence',
source_id: sourceId,
cost_delta: 0,
- ...classifyRunStop({ error: result.pagesFailed > 0 ? 'item_error' : false }),
+ ...classifyRunStop({ error: fenceStepFailed || result.pagesFailed > 0 ? 'item_error' : false }),
});
}
diff --git a/src/core/facts/unfenced-facts.ts b/src/core/facts/unfenced-facts.ts
index 231aae29d8..87be8cb903 100644
--- a/src/core/facts/unfenced-facts.ts
+++ b/src/core/facts/unfenced-facts.ts
@@ -62,6 +62,7 @@ import { FENCE_SOURCE_DEFAULT } from './extract-from-fence.ts';
import { parseMarkdown, serializePageToMarkdown } from '../markdown.ts';
import { assertSourceFilesystemActive, hasSourceFilesystemLock, withSourceFilesystemLock } from '../minions/source-filesystem.ts';
import { withPageLock } from '../page-lock.ts';
+import { LockUnavailableError } from '../db-lock.ts';
import { maintenanceTransaction } from '../persistence/attribution.ts';
import { assertLifetimeIdHeadroom } from '../persistence/journal.ts';
import { OperationError, type OperationContext } from '../ops/contract.ts';
@@ -115,6 +116,8 @@ export interface UnfencedFactsOutcome {
pages_touched: number;
/** `<slug> (<reason>)` per page whose fence write failed; its rows stay unfenced. */
failed_pages: string[];
+ /** #8568: pages in `failed_pages` whose fence write threw (I/O, lock, database), as opposed to holds the next run or another host settles. */
+ write_errors: number;
/** Slugs whose fence this run wrote. */
fenced_slugs: string[];
/** Unmanaged: `<slug> (<reason>)` per live page whose recorded canonical file is not on this host (counted in skipped_no_page). */
@@ -304,7 +307,7 @@ export async function planUnfencedFacts(engine: BrainEngine, opts: { sourceId?:
);
const outcome: UnfencedFactsOutcome = {
scanned: rows.length, fenced: 0, skipped_no_entity: 0, skipped_no_local_path: 0, skipped_no_page: 0,
- skipped_archived: 0, skipped_url_clone: 0, pages_touched: 0, failed_pages: [], fenced_slugs: [], missing_files: [], unbound_refused: [], unadoptable_rows: [],
+ skipped_archived: 0, skipped_url_clone: 0, pages_touched: 0, failed_pages: [], write_errors: 0, fenced_slugs: [], missing_files: [], unbound_refused: [], unadoptable_rows: [],
};
const groups = new Map<string, UnfencedFactRow[]>();
for (const row of rows) {
@@ -444,6 +447,14 @@ function factsChangedSincePlanning(body: string, assignments: Array<{ id: string
});
}
+/**
+ * #8568: a typed refusal (a revision conflict, a stored fence defect, an owner or host check) or a busy source lock is a
+ * hold the next run or the owning host settles; any other throw (I/O, database) is a fence write failure.
+ */
+export function isFenceHold(err: unknown): boolean {
+ return err instanceof OperationError || err instanceof LockUnavailableError;
+}
+
/** Keep the typed records and name the page once in `failed_pages` with the stable token; the rows stay as they are. */
function recordUnadoptable(outcome: UnfencedFactsOutcome, slug: string, records: UnadoptableFactRecord[]): void {
if (records.length === 0) return;
@@ -530,6 +541,7 @@ async function fenceManaged(engine: BrainEngine, plan: UnfencedFactsPlan): Promi
outcome.fenced_slugs.push(entitySlug);
} catch (err) {
outcome.failed_pages.push(`${entitySlug} (${err instanceof Error ? err.message : String(err)})`);
+ if (!isFenceHold(err)) outcome.write_errors += 1;
}
}
}
@@ -561,6 +573,7 @@ async function fenceDatabaseOnlyPage(engine: BrainEngine, sourceId: string, enti
outcome.fenced_slugs.push(entitySlug);
} catch (err) {
outcome.failed_pages.push(`${entitySlug} (${err instanceof Error ? err.message : String(err)})`);
+ if (!isFenceHold(err)) outcome.write_errors += 1;
}
}
@@ -644,6 +657,7 @@ async function fenceFilePage(engine: BrainEngine, sourceId: string, entitySlug:
await (hasSourceFilesystemLock(writeRoot) ? write() : withSourceFilesystemLock(engine, writeRoot, write));
} catch (err) {
outcome.failed_pages.push(`${entitySlug} (${err instanceof Error ? err.message : String(err)})`);
+ if (!isFenceHold(err)) outcome.write_errors += 1;
}
}
diff --git a/test/extract-facts-fence-failure-health-8758.test.ts b/test/extract-facts-fence-failure-health-8758.test.ts
new file mode 100644
index 0000000000..0d3739156d
--- /dev/null
+++ b/test/extract-facts-fence-failure-health-8758.test.ts
@@ -0,0 +1,142 @@
+/**
+ * #8568 / #8758 — a legacy fact-fence step that fails must not book a healthy
+ * extract_facts round. A fence write that errors (real I/O: the page's .tmp
+ * path is a directory) counts the page in pagesFailed, and a fence step that
+ * throws as a whole halts the round with item_error, so doctor extract_health
+ * warns. Holds by design (a page whose file lives on another host) still book
+ * a completed round. Real PGLite (and Postgres through testBackends).
+ */
+import { expect, test } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { runCycle } from '../src/core/cycle.ts';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { serializePageToMarkdown } from '../src/core/markdown.ts';
+import { computeExtractHealthCheck } from '../src/commands/doctor/checks/extraction-sync.ts';
+import { isolatedSharedSkillsEngine } from './helpers/shared-skills-engine.ts';
+import { managedBrain } from './helpers/managed-brain.ts';
+import { opError } from '../src/core/ops/contract.ts';
+import { LEGACY_FILE_SLUG, seedLegacyManagedContent } from './helpers/managed-legacy-fixture.ts';
+import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
+import { withEnv } from './helpers/with-env.ts';
+
+const SLUG = 'people/alice-example';
+
+async function unmanagedBrain(databaseUrl: string | undefined, run: (brain: { engine: BrainEngine; root: string; path: string }) => Promise<void>): Promise<void> {
+ const home = mkdtempSync(join(tmpdir(), 'gbrain-fence-health-'));
+ try {
+ await withEnv({ GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, async () => {
+ const { engine, close } = await isolatedSharedSkillsEngine(databaseUrl);
+ try {
+ const root = join(home, 'brain');
+ mkdirSync(root, { recursive: true });
+ await engine.executeRaw("UPDATE sources SET local_path = $1 WHERE id = 'default'", [root]);
+ const page = await engine.putPage(SLUG, { type: 'person', title: 'Alice Example', compiled_truth: '# Alice Example\n\nProse.' });
+ await engine.executeRaw('UPDATE pages SET source_path = $1 WHERE id = $2', [`${SLUG}.md`, page.id]);
+ const snapshot = (await engine.readPageSnapshot(SLUG, { sourceId: 'default' }))!;
+ const path = join(root, `${SLUG}.md`);
+ mkdirSync(dirname(path), { recursive: true });
+ writeFileSync(path, serializePageToMarkdown(snapshot.page, snapshot.tags));
+ await engine.executeRaw(
+ `INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability, valid_from, source, confidence)
+ VALUES ('default', $1, 'Alice example moved to Lisbon', 'fact', 'private', 'medium', '2026-02-05', 'mcp:remember', 0.9)`, [SLUG]);
+ await run({ engine, root, path });
+ } finally { await close(); }
+ });
+ } finally { rmSync(home, { recursive: true, force: true }); }
+}
+
+async function extractFactsPhase(engine: BrainEngine, root: string) {
+ const report = await runCycle(engine, { brainDir: root, sourceId: 'default', phases: ['extract_facts'] });
+ return report.phases.find(p => p.phase === 'extract_facts')!;
+}
+
+async function fenceRollup(engine: BrainEngine) {
+ const [row] = await engine.executeRaw<{ halts: number; rounds: number }>(
+ `SELECT COALESCE(SUM(halt_count), 0)::int AS halts, COALESCE(SUM(round_completed_count), 0)::int AS rounds
+ FROM extract_rollup_7d WHERE kind = 'facts.fence' AND source_id = 'default'`);
+ return { halts: Number(row!.halts), rounds: Number(row!.rounds), doctor: (await computeExtractHealthCheck(engine)).status };
+}
+
+/** The engine with one method answering `fail` when `when` matches its first argument. */
+function failing(engine: BrainEngine, method: 'executeRaw' | 'readPageSnapshot', when: (first: unknown) => boolean, message: string, typed = false): BrainEngine {
+ return new Proxy(engine, {
+ get(target, prop, receiver) {
+ const value = Reflect.get(target, prop, receiver);
+ if (typeof value !== 'function') return value;
+ if (prop !== method) return value.bind(target);
+ const error = () => typed ? opError('revision_conflict', message, 'Run maintenance again.') : new Error(message);
+ return (first: unknown, ...rest: unknown[]) => when(first) ? Promise.reject(error()) : value.call(target, first, ...rest);
+ },
+ }) as BrainEngine;
+}
+
+/** The engine with the fence planner's source read failing, so the whole fence step throws. */
+const failingFencePlanner = (engine: BrainEngine) =>
+ failing(engine, 'executeRaw', sql => sql === 'SELECT id, local_path, archived, config FROM sources', 'synthetic fence planner failure');
+
+for (const backend of testBackends()) {
+ const databaseUrl = backend === 'postgres' ? requirePostgresTestDatabase() : undefined;
+
+ test(`${backend}: a fence write that fails with an I/O error counts as a failed page and halts the round, then recovers`, async () => {
+ await unmanagedBrain(databaseUrl, async ({ engine, root, path }) => {
+ mkdirSync(`${path}.tmp`);
+ const first = await extractFactsPhase(engine, root);
+ expect((first.details as { warnings: string[] }).warnings.join('\n')).toContain('EISDIR');
+ expect(first.details).toMatchObject({ pagesFailed: 1, unfencedRowsFenced: 0 });
+ expect(await fenceRollup(engine)).toEqual({ halts: 1, rounds: 0, doctor: 'warn' });
+
+ rmSync(`${path}.tmp`, { recursive: true });
+ const second = await extractFactsPhase(engine, root);
+ expect(second.details).toMatchObject({ pagesFailed: 0, unfencedRowsFenced: 1 });
+ expect(await fenceRollup(engine)).toMatchObject({ halts: 1, rounds: 1 });
+ });
+ }, 120_000);
+
+ test(`${backend}: a fence step that throws halts the round instead of booking it healthy`, async () => {
+ await unmanagedBrain(databaseUrl, async ({ engine, root }) => {
+ const result = await extractFactsPhase(failingFencePlanner(engine), root);
+ expect((result.details as { warnings: string[] }).warnings.join('\n')).toContain('FACTS_FENCE_FAILED: synthetic fence planner failure');
+ expect(await fenceRollup(engine)).toEqual({ halts: 1, rounds: 0, doctor: 'warn' });
+ });
+ }, 120_000);
+
+ test(`${backend}: managed: one page whose fence adoption throws counts once; the other page still fences`, async () => {
+ await managedBrain(async ({ engine, root }) => {
+ const broken = failing(engine, 'readPageSnapshot', slug => slug === LEGACY_FILE_SLUG, 'synthetic snapshot failure');
+ const result = await extractFactsPhase(broken, root);
+ expect((result.details as { warnings: string[] }).warnings.join('\n')).toContain(`FACTS_FENCE_FAILED: ${LEGACY_FILE_SLUG} (synthetic snapshot failure)`);
+ expect(result.details).toMatchObject({ pagesFailed: 1, unfencedRowsFenced: 1 });
+ expect(await fenceRollup(engine)).toMatchObject({ halts: 1, rounds: 0 });
+ }, { databaseUrl, setup: async ({ engine, root }) => { await seedLegacyManagedContent(engine, root); } });
+ }, 120_000);
+
+ test(`${backend}: managed: a typed refusal (a concurrent writer moved the page) is a hold, not a failure`, async () => {
+ await managedBrain(async ({ engine, root }) => {
+ const moved = failing(engine, 'readPageSnapshot', slug => slug === LEGACY_FILE_SLUG, 'synthetic revision conflict', true);
+ const result = await extractFactsPhase(moved, root);
+ expect((result.details as { warnings: string[] }).warnings.join('\n')).toContain('synthetic revision conflict');
+ expect(result.details).toMatchObject({ pagesFailed: 0 });
+ expect(await fenceRollup(engine)).toMatchObject({ halts: 0, rounds: 1 });
+ }, { databaseUrl, setup: async ({ engine, root }) => { await seedLegacyManagedContent(engine, root); } });
+ }, 120_000);
+
+ test(`${backend}: a fence step refused with a typed error (owner checks) is a hold, not a halt`, async () => {
+ await unmanagedBrain(databaseUrl, async ({ engine, root }) => {
+ const refused = failing(engine, 'executeRaw', sql => sql === 'SELECT id, local_path, archived, config FROM sources', 'synthetic owner refusal', true);
+ const result = await extractFactsPhase(refused, root);
+ expect((result.details as { warnings: string[] }).warnings.join('\n')).toContain('FACTS_FENCE_FAILED: synthetic owner refusal');
+ expect(await fenceRollup(engine)).toMatchObject({ halts: 0, rounds: 1 });
+ });
+ }, 120_000);
+
+ test(`${backend}: a page whose file lives on another host is a hold, not a failure`, async () => {
+ await unmanagedBrain(databaseUrl, async ({ engine, root, path }) => {
+ rmSync(path);
+ const result = await extractFactsPhase(engine, root);
+ expect(result.details).toMatchObject({ pagesFailed: 0 });
+ expect(await fenceRollup(engine)).toMatchObject({ halts: 0, rounds: 1 });
+ });
+ }, 120_000);
+}
diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txt
index 72ad5657dd..a9cee05bd1 100644
--- a/test/postgres-unit-arms.txt
+++ b/test/postgres-unit-arms.txt
@@ -84,6 +84,7 @@ test/extract-atoms-input-truncated-6503.test.ts
test/extract-atoms-raised-cap-drain-7391.test.ts
test/extract-endpoint-revision.serial.test.ts
test/extract-entities-managed.test.ts
+test/extract-facts-fence-failure-health-8758.test.ts
test/extract-timeline-from-meetings-managed.test.ts
test/extraction-watermark-atoms.test.ts
test/fact-embedding-oversized-7211.test.ts