Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(doctor): atom_provenance_drift skips metadata- and fence-only edits of an unchanged body (#6825)
garrytan2e8b6e85826e
diff --git a/docs/guides/repair.md b/docs/guides/repair.mdindex fbfc157404..b8b78193e3 100644--- a/docs/guides/repair.md+++ b/docs/guides/repair.md@@ -1735,7 +1735,9 @@ with the preview and apply commands; a stale hash refuses with[`preview_changed`](#preview-changed). An apply under a different `--source`than the preview also refuses with `preview_changed`. Doctor`atom_provenance_drift` names this command when drift crosses its warn-threshold.+threshold. An atom whose page changed only in its frontmatter or in gbrain's+facts and takes sections is not drift when one of the page's recent versions+still holds the edition it was extracted from (#6825).### Atoms restorediff --git a/src/commands/doctor/checks/extraction-sync.ts b/src/commands/doctor/checks/extraction-sync.tsindex 4727fa042c..a9d7665c3e 100644--- a/src/commands/doctor/checks/extraction-sync.ts+++ b/src/commands/doctor/checks/extraction-sync.ts@@ -15,7 +15,8 @@ import { resolveHoursEnv } from '../../../core/env-number.ts';import { chunkerStamp } from '../../../core/chunkers/code.ts';import { smallBrainBacklogNote } from '../../../core/link-extraction-watermark.ts';import { extractionLagVerdict, extractionRoutes, heldExtractionReport, readExtractionLag } from '../../../core/extraction-lag.ts';-import { isMissingSchemaObject } from '../../../core/utils.ts';+import { contentHash, isMissingSchemaObject } from '../../../core/utils.ts';+import { stripFenceSections } from '../../../core/cycle/concept-publication.ts';import {loadStorageConfig,effectiveDbOnlyDirs,@@ -771,6 +772,68 @@ export async function computeExtractAtomsBacklogCheck(}}+/** The page-bound and slug-unbound atoms atom_provenance_drift reads (`ss` NULL: slug-unbound), as a CTE body. */+const ATOM_PROVENANCE_CTE = `atom AS (+ SELECT a.source_id,+ -- managed atoms keep their provisional prefix for good (#5770)+ CASE WHEN a.frontmatter->>'managed_extraction' = 'true'+ THEN regexp_replace(a.frontmatter->>'source_hash', '^pending:', '')+ ELSE a.frontmatter->>'source_hash' END AS sh,+ -- NULL = slug-unbound: transcript-minted (source_path only) or+ -- pre-binding-era. \`ss IS NULL\` is THE predicate for that+ -- population everywhere below (#4799 / #4806).+ NULLIF(a.frontmatter->>'source_slug', '') AS ss,+ CASE WHEN a.frontmatter->>'extracted_at' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}'+ THEN a.frontmatter->>'extracted_at' END AS ext+ FROM pages a+ WHERE a.type = 'atom'+ AND a.deleted_at IS NULL+ AND a.frontmatter->>'source_hash' IS NOT NULL+ -- in-flight marker written before an unmanaged extraction commits+ AND (a.frontmatter->>'source_hash' NOT LIKE 'pending:%' OR a.frontmatter->>'managed_extraction' = 'true')+ )`;++/** (page, source_hash) groups of drifted atoms unchangedBodyAtoms re-checks per run; the rest keep their drift verdict. */+const BODY_RECHECK_GROUPS = 200;+/** The newest versions of a page searched for an atom's extraction edition. */+const BODY_RECHECK_VERSIONS = 20;++/**+ * #6825: drifted atoms of a live inline page whose prose did not change. An atom's source_hash names the page edition+ * it was extracted from; when one of the page's recent versions hashes to it and its body without the gbrain-managed+ * facts and takes sections (stripFenceSections, the narrative concept synthesis compares) equals the page's, the move+ * was a metadata or fence write, not an edit of what the atom summarises. An edition no version still holds, and a+ * segmented page, keep their drift verdict.+ */+async function unchangedBodyAtoms(engine: BrainEngine): Promise<number> {+ const groups = await engine.executeRaw<{ page_id: number; sh: string; n: number }>(+ `WITH ${ATOM_PROVENANCE_CTE}, live_hashes AS MATERIALIZED (+ SELECT DISTINCT source_id, substring(content_hash from 1 for 16) AS sh FROM pages WHERE deleted_at IS NULL AND content_hash IS NOT NULL+ )+ SELECT p.id AS page_id, atom.sh, count(*)::int AS n+ FROM atom+ JOIN pages p ON p.source_id = atom.source_id AND p.slug = atom.ss AND p.deleted_at IS NULL AND p.body_storage = 'inline'+ LEFT JOIN live_hashes h ON h.source_id = atom.source_id AND h.sh = atom.sh+ WHERE h.sh IS NULL AND length(atom.sh) = 16+ GROUP BY p.id, atom.sh ORDER BY p.id, atom.sh LIMIT ${BODY_RECHECK_GROUPS}`, []);+ const json = <T>(v: unknown): T => (typeof v === 'string' ? JSON.parse(v) : v) as T;+ const narrative = (body: { compiled_truth: string; timeline: string | null }) => [stripFenceSections(body.compiled_truth), stripFenceSections(body.timeline ?? '')];+ let unchanged = 0;+ for (const group of groups) {+ const [page] = await engine.executeRaw<{ compiled_truth: string | null; timeline: string | null }>('SELECT compiled_truth, timeline FROM pages WHERE id = $1', [group.page_id]);+ if (page?.compiled_truth == null) continue;+ const versions = await engine.executeRaw<{ compiled_truth: string | null; timeline: string | null; title: string; type: string; frontmatter: unknown; tags: unknown }>(+ `SELECT compiled_truth, timeline, title, type, frontmatter, tags FROM page_versions+ WHERE page_id = $1 AND body_storage = 'inline' AND compiled_truth IS NOT NULL ORDER BY id DESC LIMIT ${BODY_RECHECK_VERSIONS}`, [group.page_id]);+ const edition = versions.find(v => contentHash({ title: v.title, type: v.type as never, compiled_truth: v.compiled_truth!, timeline: v.timeline ?? '',+ frontmatter: json<Record<string, unknown>>(v.frontmatter) ?? {}, tags: json<string[] | null>(v.tags) ?? [] }).startsWith(group.sh));+ if (!edition) continue;+ const [was, is] = [narrative({ compiled_truth: edition.compiled_truth!, timeline: edition.timeline }), narrative({ compiled_truth: page.compiled_truth, timeline: page.timeline })];+ if (was[0] === is[0] && was[1] === is[1]) unchanged += Number(group.n);+ }+ return unchanged;+}+/*** atom_provenance_drift doctor check (#4566).*@@ -833,31 +896,14 @@ export async function computeAtomProvenanceDriftCheck(// garbage from the min(); the age math happens in TS where Date// parsing can never throw (semantically-invalid dates become NaN →// metric omitted, verdict untouched).- `WITH atom AS (- SELECT a.source_id,- -- managed atoms keep their provisional prefix for good (#5770)- CASE WHEN a.frontmatter->>'managed_extraction' = 'true'- THEN regexp_replace(a.frontmatter->>'source_hash', '^pending:', '')- ELSE a.frontmatter->>'source_hash' END AS sh,- -- NULL = slug-unbound: transcript-minted (source_path only) or- -- pre-binding-era. \`ss IS NULL\` is THE predicate for that- -- population everywhere below (#4799 / #4806).- NULLIF(a.frontmatter->>'source_slug', '') AS ss,- CASE WHEN a.frontmatter->>'extracted_at' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}'- THEN a.frontmatter->>'extracted_at' END AS ext- FROM pages a- WHERE a.type = 'atom'- AND a.deleted_at IS NULL- AND a.frontmatter->>'source_hash' IS NOT NULL- -- in-flight marker written before an unmanaged extraction commits- AND (a.frontmatter->>'source_hash' NOT LIKE 'pending:%' OR a.frontmatter->>'managed_extraction' = 'true')+ `WITH ${ATOM_PROVENANCE_CTE},-- Lookup sets are built ONCE and joined (#4937). A correlated EXISTS in-- the SELECT list is not rewritten to a semi-join — Postgres re-runs it-- per atom over substring(content_hash), which no index serves, so the-- check went O(atoms x same-source pages) and blew health-check budgets.-- DISTINCT is load-bearing: duplicate live content_hash values are a-- real state and would otherwise fan one atom out into several rows.- ), live_hashes AS MATERIALIZED (+ live_hashes AS MATERIALIZED (SELECT DISTINCT source_id, substring(content_hash from 1 for 16) AS shFROM pages WHERE deleted_at IS NULL AND content_hash IS NOT NULL), live_slugs AS MATERIALIZED (@@ -891,8 +937,9 @@ export async function computeAtomProvenanceDriftCheck(const num = (v: string | number | null | undefined) => (v == null ? 0 : Number(v));const total = num(r.total);const slugUnbound = num(r.slug_unbound);- const drifted = num(r.drifted);- const sourceChanged = num(r.source_changed);+ const bodyUnchanged = num(r.source_changed) > 0 ? await unchangedBodyAtoms(engine) : 0;+ const drifted = num(r.drifted) - bodyUnchanged;+ const sourceChanged = num(r.source_changed) - bodyUnchanged;const sourceGone = num(r.source_gone);const oldestExtMs = r.oldest_ext ? new Date(String(r.oldest_ext)).getTime() : NaN;const oldestDays = Number.isFinite(oldestExtMs)@@ -905,6 +952,7 @@ export async function computeAtomProvenanceDriftCheck(drifted,source_changed: sourceChanged,source_gone: sourceGone,+ body_unchanged: bodyUnchanged,drift_pct: total > 0 ? Math.round(ratio * 1000) / 10 : 0,oldest_drifted_days: oldestDays ?? undefined,};@@ -929,7 +977,7 @@ export async function computeAtomProvenanceDriftCheck(`${drifted}/${total} atom(s) (${details.drift_pct}%) reference a source_hash no live page carries ` +`— ${sourceChanged} whose source page still exists (edited), ${sourceGone} whose source page is gone` +(oldestDays != null ? `; oldest ${oldestDays}d` : '') + su +- `. This compares source_hash only (any edit to the source page changes it); it does not re-check whether ` ++ `. Past metadata- and fence-only edits whose extracted edition a recent version still holds, this compares source_hash only; it does not re-check whether ` +`the atom's source_quote still appears in the page, so edited-source atoms may still be accurate. Fix: ${fix}`,details,};diff --git a/test/doctor-atom-provenance-drift-6825.test.ts b/test/doctor-atom-provenance-drift-6825.test.tsnew file mode 100644index 0000000000..4ae7d5a5a3--- /dev/null+++ b/test/doctor-atom-provenance-drift-6825.test.ts@@ -0,0 +1,89 @@+/**+ * #6825: atom_provenance_drift compared an atom's source_hash with the page's whole content_hash, so a+ * metadata-only frontmatter edit, or a gbrain fence write (takes or facts) that leaves the prose byte-identical, read+ * as an edited source.+ *+ * 1. Protects: when the edition the atom was extracted from is recoverable (a page_versions row whose content hash+ * carries the atom's source_hash) and its body without the gbrain-managed fence sections equals the current one,+ * the atom is not drift; a real prose edit, and an edit whose extraction edition cannot be found, still are.+ * 2. Fails when: the check counts every content_hash move as drift again, or hides an edit it cannot prove unchanged.+ * 3. Why existing coverage misses it: doctor-atom-provenance-drift.test.ts edits the prose in every drift case.+ * Real in-memory PGLite (canonical block); pages are written through importFromContent, which versions the page+ * before each write as the write paths do.+ */+import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { importFromContent } from '../src/core/import-file.ts';+import { computeAtomProvenanceDriftCheck } from '../src/commands/doctor.ts';+import { TAKES_FENCE_BEGIN, TAKES_FENCE_END } from '../src/core/takes-fence.ts';+import { resetPgliteState } from './helpers/reset-pglite.ts';++let engine: PGLiteEngine;++beforeAll(async () => {+ engine = new PGLiteEngine();+ await engine.connect({});+ await engine.initSchema();+});++afterAll(async () => {+ await engine.disconnect();+});++beforeEach(async () => {+ await resetPgliteState(engine);+});++const SOURCE = 'research/acme-example';+const PROSE = '# Acme Example\n\nAcme Example ships its widget every quarter and prices it per seat.\n';+const page = (frontmatter: string, body: string) => '---\ntitle: Acme Example\ntype: note\n' + frontmatter + '---\n\n' + body;+const write = (content: string) => importFromContent(engine, SOURCE, content, { noEmbed: true });+const TAKES = '## Takes\n\n' + TAKES_FENCE_BEGIN + '\n| # | claim | kind | who | weight | since | source |\n|---|-------|------|-----|--------|-------|--------|\n'+ + '| 1 | Acme will raise prices | bet | world | 0.6 | 2026-07 | notes |\n' + TAKES_FENCE_END + '\n';++async function extractAtom(): Promise<void> {+ const [row] = await engine.executeRaw<{ h: string }>('SELECT substring(content_hash from 1 for 16) AS h FROM pages WHERE slug = $1', [SOURCE]);+ await engine.putPage('atoms/2026-01-01/acme-000000', { type: 'atom', title: 'Acme atom', compiled_truth: 'Acme ships quarterly.',+ frontmatter: { type: 'atom', source_slug: SOURCE, source_hash: row!.h, extracted_at: new Date().toISOString() } });+}++const details = async () => (await computeAtomProvenanceDriftCheck(engine)).details as Record<string, number>;++describe('#6825 atom_provenance_drift: an unchanged body is not drift', () => {+ test('a metadata-only frontmatter edit is not drift', async () => {+ await write(page('', PROSE));+ await extractAtom();+ await write(page('facts_backstop: false\n', PROSE));+ const d = await details();+ expect(d.drifted).toBe(0);+ expect(d.source_changed).toBe(0);+ });++ test('a takes fence written under the prose is not drift', async () => {+ await write(page('', PROSE));+ await extractAtom();+ await write(page('', PROSE + '\n' + TAKES));+ const d = await details();+ expect(d.drifted).toBe(0);+ expect(d.source_changed).toBe(0);+ });++ test('a prose edit is still drift', async () => {+ await write(page('', PROSE));+ await extractAtom();+ await write(page('facts_backstop: false\n', PROSE.replace('every quarter', 'every month') + '\n' + TAKES));+ const d = await details();+ expect(d.drifted).toBe(1);+ expect(d.source_changed).toBe(1);+ });++ test('a metadata-only edit whose extraction edition is gone is still drift', async () => {+ await write(page('', PROSE));+ await extractAtom();+ await write(page('facts_backstop: false\n', PROSE));+ await engine.executeRaw('DELETE FROM page_versions WHERE page_id = (SELECT id FROM pages WHERE slug = $1)', [SOURCE]);+ const d = await details();+ expect(d.drifted).toBe(1);+ expect(d.source_changed).toBe(1);+ });+});diff --git a/test/fixtures/goldens/doctor/content-pglite-json.json b/test/fixtures/goldens/doctor/content-pglite-json.jsonindex 6274069f76..7fab65648c 100644--- a/test/fixtures/goldens/doctor/content-pglite-json.json+++ b/test/fixtures/goldens/doctor/content-pglite-json.json@@ -341,6 +341,7 @@{"category": "brain","details": {+ "body_unchanged": 0,"drift_pct": 0,"drifted": 0,"slug_unbound": 0,diff --git a/test/fixtures/goldens/doctor/hot-checks-pglite-json.json b/test/fixtures/goldens/doctor/hot-checks-pglite-json.jsonindex b979583551..de251e2af9 100644--- a/test/fixtures/goldens/doctor/hot-checks-pglite-json.json+++ b/test/fixtures/goldens/doctor/hot-checks-pglite-json.json@@ -342,6 +342,7 @@{"category": "brain","details": {+ "body_unchanged": 0,"drift_pct": 0,"drifted": 0,"slug_unbound": 0,@@ -5223,6 +5224,7 @@{"category": "brain","details": {+ "body_unchanged": 0,"drift_pct": 0,"drifted": 0,"slug_unbound": 0,diff --git a/test/fixtures/goldens/doctor/hot-checks-postgres-json.json b/test/fixtures/goldens/doctor/hot-checks-postgres-json.jsonindex 2a1c4acc2c..0192371279 100644--- a/test/fixtures/goldens/doctor/hot-checks-postgres-json.json+++ b/test/fixtures/goldens/doctor/hot-checks-postgres-json.json@@ -329,6 +329,7 @@{"category": "brain","details": {+ "body_unchanged": 0,"drift_pct": 0,"drifted": 0,"slug_unbound": 0,@@ -5211,6 +5212,7 @@{"category": "brain","details": {+ "body_unchanged": 0,"drift_pct": 0,"drifted": 0,"slug_unbound": 0,diff --git a/test/fixtures/goldens/doctor/json-pglite-degraded.json b/test/fixtures/goldens/doctor/json-pglite-degraded.jsonindex a6bbad91d0..6884c9270d 100644--- a/test/fixtures/goldens/doctor/json-pglite-degraded.json+++ b/test/fixtures/goldens/doctor/json-pglite-degraded.json@@ -338,6 +338,7 @@{"category": "brain","details": {+ "body_unchanged": 0,"drift_pct": 0,"drifted": 0,"slug_unbound": 0,diff --git a/test/fixtures/goldens/doctor/json-pglite-fresh.json b/test/fixtures/goldens/doctor/json-pglite-fresh.jsonindex ca9f480348..13377e8fa7 100644--- a/test/fixtures/goldens/doctor/json-pglite-fresh.json+++ b/test/fixtures/goldens/doctor/json-pglite-fresh.json@@ -341,6 +341,7 @@{"category": "brain","details": {+ "body_unchanged": 0,"drift_pct": 0,"drifted": 0,"slug_unbound": 0,diff --git a/test/fixtures/goldens/doctor/json-postgres-fresh.json b/test/fixtures/goldens/doctor/json-postgres-fresh.jsonindex f2354ac43d..1441092a53 100644--- a/test/fixtures/goldens/doctor/json-postgres-fresh.json+++ b/test/fixtures/goldens/doctor/json-postgres-fresh.json@@ -328,6 +328,7 @@{"category": "brain","details": {+ "body_unchanged": 0,"drift_pct": 0,"drifted": 0,"slug_unbound": 0,