Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(doctor): atom_provenance_drift skips metadata- and fence-only edits of an unchanged body (#6825)

garrytan2e8b6e85826e
diff --git a/docs/guides/repair.md b/docs/guides/repair.md
index fbfc157404..b8b78193e3 100644
--- a/docs/guides/repair.md
+++ b/docs/guides/repair.md
@@ -1735,7 +1735,9 @@ with the preview and apply commands; a stale hash refuses with
[`preview_changed`](#preview-changed). An apply under a different `--source`
than the preview also refuses with `preview_changed`. Doctor
`atom_provenance_drift` names this command when drift crosses its warn
-threshold.
+threshold. An atom whose page changed only in its frontmatter or in gbrain's
+facts and takes sections is not drift when one of the page's recent versions
+still holds the edition it was extracted from (#6825).
### Atoms restore
diff --git a/src/commands/doctor/checks/extraction-sync.ts b/src/commands/doctor/checks/extraction-sync.ts
index 4727fa042c..a9d7665c3e 100644
--- a/src/commands/doctor/checks/extraction-sync.ts
+++ b/src/commands/doctor/checks/extraction-sync.ts
@@ -15,7 +15,8 @@ import { resolveHoursEnv } from '../../../core/env-number.ts';
import { chunkerStamp } from '../../../core/chunkers/code.ts';
import { smallBrainBacklogNote } from '../../../core/link-extraction-watermark.ts';
import { extractionLagVerdict, extractionRoutes, heldExtractionReport, readExtractionLag } from '../../../core/extraction-lag.ts';
-import { isMissingSchemaObject } from '../../../core/utils.ts';
+import { contentHash, isMissingSchemaObject } from '../../../core/utils.ts';
+import { stripFenceSections } from '../../../core/cycle/concept-publication.ts';
import {
loadStorageConfig,
effectiveDbOnlyDirs,
@@ -771,6 +772,68 @@ export async function computeExtractAtomsBacklogCheck(
}
}
+/** The page-bound and slug-unbound atoms atom_provenance_drift reads (`ss` NULL: slug-unbound), as a CTE body. */
+const ATOM_PROVENANCE_CTE = `atom AS (
+ SELECT a.source_id,
+ -- managed atoms keep their provisional prefix for good (#5770)
+ CASE WHEN a.frontmatter->>'managed_extraction' = 'true'
+ THEN regexp_replace(a.frontmatter->>'source_hash', '^pending:', '')
+ ELSE a.frontmatter->>'source_hash' END AS sh,
+ -- NULL = slug-unbound: transcript-minted (source_path only) or
+ -- pre-binding-era. \`ss IS NULL\` is THE predicate for that
+ -- population everywhere below (#4799 / #4806).
+ NULLIF(a.frontmatter->>'source_slug', '') AS ss,
+ CASE WHEN a.frontmatter->>'extracted_at' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+ THEN a.frontmatter->>'extracted_at' END AS ext
+ FROM pages a
+ WHERE a.type = 'atom'
+ AND a.deleted_at IS NULL
+ AND a.frontmatter->>'source_hash' IS NOT NULL
+ -- in-flight marker written before an unmanaged extraction commits
+ AND (a.frontmatter->>'source_hash' NOT LIKE 'pending:%' OR a.frontmatter->>'managed_extraction' = 'true')
+ )`;
+
+/** (page, source_hash) groups of drifted atoms unchangedBodyAtoms re-checks per run; the rest keep their drift verdict. */
+const BODY_RECHECK_GROUPS = 200;
+/** The newest versions of a page searched for an atom's extraction edition. */
+const BODY_RECHECK_VERSIONS = 20;
+
+/**
+ * #6825: drifted atoms of a live inline page whose prose did not change. An atom's source_hash names the page edition
+ * it was extracted from; when one of the page's recent versions hashes to it and its body without the gbrain-managed
+ * facts and takes sections (stripFenceSections, the narrative concept synthesis compares) equals the page's, the move
+ * was a metadata or fence write, not an edit of what the atom summarises. An edition no version still holds, and a
+ * segmented page, keep their drift verdict.
+ */
+async function unchangedBodyAtoms(engine: BrainEngine): Promise<number> {
+ const groups = await engine.executeRaw<{ page_id: number; sh: string; n: number }>(
+ `WITH ${ATOM_PROVENANCE_CTE}, live_hashes AS MATERIALIZED (
+ SELECT DISTINCT source_id, substring(content_hash from 1 for 16) AS sh FROM pages WHERE deleted_at IS NULL AND content_hash IS NOT NULL
+ )
+ SELECT p.id AS page_id, atom.sh, count(*)::int AS n
+ FROM atom
+ JOIN pages p ON p.source_id = atom.source_id AND p.slug = atom.ss AND p.deleted_at IS NULL AND p.body_storage = 'inline'
+ LEFT JOIN live_hashes h ON h.source_id = atom.source_id AND h.sh = atom.sh
+ WHERE h.sh IS NULL AND length(atom.sh) = 16
+ GROUP BY p.id, atom.sh ORDER BY p.id, atom.sh LIMIT ${BODY_RECHECK_GROUPS}`, []);
+ const json = <T>(v: unknown): T => (typeof v === 'string' ? JSON.parse(v) : v) as T;
+ const narrative = (body: { compiled_truth: string; timeline: string | null }) => [stripFenceSections(body.compiled_truth), stripFenceSections(body.timeline ?? '')];
+ let unchanged = 0;
+ for (const group of groups) {
+ const [page] = await engine.executeRaw<{ compiled_truth: string | null; timeline: string | null }>('SELECT compiled_truth, timeline FROM pages WHERE id = $1', [group.page_id]);
+ if (page?.compiled_truth == null) continue;
+ const versions = await engine.executeRaw<{ compiled_truth: string | null; timeline: string | null; title: string; type: string; frontmatter: unknown; tags: unknown }>(
+ `SELECT compiled_truth, timeline, title, type, frontmatter, tags FROM page_versions
+ WHERE page_id = $1 AND body_storage = 'inline' AND compiled_truth IS NOT NULL ORDER BY id DESC LIMIT ${BODY_RECHECK_VERSIONS}`, [group.page_id]);
+ const edition = versions.find(v => contentHash({ title: v.title, type: v.type as never, compiled_truth: v.compiled_truth!, timeline: v.timeline ?? '',
+ frontmatter: json<Record<string, unknown>>(v.frontmatter) ?? {}, tags: json<string[] | null>(v.tags) ?? [] }).startsWith(group.sh));
+ if (!edition) continue;
+ const [was, is] = [narrative({ compiled_truth: edition.compiled_truth!, timeline: edition.timeline }), narrative({ compiled_truth: page.compiled_truth, timeline: page.timeline })];
+ if (was[0] === is[0] && was[1] === is[1]) unchanged += Number(group.n);
+ }
+ return unchanged;
+}
+
/**
* atom_provenance_drift doctor check (#4566).
*
@@ -833,31 +896,14 @@ export async function computeAtomProvenanceDriftCheck(
// garbage from the min(); the age math happens in TS where Date
// parsing can never throw (semantically-invalid dates become NaN →
// metric omitted, verdict untouched).
- `WITH atom AS (
- SELECT a.source_id,
- -- managed atoms keep their provisional prefix for good (#5770)
- CASE WHEN a.frontmatter->>'managed_extraction' = 'true'
- THEN regexp_replace(a.frontmatter->>'source_hash', '^pending:', '')
- ELSE a.frontmatter->>'source_hash' END AS sh,
- -- NULL = slug-unbound: transcript-minted (source_path only) or
- -- pre-binding-era. \`ss IS NULL\` is THE predicate for that
- -- population everywhere below (#4799 / #4806).
- NULLIF(a.frontmatter->>'source_slug', '') AS ss,
- CASE WHEN a.frontmatter->>'extracted_at' ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
- THEN a.frontmatter->>'extracted_at' END AS ext
- FROM pages a
- WHERE a.type = 'atom'
- AND a.deleted_at IS NULL
- AND a.frontmatter->>'source_hash' IS NOT NULL
- -- in-flight marker written before an unmanaged extraction commits
- AND (a.frontmatter->>'source_hash' NOT LIKE 'pending:%' OR a.frontmatter->>'managed_extraction' = 'true')
+ `WITH ${ATOM_PROVENANCE_CTE},
-- Lookup sets are built ONCE and joined (#4937). A correlated EXISTS in
-- the SELECT list is not rewritten to a semi-join — Postgres re-runs it
-- per atom over substring(content_hash), which no index serves, so the
-- check went O(atoms x same-source pages) and blew health-check budgets.
-- DISTINCT is load-bearing: duplicate live content_hash values are a
-- real state and would otherwise fan one atom out into several rows.
- ), live_hashes AS MATERIALIZED (
+ live_hashes AS MATERIALIZED (
SELECT DISTINCT source_id, substring(content_hash from 1 for 16) AS sh
FROM pages WHERE deleted_at IS NULL AND content_hash IS NOT NULL
), live_slugs AS MATERIALIZED (
@@ -891,8 +937,9 @@ export async function computeAtomProvenanceDriftCheck(
const num = (v: string | number | null | undefined) => (v == null ? 0 : Number(v));
const total = num(r.total);
const slugUnbound = num(r.slug_unbound);
- const drifted = num(r.drifted);
- const sourceChanged = num(r.source_changed);
+ const bodyUnchanged = num(r.source_changed) > 0 ? await unchangedBodyAtoms(engine) : 0;
+ const drifted = num(r.drifted) - bodyUnchanged;
+ const sourceChanged = num(r.source_changed) - bodyUnchanged;
const sourceGone = num(r.source_gone);
const oldestExtMs = r.oldest_ext ? new Date(String(r.oldest_ext)).getTime() : NaN;
const oldestDays = Number.isFinite(oldestExtMs)
@@ -905,6 +952,7 @@ export async function computeAtomProvenanceDriftCheck(
drifted,
source_changed: sourceChanged,
source_gone: sourceGone,
+ body_unchanged: bodyUnchanged,
drift_pct: total > 0 ? Math.round(ratio * 1000) / 10 : 0,
oldest_drifted_days: oldestDays ?? undefined,
};
@@ -929,7 +977,7 @@ export async function computeAtomProvenanceDriftCheck(
`${drifted}/${total} atom(s) (${details.drift_pct}%) reference a source_hash no live page carries ` +
`— ${sourceChanged} whose source page still exists (edited), ${sourceGone} whose source page is gone` +
(oldestDays != null ? `; oldest ${oldestDays}d` : '') + su +
- `. This compares source_hash only (any edit to the source page changes it); it does not re-check whether ` +
+ `. Past metadata- and fence-only edits whose extracted edition a recent version still holds, this compares source_hash only; it does not re-check whether ` +
`the atom's source_quote still appears in the page, so edited-source atoms may still be accurate. Fix: ${fix}`,
details,
};
diff --git a/test/doctor-atom-provenance-drift-6825.test.ts b/test/doctor-atom-provenance-drift-6825.test.ts
new file mode 100644
index 0000000000..4ae7d5a5a3
--- /dev/null
+++ b/test/doctor-atom-provenance-drift-6825.test.ts
@@ -0,0 +1,89 @@
+/**
+ * #6825: atom_provenance_drift compared an atom's source_hash with the page's whole content_hash, so a
+ * metadata-only frontmatter edit, or a gbrain fence write (takes or facts) that leaves the prose byte-identical, read
+ * as an edited source.
+ *
+ * 1. Protects: when the edition the atom was extracted from is recoverable (a page_versions row whose content hash
+ * carries the atom's source_hash) and its body without the gbrain-managed fence sections equals the current one,
+ * the atom is not drift; a real prose edit, and an edit whose extraction edition cannot be found, still are.
+ * 2. Fails when: the check counts every content_hash move as drift again, or hides an edit it cannot prove unchanged.
+ * 3. Why existing coverage misses it: doctor-atom-provenance-drift.test.ts edits the prose in every drift case.
+ * Real in-memory PGLite (canonical block); pages are written through importFromContent, which versions the page
+ * before each write as the write paths do.
+ */
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { importFromContent } from '../src/core/import-file.ts';
+import { computeAtomProvenanceDriftCheck } from '../src/commands/doctor.ts';
+import { TAKES_FENCE_BEGIN, TAKES_FENCE_END } from '../src/core/takes-fence.ts';
+import { resetPgliteState } from './helpers/reset-pglite.ts';
+
+let engine: PGLiteEngine;
+
+beforeAll(async () => {
+ engine = new PGLiteEngine();
+ await engine.connect({});
+ await engine.initSchema();
+});
+
+afterAll(async () => {
+ await engine.disconnect();
+});
+
+beforeEach(async () => {
+ await resetPgliteState(engine);
+});
+
+const SOURCE = 'research/acme-example';
+const PROSE = '# Acme Example\n\nAcme Example ships its widget every quarter and prices it per seat.\n';
+const page = (frontmatter: string, body: string) => '---\ntitle: Acme Example\ntype: note\n' + frontmatter + '---\n\n' + body;
+const write = (content: string) => importFromContent(engine, SOURCE, content, { noEmbed: true });
+const TAKES = '## Takes\n\n' + TAKES_FENCE_BEGIN + '\n| # | claim | kind | who | weight | since | source |\n|---|-------|------|-----|--------|-------|--------|\n'
+ + '| 1 | Acme will raise prices | bet | world | 0.6 | 2026-07 | notes |\n' + TAKES_FENCE_END + '\n';
+
+async function extractAtom(): Promise<void> {
+ const [row] = await engine.executeRaw<{ h: string }>('SELECT substring(content_hash from 1 for 16) AS h FROM pages WHERE slug = $1', [SOURCE]);
+ await engine.putPage('atoms/2026-01-01/acme-000000', { type: 'atom', title: 'Acme atom', compiled_truth: 'Acme ships quarterly.',
+ frontmatter: { type: 'atom', source_slug: SOURCE, source_hash: row!.h, extracted_at: new Date().toISOString() } });
+}
+
+const details = async () => (await computeAtomProvenanceDriftCheck(engine)).details as Record<string, number>;
+
+describe('#6825 atom_provenance_drift: an unchanged body is not drift', () => {
+ test('a metadata-only frontmatter edit is not drift', async () => {
+ await write(page('', PROSE));
+ await extractAtom();
+ await write(page('facts_backstop: false\n', PROSE));
+ const d = await details();
+ expect(d.drifted).toBe(0);
+ expect(d.source_changed).toBe(0);
+ });
+
+ test('a takes fence written under the prose is not drift', async () => {
+ await write(page('', PROSE));
+ await extractAtom();
+ await write(page('', PROSE + '\n' + TAKES));
+ const d = await details();
+ expect(d.drifted).toBe(0);
+ expect(d.source_changed).toBe(0);
+ });
+
+ test('a prose edit is still drift', async () => {
+ await write(page('', PROSE));
+ await extractAtom();
+ await write(page('facts_backstop: false\n', PROSE.replace('every quarter', 'every month') + '\n' + TAKES));
+ const d = await details();
+ expect(d.drifted).toBe(1);
+ expect(d.source_changed).toBe(1);
+ });
+
+ test('a metadata-only edit whose extraction edition is gone is still drift', async () => {
+ await write(page('', PROSE));
+ await extractAtom();
+ await write(page('facts_backstop: false\n', PROSE));
+ await engine.executeRaw('DELETE FROM page_versions WHERE page_id = (SELECT id FROM pages WHERE slug = $1)', [SOURCE]);
+ const d = await details();
+ expect(d.drifted).toBe(1);
+ expect(d.source_changed).toBe(1);
+ });
+});
diff --git a/test/fixtures/goldens/doctor/content-pglite-json.json b/test/fixtures/goldens/doctor/content-pglite-json.json
index 6274069f76..7fab65648c 100644
--- a/test/fixtures/goldens/doctor/content-pglite-json.json
+++ b/test/fixtures/goldens/doctor/content-pglite-json.json
@@ -341,6 +341,7 @@
{
"category": "brain",
"details": {
+ "body_unchanged": 0,
"drift_pct": 0,
"drifted": 0,
"slug_unbound": 0,
diff --git a/test/fixtures/goldens/doctor/hot-checks-pglite-json.json b/test/fixtures/goldens/doctor/hot-checks-pglite-json.json
index b979583551..de251e2af9 100644
--- a/test/fixtures/goldens/doctor/hot-checks-pglite-json.json
+++ b/test/fixtures/goldens/doctor/hot-checks-pglite-json.json
@@ -342,6 +342,7 @@
{
"category": "brain",
"details": {
+ "body_unchanged": 0,
"drift_pct": 0,
"drifted": 0,
"slug_unbound": 0,
@@ -5223,6 +5224,7 @@
{
"category": "brain",
"details": {
+ "body_unchanged": 0,
"drift_pct": 0,
"drifted": 0,
"slug_unbound": 0,
diff --git a/test/fixtures/goldens/doctor/hot-checks-postgres-json.json b/test/fixtures/goldens/doctor/hot-checks-postgres-json.json
index 2a1c4acc2c..0192371279 100644
--- a/test/fixtures/goldens/doctor/hot-checks-postgres-json.json
+++ b/test/fixtures/goldens/doctor/hot-checks-postgres-json.json
@@ -329,6 +329,7 @@
{
"category": "brain",
"details": {
+ "body_unchanged": 0,
"drift_pct": 0,
"drifted": 0,
"slug_unbound": 0,
@@ -5211,6 +5212,7 @@
{
"category": "brain",
"details": {
+ "body_unchanged": 0,
"drift_pct": 0,
"drifted": 0,
"slug_unbound": 0,
diff --git a/test/fixtures/goldens/doctor/json-pglite-degraded.json b/test/fixtures/goldens/doctor/json-pglite-degraded.json
index a6bbad91d0..6884c9270d 100644
--- a/test/fixtures/goldens/doctor/json-pglite-degraded.json
+++ b/test/fixtures/goldens/doctor/json-pglite-degraded.json
@@ -338,6 +338,7 @@
{
"category": "brain",
"details": {
+ "body_unchanged": 0,
"drift_pct": 0,
"drifted": 0,
"slug_unbound": 0,
diff --git a/test/fixtures/goldens/doctor/json-pglite-fresh.json b/test/fixtures/goldens/doctor/json-pglite-fresh.json
index ca9f480348..13377e8fa7 100644
--- a/test/fixtures/goldens/doctor/json-pglite-fresh.json
+++ b/test/fixtures/goldens/doctor/json-pglite-fresh.json
@@ -341,6 +341,7 @@
{
"category": "brain",
"details": {
+ "body_unchanged": 0,
"drift_pct": 0,
"drifted": 0,
"slug_unbound": 0,
diff --git a/test/fixtures/goldens/doctor/json-postgres-fresh.json b/test/fixtures/goldens/doctor/json-postgres-fresh.json
index f2354ac43d..1441092a53 100644
--- a/test/fixtures/goldens/doctor/json-postgres-fresh.json
+++ b/test/fixtures/goldens/doctor/json-postgres-fresh.json
@@ -328,6 +328,7 @@
{
"category": "brain",
"details": {
+ "body_unchanged": 0,
"drift_pct": 0,
"drifted": 0,
"slug_unbound": 0,