Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(persistence): keep the page body reader count flat for batched export reads (#7617)
garrytan75ae01233aa2
diff --git a/scripts/page-body-readers-baseline.tsv b/scripts/page-body-readers-baseline.tsvindex 9baaa5bd4b..569352aa8e 100644--- a/scripts/page-body-readers-baseline.tsv+++ b/scripts/page-body-readers-baseline.tsv@@ -207,8 +207,8 @@ src/core/enrichment-service.ts stubMarkdown call:serializeMarkdown INLINE G6: asrc/core/export-snapshot.ts inlineExportMarkdown call:serializeMarkdown SEGWsrc/core/export-snapshot.ts inlineExportMarkdown field STREAMsrc/core/export-snapshot.ts segmentedExportHead call:serializeMarkdown NONE not the page body: the frontmatter head, rendered with an empty body-src/core/export-snapshot.ts readExportPage field STREAM inline pages only: a segmented page skips the overlay here and streams through segmentedExportMarkdown (readBodyRange, trusted stored)-src/core/export-snapshot.ts readExportPage sql STREAM+src/core/export-snapshot.ts readExportPages field STREAM inline pages only: a segmented page skips the overlay here and streams through segmentedExportMarkdown (readBodyRange, trusted stored)+src/core/export-snapshot.ts readExportPages sql STREAMsrc/core/extract-ner.ts extractNerLinks field STREAMsrc/core/extract-takes-from-pages.ts extractTakesFromPages call:serializePageToMarkdown SEGWsrc/core/extract-takes-from-pages.ts extractTakesFromPages field STREAM inline rows only (SQL filter)diff --git a/src/core/export-snapshot.ts b/src/core/export-snapshot.tsindex 4628b50050..219ed2a4e9 100644--- a/src/core/export-snapshot.ts+++ b/src/core/export-snapshot.ts@@ -1,5 +1,5 @@import type { BrainEngine } from './engine.ts';-import { overlayCanonicalBodies, overlayCanonicalBodiesBatch } from './page-state/snapshot.ts';+import { overlayWithdrawalBody } from './facts/withdrawal-overlay.ts';import type { PageWithdrawal } from './page-state/types.ts';import { rowToPage } from './utils.ts';import { EXPORT_PAYLOAD_LIMIT } from './export-stage.ts';@@ -26,27 +26,16 @@ export async function readExportWithdrawals(tx: BrainEngine, sourceId: string):ORDER BY visibility,fact_hash,subject`, [sourceId]);}-export async function readExportPage(tx: BrainEngine, key: { id: string; source_id: string; slug: string }, withdrawals: ExportWithdrawal[]) {- const [row] = await tx.executeRaw<Record<string, unknown>>(`WITH export_page AS MATERIALIZED (- SELECT * FROM pages WHERE id=$1::bigint- ) SELECT p.*,- COALESCE((SELECT jsonb_agg(t.tag ORDER BY t.tag) FROM tags t WHERE t.page_id=p.id), '[]'::jsonb) AS export_tags- FROM export_page p WHERE p.source_id=$2 AND p.slug=$3 AND p.deleted_at IS NULL`,- [key.id, key.source_id, key.slug]);- if (!row) throw new Error('Export snapshot unexpectedly lost a selected page.');- const page = rowToPage(row);- if (!isSegmentedPage(page) && (page.compiled_truth.includes('gbrain:facts:') || page.timeline.includes('gbrain:facts:'))) {- Object.assign(page, await overlayCanonicalBodies(tx.executeRaw.bind(tx), page.compiled_truth, page.timeline, applicableWithdrawals(page.slug, withdrawals)));- }- return { page, tags: row.export_tags as string[] };-}+export type ExportSnapshot = { page: Page; tags: string[] };-export type ExportSnapshot = Awaited<ReturnType<typeof readExportPage>>;+export async function readExportPage(tx: BrainEngine, key: { id: string; source_id: string; slug: string }, withdrawals: ExportWithdrawal[]): Promise<ExportSnapshot> {+ return (await readExportPages(tx, [key], withdrawals))[0]!;+}/**- * readExportPage of every key, in key order, in two statements however many keys there are: the rows with their tags,- * and one fence normalization for the pages whose overlay needs it (#7617: a per-page read made a large source's- * conversion plan one round trip per page). Each snapshot is exactly the one readExportPage returns for its key.+ * The export snapshots of `keys`, in key order, in at most two statements however many keys there are (#7617: a+ * per-page read made a large source's conversion plan one round trip per page): the rows with their tags, then one+ * fence normalization for every inline page whose withdrawal overlay applies, by overlayCanonicalBodies' rules and SQL.*/export async function readExportPages(tx: BrainEngine, keys: ReadonlyArray<{ id: string; source_id: string; slug: string }>, withdrawals: ExportWithdrawal[]): Promise<ExportSnapshot[]> {if (!keys.length) return [];@@ -61,10 +50,24 @@ export async function readExportPages(tx: BrainEngine, keys: ReadonlyArray<{ id:if (!row) throw new Error('Export snapshot unexpectedly lost a selected page.');return { page: rowToPage(row), tags: row.export_tags as string[] };});- const fenced = snapshots.filter(({ page }) => !isSegmentedPage(page) && (page.compiled_truth.includes('gbrain:facts:') || page.timeline.includes('gbrain:facts:')));- const overlaid = await overlayCanonicalBodiesBatch(tx.executeRaw.bind(tx),- fenced.map(({ page }) => ({ body: page.compiled_truth, timeline: page.timeline, withdrawals: applicableWithdrawals(page.slug, withdrawals) })));- fenced.forEach(({ page }, i) => Object.assign(page, overlaid[i]));+ const fenced = snapshots.flatMap(({ page }) => {+ if (isSegmentedPage(page) || !(page.compiled_truth.includes('gbrain:facts:begin') || page.timeline.includes('gbrain:facts:begin'))) return [];+ const applicable = applicableWithdrawals(page.slug, withdrawals);+ return applicable.length ? [{ page, applicable }] : [];+ });+ if (!fenced.length) return snapshots;+ const normalized = await tx.executeRaw<{ i: number; body: string | null; timeline: string | null }>(`SELECT t.i::int AS i,+ (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)+ FROM unnest(string_to_array(t.body,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS body,+ (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)+ FROM unnest(string_to_array(t.timeline,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS timeline+ FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS t(body, timeline, i)`,+ [fenced.map(({ page }) => page.compiled_truth), fenced.map(({ page }) => page.timeline)]);+ for (const row of normalized) {+ const { page, applicable } = fenced[Number(row.i) - 1]!;+ page.compiled_truth = overlayWithdrawalBody(page.compiled_truth, row.body ?? '', applicable);+ page.timeline = overlayWithdrawalBody(page.timeline, row.timeline ?? '', applicable);+ }return snapshots;}diff --git a/src/core/page-state/snapshot.ts b/src/core/page-state/snapshot.tsindex 71eaa32b42..2f6ce8aa1c 100644--- a/src/core/page-state/snapshot.ts+++ b/src/core/page-state/snapshot.ts@@ -30,10 +30,6 @@ export async function resolveGlobalPurges(query: ReadQuery, sourceId: string, mareturn rows.map(r => typeof r.w === 'string' ? JSON.parse(r.w) as PageWithdrawal : r.w);}-/** The ledger fingerprint text of `text` (a SQL expression): each line lowercased, whitespace runs collapsed to one space. */-const normalizedLines = (text: string) => `(SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)- FROM unnest(string_to_array(${text},chr(10))) WITH ORDINALITY AS lines(line,ord))`;-/*** Normalize filesystem bytes with the exact ledger fingerprint rules before comparing them. `global` names the* snapshot's source and '*' purge marker, resolved against this text's fence rows (not the stored page's).@@ -43,34 +39,17 @@ export async function overlayCanonicalBodies(query: ReadQuery, body: string, timconst pending = global?.marker && global.marker.count > 0 ? global : undefined;if (!withdrawals.length && !pending) return { compiled_truth: body, timeline };if (!body.includes('gbrain:facts:begin') && !timeline.includes('gbrain:facts:begin')) return { compiled_truth: body, timeline };- const [normalized] = await query<{ body: string; timeline: string }>(`SELECT ${normalizedLines('$1::text')} AS body, ${normalizedLines('$2::text')} AS timeline`, [body, timeline]);+ const [normalized] = await query<{ body: string; timeline: string }>(`SELECT+ (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)+ FROM unnest(string_to_array($1::text,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS body,+ (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)+ FROM unnest(string_to_array($2::text,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS timeline`, [body, timeline]);const resolved = pending ? await resolveGlobalPurges(query, pending.sourceId, pending.marker, [normalized.body ?? '', normalized.timeline ?? '']) : [];const all = resolved.length ? [...withdrawals, ...resolved] : withdrawals;return { compiled_truth: overlayWithdrawalBody(body, normalized.body ?? '', all),timeline: overlayWithdrawalBody(timeline, normalized.timeline ?? '', all) };}-/**- * overlayCanonicalBodies (without a '*' purge marker) of many texts: the texts that need it are normalized in one- * statement, so a batch of pages costs one round trip instead of one per page. Results are in `items` order.- */-export async function overlayCanonicalBodiesBatch(query: ReadQuery, items: ReadonlyArray<{ body: string; timeline: string; withdrawals: PageWithdrawal[] }>):- Promise<Array<{ compiled_truth: string; timeline: string }>> {- const out = items.map(item => ({ compiled_truth: item.body, timeline: item.timeline }));- const pending = items.flatMap((item, i) => item.withdrawals.length- && (item.body.includes('gbrain:facts:begin') || item.timeline.includes('gbrain:facts:begin')) ? [i] : []);- if (!pending.length) return out;- const rows = await query<{ i: number; body: string | null; timeline: string | null }>(`SELECT t.i::int AS i, ${normalizedLines('t.body')} AS body,- ${normalizedLines('t.timeline')} AS timeline FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS t(body, timeline, i)`,- [pending.map(i => items[i]!.body), pending.map(i => items[i]!.timeline)]);- for (const row of rows) {- const at = pending[Number(row.i) - 1]!, item = items[at]!;- out[at] = { compiled_truth: overlayWithdrawalBody(item.body, row.body ?? '', item.withdrawals),- timeline: overlayWithdrawalBody(item.timeline, row.timeline ?? '', item.withdrawals) };- }- return out;-}-/*** One MVCC statement binds content, tags, identity and withdrawals to one revision.* The normalized body and timeline fingerprints feed only the withdrawal overlay,diff --git a/test/export-snapshot.test.ts b/test/export-snapshot.test.tsindex 2b79735fd7..ee3f7017da 100644--- a/test/export-snapshot.test.ts+++ b/test/export-snapshot.test.ts@@ -11,6 +11,26 @@ import { importFromContent } from '../src/core/import-file.ts';import { inlineExportMarkdown, readExportPage, readExportPages, readExportWithdrawals } from '../src/core/export-snapshot.ts';import { overlayCanonicalBodies } from '../src/core/page-state/snapshot.ts';import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';+import { rowToPage } from '../src/core/utils.ts';+import { isSegmentedPage } from '../src/core/page-body/page-shape.ts';+import type { ExportWithdrawal } from '../src/core/export-snapshot.ts';++/** The per-page export read as it shipped before #7617, kept as the reference the batch read must reproduce byte for byte. */+async function referenceExportPage(tx: BrainEngine, key: { id: string; source_id: string; slug: string }, withdrawals: ExportWithdrawal[]) {+ const [row] = await tx.executeRaw<Record<string, unknown>>(`WITH export_page AS MATERIALIZED (+ SELECT * FROM pages WHERE id=$1::bigint+ ) SELECT p.*,+ COALESCE((SELECT jsonb_agg(t.tag ORDER BY t.tag) FROM tags t WHERE t.page_id=p.id), '[]'::jsonb) AS export_tags+ FROM export_page p WHERE p.source_id=$2 AND p.slug=$3 AND p.deleted_at IS NULL`, [key.id, key.source_id, key.slug]);+ if (!row) throw new Error('Export snapshot unexpectedly lost a selected page.');+ const page = rowToPage(row);+ if (!isSegmentedPage(page) && (page.compiled_truth.includes('gbrain:facts:') || page.timeline.includes('gbrain:facts:'))) {+ const applicable = withdrawals.filter(w => w.subject === '*' || w.subject === page.slug)+ .map(({ visibility, fact_hash, withdrawn_at, purged }) => ({ visibility, fact_hash, withdrawn_at, ...(purged ? { purged: true } : {}) }));+ Object.assign(page, await overlayCanonicalBodies(tx.executeRaw.bind(tx), page.compiled_truth, page.timeline, applicable));+ }+ return { page, tags: row.export_tags as string[] };+}for (const backend of testBackends()) describe(`coherent export: ${backend}`, () => {let engine: BrainEngine;@@ -82,7 +102,7 @@ for (const backend of testBackends()) describe(`coherent export: ${backend}`, ()}});}, 60000);- test('#7617: readExportPages gives every page exactly the snapshot readExportPage gives, in two statements', async () => {+ test('#7617: readExportPages gives every page exactly the snapshot the per-page read gave, in two statements', async () => {await engine.executeRaw('DELETE FROM pages');await engine.executeRaw("DELETE FROM fact_withdrawals WHERE source_id='default'");await engine.executeRaw("DELETE FROM fact_purges WHERE source_id='default'");@@ -139,10 +159,11 @@ for (const backend of testBackends()) describe(`coherent export: ${backend}`, ()expect(batch.length).toBe(keys.length);let overlaid = 0;for (const [i, key] of keys.entries()) {- const single = await readExportPage(tx, key, withdrawals);+ const single = await referenceExportPage(tx, key, withdrawals);expect(batch[i]!.page.slug).toBe(key.slug);expect(JSON.stringify(batch[i])).toBe(JSON.stringify(single));expect(inlineExportMarkdown(batch[i]!)).toBe(inlineExportMarkdown(single));+ expect(JSON.stringify(await readExportPage(tx, key, withdrawals))).toBe(JSON.stringify(single));if ([single.page.compiled_truth, single.page.timeline].some(text => parseFactsFence(text).facts.some(fact => fact.active === false))) overlaid++;}// The overlay really ran: withdrawn rows read inactive in the fenced shapes and every fifth bulk page.