Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(persistence): keep the page body reader count flat for batched export reads (#7617)

garrytan75ae01233aa2
diff --git a/scripts/page-body-readers-baseline.tsv b/scripts/page-body-readers-baseline.tsv
index 9baaa5bd4b..569352aa8e 100644
--- a/scripts/page-body-readers-baseline.tsv
+++ b/scripts/page-body-readers-baseline.tsv
@@ -207,8 +207,8 @@ src/core/enrichment-service.ts stubMarkdown call:serializeMarkdown INLINE G6: a
src/core/export-snapshot.ts inlineExportMarkdown call:serializeMarkdown SEGW
src/core/export-snapshot.ts inlineExportMarkdown field STREAM
src/core/export-snapshot.ts segmentedExportHead call:serializeMarkdown NONE not the page body: the frontmatter head, rendered with an empty body
-src/core/export-snapshot.ts readExportPage field STREAM inline pages only: a segmented page skips the overlay here and streams through segmentedExportMarkdown (readBodyRange, trusted stored)
-src/core/export-snapshot.ts readExportPage sql STREAM
+src/core/export-snapshot.ts readExportPages field STREAM inline pages only: a segmented page skips the overlay here and streams through segmentedExportMarkdown (readBodyRange, trusted stored)
+src/core/export-snapshot.ts readExportPages sql STREAM
src/core/extract-ner.ts extractNerLinks field STREAM
src/core/extract-takes-from-pages.ts extractTakesFromPages call:serializePageToMarkdown SEGW
src/core/extract-takes-from-pages.ts extractTakesFromPages field STREAM inline rows only (SQL filter)
diff --git a/src/core/export-snapshot.ts b/src/core/export-snapshot.ts
index 4628b50050..219ed2a4e9 100644
--- a/src/core/export-snapshot.ts
+++ b/src/core/export-snapshot.ts
@@ -1,5 +1,5 @@
import type { BrainEngine } from './engine.ts';
-import { overlayCanonicalBodies, overlayCanonicalBodiesBatch } from './page-state/snapshot.ts';
+import { overlayWithdrawalBody } from './facts/withdrawal-overlay.ts';
import type { PageWithdrawal } from './page-state/types.ts';
import { rowToPage } from './utils.ts';
import { EXPORT_PAYLOAD_LIMIT } from './export-stage.ts';
@@ -26,27 +26,16 @@ export async function readExportWithdrawals(tx: BrainEngine, sourceId: string):
ORDER BY visibility,fact_hash,subject`, [sourceId]);
}
-export async function readExportPage(tx: BrainEngine, key: { id: string; source_id: string; slug: string }, withdrawals: ExportWithdrawal[]) {
- const [row] = await tx.executeRaw<Record<string, unknown>>(`WITH export_page AS MATERIALIZED (
- SELECT * FROM pages WHERE id=$1::bigint
- ) SELECT p.*,
- COALESCE((SELECT jsonb_agg(t.tag ORDER BY t.tag) FROM tags t WHERE t.page_id=p.id), '[]'::jsonb) AS export_tags
- FROM export_page p WHERE p.source_id=$2 AND p.slug=$3 AND p.deleted_at IS NULL`,
- [key.id, key.source_id, key.slug]);
- if (!row) throw new Error('Export snapshot unexpectedly lost a selected page.');
- const page = rowToPage(row);
- if (!isSegmentedPage(page) && (page.compiled_truth.includes('gbrain:facts:') || page.timeline.includes('gbrain:facts:'))) {
- Object.assign(page, await overlayCanonicalBodies(tx.executeRaw.bind(tx), page.compiled_truth, page.timeline, applicableWithdrawals(page.slug, withdrawals)));
- }
- return { page, tags: row.export_tags as string[] };
-}
+export type ExportSnapshot = { page: Page; tags: string[] };
-export type ExportSnapshot = Awaited<ReturnType<typeof readExportPage>>;
+export async function readExportPage(tx: BrainEngine, key: { id: string; source_id: string; slug: string }, withdrawals: ExportWithdrawal[]): Promise<ExportSnapshot> {
+ return (await readExportPages(tx, [key], withdrawals))[0]!;
+}
/**
- * readExportPage of every key, in key order, in two statements however many keys there are: the rows with their tags,
- * and one fence normalization for the pages whose overlay needs it (#7617: a per-page read made a large source's
- * conversion plan one round trip per page). Each snapshot is exactly the one readExportPage returns for its key.
+ * The export snapshots of `keys`, in key order, in at most two statements however many keys there are (#7617: a
+ * per-page read made a large source's conversion plan one round trip per page): the rows with their tags, then one
+ * fence normalization for every inline page whose withdrawal overlay applies, by overlayCanonicalBodies' rules and SQL.
*/
export async function readExportPages(tx: BrainEngine, keys: ReadonlyArray<{ id: string; source_id: string; slug: string }>, withdrawals: ExportWithdrawal[]): Promise<ExportSnapshot[]> {
if (!keys.length) return [];
@@ -61,10 +50,24 @@ export async function readExportPages(tx: BrainEngine, keys: ReadonlyArray<{ id:
if (!row) throw new Error('Export snapshot unexpectedly lost a selected page.');
return { page: rowToPage(row), tags: row.export_tags as string[] };
});
- const fenced = snapshots.filter(({ page }) => !isSegmentedPage(page) && (page.compiled_truth.includes('gbrain:facts:') || page.timeline.includes('gbrain:facts:')));
- const overlaid = await overlayCanonicalBodiesBatch(tx.executeRaw.bind(tx),
- fenced.map(({ page }) => ({ body: page.compiled_truth, timeline: page.timeline, withdrawals: applicableWithdrawals(page.slug, withdrawals) })));
- fenced.forEach(({ page }, i) => Object.assign(page, overlaid[i]));
+ const fenced = snapshots.flatMap(({ page }) => {
+ if (isSegmentedPage(page) || !(page.compiled_truth.includes('gbrain:facts:begin') || page.timeline.includes('gbrain:facts:begin'))) return [];
+ const applicable = applicableWithdrawals(page.slug, withdrawals);
+ return applicable.length ? [{ page, applicable }] : [];
+ });
+ if (!fenced.length) return snapshots;
+ const normalized = await tx.executeRaw<{ i: number; body: string | null; timeline: string | null }>(`SELECT t.i::int AS i,
+ (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)
+ FROM unnest(string_to_array(t.body,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS body,
+ (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)
+ FROM unnest(string_to_array(t.timeline,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS timeline
+ FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS t(body, timeline, i)`,
+ [fenced.map(({ page }) => page.compiled_truth), fenced.map(({ page }) => page.timeline)]);
+ for (const row of normalized) {
+ const { page, applicable } = fenced[Number(row.i) - 1]!;
+ page.compiled_truth = overlayWithdrawalBody(page.compiled_truth, row.body ?? '', applicable);
+ page.timeline = overlayWithdrawalBody(page.timeline, row.timeline ?? '', applicable);
+ }
return snapshots;
}
diff --git a/src/core/page-state/snapshot.ts b/src/core/page-state/snapshot.ts
index 71eaa32b42..2f6ce8aa1c 100644
--- a/src/core/page-state/snapshot.ts
+++ b/src/core/page-state/snapshot.ts
@@ -30,10 +30,6 @@ export async function resolveGlobalPurges(query: ReadQuery, sourceId: string, ma
return rows.map(r => typeof r.w === 'string' ? JSON.parse(r.w) as PageWithdrawal : r.w);
}
-/** The ledger fingerprint text of `text` (a SQL expression): each line lowercased, whitespace runs collapsed to one space. */
-const normalizedLines = (text: string) => `(SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)
- FROM unnest(string_to_array(${text},chr(10))) WITH ORDINALITY AS lines(line,ord))`;
-
/**
* Normalize filesystem bytes with the exact ledger fingerprint rules before comparing them. `global` names the
* snapshot's source and '*' purge marker, resolved against this text's fence rows (not the stored page's).
@@ -43,34 +39,17 @@ export async function overlayCanonicalBodies(query: ReadQuery, body: string, tim
const pending = global?.marker && global.marker.count > 0 ? global : undefined;
if (!withdrawals.length && !pending) return { compiled_truth: body, timeline };
if (!body.includes('gbrain:facts:begin') && !timeline.includes('gbrain:facts:begin')) return { compiled_truth: body, timeline };
- const [normalized] = await query<{ body: string; timeline: string }>(`SELECT ${normalizedLines('$1::text')} AS body, ${normalizedLines('$2::text')} AS timeline`, [body, timeline]);
+ const [normalized] = await query<{ body: string; timeline: string }>(`SELECT
+ (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)
+ FROM unnest(string_to_array($1::text,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS body,
+ (SELECT string_agg(regexp_replace(lower(line),'[[:space:]]+',' ','g'),chr(10) ORDER BY ord)
+ FROM unnest(string_to_array($2::text,chr(10))) WITH ORDINALITY AS lines(line,ord)) AS timeline`, [body, timeline]);
const resolved = pending ? await resolveGlobalPurges(query, pending.sourceId, pending.marker, [normalized.body ?? '', normalized.timeline ?? '']) : [];
const all = resolved.length ? [...withdrawals, ...resolved] : withdrawals;
return { compiled_truth: overlayWithdrawalBody(body, normalized.body ?? '', all),
timeline: overlayWithdrawalBody(timeline, normalized.timeline ?? '', all) };
}
-/**
- * overlayCanonicalBodies (without a '*' purge marker) of many texts: the texts that need it are normalized in one
- * statement, so a batch of pages costs one round trip instead of one per page. Results are in `items` order.
- */
-export async function overlayCanonicalBodiesBatch(query: ReadQuery, items: ReadonlyArray<{ body: string; timeline: string; withdrawals: PageWithdrawal[] }>):
- Promise<Array<{ compiled_truth: string; timeline: string }>> {
- const out = items.map(item => ({ compiled_truth: item.body, timeline: item.timeline }));
- const pending = items.flatMap((item, i) => item.withdrawals.length
- && (item.body.includes('gbrain:facts:begin') || item.timeline.includes('gbrain:facts:begin')) ? [i] : []);
- if (!pending.length) return out;
- const rows = await query<{ i: number; body: string | null; timeline: string | null }>(`SELECT t.i::int AS i, ${normalizedLines('t.body')} AS body,
- ${normalizedLines('t.timeline')} AS timeline FROM unnest($1::text[], $2::text[]) WITH ORDINALITY AS t(body, timeline, i)`,
- [pending.map(i => items[i]!.body), pending.map(i => items[i]!.timeline)]);
- for (const row of rows) {
- const at = pending[Number(row.i) - 1]!, item = items[at]!;
- out[at] = { compiled_truth: overlayWithdrawalBody(item.body, row.body ?? '', item.withdrawals),
- timeline: overlayWithdrawalBody(item.timeline, row.timeline ?? '', item.withdrawals) };
- }
- return out;
-}
-
/**
* One MVCC statement binds content, tags, identity and withdrawals to one revision.
* The normalized body and timeline fingerprints feed only the withdrawal overlay,
diff --git a/test/export-snapshot.test.ts b/test/export-snapshot.test.ts
index 2b79735fd7..ee3f7017da 100644
--- a/test/export-snapshot.test.ts
+++ b/test/export-snapshot.test.ts
@@ -11,6 +11,26 @@ import { importFromContent } from '../src/core/import-file.ts';
import { inlineExportMarkdown, readExportPage, readExportPages, readExportWithdrawals } from '../src/core/export-snapshot.ts';
import { overlayCanonicalBodies } from '../src/core/page-state/snapshot.ts';
import { requirePostgresTestDatabase, testBackends } from './helpers/test-backends.ts';
+import { rowToPage } from '../src/core/utils.ts';
+import { isSegmentedPage } from '../src/core/page-body/page-shape.ts';
+import type { ExportWithdrawal } from '../src/core/export-snapshot.ts';
+
+/** The per-page export read as it shipped before #7617, kept as the reference the batch read must reproduce byte for byte. */
+async function referenceExportPage(tx: BrainEngine, key: { id: string; source_id: string; slug: string }, withdrawals: ExportWithdrawal[]) {
+ const [row] = await tx.executeRaw<Record<string, unknown>>(`WITH export_page AS MATERIALIZED (
+ SELECT * FROM pages WHERE id=$1::bigint
+ ) SELECT p.*,
+ COALESCE((SELECT jsonb_agg(t.tag ORDER BY t.tag) FROM tags t WHERE t.page_id=p.id), '[]'::jsonb) AS export_tags
+ FROM export_page p WHERE p.source_id=$2 AND p.slug=$3 AND p.deleted_at IS NULL`, [key.id, key.source_id, key.slug]);
+ if (!row) throw new Error('Export snapshot unexpectedly lost a selected page.');
+ const page = rowToPage(row);
+ if (!isSegmentedPage(page) && (page.compiled_truth.includes('gbrain:facts:') || page.timeline.includes('gbrain:facts:'))) {
+ const applicable = withdrawals.filter(w => w.subject === '*' || w.subject === page.slug)
+ .map(({ visibility, fact_hash, withdrawn_at, purged }) => ({ visibility, fact_hash, withdrawn_at, ...(purged ? { purged: true } : {}) }));
+ Object.assign(page, await overlayCanonicalBodies(tx.executeRaw.bind(tx), page.compiled_truth, page.timeline, applicable));
+ }
+ return { page, tags: row.export_tags as string[] };
+}
for (const backend of testBackends()) describe(`coherent export: ${backend}`, () => {
let engine: BrainEngine;
@@ -82,7 +102,7 @@ for (const backend of testBackends()) describe(`coherent export: ${backend}`, ()
}
});
}, 60000);
- test('#7617: readExportPages gives every page exactly the snapshot readExportPage gives, in two statements', async () => {
+ test('#7617: readExportPages gives every page exactly the snapshot the per-page read gave, in two statements', async () => {
await engine.executeRaw('DELETE FROM pages');
await engine.executeRaw("DELETE FROM fact_withdrawals WHERE source_id='default'");
await engine.executeRaw("DELETE FROM fact_purges WHERE source_id='default'");
@@ -139,10 +159,11 @@ for (const backend of testBackends()) describe(`coherent export: ${backend}`, ()
expect(batch.length).toBe(keys.length);
let overlaid = 0;
for (const [i, key] of keys.entries()) {
- const single = await readExportPage(tx, key, withdrawals);
+ const single = await referenceExportPage(tx, key, withdrawals);
expect(batch[i]!.page.slug).toBe(key.slug);
expect(JSON.stringify(batch[i])).toBe(JSON.stringify(single));
expect(inlineExportMarkdown(batch[i]!)).toBe(inlineExportMarkdown(single));
+ expect(JSON.stringify(await readExportPage(tx, key, withdrawals))).toBe(JSON.stringify(single));
if ([single.page.compiled_truth, single.page.timeline].some(text => parseFactsFence(text).facts.some(fact => fact.active === false))) overlaid++;
}
// The overlay really ran: withdrawn rows read inactive in the fenced shapes and every fifth bulk page.