Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(export): look batched export reads up by primary key so a per-page export stays linear (#7618)
garrytan0a1e004d6c31
diff --git a/src/core/export-snapshot.ts b/src/core/export-snapshot.tsindex 219ed2a4e9..17d023b853 100644--- a/src/core/export-snapshot.ts+++ b/src/core/export-snapshot.ts@@ -36,18 +36,21 @@ export async function readExportPage(tx: BrainEngine, key: { id: string; source_* The export snapshots of `keys`, in key order, in at most two statements however many keys there are (#7617: a* per-page read made a large source's conversion plan one round trip per page): the rows with their tags, then one* fence normalization for every inline page whose withdrawal overlay applies, by overlayCanonicalBodies' rules and SQL.+ * Rows are looked up by primary key (#7618: a join on the keys let the planner scan the whole source for each key, so a+ * per-page export went quadratic), and each row must still carry its key's source and slug.*/export async function readExportPages(tx: BrainEngine, keys: ReadonlyArray<{ id: string; source_id: string; slug: string }>, withdrawals: ExportWithdrawal[]): Promise<ExportSnapshot[]> {if (!keys.length) return [];- const rows = await tx.executeRaw<Record<string, unknown>>(`SELECT p.*,+ const rows = await tx.executeRaw<Record<string, unknown>>(`WITH export_page AS MATERIALIZED (+ SELECT * FROM pages WHERE id = ANY($1::bigint[])+ ) SELECT p.*,COALESCE((SELECT jsonb_agg(t.tag ORDER BY t.tag) FROM tags t WHERE t.page_id=p.id), '[]'::jsonb) AS export_tags- FROM unnest($1::bigint[], $2::text[], $3::text[]) AS k(id, source_id, slug)- JOIN pages p ON p.id=k.id AND p.source_id=k.source_id AND p.slug=k.slug AND p.deleted_at IS NULL`,- [keys.map(key => key.id), keys.map(key => key.source_id), keys.map(key => key.slug)]);+ FROM export_page p WHERE p.deleted_at IS NULL`,+ [keys.map(key => key.id)]);const byId = new Map(rows.map(row => [String(row.id), row]));const snapshots = keys.map(key => {const row = byId.get(String(key.id));- if (!row) throw new Error('Export snapshot unexpectedly lost a selected page.');+ if (!row || row.source_id !== key.source_id || row.slug !== key.slug) throw new Error('Export snapshot unexpectedly lost a selected page.');return { page: rowToPage(row), tags: row.export_tags as string[] };});const fenced = snapshots.flatMap(({ page }) => {diff --git a/test/export-snapshot.test.ts b/test/export-snapshot.test.tsindex ee3f7017da..972367a3ef 100644--- a/test/export-snapshot.test.ts+++ b/test/export-snapshot.test.ts@@ -177,6 +177,37 @@ for (const backend of testBackends()) describe(`coherent export: ${backend}`, ()await expect(readExportPages(tx, [keys[0]!, deleted!], withdrawals)).rejects.toThrow('Export snapshot unexpectedly lost a selected page.');});}, 60000);+ /**+ * Value: protects=a per-page export read (export.ts calls readExportPage once per page) looks its row up by primary key,+ * so exporting a large source stays linear; fails_when=readExportPages joins pages on the keys and the planner answers a+ * one-key join from the source index, filtering every page of the source; seam=the plan of the statement itself.+ */+ test('#7618: a one-key export read looks its page up by primary key and examines no other page', async () => {+ await engine.executeRaw('DELETE FROM pages');+ await engine.executeRaw(`INSERT INTO pages(source_id,slug,type,title,compiled_truth,timeline,frontmatter)+ SELECT 'default','scale/p'||n,'note','Synthetic','A synthetic body.','','{}'::jsonb FROM generate_series(1,5000) n`);+ const [key] = await engine.executeRaw<{ id: string; source_id: string; slug: string }>("SELECT id::text AS id, source_id, slug FROM pages WHERE slug='scale/p2500'");+ const real = engine.executeRaw;+ let read: { query: string; params: unknown[] } | undefined;+ engine.executeRaw = async function (query, params, options) {+ if (!read && query.includes('export_tags')) read = { query, params: params ?? [] };+ return real.call(this, query, params, options) as never;+ };+ try { expect((await readExportPages(engine, [key!], []))[0]!.page.slug).toBe('scale/p2500'); }+ finally { engine.executeRaw = real; }+ const [row] = await engine.executeRaw<Record<string, unknown>>(`EXPLAIN (ANALYZE, FORMAT JSON) ${read!.query}`, read!.params);+ const raw = Object.values(row!)[0];+ const nodes: Record<string, unknown>[] = [];+ const walk = (node: Record<string, unknown>) => { nodes.push(node); for (const child of (node.Plans as Record<string, unknown>[] | undefined) ?? []) walk(child); };+ walk(((typeof raw === 'string' ? JSON.parse(raw) : raw) as { Plan: Record<string, unknown> }[])[0]!.Plan);+ const scans = nodes.filter(node => node['Relation Name'] === 'pages');+ expect(scans.length).toBeGreaterThan(0);+ for (const scan of scans) {+ expect(scan['Index Name']).toBe('pages_pkey');+ expect(Number(scan['Rows Removed by Filter'] ?? 0) + Number(scan['Rows Removed by Index Recheck'] ?? 0)).toBe(0);+ }+ }, 60000);+test('source/page/tag/raw changes after snapshot do not tear the export', async () => {await engine.executeRaw('DELETE FROM pages');await engine.executeRaw("INSERT INTO sources (id,name) VALUES ('snapshot-source','Snapshot') ON CONFLICT (id) DO UPDATE SET archived=false");