Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(export): look batched export reads up by primary key so a per-page export stays linear (#7618)

garrytan0a1e004d6c31
diff --git a/src/core/export-snapshot.ts b/src/core/export-snapshot.ts
index 219ed2a4e9..17d023b853 100644
--- a/src/core/export-snapshot.ts
+++ b/src/core/export-snapshot.ts
@@ -36,18 +36,21 @@ export async function readExportPage(tx: BrainEngine, key: { id: string; source_
* The export snapshots of `keys`, in key order, in at most two statements however many keys there are (#7617: a
* per-page read made a large source's conversion plan one round trip per page): the rows with their tags, then one
* fence normalization for every inline page whose withdrawal overlay applies, by overlayCanonicalBodies' rules and SQL.
+ * Rows are looked up by primary key (#7618: a join on the keys let the planner scan the whole source for each key, so a
+ * per-page export went quadratic), and each row must still carry its key's source and slug.
*/
export async function readExportPages(tx: BrainEngine, keys: ReadonlyArray<{ id: string; source_id: string; slug: string }>, withdrawals: ExportWithdrawal[]): Promise<ExportSnapshot[]> {
if (!keys.length) return [];
- const rows = await tx.executeRaw<Record<string, unknown>>(`SELECT p.*,
+ const rows = await tx.executeRaw<Record<string, unknown>>(`WITH export_page AS MATERIALIZED (
+ SELECT * FROM pages WHERE id = ANY($1::bigint[])
+ ) SELECT p.*,
COALESCE((SELECT jsonb_agg(t.tag ORDER BY t.tag) FROM tags t WHERE t.page_id=p.id), '[]'::jsonb) AS export_tags
- FROM unnest($1::bigint[], $2::text[], $3::text[]) AS k(id, source_id, slug)
- JOIN pages p ON p.id=k.id AND p.source_id=k.source_id AND p.slug=k.slug AND p.deleted_at IS NULL`,
- [keys.map(key => key.id), keys.map(key => key.source_id), keys.map(key => key.slug)]);
+ FROM export_page p WHERE p.deleted_at IS NULL`,
+ [keys.map(key => key.id)]);
const byId = new Map(rows.map(row => [String(row.id), row]));
const snapshots = keys.map(key => {
const row = byId.get(String(key.id));
- if (!row) throw new Error('Export snapshot unexpectedly lost a selected page.');
+ if (!row || row.source_id !== key.source_id || row.slug !== key.slug) throw new Error('Export snapshot unexpectedly lost a selected page.');
return { page: rowToPage(row), tags: row.export_tags as string[] };
});
const fenced = snapshots.flatMap(({ page }) => {
diff --git a/test/export-snapshot.test.ts b/test/export-snapshot.test.ts
index ee3f7017da..972367a3ef 100644
--- a/test/export-snapshot.test.ts
+++ b/test/export-snapshot.test.ts
@@ -177,6 +177,37 @@ for (const backend of testBackends()) describe(`coherent export: ${backend}`, ()
await expect(readExportPages(tx, [keys[0]!, deleted!], withdrawals)).rejects.toThrow('Export snapshot unexpectedly lost a selected page.');
});
}, 60000);
+ /**
+ * Value: protects=a per-page export read (export.ts calls readExportPage once per page) looks its row up by primary key,
+ * so exporting a large source stays linear; fails_when=readExportPages joins pages on the keys and the planner answers a
+ * one-key join from the source index, filtering every page of the source; seam=the plan of the statement itself.
+ */
+ test('#7618: a one-key export read looks its page up by primary key and examines no other page', async () => {
+ await engine.executeRaw('DELETE FROM pages');
+ await engine.executeRaw(`INSERT INTO pages(source_id,slug,type,title,compiled_truth,timeline,frontmatter)
+ SELECT 'default','scale/p'||n,'note','Synthetic','A synthetic body.','','{}'::jsonb FROM generate_series(1,5000) n`);
+ const [key] = await engine.executeRaw<{ id: string; source_id: string; slug: string }>("SELECT id::text AS id, source_id, slug FROM pages WHERE slug='scale/p2500'");
+ const real = engine.executeRaw;
+ let read: { query: string; params: unknown[] } | undefined;
+ engine.executeRaw = async function (query, params, options) {
+ if (!read && query.includes('export_tags')) read = { query, params: params ?? [] };
+ return real.call(this, query, params, options) as never;
+ };
+ try { expect((await readExportPages(engine, [key!], []))[0]!.page.slug).toBe('scale/p2500'); }
+ finally { engine.executeRaw = real; }
+ const [row] = await engine.executeRaw<Record<string, unknown>>(`EXPLAIN (ANALYZE, FORMAT JSON) ${read!.query}`, read!.params);
+ const raw = Object.values(row!)[0];
+ const nodes: Record<string, unknown>[] = [];
+ const walk = (node: Record<string, unknown>) => { nodes.push(node); for (const child of (node.Plans as Record<string, unknown>[] | undefined) ?? []) walk(child); };
+ walk(((typeof raw === 'string' ? JSON.parse(raw) : raw) as { Plan: Record<string, unknown> }[])[0]!.Plan);
+ const scans = nodes.filter(node => node['Relation Name'] === 'pages');
+ expect(scans.length).toBeGreaterThan(0);
+ for (const scan of scans) {
+ expect(scan['Index Name']).toBe('pages_pkey');
+ expect(Number(scan['Rows Removed by Filter'] ?? 0) + Number(scan['Rows Removed by Index Recheck'] ?? 0)).toBe(0);
+ }
+ }, 60000);
+
test('source/page/tag/raw changes after snapshot do not tear the export', async () => {
await engine.executeRaw('DELETE FROM pages');
await engine.executeRaw("INSERT INTO sources (id,name) VALUES ('snapshot-source','Snapshot') ON CONFLICT (id) DO UPDATE SET archived=false");