Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Plan durable-map scoped selects through their indexes (#2282)

Regan Bellfdcd8d0771fe
diff --git a/src/persistence/durable-map.ts b/src/persistence/durable-map.ts
index ab8563a10..fbc40891d 100644
--- a/src/persistence/durable-map.ts
+++ b/src/persistence/durable-map.ts
@@ -213,6 +213,14 @@ export function createPostgresMap<T>(
ON ${table} (id) WHERE json->>'${field}' ~ '[^\\x01-\\x7f]'`,
],
},
+ {
+ id: `durable-map/${table}/stats-${field.toLowerCase()}`,
+ statements: [
+ `SET LOCAL lock_timeout = '3s'`,
+ `CREATE STATISTICS IF NOT EXISTS ${table}_${field.toLowerCase()}_stats ON (json->>'${field}') FROM ${table}`,
+ `ANALYZE ${table}`,
+ ],
+ },
];
}),
];
@@ -282,14 +290,13 @@ export function createPostgresMap<T>(
}
let sql = `SELECT ${projection} - $1::text[] AS json FROM ${table}`;
if (query.where) {
- params.push(
- query.where.field,
- query.where.anyOfFold.map((v) => v.toLowerCase()),
- );
+ const field = query.where.field;
+ if (!/^[a-z_][a-z0-9_]*$/i.test(field)) throw new Error(`invalid select field: ${field}`);
+ params.push(query.where.anyOfFold.map((v) => v.toLowerCase()));
sql += ` WHERE id IN (
- SELECT id FROM ${table} WHERE lower(json->>$${params.length - 1}) = ANY($${params.length}::text[])
+ SELECT id FROM ${table} WHERE lower(json->>'${field}') = ANY($${params.length}::text[])
UNION
- SELECT id FROM ${table} WHERE json->>$${params.length - 1} ~ '[^\\x01-\\x7f]')`;
+ SELECT id FROM ${table} WHERE json->>'${field}' ~ '[^\\x01-\\x7f]')`;
}
if (query.afterId !== undefined) {
params.push(query.afterId);
diff --git a/test/durable-stores.test.ts b/test/durable-stores.test.ts
index a9e26b709..ab29d586b 100644
--- a/test/durable-stores.test.ts
+++ b/test/durable-stores.test.ts
@@ -179,16 +179,16 @@ test("the Postgres map select projects and filters inside SQL, never fetching th
assert.equal(reads.length, 1);
assert.match(
reads[0]!.sql,
- /^SELECT json - \$1::text\[\] AS json FROM select_probe WHERE id IN \(\s+SELECT id FROM select_probe WHERE lower\(json->>\$2\) = ANY\(\$3::text\[\]\)\s+UNION\s+SELECT id FROM select_probe WHERE json->>\$2 ~ '\[\^\\x01-\\x7f\]'\) ORDER BY id$/,
+ /^SELECT json - \$1::text\[\] AS json FROM select_probe WHERE id IN \(\s+SELECT id FROM select_probe WHERE lower\(json->>'owner'\) = ANY\(\$2::text\[\]\)\s+UNION\s+SELECT id FROM select_probe WHERE json->>'owner' ~ '\[\^\\x01-\\x7f\]'\) ORDER BY id$/,
);
- assert.deepEqual(reads[0]!.params, [["secretEnc"], "owner", ["u1", "[email protected]"]]);
+ assert.deepEqual(reads[0]!.params, [["secretEnc"], ["u1", "[email protected]"]]);
await map.select({});
assert.match(reads[1]!.sql, /^SELECT json - \$1::text\[\] AS json FROM select_probe ORDER BY id$/);
assert.deepEqual(reads[1]!.params, [[]]);
await map.select({ where: { field: "owner", anyOfFold: ["U1"] }, limit: 16, afterId: "previous" });
- assert.match(reads[2]!.sql, /\) AND id > \$4 ORDER BY id LIMIT \$5$/);
- assert.deepEqual(reads[2]!.params, [[], "owner", ["u1"], "previous", 16]);
+ assert.match(reads[2]!.sql, /\) AND id > \$3 ORDER BY id LIMIT \$4$/);
+ assert.deepEqual(reads[2]!.params, [[], ["u1"], "previous", 16]);
});
test("the Postgres map reads with ORDER BY id — heap order is not a contract", async () => {
diff --git a/test/postgres-map.test.ts b/test/postgres-map.test.ts
index 5af7d6017..b256eb03c 100644
--- a/test/postgres-map.test.ts
+++ b/test/postgres-map.test.ts
@@ -25,7 +25,7 @@ before(async () => {
const p = new pg.Pool({ connectionString: URL });
await p.query("DROP TABLE IF EXISTS qm_schema_migrations CASCADE");
await p.query(
- "DROP TABLE IF EXISTS map_webhooks, map_webhook_history, map_widgets, map_wide_widgets, map_crons, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",
+ "DROP TABLE IF EXISTS map_webhooks, map_webhook_history, map_widgets, map_wide_widgets, map_scoped_widgets, map_crons, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",
);
await p.end();
});
@@ -467,3 +467,58 @@ for (const postgres of [false, true]) {
},
);
}
+
+test("pg map: an indexed select plans both branches through their indexes", { skip }, async () => {
+ const factory = createPostgresMapFactory(URL!);
+ const pg = factory.pool;
+ const indexed = factory.map<{ owner: string; blob: string }>("map_scoped_widgets", ["owner"]);
+ await indexed.put("seed", { owner: "seed", blob: "" });
+ await pg.q(
+ `INSERT INTO map_scoped_widgets (id, json)
+ SELECT 'w' || g, jsonb_build_object('owner', 'owner-' || (g % 40), 'blob', repeat(md5(g::text), 200))
+ FROM generate_series(1, 4000) g`,
+ );
+ await pg.q("ANALYZE map_scoped_widgets");
+ const captured: Array<{ sql: string; params: unknown[] }> = [];
+ const original = pg.q.bind(pg);
+ pg.q = (async (sql: string, params?: unknown[]) => {
+ captured.push({ sql, params: params ?? [] });
+ return original(sql, params);
+ }) as typeof pg.q;
+ try {
+ const rows = await indexed.select({ omit: ["blob"], where: { field: "owner", anyOfFold: ["OWNER-7"] } });
+ assert.equal(rows.length, 100);
+ const select = captured.find((call) => call.sql.includes("ANY("));
+ assert.ok(select, "the select ran through the pool");
+ pg.q = original;
+ for (const mode of ["force_custom_plan", "force_generic_plan"]) {
+ const client = await (await pg.pool()).connect();
+ try {
+ await client.query(`SET plan_cache_mode = ${mode}`);
+ await client.query(`PREPARE scoped_select AS ${select.sql}`);
+ const args = select.params.map((value) =>
+ Array.isArray(value)
+ ? `ARRAY[${value.map((v: string) => client.escapeLiteral(v)).join(", ")}]::text[]`
+ : client.escapeLiteral(String(value)),
+ );
+ const [{ "QUERY PLAN": plan }] = (
+ await client.query(`EXPLAIN (FORMAT JSON) EXECUTE scoped_select(${args.join(", ")})`)
+ ).rows;
+ const filtered: string[] = [];
+ const walk = (node: Record<string, unknown>) => {
+ if (node["Node Type"] === "Seq Scan" && typeof node["Filter"] === "string") filtered.push(node["Filter"]);
+ for (const child of (node["Plans"] as Record<string, unknown>[] | undefined) ?? []) walk(child);
+ };
+ walk(plan[0].Plan);
+ assert.deepEqual(filtered, [], `${mode}: no branch filters by sequential scan`);
+ } finally {
+ await client.query("DEALLOCATE ALL");
+ client.release();
+ }
+ }
+ } finally {
+ pg.q = original;
+ await pg.q("DROP TABLE IF EXISTS map_scoped_widgets");
+ await factory.pool.close();
+ }
+});