Multiplayer agent harness for work.
Plan durable-map scoped selects through their indexes (#2282)
Regan Bellfdcd8d0771fe
diff --git a/src/persistence/durable-map.ts b/src/persistence/durable-map.tsindex ab8563a10..fbc40891d 100644--- a/src/persistence/durable-map.ts+++ b/src/persistence/durable-map.ts@@ -213,6 +213,14 @@ export function createPostgresMap<T>(ON ${table} (id) WHERE json->>'${field}' ~ '[^\\x01-\\x7f]'`,],},+ {+ id: `durable-map/${table}/stats-${field.toLowerCase()}`,+ statements: [+ `SET LOCAL lock_timeout = '3s'`,+ `CREATE STATISTICS IF NOT EXISTS ${table}_${field.toLowerCase()}_stats ON (json->>'${field}') FROM ${table}`,+ `ANALYZE ${table}`,+ ],+ },];}),];@@ -282,14 +290,13 @@ export function createPostgresMap<T>(}let sql = `SELECT ${projection} - $1::text[] AS json FROM ${table}`;if (query.where) {- params.push(- query.where.field,- query.where.anyOfFold.map((v) => v.toLowerCase()),- );+ const field = query.where.field;+ if (!/^[a-z_][a-z0-9_]*$/i.test(field)) throw new Error(`invalid select field: ${field}`);+ params.push(query.where.anyOfFold.map((v) => v.toLowerCase()));sql += ` WHERE id IN (- SELECT id FROM ${table} WHERE lower(json->>$${params.length - 1}) = ANY($${params.length}::text[])+ SELECT id FROM ${table} WHERE lower(json->>'${field}') = ANY($${params.length}::text[])UNION- SELECT id FROM ${table} WHERE json->>$${params.length - 1} ~ '[^\\x01-\\x7f]')`;+ SELECT id FROM ${table} WHERE json->>'${field}' ~ '[^\\x01-\\x7f]')`;}if (query.afterId !== undefined) {params.push(query.afterId);diff --git a/test/durable-stores.test.ts b/test/durable-stores.test.tsindex a9e26b709..ab29d586b 100644--- a/test/durable-stores.test.ts+++ b/test/durable-stores.test.ts@@ -179,16 +179,16 @@ test("the Postgres map select projects and filters inside SQL, never fetching thassert.equal(reads.length, 1);assert.match(reads[0]!.sql,- /^SELECT json - \$1::text\[\] AS json FROM select_probe WHERE id IN \(\s+SELECT id FROM select_probe WHERE lower\(json->>\$2\) = ANY\(\$3::text\[\]\)\s+UNION\s+SELECT id FROM select_probe WHERE json->>\$2 ~ '\[\^\\x01-\\x7f\]'\) ORDER BY id$/,+ /^SELECT json - \$1::text\[\] AS json FROM select_probe WHERE id IN \(\s+SELECT id FROM select_probe WHERE lower\(json->>'owner'\) = ANY\(\$2::text\[\]\)\s+UNION\s+SELECT id FROM select_probe WHERE json->>'owner' ~ '\[\^\\x01-\\x7f\]'\) ORDER BY id$/,);- assert.deepEqual(reads[0]!.params, [["secretEnc"], "owner", ["u1", "[email protected]"]]);+ assert.deepEqual(reads[0]!.params, [["secretEnc"], ["u1", "[email protected]"]]);await map.select({});assert.match(reads[1]!.sql, /^SELECT json - \$1::text\[\] AS json FROM select_probe ORDER BY id$/);assert.deepEqual(reads[1]!.params, [[]]);await map.select({ where: { field: "owner", anyOfFold: ["U1"] }, limit: 16, afterId: "previous" });- assert.match(reads[2]!.sql, /\) AND id > \$4 ORDER BY id LIMIT \$5$/);- assert.deepEqual(reads[2]!.params, [[], "owner", ["u1"], "previous", 16]);+ assert.match(reads[2]!.sql, /\) AND id > \$3 ORDER BY id LIMIT \$4$/);+ assert.deepEqual(reads[2]!.params, [[], ["u1"], "previous", 16]);});test("the Postgres map reads with ORDER BY id — heap order is not a contract", async () => {diff --git a/test/postgres-map.test.ts b/test/postgres-map.test.tsindex 5af7d6017..b256eb03c 100644--- a/test/postgres-map.test.ts+++ b/test/postgres-map.test.ts@@ -25,7 +25,7 @@ before(async () => {const p = new pg.Pool({ connectionString: URL });await p.query("DROP TABLE IF EXISTS qm_schema_migrations CASCADE");await p.query(- "DROP TABLE IF EXISTS map_webhooks, map_webhook_history, map_widgets, map_wide_widgets, map_crons, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",+ "DROP TABLE IF EXISTS map_webhooks, map_webhook_history, map_widgets, map_wide_widgets, map_scoped_widgets, map_crons, map_keychain_creds, map_keychain_grants, map_keychain_asks, process_sessions, durable_map_versions CASCADE",);await p.end();});@@ -467,3 +467,58 @@ for (const postgres of [false, true]) {},);}++test("pg map: an indexed select plans both branches through their indexes", { skip }, async () => {+ const factory = createPostgresMapFactory(URL!);+ const pg = factory.pool;+ const indexed = factory.map<{ owner: string; blob: string }>("map_scoped_widgets", ["owner"]);+ await indexed.put("seed", { owner: "seed", blob: "" });+ await pg.q(+ `INSERT INTO map_scoped_widgets (id, json)+ SELECT 'w' || g, jsonb_build_object('owner', 'owner-' || (g % 40), 'blob', repeat(md5(g::text), 200))+ FROM generate_series(1, 4000) g`,+ );+ await pg.q("ANALYZE map_scoped_widgets");+ const captured: Array<{ sql: string; params: unknown[] }> = [];+ const original = pg.q.bind(pg);+ pg.q = (async (sql: string, params?: unknown[]) => {+ captured.push({ sql, params: params ?? [] });+ return original(sql, params);+ }) as typeof pg.q;+ try {+ const rows = await indexed.select({ omit: ["blob"], where: { field: "owner", anyOfFold: ["OWNER-7"] } });+ assert.equal(rows.length, 100);+ const select = captured.find((call) => call.sql.includes("ANY("));+ assert.ok(select, "the select ran through the pool");+ pg.q = original;+ for (const mode of ["force_custom_plan", "force_generic_plan"]) {+ const client = await (await pg.pool()).connect();+ try {+ await client.query(`SET plan_cache_mode = ${mode}`);+ await client.query(`PREPARE scoped_select AS ${select.sql}`);+ const args = select.params.map((value) =>+ Array.isArray(value)+ ? `ARRAY[${value.map((v: string) => client.escapeLiteral(v)).join(", ")}]::text[]`+ : client.escapeLiteral(String(value)),+ );+ const [{ "QUERY PLAN": plan }] = (+ await client.query(`EXPLAIN (FORMAT JSON) EXECUTE scoped_select(${args.join(", ")})`)+ ).rows;+ const filtered: string[] = [];+ const walk = (node: Record<string, unknown>) => {+ if (node["Node Type"] === "Seq Scan" && typeof node["Filter"] === "string") filtered.push(node["Filter"]);+ for (const child of (node["Plans"] as Record<string, unknown>[] | undefined) ?? []) walk(child);+ };+ walk(plan[0].Plan);+ assert.deepEqual(filtered, [], `${mode}: no branch filters by sequential scan`);+ } finally {+ await client.query("DEALLOCATE ALL");+ client.release();+ }+ }+ } finally {+ pg.q = original;+ await pg.q("DROP TABLE IF EXISTS map_scoped_widgets");+ await factory.pool.close();+ }+});