Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
[db-perf] bind caller source ids in accessible-items and soup access gates (#9583)
teo07673da86673
diff --git a/.sqlx/query-09467ab0f1282fa21cdbc66fb01b7ab90a55d8b528699b9a88ce869dd45e4a66.json b/.sqlx/query-09467ab0f1282fa21cdbc66fb01b7ab90a55d8b528699b9a88ce869dd45e4a66.jsondeleted file mode 100644index 4cb61c05aa..0000000000--- a/.sqlx/query-09467ab0f1282fa21cdbc66fb01b7ab90a55d8b528699b9a88ce869dd45e4a66.json+++ /dev/null@@ -1,23 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n WITH user_source_ids AS (\n SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp\n WHERE cp.user_id = $1 AND cp.left_at IS NULL\n UNION ALL\n SELECT t.team_id::text FROM team_user t\n WHERE t.user_id = $1\n UNION ALL\n SELECT $1\n )\n SELECT DISTINCT ea.entity_id::text as \"item_id!\"\n FROM entity_access ea\n JOIN user_source_ids us ON us.source_id = ea.source_id\n JOIN \"Chat\" c ON c.id = ea.entity_id::text\n WHERE ea.entity_type = 'chat'\n AND c.\"deletedAt\" IS NULL\n AND (NOT $2 OR c.\"userId\" != $1)\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "item_id!",- "type_info": "Text"- }- ],- "parameters": {- "Left": [- "Text",- "Bool"- ]- },- "nullable": [- null- ]- },- "hash": "09467ab0f1282fa21cdbc66fb01b7ab90a55d8b528699b9a88ce869dd45e4a66"-}diff --git a/.sqlx/query-7e6bf8c61219e907f25c233be0c8dfd8a02cd84b2ce6746d6a16823cb52ae7de.json b/.sqlx/query-7e6bf8c61219e907f25c233be0c8dfd8a02cd84b2ce6746d6a16823cb52ae7de.jsonnew file mode 100644index 0000000000..fcec8ca8e2--- /dev/null+++ b/.sqlx/query-7e6bf8c61219e907f25c233be0c8dfd8a02cd84b2ce6746d6a16823cb52ae7de.json@@ -0,0 +1,24 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT c.id\n FROM \"Chat\" c\n WHERE c.id IN (\n SELECT ea.entity_id::text\n FROM entity_access ea\n WHERE ea.source_id = ANY($2)\n AND ea.entity_type = 'chat'\n )\n AND c.\"deletedAt\" IS NULL\n AND (NOT $3 OR c.\"userId\" != $1)\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "TextArray",+ "Bool"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "7e6bf8c61219e907f25c233be0c8dfd8a02cd84b2ce6746d6a16823cb52ae7de"+}diff --git a/.sqlx/query-a5ce3e2825e32372811a8536ddac2e8a45e3ed323cebff87e9df0ecad67f3256.json b/.sqlx/query-a5ce3e2825e32372811a8536ddac2e8a45e3ed323cebff87e9df0ecad67f3256.jsondeleted file mode 100644index 85b240cea3..0000000000--- a/.sqlx/query-a5ce3e2825e32372811a8536ddac2e8a45e3ed323cebff87e9df0ecad67f3256.json+++ /dev/null@@ -1,23 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n WITH user_source_ids AS (\n SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp\n WHERE cp.user_id = $1 AND cp.left_at IS NULL\n UNION ALL\n SELECT t.team_id::text FROM team_user t\n WHERE t.user_id = $1\n UNION ALL\n SELECT $1\n )\n SELECT DISTINCT ea.entity_id::text as \"item_id!\"\n FROM entity_access ea\n JOIN user_source_ids us ON us.source_id = ea.source_id\n JOIN \"Document\" d ON d.id = ea.entity_id::text\n WHERE ea.entity_type = 'document'\n AND d.\"deletedAt\" IS NULL\n AND (NOT $2 OR d.owner != $1)\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "item_id!",- "type_info": "Text"- }- ],- "parameters": {- "Left": [- "Text",- "Bool"- ]- },- "nullable": [- null- ]- },- "hash": "a5ce3e2825e32372811a8536ddac2e8a45e3ed323cebff87e9df0ecad67f3256"-}diff --git a/.sqlx/query-aa949da30a88081b069a571074e847c0269acd89d23b4921e6010795355989ff.json b/.sqlx/query-aa949da30a88081b069a571074e847c0269acd89d23b4921e6010795355989ff.jsonnew file mode 100644index 0000000000..1314307074--- /dev/null+++ b/.sqlx/query-aa949da30a88081b069a571074e847c0269acd89d23b4921e6010795355989ff.json@@ -0,0 +1,24 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT p.id\n FROM \"Project\" p\n WHERE p.id IN (\n SELECT ea.entity_id::text\n FROM entity_access ea\n WHERE ea.source_id = ANY($2)\n AND ea.entity_type = 'project'\n )\n AND p.\"deletedAt\" IS NULL\n AND (NOT $3 OR p.\"userId\" != $1)\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "TextArray",+ "Bool"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "aa949da30a88081b069a571074e847c0269acd89d23b4921e6010795355989ff"+}diff --git a/.sqlx/query-b48f3da320797f3254c2f0e88442261ba53a69dfaa2ba97e4ba581fe6098e52c.json b/.sqlx/query-b48f3da320797f3254c2f0e88442261ba53a69dfaa2ba97e4ba581fe6098e52c.jsondeleted file mode 100644index 7d62b4ae05..0000000000--- a/.sqlx/query-b48f3da320797f3254c2f0e88442261ba53a69dfaa2ba97e4ba581fe6098e52c.json+++ /dev/null@@ -1,23 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n WITH user_source_ids AS (\n SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp\n WHERE cp.user_id = $1 AND cp.left_at IS NULL\n UNION ALL\n SELECT t.team_id::text FROM team_user t\n WHERE t.user_id = $1\n UNION ALL\n SELECT $1\n )\n SELECT DISTINCT ea.entity_id::text as \"item_id!\"\n FROM entity_access ea\n JOIN user_source_ids us ON us.source_id = ea.source_id\n JOIN \"Project\" p ON p.id = ea.entity_id::text\n WHERE ea.entity_type = 'project'\n AND p.\"deletedAt\" IS NULL\n AND (NOT $2 OR p.\"userId\" != $1)\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "item_id!",- "type_info": "Text"- }- ],- "parameters": {- "Left": [- "Text",- "Bool"- ]- },- "nullable": [- null- ]- },- "hash": "b48f3da320797f3254c2f0e88442261ba53a69dfaa2ba97e4ba581fe6098e52c"-}diff --git a/.sqlx/query-c3359b2f83080c2f8f2af75564d72ffb27e79fa9a5ca85134c085a6d8cac41fd.json b/.sqlx/query-c3359b2f83080c2f8f2af75564d72ffb27e79fa9a5ca85134c085a6d8cac41fd.jsonnew file mode 100644index 0000000000..eed624920e--- /dev/null+++ b/.sqlx/query-c3359b2f83080c2f8f2af75564d72ffb27e79fa9a5ca85134c085a6d8cac41fd.json@@ -0,0 +1,24 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT d.id\n FROM \"Document\" d\n WHERE d.id IN (\n SELECT ea.entity_id::text\n FROM entity_access ea\n WHERE ea.source_id = ANY($2)\n AND ea.entity_type = 'document'\n )\n AND d.\"deletedAt\" IS NULL\n AND (NOT $3 OR d.owner != $1)\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "TextArray",+ "Bool"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "c3359b2f83080c2f8f2af75564d72ffb27e79fa9a5ca85134c085a6d8cac41fd"+}diff --git a/.sqlx/query-cd1dde0000615f78a4eb075ef98aec58d9a2e13d9f0bff9a20159512c8aee618.json b/.sqlx/query-cd1dde0000615f78a4eb075ef98aec58d9a2e13d9f0bff9a20159512c8aee618.jsonnew file mode 100644index 0000000000..b54c7a1b81--- /dev/null+++ b/.sqlx/query-cd1dde0000615f78a4eb075ef98aec58d9a2e13d9f0bff9a20159512c8aee618.json@@ -0,0 +1,12 @@+{+ "db_name": "PostgreSQL",+ "query": "SET LOCAL parallel_setup_cost = 10000",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": []+ },+ "nullable": []+ },+ "hash": "cd1dde0000615f78a4eb075ef98aec58d9a2e13d9f0bff9a20159512c8aee618"+}diff --git a/.sqlx/query-efd03310692ee32d62240b82e9d2d533cd0e0ecb7b5cb99b808b21da799ab4bc.json b/.sqlx/query-efd03310692ee32d62240b82e9d2d533cd0e0ecb7b5cb99b808b21da799ab4bc.jsonnew file mode 100644index 0000000000..810859dcbc--- /dev/null+++ b/.sqlx/query-efd03310692ee32d62240b82e9d2d533cd0e0ecb7b5cb99b808b21da799ab4bc.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT cp.channel_id::text FROM comms_channel_participants cp\n WHERE cp.user_id = $1 AND cp.left_at IS NULL\n UNION ALL\n SELECT t.team_id::text FROM team_user t\n WHERE t.user_id = $1\n UNION ALL\n SELECT $1\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "channel_id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Text"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "efd03310692ee32d62240b82e9d2d533cd0e0ecb7b5cb99b808b21da799ab4bc"+}diff --git a/crates/macro_db_client/src/item_access/get_accessible_items.rs b/crates/macro_db_client/src/item_access/get_accessible_items.rsindex a6c91894a8..231c038abf 100644--- a/crates/macro_db_client/src/item_access/get_accessible_items.rs+++ b/crates/macro_db_client/src/item_access/get_accessible_items.rs@@ -23,15 +23,16 @@ pub async fn get_user_accessible_items(item_type_filter: Option<String>,exclude_owned: bool,) -> anyhow::Result<Vec<UserAccessibleItem>> {+ let source_ids = user_source_ids(db, user_id).await?;match item_type_filter.as_deref() {- Some("document") => accessible_documents(db, user_id, exclude_owned).await,- Some("chat") => accessible_chats(db, user_id, exclude_owned).await,- Some("project") => accessible_projects(db, user_id, exclude_owned).await,+ Some("document") => accessible_documents(db, user_id, &source_ids, exclude_owned).await,+ Some("chat") => accessible_chats(db, user_id, &source_ids, exclude_owned).await,+ Some("project") => accessible_projects(db, user_id, &source_ids, exclude_owned).await,Some(_) => Ok(vec![]),None => {- let mut items = accessible_documents(db, user_id, exclude_owned).await?;- items.extend(accessible_chats(db, user_id, exclude_owned).await?);- items.extend(accessible_projects(db, user_id, exclude_owned).await?);+ let mut items = accessible_documents(db, user_id, &source_ids, exclude_owned).await?;+ items.extend(accessible_chats(db, user_id, &source_ids, exclude_owned).await?);+ items.extend(accessible_projects(db, user_id, &source_ids, exclude_owned).await?);Ok(items)}}@@ -46,33 +47,60 @@ fn to_items(ids: Vec<String>, item_type: &str) -> Vec<UserAccessibleItem> {.collect()}+/// The user's active channels, team, and the user id itself: every+/// `entity_access.source_id` that grants them access.+///+/// The per-type queries bind these as a constant `source_id = ANY($2)` rather+/// than joining the same union as a CTE. Postgres costs a CTE join at the+/// average source's grant count, so it planned users holding hundreds of+/// thousands of grants with one item-table probe per grant; a constant array+/// is estimated from `entity_access.source_id` statistics, which picks a hash+/// join for them. The per-type queries run as unnamed statements because a+/// cached prepared statement switches every user to one generic plan after+/// five calls.+async fn user_source_ids(db: &Pool<Postgres>, user_id: &str) -> sqlx::Result<Vec<String>> {+ let source_ids = sqlx::query_scalar!(+ r#"+ SELECT cp.channel_id::text FROM comms_channel_participants cp+ WHERE cp.user_id = $1 AND cp.left_at IS NULL+ UNION ALL+ SELECT t.team_id::text FROM team_user t+ WHERE t.user_id = $1+ UNION ALL+ SELECT $1+ "#,+ user_id+ )+ .fetch_all(db)+ .await?;++ Ok(source_ids.into_iter().flatten().collect())+}+async fn accessible_documents(db: &Pool<Postgres>,user_id: &str,+ source_ids: &[String],exclude_owned: bool,) -> anyhow::Result<Vec<UserAccessibleItem>> {let ids = sqlx::query_scalar!(r#"- WITH user_source_ids AS (- SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp- WHERE cp.user_id = $1 AND cp.left_at IS NULL- UNION ALL- SELECT t.team_id::text FROM team_user t- WHERE t.user_id = $1- UNION ALL- SELECT $1- )- SELECT DISTINCT ea.entity_id::text as "item_id!"- FROM entity_access ea- JOIN user_source_ids us ON us.source_id = ea.source_id- JOIN "Document" d ON d.id = ea.entity_id::text- WHERE ea.entity_type = 'document'+ SELECT d.id+ FROM "Document" d+ WHERE d.id IN (+ SELECT ea.entity_id::text+ FROM entity_access ea+ WHERE ea.source_id = ANY($2)+ AND ea.entity_type = 'document'+ )AND d."deletedAt" IS NULL- AND (NOT $2 OR d.owner != $1)+ AND (NOT $3 OR d.owner != $1)"#,user_id,+ source_ids,exclude_owned)+ .persistent(false).fetch_all(db).await?;@@ -82,30 +110,27 @@ async fn accessible_documents(async fn accessible_chats(db: &Pool<Postgres>,user_id: &str,+ source_ids: &[String],exclude_owned: bool,) -> anyhow::Result<Vec<UserAccessibleItem>> {let ids = sqlx::query_scalar!(r#"- WITH user_source_ids AS (- SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp- WHERE cp.user_id = $1 AND cp.left_at IS NULL- UNION ALL- SELECT t.team_id::text FROM team_user t- WHERE t.user_id = $1- UNION ALL- SELECT $1- )- SELECT DISTINCT ea.entity_id::text as "item_id!"- FROM entity_access ea- JOIN user_source_ids us ON us.source_id = ea.source_id- JOIN "Chat" c ON c.id = ea.entity_id::text- WHERE ea.entity_type = 'chat'+ SELECT c.id+ FROM "Chat" c+ WHERE c.id IN (+ SELECT ea.entity_id::text+ FROM entity_access ea+ WHERE ea.source_id = ANY($2)+ AND ea.entity_type = 'chat'+ )AND c."deletedAt" IS NULL- AND (NOT $2 OR c."userId" != $1)+ AND (NOT $3 OR c."userId" != $1)"#,user_id,+ source_ids,exclude_owned)+ .persistent(false).fetch_all(db).await?;@@ -115,30 +140,27 @@ async fn accessible_chats(async fn accessible_projects(db: &Pool<Postgres>,user_id: &str,+ source_ids: &[String],exclude_owned: bool,) -> anyhow::Result<Vec<UserAccessibleItem>> {let ids = sqlx::query_scalar!(r#"- WITH user_source_ids AS (- SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp- WHERE cp.user_id = $1 AND cp.left_at IS NULL- UNION ALL- SELECT t.team_id::text FROM team_user t- WHERE t.user_id = $1- UNION ALL- SELECT $1- )- SELECT DISTINCT ea.entity_id::text as "item_id!"- FROM entity_access ea- JOIN user_source_ids us ON us.source_id = ea.source_id- JOIN "Project" p ON p.id = ea.entity_id::text- WHERE ea.entity_type = 'project'+ SELECT p.id+ FROM "Project" p+ WHERE p.id IN (+ SELECT ea.entity_id::text+ FROM entity_access ea+ WHERE ea.source_id = ANY($2)+ AND ea.entity_type = 'project'+ )AND p."deletedAt" IS NULL- AND (NOT $2 OR p."userId" != $1)+ AND (NOT $3 OR p."userId" != $1)"#,user_id,+ source_ids,exclude_owned)+ .persistent(false).fetch_all(db).await?;diff --git a/crates/soup/AGENTS.md b/crates/soup/AGENTS.mdindex 83d6326c25..fc9e27dc0d 100644--- a/crates/soup/AGENTS.md+++ b/crates/soup/AGENTS.md@@ -107,7 +107,8 @@ rows, initiatives, ...) and get back a single unified, paginated result.- Preserve projection-backed hydration, the UUID-keyed item/fact pools, and per-group total/has-more metadata through the shared grouped response.- Flat property sorts omit calendar events, email, channels, calls, companies, and agent sessions; grouped reads retain calendar events and timestamp queries retain their existing sources.- Preserve deployed timestamp cursor encodings; client query keys restart pages when ranges or selections change.-- Keep flat timestamp top clauses lightweight; bind document-ID arrays after the sort's fixed parameters and skip subtype namespace joins when the filter proves TASK-only unless other predicates need them.+- Keep flat timestamp top clauses lightweight; bind the caller's source ids, then document-ID arrays, after the sort's fixed parameters and skip subtype namespace joins when the filter proves TASK-only unless other predicates need them.+- The dynamic query's document, chat, and project arms match `entity_access.source_id = ANY(<bound source ids>)`, not a join to the `user_source_ids` CTE: Postgres costs a CTE join at the average source's grant count and probed the item table once per grant for callers holding hundreds of thousands. The query runs in a transaction that raises `parallel_setup_cost`, because parallel plans made callers with a few thousand grants twice as slow.- Read grouping/sort definition types through uncorrelated scalar lookups; join grouping options once outside the value expansion and preserve missing-option groups.- Build option-sort order maps with an uncorrelated `jsonb_object_agg` lookup using the sort definition bind; ignore unknown option IDs and keep unknown-only values missing.- Materialize grouped `TopItems` only for property sorts so the sort key is computed before multi-value group expansion; leave timestamp CTEs inlineable.diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rsindex da98412142..da85db64c5 100644--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs@@ -46,7 +46,10 @@ use models_soup::{project::SoupProject,};use recursion::CollapsibleExt;-use sqlx::{PgPool, Postgres, QueryBuilder, Row, postgres::PgRow, prelude::FromRow};+use sqlx::{+ PgConnection, PgPool, Postgres, QueryBuilder, Row, Transaction, postgres::PgRow,+ prelude::FromRow,+};use system_properties::{StatusOption, SystemPropertyKey};use tracing::Instrument;use uuid::Uuid;@@ -95,8 +98,8 @@ fn sort_key_expression(enabled: bool) -> &'static str {}}-/// First document-ID array after the existing query parameters and optional property sort.-fn first_document_id_slot(sort: DynamicSort, grouped: bool) -> usize {+/// The caller's source-id array, after the existing query parameters and optional property sort.+fn source_ids_slot(sort: DynamicSort, grouped: bool) -> usize {let fixed_parameters = if grouped { 10 } else { 8 };let property_parameters = if matches!(sort, DynamicSort::Property { .. }) {2@@ -106,6 +109,11 @@ fn first_document_id_slot(sort: DynamicSort, grouped: bool) -> usize {fixed_parameters + property_parameters + 1}+/// First document-ID array, after the caller's source ids.+fn first_document_id_slot(sort: DynamicSort, grouped: bool) -> usize {+ source_ids_slot(sort, grouped) + 1+}+// The non-null and missing-key branches both follow SQL's NULLS LAST order.fn property_keyset(sort: DynamicSort,@@ -186,6 +194,29 @@ static PREFIX: &str = r#"),"#;+/// The `user_source_ids` rows, bound into the document, chat, and project+/// access gates by [`bound_access_semi_join`].+async fn fetch_user_source_ids(+ connection: &mut PgConnection,+ user_id: &str,+) -> Result<Vec<String>, sqlx::Error> {+ let source_ids = sqlx::query_scalar!(+ r#"+ SELECT cp.channel_id::text FROM comms_channel_participants cp+ WHERE cp.user_id = $1 AND cp.left_at IS NULL+ UNION ALL+ SELECT t.team_id::text FROM team_user t+ WHERE t.user_id = $1+ UNION ALL+ SELECT $1+ "#,+ user_id+ )+ .fetch_all(connection)+ .await?;+ Ok(source_ids.into_iter().flatten().collect())+}+// -- Lightweight top clauses: only id + sort_ts (plus filter-required joins) --static DOCUMENT_TASK_PROPERTY_JOINS: &str = r#"@@ -1614,7 +1645,23 @@ pub(in crate::outbound::pg_soup_repo) fn access_semi_join()}-fn document_top_where_clause(sort_method: SimpleSortMethod) -> String {+/// [`access_semi_join`] over the caller's source ids, bound as a constant+/// array at parameter `source_ids`. Postgres costs the `user_source_ids` join at the average+/// source's grant count, so it planned callers holding hundreds of thousands+/// of grants with one item probe per grant; it estimates the array from+/// `entity_access.source_id` statistics and hashes the grants instead.+fn bound_access_semi_join(id_sql: &str, entity_type: &str, source_ids: usize) -> String {+ format!(+ r#"{id_sql} IN (+ SELECT ea.entity_id::text+ FROM entity_access ea+ WHERE ea.source_id = ANY(${source_ids}::text[])+ AND ea.entity_type = '{entity_type}'+ )"#+ )+}++fn document_top_where_clause(sort_method: SimpleSortMethod, source_ids: usize) -> String {let user_history_join = if top_needs_user_history(sort_method) {r#" LEFT JOIN "UserHistory" uh ON uh."itemId" = d.id AND uh."itemType" = 'document' AND uh."userId" = $1"#@@ -1625,25 +1672,25 @@ fn document_top_where_clause(sort_method: SimpleSortMethod) -> String {r#"{user_history_join} WHERE d."deletedAt" IS NULLAND {}"#,- access_semi_join("d.id", "document")+ bound_access_semi_join("d.id", "document", source_ids))}-fn chat_top_where_clause() -> String {+fn chat_top_where_clause(source_ids: usize) -> String {format!(r#" WHERE c."deletedAt" IS NULLAND {}"#,- access_semi_join("c.id", "chat")+ bound_access_semi_join("c.id", "chat", source_ids))}-fn project_top_where_clause() -> String {+fn project_top_where_clause(source_ids: usize) -> String {format!(r#" WHERE p."deletedAt" IS NULLAND {}"#,- access_semi_join("p.id", "project")+ bound_access_semi_join("p.id", "project", source_ids))}@@ -1654,6 +1701,7 @@ fn build_query<'a>(document_ids: &mut DocumentIdArrays,) -> QueryBuilder<'a, Postgres> {let sort_method = sort.timestamp_method();+ let source_ids = source_ids_slot(sort, false);let property_sort = matches!(sort, DynamicSort::Property { .. });let mut builder = sqlx::QueryBuilder::new(PREFIX);@@ -1779,7 +1827,7 @@ fn build_query<'a>(if needs_task_property_joins {builder.push(DOCUMENT_TASK_PROPERTY_JOINS);}- builder.push(document_top_where_clause(sort_method));+ builder.push(document_top_where_clause(sort_method, source_ids));builder.push(build_document_filter(document_filter, Some(document_ids)));builder.push(build_properties_filter(filter_ast.properties_filter.as_deref(),@@ -1794,7 +1842,7 @@ fn build_query<'a>(if optimized_notification_predicate.is_some() && chat_notification.is_some() {builder.push(build_notification_join("c", "chat"));}- builder.push(chat_top_where_clause());+ builder.push(chat_top_where_clause(source_ids));builder.push(build_chat_filter(chat_filter));builder.push(build_properties_filter(filter_ast.properties_filter.as_deref(),@@ -1809,7 +1857,7 @@ fn build_query<'a>(if optimized_notification_predicate.is_some() && project_notification.is_some() {builder.push(build_notification_join("p", "project"));}- builder.push(project_top_where_clause());+ builder.push(project_top_where_clause(source_ids));builder.push(build_project_filter(project_filter));builder.push(build_properties_filter(filter_ast.properties_filter.as_deref(),@@ -2253,6 +2301,25 @@ pub(crate) struct ExpandedDynamicCursorArgs<'a> {pub exclude_frecency: bool,}+/// Opens the transaction a dynamic query runs in. A parallel worker costs+/// this query tens of milliseconds to start, so at the default+/// `parallel_setup_cost` callers with a few thousand grants ran twice as slow+/// in parallel; the raised cost keeps parallel plans for the largest callers.+async fn begin_dynamic_query(db: &PgPool) -> Result<Transaction<'static, Postgres>, sqlx::Error> {+ let mut transaction = db+ .begin()+ .instrument(tracing::info_span!(+ "soup.pool.acquire",+ pool_size = db.size(),+ pool_idle = db.num_idle()+ ))+ .await?;+ sqlx::query!("SET LOCAL parallel_setup_cost = 10000")+ .execute(&mut *transaction)+ .await?;+ Ok(transaction)+}+#[tracing::instrument(skip(db), err)]async fn expanded_dynamic_cursor_soup_hydrated(db: &PgPool,@@ -2274,6 +2341,8 @@ async fn expanded_dynamic_cursor_soup_hydrated(let assignees_property_id = SystemPropertyKey::ASSIGNEES_UUID;let completed_option_id = StatusOption::COMPLETED_UUID.to_string();+ let mut transaction = begin_dynamic_query(db).await?;+ let source_ids = fetch_user_source_ids(&mut transaction, user_id.as_ref()).await?;let mut document_ids =DocumentIdArrays::starting_at(first_document_id_slot(*cursor.sort_method(), false));let mut builder = build_query(@@ -2299,17 +2368,10 @@ async fn expanded_dynamic_cursor_soup_hydrated({query = query.bind(property_definition_id).bind(cursor_key);}+ query = query.bind(&source_ids);for ids in document_ids.into_binds() {query = query.bind(ids);}- let mut connection = db- .acquire()- .instrument(tracing::info_span!(- "soup.pool.acquire",- pool_size = db.size(),- pool_idle = db.num_idle()- ))- .await?;let query_span = tracing::info_span!("soup.sql.fetch_decode", rows = tracing::field::Empty);let items = query// Unnamed statement: the SQL text varies per filter shape and per@@ -2324,11 +2386,12 @@ async fn expanded_dynamic_cursor_soup_hydrated(item.sort_key = row.try_get("sort_key")?;Ok(item)})- .fetch_all(&mut *connection)+ .fetch_all(&mut *transaction).instrument(query_span.clone()).await?;query_span.record("rows", items.len());drop(query_span);+ transaction.commit().await?;Ok(items)}@@ -2493,6 +2556,7 @@ fn build_grouped_query<'a>(document_ids: &mut DocumentIdArrays,) -> (QueryBuilder<'a, Postgres>, Option<String>) {let sort_method = sort.timestamp_method();+ let source_ids = source_ids_slot(sort, true);let property_sort = matches!(sort, DynamicSort::Property { .. });let mut builder = sqlx::QueryBuilder::new(PREFIX);@@ -2558,7 +2622,7 @@ fn build_grouped_query<'a>(if needs_task_property_joins {builder.push(DOCUMENT_TASK_PROPERTY_JOINS);}- builder.push(document_top_where_clause(sort_method));+ builder.push(document_top_where_clause(sort_method, source_ids));builder.push(build_document_filter(filter_ast.document_filter.as_deref(),Some(document_ids),@@ -2572,7 +2636,7 @@ fn build_grouped_query<'a>(if include_chats {push_union_separator(&mut builder, &mut needs_separator);builder.push(grouped_chat_top_clause(sort_method));- builder.push(chat_top_where_clause());+ builder.push(chat_top_where_clause(source_ids));builder.push(build_chat_filter(filter_ast.chat_filter.as_deref()));builder.push(build_properties_filter(filter_ast.properties_filter.as_deref(),@@ -2583,7 +2647,7 @@ fn build_grouped_query<'a>(if include_projects {push_union_separator(&mut builder, &mut needs_separator);builder.push(grouped_project_top_clause(sort_method));- builder.push(project_top_where_clause());+ builder.push(project_top_where_clause(source_ids));builder.push(build_project_filter(filter_ast.project_filter.as_deref()));builder.push(build_properties_filter(filter_ast.properties_filter.as_deref(),@@ -2780,6 +2844,8 @@ pub async fn expanded_dynamic_cursor_soup_grouped(let assignees_property_id = SystemPropertyKey::ASSIGNEES_UUID;let completed_option_id = StatusOption::COMPLETED_UUID.to_string();+ let mut transaction = begin_dynamic_query(db).await?;+ let source_ids = fetch_user_source_ids(&mut transaction, user_id.as_ref()).await?;let mut document_ids =DocumentIdArrays::starting_at(first_document_id_slot(*cursor.sort_method(), true));let (mut query_builder, entity_type_bind) = build_grouped_query(@@ -2822,30 +2888,23 @@ pub async fn expanded_dynamic_cursor_soup_grouped({query = query.bind(property_definition_id).bind(cursor_key);}+ query = query.bind(&source_ids);for ids in document_ids.into_binds() {query = query.bind(ids);}- let mut connection = db- .acquire()- .instrument(tracing::info_span!(- "soup.pool.acquire",- pool_size = db.size(),- pool_idle = db.num_idle()- ))- .await?;let query_span = tracing::info_span!("soup.sql.fetch_decode", rows = tracing::field::Empty);let rows: Vec<GroupedSoupRow> = query// Unnamed statement — same rationale as expanded_dynamic_cursor_soup:// filter-shaped SQL text gets no reuse from the statement cache but// does get generic-plan flips..persistent(false)- .fetch_all(&mut *connection)+ .fetch_all(&mut *transaction).instrument(query_span.clone()).await?;query_span.record("rows", rows.len());drop(query_span);- drop(connection);+ transaction.commit().await?;let items = tracing::info_span!("soup.rows.convert", rows = rows.len()).in_scope(|| {rows.into_iter()diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/document_ids/test.rs b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/document_ids/test.rsindex 2f253f93d1..c599b2513d 100644--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/document_ids/test.rs+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/document_ids/test.rs@@ -223,16 +223,17 @@ fn arrays_follow_each_querys_fixed_binds() {)));assert!(!sql.contains(&format!("${}", first + 2)));assert!(!sql.contains("sort_parameters"));+ assert!(sql.contains(&format!("ea.source_id = ANY(${}::text[])", first - 1)));}assert!(grouped.contains("ep.entity_type = $10::property_entity_type"));if matches!(sort, DynamicSort::Property { .. }) {- assert_eq!((flat_first, grouped_first), (11, 13));+ assert_eq!((flat_first, grouped_first), (12, 14));assert!(flat.contains("WHERE id = $9::uuid"));assert!(flat.contains("$10::double precision"));assert!(grouped.contains("WHERE id = $11::uuid"));assert!(grouped.contains("$12::double precision"));} else {- assert_eq!((flat_first, grouped_first), (9, 11));+ assert_eq!((flat_first, grouped_first), (10, 12));assert!(!flat.contains("property_sort"));assert!(!grouped.contains("property_sort"));}@@ -399,7 +400,7 @@ async fn grouped_pages_match_bound_id_sets(pool: PgPool) -> anyhow::Result<()> {};let by_status = vec![(IN_PROGRESS, IN_C), ("", IN_A)];for (label, field, filter, arrays, expected) in [- // The entity type takes `$10`, the arrays `$11` and `$12`.+ // The entity type takes `$10`, the source ids `$11`, the arrays `$12` and `$13`.("status of tasks",status(Some("TASK")),@@ -407,7 +408,7 @@ async fn grouped_pages_match_bound_id_sets(pool: PgPool) -> anyhow::Result<()> {2,by_status.clone(),),- // `$10` is NULL and unreferenced before the arrays.+ // `$10` is NULL and unreferenced before the source ids.("status", status(None), associated(), 2, by_status.clone()),("entity type",@@ -416,7 +417,7 @@ async fn grouped_pages_match_bound_id_sets(pool: PgPool) -> anyhow::Result<()> {2,vec![("document", IN_C), ("document", IN_A)],),- // `$10` is NULL, unreferenced, and the last bind.+ // `$10` is NULL and unreferenced; the source ids are the last bind.("status without arrays",status(None),diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rsindex 6665aebb46..a129addd3d 100644--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs@@ -31,20 +31,22 @@ fn grouped_access_shape() {group_key: None,per_group_limit: None,};+ let sort = SimpleSortMethod::ViewedUpdated.into();let (builder, _) = build_grouped_query(&filter,false,&grouping,- SimpleSortMethod::ViewedUpdated.into(),- &mut DocumentIdArrays::starting_at(first_document_id_slot(- SimpleSortMethod::ViewedUpdated.into(),- true,- )),+ sort,+ &mut DocumentIdArrays::starting_at(first_document_id_slot(sort, true)),);let candidates = builder.sql().split("GroupedItems AS").next().unwrap();assert!(!candidates.contains("AccessibleItems"));for (id, entity_type) in [("d.id", "document"), ("c.id", "chat"), ("p.id", "project")] {- assert!(candidates.contains(&access_semi_join(id, entity_type)));+ assert!(candidates.contains(&bound_access_semi_join(+ id,+ entity_type,+ source_ids_slot(sort, true)+ )));}assert!(candidates.contains("cp.left_at IS NULL"));assert!(candidates.contains("event.owner_id = $1"));@@ -338,6 +340,10 @@ async fn grouped_query_explain_local(pool: PgPool) -> anyhow::Result<()> {calendar_event_filter: Some(Arc::new(Expr::val(CalendarEventLiteral::Id(Uuid::nil())))),..EntityFilterAst::mock_empty()};+ let owner = sqlx::query_scalar!(r#"SELECT owner FROM "Document" WHERE id = $1"#, &ids[0])+ .fetch_one(&pool)+ .await?;+ let source_ids = fetch_user_source_ids(&mut *pool.acquire().await?, &owner).await?;for sort in [SimpleSortMethod::UpdatedAt, SimpleSortMethod::ViewedUpdated] {for (label, filter, field, group_key) in [(@@ -387,7 +393,7 @@ async fn grouped_query_explain_local(pool: PgPool) -> anyhow::Result<()> {&mut DocumentIdArrays::starting_at(first_document_id_slot(sort.into(), true)),);// Dynamic AST/grouping SQL cannot use a compile-time macro.- // Mirror the production timestamp query's ten fixed parameters.+ // Mirror the production timestamp query's ten fixed parameters and source ids.let sql = format!("EXPLAIN (ANALYZE, BUFFERS, VERBOSE, FORMAT JSON) {}",builder.sql()@@ -407,7 +413,8 @@ async fn grouped_query_explain_local(pool: PgPool) -> anyhow::Result<()> {.bind(SystemPropertyKey::STATUS_UUID).bind(SystemPropertyKey::ASSIGNEES_UUID).bind(grouping.group_key.clone())- .bind(entity_type.clone());+ .bind(entity_type.clone())+ .bind(&source_ids);let plan = query.persistent(false).fetch_one(&pool).await?;println!("EXPLAIN {label} {sort} repetition={repetition}: {plan}");}