Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

[db-perf] bind caller source ids in accessible-items and soup access gates (#9583)

teo07673da86673
diff --git a/.sqlx/query-09467ab0f1282fa21cdbc66fb01b7ab90a55d8b528699b9a88ce869dd45e4a66.json b/.sqlx/query-09467ab0f1282fa21cdbc66fb01b7ab90a55d8b528699b9a88ce869dd45e4a66.json
deleted file mode 100644
index 4cb61c05aa..0000000000
--- a/.sqlx/query-09467ab0f1282fa21cdbc66fb01b7ab90a55d8b528699b9a88ce869dd45e4a66.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n WITH user_source_ids AS (\n SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp\n WHERE cp.user_id = $1 AND cp.left_at IS NULL\n UNION ALL\n SELECT t.team_id::text FROM team_user t\n WHERE t.user_id = $1\n UNION ALL\n SELECT $1\n )\n SELECT DISTINCT ea.entity_id::text as \"item_id!\"\n FROM entity_access ea\n JOIN user_source_ids us ON us.source_id = ea.source_id\n JOIN \"Chat\" c ON c.id = ea.entity_id::text\n WHERE ea.entity_type = 'chat'\n AND c.\"deletedAt\" IS NULL\n AND (NOT $2 OR c.\"userId\" != $1)\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "item_id!",
- "type_info": "Text"
- }
- ],
- "parameters": {
- "Left": [
- "Text",
- "Bool"
- ]
- },
- "nullable": [
- null
- ]
- },
- "hash": "09467ab0f1282fa21cdbc66fb01b7ab90a55d8b528699b9a88ce869dd45e4a66"
-}
diff --git a/.sqlx/query-7e6bf8c61219e907f25c233be0c8dfd8a02cd84b2ce6746d6a16823cb52ae7de.json b/.sqlx/query-7e6bf8c61219e907f25c233be0c8dfd8a02cd84b2ce6746d6a16823cb52ae7de.json
new file mode 100644
index 0000000000..fcec8ca8e2
--- /dev/null
+++ b/.sqlx/query-7e6bf8c61219e907f25c233be0c8dfd8a02cd84b2ce6746d6a16823cb52ae7de.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT c.id\n FROM \"Chat\" c\n WHERE c.id IN (\n SELECT ea.entity_id::text\n FROM entity_access ea\n WHERE ea.source_id = ANY($2)\n AND ea.entity_type = 'chat'\n )\n AND c.\"deletedAt\" IS NULL\n AND (NOT $3 OR c.\"userId\" != $1)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "TextArray",
+ "Bool"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "7e6bf8c61219e907f25c233be0c8dfd8a02cd84b2ce6746d6a16823cb52ae7de"
+}
diff --git a/.sqlx/query-a5ce3e2825e32372811a8536ddac2e8a45e3ed323cebff87e9df0ecad67f3256.json b/.sqlx/query-a5ce3e2825e32372811a8536ddac2e8a45e3ed323cebff87e9df0ecad67f3256.json
deleted file mode 100644
index 85b240cea3..0000000000
--- a/.sqlx/query-a5ce3e2825e32372811a8536ddac2e8a45e3ed323cebff87e9df0ecad67f3256.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n WITH user_source_ids AS (\n SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp\n WHERE cp.user_id = $1 AND cp.left_at IS NULL\n UNION ALL\n SELECT t.team_id::text FROM team_user t\n WHERE t.user_id = $1\n UNION ALL\n SELECT $1\n )\n SELECT DISTINCT ea.entity_id::text as \"item_id!\"\n FROM entity_access ea\n JOIN user_source_ids us ON us.source_id = ea.source_id\n JOIN \"Document\" d ON d.id = ea.entity_id::text\n WHERE ea.entity_type = 'document'\n AND d.\"deletedAt\" IS NULL\n AND (NOT $2 OR d.owner != $1)\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "item_id!",
- "type_info": "Text"
- }
- ],
- "parameters": {
- "Left": [
- "Text",
- "Bool"
- ]
- },
- "nullable": [
- null
- ]
- },
- "hash": "a5ce3e2825e32372811a8536ddac2e8a45e3ed323cebff87e9df0ecad67f3256"
-}
diff --git a/.sqlx/query-aa949da30a88081b069a571074e847c0269acd89d23b4921e6010795355989ff.json b/.sqlx/query-aa949da30a88081b069a571074e847c0269acd89d23b4921e6010795355989ff.json
new file mode 100644
index 0000000000..1314307074
--- /dev/null
+++ b/.sqlx/query-aa949da30a88081b069a571074e847c0269acd89d23b4921e6010795355989ff.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT p.id\n FROM \"Project\" p\n WHERE p.id IN (\n SELECT ea.entity_id::text\n FROM entity_access ea\n WHERE ea.source_id = ANY($2)\n AND ea.entity_type = 'project'\n )\n AND p.\"deletedAt\" IS NULL\n AND (NOT $3 OR p.\"userId\" != $1)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "TextArray",
+ "Bool"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "aa949da30a88081b069a571074e847c0269acd89d23b4921e6010795355989ff"
+}
diff --git a/.sqlx/query-b48f3da320797f3254c2f0e88442261ba53a69dfaa2ba97e4ba581fe6098e52c.json b/.sqlx/query-b48f3da320797f3254c2f0e88442261ba53a69dfaa2ba97e4ba581fe6098e52c.json
deleted file mode 100644
index 7d62b4ae05..0000000000
--- a/.sqlx/query-b48f3da320797f3254c2f0e88442261ba53a69dfaa2ba97e4ba581fe6098e52c.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n WITH user_source_ids AS (\n SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp\n WHERE cp.user_id = $1 AND cp.left_at IS NULL\n UNION ALL\n SELECT t.team_id::text FROM team_user t\n WHERE t.user_id = $1\n UNION ALL\n SELECT $1\n )\n SELECT DISTINCT ea.entity_id::text as \"item_id!\"\n FROM entity_access ea\n JOIN user_source_ids us ON us.source_id = ea.source_id\n JOIN \"Project\" p ON p.id = ea.entity_id::text\n WHERE ea.entity_type = 'project'\n AND p.\"deletedAt\" IS NULL\n AND (NOT $2 OR p.\"userId\" != $1)\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "item_id!",
- "type_info": "Text"
- }
- ],
- "parameters": {
- "Left": [
- "Text",
- "Bool"
- ]
- },
- "nullable": [
- null
- ]
- },
- "hash": "b48f3da320797f3254c2f0e88442261ba53a69dfaa2ba97e4ba581fe6098e52c"
-}
diff --git a/.sqlx/query-c3359b2f83080c2f8f2af75564d72ffb27e79fa9a5ca85134c085a6d8cac41fd.json b/.sqlx/query-c3359b2f83080c2f8f2af75564d72ffb27e79fa9a5ca85134c085a6d8cac41fd.json
new file mode 100644
index 0000000000..eed624920e
--- /dev/null
+++ b/.sqlx/query-c3359b2f83080c2f8f2af75564d72ffb27e79fa9a5ca85134c085a6d8cac41fd.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT d.id\n FROM \"Document\" d\n WHERE d.id IN (\n SELECT ea.entity_id::text\n FROM entity_access ea\n WHERE ea.source_id = ANY($2)\n AND ea.entity_type = 'document'\n )\n AND d.\"deletedAt\" IS NULL\n AND (NOT $3 OR d.owner != $1)\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "TextArray",
+ "Bool"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "c3359b2f83080c2f8f2af75564d72ffb27e79fa9a5ca85134c085a6d8cac41fd"
+}
diff --git a/.sqlx/query-cd1dde0000615f78a4eb075ef98aec58d9a2e13d9f0bff9a20159512c8aee618.json b/.sqlx/query-cd1dde0000615f78a4eb075ef98aec58d9a2e13d9f0bff9a20159512c8aee618.json
new file mode 100644
index 0000000000..b54c7a1b81
--- /dev/null
+++ b/.sqlx/query-cd1dde0000615f78a4eb075ef98aec58d9a2e13d9f0bff9a20159512c8aee618.json
@@ -0,0 +1,12 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SET LOCAL parallel_setup_cost = 10000",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": []
+ },
+ "hash": "cd1dde0000615f78a4eb075ef98aec58d9a2e13d9f0bff9a20159512c8aee618"
+}
diff --git a/.sqlx/query-efd03310692ee32d62240b82e9d2d533cd0e0ecb7b5cb99b808b21da799ab4bc.json b/.sqlx/query-efd03310692ee32d62240b82e9d2d533cd0e0ecb7b5cb99b808b21da799ab4bc.json
new file mode 100644
index 0000000000..810859dcbc
--- /dev/null
+++ b/.sqlx/query-efd03310692ee32d62240b82e9d2d533cd0e0ecb7b5cb99b808b21da799ab4bc.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT cp.channel_id::text FROM comms_channel_participants cp\n WHERE cp.user_id = $1 AND cp.left_at IS NULL\n UNION ALL\n SELECT t.team_id::text FROM team_user t\n WHERE t.user_id = $1\n UNION ALL\n SELECT $1\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "channel_id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "efd03310692ee32d62240b82e9d2d533cd0e0ecb7b5cb99b808b21da799ab4bc"
+}
diff --git a/crates/macro_db_client/src/item_access/get_accessible_items.rs b/crates/macro_db_client/src/item_access/get_accessible_items.rs
index a6c91894a8..231c038abf 100644
--- a/crates/macro_db_client/src/item_access/get_accessible_items.rs
+++ b/crates/macro_db_client/src/item_access/get_accessible_items.rs
@@ -23,15 +23,16 @@ pub async fn get_user_accessible_items(
item_type_filter: Option<String>,
exclude_owned: bool,
) -> anyhow::Result<Vec<UserAccessibleItem>> {
+ let source_ids = user_source_ids(db, user_id).await?;
match item_type_filter.as_deref() {
- Some("document") => accessible_documents(db, user_id, exclude_owned).await,
- Some("chat") => accessible_chats(db, user_id, exclude_owned).await,
- Some("project") => accessible_projects(db, user_id, exclude_owned).await,
+ Some("document") => accessible_documents(db, user_id, &source_ids, exclude_owned).await,
+ Some("chat") => accessible_chats(db, user_id, &source_ids, exclude_owned).await,
+ Some("project") => accessible_projects(db, user_id, &source_ids, exclude_owned).await,
Some(_) => Ok(vec![]),
None => {
- let mut items = accessible_documents(db, user_id, exclude_owned).await?;
- items.extend(accessible_chats(db, user_id, exclude_owned).await?);
- items.extend(accessible_projects(db, user_id, exclude_owned).await?);
+ let mut items = accessible_documents(db, user_id, &source_ids, exclude_owned).await?;
+ items.extend(accessible_chats(db, user_id, &source_ids, exclude_owned).await?);
+ items.extend(accessible_projects(db, user_id, &source_ids, exclude_owned).await?);
Ok(items)
}
}
@@ -46,33 +47,60 @@ fn to_items(ids: Vec<String>, item_type: &str) -> Vec<UserAccessibleItem> {
.collect()
}
+/// The user's active channels, team, and the user id itself: every
+/// `entity_access.source_id` that grants them access.
+///
+/// The per-type queries bind these as a constant `source_id = ANY($2)` rather
+/// than joining the same union as a CTE. Postgres costs a CTE join at the
+/// average source's grant count, so it planned users holding hundreds of
+/// thousands of grants with one item-table probe per grant; a constant array
+/// is estimated from `entity_access.source_id` statistics, which picks a hash
+/// join for them. The per-type queries run as unnamed statements because a
+/// cached prepared statement switches every user to one generic plan after
+/// five calls.
+async fn user_source_ids(db: &Pool<Postgres>, user_id: &str) -> sqlx::Result<Vec<String>> {
+ let source_ids = sqlx::query_scalar!(
+ r#"
+ SELECT cp.channel_id::text FROM comms_channel_participants cp
+ WHERE cp.user_id = $1 AND cp.left_at IS NULL
+ UNION ALL
+ SELECT t.team_id::text FROM team_user t
+ WHERE t.user_id = $1
+ UNION ALL
+ SELECT $1
+ "#,
+ user_id
+ )
+ .fetch_all(db)
+ .await?;
+
+ Ok(source_ids.into_iter().flatten().collect())
+}
+
async fn accessible_documents(
db: &Pool<Postgres>,
user_id: &str,
+ source_ids: &[String],
exclude_owned: bool,
) -> anyhow::Result<Vec<UserAccessibleItem>> {
let ids = sqlx::query_scalar!(
r#"
- WITH user_source_ids AS (
- SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp
- WHERE cp.user_id = $1 AND cp.left_at IS NULL
- UNION ALL
- SELECT t.team_id::text FROM team_user t
- WHERE t.user_id = $1
- UNION ALL
- SELECT $1
- )
- SELECT DISTINCT ea.entity_id::text as "item_id!"
- FROM entity_access ea
- JOIN user_source_ids us ON us.source_id = ea.source_id
- JOIN "Document" d ON d.id = ea.entity_id::text
- WHERE ea.entity_type = 'document'
+ SELECT d.id
+ FROM "Document" d
+ WHERE d.id IN (
+ SELECT ea.entity_id::text
+ FROM entity_access ea
+ WHERE ea.source_id = ANY($2)
+ AND ea.entity_type = 'document'
+ )
AND d."deletedAt" IS NULL
- AND (NOT $2 OR d.owner != $1)
+ AND (NOT $3 OR d.owner != $1)
"#,
user_id,
+ source_ids,
exclude_owned
)
+ .persistent(false)
.fetch_all(db)
.await?;
@@ -82,30 +110,27 @@ async fn accessible_documents(
async fn accessible_chats(
db: &Pool<Postgres>,
user_id: &str,
+ source_ids: &[String],
exclude_owned: bool,
) -> anyhow::Result<Vec<UserAccessibleItem>> {
let ids = sqlx::query_scalar!(
r#"
- WITH user_source_ids AS (
- SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp
- WHERE cp.user_id = $1 AND cp.left_at IS NULL
- UNION ALL
- SELECT t.team_id::text FROM team_user t
- WHERE t.user_id = $1
- UNION ALL
- SELECT $1
- )
- SELECT DISTINCT ea.entity_id::text as "item_id!"
- FROM entity_access ea
- JOIN user_source_ids us ON us.source_id = ea.source_id
- JOIN "Chat" c ON c.id = ea.entity_id::text
- WHERE ea.entity_type = 'chat'
+ SELECT c.id
+ FROM "Chat" c
+ WHERE c.id IN (
+ SELECT ea.entity_id::text
+ FROM entity_access ea
+ WHERE ea.source_id = ANY($2)
+ AND ea.entity_type = 'chat'
+ )
AND c."deletedAt" IS NULL
- AND (NOT $2 OR c."userId" != $1)
+ AND (NOT $3 OR c."userId" != $1)
"#,
user_id,
+ source_ids,
exclude_owned
)
+ .persistent(false)
.fetch_all(db)
.await?;
@@ -115,30 +140,27 @@ async fn accessible_chats(
async fn accessible_projects(
db: &Pool<Postgres>,
user_id: &str,
+ source_ids: &[String],
exclude_owned: bool,
) -> anyhow::Result<Vec<UserAccessibleItem>> {
let ids = sqlx::query_scalar!(
r#"
- WITH user_source_ids AS (
- SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp
- WHERE cp.user_id = $1 AND cp.left_at IS NULL
- UNION ALL
- SELECT t.team_id::text FROM team_user t
- WHERE t.user_id = $1
- UNION ALL
- SELECT $1
- )
- SELECT DISTINCT ea.entity_id::text as "item_id!"
- FROM entity_access ea
- JOIN user_source_ids us ON us.source_id = ea.source_id
- JOIN "Project" p ON p.id = ea.entity_id::text
- WHERE ea.entity_type = 'project'
+ SELECT p.id
+ FROM "Project" p
+ WHERE p.id IN (
+ SELECT ea.entity_id::text
+ FROM entity_access ea
+ WHERE ea.source_id = ANY($2)
+ AND ea.entity_type = 'project'
+ )
AND p."deletedAt" IS NULL
- AND (NOT $2 OR p."userId" != $1)
+ AND (NOT $3 OR p."userId" != $1)
"#,
user_id,
+ source_ids,
exclude_owned
)
+ .persistent(false)
.fetch_all(db)
.await?;
diff --git a/crates/soup/AGENTS.md b/crates/soup/AGENTS.md
index 83d6326c25..fc9e27dc0d 100644
--- a/crates/soup/AGENTS.md
+++ b/crates/soup/AGENTS.md
@@ -107,7 +107,8 @@ rows, initiatives, ...) and get back a single unified, paginated result.
- Preserve projection-backed hydration, the UUID-keyed item/fact pools, and per-group total/has-more metadata through the shared grouped response.
- Flat property sorts omit calendar events, email, channels, calls, companies, and agent sessions; grouped reads retain calendar events and timestamp queries retain their existing sources.
- Preserve deployed timestamp cursor encodings; client query keys restart pages when ranges or selections change.
-- Keep flat timestamp top clauses lightweight; bind document-ID arrays after the sort's fixed parameters and skip subtype namespace joins when the filter proves TASK-only unless other predicates need them.
+- Keep flat timestamp top clauses lightweight; bind the caller's source ids, then document-ID arrays, after the sort's fixed parameters and skip subtype namespace joins when the filter proves TASK-only unless other predicates need them.
+- The dynamic query's document, chat, and project arms match `entity_access.source_id = ANY(<bound source ids>)`, not a join to the `user_source_ids` CTE: Postgres costs a CTE join at the average source's grant count and probed the item table once per grant for callers holding hundreds of thousands. The query runs in a transaction that raises `parallel_setup_cost`, because parallel plans made callers with a few thousand grants twice as slow.
- Read grouping/sort definition types through uncorrelated scalar lookups; join grouping options once outside the value expansion and preserve missing-option groups.
- Build option-sort order maps with an uncorrelated `jsonb_object_agg` lookup using the sort definition bind; ignore unknown option IDs and keep unknown-only values missing.
- Materialize grouped `TopItems` only for property sorts so the sort key is computed before multi-value group expansion; leave timestamp CTEs inlineable.
diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs
index da98412142..da85db64c5 100644
--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs
+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs
@@ -46,7 +46,10 @@ use models_soup::{
project::SoupProject,
};
use recursion::CollapsibleExt;
-use sqlx::{PgPool, Postgres, QueryBuilder, Row, postgres::PgRow, prelude::FromRow};
+use sqlx::{
+ PgConnection, PgPool, Postgres, QueryBuilder, Row, Transaction, postgres::PgRow,
+ prelude::FromRow,
+};
use system_properties::{StatusOption, SystemPropertyKey};
use tracing::Instrument;
use uuid::Uuid;
@@ -95,8 +98,8 @@ fn sort_key_expression(enabled: bool) -> &'static str {
}
}
-/// First document-ID array after the existing query parameters and optional property sort.
-fn first_document_id_slot(sort: DynamicSort, grouped: bool) -> usize {
+/// The caller's source-id array, after the existing query parameters and optional property sort.
+fn source_ids_slot(sort: DynamicSort, grouped: bool) -> usize {
let fixed_parameters = if grouped { 10 } else { 8 };
let property_parameters = if matches!(sort, DynamicSort::Property { .. }) {
2
@@ -106,6 +109,11 @@ fn first_document_id_slot(sort: DynamicSort, grouped: bool) -> usize {
fixed_parameters + property_parameters + 1
}
+/// First document-ID array, after the caller's source ids.
+fn first_document_id_slot(sort: DynamicSort, grouped: bool) -> usize {
+ source_ids_slot(sort, grouped) + 1
+}
+
// The non-null and missing-key branches both follow SQL's NULLS LAST order.
fn property_keyset(
sort: DynamicSort,
@@ -186,6 +194,29 @@ static PREFIX: &str = r#"
),
"#;
+/// The `user_source_ids` rows, bound into the document, chat, and project
+/// access gates by [`bound_access_semi_join`].
+async fn fetch_user_source_ids(
+ connection: &mut PgConnection,
+ user_id: &str,
+) -> Result<Vec<String>, sqlx::Error> {
+ let source_ids = sqlx::query_scalar!(
+ r#"
+ SELECT cp.channel_id::text FROM comms_channel_participants cp
+ WHERE cp.user_id = $1 AND cp.left_at IS NULL
+ UNION ALL
+ SELECT t.team_id::text FROM team_user t
+ WHERE t.user_id = $1
+ UNION ALL
+ SELECT $1
+ "#,
+ user_id
+ )
+ .fetch_all(connection)
+ .await?;
+ Ok(source_ids.into_iter().flatten().collect())
+}
+
// -- Lightweight top clauses: only id + sort_ts (plus filter-required joins) --
static DOCUMENT_TASK_PROPERTY_JOINS: &str = r#"
@@ -1614,7 +1645,23 @@ pub(in crate::outbound::pg_soup_repo) fn access_semi_join(
)
}
-fn document_top_where_clause(sort_method: SimpleSortMethod) -> String {
+/// [`access_semi_join`] over the caller's source ids, bound as a constant
+/// array at parameter `source_ids`. Postgres costs the `user_source_ids` join at the average
+/// source's grant count, so it planned callers holding hundreds of thousands
+/// of grants with one item probe per grant; it estimates the array from
+/// `entity_access.source_id` statistics and hashes the grants instead.
+fn bound_access_semi_join(id_sql: &str, entity_type: &str, source_ids: usize) -> String {
+ format!(
+ r#"{id_sql} IN (
+ SELECT ea.entity_id::text
+ FROM entity_access ea
+ WHERE ea.source_id = ANY(${source_ids}::text[])
+ AND ea.entity_type = '{entity_type}'
+ )"#
+ )
+}
+
+fn document_top_where_clause(sort_method: SimpleSortMethod, source_ids: usize) -> String {
let user_history_join = if top_needs_user_history(sort_method) {
r#" LEFT JOIN "UserHistory" uh ON uh."itemId" = d.id AND uh."itemType" = 'document' AND uh."userId" = $1
"#
@@ -1625,25 +1672,25 @@ fn document_top_where_clause(sort_method: SimpleSortMethod) -> String {
r#"{user_history_join} WHERE d."deletedAt" IS NULL
AND {}
"#,
- access_semi_join("d.id", "document")
+ bound_access_semi_join("d.id", "document", source_ids)
)
}
-fn chat_top_where_clause() -> String {
+fn chat_top_where_clause(source_ids: usize) -> String {
format!(
r#" WHERE c."deletedAt" IS NULL
AND {}
"#,
- access_semi_join("c.id", "chat")
+ bound_access_semi_join("c.id", "chat", source_ids)
)
}
-fn project_top_where_clause() -> String {
+fn project_top_where_clause(source_ids: usize) -> String {
format!(
r#" WHERE p."deletedAt" IS NULL
AND {}
"#,
- access_semi_join("p.id", "project")
+ bound_access_semi_join("p.id", "project", source_ids)
)
}
@@ -1654,6 +1701,7 @@ fn build_query<'a>(
document_ids: &mut DocumentIdArrays,
) -> QueryBuilder<'a, Postgres> {
let sort_method = sort.timestamp_method();
+ let source_ids = source_ids_slot(sort, false);
let property_sort = matches!(sort, DynamicSort::Property { .. });
let mut builder = sqlx::QueryBuilder::new(PREFIX);
@@ -1779,7 +1827,7 @@ fn build_query<'a>(
if needs_task_property_joins {
builder.push(DOCUMENT_TASK_PROPERTY_JOINS);
}
- builder.push(document_top_where_clause(sort_method));
+ builder.push(document_top_where_clause(sort_method, source_ids));
builder.push(build_document_filter(document_filter, Some(document_ids)));
builder.push(build_properties_filter(
filter_ast.properties_filter.as_deref(),
@@ -1794,7 +1842,7 @@ fn build_query<'a>(
if optimized_notification_predicate.is_some() && chat_notification.is_some() {
builder.push(build_notification_join("c", "chat"));
}
- builder.push(chat_top_where_clause());
+ builder.push(chat_top_where_clause(source_ids));
builder.push(build_chat_filter(chat_filter));
builder.push(build_properties_filter(
filter_ast.properties_filter.as_deref(),
@@ -1809,7 +1857,7 @@ fn build_query<'a>(
if optimized_notification_predicate.is_some() && project_notification.is_some() {
builder.push(build_notification_join("p", "project"));
}
- builder.push(project_top_where_clause());
+ builder.push(project_top_where_clause(source_ids));
builder.push(build_project_filter(project_filter));
builder.push(build_properties_filter(
filter_ast.properties_filter.as_deref(),
@@ -2253,6 +2301,25 @@ pub(crate) struct ExpandedDynamicCursorArgs<'a> {
pub exclude_frecency: bool,
}
+/// Opens the transaction a dynamic query runs in. A parallel worker costs
+/// this query tens of milliseconds to start, so at the default
+/// `parallel_setup_cost` callers with a few thousand grants ran twice as slow
+/// in parallel; the raised cost keeps parallel plans for the largest callers.
+async fn begin_dynamic_query(db: &PgPool) -> Result<Transaction<'static, Postgres>, sqlx::Error> {
+ let mut transaction = db
+ .begin()
+ .instrument(tracing::info_span!(
+ "soup.pool.acquire",
+ pool_size = db.size(),
+ pool_idle = db.num_idle()
+ ))
+ .await?;
+ sqlx::query!("SET LOCAL parallel_setup_cost = 10000")
+ .execute(&mut *transaction)
+ .await?;
+ Ok(transaction)
+}
+
#[tracing::instrument(skip(db), err)]
async fn expanded_dynamic_cursor_soup_hydrated(
db: &PgPool,
@@ -2274,6 +2341,8 @@ async fn expanded_dynamic_cursor_soup_hydrated(
let assignees_property_id = SystemPropertyKey::ASSIGNEES_UUID;
let completed_option_id = StatusOption::COMPLETED_UUID.to_string();
+ let mut transaction = begin_dynamic_query(db).await?;
+ let source_ids = fetch_user_source_ids(&mut transaction, user_id.as_ref()).await?;
let mut document_ids =
DocumentIdArrays::starting_at(first_document_id_slot(*cursor.sort_method(), false));
let mut builder = build_query(
@@ -2299,17 +2368,10 @@ async fn expanded_dynamic_cursor_soup_hydrated(
{
query = query.bind(property_definition_id).bind(cursor_key);
}
+ query = query.bind(&source_ids);
for ids in document_ids.into_binds() {
query = query.bind(ids);
}
- let mut connection = db
- .acquire()
- .instrument(tracing::info_span!(
- "soup.pool.acquire",
- pool_size = db.size(),
- pool_idle = db.num_idle()
- ))
- .await?;
let query_span = tracing::info_span!("soup.sql.fetch_decode", rows = tracing::field::Empty);
let items = query
// Unnamed statement: the SQL text varies per filter shape and per
@@ -2324,11 +2386,12 @@ async fn expanded_dynamic_cursor_soup_hydrated(
item.sort_key = row.try_get("sort_key")?;
Ok(item)
})
- .fetch_all(&mut *connection)
+ .fetch_all(&mut *transaction)
.instrument(query_span.clone())
.await?;
query_span.record("rows", items.len());
drop(query_span);
+ transaction.commit().await?;
Ok(items)
}
@@ -2493,6 +2556,7 @@ fn build_grouped_query<'a>(
document_ids: &mut DocumentIdArrays,
) -> (QueryBuilder<'a, Postgres>, Option<String>) {
let sort_method = sort.timestamp_method();
+ let source_ids = source_ids_slot(sort, true);
let property_sort = matches!(sort, DynamicSort::Property { .. });
let mut builder = sqlx::QueryBuilder::new(PREFIX);
@@ -2558,7 +2622,7 @@ fn build_grouped_query<'a>(
if needs_task_property_joins {
builder.push(DOCUMENT_TASK_PROPERTY_JOINS);
}
- builder.push(document_top_where_clause(sort_method));
+ builder.push(document_top_where_clause(sort_method, source_ids));
builder.push(build_document_filter(
filter_ast.document_filter.as_deref(),
Some(document_ids),
@@ -2572,7 +2636,7 @@ fn build_grouped_query<'a>(
if include_chats {
push_union_separator(&mut builder, &mut needs_separator);
builder.push(grouped_chat_top_clause(sort_method));
- builder.push(chat_top_where_clause());
+ builder.push(chat_top_where_clause(source_ids));
builder.push(build_chat_filter(filter_ast.chat_filter.as_deref()));
builder.push(build_properties_filter(
filter_ast.properties_filter.as_deref(),
@@ -2583,7 +2647,7 @@ fn build_grouped_query<'a>(
if include_projects {
push_union_separator(&mut builder, &mut needs_separator);
builder.push(grouped_project_top_clause(sort_method));
- builder.push(project_top_where_clause());
+ builder.push(project_top_where_clause(source_ids));
builder.push(build_project_filter(filter_ast.project_filter.as_deref()));
builder.push(build_properties_filter(
filter_ast.properties_filter.as_deref(),
@@ -2780,6 +2844,8 @@ pub async fn expanded_dynamic_cursor_soup_grouped(
let assignees_property_id = SystemPropertyKey::ASSIGNEES_UUID;
let completed_option_id = StatusOption::COMPLETED_UUID.to_string();
+ let mut transaction = begin_dynamic_query(db).await?;
+ let source_ids = fetch_user_source_ids(&mut transaction, user_id.as_ref()).await?;
let mut document_ids =
DocumentIdArrays::starting_at(first_document_id_slot(*cursor.sort_method(), true));
let (mut query_builder, entity_type_bind) = build_grouped_query(
@@ -2822,30 +2888,23 @@ pub async fn expanded_dynamic_cursor_soup_grouped(
{
query = query.bind(property_definition_id).bind(cursor_key);
}
+ query = query.bind(&source_ids);
for ids in document_ids.into_binds() {
query = query.bind(ids);
}
- let mut connection = db
- .acquire()
- .instrument(tracing::info_span!(
- "soup.pool.acquire",
- pool_size = db.size(),
- pool_idle = db.num_idle()
- ))
- .await?;
let query_span = tracing::info_span!("soup.sql.fetch_decode", rows = tracing::field::Empty);
let rows: Vec<GroupedSoupRow> = query
// Unnamed statement — same rationale as expanded_dynamic_cursor_soup:
// filter-shaped SQL text gets no reuse from the statement cache but
// does get generic-plan flips.
.persistent(false)
- .fetch_all(&mut *connection)
+ .fetch_all(&mut *transaction)
.instrument(query_span.clone())
.await?;
query_span.record("rows", rows.len());
drop(query_span);
- drop(connection);
+ transaction.commit().await?;
let items = tracing::info_span!("soup.rows.convert", rows = rows.len()).in_scope(|| {
rows.into_iter()
diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/document_ids/test.rs b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/document_ids/test.rs
index 2f253f93d1..c599b2513d 100644
--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/document_ids/test.rs
+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/document_ids/test.rs
@@ -223,16 +223,17 @@ fn arrays_follow_each_querys_fixed_binds() {
)));
assert!(!sql.contains(&format!("${}", first + 2)));
assert!(!sql.contains("sort_parameters"));
+ assert!(sql.contains(&format!("ea.source_id = ANY(${}::text[])", first - 1)));
}
assert!(grouped.contains("ep.entity_type = $10::property_entity_type"));
if matches!(sort, DynamicSort::Property { .. }) {
- assert_eq!((flat_first, grouped_first), (11, 13));
+ assert_eq!((flat_first, grouped_first), (12, 14));
assert!(flat.contains("WHERE id = $9::uuid"));
assert!(flat.contains("$10::double precision"));
assert!(grouped.contains("WHERE id = $11::uuid"));
assert!(grouped.contains("$12::double precision"));
} else {
- assert_eq!((flat_first, grouped_first), (9, 11));
+ assert_eq!((flat_first, grouped_first), (10, 12));
assert!(!flat.contains("property_sort"));
assert!(!grouped.contains("property_sort"));
}
@@ -399,7 +400,7 @@ async fn grouped_pages_match_bound_id_sets(pool: PgPool) -> anyhow::Result<()> {
};
let by_status = vec![(IN_PROGRESS, IN_C), ("", IN_A)];
for (label, field, filter, arrays, expected) in [
- // The entity type takes `$10`, the arrays `$11` and `$12`.
+ // The entity type takes `$10`, the source ids `$11`, the arrays `$12` and `$13`.
(
"status of tasks",
status(Some("TASK")),
@@ -407,7 +408,7 @@ async fn grouped_pages_match_bound_id_sets(pool: PgPool) -> anyhow::Result<()> {
2,
by_status.clone(),
),
- // `$10` is NULL and unreferenced before the arrays.
+ // `$10` is NULL and unreferenced before the source ids.
("status", status(None), associated(), 2, by_status.clone()),
(
"entity type",
@@ -416,7 +417,7 @@ async fn grouped_pages_match_bound_id_sets(pool: PgPool) -> anyhow::Result<()> {
2,
vec![("document", IN_C), ("document", IN_A)],
),
- // `$10` is NULL, unreferenced, and the last bind.
+ // `$10` is NULL and unreferenced; the source ids are the last bind.
(
"status without arrays",
status(None),
diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs
index 6665aebb46..a129addd3d 100644
--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs
+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs
@@ -31,20 +31,22 @@ fn grouped_access_shape() {
group_key: None,
per_group_limit: None,
};
+ let sort = SimpleSortMethod::ViewedUpdated.into();
let (builder, _) = build_grouped_query(
&filter,
false,
&grouping,
- SimpleSortMethod::ViewedUpdated.into(),
- &mut DocumentIdArrays::starting_at(first_document_id_slot(
- SimpleSortMethod::ViewedUpdated.into(),
- true,
- )),
+ sort,
+ &mut DocumentIdArrays::starting_at(first_document_id_slot(sort, true)),
);
let candidates = builder.sql().split("GroupedItems AS").next().unwrap();
assert!(!candidates.contains("AccessibleItems"));
for (id, entity_type) in [("d.id", "document"), ("c.id", "chat"), ("p.id", "project")] {
- assert!(candidates.contains(&access_semi_join(id, entity_type)));
+ assert!(candidates.contains(&bound_access_semi_join(
+ id,
+ entity_type,
+ source_ids_slot(sort, true)
+ )));
}
assert!(candidates.contains("cp.left_at IS NULL"));
assert!(candidates.contains("event.owner_id = $1"));
@@ -338,6 +340,10 @@ async fn grouped_query_explain_local(pool: PgPool) -> anyhow::Result<()> {
calendar_event_filter: Some(Arc::new(Expr::val(CalendarEventLiteral::Id(Uuid::nil())))),
..EntityFilterAst::mock_empty()
};
+ let owner = sqlx::query_scalar!(r#"SELECT owner FROM "Document" WHERE id = $1"#, &ids[0])
+ .fetch_one(&pool)
+ .await?;
+ let source_ids = fetch_user_source_ids(&mut *pool.acquire().await?, &owner).await?;
for sort in [SimpleSortMethod::UpdatedAt, SimpleSortMethod::ViewedUpdated] {
for (label, filter, field, group_key) in [
(
@@ -387,7 +393,7 @@ async fn grouped_query_explain_local(pool: PgPool) -> anyhow::Result<()> {
&mut DocumentIdArrays::starting_at(first_document_id_slot(sort.into(), true)),
);
// Dynamic AST/grouping SQL cannot use a compile-time macro.
- // Mirror the production timestamp query's ten fixed parameters.
+ // Mirror the production timestamp query's ten fixed parameters and source ids.
let sql = format!(
"EXPLAIN (ANALYZE, BUFFERS, VERBOSE, FORMAT JSON) {}",
builder.sql()
@@ -407,7 +413,8 @@ async fn grouped_query_explain_local(pool: PgPool) -> anyhow::Result<()> {
.bind(SystemPropertyKey::STATUS_UUID)
.bind(SystemPropertyKey::ASSIGNEES_UUID)
.bind(grouping.group_key.clone())
- .bind(entity_type.clone());
+ .bind(entity_type.clone())
+ .bind(&source_ids);
let plan = query.persistent(false).fetch_one(&pool).await?;
println!("EXPLAIN {label} {sort} repetition={repetition}: {plan}");
}