Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(soup): query optimizations (#4448)

teoe95327945219
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs
index bf96ea321e..bed2d12f3e 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/filters.rs
@@ -5,6 +5,7 @@ use filter_ast::Expr;
use item_filters::ast::date::DateLiteral;
use item_filters::ast::email::{Email, EmailLiteral};
use recursion::CollapsibleExt;
+use uuid::Uuid;
fn date_predicate(col: &str, lit: &DateLiteral) -> SqlFragment {
let sql = match lit {
@@ -549,6 +550,7 @@ fn build_union_branch(
kind: AddressKind,
email: &Email,
resolved: &ResolvedFilters,
+ link_scope: Option<&[Uuid]>,
) -> Option<SqlFragment> {
let trash = build_trash_check(resolved);
match (resolved.contact_ids_for(email), email) {
@@ -581,7 +583,8 @@ fn build_union_branch(
(None, Email::Complete(_)) => None,
(None, Email::Partial(s)) => {
let pattern = format!("%{}%", escape_like_pattern(s));
- let mut f = build_union_branch_text_match(kind, "c.email_address ILIKE ", pattern);
+ let mut f =
+ build_union_branch_text_match(kind, "c.email_address ILIKE ", pattern, link_scope);
f.push_raw(" AND ");
f.extend(trash);
Some(f)
@@ -592,6 +595,7 @@ fn build_union_branch(
kind,
"LOWER(SPLIT_PART(c.email_address, '@', 2)) = ",
domain,
+ link_scope,
);
f.push_raw(" AND ");
f.extend(trash);
@@ -600,12 +604,32 @@ fn build_union_branch(
}
}
+/// `AND <alias>.link_id = ANY($links)` when a link scope applies, empty otherwise.
+fn link_scope_fragment(alias: &str, link_scope: Option<&[Uuid]>) -> SqlFragment {
+ match link_scope {
+ Some(links) => {
+ let mut f = SqlFragment::raw(format!(" AND {alias}.link_id = ANY("));
+ f.extend(SqlFragment::bind_uuid_array(links.to_vec()));
+ f.push_raw(")");
+ f
+ }
+ None => SqlFragment::empty(),
+ }
+}
+
/// Shared shape for a union-branch SELECT that joins through
/// `email_contacts` with a single bound predicate against `c.*`.
+///
+/// Text matches (ILIKE / domain equality) are not contact-id-scoped like the
+/// Complete branches are, so without `link_scope` they scan matches across
+/// every mailbox in the table before the candidate stage intersects with the
+/// caller's threads. Scoping both `c` and `m` by the caller's links bounds
+/// the CTE to the caller's own mail.
fn build_union_branch_text_match(
kind: AddressKind,
predicate_prefix: &str,
bind_value: String,
+ link_scope: Option<&[Uuid]>,
) -> SqlFragment {
match kind {
AddressKind::Sender => {
@@ -615,6 +639,8 @@ fn build_union_branch_text_match(
WHERE {predicate_prefix}"
));
f.extend(SqlFragment::bind_string(bind_value));
+ f.extend(link_scope_fragment("c", link_scope));
+ f.extend(link_scope_fragment("m", link_scope));
f
}
_ => {
@@ -627,6 +653,8 @@ fn build_union_branch_text_match(
));
f.extend(SqlFragment::bind_string(bind_value));
f.push_raw(format!(" AND mr.recipient_type = '{recipient_type}'"));
+ f.extend(link_scope_fragment("c", link_scope));
+ f.extend(link_scope_fragment("m", link_scope));
f
}
}
@@ -651,9 +679,16 @@ fn build_union_branch_text_match(
/// every conjunct).
///
/// Returns `None` when there are no pure-address conjuncts to push down.
+///
+/// `link_scope` restricts the CTE's contact/message scans to the given
+/// links. Only pass it when every candidate thread is known to belong to
+/// those links (owned-only, non-team queries) — shared and team candidate
+/// selects include threads from other users' links, which the scoped CTE
+/// would wrongly filter out.
pub(super) fn build_matching_threads_cte_body(
ast: &Expr<EmailLiteral>,
resolved: &ResolvedFilters,
+ link_scope: Option<&[Uuid]>,
) -> Option<SqlFragment> {
let conjuncts = extract_address_only_conjuncts(ast);
if conjuncts.is_empty() {
@@ -665,7 +700,7 @@ pub(super) fn build_matching_threads_cte_body(
{
let branches: Vec<SqlFragment> = literals
.into_iter()
- .filter_map(|(k, e)| build_union_branch(k, e, resolved))
+ .filter_map(|(k, e)| build_union_branch(k, e, resolved, link_scope))
.collect();
if !branches.is_empty() {
let mut iter = branches.into_iter();
@@ -694,6 +729,7 @@ pub(super) fn build_matching_threads_cte_body(
f.extend(build_trash_check(resolved));
f.push_raw(" AND ");
f.extend(predicate);
+ f.extend(link_scope_fragment("m", link_scope));
Some(f)
}
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs
index 22cedbffcd..fc25b8a030 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/query.rs
@@ -334,7 +334,13 @@ fn build_query(
let matching_threads_body = if wants_message_exists_pushdown(email_filter, view) {
None
} else {
- build_matching_threads_cte_body(email_filter, &params.resolved)
+ // Owned-only, non-team queries can scope the CTE's contact/message
+ // scans to the caller's links; shared/team candidates include
+ // threads from other links, so scoping there would drop rows.
+ let link_scope = (matches!(params.shared, SharedEmailFilter::Exclude)
+ && params.team_id.is_none())
+ .then_some(params.link_ids.as_slice());
+ build_matching_threads_cte_body(email_filter, &params.resolved, link_scope)
};
let mut builder = sqlx::QueryBuilder::new("");
@@ -737,6 +743,13 @@ pub(crate) async fn dynamic_email_thread_cursor(
);
qb.build()
+ // Unnamed statement: the SQL text varies per filter shape (and per
+ // interpolated date literal), so cached prepared statements get
+ // little reuse but do flip to generic plans after five executions —
+ // and the generic plan's inflated cost estimate also triggers JIT
+ // compilation per execution. Planning with real bind values each
+ // time keeps the plan stable for ~1ms of planning.
+ .persistent(false)
.try_map(|row| {
Ok(ThreadPreviewCursorDbRow {
id: row.try_get("id")?,
diff --git a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs
index 96019e3633..65bffb9dc9 100644
--- a/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs
+++ b/rust/cloud-storage/email/src/outbound/email_pg_repo/dynamic/tests.rs
@@ -975,7 +975,7 @@ fn test_matching_threads_cte_body_single_sender_uses_union_form() {
let id = Uuid::new_v4();
let expr = Expr::Literal(EmailLiteral::Sender(complete("[email protected]")));
let resolved = resolved_with(&[("[email protected]", id)]);
- let body = build_matching_threads_cte_body(&expr, &resolved).expect("body present");
+ let body = build_matching_threads_cte_body(&expr, &resolved, None).expect("body present");
let debug = body.to_debug_sql();
// Single sender: one UNION branch (no UNION keyword needed), index probe
@@ -1004,7 +1004,7 @@ fn test_matching_threads_cte_body_or_of_kinds_emits_one_union_branch_per_literal
Expr::Literal(EmailLiteral::Recipient(complete("[email protected]"))),
);
let resolved = resolved_with(&[("[email protected]", id)]);
- let body = build_matching_threads_cte_body(&expr, &resolved).expect("body present");
+ let body = build_matching_threads_cte_body(&expr, &resolved, None).expect("body present");
let debug = body.to_debug_sql();
// Three UNIONs join the four branches.
@@ -1028,7 +1028,7 @@ fn test_matching_threads_cte_body_skips_unresolved_complete_branches() {
Expr::Literal(EmailLiteral::Sender(complete("[email protected]"))),
);
let resolved = resolved_with(&[("[email protected]", id)]);
- let body = build_matching_threads_cte_body(&expr, &resolved).expect("body present");
+ let body = build_matching_threads_cte_body(&expr, &resolved, None).expect("body present");
let debug = body.to_debug_sql();
// Only one branch left → no UNION keyword.
@@ -1040,8 +1040,8 @@ fn test_matching_threads_cte_body_skips_unresolved_complete_branches() {
#[test]
fn test_matching_threads_cte_body_partial_emits_ilike_branch_with_email_contacts_join() {
let expr = Expr::Literal(EmailLiteral::Sender(Email::Partial("acme".into())));
- let body =
- build_matching_threads_cte_body(&expr, &ResolvedFilters::empty()).expect("body present");
+ let body = build_matching_threads_cte_body(&expr, &ResolvedFilters::empty(), None)
+ .expect("body present");
let debug = body.to_debug_sql();
assert!(debug.contains("FROM email_contacts c"));
@@ -1055,8 +1055,8 @@ fn test_matching_threads_cte_body_domain_emits_split_part_branch() {
// own copy of the address-match SQL builder. Domain on this path must
// also emit the exact-domain predicate, not ILIKE.
let expr = Expr::Literal(EmailLiteral::Sender(Email::Domain("acme.com".into())));
- let body =
- build_matching_threads_cte_body(&expr, &ResolvedFilters::empty()).expect("body present");
+ let body = build_matching_threads_cte_body(&expr, &ResolvedFilters::empty(), None)
+ .expect("body present");
let debug = body.to_debug_sql();
assert!(debug.contains("FROM email_contacts c"));
@@ -1076,8 +1076,8 @@ fn test_matching_threads_cte_body_domain_recipient_kind_uses_split_part() {
// union-branch builder. Verify the predicate shape is consistent across
// address kinds.
let expr = Expr::Literal(EmailLiteral::Recipient(Email::Domain("acme.com".into())));
- let body =
- build_matching_threads_cte_body(&expr, &ResolvedFilters::empty()).expect("body present");
+ let body = build_matching_threads_cte_body(&expr, &ResolvedFilters::empty(), None)
+ .expect("body present");
let debug = body.to_debug_sql();
assert!(debug.contains("email_message_recipients"));
@@ -1090,8 +1090,8 @@ fn test_matching_threads_cte_body_domain_recipient_kind_uses_split_part() {
#[test]
fn test_matching_threads_cte_body_domain_lowercases_bind_value() {
let expr = Expr::Literal(EmailLiteral::Sender(Email::Domain("AcMe.CoM".into())));
- let body =
- build_matching_threads_cte_body(&expr, &ResolvedFilters::empty()).expect("body present");
+ let body = build_matching_threads_cte_body(&expr, &ResolvedFilters::empty(), None)
+ .expect("body present");
assert!(body.has_bind_string("acme.com"));
assert!(!body.has_bind_string("AcMe.CoM"));
@@ -1109,7 +1109,7 @@ fn test_matching_threads_cte_body_and_of_conjuncts_uses_combined_predicate_form(
Expr::Literal(EmailLiteral::Recipient(complete("[email protected]"))),
);
let resolved = resolved_with(&[("[email protected]", sender_id), ("[email protected]", recipient_id)]);
- let body = build_matching_threads_cte_body(&expr, &resolved).expect("body present");
+ let body = build_matching_threads_cte_body(&expr, &resolved, None).expect("body present");
let debug = body.to_debug_sql();
assert!(debug.contains("SELECT DISTINCT m.thread_id"));
@@ -1131,7 +1131,7 @@ fn test_matching_threads_cte_body_uses_resolved_trash_label_id() {
let resolved = ResolvedFilters::empty()
.with_contact("[email protected]", contact_id)
.with_trash(trash_id);
- let body = build_matching_threads_cte_body(&expr, &resolved).expect("body present");
+ let body = build_matching_threads_cte_body(&expr, &resolved, None).expect("body present");
let debug = body.to_debug_sql();
assert!(debug.contains("ml.label_id = "));
@@ -1144,7 +1144,7 @@ fn test_matching_threads_cte_body_uses_resolved_trash_label_id() {
fn test_matching_threads_cte_body_none_when_no_address_literals() {
let id = Uuid::new_v4();
let expr = Expr::Literal(EmailLiteral::ThreadId(id));
- let body = build_matching_threads_cte_body(&expr, &ResolvedFilters::empty());
+ let body = build_matching_threads_cte_body(&expr, &ResolvedFilters::empty(), None);
assert!(body.is_none());
}
@@ -1322,3 +1322,68 @@ fn test_has_message_literals_false_when_only_date_filters() {
);
assert!(!has_message_literals(&expr));
}
+
+// ---------------------------------------------------------------------------
+// matching_threads link scoping (owned-only, non-team queries)
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_matching_threads_partial_branch_is_link_scoped_when_scope_given() {
+ // Partial (ILIKE) branches join email_contacts/email_messages by text
+ // match; without a link scope they scan every mailbox in the table.
+ // With a scope, both the contact and message sides must be restricted.
+ let links = vec![Uuid::new_v4(), Uuid::new_v4()];
+ let expr = Expr::Literal(EmailLiteral::Sender(Email::Partial("acme".to_string())));
+ let body = build_matching_threads_cte_body(&expr, &ResolvedFilters::empty(), Some(&links))
+ .expect("body present");
+ let debug = body.to_debug_sql();
+
+ assert!(debug.contains("ILIKE"));
+ assert!(debug.contains("c.link_id = ANY("));
+ assert!(debug.contains("m.link_id = ANY("));
+}
+
+#[test]
+fn test_matching_threads_domain_branch_is_link_scoped_when_scope_given() {
+ let links = vec![Uuid::new_v4()];
+ let expr = Expr::Literal(EmailLiteral::Recipient(Email::Domain(
+ "acme.com".to_string(),
+ )));
+ let body = build_matching_threads_cte_body(&expr, &ResolvedFilters::empty(), Some(&links))
+ .expect("body present");
+ let debug = body.to_debug_sql();
+
+ assert!(debug.contains("c.link_id = ANY("));
+ assert!(debug.contains("m.link_id = ANY("));
+}
+
+#[test]
+fn test_matching_threads_combined_fallback_is_link_scoped_when_scope_given() {
+ // Multiple AND conjuncts take the combined-predicate fallback, whose
+ // outer scan is over email_messages — the scope must land there.
+ let id = Uuid::new_v4();
+ let links = vec![Uuid::new_v4()];
+ let expr = Expr::and(
+ Expr::Literal(EmailLiteral::Sender(complete("[email protected]"))),
+ Expr::Literal(EmailLiteral::Recipient(complete("[email protected]"))),
+ );
+ let resolved = resolved_with(&[("[email protected]", id)]);
+ let body =
+ build_matching_threads_cte_body(&expr, &resolved, Some(&links)).expect("body present");
+ let debug = body.to_debug_sql();
+
+ assert!(debug.contains("SELECT DISTINCT m.thread_id FROM email_messages m"));
+ assert!(debug.contains("m.link_id = ANY("));
+}
+
+#[test]
+fn test_matching_threads_unscoped_without_link_scope() {
+ // Shared/team queries pass None — no link_id predicates may appear,
+ // since candidate threads can live on other users' links.
+ let expr = Expr::Literal(EmailLiteral::Sender(Email::Partial("acme".to_string())));
+ let body = build_matching_threads_cte_body(&expr, &ResolvedFilters::empty(), None)
+ .expect("body present");
+ let debug = body.to_debug_sql();
+
+ assert!(!debug.contains("link_id = ANY("));
+}
diff --git a/rust/cloud-storage/macro_db_client/migrations/20260702031628_entity_access_entity_first_index.sql b/rust/cloud-storage/macro_db_client/migrations/20260702031628_entity_access_entity_first_index.sql
new file mode 100644
index 0000000000..291607b9a4
--- /dev/null
+++ b/rust/cloud-storage/macro_db_client/migrations/20260702031628_entity_access_entity_first_index.sql
@@ -0,0 +1,25 @@
+-- no-transaction
+-- Entity-first lookup index for soup dynamic queries.
+--
+-- The soup TopItems arms gate each candidate row with a semi-join:
+-- d.id IN (SELECT ea.entity_id::text FROM entity_access ea
+-- JOIN user_source_ids us ON us.source_id = ea.source_id
+-- WHERE ea.entity_type = '<type>')
+-- When the arm's own filters (owner, notification join, date) are selective,
+-- the cheapest plan probes entity_access per candidate row. That probe needs
+-- an index leading on the TEXT form of entity_id (item ids are TEXT columns),
+-- then entity_type, with source_id available for the membership check.
+--
+-- idx_entity_access_source_type_entity covers the source-first direction;
+-- this covers the entity-first direction.
+--
+-- NOTE: if the concurrent build is interrupted mid-deploy it can leave an
+-- INVALID index behind, and the IF NOT EXISTS rerun will not rebuild it
+-- (this migration must stay a single statement — sqlx sends no-transaction
+-- migrations as one simple-query batch, and a multi-statement batch gets an
+-- implicit transaction, which CONCURRENTLY forbids). If that happens, fix by
+-- hand: DROP INDEX CONCURRENTLY idx_entity_access_entity_text_type_source;
+-- then re-run migrations. Check with:
+-- SELECT indexrelid::regclass FROM pg_index WHERE NOT indisvalid;
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_entity_access_entity_text_type_source
+ ON entity_access ((entity_id::text), entity_type, source_id);
diff --git a/rust/cloud-storage/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs b/rust/cloud-storage/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs
index d7c29da690..5e61073f67 100644
--- a/rust/cloud-storage/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs
+++ b/rust/cloud-storage/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs
@@ -477,6 +477,39 @@ fn clone_expr<T: Clone>(expr: &Expr<T>) -> Expr<T> {
}
}
+/// Push NOT down to the literals (De Morgan), eliminating double negation.
+/// Valid under SQL three-valued logic (De Morgan holds in Kleene logic), so
+/// this never changes which rows match. The payoff is plan quality: Postgres
+/// only turns *syntactically* top-level `NOT EXISTS (...)` conjuncts into
+/// anti-joins — a `NOT (EXISTS a OR EXISTS b)` stays a per-row SubPlan
+/// filter, which both costs more and wrecks the row estimates that
+/// join-order choice depends on.
+fn push_not_inward<T: Clone>(expr: &Expr<T>, negated: bool) -> Expr<T> {
+ match expr {
+ Expr::Not(inner) => push_not_inward(inner, !negated),
+ Expr::And(a, b) => {
+ let (a, b) = (push_not_inward(a, negated), push_not_inward(b, negated));
+ if negated {
+ Expr::or(a, b)
+ } else {
+ Expr::and(a, b)
+ }
+ }
+ Expr::Or(a, b) => {
+ let (a, b) = (push_not_inward(a, negated), push_not_inward(b, negated));
+ if negated {
+ Expr::and(a, b)
+ } else {
+ Expr::or(a, b)
+ }
+ }
+ Expr::Literal(lit) => {
+ let lit = Expr::val(lit.clone());
+ if negated { Expr::is_not(lit) } else { lit }
+ }
+ }
+}
+
fn strip_notification_conjunction<T: Clone>(
expr: &Expr<T>,
notification_predicate: impl Fn(&T) -> Option<NotificationPredicate> + Copy,
@@ -541,8 +574,14 @@ fn build_notification_items_cte(
.map(|item_type| format!("'{item_type}'"))
.collect::<Vec<_>>()
.join(", ");
+ // NOT MATERIALIZED: the set is small (a user's active notifications) and
+ // inlining it lets estimates flow into the arm joins, so the planner can
+ // drive an arm from the notification side instead of nested-looping the
+ // full accessible corpus against it (the materialized fence defaulted the
+ // CTE estimate to ~1 row, which picked a join-filter nested loop that
+ // compared every accessible item against every notification).
builder.push(format!(
- r#"NotificationItems AS MATERIALIZED (
+ r#"NotificationItems AS NOT MATERIALIZED (
SELECT DISTINCT n.event_item_type, n.event_item_id
FROM user_notification un
JOIN notification n ON n.id = un.notification_id
@@ -590,6 +629,7 @@ fn build_document_filter(ast: Option<&Expr<DocumentLiteral>>) -> String {
let Some(expr) = ast else {
return String::new();
};
+ let expr = push_not_inward(expr, false);
let formatting = expr.collapse_frames(|frame| match frame {
filter_ast::ExprFrame::And(a, b) => format!("({a} AND {b})"),
filter_ast::ExprFrame::Or(a, b) => format!("({a} OR {b})"),
@@ -631,10 +671,23 @@ fn build_document_filter(ast: Option<&Expr<DocumentLiteral>>) -> String {
filter_ast::ExprFrame::Literal(DocumentLiteral::IncludeCbmAtmNc(true)) => {
build_task_include_cbm_atm_nc_clause()
}
- // false is equivalent to disabled/no-op.
- filter_ast::ExprFrame::Literal(DocumentLiteral::IncludeCbmAtmNc(false)) => String::new(),
+ // false is equivalent to disabled/no-op. Rendered as TRUE (not an
+ // empty string): an empty operand inside And/Or/Not produces invalid
+ // SQL like `( AND x)` or `(NOT )`, and TRUE also gives NOT the right
+ // meaning (a disabled predicate matches everything, its negation
+ // nothing).
+ filter_ast::ExprFrame::Literal(DocumentLiteral::IncludeCbmAtmNc(false)) => {
+ "TRUE".to_string()
+ }
+ // EXISTS instead of a predicate on the LEFT-JOINed `dt` alias: this
+ // keeps the top clause free of the join when only SubType literals
+ // are present, and `NOT (EXISTS ...)` becomes a plannable anti-join
+ // (the old `(dt.sub_type IS NOT NULL AND ...)` OR-shape collapsed row
+ // estimates to ~1 on broad arms, picking pathological nested loops).
filter_ast::ExprFrame::Literal(DocumentLiteral::SubType(st)) => {
- format!("(dt.sub_type IS NOT NULL AND dt.sub_type = '{st}')")
+ format!(
+ "EXISTS (SELECT 1 FROM document_sub_type dst_f WHERE dst_f.document_id = d.id AND dst_f.sub_type = '{st}')"
+ )
}
filter_ast::ExprFrame::Literal(DocumentLiteral::IsEmailAttachment(true)) => {
r#"EXISTS(SELECT 1 FROM document_email WHERE document_id = d.id)"#
@@ -662,6 +715,7 @@ fn build_chat_filter(ast: Option<&Expr<ChatLiteral>>) -> String {
let Some(expr) = ast else {
return String::new();
};
+ let expr = push_not_inward(expr, false);
let formatting = expr.collapse_frames(|frame| match frame {
filter_ast::ExprFrame::And(a, b) => format!("({a} AND {b})"),
filter_ast::ExprFrame::Or(a, b) => format!("({a} OR {b})"),
@@ -669,13 +723,16 @@ fn build_chat_filter(ast: Option<&Expr<ChatLiteral>>) -> String {
filter_ast::ExprFrame::Literal(ChatLiteral::ProjectId(p)) => {
format!(r#"c."projectId" = '{p}'"#)
}
- // todo? I'm not sure what a chat role filter looks like
- filter_ast::ExprFrame::Literal(ChatLiteral::Role(_r)) => String::new(),
+ // todo? I'm not sure what a chat role filter looks like.
+ // No-op literals render as TRUE, not an empty string — an empty
+ // operand inside And/Or/Not produces invalid SQL like `( AND x)`.
+ filter_ast::ExprFrame::Literal(ChatLiteral::Role(_r)) => "TRUE".to_string(),
filter_ast::ExprFrame::Literal(ChatLiteral::ChatId(i)) => format!("c.id = '{i}'"),
filter_ast::ExprFrame::Literal(ChatLiteral::Owner(o)) => {
format!(r#"c."userId" = '{o}'"#)
}
- filter_ast::ExprFrame::Literal(ChatLiteral::Importance(true)) => String::new(),
+ // all chats are important
+ filter_ast::ExprFrame::Literal(ChatLiteral::Importance(true)) => "TRUE".to_string(),
// all chats are important, so if importance is false, exclude them
filter_ast::ExprFrame::Literal(ChatLiteral::Importance(false)) => "1=0".to_string(),
filter_ast::ExprFrame::Literal(ChatLiteral::NotificationDone(done)) => {
@@ -702,6 +759,7 @@ fn build_project_filter(ast: Option<&Expr<ProjectLiteral>>) -> String {
let Some(expr) = ast else {
return String::new();
};
+ let expr = push_not_inward(expr, false);
let formatting = expr.collapse_frames(|frame| match frame {
filter_ast::ExprFrame::And(a, b) => format!("({a} AND {b})"),
filter_ast::ExprFrame::Or(a, b) => format!("({a} OR {b})"),
@@ -715,7 +773,9 @@ fn build_project_filter(ast: Option<&Expr<ProjectLiteral>>) -> String {
filter_ast::ExprFrame::Literal(ProjectLiteral::Owner(o)) => {
format!(r#"p."userId" = '{o}'"#)
}
- filter_ast::ExprFrame::Literal(ProjectLiteral::Importance(true)) => String::new(),
+ // all projects are important; TRUE (not empty) keeps the literal
+ // valid SQL inside And/Or/Not.
+ filter_ast::ExprFrame::Literal(ProjectLiteral::Importance(true)) => "TRUE".to_string(),
// all projects are important, so if importance is false, exclude them
filter_ast::ExprFrame::Literal(ProjectLiteral::Importance(false)) => "1=0".to_string(),
filter_ast::ExprFrame::Literal(ProjectLiteral::NotificationDone(done)) => {
@@ -876,32 +936,60 @@ fn top_needs_user_history(sort_method: SimpleSortMethod) -> bool {
)
}
-fn document_top_clause(sort_method: SimpleSortMethod) -> String {
+/// Per-arm access gate. Semantically identical to the old
+/// `AccessibleItems AS MATERIALIZED` CTE + `INNER JOIN` (an item is visible
+/// iff at least one `entity_access` row matches one of the user's sources;
+/// IN dedupes exactly like the CTE's DISTINCT did), but expressed as a
+/// flattenable semi-join so the planner can pick a direction per arm:
+/// hash the user's accessible set when the arm is broad, or probe
+/// `entity_access` per candidate row (via
+/// `idx_entity_access_entity_text_type_source`) when the arm's own filters
+/// are selective. The materialized form pinned the worst plan — the whole
+/// corpus was computed and probed against the item table on every page.
+fn access_semi_join(id_sql: &str, entity_type: &str) -> String {
+ format!(
+ r#"{id_sql} IN (
+ SELECT ea.entity_id::text
+ FROM entity_access ea
+ JOIN user_source_ids us ON us.source_id = ea.source_id
+ WHERE ea.entity_type = '{entity_type}'
+ )"#
+ )
+}
+
+fn document_top_clause(sort_method: SimpleSortMethod, needs_sub_type_join: bool) -> String {
+ let sub_type_join = if needs_sub_type_join {
+ r#" LEFT JOIN document_sub_type dt ON dt.document_id = d.id
+"#
+ } else {
+ ""
+ };
format!(
r#"
SELECT
'document'::text as item_type,
d.id,
{}::timestamptz as sort_ts
- FROM AccessibleItems ai
- INNER JOIN "Document" d ON d.id = ai.item_id AND ai.item_type = 'document'
- LEFT JOIN document_sub_type dt ON dt.document_id = d.id
-"#,
- top_sort_expr("d", sort_method)
+ FROM "Document" d
+{}"#,
+ top_sort_expr("d", sort_method),
+ sub_type_join
)
}
-fn document_top_where_clause(sort_method: SimpleSortMethod) -> &'static str {
- if top_needs_user_history(sort_method) {
- r#"
- LEFT JOIN "UserHistory" uh ON uh."itemId" = d.id AND uh."itemType" = 'document' AND uh."userId" = $1
- WHERE d."deletedAt" IS NULL
+fn document_top_where_clause(sort_method: SimpleSortMethod) -> String {
+ let user_history_join = if top_needs_user_history(sort_method) {
+ r#" LEFT JOIN "UserHistory" uh ON uh."itemId" = d.id AND uh."itemType" = 'document' AND uh."userId" = $1
"#
} else {
- r#"
- WHERE d."deletedAt" IS NULL
-"#
- }
+ ""
+ };
+ format!(
+ r#"{user_history_join} WHERE d."deletedAt" IS NULL
+ AND {}
+"#,
+ access_semi_join("d.id", "document")
+ )
}
fn chat_top_clause(sort_method: SimpleSortMethod) -> String {
@@ -917,17 +1005,20 @@ fn chat_top_clause(sort_method: SimpleSortMethod) -> String {
'chat'::text as item_type,
c.id,
{}::timestamptz as sort_ts
- FROM AccessibleItems ai
- INNER JOIN "Chat" c ON c.id = ai.item_id AND ai.item_type = 'chat'
+ FROM "Chat" c
{}"#,
top_sort_expr("c", sort_method),
user_history_join
)
}
-fn chat_top_where_clause() -> &'static str {
- r#" WHERE c."deletedAt" IS NULL
-"#
+fn chat_top_where_clause() -> String {
+ format!(
+ r#" WHERE c."deletedAt" IS NULL
+ AND {}
+"#,
+ access_semi_join("c.id", "chat")
+ )
}
fn project_top_clause(sort_method: SimpleSortMethod) -> String {
@@ -946,17 +1037,20 @@ fn project_top_clause(sort_method: SimpleSortMethod) -> String {
'project'::text as item_type,
p.id,
{}::timestamptz as sort_ts
- FROM AccessibleItems ai
- INNER JOIN "Project" p ON p.id = ai.item_id AND ai.item_type = 'project'
+ FROM "Project" p
{}"#,
top_sort_expr("p", sort_method),
user_history_join
)
}
-fn project_top_where_clause() -> &'static str {
- r#" WHERE p."deletedAt" IS NULL
-"#
+fn project_top_where_clause() -> String {
+ format!(
+ r#" WHERE p."deletedAt" IS NULL
+ AND {}
+"#,
+ access_semi_join("p.id", "project")
+ )
}
fn push_accessible_items_cte(
@@ -1112,13 +1206,6 @@ fn build_query(
filter_ast.project_filter.as_deref()
};
- push_accessible_items_cte(
- &mut builder,
- include_documents,
- include_chats,
- include_projects,
- );
-
if let Some(predicate) = optimized_notification_predicate {
let mut item_types = Vec::with_capacity(3);
if include_documents && document_notification.is_some() {
@@ -1141,12 +1228,15 @@ fn build_query(
if include_documents {
push_union_separator(&mut builder, &mut needs_separator);
- // Document top clause (lightweight)
- builder.push(document_top_clause(sort_method));
+ // Document top clause (lightweight). The document_sub_type join is
+ // only needed by filters that reference `dt` (Importance / CBM);
+ // SubType literals render as their own EXISTS probes.
+ let needs_task_property_joins = document_filter_needs_task_property_joins(document_filter);
+ builder.push(document_top_clause(sort_method, needs_task_property_joins));
if optimized_notification_predicate.is_some() && document_notification.is_some() {
builder.push(build_notification_join("d", "document"));
}
- if document_filter_needs_task_property_joins(document_filter) {
+ if needs_task_property_joins {
builder.push(DOCUMENT_TASK_PROPERTY_JOINS);
}
builder.push(document_top_where_clause(sort_method));
@@ -1488,6 +1578,13 @@ pub(crate) async fn expanded_dynamic_cursor_soup(
.bind(completed_option_id)
.bind(status_property_id)
.bind(assignees_property_id)
+ // Unnamed statement: the SQL text varies per filter shape and per
+ // interpolated literal (dates change daily), so a cached prepared
+ // statement is rarely reused but flips to a generic plan after five
+ // executions — and the generic plan misestimates the CTEs badly
+ // enough to run 10x slower. Planning with the real bind values every
+ // time costs ~1ms and keeps the plan stable.
+ .persistent(false)
.try_map(|row| SoupRow::from_row(&row)?.into_soup_item())
.fetch_all(db)
.await?;
@@ -1820,6 +1917,10 @@ pub async fn expanded_dynamic_cursor_soup_grouped(
}
let rows: Vec<GroupedSoupRow> = query
+ // Unnamed statement — same rationale as expanded_dynamic_cursor_soup:
+ // filter-shaped SQL text gets no reuse from the statement cache but
+ // does get generic-plan flips.
+ .persistent(false)
.try_map(|row| GroupedSoupRow::from_row(&row))
.fetch_all(db)
.await?;