Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(soup): linearize the notified-at candidate query (#6162)
gbirman158ddaa90497
diff --git a/crates/soup/src/outbound/pg_soup_repo/candidate_gates.rs b/crates/soup/src/outbound/pg_soup_repo/candidate_gates.rsindex aa5f230841..265a3b4481 100644--- a/crates/soup/src/outbound/pg_soup_repo/candidate_gates.rs+++ b/crates/soup/src/outbound/pg_soup_repo/candidate_gates.rs@@ -10,7 +10,7 @@use filter_ast::Expr;use item_filters::ast::EntityFilterAst;-use item_filters::ast::channel::ChannelLiteral;+use item_filters::ast::channel::{ChannelLiteral, ChannelThreadLiteral};use item_filters::ast::email::EmailLiteral;use item_filters::ast::properties::{PropertyEntityType, properties_filter_matches_propertyless};use system_properties::SystemPropertyKey;@@ -41,7 +41,10 @@ fn and_conjuncts<'a, T>(expr: &'a Expr<T>, out: &mut Vec<&'a T>) {/// Renders the implied conjuncts of `tree` that `fold` knows how to express/// as ` AND ...` clauses, in tree order.-fn implied_conjuncts_sql<T>(tree: Option<&Expr<T>>, fold: impl Fn(&T) -> Option<String>) -> String {+pub(super) fn implied_conjuncts_sql<T>(+ tree: Option<&Expr<T>>,+ fold: impl Fn(&T) -> Option<String>,+) -> String {let Some(tree) = tree else {return String::new();};@@ -190,8 +193,29 @@ pub(super) fn channel_gate(id_sql: &str, filter: Option<&EntityFilterAst>) -> St/// Channel-thread rows are root messages; one is visible iff it is undeleted/// and the caller is an active participant of its channel. The thread tree-/// folds in the channels crate, which hydration applies in full.-pub(super) fn channel_thread_gate(id_sql: &str) -> String {+/// folds in the channels crate, which hydration applies in full; the+/// notification-state conjuncts it implies are pre-applied here over the+/// thread's own notifications (the channel notifications naming this root+/// as their secondary item, the same rows the channels fold predicates on).+pub(super) fn channel_thread_gate(id_sql: &str, filter: Option<&EntityFilterAst>) -> String {+ let implied = implied_conjuncts_sql(+ filter.and_then(|f| f.channel_thread_filter.as_deref()),+ |literal| {+ let predicate = match literal {+ ChannelThreadLiteral::NotificationDone(done) => NotificationPredicate::Done(*done),+ ChannelThreadLiteral::NotificationSeen(seen) => NotificationPredicate::Seen(*seen),+ _ => return None,+ };+ Some(build_notification_exists_clause(+ "m.channel_id",+ "channel",+ &format!(+ "n.secondary_event_item_type = 'channel_message' AND n.secondary_event_item_id = m.id::text AND {}",+ predicate.sql()+ ),+ ))+ },+ );uuid_guarded(id_sql,format!(@@ -204,6 +228,7 @@ pub(super) fn channel_thread_gate(id_sql: &str) -> String {WHERE m.id = {id_sql}::uuidAND m.thread_id IS NULLAND m.deleted_at IS NULL+ {implied})"#),)diff --git a/crates/soup/src/outbound/pg_soup_repo/notified.rs b/crates/soup/src/outbound/pg_soup_repo/notified.rsindex 7c2b74b226..5d77a94a17 100644--- a/crates/soup/src/outbound/pg_soup_repo/notified.rs+++ b/crates/soup/src/outbound/pg_soup_repo/notified.rs@@ -4,9 +4,9 @@//! Every candidate is gated on existence, deletion and access, and on the//! request's filters where soup owns the fold (documents, chats, projects,//! calendar events, properties). Channel, channel-thread, email,-//! foreign-entity and reminder candidates are gated on access (plus, for-//! channels and emails, the notification-state and importance conjuncts-//! their trees imply): their filter trees fold in their own domains' query+//! foreign-entity and reminder candidates are gated on access (plus the+//! notification-state and, for emails, importance conjuncts their trees+//! imply): their filter trees fold in their own domains' query//! builders, so the service applies them in full when it hydrates the page//! through those legs and drops the candidates that fail, refilling from the//! next candidate page. A short page from this query therefore still means@@ -30,9 +30,9 @@ use std::str::FromStr;use crate::domain::models::{NotifiedEntity, NotifiedSoupRequest};use crate::outbound::pg_soup_repo::candidate_gates::{- channel_gate, channel_thread_gate, chat_gate, document_gate, email_gate, includes_channels,- includes_chats, includes_documents, includes_email_threads, includes_projects, project_gate,- uuid_guarded,+ channel_gate, channel_thread_gate, chat_gate, document_gate, email_gate, implied_conjuncts_sql,+ includes_channels, includes_chats, includes_documents, includes_email_threads,+ includes_projects, project_gate, uuid_guarded,};use crate::outbound::pg_soup_repo::expanded::dynamic::{build_notification_done_clause, build_notification_seen_clause, build_properties_filter,@@ -102,8 +102,23 @@ fn calendar_event_gate(filter: Option<&EntityFilterAst>) -> String {/// Foreign entities are visible through the source they were stored for:/// the caller themselves or a team they belong to (`$7`/`$8` are the-/// parallel id / auth-entity arrays of those sources).-fn foreign_entity_gate() -> String {+/// parallel id / auth-entity arrays of those sources). The foreign-entity+/// tree folds in its own crate, which hydration applies in full; the+/// notification-state conjuncts it implies are pre-applied here.+fn foreign_entity_gate(filter: Option<&EntityFilterAst>) -> String {+ let implied =+ implied_conjuncts_sql(+ filter.and_then(|f| f.foreign_entity_filter.as_deref()),+ |literal| match literal {+ ForeignEntityLiteral::NotificationDone(done) => Some(+ build_notification_done_clause("fe.id", "foreign_entity", *done),+ ),+ ForeignEntityLiteral::NotificationSeen(seen) => Some(+ build_notification_seen_clause("fe.id", "foreign_entity", *seen),+ ),+ _ => None,+ },+ );uuid_guarded(ID_SQL,format!(@@ -115,6 +130,7 @@ fn foreign_entity_gate() -> String {WHERE source.id = fe.stored_for_idAND source.auth_entity = fe.stored_for_auth_entity)+ {implied})"#),)@@ -199,11 +215,14 @@ fn included_types(req: &NotifiedSoupRequest<'_>) -> Vec<&'static str> {/// Fetches one page of notified-at candidates.///-/// Shape: keyset scan of the user's live notifications in `created_at DESC`-/// order over the `notified` CTE's derived entity key, keeping only each-/// entity's newest notification (the `NOT EXISTS` group-max), gated per-/// entity type on existence/deletion/access and the soup-owned filters so-/// `LIMIT` lands after gating.+/// Shape: the user's live notifications collapse to one row per derived+/// entity key, its newest notification (the `latest` window's first row per+/// partition), which the fenced subquery orders and keysets; the outer query+/// gates those per entity type on existence/deletion/access and the+/// soup-owned filters in that order until `LIMIT` is met. The `OFFSET 0`+/// fence keeps the gates outside the dedupe and the inner order intact, so+/// the planner neither pushes the gates down onto every notification nor+/// re-sorts after them.////// `user_notification.created_at` is a naive `TIMESTAMP` written in UTC, so/// the keyset binds and the returned value stay naive and are re-tagged as@@ -245,18 +264,29 @@ pub(super) async fn notified_soup_page(JOIN notification n ON n.id = un.notification_idWHERE un.user_id = $1AND un.deleted_at IS NULL+ ),+ latest AS NOT MATERIALIZED (+ SELECT+ entity_type,+ entity_id,+ created_at,+ row_number() OVER (+ PARTITION BY entity_type, entity_id+ ORDER BY created_at DESC, notification_id DESC+ ) AS rn+ FROM notified+ WHERE entity_type = ANY($2))- SELECT nc.entity_type, nc.entity_id, nc.created_at AS notified_at- FROM notified nc- WHERE nc.entity_type = ANY($2)- AND ($3::timestamp IS NULL OR (nc.created_at, nc.entity_id) < ($3, $4))- AND NOT EXISTS (- SELECT 1 FROM notified newer- WHERE newer.entity_type = nc.entity_type- AND newer.entity_id = nc.entity_id- AND (newer.created_at, newer.notification_id) > (nc.created_at, nc.notification_id)- )- AND CASE nc.entity_type+ SELECT nc.entity_type, nc.entity_id, nc.notified_at+ FROM (+ SELECT entity_type, entity_id, created_at AS notified_at+ FROM latest+ WHERE rn = 1+ AND ($3::timestamp IS NULL OR (created_at, entity_id) < ($3, $4))+ ORDER BY created_at DESC, entity_id DESC+ OFFSET 0+ ) nc+ WHERE CASE nc.entity_typeWHEN 'document' THEN {document_gate}WHEN 'chat' THEN {chat_gate}WHEN 'project' THEN {project_gate}@@ -268,17 +298,17 @@ pub(super) async fn notified_soup_page(WHEN 'reminder' THEN {reminder_gate}ELSE FALSEEND- ORDER BY nc.created_at DESC, nc.entity_id DESC+ ORDER BY nc.notified_at DESC, nc.entity_id DESCLIMIT $6"#,document_gate = document_gate(ID_SQL, req.filter),chat_gate = chat_gate(ID_SQL, req.filter),project_gate = project_gate(ID_SQL, req.filter),channel_gate = channel_gate(ID_SQL, req.filter),- channel_thread_gate = channel_thread_gate(ID_SQL),+ channel_thread_gate = channel_thread_gate(ID_SQL, req.filter),email_gate = email_gate(ID_SQL, req.filter),calendar_event_gate = calendar_event_gate(req.filter),- foreign_entity_gate = foreign_entity_gate(),+ foreign_entity_gate = foreign_entity_gate(req.filter),reminder_gate = reminder_gate(),);diff --git a/crates/soup/src/outbound/pg_soup_repo/notified/test.rs b/crates/soup/src/outbound/pg_soup_repo/notified/test.rsindex 5ffb91c968..ea3da616d4 100644--- a/crates/soup/src/outbound/pg_soup_repo/notified/test.rs+++ b/crates/soup/src/outbound/pg_soup_repo/notified/test.rs@@ -25,6 +25,8 @@ const REMINDER_R1: &str = "88888888-aaaa-aaaa-aaaa-aaaaaaaaaaaa";const LINK_1: &str = "55555555-aaaa-aaaa-aaaa-aaaaaaaaaaaa";const TEAM_T: &str = "eeeeeeee-1111-1111-1111-111111111111";const CHANNEL_X_INVITE: &str = "0190a000-0000-7000-8000-000000000000";+const THREAD_M_MENTION: &str = "0190a000-0000-7000-8000-000000000006";+const PR_F1_EVENT: &str = "0190a000-0000-7000-8000-000000000003";const EVERYTHING: NotifiedHydratableTypes = NotifiedHydratableTypes {channels: true,@@ -363,6 +365,72 @@ async fn channel_conjuncts_ignore_thread_scoped_notifications(Ok(())}+/// Channel-thread and foreign-entity trees fold at hydration, but the+/// notification-state conjuncts they imply pre-filter the candidates over+/// the rows their folds predicate on: the thread's own notifications (not+/// its channel's) and the foreign entity's notifications.+#[sqlx::test(+ fixtures(path = "../../../../fixtures", scripts("notified_at")),+ migrator = "MACRO_DB_MIGRATIONS"+)]+async fn thread_and_foreign_entity_conjuncts_prefilter_candidates(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ use item_filters::ast::channel::ChannelThreadLiteral;++ sqlx::query("UPDATE user_notification SET done = TRUE WHERE notification_id = ANY($1::uuid[])")+ .bind(vec![+ Uuid::parse_str(THREAD_M_MENTION)?,+ Uuid::parse_str(PR_F1_EVENT)?,+ ])+ .execute(&pool)+ .await?;++ let link_ids = [Uuid::parse_str(LINK_1)?];+ let sources = sources();+ let thread = (EntityType::ChannelMessage, THREAD_M.to_string());+ let pr_f1 = (EntityType::ForeignEntity, PR_F1.to_string());+ let pr_f3 = (EntityType::ForeignEntity, PR_F3.to_string());++ // Not-done trees drop the done mention and the done pull request; the+ // channel's live invite does not keep the thread row.+ let filter = EntityFilterAst {+ channel_thread_filter: Some(Arc::new(filter_ast::Expr::val(+ ChannelThreadLiteral::NotificationDone(false),+ ))),+ foreign_entity_filter: Some(Arc::new(filter_ast::Expr::val(+ ForeignEntityLiteral::NotificationDone(false),+ ))),+ ..EntityFilterAst::default()+ };+ let page =+ notified_soup_page(&pool, req(Some(&filter), &link_ids, &sources, EVERYTHING)).await?;+ assert!(!keys(&page).contains(&thread));+ assert!(!keys(&page).contains(&pr_f1));+ assert!(keys(&page).contains(&pr_f3));+ assert!(keys(&page).contains(&(EntityType::Channel, CHANNEL_X.to_string())));+ assert_eq!(page.len(), 9);++ // Done trees keep them and drop the live pull request instead.+ let filter = EntityFilterAst {+ channel_thread_filter: Some(Arc::new(filter_ast::Expr::val(+ ChannelThreadLiteral::NotificationDone(true),+ ))),+ foreign_entity_filter: Some(Arc::new(filter_ast::Expr::val(+ ForeignEntityLiteral::NotificationDone(true),+ ))),+ ..EntityFilterAst::default()+ };+ let page =+ notified_soup_page(&pool, req(Some(&filter), &link_ids, &sources, EVERYTHING)).await?;+ assert!(keys(&page).contains(&thread));+ assert!(keys(&page).contains(&pr_f1));+ assert!(!keys(&page).contains(&pr_f3));+ assert_eq!(page.len(), 10);++ Ok(())+}+#[test]fn every_type_with_every_leg_active() {let link_ids = [Uuid::from_u128(1)];