Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/fix/less query (#6890)
Sean Aye633a16b6ea2d
diff --git a/.sqlx/query-1ace84dbc2133ca52b88a247f96babb8b79a15c91403762d37651d2b0053fa25.json b/.sqlx/query-1ace84dbc2133ca52b88a247f96babb8b79a15c91403762d37651d2b0053fa25.jsonnew file mode 100644index 0000000000..e1c160f0b1--- /dev/null+++ b/.sqlx/query-1ace84dbc2133ca52b88a247f96babb8b79a15c91403762d37651d2b0053fa25.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO user_notification (user_id, notification_id, created_at, sent, state) SELECT $1, id, created_at, TRUE, 'unseen' FROM notification WHERE id = ANY($2::uuid[])",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "UuidArray"+ ]+ },+ "nullable": []+ },+ "hash": "1ace84dbc2133ca52b88a247f96babb8b79a15c91403762d37651d2b0053fa25"+}diff --git a/.sqlx/query-217acdbed6242ff0c4b45c2d2d20fabf7819587af3f045b0bbcd500b6f9c5107.json b/.sqlx/query-217acdbed6242ff0c4b45c2d2d20fabf7819587af3f045b0bbcd500b6f9c5107.jsonnew file mode 100644index 0000000000..32825ab260--- /dev/null+++ b/.sqlx/query-217acdbed6242ff0c4b45c2d2d20fabf7819587af3f045b0bbcd500b6f9c5107.json@@ -0,0 +1,12 @@+{+ "db_name": "PostgreSQL",+ "query": "ANALYZE",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": []+ },+ "nullable": []+ },+ "hash": "217acdbed6242ff0c4b45c2d2d20fabf7819587af3f045b0bbcd500b6f9c5107"+}diff --git a/.sqlx/query-55c45e9910ee3872af6feffea842117180eeb1eb8aa32f87538871443105a41b.json b/.sqlx/query-55c45e9910ee3872af6feffea842117180eeb1eb8aa32f87538871443105a41b.jsonnew file mode 100644index 0000000000..1ff1ebad09--- /dev/null+++ b/.sqlx/query-55c45e9910ee3872af6feffea842117180eeb1eb8aa32f87538871443105a41b.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO \"Document\" (id, name, owner, \"createdAt\", \"updatedAt\")\n SELECT id, 'notified benchmark', $2, '2024-06-02', '2024-06-02'\n FROM unnest($1::text[]) id",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "TextArray",+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "55c45e9910ee3872af6feffea842117180eeb1eb8aa32f87538871443105a41b"+}diff --git a/.sqlx/query-a990c42683ac176337a16987d448fe562b420e692135be77f966c52e2a2c2495.json b/.sqlx/query-a990c42683ac176337a16987d448fe562b420e692135be77f966c52e2a2c2495.jsonnew file mode 100644index 0000000000..3e31daae59--- /dev/null+++ b/.sqlx/query-a990c42683ac176337a16987d448fe562b420e692135be77f966c52e2a2c2495.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO entity_access (entity_id, entity_type, source_id, source_type, access_level) SELECT id::uuid, 'document', $2, 'user', 'owner' FROM unnest($1::text[]) id",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "TextArray",+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "a990c42683ac176337a16987d448fe562b420e692135be77f966c52e2a2c2495"+}diff --git a/.sqlx/query-ccffcdc346ba29f62505162f7df86e7352e50be067678781ce6386cf59cc61fd.json b/.sqlx/query-ccffcdc346ba29f62505162f7df86e7352e50be067678781ce6386cf59cc61fd.jsonnew file mode 100644index 0000000000..1fadfe4c60--- /dev/null+++ b/.sqlx/query-ccffcdc346ba29f62505162f7df86e7352e50be067678781ce6386cf59cc61fd.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO notification (id, notification_event_type, event_item_id, event_item_type, service_sender, created_at)\n SELECT id, 'document_comment', entity_id, 'document', 'test',\n timestamp '2024-06-02' + position * interval '1 second'\n FROM unnest($1::uuid[], $2::text[]) WITH ORDINALITY seeded(id, entity_id, position)",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray",+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "ccffcdc346ba29f62505162f7df86e7352e50be067678781ce6386cf59cc61fd"+}diff --git a/.sqlx/query-d2408d7cca5a71f14325af6e58c42cc70cd7ec651701f49573c944f0a1610213.json b/.sqlx/query-d2408d7cca5a71f14325af6e58c42cc70cd7ec651701f49573c944f0a1610213.jsonnew file mode 100644index 0000000000..0191a1ac91--- /dev/null+++ b/.sqlx/query-d2408d7cca5a71f14325af6e58c42cc70cd7ec651701f49573c944f0a1610213.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT current_setting('work_mem')",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "current_setting",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ null+ ]+ },+ "hash": "d2408d7cca5a71f14325af6e58c42cc70cd7ec651701f49573c944f0a1610213"+}diff --git a/.sqlx/query-df1f1d15d442789a5b9c81cdddf44d88d5748499cc48865023ddc1ff1587d0f6.json b/.sqlx/query-df1f1d15d442789a5b9c81cdddf44d88d5748499cc48865023ddc1ff1587d0f6.jsonnew file mode 100644index 0000000000..7fe6da3923--- /dev/null+++ b/.sqlx/query-df1f1d15d442789a5b9c81cdddf44d88d5748499cc48865023ddc1ff1587d0f6.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT version()",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "version",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ null+ ]+ },+ "hash": "df1f1d15d442789a5b9c81cdddf44d88d5748499cc48865023ddc1ff1587d0f6"+}diff --git a/.sqlx/query-eba001bed3f012a9b5536a865d1dbf5bb3196a757e160895209a2d98db1b45cd.json b/.sqlx/query-eba001bed3f012a9b5536a865d1dbf5bb3196a757e160895209a2d98db1b45cd.jsonnew file mode 100644index 0000000000..dca549a5b9--- /dev/null+++ b/.sqlx/query-eba001bed3f012a9b5536a865d1dbf5bb3196a757e160895209a2d98db1b45cd.json@@ -0,0 +1,12 @@+{+ "db_name": "PostgreSQL",+ "query": "SET work_mem = '16MB'",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": []+ },+ "nullable": []+ },+ "hash": "eba001bed3f012a9b5536a865d1dbf5bb3196a757e160895209a2d98db1b45cd"+}diff --git a/crates/soup/fixtures/notified_latest.sql b/crates/soup/fixtures/notified_latest.sqlnew file mode 100644index 0000000000..b413ff3cd0--- /dev/null+++ b/crates/soup/fixtures/notified_latest.sql@@ -0,0 +1,30 @@+-- Extends notified_at: timestamp ties, an older notification with a larger id,+-- and a chat sharing a document's id. Notification-table timestamps deliberately+-- differ from the recipient timestamps that define notified_at.++INSERT INTO public."Chat" ("id", "name", "userId", "createdAt", "updatedAt")+VALUES ('11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'Same-id chat', 'macro|[email protected]', '2024-06-01 10:00:00', '2024-06-01 10:00:00');++INSERT INTO public.entity_access (entity_id, entity_type, source_id, source_type, access_level)+VALUES ('11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'chat', 'macro|[email protected]', 'user', 'owner');++INSERT INTO public.notification (id, notification_event_type, event_item_id, event_item_type, service_sender, created_at)+VALUES+('0190b000-0000-7000-8000-000000000001', 'document_comment', '11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'document', 'test', '2024-06-01 11:00:00'),+('0190b000-0000-7000-8000-000000000002', 'document_comment', '11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'document', 'test', '2024-06-01 11:00:00'),+('0190b000-0000-7000-8000-000000000003', 'document_comment', '11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'document', 'test', '2024-06-01 11:00:00'),+('0190b000-0000-7000-8000-000000000004', 'chat_complete', '11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'chat', 'test', '2024-06-01 11:00:00');++INSERT INTO public.user_notification (user_id, notification_id, created_at, sent, state)+VALUES+('macro|[email protected]', '0190b000-0000-7000-8000-000000000001', '2024-06-01 10:09:00', TRUE, 'done'),+('macro|[email protected]', '0190b000-0000-7000-8000-000000000002', '2024-06-01 10:09:00', TRUE, 'done'),+('macro|[email protected]', '0190b000-0000-7000-8000-000000000003', '2024-06-01 10:01:00', TRUE, 'unseen'),+('macro|[email protected]', '0190b000-0000-7000-8000-000000000004', '2024-06-01 10:10:00', TRUE, 'unseen');++-- Only old document notifications match "unseen". The document must still be+-- ordered at T9, not at the matching notification's T1.+UPDATE public.user_notification+SET state = 'done'+WHERE user_id = 'macro|[email protected]'+ AND notification_id = '0190a000-0000-7000-8000-000000000009';diff --git a/crates/soup/src/domain/models.rs b/crates/soup/src/domain/models.rsindex 6b598d861f..b9c248db65 100644--- a/crates/soup/src/domain/models.rs+++ b/crates/soup/src/domain/models.rs@@ -1,5 +1,7 @@pub mod grouping;+mod exclusions;+#[cfg(test)]mod test;@@ -427,6 +429,11 @@ impl SoupRequest<Option<EntityFilterAst>> {if self.link_ids.is_empty() {return None;}+ // CRM-scoped requests must still reach email's authorization precheck,+ // even when their id filter cannot match any threads.+ if crm_scope.is_none() && exclusions::email(entity_ast) {+ return None;+ }// A properties filter that cannot match threads makes the email// branch empty, so the sub-request is skipped. Otherwise it is ANDed@@ -514,6 +521,9 @@ impl SoupRequest<Option<EntityFilterAst>> {}pub(crate) fn build_call_request(&self) -> Option<GetCallRecordsRequest> {+ if exclusions::call(self.entity_ast()) {+ return None;+ }// A def-less tag filter (entity_type None on every literal) applies to// calls, which carry tags; it is folded into the call filter below and// rendered as a `properties.values` EXISTS by the call query. Any other@@ -695,7 +705,7 @@ impl SoupRequest<Option<EntityFilterAst>> {}pub(crate) fn build_comms_request(&self) -> Option<GetChannelsRequest> {- if self.properties_filter_blocks_propertyless() {+ if self.properties_filter_blocks_propertyless() || exclusions::channel(self.entity_ast()) {return None;}Some(GetChannelsRequest {@@ -736,7 +746,9 @@ impl SoupRequest<Option<EntityFilterAst>> {}pub(crate) fn build_comms_thread_request(&self) -> Option<GetThreadReplyRowsRequest> {- if self.properties_filter_blocks_propertyless() {+ if self.properties_filter_blocks_propertyless()+ || exclusions::channel_thread(self.entity_ast())+ {return None;}let query = match &self.cursor {@@ -782,7 +794,9 @@ impl SoupRequest<Option<EntityFilterAst>> {}pub(crate) fn build_foreign_entity_query(&self) -> Option<ForeignEntityListQuery> {- if self.properties_filter_blocks_propertyless() {+ if self.properties_filter_blocks_propertyless()+ || exclusions::foreign_entity(self.entity_ast())+ {return None;}match &self.cursor {diff --git a/crates/soup/src/domain/models/exclusions.rs b/crates/soup/src/domain/models/exclusions.rsnew file mode 100644index 0000000000..9df854813d--- /dev/null+++ b/crates/soup/src/domain/models/exclusions.rs@@ -0,0 +1,61 @@+//! Recognize the nil-primary-id filters clients use to exclude an entity type.+//! Prune these legs before calling their domain services, not after acquiring+//! database connections for queries that can only return an empty page.++use filter_ast::{Expr, ExprFrame};+use item_filters::ast::{+ EntityFilterAst,+ call::CallLiteral,+ channel::{ChannelLiteral, ChannelThreadLiteral},+ email::EmailLiteral,+ foreign_entity::ForeignEntityLiteral,+};+use recursion::CollapsibleExt;++/// Only prove exclusions: an unknown literal or a negation may still match.+/// In particular, `nil OR real_id` must never suppress the whole request.+fn excludes_all<L: Clone>(tree: Option<&Expr<L>>, excludes: impl Fn(L) -> bool) -> bool {+ tree.is_some_and(|tree| {+ tree.collapse_frames(|frame| match frame {+ ExprFrame::And(a, b) => a || b,+ ExprFrame::Or(a, b) => a && b,+ ExprFrame::Not(_) => false,+ ExprFrame::Literal(literal) => excludes(literal),+ })+ })+}++pub(super) fn email(filter: Option<&EntityFilterAst>) -> bool {+ excludes_all(+ filter.and_then(|filter| filter.email_filter.tree.as_deref()),+ |literal| matches!(literal, EmailLiteral::ThreadId(id) if id.is_nil()),+ )+}++pub(super) fn channel(filter: Option<&EntityFilterAst>) -> bool {+ excludes_all(+ filter.and_then(|filter| filter.channel_filter.as_deref()),+ |literal| matches!(literal, ChannelLiteral::ChannelId(id) if id.is_nil()),+ )+}++pub(super) fn channel_thread(filter: Option<&EntityFilterAst>) -> bool {+ excludes_all(+ filter.and_then(|filter| filter.channel_thread_filter.as_deref()),+ |literal| matches!(literal, ChannelThreadLiteral::ThreadId(id) if id.is_nil()),+ )+}++pub(super) fn call(filter: Option<&EntityFilterAst>) -> bool {+ excludes_all(+ filter.and_then(|filter| filter.call_filter.as_deref()),+ |literal| matches!(literal, CallLiteral::CallId(id) if id.is_nil()),+ )+}++pub(super) fn foreign_entity(filter: Option<&EntityFilterAst>) -> bool {+ excludes_all(+ filter.and_then(|filter| filter.foreign_entity_filter.as_deref()),+ |literal| matches!(literal, ForeignEntityLiteral::Id(id) if id.is_nil()),+ )+}diff --git a/crates/soup/src/domain/models/test.rs b/crates/soup/src/domain/models/test.rsindex 395f3d47bc..793de79e9e 100644--- a/crates/soup/src/domain/models/test.rs+++ b/crates/soup/src/domain/models/test.rs@@ -1,5 +1,7 @@use super::*;+mod exclusions;+fn extract(expr: &Expr<ReminderLiteral>) -> Option<ReminderFilterExtract> {let mut out = ReminderFilterExtract::default();extract_reminder_filter(expr, &mut out).then_some(out)diff --git a/crates/soup/src/domain/models/test/exclusions.rs b/crates/soup/src/domain/models/test/exclusions.rsnew file mode 100644index 0000000000..23fe480942--- /dev/null+++ b/crates/soup/src/domain/models/test/exclusions.rs@@ -0,0 +1,201 @@+use super::*;+use item_filters::ast::channel::{ChannelLiteral, ChannelThreadLiteral};+use item_filters::ast::foreign_entity::ForeignEntityLiteral;+use models_pagination::CursorVal;++#[derive(Clone, Copy, Debug)]+enum Leg {+ Email,+ Channel,+ ChannelThread,+ Call,+ ForeignEntity,+}++const LEGS: [Leg; 5] = [+ Leg::Email,+ Leg::Channel,+ Leg::ChannelThread,+ Leg::Call,+ Leg::ForeignEntity,+];++fn map_ids<T>(tree: &Expr<Uuid>, literal: impl Fn(Uuid) -> T) -> Arc<Expr<T>> {+ Arc::new(tree.collapse_frames(|frame| match frame {+ filter_ast::ExprFrame::And(a, b) => Expr::and(a, b),+ filter_ast::ExprFrame::Or(a, b) => Expr::or(a, b),+ filter_ast::ExprFrame::Not(a) => Expr::is_not(a),+ filter_ast::ExprFrame::Literal(id) => Expr::val(literal(id)),+ }))+}++fn filter_for(leg: Leg, tree: &Expr<Uuid>) -> EntityFilterAst {+ let mut filter = EntityFilterAst::default();+ match leg {+ Leg::Email => filter.email_filter.tree = Some(map_ids(tree, EmailLiteral::ThreadId)),+ Leg::Channel => filter.channel_filter = Some(map_ids(tree, ChannelLiteral::ChannelId)),+ Leg::ChannelThread => {+ filter.channel_thread_filter = Some(map_ids(tree, ChannelThreadLiteral::ThreadId));+ }+ Leg::Call => filter.call_filter = Some(map_ids(tree, CallLiteral::CallId)),+ Leg::ForeignEntity => {+ filter.foreign_entity_filter = Some(map_ids(tree, ForeignEntityLiteral::Id));+ }+ }+ filter+}++fn request<T>(filter: T, follow_up: bool) -> SoupRequest<T> {+ let cursor = if follow_up {+ Query::Cursor(CursorWithValAndFilter {+ id: id(2),+ limit: 50,+ val: CursorVal {+ sort_type: SimpleSortMethod::UpdatedAt,+ last_val: chrono::DateTime::from_timestamp(1_700_000_000, 0).unwrap(),+ },+ filter,+ })+ } else {+ Query::Sort(SimpleSortMethod::UpdatedAt, filter)+ };+ SoupRequest {+ soup_type: SoupType::Expanded,+ limit: 50,+ cursor: SoupQuery::Simple(SimpleQueryInner(cursor)),+ sort_direction: SoupSortDirection::Desc,+ user: MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap(),+ email_preview_view: PreviewView::default(),+ link_ids: vec![id(3)],+ }+}++fn builds_request(leg: Leg, request: &SoupRequest<Option<EntityFilterAst>>) -> bool {+ match leg {+ Leg::Email => request.build_email_request(None).is_some(),+ Leg::Channel => request.build_comms_request().is_some(),+ Leg::ChannelThread => request.build_comms_thread_request().is_some(),+ Leg::Call => request.build_call_request().is_some(),+ Leg::ForeignEntity => request.build_foreign_entity_query().is_some(),+ }+}++#[test]+fn nil_entity_ids_skip_domain_requests_on_initial_and_cursor_pages() {+ for leg in LEGS {+ for follow_up in [false, true] {+ let req = request(Some(filter_for(leg, &Expr::val(Uuid::nil()))), follow_up);+ assert!(+ !builds_request(leg, &req),+ "excluded {leg:?} must not call its service (follow_up={follow_up})"+ );+ }+ }+}++#[test]+fn legacy_rest_exclusions_skip_requests_after_ast_expansion() {+ let nil = Uuid::nil().to_string();+ let filters: EntityFilters = serde_json::from_value(serde_json::json!({+ "email_filters": { "email_thread_ids": [nil] },+ "channel_filters": { "channel_ids": [nil] },+ "channel_thread_filters": { "thread_ids": [nil] },+ "call_filters": { "call_ids": [nil] },+ "foreign_entity_filters": { "ids": [nil] }+ }))+ .unwrap();+ for follow_up in [false, true] {+ let req = request(filters.clone(), follow_up).into_ast().unwrap();+ for leg in LEGS {+ assert!(!builds_request(leg, &req), "excluded REST leg {leg:?}");+ }+ }+}++#[test]+fn absent_filters_and_real_entity_ids_keep_domain_requests() {+ for leg in LEGS {+ for follow_up in [false, true] {+ for filter in [None, Some(filter_for(leg, &Expr::val(id(1))))] {+ assert!(builds_request(leg, &request(filter, follow_up)), "{leg:?}");+ }+ }+ }+}++#[test]+fn exclusion_respects_and_or_and_not_without_dropping_possible_matches() {+ let excluded = Expr::val(Uuid::nil());+ let included = Expr::val(id(1));+ let cases = [+ (Expr::and(excluded.clone(), included.clone()), false),+ (Expr::and(included.clone(), excluded.clone()), false),+ (Expr::or(excluded.clone(), excluded.clone()), false),+ (Expr::or(excluded.clone(), included.clone()), true),+ (Expr::or(included.clone(), excluded.clone()), true),+ (Expr::is_not(excluded.clone()), true),+ (Expr::is_not(included), true),+ // Negations are deliberately not simplified: pruning is conservative.+ (Expr::is_not(Expr::is_not(excluded)), true),+ ];+ for leg in LEGS {+ for (tree, expected) in &cases {+ for follow_up in [false, true] {+ let req = request(Some(filter_for(leg, tree)), follow_up);+ assert_eq!(+ builds_request(leg, &req),+ *expected,+ "{leg:?}: {tree:?} (follow_up={follow_up})"+ );+ }+ }+ }+}++#[test]+fn crm_scoped_email_exclusions_preserve_authorization_prechecks() {+ for scope in [+ item_filters::ast::CrmScope::Domains(vec!["example.com".to_string()]),+ item_filters::ast::CrmScope::Addresses(vec!["[email protected]".to_string()]),+ ] {+ let mut filter = filter_for(Leg::Email, &Expr::val(Uuid::nil()));+ filter.email_filter.crm_scope = Some(scope);+ for follow_up in [false, true] {+ let req = request(Some(filter.clone()), follow_up);+ let email = req+ .build_email_request(None)+ .expect("email service must still validate the CRM scope");+ assert!(email.crm_scope.is_some());+ }+ }+}++#[test]+fn exclusion_only_skips_the_targeted_entity_type() {+ for excluded_leg in LEGS {+ let req = request(+ Some(filter_for(excluded_leg, &Expr::val(Uuid::nil()))),+ false,+ );+ let built = LEGS.map(|leg| builds_request(leg, &req));+ assert_eq!(built.into_iter().filter(|built| *built).count(), 4);+ }+}++#[test]+fn notified_hydration_legs_also_skip_excluded_entities() {+ for leg in [+ Leg::Email,+ Leg::Channel,+ Leg::ChannelThread,+ Leg::ForeignEntity,+ ] {+ for tree in [Expr::val(Uuid::nil()), Expr::val(id(1))] {+ let expected = matches!(&tree, Expr::Literal(id) if !id.is_nil());+ let filter = Some(filter_for(leg, &tree));+ let mut req = request(None, false);+ req.cursor = SoupQuery::Notified(NotifiedQueryInner(Query::Sort(NotifiedAt, filter)));+ assert_eq!(builds_request(leg, &req), expected, "{leg:?}: {tree:?}");+ }+ }+}diff --git a/crates/soup/src/domain/service/tests.rs b/crates/soup/src/domain/service/tests.rsindex 91e506e409..281f2eaf21 100644--- a/crates/soup/src/domain/service/tests.rs+++ b/crates/soup/src/domain/service/tests.rs@@ -48,6 +48,7 @@ use uuid::Uuid;use super::*;mod agent_metadata;+mod exclusions;struct NoopEmailPreviewService;diff --git a/crates/soup/src/domain/service/tests/exclusions.rs b/crates/soup/src/domain/service/tests/exclusions.rsnew file mode 100644index 0000000000..2acf7023fc--- /dev/null+++ b/crates/soup/src/domain/service/tests/exclusions.rs@@ -0,0 +1,68 @@+use super::*;++#[tokio::test]+async fn document_only_soup_never_calls_excluded_entity_services() {+ let nil = Uuid::nil().to_string();+ let document_id = Uuid::from_u128(1);+ let filters: EntityFilters = serde_json::from_value(serde_json::json!({+ "document_filters": { "document_ids": [document_id] },+ "email_filters": { "email_thread_ids": [nil] },+ "channel_filters": { "channel_ids": [nil] },+ "channel_thread_filters": { "thread_ids": [nil] },+ "call_filters": { "call_ids": [nil] },+ "foreign_entity_filters": { "ids": [nil] }+ }))+ .unwrap();+ let emails = RecordingEmailPreviewService::default();+ let channels = RecordingCommsService::new(Vec::new());+ let calls = RecordingCallRecordQueryService::new(Vec::new());+ let foreign_entities = RecordingForeignEntityService::new(Vec::new());+ let mut soup = MockSoupRepo::new();+ soup.expect_unexpanded_generic_cursor_soup()+ .times(1)+ .returning(move |_| {+ Box::pin(async move {+ Ok(vec![SoupItem::Document(soup_document_uuid_with_updated(+ document_id,+ DateTime::default(),+ ))])+ })+ });++ let page = SoupImpl::new(+ soup,+ FrecencyQueryServiceImpl::new(MockFrecencyStorage::new()),+ emails.clone(),+ channels.clone(),+ calls.clone(),+ NoOpCrmService,+ foreign_entities.clone(),+ NoOpRemindersService,+ )+ .get_user_soup(+ SoupRequest {+ soup_type: SoupType::UnExpanded,+ limit: 50,+ cursor: SoupQuery::new_sort_simple(SimpleSortMethod::UpdatedAt, filters),+ sort_direction: SoupSortDirection::Desc,+ user: MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap(),+ email_preview_view: PreviewView::default(),+ link_ids: vec![Uuid::from_u128(2)],+ },+ None,+ )+ .await+ .unwrap()+ .into_simple()+ .unwrap();++ assert_eq!(page.items.len(), 1);+ assert_matches!(&page.items[0], SoupItem::Document(doc) => {+ assert_eq!(doc.id, document_id);+ });+ assert!(emails.requests.lock().unwrap().is_empty());+ assert_eq!(channels.channel_calls(), 0);+ assert!(channels.thread_filters().is_empty());+ assert_eq!(calls.calls(), 0);+ assert!(foreign_entities.calls().is_empty());+}diff --git a/crates/soup/src/outbound/pg_soup_repo/notified.rs b/crates/soup/src/outbound/pg_soup_repo/notified.rsindex 57ae29d0a7..afc7dc479a 100644--- a/crates/soup/src/outbound/pg_soup_repo/notified.rs+++ b/crates/soup/src/outbound/pg_soup_repo/notified.rs@@ -238,11 +238,32 @@ fn included_types(req: &NotifiedSoupRequest<'_>) -> Vec<&'static str> {types}+// Entity-specific AST gates make this genuinely dynamic SQL, so a static SQLx+// macro cannot check the whole statement. Keep request values bound below and+// generate the gates through the existing typed filter builders.+fn build_query(filter: Option<&EntityFilterAst>) -> String {+ format!(+ include_str!("notified/query.sql"),+ document_gate = document_gate(ID_SQL, filter),+ chat_gate = chat_gate(ID_SQL, filter),+ project_gate = project_gate(ID_SQL, filter),+ channel_gate = channel_gate(ID_SQL, filter),+ channel_thread_gate = channel_thread_gate(ID_SQL, filter),+ email_gate = email_gate(ID_SQL, filter),+ calendar_event_gate = calendar_event_gate(filter),+ foreign_entity_gate = foreign_entity_gate(filter),+ reminder_gate = reminder_gate(),+ agent_session_gate = agent_session_gate(),+ )+}+/// Fetches one page of notified-at candidates.////// Shape: the user's live notifications collapse to one row per derived-/// entity key, its newest notification (the `latest` window's first row per-/// partition), which the fenced subquery orders and keysets; the outer query+/// entity key using its maximum notification timestamp. No other column from+/// the latest notification is needed, so grouping avoids sorting every+/// notification for a row-number window. The fenced subquery orders and+/// keysets these aggregates (never the raw notifications); the outer query/// gates those per entity type on existence/deletion/access and the/// soup-owned filters in that order until `LIMIT` is met. The `OFFSET 0`/// fence keeps the gates outside the dedupe and the inner order intact, so@@ -262,82 +283,7 @@ pub(super) async fn notified_soup_page(return Ok(Vec::new());}- let sql = format!(- r#"- WITH user_source_ids AS (- SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp- WHERE cp.user_id = $1 AND cp.left_at IS NULL- UNION ALL- SELECT t.team_id::text FROM team_user t- WHERE t.user_id = $1- UNION ALL- SELECT $1- ),- notified AS NOT MATERIALIZED (- SELECT- un.created_at,- n.id AS notification_id,- CASE WHEN n.event_item_type = 'channel'- AND n.secondary_event_item_type = 'channel_message'- THEN 'channel_message' ELSE n.event_item_type- END AS entity_type,- CASE WHEN n.event_item_type = 'channel'- AND n.secondary_event_item_type = 'channel_message'- THEN n.secondary_event_item_id ELSE n.event_item_id- END AS entity_id- FROM user_notification un- JOIN notification n ON n.id = un.notification_id- WHERE un.user_id = $1- AND un.deleted_at IS NULL- ),- latest AS NOT MATERIALIZED (- SELECT- entity_type,- entity_id,- created_at,- row_number() OVER (- PARTITION BY entity_type, entity_id- ORDER BY created_at DESC, notification_id DESC- ) AS rn- FROM notified- WHERE entity_type = ANY($2)- )- SELECT nc.entity_type, nc.entity_id, nc.notified_at- FROM (- SELECT entity_type, entity_id, created_at AS notified_at- FROM latest- WHERE rn = 1- AND ($3::timestamp IS NULL OR (created_at, entity_id) < ($3, $4))- ORDER BY created_at DESC, entity_id DESC- OFFSET 0- ) nc- WHERE CASE nc.entity_type- WHEN 'document' THEN {document_gate}- WHEN 'chat' THEN {chat_gate}- WHEN 'project' THEN {project_gate}- WHEN 'channel' THEN {channel_gate}- WHEN 'channel_message' THEN {channel_thread_gate}- WHEN 'email_thread' THEN {email_gate}- WHEN 'calendar_event' THEN {calendar_event_gate}- WHEN 'foreign_entity' THEN {foreign_entity_gate}- WHEN 'reminder' THEN {reminder_gate}- WHEN 'agent_session' THEN {agent_session_gate}- ELSE FALSE- END- ORDER BY nc.notified_at DESC, nc.entity_id DESC- LIMIT $6- "#,- document_gate = document_gate(ID_SQL, req.filter),- chat_gate = chat_gate(ID_SQL, req.filter),- project_gate = project_gate(ID_SQL, req.filter),- channel_gate = channel_gate(ID_SQL, req.filter),- channel_thread_gate = channel_thread_gate(ID_SQL, req.filter),- email_gate = email_gate(ID_SQL, req.filter),- calendar_event_gate = calendar_event_gate(req.filter),- foreign_entity_gate = foreign_entity_gate(req.filter),- reminder_gate = reminder_gate(),- agent_session_gate = agent_session_gate(),- );+ let sql = build_query(req.filter);let after_ts = req.after.as_ref().map(|a| a.notified_at.naive_utc());let after_id = req.after.map(|a| a.entity_id);diff --git a/crates/soup/src/outbound/pg_soup_repo/notified/query.sql b/crates/soup/src/outbound/pg_soup_repo/notified/query.sqlnew file mode 100644index 0000000000..72c7a9ae66--- /dev/null+++ b/crates/soup/src/outbound/pg_soup_repo/notified/query.sql@@ -0,0 +1,57 @@+WITH user_source_ids AS (+ SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp+ WHERE cp.user_id = $1 AND cp.left_at IS NULL+ UNION ALL+ SELECT t.team_id::text FROM team_user t+ WHERE t.user_id = $1+ UNION ALL+ SELECT $1+),+notified AS NOT MATERIALIZED (+ SELECT+ un.created_at,+ CASE WHEN n.event_item_type = 'channel'+ AND n.secondary_event_item_type = 'channel_message'+ THEN 'channel_message' ELSE n.event_item_type+ END AS entity_type,+ CASE WHEN n.event_item_type = 'channel'+ AND n.secondary_event_item_type = 'channel_message'+ THEN n.secondary_event_item_id ELSE n.event_item_id+ END AS entity_id+ FROM user_notification un+ JOIN notification n ON n.id = un.notification_id+ WHERE un.user_id = $1+ AND un.deleted_at IS NULL+),+latest AS NOT MATERIALIZED (+ SELECT+ entity_type,+ entity_id,+ max(created_at) AS created_at+ FROM notified+ WHERE entity_type = ANY($2)+ GROUP BY entity_type, entity_id+)+SELECT nc.entity_type, nc.entity_id, nc.notified_at+FROM (+ SELECT entity_type, entity_id, created_at AS notified_at+ FROM latest+ WHERE ($3::timestamp IS NULL OR (created_at, entity_id) < ($3, $4))+ ORDER BY created_at DESC, entity_id DESC+ OFFSET 0+) nc+WHERE CASE nc.entity_type+ WHEN 'document' THEN {document_gate}+ WHEN 'chat' THEN {chat_gate}+ WHEN 'project' THEN {project_gate}+ WHEN 'channel' THEN {channel_gate}+ WHEN 'channel_message' THEN {channel_thread_gate}+ WHEN 'email_thread' THEN {email_gate}+ WHEN 'calendar_event' THEN {calendar_event_gate}+ WHEN 'foreign_entity' THEN {foreign_entity_gate}+ WHEN 'reminder' THEN {reminder_gate}+ WHEN 'agent_session' THEN {agent_session_gate}+ ELSE FALSE+END+ORDER BY nc.notified_at DESC, nc.entity_id DESC+LIMIT $6diff --git a/crates/soup/src/outbound/pg_soup_repo/notified/test.rs b/crates/soup/src/outbound/pg_soup_repo/notified/test.rsindex 2c28a1f757..44c9bbee23 100644--- a/crates/soup/src/outbound/pg_soup_repo/notified/test.rs+++ b/crates/soup/src/outbound/pg_soup_repo/notified/test.rs@@ -11,6 +11,9 @@ use sqlx::{Pool, Postgres};use std::sync::Arc;use uuid::Uuid;+mod explain;+mod latest;+const USER_1: &str = "macro|[email protected]";const DOC_A: &str = "11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa";const CHAT_A: &str = "22222222-aaaa-aaaa-aaaa-aaaaaaaaaaaa";diff --git a/crates/soup/src/outbound/pg_soup_repo/notified/test/explain.rs b/crates/soup/src/outbound/pg_soup_repo/notified/test/explain.rsnew file mode 100644index 0000000000..7ecba7806c--- /dev/null+++ b/crates/soup/src/outbound/pg_soup_repo/notified/test/explain.rs@@ -0,0 +1,174 @@+use super::*;+use chrono::NaiveDateTime;+use serde_json::Value;+use sqlx::{postgres::PgArguments, query::Query};++const DOCUMENTS: usize = 5_000;+const NOTIFICATIONS: usize = 50_000;+const PAGE_SIZE: i64 = 100;++// Both alternatives use the production query's bindings and access scopes.+// The SQL is generated from AST gates, so this cannot use a static SQLx macro.+fn bind_query(sql: &str) -> Query<'_, Postgres, PgArguments> {+ let links = vec![Uuid::parse_str(LINK_1).unwrap()];+ let sources = sources();+ let types = included_types(&req(None, &links, &sources, EVERYTHING));+ sqlx::query(sql)+ .bind(USER_1)+ .bind(types)+ .bind(None::<NaiveDateTime>)+ .bind(None::<String>)+ .bind(links)+ .bind(PAGE_SIZE)+ .bind(vec![USER_1, TEAM_T])+ .bind(vec!["user", "team"])+}++fn keys_with_timestamps(rows: &[sqlx::postgres::PgRow]) -> Vec<(String, String, NaiveDateTime)> {+ rows.iter()+ .map(|row| {+ (+ row.get("entity_type"),+ row.get("entity_id"),+ row.get("notified_at"),+ )+ })+ .collect()+}++fn plan_nodes(plan: &Value, nodes: &mut Vec<Value>) {+ nodes.push(serde_json::json!({+ "node": plan["Node Type"],+ "strategy": plan["Strategy"],+ "rows": plan["Actual Rows"],+ "loops": plan["Actual Loops"],+ "sort_kb": plan["Sort Space Used"],+ "hash_kb": plan["Peak Memory Usage"],+ }));+ if let Some(children) = plan["Plans"].as_array() {+ for child in children {+ plan_nodes(child, nodes);+ }+ }+}++/// Explicit local diagnostic: SQLx supplies an isolated, migrated database.+/// Compare complete candidate queries, including access gates, using the dev+/// replica's 16 MiB work_mem and no forced plans or timing assertions. The setting+/// is session-only. Never point the test harness at hosted data.+#[sqlx::test(+ fixtures(path = "../../../../../fixtures", scripts("notified_at")),+ migrator = "MACRO_DB_MIGRATIONS"+)]+#[ignore = "local EXPLAIN diagnostic; run explicitly with --ignored --nocapture"]+async fn notified_query_explain_local(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let document_ids = (0..DOCUMENTS)+ .map(|_| Uuid::now_v7().to_string())+ .collect::<Vec<_>>();+ let notification_ids = (0..NOTIFICATIONS)+ .map(|_| Uuid::now_v7())+ .collect::<Vec<_>>();+ let entity_ids = (0..NOTIFICATIONS)+ .map(|index| document_ids[index % DOCUMENTS].clone())+ .collect::<Vec<_>>();+ sqlx::query!(+ r#"INSERT INTO "Document" (id, name, owner, "createdAt", "updatedAt")+ SELECT id, 'notified benchmark', $2, '2024-06-02', '2024-06-02'+ FROM unnest($1::text[]) id"#,+ &document_ids,+ USER_1,+ )+ .execute(&pool)+ .await?;+ sqlx::query!(+ "INSERT INTO entity_access (entity_id, entity_type, source_id, source_type, access_level) SELECT id::uuid, 'document', $2, 'user', 'owner' FROM unnest($1::text[]) id",+ &document_ids,+ USER_1,+ )+ .execute(&pool)+ .await?;+ sqlx::query!(+ r#"INSERT INTO notification (id, notification_event_type, event_item_id, event_item_type, service_sender, created_at)+ SELECT id, 'document_comment', entity_id, 'document', 'test',+ timestamp '2024-06-02' + position * interval '1 second'+ FROM unnest($1::uuid[], $2::text[]) WITH ORDINALITY seeded(id, entity_id, position)"#,+ ¬ification_ids,+ &entity_ids,+ )+ .execute(&pool)+ .await?;+ sqlx::query!(+ "INSERT INTO user_notification (user_id, notification_id, created_at, sent, state) SELECT $1, id, created_at, TRUE, 'unseen' FROM notification WHERE id = ANY($2::uuid[])",+ USER_1,+ ¬ification_ids,+ )+ .execute(&pool)+ .await?;+ sqlx::query!("ANALYZE").execute(&pool).await?;+ let mut connection = pool.acquire().await?;+ sqlx::query!("SET work_mem = '16MB'")+ .execute(&mut *connection)+ .await?;+ let version = sqlx::query_scalar!("SELECT version()")+ .fetch_one(&mut *connection)+ .await?;+ let work_mem = sqlx::query_scalar!("SELECT current_setting('work_mem')")+ .fetch_one(&mut *connection)+ .await?;+ println!(+ "database={version:?}; work_mem={work_mem:?}; documents={DOCUMENTS}; notifications={NOTIFICATIONS}"+ );++ let filter = EntityFilterAst {+ document_filter: Some(Arc::new(Expr::val(DocumentLiteral::NotificationState(+ item_filters::NotificationState::Unseen,+ )))),+ ..EntityFilterAst::default()+ };+ for (label, filter) in [("unfiltered", None), ("unseen", Some(&filter))] {+ let grouped = build_query(filter);+ // Reconstruct only the previous deduplication stage. All joins, gates,+ // cursor handling, and bindings are identical to the optimized query.+ let window = grouped+ .replace("un.created_at,", "un.created_at, n.id AS notification_id,")+ .replace(+ "max(created_at) AS created_at",+ "created_at, row_number() OVER (PARTITION BY entity_type, entity_id ORDER BY created_at DESC, notification_id DESC) AS rn",+ )+ .replace("GROUP BY entity_type, entity_id", "")+ .replace("WHERE ($3::timestamp", "WHERE rn = 1 AND ($3::timestamp");+ let expected =+ keys_with_timestamps(&bind_query(&window).fetch_all(&mut *connection).await?);+ let actual = keys_with_timestamps(&bind_query(&grouped).fetch_all(&mut *connection).await?);+ assert_eq!(actual, expected);+ assert_eq!(actual.len(), PAGE_SIZE as usize);++ for repetition in 0..5 {+ let variants = if repetition % 2 == 0 {+ [("window", &window), ("group_max", &grouped)]+ } else {+ [("group_max", &grouped), ("window", &window)]+ };+ for (variant, sql) in variants {+ let explain = format!("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) {sql}");+ let row = bind_query(&explain).fetch_one(&mut *connection).await?;+ let plan: Value = row.try_get(0)?;+ let mut nodes = Vec::new();+ plan_nodes(&plan[0]["Plan"], &mut nodes);+ println!(+ "{}",+ serde_json::json!({+ "case": label,+ "variant": variant,+ "repetition": repetition,+ "execution_ms": plan[0]["Execution Time"],+ "planning_ms": plan[0]["Planning Time"],+ "shared_hit_blocks": plan[0]["Plan"]["Shared Hit Blocks"],+ "nodes": nodes,+ })+ );+ }+ }+ }+ Ok(())+}diff --git a/crates/soup/src/outbound/pg_soup_repo/notified/test/latest.rs b/crates/soup/src/outbound/pg_soup_repo/notified/test/latest.rsnew file mode 100644index 0000000000..6cc9d379b4--- /dev/null+++ b/crates/soup/src/outbound/pg_soup_repo/notified/test/latest.rs@@ -0,0 +1,82 @@+use super::*;+use chrono::Timelike;++#[sqlx::test(+ fixtures(+ path = "../../../../../fixtures",+ scripts("notified_at", "notified_latest")+ ),+ migrator = "MACRO_DB_MIGRATIONS"+)]+async fn timestamp_ties_deduplicate_without_merging_entity_types(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let links = [Uuid::parse_str(LINK_1)?];+ let sources = sources();+ let page = notified_soup_page(&pool, req(None, &links, &sources, EVERYTHING)).await?;++ let same_id = page+ .iter()+ .filter(|item| item.entity.entity_id == DOC_A)+ .collect::<Vec<_>>();+ assert_eq!(same_id.len(), 2);+ assert_eq!(same_id[0].entity.entity_type, EntityType::Chat);+ assert_eq!(same_id[0].notified_at.minute(), 10);+ assert_eq!(same_id[1].entity.entity_type, EntityType::Document);+ assert_eq!(same_id[1].notified_at.minute(), 9);+ assert_eq!(page.len(), 12);+ Ok(())+}++#[sqlx::test(+ fixtures(+ path = "../../../../../fixtures",+ scripts("notified_at", "notified_latest")+ ),+ migrator = "MACRO_DB_MIGRATIONS"+)]+async fn cursor_does_not_resurrect_older_notifications(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let links = [Uuid::parse_str(LINK_1)?];+ let sources = sources();+ let mut request = req(None, &links, &sources, EVERYTHING);+ request.after = Some(NotifiedPagePosition {+ notified_at: "2024-06-01T10:09:00Z".parse()?,+ entity_id: DOC_A.to_string(),+ });+ let page = notified_soup_page(&pool, request).await?;++ // Neither the document's T1 notifications nor its duplicate T9 rows may+ // reintroduce it after its T9 cursor. The same-id chat is newer still.+ assert!(page.iter().all(|item| item.entity.entity_id != DOC_A));+ assert_eq!(minutes(&page), vec![8, 7, 6, 5, 4, 3, 2, 0]);+ Ok(())+}++#[sqlx::test(+ fixtures(+ path = "../../../../../fixtures",+ scripts("notified_at", "notified_latest")+ ),+ migrator = "MACRO_DB_MIGRATIONS"+)]+async fn state_filter_does_not_move_latest_timestamp(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let links = [Uuid::parse_str(LINK_1)?];+ let sources = sources();+ let filter = EntityFilterAst {+ document_filter: Some(Arc::new(Expr::val(DocumentLiteral::NotificationState(+ item_filters::NotificationState::Unseen,+ )))),+ ..EntityFilterAst::default()+ };+ let page = notified_soup_page(&pool, req(Some(&filter), &links, &sources, EVERYTHING)).await?;+ let document = page+ .iter()+ .find(|item| {+ item.entity.entity_type == EntityType::Document && item.entity.entity_id == DOC_A+ })+ .expect("an older unseen notification still qualifies the document");+ // Notification state is a membership filter, not a filter on the input+ // to max(created_at); the latest (done) notification still sets the key.+ assert_eq!(document.notified_at.minute(), 9);+ Ok(())+}