Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/fix/less query (#6890)

Sean Aye633a16b6ea2d
diff --git a/.sqlx/query-1ace84dbc2133ca52b88a247f96babb8b79a15c91403762d37651d2b0053fa25.json b/.sqlx/query-1ace84dbc2133ca52b88a247f96babb8b79a15c91403762d37651d2b0053fa25.json
new file mode 100644
index 0000000000..e1c160f0b1
--- /dev/null
+++ b/.sqlx/query-1ace84dbc2133ca52b88a247f96babb8b79a15c91403762d37651d2b0053fa25.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO user_notification (user_id, notification_id, created_at, sent, state) SELECT $1, id, created_at, TRUE, 'unseen' FROM notification WHERE id = ANY($2::uuid[])",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "UuidArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "1ace84dbc2133ca52b88a247f96babb8b79a15c91403762d37651d2b0053fa25"
+}
diff --git a/.sqlx/query-217acdbed6242ff0c4b45c2d2d20fabf7819587af3f045b0bbcd500b6f9c5107.json b/.sqlx/query-217acdbed6242ff0c4b45c2d2d20fabf7819587af3f045b0bbcd500b6f9c5107.json
new file mode 100644
index 0000000000..32825ab260
--- /dev/null
+++ b/.sqlx/query-217acdbed6242ff0c4b45c2d2d20fabf7819587af3f045b0bbcd500b6f9c5107.json
@@ -0,0 +1,12 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "ANALYZE",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": []
+ },
+ "hash": "217acdbed6242ff0c4b45c2d2d20fabf7819587af3f045b0bbcd500b6f9c5107"
+}
diff --git a/.sqlx/query-55c45e9910ee3872af6feffea842117180eeb1eb8aa32f87538871443105a41b.json b/.sqlx/query-55c45e9910ee3872af6feffea842117180eeb1eb8aa32f87538871443105a41b.json
new file mode 100644
index 0000000000..1ff1ebad09
--- /dev/null
+++ b/.sqlx/query-55c45e9910ee3872af6feffea842117180eeb1eb8aa32f87538871443105a41b.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO \"Document\" (id, name, owner, \"createdAt\", \"updatedAt\")\n SELECT id, 'notified benchmark', $2, '2024-06-02', '2024-06-02'\n FROM unnest($1::text[]) id",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "55c45e9910ee3872af6feffea842117180eeb1eb8aa32f87538871443105a41b"
+}
diff --git a/.sqlx/query-a990c42683ac176337a16987d448fe562b420e692135be77f966c52e2a2c2495.json b/.sqlx/query-a990c42683ac176337a16987d448fe562b420e692135be77f966c52e2a2c2495.json
new file mode 100644
index 0000000000..3e31daae59
--- /dev/null
+++ b/.sqlx/query-a990c42683ac176337a16987d448fe562b420e692135be77f966c52e2a2c2495.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO entity_access (entity_id, entity_type, source_id, source_type, access_level) SELECT id::uuid, 'document', $2, 'user', 'owner' FROM unnest($1::text[]) id",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "TextArray",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "a990c42683ac176337a16987d448fe562b420e692135be77f966c52e2a2c2495"
+}
diff --git a/.sqlx/query-ccffcdc346ba29f62505162f7df86e7352e50be067678781ce6386cf59cc61fd.json b/.sqlx/query-ccffcdc346ba29f62505162f7df86e7352e50be067678781ce6386cf59cc61fd.json
new file mode 100644
index 0000000000..1fadfe4c60
--- /dev/null
+++ b/.sqlx/query-ccffcdc346ba29f62505162f7df86e7352e50be067678781ce6386cf59cc61fd.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "INSERT INTO notification (id, notification_event_type, event_item_id, event_item_type, service_sender, created_at)\n SELECT id, 'document_comment', entity_id, 'document', 'test',\n timestamp '2024-06-02' + position * interval '1 second'\n FROM unnest($1::uuid[], $2::text[]) WITH ORDINALITY seeded(id, entity_id, position)",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "UuidArray",
+ "TextArray"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "ccffcdc346ba29f62505162f7df86e7352e50be067678781ce6386cf59cc61fd"
+}
diff --git a/.sqlx/query-d2408d7cca5a71f14325af6e58c42cc70cd7ec651701f49573c944f0a1610213.json b/.sqlx/query-d2408d7cca5a71f14325af6e58c42cc70cd7ec651701f49573c944f0a1610213.json
new file mode 100644
index 0000000000..0191a1ac91
--- /dev/null
+++ b/.sqlx/query-d2408d7cca5a71f14325af6e58c42cc70cd7ec651701f49573c944f0a1610213.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT current_setting('work_mem')",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "current_setting",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "d2408d7cca5a71f14325af6e58c42cc70cd7ec651701f49573c944f0a1610213"
+}
diff --git a/.sqlx/query-df1f1d15d442789a5b9c81cdddf44d88d5748499cc48865023ddc1ff1587d0f6.json b/.sqlx/query-df1f1d15d442789a5b9c81cdddf44d88d5748499cc48865023ddc1ff1587d0f6.json
new file mode 100644
index 0000000000..7fe6da3923
--- /dev/null
+++ b/.sqlx/query-df1f1d15d442789a5b9c81cdddf44d88d5748499cc48865023ddc1ff1587d0f6.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT version()",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "version",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "df1f1d15d442789a5b9c81cdddf44d88d5748499cc48865023ddc1ff1587d0f6"
+}
diff --git a/.sqlx/query-eba001bed3f012a9b5536a865d1dbf5bb3196a757e160895209a2d98db1b45cd.json b/.sqlx/query-eba001bed3f012a9b5536a865d1dbf5bb3196a757e160895209a2d98db1b45cd.json
new file mode 100644
index 0000000000..dca549a5b9
--- /dev/null
+++ b/.sqlx/query-eba001bed3f012a9b5536a865d1dbf5bb3196a757e160895209a2d98db1b45cd.json
@@ -0,0 +1,12 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SET work_mem = '16MB'",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": []
+ },
+ "hash": "eba001bed3f012a9b5536a865d1dbf5bb3196a757e160895209a2d98db1b45cd"
+}
diff --git a/crates/soup/fixtures/notified_latest.sql b/crates/soup/fixtures/notified_latest.sql
new file mode 100644
index 0000000000..b413ff3cd0
--- /dev/null
+++ b/crates/soup/fixtures/notified_latest.sql
@@ -0,0 +1,30 @@
+-- Extends notified_at: timestamp ties, an older notification with a larger id,
+-- and a chat sharing a document's id. Notification-table timestamps deliberately
+-- differ from the recipient timestamps that define notified_at.
+
+INSERT INTO public."Chat" ("id", "name", "userId", "createdAt", "updatedAt")
+VALUES ('11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'Same-id chat', 'macro|[email protected]', '2024-06-01 10:00:00', '2024-06-01 10:00:00');
+
+INSERT INTO public.entity_access (entity_id, entity_type, source_id, source_type, access_level)
+VALUES ('11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'chat', 'macro|[email protected]', 'user', 'owner');
+
+INSERT INTO public.notification (id, notification_event_type, event_item_id, event_item_type, service_sender, created_at)
+VALUES
+('0190b000-0000-7000-8000-000000000001', 'document_comment', '11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'document', 'test', '2024-06-01 11:00:00'),
+('0190b000-0000-7000-8000-000000000002', 'document_comment', '11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'document', 'test', '2024-06-01 11:00:00'),
+('0190b000-0000-7000-8000-000000000003', 'document_comment', '11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'document', 'test', '2024-06-01 11:00:00'),
+('0190b000-0000-7000-8000-000000000004', 'chat_complete', '11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'chat', 'test', '2024-06-01 11:00:00');
+
+INSERT INTO public.user_notification (user_id, notification_id, created_at, sent, state)
+VALUES
+('macro|[email protected]', '0190b000-0000-7000-8000-000000000001', '2024-06-01 10:09:00', TRUE, 'done'),
+('macro|[email protected]', '0190b000-0000-7000-8000-000000000002', '2024-06-01 10:09:00', TRUE, 'done'),
+('macro|[email protected]', '0190b000-0000-7000-8000-000000000003', '2024-06-01 10:01:00', TRUE, 'unseen'),
+('macro|[email protected]', '0190b000-0000-7000-8000-000000000004', '2024-06-01 10:10:00', TRUE, 'unseen');
+
+-- Only old document notifications match "unseen". The document must still be
+-- ordered at T9, not at the matching notification's T1.
+UPDATE public.user_notification
+SET state = 'done'
+WHERE user_id = 'macro|[email protected]'
+ AND notification_id = '0190a000-0000-7000-8000-000000000009';
diff --git a/crates/soup/src/domain/models.rs b/crates/soup/src/domain/models.rs
index 6b598d861f..b9c248db65 100644
--- a/crates/soup/src/domain/models.rs
+++ b/crates/soup/src/domain/models.rs
@@ -1,5 +1,7 @@
pub mod grouping;
+mod exclusions;
+
#[cfg(test)]
mod test;
@@ -427,6 +429,11 @@ impl SoupRequest<Option<EntityFilterAst>> {
if self.link_ids.is_empty() {
return None;
}
+ // CRM-scoped requests must still reach email's authorization precheck,
+ // even when their id filter cannot match any threads.
+ if crm_scope.is_none() && exclusions::email(entity_ast) {
+ return None;
+ }
// A properties filter that cannot match threads makes the email
// branch empty, so the sub-request is skipped. Otherwise it is ANDed
@@ -514,6 +521,9 @@ impl SoupRequest<Option<EntityFilterAst>> {
}
pub(crate) fn build_call_request(&self) -> Option<GetCallRecordsRequest> {
+ if exclusions::call(self.entity_ast()) {
+ return None;
+ }
// A def-less tag filter (entity_type None on every literal) applies to
// calls, which carry tags; it is folded into the call filter below and
// rendered as a `properties.values` EXISTS by the call query. Any other
@@ -695,7 +705,7 @@ impl SoupRequest<Option<EntityFilterAst>> {
}
pub(crate) fn build_comms_request(&self) -> Option<GetChannelsRequest> {
- if self.properties_filter_blocks_propertyless() {
+ if self.properties_filter_blocks_propertyless() || exclusions::channel(self.entity_ast()) {
return None;
}
Some(GetChannelsRequest {
@@ -736,7 +746,9 @@ impl SoupRequest<Option<EntityFilterAst>> {
}
pub(crate) fn build_comms_thread_request(&self) -> Option<GetThreadReplyRowsRequest> {
- if self.properties_filter_blocks_propertyless() {
+ if self.properties_filter_blocks_propertyless()
+ || exclusions::channel_thread(self.entity_ast())
+ {
return None;
}
let query = match &self.cursor {
@@ -782,7 +794,9 @@ impl SoupRequest<Option<EntityFilterAst>> {
}
pub(crate) fn build_foreign_entity_query(&self) -> Option<ForeignEntityListQuery> {
- if self.properties_filter_blocks_propertyless() {
+ if self.properties_filter_blocks_propertyless()
+ || exclusions::foreign_entity(self.entity_ast())
+ {
return None;
}
match &self.cursor {
diff --git a/crates/soup/src/domain/models/exclusions.rs b/crates/soup/src/domain/models/exclusions.rs
new file mode 100644
index 0000000000..9df854813d
--- /dev/null
+++ b/crates/soup/src/domain/models/exclusions.rs
@@ -0,0 +1,61 @@
+//! Recognize the nil-primary-id filters clients use to exclude an entity type.
+//! Prune these legs before calling their domain services, not after acquiring
+//! database connections for queries that can only return an empty page.
+
+use filter_ast::{Expr, ExprFrame};
+use item_filters::ast::{
+ EntityFilterAst,
+ call::CallLiteral,
+ channel::{ChannelLiteral, ChannelThreadLiteral},
+ email::EmailLiteral,
+ foreign_entity::ForeignEntityLiteral,
+};
+use recursion::CollapsibleExt;
+
+/// Only prove exclusions: an unknown literal or a negation may still match.
+/// In particular, `nil OR real_id` must never suppress the whole request.
+fn excludes_all<L: Clone>(tree: Option<&Expr<L>>, excludes: impl Fn(L) -> bool) -> bool {
+ tree.is_some_and(|tree| {
+ tree.collapse_frames(|frame| match frame {
+ ExprFrame::And(a, b) => a || b,
+ ExprFrame::Or(a, b) => a && b,
+ ExprFrame::Not(_) => false,
+ ExprFrame::Literal(literal) => excludes(literal),
+ })
+ })
+}
+
+pub(super) fn email(filter: Option<&EntityFilterAst>) -> bool {
+ excludes_all(
+ filter.and_then(|filter| filter.email_filter.tree.as_deref()),
+ |literal| matches!(literal, EmailLiteral::ThreadId(id) if id.is_nil()),
+ )
+}
+
+pub(super) fn channel(filter: Option<&EntityFilterAst>) -> bool {
+ excludes_all(
+ filter.and_then(|filter| filter.channel_filter.as_deref()),
+ |literal| matches!(literal, ChannelLiteral::ChannelId(id) if id.is_nil()),
+ )
+}
+
+pub(super) fn channel_thread(filter: Option<&EntityFilterAst>) -> bool {
+ excludes_all(
+ filter.and_then(|filter| filter.channel_thread_filter.as_deref()),
+ |literal| matches!(literal, ChannelThreadLiteral::ThreadId(id) if id.is_nil()),
+ )
+}
+
+pub(super) fn call(filter: Option<&EntityFilterAst>) -> bool {
+ excludes_all(
+ filter.and_then(|filter| filter.call_filter.as_deref()),
+ |literal| matches!(literal, CallLiteral::CallId(id) if id.is_nil()),
+ )
+}
+
+pub(super) fn foreign_entity(filter: Option<&EntityFilterAst>) -> bool {
+ excludes_all(
+ filter.and_then(|filter| filter.foreign_entity_filter.as_deref()),
+ |literal| matches!(literal, ForeignEntityLiteral::Id(id) if id.is_nil()),
+ )
+}
diff --git a/crates/soup/src/domain/models/test.rs b/crates/soup/src/domain/models/test.rs
index 395f3d47bc..793de79e9e 100644
--- a/crates/soup/src/domain/models/test.rs
+++ b/crates/soup/src/domain/models/test.rs
@@ -1,5 +1,7 @@
use super::*;
+mod exclusions;
+
fn extract(expr: &Expr<ReminderLiteral>) -> Option<ReminderFilterExtract> {
let mut out = ReminderFilterExtract::default();
extract_reminder_filter(expr, &mut out).then_some(out)
diff --git a/crates/soup/src/domain/models/test/exclusions.rs b/crates/soup/src/domain/models/test/exclusions.rs
new file mode 100644
index 0000000000..23fe480942
--- /dev/null
+++ b/crates/soup/src/domain/models/test/exclusions.rs
@@ -0,0 +1,201 @@
+use super::*;
+use item_filters::ast::channel::{ChannelLiteral, ChannelThreadLiteral};
+use item_filters::ast::foreign_entity::ForeignEntityLiteral;
+use models_pagination::CursorVal;
+
+#[derive(Clone, Copy, Debug)]
+enum Leg {
+ Email,
+ Channel,
+ ChannelThread,
+ Call,
+ ForeignEntity,
+}
+
+const LEGS: [Leg; 5] = [
+ Leg::Email,
+ Leg::Channel,
+ Leg::ChannelThread,
+ Leg::Call,
+ Leg::ForeignEntity,
+];
+
+fn map_ids<T>(tree: &Expr<Uuid>, literal: impl Fn(Uuid) -> T) -> Arc<Expr<T>> {
+ Arc::new(tree.collapse_frames(|frame| match frame {
+ filter_ast::ExprFrame::And(a, b) => Expr::and(a, b),
+ filter_ast::ExprFrame::Or(a, b) => Expr::or(a, b),
+ filter_ast::ExprFrame::Not(a) => Expr::is_not(a),
+ filter_ast::ExprFrame::Literal(id) => Expr::val(literal(id)),
+ }))
+}
+
+fn filter_for(leg: Leg, tree: &Expr<Uuid>) -> EntityFilterAst {
+ let mut filter = EntityFilterAst::default();
+ match leg {
+ Leg::Email => filter.email_filter.tree = Some(map_ids(tree, EmailLiteral::ThreadId)),
+ Leg::Channel => filter.channel_filter = Some(map_ids(tree, ChannelLiteral::ChannelId)),
+ Leg::ChannelThread => {
+ filter.channel_thread_filter = Some(map_ids(tree, ChannelThreadLiteral::ThreadId));
+ }
+ Leg::Call => filter.call_filter = Some(map_ids(tree, CallLiteral::CallId)),
+ Leg::ForeignEntity => {
+ filter.foreign_entity_filter = Some(map_ids(tree, ForeignEntityLiteral::Id));
+ }
+ }
+ filter
+}
+
+fn request<T>(filter: T, follow_up: bool) -> SoupRequest<T> {
+ let cursor = if follow_up {
+ Query::Cursor(CursorWithValAndFilter {
+ id: id(2),
+ limit: 50,
+ val: CursorVal {
+ sort_type: SimpleSortMethod::UpdatedAt,
+ last_val: chrono::DateTime::from_timestamp(1_700_000_000, 0).unwrap(),
+ },
+ filter,
+ })
+ } else {
+ Query::Sort(SimpleSortMethod::UpdatedAt, filter)
+ };
+ SoupRequest {
+ soup_type: SoupType::Expanded,
+ limit: 50,
+ cursor: SoupQuery::Simple(SimpleQueryInner(cursor)),
+ sort_direction: SoupSortDirection::Desc,
+ user: MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap(),
+ email_preview_view: PreviewView::default(),
+ link_ids: vec![id(3)],
+ }
+}
+
+fn builds_request(leg: Leg, request: &SoupRequest<Option<EntityFilterAst>>) -> bool {
+ match leg {
+ Leg::Email => request.build_email_request(None).is_some(),
+ Leg::Channel => request.build_comms_request().is_some(),
+ Leg::ChannelThread => request.build_comms_thread_request().is_some(),
+ Leg::Call => request.build_call_request().is_some(),
+ Leg::ForeignEntity => request.build_foreign_entity_query().is_some(),
+ }
+}
+
+#[test]
+fn nil_entity_ids_skip_domain_requests_on_initial_and_cursor_pages() {
+ for leg in LEGS {
+ for follow_up in [false, true] {
+ let req = request(Some(filter_for(leg, &Expr::val(Uuid::nil()))), follow_up);
+ assert!(
+ !builds_request(leg, &req),
+ "excluded {leg:?} must not call its service (follow_up={follow_up})"
+ );
+ }
+ }
+}
+
+#[test]
+fn legacy_rest_exclusions_skip_requests_after_ast_expansion() {
+ let nil = Uuid::nil().to_string();
+ let filters: EntityFilters = serde_json::from_value(serde_json::json!({
+ "email_filters": { "email_thread_ids": [nil] },
+ "channel_filters": { "channel_ids": [nil] },
+ "channel_thread_filters": { "thread_ids": [nil] },
+ "call_filters": { "call_ids": [nil] },
+ "foreign_entity_filters": { "ids": [nil] }
+ }))
+ .unwrap();
+ for follow_up in [false, true] {
+ let req = request(filters.clone(), follow_up).into_ast().unwrap();
+ for leg in LEGS {
+ assert!(!builds_request(leg, &req), "excluded REST leg {leg:?}");
+ }
+ }
+}
+
+#[test]
+fn absent_filters_and_real_entity_ids_keep_domain_requests() {
+ for leg in LEGS {
+ for follow_up in [false, true] {
+ for filter in [None, Some(filter_for(leg, &Expr::val(id(1))))] {
+ assert!(builds_request(leg, &request(filter, follow_up)), "{leg:?}");
+ }
+ }
+ }
+}
+
+#[test]
+fn exclusion_respects_and_or_and_not_without_dropping_possible_matches() {
+ let excluded = Expr::val(Uuid::nil());
+ let included = Expr::val(id(1));
+ let cases = [
+ (Expr::and(excluded.clone(), included.clone()), false),
+ (Expr::and(included.clone(), excluded.clone()), false),
+ (Expr::or(excluded.clone(), excluded.clone()), false),
+ (Expr::or(excluded.clone(), included.clone()), true),
+ (Expr::or(included.clone(), excluded.clone()), true),
+ (Expr::is_not(excluded.clone()), true),
+ (Expr::is_not(included), true),
+ // Negations are deliberately not simplified: pruning is conservative.
+ (Expr::is_not(Expr::is_not(excluded)), true),
+ ];
+ for leg in LEGS {
+ for (tree, expected) in &cases {
+ for follow_up in [false, true] {
+ let req = request(Some(filter_for(leg, tree)), follow_up);
+ assert_eq!(
+ builds_request(leg, &req),
+ *expected,
+ "{leg:?}: {tree:?} (follow_up={follow_up})"
+ );
+ }
+ }
+ }
+}
+
+#[test]
+fn crm_scoped_email_exclusions_preserve_authorization_prechecks() {
+ for scope in [
+ item_filters::ast::CrmScope::Domains(vec!["example.com".to_string()]),
+ item_filters::ast::CrmScope::Addresses(vec!["[email protected]".to_string()]),
+ ] {
+ let mut filter = filter_for(Leg::Email, &Expr::val(Uuid::nil()));
+ filter.email_filter.crm_scope = Some(scope);
+ for follow_up in [false, true] {
+ let req = request(Some(filter.clone()), follow_up);
+ let email = req
+ .build_email_request(None)
+ .expect("email service must still validate the CRM scope");
+ assert!(email.crm_scope.is_some());
+ }
+ }
+}
+
+#[test]
+fn exclusion_only_skips_the_targeted_entity_type() {
+ for excluded_leg in LEGS {
+ let req = request(
+ Some(filter_for(excluded_leg, &Expr::val(Uuid::nil()))),
+ false,
+ );
+ let built = LEGS.map(|leg| builds_request(leg, &req));
+ assert_eq!(built.into_iter().filter(|built| *built).count(), 4);
+ }
+}
+
+#[test]
+fn notified_hydration_legs_also_skip_excluded_entities() {
+ for leg in [
+ Leg::Email,
+ Leg::Channel,
+ Leg::ChannelThread,
+ Leg::ForeignEntity,
+ ] {
+ for tree in [Expr::val(Uuid::nil()), Expr::val(id(1))] {
+ let expected = matches!(&tree, Expr::Literal(id) if !id.is_nil());
+ let filter = Some(filter_for(leg, &tree));
+ let mut req = request(None, false);
+ req.cursor = SoupQuery::Notified(NotifiedQueryInner(Query::Sort(NotifiedAt, filter)));
+ assert_eq!(builds_request(leg, &req), expected, "{leg:?}: {tree:?}");
+ }
+ }
+}
diff --git a/crates/soup/src/domain/service/tests.rs b/crates/soup/src/domain/service/tests.rs
index 91e506e409..281f2eaf21 100644
--- a/crates/soup/src/domain/service/tests.rs
+++ b/crates/soup/src/domain/service/tests.rs
@@ -48,6 +48,7 @@ use uuid::Uuid;
use super::*;
mod agent_metadata;
+mod exclusions;
struct NoopEmailPreviewService;
diff --git a/crates/soup/src/domain/service/tests/exclusions.rs b/crates/soup/src/domain/service/tests/exclusions.rs
new file mode 100644
index 0000000000..2acf7023fc
--- /dev/null
+++ b/crates/soup/src/domain/service/tests/exclusions.rs
@@ -0,0 +1,68 @@
+use super::*;
+
+#[tokio::test]
+async fn document_only_soup_never_calls_excluded_entity_services() {
+ let nil = Uuid::nil().to_string();
+ let document_id = Uuid::from_u128(1);
+ let filters: EntityFilters = serde_json::from_value(serde_json::json!({
+ "document_filters": { "document_ids": [document_id] },
+ "email_filters": { "email_thread_ids": [nil] },
+ "channel_filters": { "channel_ids": [nil] },
+ "channel_thread_filters": { "thread_ids": [nil] },
+ "call_filters": { "call_ids": [nil] },
+ "foreign_entity_filters": { "ids": [nil] }
+ }))
+ .unwrap();
+ let emails = RecordingEmailPreviewService::default();
+ let channels = RecordingCommsService::new(Vec::new());
+ let calls = RecordingCallRecordQueryService::new(Vec::new());
+ let foreign_entities = RecordingForeignEntityService::new(Vec::new());
+ let mut soup = MockSoupRepo::new();
+ soup.expect_unexpanded_generic_cursor_soup()
+ .times(1)
+ .returning(move |_| {
+ Box::pin(async move {
+ Ok(vec![SoupItem::Document(soup_document_uuid_with_updated(
+ document_id,
+ DateTime::default(),
+ ))])
+ })
+ });
+
+ let page = SoupImpl::new(
+ soup,
+ FrecencyQueryServiceImpl::new(MockFrecencyStorage::new()),
+ emails.clone(),
+ channels.clone(),
+ calls.clone(),
+ NoOpCrmService,
+ foreign_entities.clone(),
+ NoOpRemindersService,
+ )
+ .get_user_soup(
+ SoupRequest {
+ soup_type: SoupType::UnExpanded,
+ limit: 50,
+ cursor: SoupQuery::new_sort_simple(SimpleSortMethod::UpdatedAt, filters),
+ sort_direction: SoupSortDirection::Desc,
+ user: MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap(),
+ email_preview_view: PreviewView::default(),
+ link_ids: vec![Uuid::from_u128(2)],
+ },
+ None,
+ )
+ .await
+ .unwrap()
+ .into_simple()
+ .unwrap();
+
+ assert_eq!(page.items.len(), 1);
+ assert_matches!(&page.items[0], SoupItem::Document(doc) => {
+ assert_eq!(doc.id, document_id);
+ });
+ assert!(emails.requests.lock().unwrap().is_empty());
+ assert_eq!(channels.channel_calls(), 0);
+ assert!(channels.thread_filters().is_empty());
+ assert_eq!(calls.calls(), 0);
+ assert!(foreign_entities.calls().is_empty());
+}
diff --git a/crates/soup/src/outbound/pg_soup_repo/notified.rs b/crates/soup/src/outbound/pg_soup_repo/notified.rs
index 57ae29d0a7..afc7dc479a 100644
--- a/crates/soup/src/outbound/pg_soup_repo/notified.rs
+++ b/crates/soup/src/outbound/pg_soup_repo/notified.rs
@@ -238,11 +238,32 @@ fn included_types(req: &NotifiedSoupRequest<'_>) -> Vec<&'static str> {
types
}
+// Entity-specific AST gates make this genuinely dynamic SQL, so a static SQLx
+// macro cannot check the whole statement. Keep request values bound below and
+// generate the gates through the existing typed filter builders.
+fn build_query(filter: Option<&EntityFilterAst>) -> String {
+ format!(
+ include_str!("notified/query.sql"),
+ document_gate = document_gate(ID_SQL, filter),
+ chat_gate = chat_gate(ID_SQL, filter),
+ project_gate = project_gate(ID_SQL, filter),
+ channel_gate = channel_gate(ID_SQL, filter),
+ channel_thread_gate = channel_thread_gate(ID_SQL, filter),
+ email_gate = email_gate(ID_SQL, filter),
+ calendar_event_gate = calendar_event_gate(filter),
+ foreign_entity_gate = foreign_entity_gate(filter),
+ reminder_gate = reminder_gate(),
+ agent_session_gate = agent_session_gate(),
+ )
+}
+
/// Fetches one page of notified-at candidates.
///
/// Shape: the user's live notifications collapse to one row per derived
-/// entity key, its newest notification (the `latest` window's first row per
-/// partition), which the fenced subquery orders and keysets; the outer query
+/// entity key using its maximum notification timestamp. No other column from
+/// the latest notification is needed, so grouping avoids sorting every
+/// notification for a row-number window. The fenced subquery orders and
+/// keysets these aggregates (never the raw notifications); the outer query
/// gates those per entity type on existence/deletion/access and the
/// soup-owned filters in that order until `LIMIT` is met. The `OFFSET 0`
/// fence keeps the gates outside the dedupe and the inner order intact, so
@@ -262,82 +283,7 @@ pub(super) async fn notified_soup_page(
return Ok(Vec::new());
}
- let sql = format!(
- r#"
- WITH user_source_ids AS (
- SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp
- WHERE cp.user_id = $1 AND cp.left_at IS NULL
- UNION ALL
- SELECT t.team_id::text FROM team_user t
- WHERE t.user_id = $1
- UNION ALL
- SELECT $1
- ),
- notified AS NOT MATERIALIZED (
- SELECT
- un.created_at,
- n.id AS notification_id,
- CASE WHEN n.event_item_type = 'channel'
- AND n.secondary_event_item_type = 'channel_message'
- THEN 'channel_message' ELSE n.event_item_type
- END AS entity_type,
- CASE WHEN n.event_item_type = 'channel'
- AND n.secondary_event_item_type = 'channel_message'
- THEN n.secondary_event_item_id ELSE n.event_item_id
- END AS entity_id
- FROM user_notification un
- JOIN notification n ON n.id = un.notification_id
- WHERE un.user_id = $1
- AND un.deleted_at IS NULL
- ),
- latest AS NOT MATERIALIZED (
- SELECT
- entity_type,
- entity_id,
- created_at,
- row_number() OVER (
- PARTITION BY entity_type, entity_id
- ORDER BY created_at DESC, notification_id DESC
- ) AS rn
- FROM notified
- WHERE entity_type = ANY($2)
- )
- SELECT nc.entity_type, nc.entity_id, nc.notified_at
- FROM (
- SELECT entity_type, entity_id, created_at AS notified_at
- FROM latest
- WHERE rn = 1
- AND ($3::timestamp IS NULL OR (created_at, entity_id) < ($3, $4))
- ORDER BY created_at DESC, entity_id DESC
- OFFSET 0
- ) nc
- WHERE CASE nc.entity_type
- WHEN 'document' THEN {document_gate}
- WHEN 'chat' THEN {chat_gate}
- WHEN 'project' THEN {project_gate}
- WHEN 'channel' THEN {channel_gate}
- WHEN 'channel_message' THEN {channel_thread_gate}
- WHEN 'email_thread' THEN {email_gate}
- WHEN 'calendar_event' THEN {calendar_event_gate}
- WHEN 'foreign_entity' THEN {foreign_entity_gate}
- WHEN 'reminder' THEN {reminder_gate}
- WHEN 'agent_session' THEN {agent_session_gate}
- ELSE FALSE
- END
- ORDER BY nc.notified_at DESC, nc.entity_id DESC
- LIMIT $6
- "#,
- document_gate = document_gate(ID_SQL, req.filter),
- chat_gate = chat_gate(ID_SQL, req.filter),
- project_gate = project_gate(ID_SQL, req.filter),
- channel_gate = channel_gate(ID_SQL, req.filter),
- channel_thread_gate = channel_thread_gate(ID_SQL, req.filter),
- email_gate = email_gate(ID_SQL, req.filter),
- calendar_event_gate = calendar_event_gate(req.filter),
- foreign_entity_gate = foreign_entity_gate(req.filter),
- reminder_gate = reminder_gate(),
- agent_session_gate = agent_session_gate(),
- );
+ let sql = build_query(req.filter);
let after_ts = req.after.as_ref().map(|a| a.notified_at.naive_utc());
let after_id = req.after.map(|a| a.entity_id);
diff --git a/crates/soup/src/outbound/pg_soup_repo/notified/query.sql b/crates/soup/src/outbound/pg_soup_repo/notified/query.sql
new file mode 100644
index 0000000000..72c7a9ae66
--- /dev/null
+++ b/crates/soup/src/outbound/pg_soup_repo/notified/query.sql
@@ -0,0 +1,57 @@
+WITH user_source_ids AS (
+ SELECT cp.channel_id::text as source_id FROM comms_channel_participants cp
+ WHERE cp.user_id = $1 AND cp.left_at IS NULL
+ UNION ALL
+ SELECT t.team_id::text FROM team_user t
+ WHERE t.user_id = $1
+ UNION ALL
+ SELECT $1
+),
+notified AS NOT MATERIALIZED (
+ SELECT
+ un.created_at,
+ CASE WHEN n.event_item_type = 'channel'
+ AND n.secondary_event_item_type = 'channel_message'
+ THEN 'channel_message' ELSE n.event_item_type
+ END AS entity_type,
+ CASE WHEN n.event_item_type = 'channel'
+ AND n.secondary_event_item_type = 'channel_message'
+ THEN n.secondary_event_item_id ELSE n.event_item_id
+ END AS entity_id
+ FROM user_notification un
+ JOIN notification n ON n.id = un.notification_id
+ WHERE un.user_id = $1
+ AND un.deleted_at IS NULL
+),
+latest AS NOT MATERIALIZED (
+ SELECT
+ entity_type,
+ entity_id,
+ max(created_at) AS created_at
+ FROM notified
+ WHERE entity_type = ANY($2)
+ GROUP BY entity_type, entity_id
+)
+SELECT nc.entity_type, nc.entity_id, nc.notified_at
+FROM (
+ SELECT entity_type, entity_id, created_at AS notified_at
+ FROM latest
+ WHERE ($3::timestamp IS NULL OR (created_at, entity_id) < ($3, $4))
+ ORDER BY created_at DESC, entity_id DESC
+ OFFSET 0
+) nc
+WHERE CASE nc.entity_type
+ WHEN 'document' THEN {document_gate}
+ WHEN 'chat' THEN {chat_gate}
+ WHEN 'project' THEN {project_gate}
+ WHEN 'channel' THEN {channel_gate}
+ WHEN 'channel_message' THEN {channel_thread_gate}
+ WHEN 'email_thread' THEN {email_gate}
+ WHEN 'calendar_event' THEN {calendar_event_gate}
+ WHEN 'foreign_entity' THEN {foreign_entity_gate}
+ WHEN 'reminder' THEN {reminder_gate}
+ WHEN 'agent_session' THEN {agent_session_gate}
+ ELSE FALSE
+END
+ORDER BY nc.notified_at DESC, nc.entity_id DESC
+LIMIT $6
diff --git a/crates/soup/src/outbound/pg_soup_repo/notified/test.rs b/crates/soup/src/outbound/pg_soup_repo/notified/test.rs
index 2c28a1f757..44c9bbee23 100644
--- a/crates/soup/src/outbound/pg_soup_repo/notified/test.rs
+++ b/crates/soup/src/outbound/pg_soup_repo/notified/test.rs
@@ -11,6 +11,9 @@ use sqlx::{Pool, Postgres};
use std::sync::Arc;
use uuid::Uuid;
+mod explain;
+mod latest;
+
const USER_1: &str = "macro|[email protected]";
const DOC_A: &str = "11111111-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
const CHAT_A: &str = "22222222-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
diff --git a/crates/soup/src/outbound/pg_soup_repo/notified/test/explain.rs b/crates/soup/src/outbound/pg_soup_repo/notified/test/explain.rs
new file mode 100644
index 0000000000..7ecba7806c
--- /dev/null
+++ b/crates/soup/src/outbound/pg_soup_repo/notified/test/explain.rs
@@ -0,0 +1,174 @@
+use super::*;
+use chrono::NaiveDateTime;
+use serde_json::Value;
+use sqlx::{postgres::PgArguments, query::Query};
+
+const DOCUMENTS: usize = 5_000;
+const NOTIFICATIONS: usize = 50_000;
+const PAGE_SIZE: i64 = 100;
+
+// Both alternatives use the production query's bindings and access scopes.
+// The SQL is generated from AST gates, so this cannot use a static SQLx macro.
+fn bind_query(sql: &str) -> Query<'_, Postgres, PgArguments> {
+ let links = vec![Uuid::parse_str(LINK_1).unwrap()];
+ let sources = sources();
+ let types = included_types(&req(None, &links, &sources, EVERYTHING));
+ sqlx::query(sql)
+ .bind(USER_1)
+ .bind(types)
+ .bind(None::<NaiveDateTime>)
+ .bind(None::<String>)
+ .bind(links)
+ .bind(PAGE_SIZE)
+ .bind(vec![USER_1, TEAM_T])
+ .bind(vec!["user", "team"])
+}
+
+fn keys_with_timestamps(rows: &[sqlx::postgres::PgRow]) -> Vec<(String, String, NaiveDateTime)> {
+ rows.iter()
+ .map(|row| {
+ (
+ row.get("entity_type"),
+ row.get("entity_id"),
+ row.get("notified_at"),
+ )
+ })
+ .collect()
+}
+
+fn plan_nodes(plan: &Value, nodes: &mut Vec<Value>) {
+ nodes.push(serde_json::json!({
+ "node": plan["Node Type"],
+ "strategy": plan["Strategy"],
+ "rows": plan["Actual Rows"],
+ "loops": plan["Actual Loops"],
+ "sort_kb": plan["Sort Space Used"],
+ "hash_kb": plan["Peak Memory Usage"],
+ }));
+ if let Some(children) = plan["Plans"].as_array() {
+ for child in children {
+ plan_nodes(child, nodes);
+ }
+ }
+}
+
+/// Explicit local diagnostic: SQLx supplies an isolated, migrated database.
+/// Compare complete candidate queries, including access gates, using the dev
+/// replica's 16 MiB work_mem and no forced plans or timing assertions. The setting
+/// is session-only. Never point the test harness at hosted data.
+#[sqlx::test(
+ fixtures(path = "../../../../../fixtures", scripts("notified_at")),
+ migrator = "MACRO_DB_MIGRATIONS"
+)]
+#[ignore = "local EXPLAIN diagnostic; run explicitly with --ignored --nocapture"]
+async fn notified_query_explain_local(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let document_ids = (0..DOCUMENTS)
+ .map(|_| Uuid::now_v7().to_string())
+ .collect::<Vec<_>>();
+ let notification_ids = (0..NOTIFICATIONS)
+ .map(|_| Uuid::now_v7())
+ .collect::<Vec<_>>();
+ let entity_ids = (0..NOTIFICATIONS)
+ .map(|index| document_ids[index % DOCUMENTS].clone())
+ .collect::<Vec<_>>();
+ sqlx::query!(
+ r#"INSERT INTO "Document" (id, name, owner, "createdAt", "updatedAt")
+ SELECT id, 'notified benchmark', $2, '2024-06-02', '2024-06-02'
+ FROM unnest($1::text[]) id"#,
+ &document_ids,
+ USER_1,
+ )
+ .execute(&pool)
+ .await?;
+ sqlx::query!(
+ "INSERT INTO entity_access (entity_id, entity_type, source_id, source_type, access_level) SELECT id::uuid, 'document', $2, 'user', 'owner' FROM unnest($1::text[]) id",
+ &document_ids,
+ USER_1,
+ )
+ .execute(&pool)
+ .await?;
+ sqlx::query!(
+ r#"INSERT INTO notification (id, notification_event_type, event_item_id, event_item_type, service_sender, created_at)
+ SELECT id, 'document_comment', entity_id, 'document', 'test',
+ timestamp '2024-06-02' + position * interval '1 second'
+ FROM unnest($1::uuid[], $2::text[]) WITH ORDINALITY seeded(id, entity_id, position)"#,
+ &notification_ids,
+ &entity_ids,
+ )
+ .execute(&pool)
+ .await?;
+ sqlx::query!(
+ "INSERT INTO user_notification (user_id, notification_id, created_at, sent, state) SELECT $1, id, created_at, TRUE, 'unseen' FROM notification WHERE id = ANY($2::uuid[])",
+ USER_1,
+ &notification_ids,
+ )
+ .execute(&pool)
+ .await?;
+ sqlx::query!("ANALYZE").execute(&pool).await?;
+ let mut connection = pool.acquire().await?;
+ sqlx::query!("SET work_mem = '16MB'")
+ .execute(&mut *connection)
+ .await?;
+ let version = sqlx::query_scalar!("SELECT version()")
+ .fetch_one(&mut *connection)
+ .await?;
+ let work_mem = sqlx::query_scalar!("SELECT current_setting('work_mem')")
+ .fetch_one(&mut *connection)
+ .await?;
+ println!(
+ "database={version:?}; work_mem={work_mem:?}; documents={DOCUMENTS}; notifications={NOTIFICATIONS}"
+ );
+
+ let filter = EntityFilterAst {
+ document_filter: Some(Arc::new(Expr::val(DocumentLiteral::NotificationState(
+ item_filters::NotificationState::Unseen,
+ )))),
+ ..EntityFilterAst::default()
+ };
+ for (label, filter) in [("unfiltered", None), ("unseen", Some(&filter))] {
+ let grouped = build_query(filter);
+ // Reconstruct only the previous deduplication stage. All joins, gates,
+ // cursor handling, and bindings are identical to the optimized query.
+ let window = grouped
+ .replace("un.created_at,", "un.created_at, n.id AS notification_id,")
+ .replace(
+ "max(created_at) AS created_at",
+ "created_at, row_number() OVER (PARTITION BY entity_type, entity_id ORDER BY created_at DESC, notification_id DESC) AS rn",
+ )
+ .replace("GROUP BY entity_type, entity_id", "")
+ .replace("WHERE ($3::timestamp", "WHERE rn = 1 AND ($3::timestamp");
+ let expected =
+ keys_with_timestamps(&bind_query(&window).fetch_all(&mut *connection).await?);
+ let actual = keys_with_timestamps(&bind_query(&grouped).fetch_all(&mut *connection).await?);
+ assert_eq!(actual, expected);
+ assert_eq!(actual.len(), PAGE_SIZE as usize);
+
+ for repetition in 0..5 {
+ let variants = if repetition % 2 == 0 {
+ [("window", &window), ("group_max", &grouped)]
+ } else {
+ [("group_max", &grouped), ("window", &window)]
+ };
+ for (variant, sql) in variants {
+ let explain = format!("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) {sql}");
+ let row = bind_query(&explain).fetch_one(&mut *connection).await?;
+ let plan: Value = row.try_get(0)?;
+ let mut nodes = Vec::new();
+ plan_nodes(&plan[0]["Plan"], &mut nodes);
+ println!(
+ "{}",
+ serde_json::json!({
+ "case": label,
+ "variant": variant,
+ "repetition": repetition,
+ "execution_ms": plan[0]["Execution Time"],
+ "planning_ms": plan[0]["Planning Time"],
+ "shared_hit_blocks": plan[0]["Plan"]["Shared Hit Blocks"],
+ "nodes": nodes,
+ })
+ );
+ }
+ }
+ }
+ Ok(())
+}
diff --git a/crates/soup/src/outbound/pg_soup_repo/notified/test/latest.rs b/crates/soup/src/outbound/pg_soup_repo/notified/test/latest.rs
new file mode 100644
index 0000000000..6cc9d379b4
--- /dev/null
+++ b/crates/soup/src/outbound/pg_soup_repo/notified/test/latest.rs
@@ -0,0 +1,82 @@
+use super::*;
+use chrono::Timelike;
+
+#[sqlx::test(
+ fixtures(
+ path = "../../../../../fixtures",
+ scripts("notified_at", "notified_latest")
+ ),
+ migrator = "MACRO_DB_MIGRATIONS"
+)]
+async fn timestamp_ties_deduplicate_without_merging_entity_types(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let links = [Uuid::parse_str(LINK_1)?];
+ let sources = sources();
+ let page = notified_soup_page(&pool, req(None, &links, &sources, EVERYTHING)).await?;
+
+ let same_id = page
+ .iter()
+ .filter(|item| item.entity.entity_id == DOC_A)
+ .collect::<Vec<_>>();
+ assert_eq!(same_id.len(), 2);
+ assert_eq!(same_id[0].entity.entity_type, EntityType::Chat);
+ assert_eq!(same_id[0].notified_at.minute(), 10);
+ assert_eq!(same_id[1].entity.entity_type, EntityType::Document);
+ assert_eq!(same_id[1].notified_at.minute(), 9);
+ assert_eq!(page.len(), 12);
+ Ok(())
+}
+
+#[sqlx::test(
+ fixtures(
+ path = "../../../../../fixtures",
+ scripts("notified_at", "notified_latest")
+ ),
+ migrator = "MACRO_DB_MIGRATIONS"
+)]
+async fn cursor_does_not_resurrect_older_notifications(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let links = [Uuid::parse_str(LINK_1)?];
+ let sources = sources();
+ let mut request = req(None, &links, &sources, EVERYTHING);
+ request.after = Some(NotifiedPagePosition {
+ notified_at: "2024-06-01T10:09:00Z".parse()?,
+ entity_id: DOC_A.to_string(),
+ });
+ let page = notified_soup_page(&pool, request).await?;
+
+ // Neither the document's T1 notifications nor its duplicate T9 rows may
+ // reintroduce it after its T9 cursor. The same-id chat is newer still.
+ assert!(page.iter().all(|item| item.entity.entity_id != DOC_A));
+ assert_eq!(minutes(&page), vec![8, 7, 6, 5, 4, 3, 2, 0]);
+ Ok(())
+}
+
+#[sqlx::test(
+ fixtures(
+ path = "../../../../../fixtures",
+ scripts("notified_at", "notified_latest")
+ ),
+ migrator = "MACRO_DB_MIGRATIONS"
+)]
+async fn state_filter_does_not_move_latest_timestamp(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let links = [Uuid::parse_str(LINK_1)?];
+ let sources = sources();
+ let filter = EntityFilterAst {
+ document_filter: Some(Arc::new(Expr::val(DocumentLiteral::NotificationState(
+ item_filters::NotificationState::Unseen,
+ )))),
+ ..EntityFilterAst::default()
+ };
+ let page = notified_soup_page(&pool, req(Some(&filter), &links, &sources, EVERYTHING)).await?;
+ let document = page
+ .iter()
+ .find(|item| {
+ item.entity.entity_type == EntityType::Document && item.entity.entity_id == DOC_A
+ })
+ .expect("an older unseen notification still qualifies the document");
+ // Notification state is a membership filter, not a filter on the input
+ // to max(created_at); the latest (done) notification still sets the key.
+ assert_eq!(document.notified_at.minute(), 9);
+ Ok(())
+}