Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/fix/channel list perf (#5662)
Sean Aye4067868a4a88
diff --git a/crates/channels/fixtures/channel_list_pagination.sql b/crates/channels/fixtures/channel_list_pagination.sqlnew file mode 100644index 0000000000..f810dac329--- /dev/null+++ b/crates/channels/fixtures/channel_list_pagination.sql@@ -0,0 +1,11 @@+-- Give user-a's channels different created_at and updated_at orderings so the+-- channel-list cursor regression test exercises both sort methods.+UPDATE comms_channels+SET created_at = '2024-01-01 00:00:00+00',+ updated_at = '2024-01-03 00:00:00+00'+WHERE id = '00000000-0000-0000-0000-000000000c01';++UPDATE comms_channels+SET created_at = '2024-01-02 00:00:00+00',+ updated_at = '2024-01-02 00:00:00+00'+WHERE id = '00000000-0000-0000-0000-000000000c03';diff --git a/crates/channels/src/outbound/pg_channels_repo.rs b/crates/channels/src/outbound/pg_channels_repo.rsindex fe5ffee9e6..3b2bc1e87b 100644--- a/crates/channels/src/outbound/pg_channels_repo.rs+++ b/crates/channels/src/outbound/pg_channels_repo.rs@@ -731,7 +731,7 @@ fn id_to_display_name(user_id: &MacroUserIdStr<'static>, name_lookup: &NameLooku#[cfg(feature = "list")]static CHANNEL_LIST_PREFIX: &str = r#"WITH user_channels AS (- SELECT DISTINCT c.*+ SELECT c.*FROM comms_channels cINNER JOIN comms_channel_participants cp ON cp.channel_id = c.idWHERE cp.user_id = $1 AND cp.left_at IS NULL@@ -743,7 +743,7 @@ static CHANNEL_LIST_PREFIX: &str = r#"#[cfg(feature = "list")]static CHANNEL_LIST_PREFIX_WITH_TEAM_CHANNELS: &str = r#"WITH user_channels AS (- SELECT DISTINCT c.*+ SELECT c.*FROM comms_channels cWHERE (EXISTS (@@ -768,9 +768,19 @@ static CHANNEL_LIST_PREFIX_WITH_TEAM_CHANNELS: &str = r#"#[cfg(feature = "list")]static CHANNEL_LIST_SELECT: &str = r#"),+ paged_channels AS (+ SELECT uc.*+ FROM user_channels uc+ WHERE+ ($4::timestamptz IS NULL)+ OR+ ((CASE $2 WHEN 'created_at' THEN uc.created_at ELSE uc.updated_at END), uc.id::text) < ($4, $5)+ ORDER BY (CASE $2 WHEN 'created_at' THEN uc.created_at ELSE uc.updated_at END) DESC, uc.id::text DESC+ LIMIT $3+ ),channel_participants_json AS (SELECT- uc.id as channel_id,+ pc.id as channel_id,ARRAY_AGG(json_build_object('channel_id', cp.channel_id,@@ -780,37 +790,32 @@ static CHANNEL_LIST_SELECT: &str = r#"'left_at', cp.left_at)) as participants- FROM user_channels uc- JOIN comms_channel_participants cp ON cp.channel_id = uc.id+ FROM paged_channels pc+ JOIN comms_channel_participants cp ON cp.channel_id = pc.idWHERE cp.left_at IS NULL- GROUP BY uc.id+ GROUP BY pc.id)SELECT- uc.id as "id",- uc.name as "name",- uc.channel_type as "channel_type",- uc.org_id as "org_id",- uc.team_id as "team_id",- uc.auto_join_team as "auto_join_team",- uc.created_at as "created_at",- uc.updated_at as "updated_at",- uc.owner_id as "owner_id",+ pc.id as "id",+ pc.name as "name",+ pc.channel_type as "channel_type",+ pc.org_id as "org_id",+ pc.team_id as "team_id",+ pc.auto_join_team as "auto_join_team",+ pc.created_at as "created_at",+ pc.updated_at as "updated_at",+ pc.owner_id as "owner_id",cpj.participants as "participants_json",EXISTS (SELECT 1FROM comms_channel_participants cp_active- WHERE cp_active.channel_id = uc.id+ WHERE cp_active.channel_id = pc.idAND cp_active.user_id = $1AND cp_active.left_at IS NULL) as "is_participant"- FROM user_channels uc- LEFT JOIN channel_participants_json cpj ON cpj.channel_id = uc.id- WHERE- ($4::timestamptz IS NULL)- OR- ((CASE $2 WHEN 'created_at' THEN uc.created_at ELSE uc.updated_at END), uc.id::text) < ($4, $5)- ORDER BY (CASE $2 WHEN 'created_at' THEN uc.created_at ELSE uc.updated_at END) DESC, uc.id::text DESC- LIMIT $3+ FROM paged_channels pc+ LEFT JOIN channel_participants_json cpj ON cpj.channel_id = pc.id+ ORDER BY (CASE $2 WHEN 'created_at' THEN pc.created_at ELSE pc.updated_at END) DESC, pc.id::text DESC"#;#[cfg(feature = "list")]@@ -930,6 +935,8 @@ fn channel_filter_mentions_participation(expr: &Expr<ChannelLiteral>) -> bool {fn build_channel_list_query(filter_ast: &LiteralTree<ChannelLiteral>,) -> QueryBuilder<'_, Postgres> {+ // QueryBuilder is required because the channel filter is an AST with arbitrary+ // AND/OR/NOT shape. Dynamic literals are constrained to parsed domain types.let prefix = if filter_ast.as_deref().is_some_and(channel_filter_mentions_participation)diff --git a/crates/channels/src/outbound/pg_channels_repo/tests.rs b/crates/channels/src/outbound/pg_channels_repo/tests.rsindex 8f04890cc8..d6a6cdfa3c 100644--- a/crates/channels/src/outbound/pg_channels_repo/tests.rs+++ b/crates/channels/src/outbound/pg_channels_repo/tests.rs@@ -1,8 +1,8 @@use crate::domain::models::{AttachmentEntityReference, BotId, BotSenderProfile, ChannelMessageFilters, ChannelType,- CreateChannelRequest, CreateEntityMentionOptions, GetChannelsParams, GetChannelsRequest,- GetThreadReplyRowsRequest, MessagePageDirection, NotificationFilters, ParticipantRole,- PatchChannelRequest,+ ChannelWithParticipants, CreateChannelRequest, CreateEntityMentionOptions, GetChannelsParams,+ GetChannelsRequest, GetThreadReplyRowsRequest, MessagePageDirection, NotificationFilters,+ ParticipantRole, PatchChannelRequest,};use crate::domain::ports::{ChannelListRepo, ChannelRepo};use crate::outbound::pg_channels_repo::PgChannelsRepo;@@ -35,6 +35,7 @@ const NO_FILTERS: ChannelMessageFilters = ChannelMessageFilters {const CH1: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c01);const CH2: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c02);+const CH3: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c03);const TEAM_A: Uuid = Uuid::from_u128(0x11111111_1111_1111_1111_111111111111);const TEAM_A_AUTO_ACTIVE: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c11);const TEAM_A_AUTO_LEFT: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c12);@@ -105,6 +106,20 @@ fn channel_filter(literal: ChannelLiteral) -> LiteralTree<ChannelLiteral> {Some(Arc::new(Expr::val(literal)))}+fn participant_roles(channel: &ChannelWithParticipants) -> Vec<(String, ParticipantRole)> {+ let mut participants = channel+ .participants+ .iter()+ .map(|participant| {+ assert_eq!(participant.channel_id, channel.channel.id);+ assert!(participant.left_at.is_none());+ (participant.user_id.clone(), participant.role)+ })+ .collect::<Vec<_>>();+ participants.sort_by(|a, b| a.0.cmp(&b.0));+ participants+}+#[sqlx::test(fixtures(path = "../../../fixtures", scripts("channels_repo")),migrator = "MACRO_DB_MIGRATIONS"@@ -240,6 +255,105 @@ async fn is_participant_filter_inside_not_widens_candidates(pool: Pool<Postgres>assert!(!channels[0].is_participant);}+#[sqlx::test(+ fixtures(+ path = "../../../fixtures",+ scripts("channels_repo", "channel_list_pagination")+ ),+ migrator = "MACRO_DB_MIGRATIONS"+)]+async fn channel_list_cursor_pagination_matches_unpaginated_results(pool: Pool<Postgres>) {+ let repo = repo(pool);++ for (sort, expected_ids) in [+ (SimpleSortMethod::CreatedAt, vec![CH3, CH1]),+ (SimpleSortMethod::UpdatedAt, vec![CH1, CH3]),+ ] {+ let unpaginated = repo+ .get_user_channels_with_participants(+ GetChannelsRequest {+ macro_id: macro_user_id(USER_A),+ limit: Some(50),+ include_frecency: false,+ query: Query::Sort(sort, None),+ }+ .into_params(),+ )+ .await+ .unwrap();+ let unpaginated_ids = unpaginated+ .iter()+ .map(|channel| channel.channel.id)+ .collect::<Vec<_>>();+ assert_eq!(unpaginated_ids, expected_ids);++ let mut query = Query::Sort(sort, None);+ let mut paginated = Vec::new();+ loop {+ let page = repo+ .get_user_channels_with_participants(+ GetChannelsRequest {+ macro_id: macro_user_id(USER_A),+ limit: Some(1),+ include_frecency: false,+ query,+ }+ .into_params(),+ )+ .await+ .unwrap();+ let Some(last) = page.last() else {+ break;+ };+ assert_eq!(page.len(), 1);++ query = Query::Cursor(Cursor {+ id: last.channel.id,+ limit: 1,+ val: CursorVal {+ sort_type: sort,+ last_val: match sort {+ SimpleSortMethod::CreatedAt => last.channel.created_at,+ SimpleSortMethod::UpdatedAt => last.channel.updated_at,+ _ => unreachable!("channel list test only uses supported sort methods"),+ },+ },+ filter: None,+ });+ paginated.extend(page);+ assert!(+ paginated.len() <= unpaginated.len(),+ "cursor pagination returned duplicate rows"+ );+ }++ let paginated_ids = paginated+ .iter()+ .map(|channel| channel.channel.id)+ .collect::<Vec<_>>();+ assert_eq!(paginated_ids, unpaginated_ids);+ assert_eq!(paginated.len(), unpaginated.len());++ for (expected, actual) in unpaginated.iter().zip(&paginated) {+ assert_eq!(actual.channel.id, expected.channel.id);+ assert_eq!(actual.is_participant, expected.is_participant);+ assert_eq!(participant_roles(actual), participant_roles(expected));+ assert!(actual.is_participant);++ let expected_participants = match actual.channel.id {+ CH1 => vec![+ (USER_A.to_string(), ParticipantRole::Owner),+ (USER_B.to_string(), ParticipantRole::Admin),+ (USER_C.to_string(), ParticipantRole::Member),+ ],+ CH3 => vec![(USER_A.to_string(), ParticipantRole::Owner)],+ id => panic!("unexpected channel {id}"),+ };+ assert_eq!(participant_roles(actual), expected_participants);+ }+ }+}+#[sqlx::test(fixtures(path = "../../../fixtures", scripts("channels_repo")),migrator = "MACRO_DB_MIGRATIONS"