Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/fix/channel list perf (#5662)

Sean Aye4067868a4a88
diff --git a/crates/channels/fixtures/channel_list_pagination.sql b/crates/channels/fixtures/channel_list_pagination.sql
new file mode 100644
index 0000000000..f810dac329
--- /dev/null
+++ b/crates/channels/fixtures/channel_list_pagination.sql
@@ -0,0 +1,11 @@
+-- Give user-a's channels different created_at and updated_at orderings so the
+-- channel-list cursor regression test exercises both sort methods.
+UPDATE comms_channels
+SET created_at = '2024-01-01 00:00:00+00',
+ updated_at = '2024-01-03 00:00:00+00'
+WHERE id = '00000000-0000-0000-0000-000000000c01';
+
+UPDATE comms_channels
+SET created_at = '2024-01-02 00:00:00+00',
+ updated_at = '2024-01-02 00:00:00+00'
+WHERE id = '00000000-0000-0000-0000-000000000c03';
diff --git a/crates/channels/src/outbound/pg_channels_repo.rs b/crates/channels/src/outbound/pg_channels_repo.rs
index fe5ffee9e6..3b2bc1e87b 100644
--- a/crates/channels/src/outbound/pg_channels_repo.rs
+++ b/crates/channels/src/outbound/pg_channels_repo.rs
@@ -731,7 +731,7 @@ fn id_to_display_name(user_id: &MacroUserIdStr<'static>, name_lookup: &NameLooku
#[cfg(feature = "list")]
static CHANNEL_LIST_PREFIX: &str = r#"
WITH user_channels AS (
- SELECT DISTINCT c.*
+ SELECT c.*
FROM comms_channels c
INNER JOIN comms_channel_participants cp ON cp.channel_id = c.id
WHERE cp.user_id = $1 AND cp.left_at IS NULL
@@ -743,7 +743,7 @@ static CHANNEL_LIST_PREFIX: &str = r#"
#[cfg(feature = "list")]
static CHANNEL_LIST_PREFIX_WITH_TEAM_CHANNELS: &str = r#"
WITH user_channels AS (
- SELECT DISTINCT c.*
+ SELECT c.*
FROM comms_channels c
WHERE (
EXISTS (
@@ -768,9 +768,19 @@ static CHANNEL_LIST_PREFIX_WITH_TEAM_CHANNELS: &str = r#"
#[cfg(feature = "list")]
static CHANNEL_LIST_SELECT: &str = r#"
),
+ paged_channels AS (
+ SELECT uc.*
+ FROM user_channels uc
+ WHERE
+ ($4::timestamptz IS NULL)
+ OR
+ ((CASE $2 WHEN 'created_at' THEN uc.created_at ELSE uc.updated_at END), uc.id::text) < ($4, $5)
+ ORDER BY (CASE $2 WHEN 'created_at' THEN uc.created_at ELSE uc.updated_at END) DESC, uc.id::text DESC
+ LIMIT $3
+ ),
channel_participants_json AS (
SELECT
- uc.id as channel_id,
+ pc.id as channel_id,
ARRAY_AGG(
json_build_object(
'channel_id', cp.channel_id,
@@ -780,37 +790,32 @@ static CHANNEL_LIST_SELECT: &str = r#"
'left_at', cp.left_at
)
) as participants
- FROM user_channels uc
- JOIN comms_channel_participants cp ON cp.channel_id = uc.id
+ FROM paged_channels pc
+ JOIN comms_channel_participants cp ON cp.channel_id = pc.id
WHERE cp.left_at IS NULL
- GROUP BY uc.id
+ GROUP BY pc.id
)
SELECT
- uc.id as "id",
- uc.name as "name",
- uc.channel_type as "channel_type",
- uc.org_id as "org_id",
- uc.team_id as "team_id",
- uc.auto_join_team as "auto_join_team",
- uc.created_at as "created_at",
- uc.updated_at as "updated_at",
- uc.owner_id as "owner_id",
+ pc.id as "id",
+ pc.name as "name",
+ pc.channel_type as "channel_type",
+ pc.org_id as "org_id",
+ pc.team_id as "team_id",
+ pc.auto_join_team as "auto_join_team",
+ pc.created_at as "created_at",
+ pc.updated_at as "updated_at",
+ pc.owner_id as "owner_id",
cpj.participants as "participants_json",
EXISTS (
SELECT 1
FROM comms_channel_participants cp_active
- WHERE cp_active.channel_id = uc.id
+ WHERE cp_active.channel_id = pc.id
AND cp_active.user_id = $1
AND cp_active.left_at IS NULL
) as "is_participant"
- FROM user_channels uc
- LEFT JOIN channel_participants_json cpj ON cpj.channel_id = uc.id
- WHERE
- ($4::timestamptz IS NULL)
- OR
- ((CASE $2 WHEN 'created_at' THEN uc.created_at ELSE uc.updated_at END), uc.id::text) < ($4, $5)
- ORDER BY (CASE $2 WHEN 'created_at' THEN uc.created_at ELSE uc.updated_at END) DESC, uc.id::text DESC
- LIMIT $3
+ FROM paged_channels pc
+ LEFT JOIN channel_participants_json cpj ON cpj.channel_id = pc.id
+ ORDER BY (CASE $2 WHEN 'created_at' THEN pc.created_at ELSE pc.updated_at END) DESC, pc.id::text DESC
"#;
#[cfg(feature = "list")]
@@ -930,6 +935,8 @@ fn channel_filter_mentions_participation(expr: &Expr<ChannelLiteral>) -> bool {
fn build_channel_list_query(
filter_ast: &LiteralTree<ChannelLiteral>,
) -> QueryBuilder<'_, Postgres> {
+ // QueryBuilder is required because the channel filter is an AST with arbitrary
+ // AND/OR/NOT shape. Dynamic literals are constrained to parsed domain types.
let prefix = if filter_ast
.as_deref()
.is_some_and(channel_filter_mentions_participation)
diff --git a/crates/channels/src/outbound/pg_channels_repo/tests.rs b/crates/channels/src/outbound/pg_channels_repo/tests.rs
index 8f04890cc8..d6a6cdfa3c 100644
--- a/crates/channels/src/outbound/pg_channels_repo/tests.rs
+++ b/crates/channels/src/outbound/pg_channels_repo/tests.rs
@@ -1,8 +1,8 @@
use crate::domain::models::{
AttachmentEntityReference, BotId, BotSenderProfile, ChannelMessageFilters, ChannelType,
- CreateChannelRequest, CreateEntityMentionOptions, GetChannelsParams, GetChannelsRequest,
- GetThreadReplyRowsRequest, MessagePageDirection, NotificationFilters, ParticipantRole,
- PatchChannelRequest,
+ ChannelWithParticipants, CreateChannelRequest, CreateEntityMentionOptions, GetChannelsParams,
+ GetChannelsRequest, GetThreadReplyRowsRequest, MessagePageDirection, NotificationFilters,
+ ParticipantRole, PatchChannelRequest,
};
use crate::domain::ports::{ChannelListRepo, ChannelRepo};
use crate::outbound::pg_channels_repo::PgChannelsRepo;
@@ -35,6 +35,7 @@ const NO_FILTERS: ChannelMessageFilters = ChannelMessageFilters {
const CH1: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c01);
const CH2: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c02);
+const CH3: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c03);
const TEAM_A: Uuid = Uuid::from_u128(0x11111111_1111_1111_1111_111111111111);
const TEAM_A_AUTO_ACTIVE: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c11);
const TEAM_A_AUTO_LEFT: Uuid = Uuid::from_u128(0x00000000_0000_0000_0000_000000000c12);
@@ -105,6 +106,20 @@ fn channel_filter(literal: ChannelLiteral) -> LiteralTree<ChannelLiteral> {
Some(Arc::new(Expr::val(literal)))
}
+fn participant_roles(channel: &ChannelWithParticipants) -> Vec<(String, ParticipantRole)> {
+ let mut participants = channel
+ .participants
+ .iter()
+ .map(|participant| {
+ assert_eq!(participant.channel_id, channel.channel.id);
+ assert!(participant.left_at.is_none());
+ (participant.user_id.clone(), participant.role)
+ })
+ .collect::<Vec<_>>();
+ participants.sort_by(|a, b| a.0.cmp(&b.0));
+ participants
+}
+
#[sqlx::test(
fixtures(path = "../../../fixtures", scripts("channels_repo")),
migrator = "MACRO_DB_MIGRATIONS"
@@ -240,6 +255,105 @@ async fn is_participant_filter_inside_not_widens_candidates(pool: Pool<Postgres>
assert!(!channels[0].is_participant);
}
+#[sqlx::test(
+ fixtures(
+ path = "../../../fixtures",
+ scripts("channels_repo", "channel_list_pagination")
+ ),
+ migrator = "MACRO_DB_MIGRATIONS"
+)]
+async fn channel_list_cursor_pagination_matches_unpaginated_results(pool: Pool<Postgres>) {
+ let repo = repo(pool);
+
+ for (sort, expected_ids) in [
+ (SimpleSortMethod::CreatedAt, vec![CH3, CH1]),
+ (SimpleSortMethod::UpdatedAt, vec![CH1, CH3]),
+ ] {
+ let unpaginated = repo
+ .get_user_channels_with_participants(
+ GetChannelsRequest {
+ macro_id: macro_user_id(USER_A),
+ limit: Some(50),
+ include_frecency: false,
+ query: Query::Sort(sort, None),
+ }
+ .into_params(),
+ )
+ .await
+ .unwrap();
+ let unpaginated_ids = unpaginated
+ .iter()
+ .map(|channel| channel.channel.id)
+ .collect::<Vec<_>>();
+ assert_eq!(unpaginated_ids, expected_ids);
+
+ let mut query = Query::Sort(sort, None);
+ let mut paginated = Vec::new();
+ loop {
+ let page = repo
+ .get_user_channels_with_participants(
+ GetChannelsRequest {
+ macro_id: macro_user_id(USER_A),
+ limit: Some(1),
+ include_frecency: false,
+ query,
+ }
+ .into_params(),
+ )
+ .await
+ .unwrap();
+ let Some(last) = page.last() else {
+ break;
+ };
+ assert_eq!(page.len(), 1);
+
+ query = Query::Cursor(Cursor {
+ id: last.channel.id,
+ limit: 1,
+ val: CursorVal {
+ sort_type: sort,
+ last_val: match sort {
+ SimpleSortMethod::CreatedAt => last.channel.created_at,
+ SimpleSortMethod::UpdatedAt => last.channel.updated_at,
+ _ => unreachable!("channel list test only uses supported sort methods"),
+ },
+ },
+ filter: None,
+ });
+ paginated.extend(page);
+ assert!(
+ paginated.len() <= unpaginated.len(),
+ "cursor pagination returned duplicate rows"
+ );
+ }
+
+ let paginated_ids = paginated
+ .iter()
+ .map(|channel| channel.channel.id)
+ .collect::<Vec<_>>();
+ assert_eq!(paginated_ids, unpaginated_ids);
+ assert_eq!(paginated.len(), unpaginated.len());
+
+ for (expected, actual) in unpaginated.iter().zip(&paginated) {
+ assert_eq!(actual.channel.id, expected.channel.id);
+ assert_eq!(actual.is_participant, expected.is_participant);
+ assert_eq!(participant_roles(actual), participant_roles(expected));
+ assert!(actual.is_participant);
+
+ let expected_participants = match actual.channel.id {
+ CH1 => vec![
+ (USER_A.to_string(), ParticipantRole::Owner),
+ (USER_B.to_string(), ParticipantRole::Admin),
+ (USER_C.to_string(), ParticipantRole::Member),
+ ],
+ CH3 => vec![(USER_A.to_string(), ParticipantRole::Owner)],
+ id => panic!("unexpected channel {id}"),
+ };
+ assert_eq!(participant_roles(actual), expected_participants);
+ }
+ }
+}
+
#[sqlx::test(
fixtures(path = "../../../fixtures", scripts("channels_repo")),
migrator = "MACRO_DB_MIGRATIONS"