Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(channels): paginate list channels (temp) (#5194)
teod71fb4a0932d
diff --git a/apps/web/src/lib/queries/channel/channels.ts b/apps/web/src/lib/queries/channel/channels.tsindex ffff5a0032..f1bf076e22 100644--- a/apps/web/src/lib/queries/channel/channels.ts+++ b/apps/web/src/lib/queries/channel/channels.ts@@ -16,10 +16,33 @@ import { useMutation, useQuery } from '@tanstack/solid-query';import { invalidateChannelParticipants } from './channel-participants';import { channelKeys } from './keys';+const CHANNEL_LIST_PAGE_SIZE = 100;++export async function fetchAllChannels(+ signal?: AbortSignal+): Promise<ApiChannelWithLatest[]> {+ const channels: ApiChannelWithLatest[] = [];+ let cursor: string | undefined;++ do {+ const page = await throwOnErr(() =>+ storageServiceClient.getChannels({+ cursor,+ limit: CHANNEL_LIST_PAGE_SIZE,+ signal,+ })+ );+ channels.push(...page.items);+ cursor = page.next_cursor ?? undefined;+ } while (cursor);++ return channels;+}+export function useListChannelsQuery() {return useQuery(() => ({queryKey: channelKeys.listChannels.queryKey,- queryFn: async () => await throwOnErr(storageServiceClient.getChannels),+ queryFn: ({ signal }) => fetchAllChannels(signal),}));}diff --git a/apps/web/src/lib/queries/channel/tests/channels.test.ts b/apps/web/src/lib/queries/channel/tests/channels.test.tsnew file mode 100644index 0000000000..8c72b1b52d--- /dev/null+++ b/apps/web/src/lib/queries/channel/tests/channels.test.ts@@ -0,0 +1,51 @@+import type { ApiChannelWithLatest } from '@service-storage/channel-list-types';+import { storageServiceClient } from '@service-storage/client';+import { ok } from 'neverthrow';+import { beforeEach, describe, expect, it, vi } from 'vitest';+import { fetchAllChannels } from '../channels';++vi.mock('@service-storage/client', () => ({+ storageServiceClient: {+ getChannels: vi.fn(),+ },+}));++describe('fetchAllChannels', () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ it('fetches every channel page and returns one complete list', async () => {+ const firstChannel = { id: 'channel-1' } as ApiChannelWithLatest;+ const secondChannel = { id: 'channel-2' } as ApiChannelWithLatest;+ vi.mocked(storageServiceClient.getChannels)+ .mockResolvedValueOnce(+ ok({+ items: [firstChannel],+ next_cursor: 'next-page',+ })+ )+ .mockResolvedValueOnce(+ ok({+ items: [secondChannel],+ next_cursor: null,+ })+ );+ const controller = new AbortController();++ await expect(fetchAllChannels(controller.signal)).resolves.toEqual([+ firstChannel,+ secondChannel,+ ]);+ expect(storageServiceClient.getChannels).toHaveBeenNthCalledWith(1, {+ cursor: undefined,+ limit: 100,+ signal: controller.signal,+ });+ expect(storageServiceClient.getChannels).toHaveBeenNthCalledWith(2, {+ cursor: 'next-page',+ limit: 100,+ signal: controller.signal,+ });+ });+});diff --git a/apps/web/src/lib/service-clients/service-storage/channel-list-types.ts b/apps/web/src/lib/service-clients/service-storage/channel-list-types.tsindex d477238fde..be19318f9b 100644--- a/apps/web/src/lib/service-clients/service-storage/channel-list-types.ts+++ b/apps/web/src/lib/service-clients/service-storage/channel-list-types.ts@@ -24,3 +24,9 @@ export type ApiChannelWithLatest = ChannelWithParticipants &interacted_at?: string | null;viewed_at?: string | null;};++/** A cursor-paginated page from the channel-list endpoint. */+export type ApiChannelListPage = {+ items: ApiChannelWithLatest[];+ next_cursor: string | null;+};diff --git a/apps/web/src/lib/service-clients/service-storage/client.ts b/apps/web/src/lib/service-clients/service-storage/client.tsindex 932c01f95a..048fce3e21 100644--- a/apps/web/src/lib/service-clients/service-storage/client.ts+++ b/apps/web/src/lib/service-clients/service-storage/client.ts@@ -28,7 +28,7 @@ import type { SafeFetchInit } from '@core/util/safeFetch';import type { IDocumentStorageServiceFile } from '@filesystem/file';import type { SerializedEditorState } from 'lexical';import { err, ok, type Result } from 'neverthrow';-import type { ApiChannelWithLatest } from './channel-list-types';+import type { ApiChannelListPage } from './channel-list-types';import type {AccessLevel,AddFavoriteRequest,@@ -514,11 +514,24 @@ export const storageServiceClient = {// The channel list is still served by the comms hex, mounted at// `/comms/channels` on the same DSS host. Repoint to `/channels` once the// list moves into the channels hex (alongside the comms teardown).- async getChannels() {+ async getChannels(args: {+ cursor?: string;+ limit?: number;+ signal?: AbortSignal;+ }) {+ const params = new URLSearchParams();+ if (args.cursor) params.set('cursor', args.cursor);+ if (args.limit != null) params.set('limit', String(args.limit));+ const query = params.toString();+return (- await dssFetch<ApiChannelWithLatest[]>(`/comms/channels`, {- method: 'GET',- })+ await dssFetch<ApiChannelListPage>(+ `/comms/channels${query ? `?${query}` : ''}`,+ {+ method: 'GET',+ signal: args.signal,+ }+ )).map((result) => result);},diff --git a/crates/channels/src/domain/models.rs b/crates/channels/src/domain/models.rsindex 83bee88715..955033f776 100644--- a/crates/channels/src/domain/models.rs+++ b/crates/channels/src/domain/models.rs@@ -1340,6 +1340,36 @@ pub struct ChannelWithLatest {pub frecency_score: Option<frecency::domain::models::AggregateFrecency>,}+#[cfg(feature = "list")]+impl Identify for ChannelWithLatest {+ type Id = Uuid;++ fn id(&self) -> Self::Id {+ self.channel.channel.id+ }+}++#[cfg(feature = "list")]+impl SortOn<SimpleSortMethod> for ChannelWithLatest {+ fn sort_on(sort: SimpleSortMethod) -> impl FnMut(&Self) -> CursorVal<SimpleSortMethod> {+ move |channel| {+ let last_val = match sort {+ SimpleSortMethod::ViewedAt => channel.viewed_at.unwrap_or_default(),+ SimpleSortMethod::UpdatedAt => channel.channel.channel.updated_at,+ SimpleSortMethod::CreatedAt => channel.channel.channel.created_at,+ SimpleSortMethod::ViewedUpdated => channel+ .viewed_at+ .unwrap_or(channel.channel.channel.updated_at),+ };++ CursorVal {+ sort_type: sort,+ last_val,+ }+ }+ }+}+/// Raw preview row returned from the repository.#[derive(Debug, Clone, PartialEq, Eq)]pub struct ChannelPreviewRow {diff --git a/crates/channels/src/inbound/list_router.rs b/crates/channels/src/inbound/list_router.rsindex aa4b99e617..708a8c607c 100644--- a/crates/channels/src/inbound/list_router.rs+++ b/crates/channels/src/inbound/list_router.rs@@ -5,7 +5,7 @@ use std::sync::Arc;use axum::{Json, Router,- extract::{FromRef, State},+ extract::{FromRef, Query, State},http::StatusCode,response::IntoResponse,routing::get,@@ -15,7 +15,11 @@ use macro_authorization::{MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState, UserOrInternal,};use macro_user_id::user_id::MacroUserIdStr;-use serde::Serialize;+use models_pagination::{+ CursorOptionExt, CursorWithValAndFilter, PaginateOn, Paginated, SimpleSortMethod,+ TypeEraseCursor,+};+use serde::{Deserialize, Serialize};use thiserror::Error;use utoipa::ToSchema;use uuid::Uuid;@@ -29,6 +33,7 @@ use crate::domain::{};const DEFAULT_CHANNEL_LIST_LIMIT: u32 = 100;+const MAX_CHANNEL_LIST_LIMIT: u32 = 100;/// Router state for legacy channel-list endpoints.pub struct ChannelListRouterState<S, Auth> {@@ -99,8 +104,13 @@ impl IntoResponse for ChannelListRouterErr {path = "/channels",tag = "channels",operation_id = "get_channels",+ params(+ ("limit" = Option<u32>, Query, description = "Page size (1-100, default 100)"),+ ("cursor" = Option<String>, Query, description = "Opaque cursor for the next page"),+ ),responses(- (status = 200, body=Vec<ApiChannelWithLatest>),+ (status = 200, body=ApiChannelListPage),+ (status = 400, body=String),(status = 401, body=String),(status = 404, body=String),(status = 500, body=String),@@ -109,31 +119,55 @@ impl IntoResponse for ChannelListRouterErr {async fn get_channels_handler<S, Auth>(State(service): State<ChannelListRouterState<S, Auth>>,authorization: MacroAuthorizationExtractor<Auth, UserOrInternal>,-) -> Result<Json<Vec<ApiChannelWithLatest>>, ChannelListRouterErr>+ Query(params): Query<GetChannelsQueryParams>,+ cursor: Option<CursorWithValAndFilter<Uuid, SimpleSortMethod, ()>>,+) -> Result<Json<ApiChannelListPage>, ChannelListRouterErr>whereS: ChannelListService,Auth: MacroAuthorizationService,{let user = &authorization.authorization.user;+ let limit = params+ .limit+ .unwrap_or(DEFAULT_CHANNEL_LIST_LIMIT)+ .clamp(1, MAX_CHANNEL_LIST_LIMIT);let res = service.inner.get_channels(GetChannelsRequest {macro_id: user.macro_user_id.clone(),- limit: Some(DEFAULT_CHANNEL_LIST_LIMIT),+ // Fetch one extra row so pagination can distinguish a full final+ // page from a page with more results.+ limit: Some(limit.saturating_add(1)),include_frecency: true,- query: models_pagination::Query::Sort(- models_pagination::SimpleSortMethod::UpdatedAt,- None,- ),+ query: cursor+ .into_query(SimpleSortMethod::UpdatedAt, ())+ .map_filter(|_| None),}).await.map_err(|_| ChannelListRouterErr::Internal)?;- Ok(Json(- res.into_iter()+ let has_more = res.len() > limit as usize;+ let Paginated {+ items, next_cursor, ..+ } = res+ .into_iter()+ .paginate_on(limit as usize, SimpleSortMethod::UpdatedAt)+ .filter_on(())+ .into_page()+ .type_erase();++ Ok(Json(ApiChannelListPage {+ items: items+ .into_iter().map(ApiChannelWithLatest::new_from_domain).collect(),- ))+ next_cursor: next_cursor.filter(|_| has_more),+ }))+}++#[derive(Debug, Deserialize)]+struct GetChannelsQueryParams {+ limit: Option<u32>,}#[tracing::instrument(skip(service, authorization))]@@ -245,6 +279,15 @@ pub struct ApiChannelWithLatest {pub frecency_score: Option<f64>,}+/// A cursor-paginated channel list response.+#[derive(Debug, Clone, Serialize, ToSchema)]+pub struct ApiChannelListPage {+ /// Channels in this page.+ pub items: Vec<ApiChannelWithLatest>,+ /// Opaque cursor for the next page, if one exists.+ pub next_cursor: Option<String>,+}+impl ApiChannelWithLatest {fn new_from_domain(value: ChannelWithLatest) -> Self {Self {diff --git a/crates/channels/src/inbound/list_router/test.rs b/crates/channels/src/inbound/list_router/test.rsindex 15c61c8a38..d27c4a6366 100644--- a/crates/channels/src/inbound/list_router/test.rs+++ b/crates/channels/src/inbound/list_router/test.rs@@ -8,17 +8,23 @@ use axum::{body::Body,http::{Request, StatusCode, header},};+use chrono::{TimeZone, Utc};+use http_body_util::BodyExt;use macro_authorization::{INTERNAL_API_KEY_HEADER, INTERNAL_MACRO_USER_ID_HEADER, InternalAuthConfig, JwtValidator,MacroAuthorizationError, MacroAuthorizationServiceImpl, MacroAuthorizationState,ValidatedIdentity,};use macro_user_id::user_id::MacroUserIdStr;+use models_pagination::Base64Str;use rootcause::Report;use tower::ServiceExt;+use uuid::Uuid;use super::*;-use crate::domain::models::{ChannelMessage, GetThreadReplyRowsRequest, UserName};+use crate::domain::models::{+ ChannelMessage, ChannelWithParticipants, GetThreadReplyRowsRequest, LatestMessage, UserName,+};const VALID_BEARER_TOKEN: &str = "valid-token";const INVALID_BEARER_TOKEN: &str = "invalid-token";@@ -30,7 +36,11 @@ const DEFAULT_USER_ID: &str = "macro|[email protected]";#[derive(Clone, Debug, Eq, PartialEq)]enum ServiceCall {- GetChannels(String),+ GetChannels {+ user_id: String,+ limit: Option<u32>,+ cursor_id: Option<Uuid>,+ },GetActivities(String),}@@ -51,6 +61,7 @@ impl ServiceCallTracker {struct FakeChannelListService {tracker: ServiceCallTracker,+ channels: Vec<ChannelWithLatest>,}impl ChannelListService for FakeChannelListService {@@ -58,9 +69,14 @@ impl ChannelListService for FakeChannelListService {&self,request: GetChannelsRequest,) -> Result<Vec<ChannelWithLatest>, Report> {- self.tracker- .record(ServiceCall::GetChannels(request.macro_id.to_string()));- Ok(Vec::new())+ let (cursor_id, _) = request.query.vals();+ let limit = request.limit.map_or(usize::MAX, |limit| limit as usize);+ self.tracker.record(ServiceCall::GetChannels {+ user_id: request.macro_id.to_string(),+ limit: request.limit,+ cursor_id: cursor_id.copied(),+ });+ Ok(self.channels.iter().take(limit).cloned().collect())}async fn get_activities(&self, user_id: MacroUserIdStr<'_>) -> Result<Vec<Activity>, Report> {@@ -102,10 +118,14 @@ impl JwtValidator for FakeJwtValidator {}}-fn test_router(default_user_id: Option<&str>) -> (Router, ServiceCallTracker) {+fn test_router_with_channels(+ default_user_id: Option<&str>,+ channels: Vec<ChannelWithLatest>,+) -> (Router, ServiceCallTracker) {let tracker = ServiceCallTracker::default();let list_service = FakeChannelListService {tracker: tracker.clone(),+ channels,};let authorization_service = MacroAuthorizationServiceImpl::new(FakeJwtValidator,@@ -123,6 +143,10 @@ fn test_router(default_user_id: Option<&str>) -> (Router, ServiceCallTracker) {(channel_list_router::<_, _, ()>(state), tracker)}+fn test_router(default_user_id: Option<&str>) -> (Router, ServiceCallTracker) {+ test_router_with_channels(default_user_id, Vec::new())+}+fn bearer_request(path: &str, token: &str) -> Request<Body> {Request::get(path).header(header::AUTHORIZATION, format!("Bearer {token}"))@@ -142,7 +166,11 @@ async fn valid_bearer_credentials_pass_user_id_to_channel_list_service() {assert_eq!(response.status(), StatusCode::OK);assert_eq!(tracker.calls(),- vec![ServiceCall::GetChannels(BEARER_USER_ID.to_string())]+ vec![ServiceCall::GetChannels {+ user_id: BEARER_USER_ID.to_string(),+ limit: Some(DEFAULT_CHANNEL_LIST_LIMIT.saturating_add(1)),+ cursor_id: None,+ }]);}@@ -160,7 +188,122 @@ async fn standard_internal_credentials_pass_acting_user_to_channel_list_service(assert_eq!(response.status(), StatusCode::OK);assert_eq!(tracker.calls(),- vec![ServiceCall::GetChannels(ACTING_USER_ID.to_string())]+ vec![ServiceCall::GetChannels {+ user_id: ACTING_USER_ID.to_string(),+ limit: Some(DEFAULT_CHANNEL_LIST_LIMIT.saturating_add(1)),+ cursor_id: None,+ }]+ );+}++fn list_channel(id: Uuid, updated_at_seconds: i64) -> ChannelWithLatest {+ let timestamp = Utc.timestamp_opt(updated_at_seconds, 0).unwrap();+ ChannelWithLatest {+ channel: ChannelWithParticipants {+ channel: ChannelListItem {+ id,+ name: Some(format!("channel-{id}")),+ channel_type: DomainChannelType::Private,+ org_id: None,+ team_id: None,+ auto_join_team: false,+ created_at: timestamp,+ updated_at: timestamp,+ owner_id: MacroUserIdStr::try_from(BEARER_USER_ID.to_string()).unwrap(),+ },+ participants: Vec::new(),+ is_participant: true,+ },+ latest_message: LatestMessage::default(),+ viewed_at: None,+ interacted_at: None,+ frecency_score: None,+ }+}++#[tokio::test]+async fn channel_list_returns_an_opaque_cursor_page_and_honors_limit() {+ let first_id = Uuid::from_u128(1);+ let second_id = Uuid::from_u128(2);+ let (router, tracker) = test_router_with_channels(+ None,+ vec![list_channel(first_id, 2), list_channel(second_id, 1)],+ );++ let response = router+ .oneshot(bearer_request("/channels?limit=1", VALID_BEARER_TOKEN))+ .await+ .unwrap();++ assert_eq!(response.status(), StatusCode::OK);+ let body = response.into_body().collect().await.unwrap().to_bytes();+ let page: serde_json::Value = serde_json::from_slice(&body).unwrap();+ assert_eq!(page["items"].as_array().unwrap().len(), 1);+ assert_eq!(page["items"][0]["id"], first_id.to_string());+ assert!(page["next_cursor"].as_str().is_some());+ assert_eq!(+ tracker.calls(),+ vec![ServiceCall::GetChannels {+ user_id: BEARER_USER_ID.to_string(),+ limit: Some(2),+ cursor_id: None,+ }]+ );+}++#[tokio::test]+async fn channel_list_omits_cursor_when_a_full_page_exhausts_results() {+ let channel_id = Uuid::from_u128(1);+ let (router, _) = test_router_with_channels(None, vec![list_channel(channel_id, 1)]);++ let response = router+ .oneshot(bearer_request("/channels?limit=1", VALID_BEARER_TOKEN))+ .await+ .unwrap();++ assert_eq!(response.status(), StatusCode::OK);+ let body = response.into_body().collect().await.unwrap().to_bytes();+ let page: serde_json::Value = serde_json::from_slice(&body).unwrap();+ assert_eq!(page["items"].as_array().unwrap().len(), 1);+ assert_eq!(page["items"][0]["id"], channel_id.to_string());+ assert!(page["next_cursor"].is_null());+}++#[tokio::test]+async fn channel_list_passes_the_next_page_cursor_to_the_service() {+ let cursor_id = Uuid::from_u128(42);+ let cursor = Base64Str::encode_json(models_pagination::Cursor {+ id: cursor_id,+ limit: 25,+ val: models_pagination::CursorVal {+ sort_type: SimpleSortMethod::UpdatedAt,+ last_val: Utc.timestamp_opt(10, 0).unwrap(),+ },+ filter: (),+ });+ let encoded_cursor = cursor+ .to_string()+ .replace('+', "%2B")+ .replace('/', "%2F")+ .replace('=', "%3D");+ let (router, tracker) = test_router(None);++ let response = router+ .oneshot(bearer_request(+ &format!("/channels?limit=25&cursor={encoded_cursor}"),+ VALID_BEARER_TOKEN,+ ))+ .await+ .unwrap();++ assert_eq!(response.status(), StatusCode::OK);+ assert_eq!(+ tracker.calls(),+ vec![ServiceCall::GetChannels {+ user_id: BEARER_USER_ID.to_string(),+ limit: Some(26),+ cursor_id: Some(cursor_id),+ }]);}