Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(channels): paginate list channels (temp) (#5194)

teod71fb4a0932d
diff --git a/apps/web/src/lib/queries/channel/channels.ts b/apps/web/src/lib/queries/channel/channels.ts
index ffff5a0032..f1bf076e22 100644
--- a/apps/web/src/lib/queries/channel/channels.ts
+++ b/apps/web/src/lib/queries/channel/channels.ts
@@ -16,10 +16,33 @@ import { useMutation, useQuery } from '@tanstack/solid-query';
import { invalidateChannelParticipants } from './channel-participants';
import { channelKeys } from './keys';
+const CHANNEL_LIST_PAGE_SIZE = 100;
+
+export async function fetchAllChannels(
+ signal?: AbortSignal
+): Promise<ApiChannelWithLatest[]> {
+ const channels: ApiChannelWithLatest[] = [];
+ let cursor: string | undefined;
+
+ do {
+ const page = await throwOnErr(() =>
+ storageServiceClient.getChannels({
+ cursor,
+ limit: CHANNEL_LIST_PAGE_SIZE,
+ signal,
+ })
+ );
+ channels.push(...page.items);
+ cursor = page.next_cursor ?? undefined;
+ } while (cursor);
+
+ return channels;
+}
+
export function useListChannelsQuery() {
return useQuery(() => ({
queryKey: channelKeys.listChannels.queryKey,
- queryFn: async () => await throwOnErr(storageServiceClient.getChannels),
+ queryFn: ({ signal }) => fetchAllChannels(signal),
}));
}
diff --git a/apps/web/src/lib/queries/channel/tests/channels.test.ts b/apps/web/src/lib/queries/channel/tests/channels.test.ts
new file mode 100644
index 0000000000..8c72b1b52d
--- /dev/null
+++ b/apps/web/src/lib/queries/channel/tests/channels.test.ts
@@ -0,0 +1,51 @@
+import type { ApiChannelWithLatest } from '@service-storage/channel-list-types';
+import { storageServiceClient } from '@service-storage/client';
+import { ok } from 'neverthrow';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { fetchAllChannels } from '../channels';
+
+vi.mock('@service-storage/client', () => ({
+ storageServiceClient: {
+ getChannels: vi.fn(),
+ },
+}));
+
+describe('fetchAllChannels', () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ it('fetches every channel page and returns one complete list', async () => {
+ const firstChannel = { id: 'channel-1' } as ApiChannelWithLatest;
+ const secondChannel = { id: 'channel-2' } as ApiChannelWithLatest;
+ vi.mocked(storageServiceClient.getChannels)
+ .mockResolvedValueOnce(
+ ok({
+ items: [firstChannel],
+ next_cursor: 'next-page',
+ })
+ )
+ .mockResolvedValueOnce(
+ ok({
+ items: [secondChannel],
+ next_cursor: null,
+ })
+ );
+ const controller = new AbortController();
+
+ await expect(fetchAllChannels(controller.signal)).resolves.toEqual([
+ firstChannel,
+ secondChannel,
+ ]);
+ expect(storageServiceClient.getChannels).toHaveBeenNthCalledWith(1, {
+ cursor: undefined,
+ limit: 100,
+ signal: controller.signal,
+ });
+ expect(storageServiceClient.getChannels).toHaveBeenNthCalledWith(2, {
+ cursor: 'next-page',
+ limit: 100,
+ signal: controller.signal,
+ });
+ });
+});
diff --git a/apps/web/src/lib/service-clients/service-storage/channel-list-types.ts b/apps/web/src/lib/service-clients/service-storage/channel-list-types.ts
index d477238fde..be19318f9b 100644
--- a/apps/web/src/lib/service-clients/service-storage/channel-list-types.ts
+++ b/apps/web/src/lib/service-clients/service-storage/channel-list-types.ts
@@ -24,3 +24,9 @@ export type ApiChannelWithLatest = ChannelWithParticipants &
interacted_at?: string | null;
viewed_at?: string | null;
};
+
+/** A cursor-paginated page from the channel-list endpoint. */
+export type ApiChannelListPage = {
+ items: ApiChannelWithLatest[];
+ next_cursor: string | null;
+};
diff --git a/apps/web/src/lib/service-clients/service-storage/client.ts b/apps/web/src/lib/service-clients/service-storage/client.ts
index 932c01f95a..048fce3e21 100644
--- a/apps/web/src/lib/service-clients/service-storage/client.ts
+++ b/apps/web/src/lib/service-clients/service-storage/client.ts
@@ -28,7 +28,7 @@ import type { SafeFetchInit } from '@core/util/safeFetch';
import type { IDocumentStorageServiceFile } from '@filesystem/file';
import type { SerializedEditorState } from 'lexical';
import { err, ok, type Result } from 'neverthrow';
-import type { ApiChannelWithLatest } from './channel-list-types';
+import type { ApiChannelListPage } from './channel-list-types';
import type {
AccessLevel,
AddFavoriteRequest,
@@ -514,11 +514,24 @@ export const storageServiceClient = {
// The channel list is still served by the comms hex, mounted at
// `/comms/channels` on the same DSS host. Repoint to `/channels` once the
// list moves into the channels hex (alongside the comms teardown).
- async getChannels() {
+ async getChannels(args: {
+ cursor?: string;
+ limit?: number;
+ signal?: AbortSignal;
+ }) {
+ const params = new URLSearchParams();
+ if (args.cursor) params.set('cursor', args.cursor);
+ if (args.limit != null) params.set('limit', String(args.limit));
+ const query = params.toString();
+
return (
- await dssFetch<ApiChannelWithLatest[]>(`/comms/channels`, {
- method: 'GET',
- })
+ await dssFetch<ApiChannelListPage>(
+ `/comms/channels${query ? `?${query}` : ''}`,
+ {
+ method: 'GET',
+ signal: args.signal,
+ }
+ )
).map((result) => result);
},
diff --git a/crates/channels/src/domain/models.rs b/crates/channels/src/domain/models.rs
index 83bee88715..955033f776 100644
--- a/crates/channels/src/domain/models.rs
+++ b/crates/channels/src/domain/models.rs
@@ -1340,6 +1340,36 @@ pub struct ChannelWithLatest {
pub frecency_score: Option<frecency::domain::models::AggregateFrecency>,
}
+#[cfg(feature = "list")]
+impl Identify for ChannelWithLatest {
+ type Id = Uuid;
+
+ fn id(&self) -> Self::Id {
+ self.channel.channel.id
+ }
+}
+
+#[cfg(feature = "list")]
+impl SortOn<SimpleSortMethod> for ChannelWithLatest {
+ fn sort_on(sort: SimpleSortMethod) -> impl FnMut(&Self) -> CursorVal<SimpleSortMethod> {
+ move |channel| {
+ let last_val = match sort {
+ SimpleSortMethod::ViewedAt => channel.viewed_at.unwrap_or_default(),
+ SimpleSortMethod::UpdatedAt => channel.channel.channel.updated_at,
+ SimpleSortMethod::CreatedAt => channel.channel.channel.created_at,
+ SimpleSortMethod::ViewedUpdated => channel
+ .viewed_at
+ .unwrap_or(channel.channel.channel.updated_at),
+ };
+
+ CursorVal {
+ sort_type: sort,
+ last_val,
+ }
+ }
+ }
+}
+
/// Raw preview row returned from the repository.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ChannelPreviewRow {
diff --git a/crates/channels/src/inbound/list_router.rs b/crates/channels/src/inbound/list_router.rs
index aa4b99e617..708a8c607c 100644
--- a/crates/channels/src/inbound/list_router.rs
+++ b/crates/channels/src/inbound/list_router.rs
@@ -5,7 +5,7 @@ use std::sync::Arc;
use axum::{
Json, Router,
- extract::{FromRef, State},
+ extract::{FromRef, Query, State},
http::StatusCode,
response::IntoResponse,
routing::get,
@@ -15,7 +15,11 @@ use macro_authorization::{
MacroAuthorizationExtractor, MacroAuthorizationService, MacroAuthorizationState, UserOrInternal,
};
use macro_user_id::user_id::MacroUserIdStr;
-use serde::Serialize;
+use models_pagination::{
+ CursorOptionExt, CursorWithValAndFilter, PaginateOn, Paginated, SimpleSortMethod,
+ TypeEraseCursor,
+};
+use serde::{Deserialize, Serialize};
use thiserror::Error;
use utoipa::ToSchema;
use uuid::Uuid;
@@ -29,6 +33,7 @@ use crate::domain::{
};
const DEFAULT_CHANNEL_LIST_LIMIT: u32 = 100;
+const MAX_CHANNEL_LIST_LIMIT: u32 = 100;
/// Router state for legacy channel-list endpoints.
pub struct ChannelListRouterState<S, Auth> {
@@ -99,8 +104,13 @@ impl IntoResponse for ChannelListRouterErr {
path = "/channels",
tag = "channels",
operation_id = "get_channels",
+ params(
+ ("limit" = Option<u32>, Query, description = "Page size (1-100, default 100)"),
+ ("cursor" = Option<String>, Query, description = "Opaque cursor for the next page"),
+ ),
responses(
- (status = 200, body=Vec<ApiChannelWithLatest>),
+ (status = 200, body=ApiChannelListPage),
+ (status = 400, body=String),
(status = 401, body=String),
(status = 404, body=String),
(status = 500, body=String),
@@ -109,31 +119,55 @@ impl IntoResponse for ChannelListRouterErr {
async fn get_channels_handler<S, Auth>(
State(service): State<ChannelListRouterState<S, Auth>>,
authorization: MacroAuthorizationExtractor<Auth, UserOrInternal>,
-) -> Result<Json<Vec<ApiChannelWithLatest>>, ChannelListRouterErr>
+ Query(params): Query<GetChannelsQueryParams>,
+ cursor: Option<CursorWithValAndFilter<Uuid, SimpleSortMethod, ()>>,
+) -> Result<Json<ApiChannelListPage>, ChannelListRouterErr>
where
S: ChannelListService,
Auth: MacroAuthorizationService,
{
let user = &authorization.authorization.user;
+ let limit = params
+ .limit
+ .unwrap_or(DEFAULT_CHANNEL_LIST_LIMIT)
+ .clamp(1, MAX_CHANNEL_LIST_LIMIT);
let res = service
.inner
.get_channels(GetChannelsRequest {
macro_id: user.macro_user_id.clone(),
- limit: Some(DEFAULT_CHANNEL_LIST_LIMIT),
+ // Fetch one extra row so pagination can distinguish a full final
+ // page from a page with more results.
+ limit: Some(limit.saturating_add(1)),
include_frecency: true,
- query: models_pagination::Query::Sort(
- models_pagination::SimpleSortMethod::UpdatedAt,
- None,
- ),
+ query: cursor
+ .into_query(SimpleSortMethod::UpdatedAt, ())
+ .map_filter(|_| None),
})
.await
.map_err(|_| ChannelListRouterErr::Internal)?;
- Ok(Json(
- res.into_iter()
+ let has_more = res.len() > limit as usize;
+ let Paginated {
+ items, next_cursor, ..
+ } = res
+ .into_iter()
+ .paginate_on(limit as usize, SimpleSortMethod::UpdatedAt)
+ .filter_on(())
+ .into_page()
+ .type_erase();
+
+ Ok(Json(ApiChannelListPage {
+ items: items
+ .into_iter()
.map(ApiChannelWithLatest::new_from_domain)
.collect(),
- ))
+ next_cursor: next_cursor.filter(|_| has_more),
+ }))
+}
+
+#[derive(Debug, Deserialize)]
+struct GetChannelsQueryParams {
+ limit: Option<u32>,
}
#[tracing::instrument(skip(service, authorization))]
@@ -245,6 +279,15 @@ pub struct ApiChannelWithLatest {
pub frecency_score: Option<f64>,
}
+/// A cursor-paginated channel list response.
+#[derive(Debug, Clone, Serialize, ToSchema)]
+pub struct ApiChannelListPage {
+ /// Channels in this page.
+ pub items: Vec<ApiChannelWithLatest>,
+ /// Opaque cursor for the next page, if one exists.
+ pub next_cursor: Option<String>,
+}
+
impl ApiChannelWithLatest {
fn new_from_domain(value: ChannelWithLatest) -> Self {
Self {
diff --git a/crates/channels/src/inbound/list_router/test.rs b/crates/channels/src/inbound/list_router/test.rs
index 15c61c8a38..d27c4a6366 100644
--- a/crates/channels/src/inbound/list_router/test.rs
+++ b/crates/channels/src/inbound/list_router/test.rs
@@ -8,17 +8,23 @@ use axum::{
body::Body,
http::{Request, StatusCode, header},
};
+use chrono::{TimeZone, Utc};
+use http_body_util::BodyExt;
use macro_authorization::{
INTERNAL_API_KEY_HEADER, INTERNAL_MACRO_USER_ID_HEADER, InternalAuthConfig, JwtValidator,
MacroAuthorizationError, MacroAuthorizationServiceImpl, MacroAuthorizationState,
ValidatedIdentity,
};
use macro_user_id::user_id::MacroUserIdStr;
+use models_pagination::Base64Str;
use rootcause::Report;
use tower::ServiceExt;
+use uuid::Uuid;
use super::*;
-use crate::domain::models::{ChannelMessage, GetThreadReplyRowsRequest, UserName};
+use crate::domain::models::{
+ ChannelMessage, ChannelWithParticipants, GetThreadReplyRowsRequest, LatestMessage, UserName,
+};
const VALID_BEARER_TOKEN: &str = "valid-token";
const INVALID_BEARER_TOKEN: &str = "invalid-token";
@@ -30,7 +36,11 @@ const DEFAULT_USER_ID: &str = "macro|[email protected]";
#[derive(Clone, Debug, Eq, PartialEq)]
enum ServiceCall {
- GetChannels(String),
+ GetChannels {
+ user_id: String,
+ limit: Option<u32>,
+ cursor_id: Option<Uuid>,
+ },
GetActivities(String),
}
@@ -51,6 +61,7 @@ impl ServiceCallTracker {
struct FakeChannelListService {
tracker: ServiceCallTracker,
+ channels: Vec<ChannelWithLatest>,
}
impl ChannelListService for FakeChannelListService {
@@ -58,9 +69,14 @@ impl ChannelListService for FakeChannelListService {
&self,
request: GetChannelsRequest,
) -> Result<Vec<ChannelWithLatest>, Report> {
- self.tracker
- .record(ServiceCall::GetChannels(request.macro_id.to_string()));
- Ok(Vec::new())
+ let (cursor_id, _) = request.query.vals();
+ let limit = request.limit.map_or(usize::MAX, |limit| limit as usize);
+ self.tracker.record(ServiceCall::GetChannels {
+ user_id: request.macro_id.to_string(),
+ limit: request.limit,
+ cursor_id: cursor_id.copied(),
+ });
+ Ok(self.channels.iter().take(limit).cloned().collect())
}
async fn get_activities(&self, user_id: MacroUserIdStr<'_>) -> Result<Vec<Activity>, Report> {
@@ -102,10 +118,14 @@ impl JwtValidator for FakeJwtValidator {
}
}
-fn test_router(default_user_id: Option<&str>) -> (Router, ServiceCallTracker) {
+fn test_router_with_channels(
+ default_user_id: Option<&str>,
+ channels: Vec<ChannelWithLatest>,
+) -> (Router, ServiceCallTracker) {
let tracker = ServiceCallTracker::default();
let list_service = FakeChannelListService {
tracker: tracker.clone(),
+ channels,
};
let authorization_service = MacroAuthorizationServiceImpl::new(
FakeJwtValidator,
@@ -123,6 +143,10 @@ fn test_router(default_user_id: Option<&str>) -> (Router, ServiceCallTracker) {
(channel_list_router::<_, _, ()>(state), tracker)
}
+fn test_router(default_user_id: Option<&str>) -> (Router, ServiceCallTracker) {
+ test_router_with_channels(default_user_id, Vec::new())
+}
+
fn bearer_request(path: &str, token: &str) -> Request<Body> {
Request::get(path)
.header(header::AUTHORIZATION, format!("Bearer {token}"))
@@ -142,7 +166,11 @@ async fn valid_bearer_credentials_pass_user_id_to_channel_list_service() {
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
tracker.calls(),
- vec![ServiceCall::GetChannels(BEARER_USER_ID.to_string())]
+ vec![ServiceCall::GetChannels {
+ user_id: BEARER_USER_ID.to_string(),
+ limit: Some(DEFAULT_CHANNEL_LIST_LIMIT.saturating_add(1)),
+ cursor_id: None,
+ }]
);
}
@@ -160,7 +188,122 @@ async fn standard_internal_credentials_pass_acting_user_to_channel_list_service(
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
tracker.calls(),
- vec![ServiceCall::GetChannels(ACTING_USER_ID.to_string())]
+ vec![ServiceCall::GetChannels {
+ user_id: ACTING_USER_ID.to_string(),
+ limit: Some(DEFAULT_CHANNEL_LIST_LIMIT.saturating_add(1)),
+ cursor_id: None,
+ }]
+ );
+}
+
+fn list_channel(id: Uuid, updated_at_seconds: i64) -> ChannelWithLatest {
+ let timestamp = Utc.timestamp_opt(updated_at_seconds, 0).unwrap();
+ ChannelWithLatest {
+ channel: ChannelWithParticipants {
+ channel: ChannelListItem {
+ id,
+ name: Some(format!("channel-{id}")),
+ channel_type: DomainChannelType::Private,
+ org_id: None,
+ team_id: None,
+ auto_join_team: false,
+ created_at: timestamp,
+ updated_at: timestamp,
+ owner_id: MacroUserIdStr::try_from(BEARER_USER_ID.to_string()).unwrap(),
+ },
+ participants: Vec::new(),
+ is_participant: true,
+ },
+ latest_message: LatestMessage::default(),
+ viewed_at: None,
+ interacted_at: None,
+ frecency_score: None,
+ }
+}
+
+#[tokio::test]
+async fn channel_list_returns_an_opaque_cursor_page_and_honors_limit() {
+ let first_id = Uuid::from_u128(1);
+ let second_id = Uuid::from_u128(2);
+ let (router, tracker) = test_router_with_channels(
+ None,
+ vec![list_channel(first_id, 2), list_channel(second_id, 1)],
+ );
+
+ let response = router
+ .oneshot(bearer_request("/channels?limit=1", VALID_BEARER_TOKEN))
+ .await
+ .unwrap();
+
+ assert_eq!(response.status(), StatusCode::OK);
+ let body = response.into_body().collect().await.unwrap().to_bytes();
+ let page: serde_json::Value = serde_json::from_slice(&body).unwrap();
+ assert_eq!(page["items"].as_array().unwrap().len(), 1);
+ assert_eq!(page["items"][0]["id"], first_id.to_string());
+ assert!(page["next_cursor"].as_str().is_some());
+ assert_eq!(
+ tracker.calls(),
+ vec![ServiceCall::GetChannels {
+ user_id: BEARER_USER_ID.to_string(),
+ limit: Some(2),
+ cursor_id: None,
+ }]
+ );
+}
+
+#[tokio::test]
+async fn channel_list_omits_cursor_when_a_full_page_exhausts_results() {
+ let channel_id = Uuid::from_u128(1);
+ let (router, _) = test_router_with_channels(None, vec![list_channel(channel_id, 1)]);
+
+ let response = router
+ .oneshot(bearer_request("/channels?limit=1", VALID_BEARER_TOKEN))
+ .await
+ .unwrap();
+
+ assert_eq!(response.status(), StatusCode::OK);
+ let body = response.into_body().collect().await.unwrap().to_bytes();
+ let page: serde_json::Value = serde_json::from_slice(&body).unwrap();
+ assert_eq!(page["items"].as_array().unwrap().len(), 1);
+ assert_eq!(page["items"][0]["id"], channel_id.to_string());
+ assert!(page["next_cursor"].is_null());
+}
+
+#[tokio::test]
+async fn channel_list_passes_the_next_page_cursor_to_the_service() {
+ let cursor_id = Uuid::from_u128(42);
+ let cursor = Base64Str::encode_json(models_pagination::Cursor {
+ id: cursor_id,
+ limit: 25,
+ val: models_pagination::CursorVal {
+ sort_type: SimpleSortMethod::UpdatedAt,
+ last_val: Utc.timestamp_opt(10, 0).unwrap(),
+ },
+ filter: (),
+ });
+ let encoded_cursor = cursor
+ .to_string()
+ .replace('+', "%2B")
+ .replace('/', "%2F")
+ .replace('=', "%3D");
+ let (router, tracker) = test_router(None);
+
+ let response = router
+ .oneshot(bearer_request(
+ &format!("/channels?limit=25&cursor={encoded_cursor}"),
+ VALID_BEARER_TOKEN,
+ ))
+ .await
+ .unwrap();
+
+ assert_eq!(response.status(), StatusCode::OK);
+ assert_eq!(
+ tracker.calls(),
+ vec![ServiceCall::GetChannels {
+ user_id: BEARER_USER_ID.to_string(),
+ limit: Some(26),
+ cursor_id: Some(cursor_id),
+ }]
);
}