Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

perf(guild): fetch guild members a thousand at a time (#2201)

Hampuse993a477209c
diff --git a/fluxer_api/src/api/rpc/RpcService.ts b/fluxer_api/src/api/rpc/RpcService.ts
index e17224d59..615cca4c4 100644
--- a/fluxer_api/src/api/rpc/RpcService.ts
+++ b/fluxer_api/src/api/rpc/RpcService.ts
@@ -1360,7 +1360,9 @@ export class RpcService {
afterUserId?: UserID;
limit?: number;
}): Promise<RpcResponseGuildCollectionData> {
- await this.getGuildOrThrow(guildId);
+ if (!afterUserId) {
+ await this.getGuildOrThrow(guildId);
+ }
const chunkSize = this.resolveGuildCollectionLimit(limit);
const members = await this.guildRepository.listMembersPaginated(guildId, chunkSize + 1, afterUserId);
const hasMore = members.length > chunkSize;
diff --git a/fluxer_api/src/api/rpc/tests/RpcGuildCollectionMembers.test.ts b/fluxer_api/src/api/rpc/tests/RpcGuildCollectionMembers.test.ts
new file mode 100644
index 000000000..5a80f0d67
--- /dev/null
+++ b/fluxer_api/src/api/rpc/tests/RpcGuildCollectionMembers.test.ts
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {RpcRequest} from '@fluxer/schema/src/domains/rpc/RpcSchemas';
+import {afterEach, beforeEach, describe, expect, test} from 'vitest';
+import {setupTestGuildWithMembers} from '../../guild/tests/GuildTestUtils';
+import {type ApiTestHarness, createApiTestHarness} from '../../test/ApiTestHarness';
+import {HTTP_STATUS, TEST_IDS} from '../../test/TestConstants';
+import {createBuilder} from '../../test/TestRequestBuilder';
+
+interface RpcGuildCollectionMembersResponse {
+ type: 'guild_collection';
+ data: {
+ collection: 'members';
+ members: Array<{
+ user: {
+ id: string;
+ };
+ }>;
+ has_more: boolean;
+ next_after_user_id: string | null;
+ };
+}
+
+async function fetchMemberPage(
+ harness: ApiTestHarness,
+ guildId: string,
+ limit: number,
+ afterUserId?: string,
+): Promise<RpcGuildCollectionMembersResponse['data']> {
+ const response = await createBuilder<RpcGuildCollectionMembersResponse>(harness, '')
+ .post('/test/rpc-session-init')
+ .body({
+ type: 'guild_collection',
+ guild_id: guildId,
+ collection: 'members',
+ limit,
+ ...(afterUserId ? {after_user_id: afterUserId} : {}),
+ })
+ .expect(HTTP_STATUS.OK)
+ .execute();
+ return response.data;
+}
+
+describe('RpcService guild member collection pagination', () => {
+ let harness: ApiTestHarness;
+ beforeEach(async () => {
+ harness = await createApiTestHarness();
+ });
+ afterEach(async () => {
+ await harness?.shutdown();
+ });
+
+ test('walks every member across cursor pages', async () => {
+ const {owner, members, guild} = await setupTestGuildWithMembers(harness, 2);
+ const expectedIds = [owner.userId, ...members.map((member) => member.userId)];
+ const seenIds: Array<string> = [];
+ let afterUserId: string | undefined;
+ for (let page = 0; page <= expectedIds.length; page++) {
+ const data = await fetchMemberPage(harness, guild.id, 1, afterUserId);
+ expect(data.collection).toBe('members');
+ seenIds.push(...data.members.map((member) => member.user.id));
+ if (!data.has_more) {
+ expect(data.next_after_user_id).toBeNull();
+ break;
+ }
+ expect(data.next_after_user_id).toBe(seenIds[seenIds.length - 1]);
+ afterUserId = data.next_after_user_id ?? undefined;
+ }
+ expect(seenIds.slice().sort()).toStrictEqual(expectedIds.slice().sort());
+ });
+
+ test('rejects an unknown guild on the first member page', async () => {
+ await createBuilder(harness, '')
+ .post('/test/rpc-session-init')
+ .body({
+ type: 'guild_collection',
+ guild_id: TEST_IDS.NONEXISTENT_GUILD,
+ collection: 'members',
+ })
+ .expect(HTTP_STATUS.NOT_FOUND)
+ .execute();
+ });
+
+ test('accepts the member page size the gateway requests', () => {
+ const atCeiling = RpcRequest.safeParse({
+ type: 'guild_collection',
+ guild_id: TEST_IDS.NONEXISTENT_GUILD,
+ collection: 'members',
+ limit: 1000,
+ });
+ expect(atCeiling.success).toBe(true);
+ const aboveCeiling = RpcRequest.safeParse({
+ type: 'guild_collection',
+ guild_id: TEST_IDS.NONEXISTENT_GUILD,
+ collection: 'members',
+ limit: 1001,
+ });
+ expect(aboveCeiling.success).toBe(false);
+ });
+});
diff --git a/fluxer_gateway/src/guild/guild_manager_shard_fetch.erl b/fluxer_gateway/src/guild/guild_manager_shard_fetch.erl
index 9c3bfeda8..efa56bb43 100644
--- a/fluxer_gateway/src/guild/guild_manager_shard_fetch.erl
+++ b/fluxer_gateway/src/guild/guild_manager_shard_fetch.erl
@@ -11,7 +11,7 @@
-type fetch_result() :: {ok, map()} | {error, term()}.
-define(GUILD_COLLECTION_FETCH_TIMEOUT_MS, 120000).
--define(GUILD_MEMBER_COLLECTION_LIMIT, 250).
+-define(GUILD_MEMBER_COLLECTION_LIMIT, 1000).
-define(GUILD_COLLECTIONS, [
<<"guild">>,
<<"roles">>,