A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
fix(api): read user content pages in descending order
Hampus733f16cd3ec8
diff --git a/fluxer_api/src/api/user/repositories/RecentMentionRepository.ts b/fluxer_api/src/api/user/repositories/RecentMentionRepository.tsindex 377bc0276..1428bc3b5 100644--- a/fluxer_api/src/api/user/repositories/RecentMentionRepository.ts+++ b/fluxer_api/src/api/user/repositories/RecentMentionRepository.ts@@ -14,6 +14,7 @@ const FETCH_RECENT_MENTION_CQL = RecentMentions.selectCql({const createFetchRecentMentionsQuery = (limit: number) =>RecentMentions.select({where: [RecentMentions.where.eq('user_id'), RecentMentions.where.lt('message_id', 'before_message_id')],+ orderBy: {col: 'message_id', direction: 'DESC'},limit,});const BULK_RECENT_MENTION_BATCH_QUERY_LIMIT = 50;diff --git a/fluxer_api/src/api/user/repositories/SavedMessageRepository.ts b/fluxer_api/src/api/user/repositories/SavedMessageRepository.tsindex e09b20b28..927af49d8 100644--- a/fluxer_api/src/api/user/repositories/SavedMessageRepository.ts+++ b/fluxer_api/src/api/user/repositories/SavedMessageRepository.ts@@ -10,6 +10,7 @@ import {generateSnowflake} from '@fluxer/snowflake/src/Snowflake';const createFetchSavedMessagesQuery = (limit: number) =>SavedMessages.select({where: [SavedMessages.where.eq('user_id'), SavedMessages.where.lt('message_id', 'before_message_id')],+ orderBy: {col: 'message_id', direction: 'DESC'},limit,});diff --git a/fluxer_api/src/api/user/services/UserContentService.ts b/fluxer_api/src/api/user/services/UserContentService.tsindex 321689366..de0c3970a 100644--- a/fluxer_api/src/api/user/services/UserContentService.ts+++ b/fluxer_api/src/api/user/services/UserContentService.ts@@ -19,12 +19,10 @@ import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';import type {IStorageService} from '@app/api/infrastructure/IStorageService';import type {KVBulkMessageDeletionQueueService} from '@app/api/infrastructure/KVBulkMessageDeletionQueueService';-import type {UserCacheService} from '@app/api/infrastructure/UserCacheService';import {Logger} from '@app/api/Logger';import type {LimitConfigService} from '@app/api/limits/LimitConfigService';import {resolveLimitSafe} from '@app/api/limits/LimitConfigUtils';import {createLimitMatchContext} from '@app/api/limits/LimitMatchContextBuilder';-import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware';import type {Channel} from '@app/api/models/Channel';import type {Message} from '@app/api/models/Message';import {PushSubscription} from '@app/api/models/PushSubscription';@@ -63,7 +61,6 @@ import {HarvestFailedError} from '@fluxer/errors/src/domains/moderation/HarvestFimport {HarvestNotReadyError} from '@fluxer/errors/src/domains/moderation/HarvestNotReadyError';import {NsfwContentRequiresAgeVerificationError} from '@fluxer/errors/src/domains/moderation/NsfwContentRequiresAgeVerificationError';import {UnknownHarvestError} from '@fluxer/errors/src/domains/moderation/UnknownHarvestError';-import {UnknownUserError} from '@fluxer/errors/src/domains/user/UnknownUserError';import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas';import type {HarvestCreationResponse, HarvestStatusResponse} from '@fluxer/schema/src/domains/user/UserHarvestSchemas';import type {@@ -74,6 +71,7 @@ import type {} from '@fluxer/schema/src/domains/user/UserRequestSchemas';import type {SavedMessageStatus} from '@fluxer/schema/src/domains/user/UserResponseSchemas';import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';+import {compareSnowflakes} from '@fluxer/snowflake/src/SnowflakeComparison';import {isPubliclyRoutableUrlShape} from '@pkgs/http_client/src/PublicInternetRequestUrlPolicy';import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';@@ -103,6 +101,16 @@ const DEFAULT_MOBILE_APP_ID = 'stable';const DEFAULT_APNS_PROVIDER_ENVIRONMENT = 'production';const MAX_HIDDEN_REFILL_PAGES = 3;+function oldestMessageId(entries: ReadonlyArray<{messageId: MessageID}>): MessageID | undefined {+ let oldest: MessageID | undefined;+ for (const {messageId} of entries) {+ if (oldest === undefined || messageId < oldest) {+ oldest = messageId;+ }+ }+ return oldest;+}+function savedMessageCreatePayloads(channel: Channel, data: MessageResponse | undefined): Array<unknown> {const guildId = channel.guildId?.toString();if (!data || !guildId) return [data];@@ -127,7 +135,10 @@ function assertPublicPushEndpoint(endpoint: string, fieldName: string): void {let parsedUrl: URL;try {parsedUrl = new URL(endpoint);- } catch {+ } catch (error) {+ if (!(error instanceof TypeError)) {+ Logger.warn({error, fieldName}, 'Failed to parse a public push endpoint');+ }throw InputValidationError.fromCode(fieldName, ValidationErrorCodes.INVALID_URL_FORMAT);}if (!isPubliclyRoutableUrlShape(parsedUrl)) {@@ -192,10 +203,10 @@ export class UserContentService {constructor(apiContext: ApiContext,- private channelService: ChannelService,- private channelRepository: ChannelRepository,- private bulkMessageDeletionQueue: KVBulkMessageDeletionQueueService,- private limitConfigService: LimitConfigService,+ private readonly channelService: ChannelService,+ private readonly channelRepository: ChannelRepository,+ private readonly bulkMessageDeletionQueue: KVBulkMessageDeletionQueueService,+ private readonly limitConfigService: LimitConfigService,) {const {users, gateway, worker, snowflake, kv} = apiContext.services;this.userRepository = users;@@ -232,10 +243,7 @@ export class UserContentService {if (message) messages.push(message);}if (hidden === 0 || mentions.length < pageLimit || refills === MAX_HIDDEN_REFILL_PAGES) break;- cursor = mentions.reduce(- (oldest, mention) => (mention.messageId < oldest ? mention.messageId : oldest),- mentions[0]!.messageId,- );+ cursor = oldestMessageId(mentions);pageLimit = hidden;}return messages.sort((a, b) => (b.id > a.id ? 1 : -1));@@ -323,13 +331,10 @@ export class UserContentService {const savedMessages = await this.userRepository.listSavedMessages(userId, pageLimit, cursor);const hidden = await this.collectSavedMessages(userId, viewer, savedMessages, results);if (hidden === 0 || savedMessages.length < pageLimit || refills === MAX_HIDDEN_REFILL_PAGES) break;- cursor = savedMessages.reduce(- (oldest, savedMessage) => (savedMessage.messageId < oldest ? savedMessage.messageId : oldest),- savedMessages[0]!.messageId,- );+ cursor = oldestMessageId(savedMessages);pageLimit = hidden;}- return results.sort((a, b) => (b.messageId > a.messageId ? 1 : a.messageId > b.messageId ? -1 : 0));+ return results.sort((a, b) => compareSnowflakes(b.messageId, a.messageId));}private async collectSavedMessages(@@ -408,15 +413,11 @@ export class UserContentService {viewer,channelId,messageId,- userCacheService,- requestCache,}: {userId: UserID;viewer: ThreadViewer;channelId: ChannelID;messageId: MessageID;- userCacheService: UserCacheService;- requestCache: RequestCache;}): Promise<void> {const user = await this.userRepository.findUnique(userId);if (!user) {@@ -439,7 +440,7 @@ export class UserContentService {throw new UnknownMessageError();}await this.userRepository.createSavedMessage(userId, channelId, messageId);- await this.dispatchSavedMessageCreate({userId, message, channel, userCacheService, requestCache});+ await this.dispatchSavedMessageCreate({userId, message, channel});}async unsaveMessage({userId, messageId}: {userId: UserID; messageId: MessageID}): Promise<void> {@@ -770,7 +771,10 @@ export class UserContentService {let userId: UserID;try {userId = createUserID(BigInt(payload.userId));- } catch {+ } catch (error) {+ if (!(error instanceof SyntaxError)) {+ Logger.warn({error, harvestId: params.harvestId}, 'Failed to read a harvest download user id');+ }return null;}const harvestRepository = new UserHarvestRepository();@@ -894,7 +898,13 @@ export class UserContentService {};}- async dispatchRecentMentionDelete({userId, messageId}: {userId: UserID; messageId: MessageID}): Promise<void> {+ private async dispatchRecentMentionDelete({+ userId,+ messageId,+ }: {+ userId: UserID;+ messageId: MessageID;+ }): Promise<void> {await this.gatewayService.dispatchPresence({userId,@@ -910,7 +920,7 @@ export class UserContentService {});}- async dispatchSavedMessageCreate({+ private async dispatchSavedMessageCreate({userId,message,channel,@@ -918,8 +928,6 @@ export class UserContentService {userId: UserID;message: Message;channel: Channel;- userCacheService: UserCacheService;- requestCache: RequestCache;}): Promise<void> {const data = (await this.buildUnmaskedResponses(userId, [message])).responses[0];if (!data) return;@@ -949,7 +957,7 @@ export class UserContentService {return maskThreadArtifactsByChannel(viewer, channelById, responses);}- async buildUnmaskedResponses(+ private async buildUnmaskedResponses(userId: UserID,messages: Array<Message>,): Promise<{responses: Array<MessageResponse>; channelById: Map<string, Channel>}> {@@ -967,7 +975,7 @@ export class UserContentService {return {responses, channelById};}- async dispatchSavedMessageDelete({userId, messageId}: {userId: UserID; messageId: MessageID}): Promise<void> {+ private async dispatchSavedMessageDelete({userId, messageId}: {userId: UserID; messageId: MessageID}): Promise<void> {await this.gatewayService.dispatchPresence({userId,diff --git a/fluxer_api/src/api/user/tests/HarvestDownloadStreaming.test.ts b/fluxer_api/src/api/user/tests/HarvestDownloadStreaming.test.tsindex 932ca3ba1..fb1ed7dad 100644--- a/fluxer_api/src/api/user/tests/HarvestDownloadStreaming.test.ts+++ b/fluxer_api/src/api/user/tests/HarvestDownloadStreaming.test.ts@@ -5,6 +5,8 @@ import {getConfig} from '@app/api/Config';import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';import {signHarvestDownloadToken} from '@app/api/user/services/HarvestDownloadToken';import {fetchHarvestDownload, markHarvestCompleted, requestHarvest} from '@app/api/user/tests/HarvestTestUtils';+import {HttpStatus} from '@fluxer/constants/src/HttpConstants';+import {MS_PER_DAY, MS_PER_MINUTE} from '@fluxer/date_utils/src/DateConstants';import {beforeEach, describe, expect, test} from 'vitest';const ZIP_BYTES = new TextEncoder().encode('PK pretend zip');@@ -23,7 +25,7 @@ async function withStreamingHarvestDownloads(callback: () => Promise<void>): Proasync function completedHarvest(harness: ApiTestHarness) {const account = await createTestAccount(harness);const {harvest_id} = await requestHarvest(harness, account.token);- await markHarvestCompleted(account.userId, harvest_id, new Date(Date.now() + 6 * 24 * 60 * 60 * 1000));+ await markHarvestCompleted(account.userId, harvest_id, new Date(Date.now() + 6 * MS_PER_DAY));const storageKey = `test/${harvest_id}.zip`;await harness.storageService.uploadObject({bucket: getConfig().s3.buckets.harvests,@@ -56,11 +58,11 @@ describe('Harvest download streaming', () => {expect(download.download_url).toBe('https://presigned.url/test');const token = signHarvestDownloadToken(- {userId: account.userId, harvestId, storageKey, expiresAt: Date.now() + 60_000},+ {userId: account.userId, harvestId, storageKey, expiresAt: Date.now() + MS_PER_MINUTE},getConfig().auth.connectionInitiationSecret,);const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);- expect(response.status).toBe(404);+ expect(response.status).toBe(HttpStatus.NOT_FOUND);});test('serves the archive through the API when presigned downloads are disabled', async () => {@@ -73,7 +75,7 @@ describe('Harvest download streaming', () => {expect(download.download_url).toContain('token=');const response = await get(harness, downloadPath(download.download_url));- expect(response.status).toBe(200);+ expect(response.status).toBe(HttpStatus.OK);expect(response.headers.get('content-type')).toBe('application/zip');expect(response.headers.get('content-disposition')).toContain('attachment');expect(response.headers.get('cache-control')).toBe('private, no-store');@@ -85,11 +87,11 @@ describe('Harvest download streaming', () => {await withStreamingHarvestDownloads(async () => {const {account, harvestId, storageKey} = await completedHarvest(harness);const token = signHarvestDownloadToken(- {userId: account.userId, harvestId, storageKey, expiresAt: Date.now() + 60_000},+ {userId: account.userId, harvestId, storageKey, expiresAt: Date.now() + MS_PER_MINUTE},'not-the-configured-secret',);const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);- expect(response.status).toBe(404);+ expect(response.status).toBe(HttpStatus.NOT_FOUND);});});@@ -102,12 +104,12 @@ describe('Harvest download streaming', () => {userId: first.account.userId,harvestId: first.harvestId,storageKey: first.storageKey,- expiresAt: Date.now() + 60_000,+ expiresAt: Date.now() + MS_PER_MINUTE,},getConfig().auth.connectionInitiationSecret,);const response = await get(harness, `/harvest-downloads/${second.harvestId}?token=${encodeURIComponent(token)}`);- expect(response.status).toBe(404);+ expect(response.status).toBe(HttpStatus.NOT_FOUND);});});@@ -115,11 +117,16 @@ describe('Harvest download streaming', () => {await withStreamingHarvestDownloads(async () => {const {account, harvestId} = await completedHarvest(harness);const token = signHarvestDownloadToken(- {userId: account.userId, harvestId, storageKey: 'test/some-other-key.zip', expiresAt: Date.now() + 60_000},+ {+ userId: account.userId,+ harvestId,+ storageKey: 'test/some-other-key.zip',+ expiresAt: Date.now() + MS_PER_MINUTE,+ },getConfig().auth.connectionInitiationSecret,);const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);- expect(response.status).toBe(404);+ expect(response.status).toBe(HttpStatus.NOT_FOUND);});});@@ -128,11 +135,11 @@ describe('Harvest download streaming', () => {const {harvestId, storageKey} = await completedHarvest(harness);const other = await createTestAccount(harness);const token = signHarvestDownloadToken(- {userId: other.userId, harvestId, storageKey, expiresAt: Date.now() + 60_000},+ {userId: other.userId, harvestId, storageKey, expiresAt: Date.now() + MS_PER_MINUTE},getConfig().auth.connectionInitiationSecret,);const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);- expect(response.status).toBe(404);+ expect(response.status).toBe(HttpStatus.NOT_FOUND);});});@@ -140,7 +147,20 @@ describe('Harvest download streaming', () => {await withStreamingHarvestDownloads(async () => {const {harvestId} = await completedHarvest(harness);const response = await get(harness, `/harvest-downloads/${harvestId}`);- expect(response.status).toBe(404);+ expect(response.status).toBe(HttpStatus.NOT_FOUND);+ expect(await response.text()).toBe('Not Found');+ });+ });++ test('rejects a signed token with a malformed user id', async () => {+ await withStreamingHarvestDownloads(async () => {+ const {harvestId, storageKey} = await completedHarvest(harness);+ const token = signHarvestDownloadToken(+ {userId: 'invalid-user', harvestId, storageKey, expiresAt: Date.now() + MS_PER_MINUTE},+ getConfig().auth.connectionInitiationSecret,+ );+ const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);+ expect(response.status).toBe(HttpStatus.NOT_FOUND);expect(await response.text()).toBe('Not Found');});});@@ -153,7 +173,7 @@ describe('Harvest download streaming', () => {await withStreamingHarvestDownloads(async () => {const {harvestId} = await completedHarvest(harness);const response = await get(harness, `/harvest-downloads/${harvestId}${query}`);- expect(response.status).toBe(404);+ expect(response.status).toBe(HttpStatus.NOT_FOUND);expect(await response.text()).toBe('Not Found');});});@@ -163,7 +183,7 @@ describe('Harvest download streaming', () => {const {account, harvestId} = await completedHarvest(harness);const download = await fetchHarvestDownload(harness, account.token, harvestId);const response = await get(harness, `${downloadPath(download.download_url)}&source=email`);- expect(response.status).toBe(200);+ expect(response.status).toBe(HttpStatus.OK);expect(new Uint8Array(await response.arrayBuffer())).toEqual(ZIP_BYTES);});});diff --git a/fluxer_api/src/api/user/tests/UserContentPagination.test.ts b/fluxer_api/src/api/user/tests/UserContentPagination.test.tsnew file mode 100644index 000000000..aeda9e510--- /dev/null+++ b/fluxer_api/src/api/user/tests/UserContentPagination.test.ts@@ -0,0 +1,110 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import assert from 'node:assert/strict';+import type {TestAccount} from '@app/api/auth/tests/AuthTestUtils';+import {createChannelID, createGuildID, createMessageID, createUserID} from '@app/api/BrandedTypes';+import {setupTestGuildWithMembers} from '@app/api/channel/tests/ChannelTestUtils';+import {threadsRequest} from '@app/api/channel/tests/ThreadTestUtils';+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';+import {UserRepository} from '@app/api/user/repositories/UserRepository';+import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants';+import {HttpStatus} from '@fluxer/constants/src/HttpConstants';+import type {ThreadChannelResponse} from '@fluxer/schema/src/domains/channel/ThreadRequestSchemas';+import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas';+import {afterEach, beforeEach, describe, expect, it} from 'vitest';++describe.each(['mentions', 'saved-messages'])('%s pagination', (collection) => {+ let harness: ApiTestHarness;+ let users: UserRepository;++ beforeEach(async () => {+ harness = await createApiTestHarness();+ users = new UserRepository();+ });++ afterEach(async () => {+ await harness.shutdown();+ });++ async function world() {+ const {owner, members, guild, systemChannel} = await setupTestGuildWithMembers(harness, 1);+ const [member] = members;+ assert(member, 'Expected a guild member');+ return {owner, member, guild, channel: systemChannel};+ }++ async function send(account: TestAccount, channelId: string) {+ return threadsRequest<MessageResponse>(harness, account.token)+ .post(`/channels/${channelId}/messages`)+ .body({content: 'Stored message'})+ .expect(HttpStatus.OK)+ .execute();+ }++ async function thread(account: TestAccount, channelId: string) {+ return threadsRequest<ThreadChannelResponse>(harness, account.token)+ .post(`/channels/${channelId}/threads`)+ .body({name: 'Saved thread', type: ChannelTypes.PUBLIC_THREAD})+ .expect(HttpStatus.CREATED)+ .execute();+ }++ async function seed(account: TestAccount, guildId: string, messages: Array<MessageResponse>) {+ const userId = createUserID(BigInt(account.userId));+ if (collection === 'mentions') {+ await users.createRecentMentions(+ messages.map((message) => ({+ user_id: userId,+ channel_id: createChannelID(BigInt(message.channel_id)),+ message_id: createMessageID(BigInt(message.id)),+ guild_id: createGuildID(BigInt(guildId)),+ is_everyone: false,+ is_role: false,+ })),+ );+ return;+ }+ for (const message of messages) {+ await users.createSavedMessage(+ userId,+ createChannelID(BigInt(message.channel_id)),+ createMessageID(BigInt(message.id)),+ );+ }+ }++ async function list(account: TestAccount, options: {limit: number; before?: string; capable?: boolean}) {+ const query = new URLSearchParams({limit: String(options.limit)});+ if (options.before !== undefined) {+ query.set('before', options.before);+ }+ const entries = await threadsRequest<Array<{id: string}>>(harness, account.token, options)+ .get(`/users/@me/${collection}?${query}`)+ .expect(HttpStatus.OK)+ .execute();+ return entries.map((entry) => entry.id);+ }++ it('returns the newest page and continues strictly before its cursor', async () => {+ const w = await world();+ const oldest = await send(w.owner, w.channel.id);+ const middle = await send(w.owner, w.channel.id);+ const newest = await send(w.owner, w.channel.id);+ await seed(w.member, w.guild.id, [middle, newest, oldest]);+ expect(await list(w.member, {limit: 2})).toEqual([newest.id, middle.id]);+ expect(await list(w.member, {limit: 2, before: middle.id})).toEqual([oldest.id]);+ });++ it('stops after four hidden pages and can reach older content with an explicit cursor', async () => {+ const w = await world();+ const t = await thread(w.owner, w.channel.id);+ const visible = await send(w.owner, w.channel.id);+ const firstHidden = await send(w.owner, t.id);+ const secondHidden = await send(w.owner, t.id);+ const thirdHidden = await send(w.owner, t.id);+ const fourthHidden = await send(w.owner, t.id);+ await seed(w.member, w.guild.id, [visible, thirdHidden, firstHidden, fourthHidden, secondHidden]);+ expect(await list(w.member, {limit: 1, capable: false})).toEqual([]);+ expect(await list(w.member, {limit: 1, before: firstHidden.id, capable: false})).toEqual([visible.id]);+ });+});