Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp

fix(api): read user content pages in descending order

Hampus733f16cd3ec8
diff --git a/fluxer_api/src/api/user/repositories/RecentMentionRepository.ts b/fluxer_api/src/api/user/repositories/RecentMentionRepository.ts
index 377bc0276..1428bc3b5 100644
--- a/fluxer_api/src/api/user/repositories/RecentMentionRepository.ts
+++ b/fluxer_api/src/api/user/repositories/RecentMentionRepository.ts
@@ -14,6 +14,7 @@ const FETCH_RECENT_MENTION_CQL = RecentMentions.selectCql({
const createFetchRecentMentionsQuery = (limit: number) =>
RecentMentions.select({
where: [RecentMentions.where.eq('user_id'), RecentMentions.where.lt('message_id', 'before_message_id')],
+ orderBy: {col: 'message_id', direction: 'DESC'},
limit,
});
const BULK_RECENT_MENTION_BATCH_QUERY_LIMIT = 50;
diff --git a/fluxer_api/src/api/user/repositories/SavedMessageRepository.ts b/fluxer_api/src/api/user/repositories/SavedMessageRepository.ts
index e09b20b28..927af49d8 100644
--- a/fluxer_api/src/api/user/repositories/SavedMessageRepository.ts
+++ b/fluxer_api/src/api/user/repositories/SavedMessageRepository.ts
@@ -10,6 +10,7 @@ import {generateSnowflake} from '@fluxer/snowflake/src/Snowflake';
const createFetchSavedMessagesQuery = (limit: number) =>
SavedMessages.select({
where: [SavedMessages.where.eq('user_id'), SavedMessages.where.lt('message_id', 'before_message_id')],
+ orderBy: {col: 'message_id', direction: 'DESC'},
limit,
});
diff --git a/fluxer_api/src/api/user/services/UserContentService.ts b/fluxer_api/src/api/user/services/UserContentService.ts
index 321689366..de0c3970a 100644
--- a/fluxer_api/src/api/user/services/UserContentService.ts
+++ b/fluxer_api/src/api/user/services/UserContentService.ts
@@ -19,12 +19,10 @@ import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import type {KVBulkMessageDeletionQueueService} from '@app/api/infrastructure/KVBulkMessageDeletionQueueService';
-import type {UserCacheService} from '@app/api/infrastructure/UserCacheService';
import {Logger} from '@app/api/Logger';
import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
import {resolveLimitSafe} from '@app/api/limits/LimitConfigUtils';
import {createLimitMatchContext} from '@app/api/limits/LimitMatchContextBuilder';
-import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware';
import type {Channel} from '@app/api/models/Channel';
import type {Message} from '@app/api/models/Message';
import {PushSubscription} from '@app/api/models/PushSubscription';
@@ -63,7 +61,6 @@ import {HarvestFailedError} from '@fluxer/errors/src/domains/moderation/HarvestF
import {HarvestNotReadyError} from '@fluxer/errors/src/domains/moderation/HarvestNotReadyError';
import {NsfwContentRequiresAgeVerificationError} from '@fluxer/errors/src/domains/moderation/NsfwContentRequiresAgeVerificationError';
import {UnknownHarvestError} from '@fluxer/errors/src/domains/moderation/UnknownHarvestError';
-import {UnknownUserError} from '@fluxer/errors/src/domains/user/UnknownUserError';
import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas';
import type {HarvestCreationResponse, HarvestStatusResponse} from '@fluxer/schema/src/domains/user/UserHarvestSchemas';
import type {
@@ -74,6 +71,7 @@ import type {
} from '@fluxer/schema/src/domains/user/UserRequestSchemas';
import type {SavedMessageStatus} from '@fluxer/schema/src/domains/user/UserResponseSchemas';
import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';
+import {compareSnowflakes} from '@fluxer/snowflake/src/SnowflakeComparison';
import {isPubliclyRoutableUrlShape} from '@pkgs/http_client/src/PublicInternetRequestUrlPolicy';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';
@@ -103,6 +101,16 @@ const DEFAULT_MOBILE_APP_ID = 'stable';
const DEFAULT_APNS_PROVIDER_ENVIRONMENT = 'production';
const MAX_HIDDEN_REFILL_PAGES = 3;
+function oldestMessageId(entries: ReadonlyArray<{messageId: MessageID}>): MessageID | undefined {
+ let oldest: MessageID | undefined;
+ for (const {messageId} of entries) {
+ if (oldest === undefined || messageId < oldest) {
+ oldest = messageId;
+ }
+ }
+ return oldest;
+}
+
function savedMessageCreatePayloads(channel: Channel, data: MessageResponse | undefined): Array<unknown> {
const guildId = channel.guildId?.toString();
if (!data || !guildId) return [data];
@@ -127,7 +135,10 @@ function assertPublicPushEndpoint(endpoint: string, fieldName: string): void {
let parsedUrl: URL;
try {
parsedUrl = new URL(endpoint);
- } catch {
+ } catch (error) {
+ if (!(error instanceof TypeError)) {
+ Logger.warn({error, fieldName}, 'Failed to parse a public push endpoint');
+ }
throw InputValidationError.fromCode(fieldName, ValidationErrorCodes.INVALID_URL_FORMAT);
}
if (!isPubliclyRoutableUrlShape(parsedUrl)) {
@@ -192,10 +203,10 @@ export class UserContentService {
constructor(
apiContext: ApiContext,
- private channelService: ChannelService,
- private channelRepository: ChannelRepository,
- private bulkMessageDeletionQueue: KVBulkMessageDeletionQueueService,
- private limitConfigService: LimitConfigService,
+ private readonly channelService: ChannelService,
+ private readonly channelRepository: ChannelRepository,
+ private readonly bulkMessageDeletionQueue: KVBulkMessageDeletionQueueService,
+ private readonly limitConfigService: LimitConfigService,
) {
const {users, gateway, worker, snowflake, kv} = apiContext.services;
this.userRepository = users;
@@ -232,10 +243,7 @@ export class UserContentService {
if (message) messages.push(message);
}
if (hidden === 0 || mentions.length < pageLimit || refills === MAX_HIDDEN_REFILL_PAGES) break;
- cursor = mentions.reduce(
- (oldest, mention) => (mention.messageId < oldest ? mention.messageId : oldest),
- mentions[0]!.messageId,
- );
+ cursor = oldestMessageId(mentions);
pageLimit = hidden;
}
return messages.sort((a, b) => (b.id > a.id ? 1 : -1));
@@ -323,13 +331,10 @@ export class UserContentService {
const savedMessages = await this.userRepository.listSavedMessages(userId, pageLimit, cursor);
const hidden = await this.collectSavedMessages(userId, viewer, savedMessages, results);
if (hidden === 0 || savedMessages.length < pageLimit || refills === MAX_HIDDEN_REFILL_PAGES) break;
- cursor = savedMessages.reduce(
- (oldest, savedMessage) => (savedMessage.messageId < oldest ? savedMessage.messageId : oldest),
- savedMessages[0]!.messageId,
- );
+ cursor = oldestMessageId(savedMessages);
pageLimit = hidden;
}
- return results.sort((a, b) => (b.messageId > a.messageId ? 1 : a.messageId > b.messageId ? -1 : 0));
+ return results.sort((a, b) => compareSnowflakes(b.messageId, a.messageId));
}
private async collectSavedMessages(
@@ -408,15 +413,11 @@ export class UserContentService {
viewer,
channelId,
messageId,
- userCacheService,
- requestCache,
}: {
userId: UserID;
viewer: ThreadViewer;
channelId: ChannelID;
messageId: MessageID;
- userCacheService: UserCacheService;
- requestCache: RequestCache;
}): Promise<void> {
const user = await this.userRepository.findUnique(userId);
if (!user) {
@@ -439,7 +440,7 @@ export class UserContentService {
throw new UnknownMessageError();
}
await this.userRepository.createSavedMessage(userId, channelId, messageId);
- await this.dispatchSavedMessageCreate({userId, message, channel, userCacheService, requestCache});
+ await this.dispatchSavedMessageCreate({userId, message, channel});
}
async unsaveMessage({userId, messageId}: {userId: UserID; messageId: MessageID}): Promise<void> {
@@ -770,7 +771,10 @@ export class UserContentService {
let userId: UserID;
try {
userId = createUserID(BigInt(payload.userId));
- } catch {
+ } catch (error) {
+ if (!(error instanceof SyntaxError)) {
+ Logger.warn({error, harvestId: params.harvestId}, 'Failed to read a harvest download user id');
+ }
return null;
}
const harvestRepository = new UserHarvestRepository();
@@ -894,7 +898,13 @@ export class UserContentService {
};
}
- async dispatchRecentMentionDelete({userId, messageId}: {userId: UserID; messageId: MessageID}): Promise<void> {
+ private async dispatchRecentMentionDelete({
+ userId,
+ messageId,
+ }: {
+ userId: UserID;
+ messageId: MessageID;
+ }): Promise<void> {
await this.gatewayService
.dispatchPresence({
userId,
@@ -910,7 +920,7 @@ export class UserContentService {
});
}
- async dispatchSavedMessageCreate({
+ private async dispatchSavedMessageCreate({
userId,
message,
channel,
@@ -918,8 +928,6 @@ export class UserContentService {
userId: UserID;
message: Message;
channel: Channel;
- userCacheService: UserCacheService;
- requestCache: RequestCache;
}): Promise<void> {
const data = (await this.buildUnmaskedResponses(userId, [message])).responses[0];
if (!data) return;
@@ -949,7 +957,7 @@ export class UserContentService {
return maskThreadArtifactsByChannel(viewer, channelById, responses);
}
- async buildUnmaskedResponses(
+ private async buildUnmaskedResponses(
userId: UserID,
messages: Array<Message>,
): Promise<{responses: Array<MessageResponse>; channelById: Map<string, Channel>}> {
@@ -967,7 +975,7 @@ export class UserContentService {
return {responses, channelById};
}
- async dispatchSavedMessageDelete({userId, messageId}: {userId: UserID; messageId: MessageID}): Promise<void> {
+ private async dispatchSavedMessageDelete({userId, messageId}: {userId: UserID; messageId: MessageID}): Promise<void> {
await this.gatewayService
.dispatchPresence({
userId,
diff --git a/fluxer_api/src/api/user/tests/HarvestDownloadStreaming.test.ts b/fluxer_api/src/api/user/tests/HarvestDownloadStreaming.test.ts
index 932ca3ba1..fb1ed7dad 100644
--- a/fluxer_api/src/api/user/tests/HarvestDownloadStreaming.test.ts
+++ b/fluxer_api/src/api/user/tests/HarvestDownloadStreaming.test.ts
@@ -5,6 +5,8 @@ import {getConfig} from '@app/api/Config';
import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
import {signHarvestDownloadToken} from '@app/api/user/services/HarvestDownloadToken';
import {fetchHarvestDownload, markHarvestCompleted, requestHarvest} from '@app/api/user/tests/HarvestTestUtils';
+import {HttpStatus} from '@fluxer/constants/src/HttpConstants';
+import {MS_PER_DAY, MS_PER_MINUTE} from '@fluxer/date_utils/src/DateConstants';
import {beforeEach, describe, expect, test} from 'vitest';
const ZIP_BYTES = new TextEncoder().encode('PK pretend zip');
@@ -23,7 +25,7 @@ async function withStreamingHarvestDownloads(callback: () => Promise<void>): Pro
async function completedHarvest(harness: ApiTestHarness) {
const account = await createTestAccount(harness);
const {harvest_id} = await requestHarvest(harness, account.token);
- await markHarvestCompleted(account.userId, harvest_id, new Date(Date.now() + 6 * 24 * 60 * 60 * 1000));
+ await markHarvestCompleted(account.userId, harvest_id, new Date(Date.now() + 6 * MS_PER_DAY));
const storageKey = `test/${harvest_id}.zip`;
await harness.storageService.uploadObject({
bucket: getConfig().s3.buckets.harvests,
@@ -56,11 +58,11 @@ describe('Harvest download streaming', () => {
expect(download.download_url).toBe('https://presigned.url/test');
const token = signHarvestDownloadToken(
- {userId: account.userId, harvestId, storageKey, expiresAt: Date.now() + 60_000},
+ {userId: account.userId, harvestId, storageKey, expiresAt: Date.now() + MS_PER_MINUTE},
getConfig().auth.connectionInitiationSecret,
);
const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);
- expect(response.status).toBe(404);
+ expect(response.status).toBe(HttpStatus.NOT_FOUND);
});
test('serves the archive through the API when presigned downloads are disabled', async () => {
@@ -73,7 +75,7 @@ describe('Harvest download streaming', () => {
expect(download.download_url).toContain('token=');
const response = await get(harness, downloadPath(download.download_url));
- expect(response.status).toBe(200);
+ expect(response.status).toBe(HttpStatus.OK);
expect(response.headers.get('content-type')).toBe('application/zip');
expect(response.headers.get('content-disposition')).toContain('attachment');
expect(response.headers.get('cache-control')).toBe('private, no-store');
@@ -85,11 +87,11 @@ describe('Harvest download streaming', () => {
await withStreamingHarvestDownloads(async () => {
const {account, harvestId, storageKey} = await completedHarvest(harness);
const token = signHarvestDownloadToken(
- {userId: account.userId, harvestId, storageKey, expiresAt: Date.now() + 60_000},
+ {userId: account.userId, harvestId, storageKey, expiresAt: Date.now() + MS_PER_MINUTE},
'not-the-configured-secret',
);
const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);
- expect(response.status).toBe(404);
+ expect(response.status).toBe(HttpStatus.NOT_FOUND);
});
});
@@ -102,12 +104,12 @@ describe('Harvest download streaming', () => {
userId: first.account.userId,
harvestId: first.harvestId,
storageKey: first.storageKey,
- expiresAt: Date.now() + 60_000,
+ expiresAt: Date.now() + MS_PER_MINUTE,
},
getConfig().auth.connectionInitiationSecret,
);
const response = await get(harness, `/harvest-downloads/${second.harvestId}?token=${encodeURIComponent(token)}`);
- expect(response.status).toBe(404);
+ expect(response.status).toBe(HttpStatus.NOT_FOUND);
});
});
@@ -115,11 +117,16 @@ describe('Harvest download streaming', () => {
await withStreamingHarvestDownloads(async () => {
const {account, harvestId} = await completedHarvest(harness);
const token = signHarvestDownloadToken(
- {userId: account.userId, harvestId, storageKey: 'test/some-other-key.zip', expiresAt: Date.now() + 60_000},
+ {
+ userId: account.userId,
+ harvestId,
+ storageKey: 'test/some-other-key.zip',
+ expiresAt: Date.now() + MS_PER_MINUTE,
+ },
getConfig().auth.connectionInitiationSecret,
);
const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);
- expect(response.status).toBe(404);
+ expect(response.status).toBe(HttpStatus.NOT_FOUND);
});
});
@@ -128,11 +135,11 @@ describe('Harvest download streaming', () => {
const {harvestId, storageKey} = await completedHarvest(harness);
const other = await createTestAccount(harness);
const token = signHarvestDownloadToken(
- {userId: other.userId, harvestId, storageKey, expiresAt: Date.now() + 60_000},
+ {userId: other.userId, harvestId, storageKey, expiresAt: Date.now() + MS_PER_MINUTE},
getConfig().auth.connectionInitiationSecret,
);
const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);
- expect(response.status).toBe(404);
+ expect(response.status).toBe(HttpStatus.NOT_FOUND);
});
});
@@ -140,7 +147,20 @@ describe('Harvest download streaming', () => {
await withStreamingHarvestDownloads(async () => {
const {harvestId} = await completedHarvest(harness);
const response = await get(harness, `/harvest-downloads/${harvestId}`);
- expect(response.status).toBe(404);
+ expect(response.status).toBe(HttpStatus.NOT_FOUND);
+ expect(await response.text()).toBe('Not Found');
+ });
+ });
+
+ test('rejects a signed token with a malformed user id', async () => {
+ await withStreamingHarvestDownloads(async () => {
+ const {harvestId, storageKey} = await completedHarvest(harness);
+ const token = signHarvestDownloadToken(
+ {userId: 'invalid-user', harvestId, storageKey, expiresAt: Date.now() + MS_PER_MINUTE},
+ getConfig().auth.connectionInitiationSecret,
+ );
+ const response = await get(harness, `/harvest-downloads/${harvestId}?token=${encodeURIComponent(token)}`);
+ expect(response.status).toBe(HttpStatus.NOT_FOUND);
expect(await response.text()).toBe('Not Found');
});
});
@@ -153,7 +173,7 @@ describe('Harvest download streaming', () => {
await withStreamingHarvestDownloads(async () => {
const {harvestId} = await completedHarvest(harness);
const response = await get(harness, `/harvest-downloads/${harvestId}${query}`);
- expect(response.status).toBe(404);
+ expect(response.status).toBe(HttpStatus.NOT_FOUND);
expect(await response.text()).toBe('Not Found');
});
});
@@ -163,7 +183,7 @@ describe('Harvest download streaming', () => {
const {account, harvestId} = await completedHarvest(harness);
const download = await fetchHarvestDownload(harness, account.token, harvestId);
const response = await get(harness, `${downloadPath(download.download_url)}&source=email`);
- expect(response.status).toBe(200);
+ expect(response.status).toBe(HttpStatus.OK);
expect(new Uint8Array(await response.arrayBuffer())).toEqual(ZIP_BYTES);
});
});
diff --git a/fluxer_api/src/api/user/tests/UserContentPagination.test.ts b/fluxer_api/src/api/user/tests/UserContentPagination.test.ts
new file mode 100644
index 000000000..aeda9e510
--- /dev/null
+++ b/fluxer_api/src/api/user/tests/UserContentPagination.test.ts
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import assert from 'node:assert/strict';
+import type {TestAccount} from '@app/api/auth/tests/AuthTestUtils';
+import {createChannelID, createGuildID, createMessageID, createUserID} from '@app/api/BrandedTypes';
+import {setupTestGuildWithMembers} from '@app/api/channel/tests/ChannelTestUtils';
+import {threadsRequest} from '@app/api/channel/tests/ThreadTestUtils';
+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';
+import {UserRepository} from '@app/api/user/repositories/UserRepository';
+import {ChannelTypes} from '@fluxer/constants/src/ChannelConstants';
+import {HttpStatus} from '@fluxer/constants/src/HttpConstants';
+import type {ThreadChannelResponse} from '@fluxer/schema/src/domains/channel/ThreadRequestSchemas';
+import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageResponseSchemas';
+import {afterEach, beforeEach, describe, expect, it} from 'vitest';
+
+describe.each(['mentions', 'saved-messages'])('%s pagination', (collection) => {
+ let harness: ApiTestHarness;
+ let users: UserRepository;
+
+ beforeEach(async () => {
+ harness = await createApiTestHarness();
+ users = new UserRepository();
+ });
+
+ afterEach(async () => {
+ await harness.shutdown();
+ });
+
+ async function world() {
+ const {owner, members, guild, systemChannel} = await setupTestGuildWithMembers(harness, 1);
+ const [member] = members;
+ assert(member, 'Expected a guild member');
+ return {owner, member, guild, channel: systemChannel};
+ }
+
+ async function send(account: TestAccount, channelId: string) {
+ return threadsRequest<MessageResponse>(harness, account.token)
+ .post(`/channels/${channelId}/messages`)
+ .body({content: 'Stored message'})
+ .expect(HttpStatus.OK)
+ .execute();
+ }
+
+ async function thread(account: TestAccount, channelId: string) {
+ return threadsRequest<ThreadChannelResponse>(harness, account.token)
+ .post(`/channels/${channelId}/threads`)
+ .body({name: 'Saved thread', type: ChannelTypes.PUBLIC_THREAD})
+ .expect(HttpStatus.CREATED)
+ .execute();
+ }
+
+ async function seed(account: TestAccount, guildId: string, messages: Array<MessageResponse>) {
+ const userId = createUserID(BigInt(account.userId));
+ if (collection === 'mentions') {
+ await users.createRecentMentions(
+ messages.map((message) => ({
+ user_id: userId,
+ channel_id: createChannelID(BigInt(message.channel_id)),
+ message_id: createMessageID(BigInt(message.id)),
+ guild_id: createGuildID(BigInt(guildId)),
+ is_everyone: false,
+ is_role: false,
+ })),
+ );
+ return;
+ }
+ for (const message of messages) {
+ await users.createSavedMessage(
+ userId,
+ createChannelID(BigInt(message.channel_id)),
+ createMessageID(BigInt(message.id)),
+ );
+ }
+ }
+
+ async function list(account: TestAccount, options: {limit: number; before?: string; capable?: boolean}) {
+ const query = new URLSearchParams({limit: String(options.limit)});
+ if (options.before !== undefined) {
+ query.set('before', options.before);
+ }
+ const entries = await threadsRequest<Array<{id: string}>>(harness, account.token, options)
+ .get(`/users/@me/${collection}?${query}`)
+ .expect(HttpStatus.OK)
+ .execute();
+ return entries.map((entry) => entry.id);
+ }
+
+ it('returns the newest page and continues strictly before its cursor', async () => {
+ const w = await world();
+ const oldest = await send(w.owner, w.channel.id);
+ const middle = await send(w.owner, w.channel.id);
+ const newest = await send(w.owner, w.channel.id);
+ await seed(w.member, w.guild.id, [middle, newest, oldest]);
+ expect(await list(w.member, {limit: 2})).toEqual([newest.id, middle.id]);
+ expect(await list(w.member, {limit: 2, before: middle.id})).toEqual([oldest.id]);
+ });
+
+ it('stops after four hidden pages and can reach older content with an explicit cursor', async () => {
+ const w = await world();
+ const t = await thread(w.owner, w.channel.id);
+ const visible = await send(w.owner, w.channel.id);
+ const firstHidden = await send(w.owner, t.id);
+ const secondHidden = await send(w.owner, t.id);
+ const thirdHidden = await send(w.owner, t.id);
+ const fourthHidden = await send(w.owner, t.id);
+ await seed(w.member, w.guild.id, [visible, thirdHidden, firstHidden, fourthHidden, secondHidden]);
+ expect(await list(w.member, {limit: 1, capable: false})).toEqual([]);
+ expect(await list(w.member, {limit: 1, before: firstHidden.id, capable: false})).toEqual([visible.id]);
+ });
+});