A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
feat(api/admin): serve archive and report files via signed links
Hampusa32b6f8fe81e
diff --git a/fluxer_api/src/api/admin/AdminDownloadController.ts b/fluxer_api/src/api/admin/AdminDownloadController.tsnew file mode 100644index 000000000..0a984be1f--- /dev/null+++ b/fluxer_api/src/api/admin/AdminDownloadController.ts@@ -0,0 +1,77 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {Readable} from 'node:stream';+import {ADMIN_DOWNLOAD_PATH, verifyAdminDownloadToken} from '@app/api/admin/AdminSignedDownloads';+import {Config} from '@app/api/Config';+import {StorageObjectRangeNotSatisfiableError} from '@app/api/infrastructure/IStorageService';+import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';+import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';+import {RateLimitConfigs} from '@app/api/RateLimitConfig';+import type {HonoApp} from '@app/api/types/HonoEnv';+import {Validator} from '@app/api/Validator';+import {AdminSignedDownloadParam, AdminSignedDownloadResponse} from '@fluxer/schema/src/domains/admin/AdminSchemas';+import {HarvestDownloadQuery} from '@fluxer/schema/src/domains/user/UserHarvestSchemas';++export function AdminDownloadController(app: HonoApp) {+ app.get(+ `${ADMIN_DOWNLOAD_PATH}/:filename`,+ RateLimitMiddleware(RateLimitConfigs.USER_HARVEST_DOWNLOAD_FILE),+ Validator('param', AdminSignedDownloadParam),+ Validator('query', HarvestDownloadQuery, (result, ctx) =>+ result.success ? undefined : ctx.text('Not Found', 404),+ ),+ OpenAPI({+ operationId: 'download_admin_signed_file',+ summary: 'Download an admin archive or report file',+ responseSchema: AdminSignedDownloadResponse,+ responseContentType: 'application/octet-stream',+ statusCode: [200, 206],+ security: [],+ tags: ['Admin'],+ description:+ 'Streams an admin archive or a file kept as report evidence. Authorised by a signed, expiring token that the admin API hands out in place of a storage link. Only active when presigned harvest downloads are disabled.',+ }),+ async (ctx) => {+ if (Config.presignedHarvestDownloadsEnabled) {+ return ctx.text('Not Found', 404);+ }+ const payload = verifyAdminDownloadToken(ctx.req.valid('query').token);+ if (!payload) {+ return ctx.text('Not Found', 404);+ }+ try {+ const object = await ctx.get('storageService').streamObject({+ bucket: Config.s3.buckets[payload.bucket],+ key: payload.key,+ range: ctx.req.header('range') ?? undefined,+ });+ if (!object) {+ return ctx.text('Not Found', 404);+ }+ const headers = new Headers();+ headers.set('Content-Type', object.contentType ?? 'application/octet-stream');+ headers.set(+ 'Content-Disposition',+ `attachment; filename="${encodeURIComponent(ctx.req.valid('param').filename)}"`,+ );+ headers.set('Content-Security-Policy', "sandbox; default-src 'none'");+ headers.set('Content-Length', String(object.contentLength));+ headers.set('Cache-Control', 'private, no-store');+ headers.set('X-Content-Type-Options', 'nosniff');+ headers.set('Accept-Ranges', 'bytes');+ if (object.contentRange) {+ headers.set('Content-Range', object.contentRange);+ }+ return new Response(Readable.toWeb(object.body) as ReadableStream, {+ status: object.contentRange ? 206 : 200,+ headers,+ });+ } catch (error) {+ if (error instanceof StorageObjectRangeNotSatisfiableError) {+ return ctx.text('Range Not Satisfiable', 416);+ }+ throw error;+ }+ },+ );+}diff --git a/fluxer_api/src/api/admin/AdminSignedDownloads.ts b/fluxer_api/src/api/admin/AdminSignedDownloads.tsnew file mode 100644index 000000000..f0036d589--- /dev/null+++ b/fluxer_api/src/api/admin/AdminSignedDownloads.ts@@ -0,0 +1,55 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {createHmac} from 'node:crypto';+import {Config} from '@app/api/Config';+import type {IStorageService} from '@app/api/infrastructure/IStorageService';+import {isJsonRecord} from '@app/api/utils/JsonBoundaryUtils';+import {signExpiringToken, verifyExpiringToken} from '@app/api/utils/SignedExpiringToken';++const KEY_DERIVATION_LABEL = 'fluxer.admin-download.v1';+export const ADMIN_DOWNLOAD_PATH = '/signed-downloads';++type AdminDownloadBucket = 'harvests' | 'reports';++interface AdminDownloadTokenPayload {+ bucket: AdminDownloadBucket;+ key: string;+ expiresAt: number;+}++function deriveKey(): Buffer {+ return createHmac('sha256', Config.auth.connectionInitiationSecret).update(KEY_DERIVATION_LABEL).digest();+}++function isAdminDownloadTokenPayload(value: unknown): value is AdminDownloadTokenPayload {+ return (+ isJsonRecord(value) &&+ (value['bucket'] === 'harvests' || value['bucket'] === 'reports') &&+ typeof value['key'] === 'string' &&+ typeof value['expiresAt'] === 'number' &&+ Number.isFinite(value['expiresAt'])+ );+}++export function verifyAdminDownloadToken(token: string): AdminDownloadTokenPayload | null {+ return verifyExpiringToken(token, deriveKey(), isAdminDownloadTokenPayload);+}++export async function buildAdminDownloadUrl(+ storageService: Pick<IStorageService, 'getPresignedDownloadURL'>,+ params: {bucket: AdminDownloadBucket; key: string; filename: string; expiresInSeconds: number},+): Promise<string> {+ if (Config.presignedHarvestDownloadsEnabled) {+ return storageService.getPresignedDownloadURL({+ bucket: Config.s3.buckets[params.bucket],+ key: params.key,+ expiresIn: params.expiresInSeconds,+ });+ }+ const token = signExpiringToken(+ {bucket: params.bucket, key: params.key, expiresAt: Date.now() + params.expiresInSeconds * 1000},+ deriveKey(),+ );+ const base = Config.endpoints.apiPublic.replace(/\/+$/u, '');+ return `${base}${ADMIN_DOWNLOAD_PATH}/${encodeURIComponent(params.filename)}?token=${encodeURIComponent(token)}`;+}diff --git a/fluxer_api/src/api/admin/services/AdminArchiveService.ts b/fluxer_api/src/api/admin/services/AdminArchiveService.tsindex 2c610e779..b17d06930 100644--- a/fluxer_api/src/api/admin/services/AdminArchiveService.ts+++ b/fluxer_api/src/api/admin/services/AdminArchiveService.ts@@ -1,9 +1,9 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {buildAdminDownloadUrl} from '@app/api/admin/AdminSignedDownloads';import {AdminArchive} from '@app/api/admin/models/AdminArchiveModel';import type {AdminArchiveRepository} from '@app/api/admin/repositories/AdminArchiveRepository';import type {GuildID, UserID} from '@app/api/BrandedTypes';-import {Config} from '@app/api/Config';import type {GuildRepository} from '@app/api/guild/repositories/GuildRepository';import type {ISnowflakeService} from '@app/api/infrastructure/ISnowflakeService';import type {IStorageService} from '@app/api/infrastructure/IStorageService';@@ -15,7 +15,6 @@ import {HarvestExpiredError} from '@fluxer/errors/src/domains/moderation/Harvestimport {HarvestFailedError} from '@fluxer/errors/src/domains/moderation/HarvestFailedError';import {HarvestNotReadyError} from '@fluxer/errors/src/domains/moderation/HarvestNotReadyError';import {UnknownHarvestError} from '@fluxer/errors/src/domains/moderation/UnknownHarvestError';-import {UnknownUserError} from '@fluxer/errors/src/domains/user/UnknownUserError';import type {AdminArchiveResponse, ArchiveSubjectType} from '@fluxer/schema/src/domains/admin/AdminArchiveSchemas';import type {IWorkerService} from '@pkgs/worker/src/contracts/IWorkerService';import {ms, seconds} from 'itty-time';@@ -180,10 +179,11 @@ export class AdminArchiveService {if (archive.expiresAt && archive.expiresAt < new Date()) {throw new HarvestExpiredError();}- const downloadUrl = await this.storageService.getPresignedDownloadURL({- bucket: Config.s3.buckets.harvests,+ const downloadUrl = await buildAdminDownloadUrl(this.storageService, {+ bucket: 'harvests',key: archive.storageKey,- expiresIn: DOWNLOAD_LINK_SECONDS,+ filename: `${subjectType}-${subjectId}-${archiveId}.zip`,+ expiresInSeconds: DOWNLOAD_LINK_SECONDS,});const expiresAt = new Date(Date.now() + ms('7 days'));return {downloadUrl, expiresAt: expiresAt.toISOString()};diff --git a/fluxer_api/src/api/admin/services/AdminReportService.ts b/fluxer_api/src/api/admin/services/AdminReportService.tsindex e5c8e0c09..c0e5feed5 100644--- a/fluxer_api/src/api/admin/services/AdminReportService.ts+++ b/fluxer_api/src/api/admin/services/AdminReportService.ts@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport type {ApiContext} from '@app/api/ApiContext';+import {buildAdminDownloadUrl} from '@app/api/admin/AdminSignedDownloads';import type {AdminAuditService} from '@app/api/admin/services/AdminAuditService';import {describeReporterNotice,@@ -14,7 +15,6 @@ import {type ReportID,type UserID,} from '@app/api/BrandedTypes';-import {Config} from '@app/api/Config';import type {ChannelRepository} from '@app/api/channel/ChannelRepository';import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';import {@@ -49,6 +49,7 @@ import type {ReportSearchFilters} from '@fluxer/schema/src/contracts/search/Searimport type {ReportNoteListResponse, ReportNoteResponse} from '@fluxer/schema/src/domains/admin/AdminReportNoteSchemas';import type {AdminReportReasonsResponse,+ ReportAdminResponseSchema,ReportItemOutcome,SearchReportsRequest,UpdateReportRequest,@@ -57,6 +58,7 @@ import type {MessageResponse} from '@fluxer/schema/src/domains/message/MessageReimport type {ReportProfileAssetSnapshot} from '@fluxer/schema/src/domains/report/ReportProfileSnapshotSchemas';import type {UserPartialResponse} from '@fluxer/schema/src/domains/user/UserResponseSchemas';import {seconds} from 'itty-time';+import type {z} from 'zod';interface AdminReportServiceDeps {apiContext: ApiContext;@@ -395,16 +397,16 @@ export class AdminReportService {const reporterInfo = await this.buildUserTag(report.reporterId, requestCache);const reportedUserInfo = await this.buildUserTag(report.reportedUserId, requestCache);const canViewReporterPii = acls.has(AdminACLs.REPORT_VIEW_REPORTER_PII) || acls.has(AdminACLs.WILDCARD);- const reportedGuildNsfwLevel =- report.reportedGuildNsfw !== null- ? report.reportedGuildNsfw- ? 3- : 0- : await this.getGuildNsfwLevelForContext(- report.reportedChannelId,- report.reportedGuildId ?? report.guildContextId ?? null,- reportLookupCache,- );+ let reportedGuildNsfwLevel: number | null;+ if (report.reportedGuildNsfw !== null) {+ reportedGuildNsfwLevel = report.reportedGuildNsfw ? 3 : 0;+ } else {+ reportedGuildNsfwLevel = await this.getGuildNsfwLevelForContext(+ report.reportedChannelId,+ report.reportedGuildId ?? report.guildContextId ?? null,+ reportLookupCache,+ );+ }const reportedChannelNsfw =report.reportedChannelEffectiveNsfw !== null? report.reportedChannelEffectiveNsfw@@ -536,20 +538,24 @@ export class AdminReportService {}const asset = (value: ReportProfileAssetSnapshot | null) => this.mapProfileSnapshotAsset(report.reportId, value);const {user, member, guild} = snapshot;+ const capturedAt = snapshot.captured_at;+ let userResponse: NonNullable<z.infer<typeof ReportAdminResponseSchema>['reported_profile_snapshot']>['user'] =+ null;+ if (user) {+ userResponse = {+ id: user.id,+ username: user.username,+ discriminator: user.discriminator === null ? null : user.discriminator.toString().padStart(4, '0'),+ global_name: user.global_name,+ bio: user.bio,+ pronouns: user.pronouns,+ avatar: await asset(user.avatar),+ banner: await asset(user.banner),+ };+ }return {- captured_at: snapshot.captured_at,- user: user- ? {- id: user.id,- username: user.username,- discriminator: user.discriminator === null ? null : user.discriminator.toString().padStart(4, '0'),- global_name: user.global_name,- bio: user.bio,- pronouns: user.pronouns,- avatar: await asset(user.avatar),- banner: await asset(user.banner),- }- : null,+ captured_at: capturedAt,+ user: userResponse,member: member? {guild_id: member.guild_id,@@ -585,10 +591,11 @@ export class AdminReportService {return {hash: asset.hash, url: null};}try {- const url = await this.deps.storageService.getPresignedDownloadURL({- bucket: Config.s3.buckets.reports,+ const url = await buildAdminDownloadUrl(this.deps.storageService, {+ bucket: 'reports',key: asset.key,- expiresIn: seconds('5 minutes'),+ filename: asset.key.split('/').at(-1) ?? asset.hash,+ expiresInSeconds: seconds('5 minutes'),});return {hash: asset.hash, url};} catch (error) {@@ -779,10 +786,11 @@ export class AdminReportService {const filename = String(attachment.filename);const key = makeAttachmentCdnKey(channelId, attachmentId, filename);try {- const url = await storageService.getPresignedDownloadURL({- bucket: Config.s3.buckets.reports,+ const url = await buildAdminDownloadUrl(storageService, {+ bucket: 'reports',key,- expiresIn: seconds('5 minutes'),+ filename,+ expiresInSeconds: seconds('5 minutes'),});return {id: attachment.attachment_id.toString(),diff --git a/fluxer_api/src/api/admin/tests/AdminSignedDownloads.test.ts b/fluxer_api/src/api/admin/tests/AdminSignedDownloads.test.tsnew file mode 100644index 000000000..2ddf2bd3e--- /dev/null+++ b/fluxer_api/src/api/admin/tests/AdminSignedDownloads.test.ts@@ -0,0 +1,73 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {AdminArchiveRepository} from '@app/api/admin/repositories/AdminArchiveRepository';+import {createTestAccount, setUserACLs, type TestAccount} from '@app/api/auth/tests/AuthTestUtils';+import {getConfig} from '@app/api/Config';+import {createTestGuild} from '@app/api/emoji/tests/EmojiTestUtils';+import {type ApiTestHarness, createApiTestHarness} from '@app/api/test/ApiTestHarness';+import {HTTP_STATUS} from '@app/api/test/TestConstants';+import {createBuilder} from '@app/api/test/TestRequestBuilder';+import {AdminACLs} from '@fluxer/constants/src/AdminACLs';+import {afterAll, afterEach, beforeAll, beforeEach, describe, expect, it} from 'vitest';++const ZIP_BYTES = new TextEncoder().encode('PK pretend guild archive');++describe('admin archive downloads', () => {+ let harness: ApiTestHarness;+ let admin: TestAccount;+ const presigned = getConfig().presignedHarvestDownloadsEnabled;++ beforeAll(async () => {+ harness = await createApiTestHarness();+ });++ beforeEach(async () => {+ await harness.reset();+ admin = await setUserACLs(harness, await createTestAccount(harness), [AdminACLs.WILDCARD]);+ });++ afterEach(() => {+ getConfig().presignedHarvestDownloadsEnabled = presigned;+ });++ afterAll(async () => {+ await harness.shutdown();+ });++ async function completedGuildArchive(): Promise<string> {+ const owner = await createTestAccount(harness);+ const guild = await createTestGuild(harness, owner.token);+ const {archive_id: archiveId} = await createBuilder<{archive_id: string}>(harness, admin.token)+ .post(`/admin/guilds/${guild.id}/archives`)+ .body({})+ .execute();+ const repository = new AdminArchiveRepository();+ const queued = await repository.findBySubjectAndArchiveId('guild', BigInt(guild.id), BigInt(archiveId));+ const storageKey = `test/${archiveId}.zip`;+ await harness.storageService.uploadObject({+ bucket: getConfig().s3.buckets.harvests,+ key: storageKey,+ body: ZIP_BYTES,+ contentType: 'application/zip',+ });+ await repository.markAsCompleted(+ await repository.markAsStarted(queued!),+ storageKey,+ BigInt(ZIP_BYTES.length),+ new Date(Date.now() + 6 * 24 * 3600 * 1000),+ );+ return `/admin/archives/guild/${guild.id}/${archiveId}/download`;+ }++ async function fetchDownloadUrl(path: string): Promise<string> {+ return (await createBuilder<{downloadUrl: string}>(harness, admin.token).get(path).execute()).downloadUrl;+ }++ it('refuses a changed token', async () => {+ getConfig().presignedHarvestDownloadsEnabled = false;+ const url = await fetchDownloadUrl(await completedGuildArchive());+ const tampered = url.slice(url.indexOf('/signed-downloads/')).replace(/token=([^&]+)/, 'token=$1x');+ const response = await harness.app.request(tampered, {headers: {'x-forwarded-for': '203.0.113.9'}});+ expect(response.status).toBe(HTTP_STATUS.NOT_FOUND);+ });+});