A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
feat(media-proxy): sign attachment URLs and gate origins (#2830)
Hampus522cf08e6152
diff --git a/Cargo.lock b/Cargo.lockindex a4716d8d2..61e9cfd04 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1870,6 +1870,7 @@ name = "fluxer-messages"version = "0.1.0"dependencies = ["anyhow",+ "base64 0.23.1","chrono","criterion","fluxer-svc",@@ -2038,6 +2039,7 @@ dependencies = ["aws-sigv4","axum","base64 0.23.1",+ "hex","hmac 0.13.0","maxminddb","moka",diff --git a/config/env/development.env b/config/env/development.envindex a06df9451..c67d3b22e 100644--- a/config/env/development.env+++ b/config/env/development.env@@ -130,6 +130,7 @@ PUBLIC_RELEASE_CHANNEL=canaryPUBLIC_BOOTSTRAP_API_ENDPOINT=/apiPUBLIC_BOOTSTRAP_API_PUBLIC_ENDPOINT=http://localhost:8088/apiFLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64=Zmx1eGVyLWRldi11cGxvYWQtcmVsYXktc2VjcmV0LTAwMDA=+FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64=Zmx1eGVyLWRldi1hdHRhY2htZW50LXVybC1zZWNyZXQ=AWS_EC2_METADATA_DISABLED=trueAWS_ACCESS_KEY_ID=fluxerAWS_SECRET_ACCESS_KEY=fluxer-secretdiff --git a/deploy/self-hosting/.env.example b/deploy/self-hosting/.env.exampleindex 62a868726..a87b89529 100644--- a/deploy/self-hosting/.env.example+++ b/deploy/self-hosting/.env.example@@ -198,6 +198,36 @@ FLUXER_VAPID_PRIVATE_KEY=CHANGE_ME#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=https://chat.example.com#FLUXER_PASSKEY_ADDITIONAL_ALLOWED_ORIGINS=http://chat.example.com:19080+# Two optional media policies, both off unless you turn them on. Nothing below is+# needed for a working instance, and an upgrade never adds any of it.+#+# The first limits which web origins may read media through CORS. A request with+# no Origin header is always served, so direct links, image tags and native+# clients keep working. The allowlist defaults to the public origin above, which+# is where this instance serves its web app. The hosted web client and the+# desktop app run on https://web.fluxer.app, so add that origin, as in the second+# allowlist line, if people use them with this instance.+#+# The second makes an attachment read need a signed URL, which stops a copied+# link working forever elsewhere. Turning it on takes two settings: a secret, and+# the mode. Generate the secret with openssl rand -base64 32. It is a comma+# separated list, the first entry signs and every entry verifies. To rotate, add+# the new secret second and run docker compose up -d, then move it first and run+# again. Remove the old secret no sooner than a day after that, because an+# ordinary URL it signed stays valid for up to a day. A data package URL inside a+# harvest export never expires, so removing a secret ends every data package URL+# it signed and those exports have to be rebuilt.+#+# Each mode is off, report or enforce on its own. Set a mode to report first to+# log what enforce would refuse while refusing nothing. media-proxy reads these+# at container start, so apply a change with docker compose up -d media-proxy.+# docker compose restart media-proxy keeps the old environment.+#FLUXER_MEDIA_PROXY_CORS_MODE=enforce+#FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com+#FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS=https://chat.example.com,https://web.fluxer.app+#FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64=+#FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE=enforce+# Extra Content-Security-Policy sources, appended to the built-in ones. Set these# only when a browser must reach an origin the defaults do not cover, such as a# voice server hosted on a domain other than FLUXER_DOMAIN. Separate severaldiff --git a/deploy/self-hosting/docker-compose.yml b/deploy/self-hosting/docker-compose.ymlindex 4ae273908..b52046b31 100644--- a/deploy/self-hosting/docker-compose.yml+++ b/deploy/self-hosting/docker-compose.yml@@ -97,6 +97,7 @@ x-fluxer-env: &fluxer-envFLUXER_GATEWAY_RPC_AUTH_TOKEN: ${FLUXER_GATEWAY_RPC_AUTH_TOKEN:?set FLUXER_GATEWAY_RPC_AUTH_TOKEN in .env}FLUXER_MEDIA_PROXY_SECRET_KEY: ${FLUXER_MEDIA_PROXY_SECRET_KEY:?set FLUXER_MEDIA_PROXY_SECRET_KEY in .env}FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64: ${FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64:?set FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 in .env}+ FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64: ${FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64:-}FLUXER_ADMIN_SECRET_KEY_BASE: ${FLUXER_ADMIN_SECRET_KEY_BASE:?set FLUXER_ADMIN_SECRET_KEY_BASE in .env}FLUXER_ADMIN_OAUTH_CLIENT_SECRET: ${FLUXER_ADMIN_OAUTH_CLIENT_SECRET:?set FLUXER_ADMIN_OAUTH_CLIENT_SECRET in .env}@@ -473,6 +474,9 @@ services:FLUXER_MEDIA_PROXY_MODE: uploadFLUXER_MEDIA_PROXY_STORAGE_BACKEND: s3FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT: ${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}}/media+ FLUXER_MEDIA_PROXY_CORS_MODE: ${FLUXER_MEDIA_PROXY_CORS_MODE:-off}+ FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS: ${FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS:-${FLUXER_PUBLIC_ORIGIN:-${FLUXER_PUBLIC_SCHEME:-https}://${FLUXER_DOMAIN}:${FLUXER_PUBLIC_PORT:-443}}}+ FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE: ${FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE:-off}FLUXER_S3_READ_SIGNED: "true"depends_on:seaweedfs-init: {condition: service_completed_successfully}diff --git a/fluxer_api/pkgs/media_proxy_utils/package.json b/fluxer_api/pkgs/media_proxy_utils/package.jsonindex df03a0d56..022329509 100644--- a/fluxer_api/pkgs/media_proxy_utils/package.json+++ b/fluxer_api/pkgs/media_proxy_utils/package.json@@ -24,6 +24,10 @@"import": "./src/MediaProxySigner.ts","types": "./src/MediaProxySigner.ts"},+ "./src/AttachmentUrlSignature": {+ "import": "./src/AttachmentUrlSignature.ts",+ "types": "./src/AttachmentUrlSignature.ts"+ },"./*": "./*"},"main": "./src/MediaProxyUtils.ts",diff --git a/fluxer_api/pkgs/media_proxy_utils/src/AttachmentUrlSignature.ts b/fluxer_api/pkgs/media_proxy_utils/src/AttachmentUrlSignature.tsnew file mode 100644index 000000000..3c3c7e74d--- /dev/null+++ b/fluxer_api/pkgs/media_proxy_utils/src/AttachmentUrlSignature.ts@@ -0,0 +1,211 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import crypto from 'node:crypto';++export const ATTACHMENT_URL_TTL_SECS = 86_400;+export const ATTACHMENT_URL_BUCKET_SECS = 43_200;++export const ORDINARY_USAGE = '';+export const DATA_PACKAGE_USAGE = 'dp';++export type AttachmentUrlUsage = typeof ORDINARY_USAGE | typeof DATA_PACKAGE_USAGE;+export type SignatureParameterName = 'ex' | 'is' | 'hm' | 'uc';++const SIGNATURE_DOMAIN = 'fluxer-attachment-url-v1';+const ATTACHMENT_PATH_PREFIX = '/attachments/';+const SIGNATURE_PARAMETER_NAMES: ReadonlyArray<SignatureParameterName> = ['ex', 'is', 'hm', 'uc'];+const DATA_PACKAGE_EXPIRES = '0';+const WINDOW_HEX_LENGTH = 8;+const MAX_WINDOW_SECS = 0xff_ff_ff_ff;+const LEADING_SLASHES_REGEX = /^\/+/u;+const TRAILING_SLASHES_REGEX = /\/+$/u;++const textEncoder = new TextEncoder();+const strictTextDecoder = new TextDecoder('utf-8', {fatal: true, ignoreBOM: true});++export interface AttachmentUrlWindow {+ issued: number;+ expires: number;+}++export interface SignAttachmentUrlOptions {+ mediaEndpoint: string;+ secret: Uint8Array;+ nowSecs: number;+ anchorSecs: number;+}++function hexNibble(byte: number): number {+ if (byte >= 0x30 && byte <= 0x39) return byte - 0x30;+ if (byte >= 0x41 && byte <= 0x46) return byte - 0x41 + 10;+ if (byte >= 0x61 && byte <= 0x66) return byte - 0x61 + 10;+ return -1;+}++function percentDecodeBytes(value: string, plusAsSpace: boolean): Uint8Array {+ const bytes = textEncoder.encode(value);+ const decoded = new Uint8Array(bytes.length);+ let length = 0;+ let index = 0;+ while (index < bytes.length) {+ const byte = bytes[index] as number;+ if (byte === 0x25 && index + 2 < bytes.length) {+ const high = hexNibble(bytes[index + 1] as number);+ const low = hexNibble(bytes[index + 2] as number);+ if (high >= 0 && low >= 0) {+ decoded[length] = (high << 4) | low;+ length += 1;+ index += 3;+ continue;+ }+ }+ decoded[length] = plusAsSpace && byte === 0x2b ? 0x20 : byte;+ length += 1;+ index += 1;+ }+ return decoded.subarray(0, length);+}++export function percentDecodeStorageKey(path: string): string | null {+ try {+ return strictTextDecoder.decode(percentDecodeBytes(path.replace(LEADING_SLASHES_REGEX, ''), false));+ } catch {+ return null;+ }+}++export function signatureParameterName(name: string): SignatureParameterName | null {+ const decoded = percentDecodeBytes(name, true);+ if (decoded.length !== 2) return null;+ const candidate = String.fromCharCode(decoded[0] as number, decoded[1] as number);+ return SIGNATURE_PARAMETER_NAMES.find((entry) => entry === candidate) ?? null;+}++export function isSignatureParameterName(name: string): boolean {+ return signatureParameterName(name) !== null;+}++function isSafeStorageKey(key: string): boolean {+ if (key.length === 0 || key.startsWith('/')) return false;+ return key+ .split('/')+ .every((component) => component.length > 0 && component !== '.' && component !== '..' && !component.includes('\0'));+}++function firstIndexOf(value: string, characters: ReadonlyArray<string>): number {+ let found = -1;+ for (const character of characters) {+ const index = value.indexOf(character);+ if (index >= 0 && (found < 0 || index < found)) {+ found = index;+ }+ }+ return found;+}++function rawPathFromUrl(url: string): string | null {+ const schemeIndex = url.indexOf('://');+ if (schemeIndex < 0) return null;+ const afterAuthority = url.slice(schemeIndex + 3);+ const boundary = firstIndexOf(afterAuthority, ['/', '?', '#']);+ if (boundary < 0 || afterAuthority[boundary] !== '/') return '';+ const path = afterAuthority.slice(boundary);+ const queryIndex = firstIndexOf(path, ['?', '#']);+ return queryIndex < 0 ? path : path.slice(0, queryIndex);+}++function parseWebUrl(value: string): URL | null {+ try {+ const parsed = new URL(value);+ return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? parsed : null;+ } catch {+ return null;+ }+}++export function attachmentStorageKeyFromUrl(url: string, mediaEndpoint: string): string | null {+ const target = parseWebUrl(url);+ const endpoint = parseWebUrl(mediaEndpoint);+ if (!target || !endpoint || target.origin !== endpoint.origin) return null;+ const path = rawPathFromUrl(url);+ if (path === null) return null;+ const endpointPath = (rawPathFromUrl(mediaEndpoint) ?? '').replace(TRAILING_SLASHES_REGEX, '');+ if (!path.startsWith(`${endpointPath}${ATTACHMENT_PATH_PREFIX}`)) return null;+ const storageKey = percentDecodeStorageKey(path.slice(endpointPath.length));+ if (storageKey === null || !isSafeStorageKey(storageKey)) return null;+ return storageKey;+}++interface SplitUrl {+ base: string;+ query: string;+ fragment: string;+}++function splitUrl(url: string): SplitUrl {+ const fragmentIndex = url.indexOf('#');+ const head = fragmentIndex < 0 ? url : url.slice(0, fragmentIndex);+ const fragment = fragmentIndex < 0 ? '' : url.slice(fragmentIndex);+ const queryIndex = head.indexOf('?');+ if (queryIndex < 0) return {base: head, query: '', fragment};+ return {base: head.slice(0, queryIndex), query: head.slice(queryIndex + 1), fragment};+}++function preservedFields(query: string): Array<string> {+ if (query.length === 0) return [];+ return query.split('&').filter((field) => {+ if (field.length === 0 || field === '=') return false;+ const separator = field.indexOf('=');+ return !isSignatureParameterName(separator < 0 ? field : field.slice(0, separator));+ });+}++export function stripAttachmentSignature(url: string): string {+ const {base, query, fragment} = splitUrl(url);+ const preserved = preservedFields(query);+ if (preserved.length === 0) return `${base}${fragment}`;+ return `${base}?${preserved.join('&')}${fragment}`;+}++export function issueWindow(anchorSecs: number, nowSecs: number): AttachmentUrlWindow {+ const elapsed = Math.max(0, nowSecs - anchorSecs);+ const issued = anchorSecs + Math.floor(elapsed / ATTACHMENT_URL_BUCKET_SECS) * ATTACHMENT_URL_BUCKET_SECS;+ return {issued, expires: issued + ATTACHMENT_URL_TTL_SECS};+}++export function canonicalInput(storageKey: string, exHex: string, isHex: string, usage: AttachmentUrlUsage): string {+ return `${SIGNATURE_DOMAIN}\n${exHex}\n${isHex}\n${usage}\n${storageKey}`;+}++function windowHex(value: number): string {+ return value.toString(16).padStart(WINDOW_HEX_LENGTH, '0');+}++function signUsage(url: string, options: SignAttachmentUrlOptions, usage: AttachmentUrlUsage): string {+ const storageKey = attachmentStorageKeyFromUrl(url, options.mediaEndpoint);+ if (storageKey === null) return url;+ const {issued, expires} = issueWindow(options.anchorSecs, options.nowSecs);+ if (!Number.isSafeInteger(issued) || issued < 0 || expires > MAX_WINDOW_SECS) return url;+ const isDataPackage = usage === DATA_PACKAGE_USAGE;+ const exHex = windowHex(isDataPackage ? 0 : expires);+ const isHex = windowHex(issued);+ const signature = crypto+ .createHmac('sha256', options.secret)+ .update(canonicalInput(storageKey, exHex, isHex, usage))+ .digest('hex');+ const signatureFields = isDataPackage+ ? `ex=${DATA_PACKAGE_EXPIRES}&is=${isHex}&hm=${signature}&uc=${DATA_PACKAGE_USAGE}`+ : `ex=${exHex}&is=${isHex}&hm=${signature}`;+ const {base, query, fragment} = splitUrl(url);+ const preserved = preservedFields(query);+ const fields = preserved.length === 0 ? signatureFields : `${signatureFields}&${preserved.join('&')}`;+ return `${base}?${fields}${fragment}`;+}++export function signAttachmentUrl(url: string, options: SignAttachmentUrlOptions): string {+ return signUsage(url, options, ORDINARY_USAGE);+}++export function signDataPackageAttachmentUrl(url: string, options: SignAttachmentUrlOptions): string {+ return signUsage(url, options, DATA_PACKAGE_USAGE);+}diff --git a/fluxer_api/src/api/Config.ts b/fluxer_api/src/api/Config.tsindex d252f5aa7..2694cc1d2 100644--- a/fluxer_api/src/api/Config.ts+++ b/fluxer_api/src/api/Config.ts@@ -263,6 +263,9 @@ export function buildAPIConfigFromMaster(master: MasterConfig): APIConfig {tokenTtlSecs: uploadRelayConfig.token_ttl_secs,keepDirectCountries: uploadRelayConfig.keep_direct_countries,},+ attachmentUrls: {+ secretsBase64: master.services.media_proxy.attachment_urls.secrets_base64,+ },},geoip: geoipSourceConfig,proxy: {diff --git a/fluxer_api/src/api/app/ControllerRegistry.ts b/fluxer_api/src/api/app/ControllerRegistry.tsindex 868a841b3..51d466750 100644--- a/fluxer_api/src/api/app/ControllerRegistry.ts+++ b/fluxer_api/src/api/app/ControllerRegistry.ts@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport {registerAdminControllers} from '@app/api/admin/controllers/index';+import {AttachmentController} from '@app/api/attachment/AttachmentController';import {AuthController} from '@app/api/auth/AuthController';import {BlueskyOAuthController} from '@app/api/bluesky/BlueskyOAuthController';import {Config} from '@app/api/Config';@@ -45,6 +46,7 @@ export function registerControllers(routes: HonoApp, config: APIConfig): void {GeolocationController(routes);registerAdminControllers(routes);AuthController(routes);+ AttachmentController(routes);ChannelController(routes);ConnectionController(routes);BlueskyOAuthController(routes);diff --git a/fluxer_api/src/api/attachment/AttachmentController.ts b/fluxer_api/src/api/attachment/AttachmentController.tsnew file mode 100644index 000000000..b51374207--- /dev/null+++ b/fluxer_api/src/api/attachment/AttachmentController.ts@@ -0,0 +1,38 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {signAttachmentUrl} from '@app/api/attachment/AttachmentUrls';+import {LoginRequired} from '@app/api/middleware/AuthMiddleware';+import {RateLimitMiddleware} from '@app/api/middleware/RateLimitMiddleware';+import {OpenAPI} from '@app/api/middleware/ResponseTypeMiddleware';+import {RateLimitConfigs} from '@app/api/RateLimitConfig';+import type {HonoApp} from '@app/api/types/HonoEnv';+import {Validator} from '@app/api/Validator';+import {+ RefreshAttachmentUrlsRequest,+ RefreshAttachmentUrlsResponse,+} from '@fluxer/schema/src/domains/message/AttachmentSchemas';++export function AttachmentController(app: HonoApp) {+ app.post(+ '/attachments/refresh-urls',+ RateLimitMiddleware(RateLimitConfigs.ATTACHMENT_URLS_REFRESH),+ LoginRequired,+ Validator('json', RefreshAttachmentUrlsRequest),+ OpenAPI({+ operationId: 'refresh_attachment_urls',+ summary: 'Refresh attachment URLs',+ responseSchema: RefreshAttachmentUrlsResponse,+ statusCode: 200,+ security: ['botToken', 'sessionToken'],+ tags: ['Messages'],+ description:+ 'Reissues the expiring signature on attachment URLs. Returns one entry per requested URL, in the order they were requested, each pairing the URL exactly as it was sent with a freshly signed copy. A URL that is not an attachment URL of this instance is returned unchanged. No membership or existence check is performed.',+ }),+ async (ctx) => {+ const urls = ctx.req.valid('json').attachment_urls;+ return ctx.json({+ refreshed_urls: urls.map((original) => ({original, refreshed: signAttachmentUrl(original)})),+ });+ },+ );+}diff --git a/fluxer_api/src/api/attachment/AttachmentUrls.ts b/fluxer_api/src/api/attachment/AttachmentUrls.tsnew file mode 100644index 000000000..ae0c5ec65--- /dev/null+++ b/fluxer_api/src/api/attachment/AttachmentUrls.ts@@ -0,0 +1,76 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import type {AttachmentID, ChannelID} from '@app/api/BrandedTypes';+import {Config} from '@app/api/Config';+import {makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers';+import {extractTimestampBigInt} from '@fluxer/snowflake/src/SnowflakeUtils';+import {+ attachmentStorageKeyFromUrl,+ type SignAttachmentUrlOptions,+ signDataPackageAttachmentUrl as signDataPackageWithSecret,+ signAttachmentUrl as signWithSecret,+ stripAttachmentSignature as stripSignature,+} from '@pkgs/media_proxy_utils/src/AttachmentUrlSignature';++const SNOWFLAKE_SEGMENT_REGEX = /^[0-9]{1,20}$/u;+const STORAGE_KEY_MIN_SEGMENTS = 4;++function signingSecret(): Buffer | null {+ const configured = Config.mediaProxy.attachmentUrls.secretsBase64[0];+ if (!configured) return null;+ const secret = Buffer.from(configured, 'base64');+ return secret.length === 0 ? null : secret;+}++function anchorSecsFromStorageKey(storageKey: string): number | null {+ const segments = storageKey.split('/');+ if (segments.length < STORAGE_KEY_MIN_SEGMENTS || segments[0] !== 'attachments') return null;+ const attachmentId = segments[2] as string;+ if (!SNOWFLAKE_SEGMENT_REGEX.test(segments[1] as string) || !SNOWFLAKE_SEGMENT_REGEX.test(attachmentId)) return null;+ return Math.floor(extractTimestampBigInt(BigInt(attachmentId)) / 1000);+}++function signingOptions(url: string, nowSecs?: number): SignAttachmentUrlOptions | null {+ const secret = signingSecret();+ if (secret === null) return null;+ const mediaEndpoint = Config.endpoints.media;+ const storageKey = attachmentStorageKeyFromUrl(url, mediaEndpoint);+ if (storageKey === null) return null;+ const anchorSecs = anchorSecsFromStorageKey(storageKey);+ if (anchorSecs === null) return null;+ return {mediaEndpoint, secret, nowSecs: nowSecs ?? Math.floor(Date.now() / 1000), anchorSecs};+}++export function signAttachmentUrl(url: string, nowSecs?: number): string {+ const options = signingOptions(url, nowSecs);+ return options === null ? url : signWithSecret(url, options);+}++export function signDataPackageAttachmentUrl(url: string, nowSecs?: number): string {+ const options = signingOptions(url, nowSecs);+ return options === null ? url : signDataPackageWithSecret(url, options);+}++export function stripAttachmentSignature(url: string): string {+ return stripSignature(url);+}++export function stripOwnAttachmentSignature(url: string): string {+ return attachmentStorageKeyFromUrl(url, Config.endpoints.media) === null ? url : stripSignature(url);+}++export function makeSignedAttachmentCdnUrl(+ channelId: ChannelID,+ attachmentId: AttachmentID | bigint,+ filename: string,+): string {+ return signAttachmentUrl(makeAttachmentCdnUrl(channelId, attachmentId, filename));+}++export function makeDataPackageAttachmentCdnUrl(+ channelId: ChannelID,+ attachmentId: AttachmentID | bigint,+ filename: string,+): string {+ return signDataPackageAttachmentUrl(makeAttachmentCdnUrl(channelId, attachmentId, filename));+}diff --git a/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts b/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.tsindex 420664544..88b5bcfbe 100644--- a/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts+++ b/fluxer_api/src/api/channel/services/message/AttachmentProcessingService.ts@@ -1,7 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport fs from 'node:fs';-import {createAttachmentID, type UserID} from '@app/api/BrandedTypes';+import {createAttachmentID, createGuildID, createUserID, type UserID} from '@app/api/BrandedTypes';import {Config} from '@app/api/Config';import type {AttachmentToProcess} from '@app/api/channel/AttachmentDTOs';import type {AttachmentUploadTraceRepository} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository';@@ -11,6 +11,7 @@ import {makeAttachmentCdnKey,validateAttachmentIds,} from '@app/api/channel/services/message/MessageHelpers';+import {scheduleUploadSegmentSignal} from '@app/api/channel/services/message/UploadSegmentSignal';import type {MessageAttachment} from '@app/api/database/types/MessageTypes';import {contentModerationService, type ModerationContext} from '@app/api/infrastructure/ContentModerationService';import type {@@ -65,6 +66,7 @@ interface ProcessedAttachment {hasVirusDetected: boolean;applyFinalObjectMetadata: boolean;sourceLocalPath: string | null;+ sniffedContentType: string | null;}export class AttachmentProcessingService {@@ -172,6 +174,24 @@ export class AttachmentProcessingService {}return result.attachment;});+ scheduleUploadSegmentSignal({+ userId: params.uploadUserId,+ guildId: params.guild ? createGuildID(BigInt(params.guild.id)) : null,+ guildOwnerId: params.guild ? createUserID(BigInt(params.guild.owner_id)) : null,+ channelId: params.message.channelId,+ messageId: params.message.id,+ attachments: processedAttachments.map((attachment, index) => ({+ attachmentId: attachment.attachment_id,+ uploadKey: results[index].copyOperation.sourceKey,+ filename: attachment.filename,+ contentType: attachment.content_type,+ size: attachment.size,+ duration: attachment.duration ?? null,+ waveform: attachment.waveform ?? null,+ sniffedContentType: results[index].sniffedContentType,+ requestIp: bindingResults[index].bound?.request_ip ?? null,+ })),+ });return {attachments: processedAttachments, hasVirusDetected: false};}@@ -213,7 +233,31 @@ export class AttachmentProcessingService {let applyFinalObjectMetadata = false;const clientDuration: number | null = attachment.duration ?? null;const waveform: string | null = attachment.waveform ?? null;- const isMedia = isMediaFile(contentType);+ const sniffedContentType = isMediaFile(contentType)+ ? null+ : await this.sniffAttachmentMediaType({+ index,+ uploadFilename: attachment.upload_filename,+ filename: attachment.filename,+ });+ if (sniffedContentType !== null) {+ Logger.warn(+ {+ surface: 'message_attachment',+ userId: params.uploadUserId.toString(),+ guildId: params.guild?.id ?? null,+ channelId: message.channelId.toString(),+ messageId: message.id.toString(),+ attachmentId: attachmentId.toString(),+ uploadKey: attachment.upload_filename,+ filename: attachment.filename,+ filenameContentType: contentType,+ sniffedContentType,+ },+ 'content_moderation.attachment_type_mismatch',+ );+ }+ const isMedia = isMediaFile(contentType) || sniffedContentType !== null;let metadata: MediaProxyMetadataResponse | null = null;if (isMedia) {metadata = await this.getAttachmentMediaMetadata({@@ -303,6 +347,7 @@ export class AttachmentProcessingService {hasVirusDetected,applyFinalObjectMetadata,sourceLocalPath: null,+ sniffedContentType,};}const isAudio = contentType.startsWith('audio/');@@ -341,6 +386,7 @@ export class AttachmentProcessingService {hasVirusDetected,applyFinalObjectMetadata,sourceLocalPath: retainedLocalPath,+ sniffedContentType,};} catch (error) {if (sourceLocalPath) {@@ -350,6 +396,27 @@ export class AttachmentProcessingService {}}+ private async sniffAttachmentMediaType(params: {+ index: number;+ uploadFilename: string;+ filename: string;+ }): Promise<string | null> {+ const sniff = await this.mediaService.sniffUpload(params.uploadFilename);+ if (sniff) {+ return sniff.content_type;+ }+ Logger.warn(+ {+ context: METADATA_PROBE_DEGRADED_CONTEXT,+ attachmentIndex: params.index,+ uploadFilename: params.uploadFilename,+ filename: params.filename,+ },+ 'Attachment content sniff unavailable, storing attachment with its filename type',+ );+ return null;+ }+private async getAttachmentMediaMetadata(params: {index: number;uploadFilename: string;diff --git a/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts b/fluxer_api/src/api/channel/services/message/MessageResponseDataService.tsindex e0fd644d9..21bad3d5e 100644--- a/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts+++ b/fluxer_api/src/api/channel/services/message/MessageResponseDataService.ts@@ -107,6 +107,7 @@ export class MessageResponseDataService {can_read_message_history: params.access.canReadMessageHistory,media_endpoint: Config.endpoints.media,media_proxy_secret_key: Config.mediaProxy.secretKey,+ attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0],include_reactions: true,});if (typeof response === 'object' && 'FoundApiMany' in response) {@@ -147,6 +148,7 @@ export class MessageResponseDataService {can_read_message_history: params.access.canReadMessageHistory,media_endpoint: Config.endpoints.media,media_proxy_secret_key: Config.mediaProxy.secretKey,+ attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0],include_reactions: true,nonce: params.nonce,tts: params.tts,@@ -177,6 +179,7 @@ export class MessageResponseDataService {can_read_message_history: params.access.canReadMessageHistory,media_endpoint: Config.endpoints.media,media_proxy_secret_key: Config.mediaProxy.secretKey,+ attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0],include_reactions: params.includeReactions ?? true,nonce: params.nonce,tts: params.tts,@@ -244,6 +247,7 @@ export class MessageResponseDataService {can_read_message_history: params.access.canReadMessageHistory,media_endpoint: Config.endpoints.media,media_proxy_secret_key: Config.mediaProxy.secretKey,+ attachment_url_secret_base64: Config.mediaProxy.attachmentUrls.secretsBase64[0],include_reactions: params.includeReactions ?? true,});if (typeof response !== 'object' || !('FoundApiMany' in response)) {diff --git a/fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts b/fluxer_api/src/api/channel/services/message/UploadSegmentSignal.tsnew file mode 100644index 000000000..b5b3b0bb4--- /dev/null+++ b/fluxer_api/src/api/channel/services/message/UploadSegmentSignal.ts@@ -0,0 +1,209 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import type {AttachmentID, ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes';+import {Logger} from '@app/api/Logger';+import {getKVClient} from '@app/api/middleware/ServiceRegistry';+import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';++const WINDOW_MS = 600000;+const USER_THRESHOLD = 60;+const GUILD_THRESHOLD = 120;+const FRESH_GUILD_THRESHOLD = 20;+const FRESH_GUILD_MAX_AGE_MS = 604800000;+const SEGMENT_MAX_BYTES = 16 * 1024 * 1024;+const SEGMENT_MAX_DURATION_SECONDS = 30;++const KEY_PREFIX = 'abuse:upload_segment:';+const WINDOW_TTL_SECONDS = WINDOW_MS / 1000;+const MAX_IN_FLIGHT_SIGNALS = 32;+const SURFACE = 'message_attachment';+const SIGNAL_MESSAGE = 'content_moderation.upload_segment_pattern';+const FAILURE_MESSAGE = 'content_moderation.upload_segment_signal_failed';+const DROPPED_MESSAGE = 'content_moderation.upload_segment_signal_dropped';+const PLAYLIST_CONTENT_TYPES = new Set(['application/vnd.apple.mpegurl', 'application/x-mpegurl']);++export type UploadSegmentScope = 'user' | 'guild' | 'fresh_guild';++export interface UploadSegmentAttachment {+ attachmentId: AttachmentID;+ uploadKey: string;+ filename: string;+ contentType: string;+ size: bigint;+ duration: number | null;+ waveform: string | null;+ sniffedContentType: string | null;+ requestIp: string | null;+}++export interface UploadSegmentSignalInput {+ userId: UserID;+ guildId: GuildID | null;+ guildOwnerId: UserID | null;+ channelId: ChannelID;+ messageId: MessageID;+ attachments: ReadonlyArray<UploadSegmentAttachment>;+}++interface CountedScope {+ scope: UploadSegmentScope;+ key: string;+ threshold: number;+}++function baseContentType(contentType: string): string {+ const [base] = contentType.split(';');+ return (base ?? '').trim().toLowerCase();+}++export function isSegmentShapedAttachment(attachment: UploadSegmentAttachment): boolean {+ if (attachment.waveform !== null) {+ return false;+ }+ if (attachment.sniffedContentType !== null) {+ return true;+ }+ const contentType = baseContentType(attachment.contentType);+ const isSegmentType =+ contentType.startsWith('video/') || contentType.startsWith('audio/') || PLAYLIST_CONTENT_TYPES.has(contentType);+ if (!isSegmentType) {+ return false;+ }+ if (attachment.size > BigInt(SEGMENT_MAX_BYTES)) {+ return false;+ }+ return attachment.duration === null || attachment.duration <= SEGMENT_MAX_DURATION_SECONDS;+}++export function isRungValue(count: number, threshold: number): boolean {+ if (count < threshold || count % threshold !== 0) {+ return false;+ }+ const multiple = count / threshold;+ return (multiple & (multiple - 1)) === 0;+}++function uploaderOwnsGuild(input: UploadSegmentSignalInput): boolean {+ return input.guildOwnerId !== null && input.guildOwnerId === input.userId;+}++function resolveScopes(input: UploadSegmentSignalInput, windowIndex: number, nowMs: number): Array<CountedScope> {+ const scopes: Array<CountedScope> = [+ {+ scope: 'user',+ key: `${KEY_PREFIX}user:${input.userId.toString()}:${windowIndex}`,+ threshold: USER_THRESHOLD,+ },+ ];+ if (input.guildId === null) {+ return scopes;+ }+ const guildAgeMs = nowMs - snowflakeToDate(input.guildId).getTime();+ const isFreshGuild = uploaderOwnsGuild(input) && guildAgeMs < FRESH_GUILD_MAX_AGE_MS;+ scopes.push({+ scope: isFreshGuild ? 'fresh_guild' : 'guild',+ key: `${KEY_PREFIX}guild:${input.guildId.toString()}:${windowIndex}`,+ threshold: isFreshGuild ? FRESH_GUILD_THRESHOLD : GUILD_THRESHOLD,+ });+ return scopes;+}++function buildSignalFields(params: {+ input: UploadSegmentSignalInput;+ segments: ReadonlyArray<UploadSegmentAttachment>;+ windowIndex: number;+ scope: CountedScope;+ count: number;+}): Record<string, unknown> {+ const {input, segments, windowIndex, scope, count} = params;+ const requestIps = new Set<string>();+ for (const segment of segments) {+ if (segment.requestIp !== null) {+ requestIps.add(segment.requestIp);+ }+ }+ return {+ surface: SURFACE,+ scope: scope.scope,+ count,+ threshold: scope.threshold,+ windowMs: WINDOW_MS,+ windowStartedAt: new Date(windowIndex * WINDOW_MS).toISOString(),+ userId: input.userId.toString(),+ userCreatedAt: snowflakeToDate(input.userId).toISOString(),+ guildId: input.guildId === null ? null : input.guildId.toString(),+ guildCreatedAt: input.guildId === null ? null : snowflakeToDate(input.guildId).toISOString(),+ guildOwnerId: input.guildOwnerId === null ? null : input.guildOwnerId.toString(),+ uploaderOwnsGuild: uploaderOwnsGuild(input),+ channelId: input.channelId.toString(),+ messageId: input.messageId.toString(),+ attachmentIds: segments.map((segment) => segment.attachmentId.toString()),+ uploadKeys: segments.map((segment) => segment.uploadKey),+ requestIps: Array.from(requestIps),+ filenames: segments.map((segment) => segment.filename),+ contentTypes: segments.map((segment) => segment.contentType),+ disguisedCount: segments.filter((segment) => segment.sniffedContentType !== null).length,+ };+}++function scopeFields(input: UploadSegmentSignalInput): Record<string, unknown> {+ return {+ surface: SURFACE,+ userId: input.userId.toString(),+ guildId: input.guildId === null ? null : input.guildId.toString(),+ channelId: input.channelId.toString(),+ messageId: input.messageId.toString(),+ };+}++const inFlightSignals = new Set<Promise<void>>();++export function scheduleUploadSegmentSignal(input: UploadSegmentSignalInput): void {+ if (!input.attachments.some(isSegmentShapedAttachment)) {+ return;+ }+ if (inFlightSignals.size >= MAX_IN_FLIGHT_SIGNALS) {+ Logger.warn(scopeFields(input), DROPPED_MESSAGE);+ return;+ }+ const pending = recordUploadSegmentSignal(input).finally(() => {+ inFlightSignals.delete(pending);+ });+ inFlightSignals.add(pending);+}++export async function flushUploadSegmentSignals(): Promise<void> {+ while (inFlightSignals.size > 0) {+ await Promise.all([...inFlightSignals]);+ }+}++export async function recordUploadSegmentSignal(input: UploadSegmentSignalInput): Promise<void> {+ try {+ const segments = input.attachments.filter(isSegmentShapedAttachment);+ if (segments.length === 0) {+ return;+ }+ const nowMs = Date.now();+ const windowIndex = Math.floor(nowMs / WINDOW_MS);+ const scopes = resolveScopes(input, windowIndex, nowMs);+ const kv = getKVClient();+ const counts: Array<number> = [];+ for (const scope of scopes) {+ const count = await kv.incr(scope.key);+ counts.push(count);+ if (count === 1) {+ await kv.expire(scope.key, WINDOW_TTL_SECONDS);+ }+ }+ for (const [index, scope] of scopes.entries()) {+ const count = counts[index];+ if (!isRungValue(count, scope.threshold)) {+ continue;+ }+ Logger.warn(buildSignalFields({input, segments, windowIndex, scope, count}), SIGNAL_MESSAGE);+ }+ } catch (error) {+ Logger.warn({error, ...scopeFields(input)}, FAILURE_MESSAGE);+ }+}diff --git a/fluxer_api/src/api/config/APIConfig.ts b/fluxer_api/src/api/config/APIConfig.tsindex 0f4019278..26b68a04e 100644--- a/fluxer_api/src/api/config/APIConfig.ts+++ b/fluxer_api/src/api/config/APIConfig.ts@@ -118,6 +118,9 @@ export interface APIConfig {tokenTtlSecs: number;keepDirectCountries: Array<string>;};+ attachmentUrls: {+ secretsBase64: Array<string>;+ };};geoip: APIGeoipConfig;proxy: {diff --git a/fluxer_api/src/api/favorite_gif/FavoriteGifResolver.ts b/fluxer_api/src/api/favorite_gif/FavoriteGifResolver.tsindex e74c06611..d08aa3233 100644--- a/fluxer_api/src/api/favorite_gif/FavoriteGifResolver.ts+++ b/fluxer_api/src/api/favorite_gif/FavoriteGifResolver.ts@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {signAttachmentUrl} from '@app/api/attachment/AttachmentUrls';import {tryExtractGifProviderSlug} from '@app/api/gif/GifProviderUtils';import type {GifService} from '@app/api/gif/GifService';import type {IGifProvider} from '@app/api/gif/IGifProvider';@@ -108,10 +109,11 @@ function favoriteGifEntryFromExternalMedia({mediaService: IMediaService;metadata: MediaProxyMetadataResponse | null;}): ResolvedGifEntrySchema {- const proxyUrl = mediaService.getExternalMediaProxyURL(url);- const media = directMediaFormatFromMetadata({url, proxyUrl, metadata});+ const signedUrl = signAttachmentUrl(url);+ const proxyUrl = signAttachmentUrl(mediaService.getExternalMediaProxyURL(url));+ const media = directMediaFormatFromMetadata({url: signedUrl, proxyUrl, metadata});return {- url,+ url: signedUrl,proxy_url: proxyUrl,width: metadata?.width ?? 0,height: metadata?.height ?? 0,@@ -153,16 +155,16 @@ function favoriteGifEntryFromEmbedMedia({mediaService: IMediaService;media: EmbedMediaResponse;}): ResolvedGifEntrySchema {- const proxyUrl = media.proxy_url ?? mediaService.getExternalMediaProxyURL(media.url);+ const proxyUrl = signAttachmentUrl(media.proxy_url ?? mediaService.getExternalMediaProxyURL(media.url));const width = media.width ?? 0;const height = media.height ?? 0;const contentType = media.content_type ?? '';return {- url,+ url: signAttachmentUrl(url),proxy_url: proxyUrl,width,height,- media: directMediaFormatFromDetails({url: media.url, proxyUrl, contentType, width, height}),+ media: directMediaFormatFromDetails({url: signAttachmentUrl(media.url), proxyUrl, contentType, width, height}),content_type: contentType,placeholder: media.placeholder ?? null,};diff --git a/fluxer_api/src/api/favorite_meme/FavoriteMemeModel.ts b/fluxer_api/src/api/favorite_meme/FavoriteMemeModel.tsindex 306efdf4f..237c9485f 100644--- a/fluxer_api/src/api/favorite_meme/FavoriteMemeModel.ts+++ b/fluxer_api/src/api/favorite_meme/FavoriteMemeModel.ts@@ -1,13 +1,13 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {makeSignedAttachmentCdnUrl} from '@app/api/attachment/AttachmentUrls';import {userIdToChannelId} from '@app/api/BrandedTypes';-import {makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers';import type {FavoriteMeme} from '@app/api/models/FavoriteMeme';import {assertSafeByteSize} from '@app/api/utils/ByteSizeUtils';import type {FavoriteMemeResponse} from '@fluxer/schema/src/domains/meme/MemeSchemas';export function mapFavoriteMemeToResponse(meme: FavoriteMeme): FavoriteMemeResponse {- const url = makeAttachmentCdnUrl(userIdToChannelId(meme.userId), meme.attachmentId, meme.filename);+ const url = makeSignedAttachmentCdnUrl(userIdToChannelId(meme.userId), meme.attachmentId, meme.filename);return {id: meme.id.toString(),user_id: meme.userId.toString(),diff --git a/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts b/fluxer_api/src/api/favorite_meme/FavoriteMemeService.tsindex ce646688a..0c989f454 100644--- a/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts+++ b/fluxer_api/src/api/favorite_meme/FavoriteMemeService.ts@@ -1,11 +1,12 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport type {ApiContext} from '@app/api/ApiContext';+import {makeSignedAttachmentCdnUrl} from '@app/api/attachment/AttachmentUrls';import type {ChannelID, MemeID, MessageID, UserID} from '@app/api/BrandedTypes';import {createAttachmentID, createMemeID, userIdToChannelId} from '@app/api/BrandedTypes';import {Config} from '@app/api/Config';import type {ChannelService} from '@app/api/channel/services/ChannelService';-import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers';+import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';import {mapFavoriteMemeToResponse} from '@app/api/favorite_meme/FavoriteMemeModel';import type {IFavoriteMemeRepository} from '@app/api/favorite_meme/IFavoriteMemeRepository';import {@@ -35,6 +36,7 @@ import {MediaMetadataError} from '@fluxer/errors/src/domains/core/MediaMetadataEimport {UnknownFavoriteMemeError} from '@fluxer/errors/src/domains/core/UnknownFavoriteMemeError';import type {GifMediaFormat} from '@fluxer/schema/src/domains/gif/GifSchemas';import {normalizeFilename} from '@fluxer/schema/src/primitives/FileValidators';+import {attachmentStorageKeyFromUrl} from '@pkgs/media_proxy_utils/src/AttachmentUrlSignature';import mime from 'mime';type MessageAttachmentCandidate = Message['attachments'][number];@@ -78,6 +80,11 @@ function isAnimatedEmbedMedia(contentType: string | null | undefined, flags: numreturn ((flags ?? 0) & EmbedMediaFlags.IS_ANIMATED) !== 0;}+function isAttachmentKeyInChannel(storageKey: string, channelId: ChannelID): boolean {+ const segments = storageKey.split('/');+ return segments.length === 4 && segments[0] === 'attachments' && segments[1] === channelId.toString();+}+function resolveFavoriteMemeAnimationFlag(media: Pick<FavoriteMemeMedia, 'isGifv'>,metadata: Pick<MediaProxyMetadataResponse, 'animated'> | null | undefined,@@ -315,7 +322,7 @@ export class FavoriteMemeService {this.ensureFavoriteMemeTagLimit(user, urlTags);const metadata = await this.apiContext.services.media.getMetadata({type: 'external',- url,+ url: url,with_base64: true,nsfw: 'allow',});@@ -544,7 +551,7 @@ export class FavoriteMemeService {embedCount: embeds.length,});}- return this.mediaFromEmbed(embeds[preferredEmbedIndex], `embed_${preferredEmbedIndex}`);+ return this.mediaFromEmbed(embeds[preferredEmbedIndex], `embed_${preferredEmbedIndex}`, message.channelId);}if (attachments.length > 0) {let attachment: MessageAttachmentCandidate | undefined;@@ -566,7 +573,7 @@ export class FavoriteMemeService {}}for (const embed of embeds) {- const media = await this.mediaFromEmbed(embed, 'media');+ const media = await this.mediaFromEmbed(embed, 'media', message.channelId);if (media) return media;}return null;@@ -587,7 +594,7 @@ export class FavoriteMemeService {const isGifv = isAnimatedAttachment(attachment.contentType, attachment.flags);return {isExternal: false,- url: makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename),+ url: makeSignedAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename),sourceKey: makeAttachmentCdnKey(message.channelId, attachment.id, attachment.filename),filename: attachment.filename,contentType: attachment.contentType,@@ -607,6 +614,7 @@ export class FavoriteMemeService {private async mediaFromEmbed(embed: MessageEmbedCandidate,fallbackFilename: string,+ channelId: ChannelID,): Promise<FavoriteMemeMedia | null> {const media = embed.image || embed.video || embed.thumbnail;if (!media?.url) {@@ -617,13 +625,14 @@ export class FavoriteMemeService {if (!this.isValidMediaType(contentType)) {return null;}- const isExternal = !this.isInternalCDNUrl(media.url);+ const candidateKey = attachmentStorageKeyFromUrl(media.url, Config.endpoints.media);+ const sourceKey = candidateKey !== null && isAttachmentKeyInChannel(candidateKey, channelId) ? candidateKey : null;const isGifv = embed.type === 'gifv' || isAnimatedEmbedMedia(media.contentType, media.flags);const detectedGif = embed.type === 'gifv' ? await this.detectGifFromUrl(media.url) : null;return {- isExternal,+ isExternal: sourceKey === null,url: media.url,- sourceKey: isExternal ? '' : this.extractStorageKeyFromUrl(media.url) || '',+ sourceKey: sourceKey ?? '',filename,contentType,size: BigInt(0),@@ -639,10 +648,6 @@ export class FavoriteMemeService {};}- private isInternalCDNUrl(url: string): boolean {- return url.startsWith(`${Config.endpoints.media}/`);- }-private isValidMediaType(contentType: string): boolean {return contentType.startsWith('image/') || contentType.startsWith('video/') || contentType.startsWith('audio/');}@@ -661,15 +666,6 @@ export class FavoriteMemeService {}}- private extractStorageKeyFromUrl(url: string): string | null {- try {- const urlObj = new URL(url);- return urlObj.pathname.substring(1);- } catch {- return null;- }- }-private ensureFavoriteMemeTagLimit(user: User, tags?: Array<string>): void {const limit = this.resolveUserLimit(user, 'max_favorite_meme_tags', MAX_FAVORITE_MEME_TAGS);if ((tags?.length ?? 0) > limit) {diff --git a/fluxer_api/src/api/infrastructure/EmbedService.ts b/fluxer_api/src/api/infrastructure/EmbedService.tsindex 5c26c3dae..f02f9f205 100644--- a/fluxer_api/src/api/infrastructure/EmbedService.ts+++ b/fluxer_api/src/api/infrastructure/EmbedService.ts@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {stripOwnAttachmentSignature} from '@app/api/attachment/AttachmentUrls';import type {ChannelID, MessageID} from '@app/api/BrandedTypes';import type {RichEmbedMediaWithMetadata} from '@app/api/channel/EmbedTypes';import type {IChannelRepository} from '@app/api/channel/IChannelRepository';@@ -53,6 +54,15 @@ interface InitialUrlEmbedResult {hasUncachedUrls: boolean;}+type RichEmbedRequestWithMedia = RichEmbedRequest & {+ image?: RichEmbedMediaWithMetadata | null;+ thumbnail?: RichEmbedMediaWithMetadata | null;+};++function canonicalUrl(url: string | null | undefined): string | null {+ return url == null ? null : stripOwnAttachmentSignature(url);+}+export class EmbedService {private readonly MAX_EMBED_CHARACTERS = 6000;private readonly MAX_EMBED_CHARACTERS_BUG_HUNTER = 12000;@@ -228,20 +238,20 @@ export class EmbedService {type: embed.type ?? null,title: embed.title ?? null,description: embed.description ?? null,- url: embed.url ?? null,+ url: canonicalUrl(embed.url),timestamp: embed.timestamp ? new Date(embed.timestamp) : null,color: embed.color ?? null,author: embed.author? {name: embed.author.name ?? null,- url: embed.author.url ?? null,- icon_url: embed.author.icon_url ?? null,+ url: canonicalUrl(embed.author.url),+ icon_url: canonicalUrl(embed.author.icon_url),}: null,provider: embed.provider? {name: embed.provider.name ?? null,- url: embed.provider.url ?? null,+ url: canonicalUrl(embed.provider.url),}: null,thumbnail: this.mapResponseMedia(embed.thumbnail),@@ -251,7 +261,7 @@ export class EmbedService {footer: embed.footer? {text: embed.footer.text ?? null,- icon_url: embed.footer.icon_url ?? null,+ icon_url: canonicalUrl(embed.footer.icon_url),}: null,fields:@@ -272,7 +282,7 @@ export class EmbedService {private mapResponseMedia(media?: MessageEmbedResponse['image']): MessageEmbed['image'] {if (!media) return null;return {- url: media.url,+ url: stripOwnAttachmentSignature(media.url),content_type: media.content_type ?? null,content_hash: media.content_hash ?? null,width: media.width ?? null,@@ -303,13 +313,7 @@ export class EmbedService {}}- private async createEmbed(- embed: RichEmbedRequest & {- image?: RichEmbedMediaWithMetadata | null;- thumbnail?: RichEmbedMediaWithMetadata | null;- },- nsfwMode: MediaProxyNsfwMode,- ): Promise<Embed> {+ private async createEmbed(embed: RichEmbedRequestWithMedia, nsfwMode: MediaProxyNsfwMode): Promise<Embed> {const [author, footer, imageResult, thumbnailResult] = await Promise.all([this.processAuthor(embed.author ?? undefined, nsfwMode),this.processFooter(embed.footer ?? undefined, nsfwMode),@@ -326,7 +330,7 @@ export class EmbedService {type: 'rich',title: embed.title ?? null,description: embed.description ?? null,- url: embed.url ?? null,+ url: canonicalUrl(embed.url),timestamp: embed.timestamp ?? null,color: embed.color ?? 0,footer: footer?.toMessageEmbedFooter() ?? null,@@ -363,7 +367,7 @@ export class EmbedService {if (attachmentMetadata) {return {media: new EmbedMedia({- url: request.url,+ url: stripOwnAttachmentSignature(request.url),width: attachmentMetadata.width,height: attachmentMetadata.height,description: request.description ?? null,@@ -376,7 +380,7 @@ export class EmbedService {nsfw: attachmentMetadata.nsfw ?? false,};}- const {url, metadata} = await this.resolveExternalMedia(request.url, nsfwMode);+ const {url, metadata} = await this.resolveExternalMedia(stripOwnAttachmentSignature(request.url), nsfwMode);if (!metadata) {return {media: new EmbedMedia({@@ -418,13 +422,14 @@ export class EmbedService {url: string;metadata: MediaProxyMetadataResponse | null;}> {+ const stored = stripOwnAttachmentSignature(url);const directMetadata = await this.mediaService.getMetadata({type: 'external',url,...mediaProxyMetadataPolicy(nsfwMode),});if (this.isRenderableMediaType(directMetadata?.content_type)) {- return {url, metadata: directMetadata};+ return {url: stored, metadata: directMetadata};}const unfurled = await this.unfurlerService.unfurl(url, nsfwMode);for (const embed of unfurled) {@@ -437,11 +442,11 @@ export class EmbedService {...mediaProxyMetadataPolicy(nsfwMode),});if (this.isRenderableMediaType(candidateMetadata?.content_type)) {- return {url: candidate, metadata: candidateMetadata};+ return {url: stripOwnAttachmentSignature(candidate), metadata: candidateMetadata};}}}- return {url, metadata: directMetadata};+ return {url: stored, metadata: directMetadata};}private isRenderableMediaType(contentType: string | null | undefined): boolean {@@ -461,11 +466,11 @@ export class EmbedService {url: author.icon_url,...mediaProxyMetadataPolicy(nsfwMode),});- if (metadata) iconUrl = author.icon_url;+ if (metadata) iconUrl = stripOwnAttachmentSignature(author.icon_url);}return new EmbedAuthor({name: author.name,- url: author.url ?? null,+ url: canonicalUrl(author.url),icon_url: iconUrl,});}@@ -482,7 +487,7 @@ export class EmbedService {url: footer.icon_url,...mediaProxyMetadataPolicy(nsfwMode),});- if (metadata) iconUrl = footer.icon_url;+ if (metadata) iconUrl = stripOwnAttachmentSignature(footer.icon_url);}return new EmbedFooter({text: footer.text,diff --git a/fluxer_api/src/api/infrastructure/IMediaService.ts b/fluxer_api/src/api/infrastructure/IMediaService.tsindex f611158ed..cf40d66fb 100644--- a/fluxer_api/src/api/infrastructure/IMediaService.ts+++ b/fluxer_api/src/api/infrastructure/IMediaService.ts@@ -57,6 +57,10 @@ export interface MediaProxyMetadataResponse {nsfw_probability?: number;}+export interface MediaProxySniffResponse {+ content_type: string | null;+}+export type MediaProxyFrameRequest =| {type: 'upload';@@ -85,5 +89,7 @@ export abstract class IMediaService {abstract getThumbnail(uploadFilename: string): Promise<Buffer | null>;+ abstract sniffUpload(uploadFilename: string): Promise<MediaProxySniffResponse | null>;+abstract extractFrames(request: MediaProxyFrameRequest): Promise<MediaProxyFrameResponse>;}diff --git a/fluxer_api/src/api/infrastructure/MediaService.ts b/fluxer_api/src/api/infrastructure/MediaService.tsindex 1a779a113..51682a312 100644--- a/fluxer_api/src/api/infrastructure/MediaService.ts+++ b/fluxer_api/src/api/infrastructure/MediaService.ts@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {signAttachmentUrl} from '@app/api/attachment/AttachmentUrls';import {Config} from '@app/api/Config';import {IMediaService,@@ -9,11 +10,13 @@ import {type MediaProxyMetadataRequest,type MediaProxyMetadataResponse,type MediaProxyNsfwMode,+ type MediaProxySniffResponse,} from '@app/api/infrastructure/IMediaService';import {Logger} from '@app/api/Logger';import * as FetchUtils from '@app/api/utils/FetchUtils';import {isJsonRecord, parseJsonWithGuard} from '@app/api/utils/JsonBoundaryUtils';import {ExplicitContentCannotBeSentError} from '@fluxer/errors/src/domains/moderation/ExplicitContentCannotBeSentError';+import {attachmentStorageKeyFromUrl} from '@pkgs/media_proxy_utils/src/AttachmentUrlSignature';import * as MediaProxyUtils from '@pkgs/media_proxy_utils/src/MediaProxyUtils';import {ms} from 'itty-time';@@ -30,7 +33,13 @@ const MEDIA_PROXY_METADATA_WITH_BASE64_MAX_BYTES = 64 * 1024 * 1024;const MEDIA_PROXY_ERROR_MAX_BYTES = 16 * 1024;const MEDIA_PROXY_THUMBNAIL_MAX_BYTES = 8 * 1024 * 1024;const MEDIA_PROXY_FRAMES_MAX_BYTES = 512 * 1024;+const MEDIA_PROXY_SNIFF_MAX_BYTES = 1024;const MEDIA_PROXY_REQUEST_TIMEOUT_MS = ms('30 seconds');+const MEDIA_PROXY_SNIFF_TIMEOUT_MS = ms('2 seconds');++function isMediaProxySniffResponse(value: unknown): value is MediaProxySniffResponse {+ return isJsonRecord(value) && (value.content_type === null || typeof value.content_type === 'string');+}function isMediaProxyMetadataResponse(value: unknown): value is MediaProxyMetadataResponse {if (!isJsonRecord(value)) return false;@@ -135,6 +144,9 @@ export class MediaService extends IMediaService {}getExternalMediaProxyURL(url: string): string {+ if (attachmentStorageKeyFromUrl(url, Config.endpoints.media) !== null) {+ return signAttachmentUrl(url);+ }let urlObj: URL;try {urlObj = new URL(url);@@ -166,6 +178,34 @@ export class MediaService extends IMediaService {}}+ async sniffUpload(uploadFilename: string): Promise<MediaProxySniffResponse | null> {+ const response = await this.makeRequest(+ '/_sniff',+ {+ type: 'upload',+ upload_filename: uploadFilename,+ },+ MEDIA_PROXY_SNIFF_TIMEOUT_MS,+ );+ if (!response) return null;+ try {+ const responseText = await FetchUtils.streamToStringWithLimit(response.body, {+ maxBytes: MEDIA_PROXY_SNIFF_MAX_BYTES,+ headers: response.headers,+ description: 'Media proxy sniff response',+ });+ const sniff = parseJsonWithGuard(responseText, isMediaProxySniffResponse);+ if (!sniff) {+ Logger.error({uploadFilename}, 'Media proxy returned invalid sniff response');+ return null;+ }+ return {content_type: sniff.content_type};+ } catch (error) {+ Logger.error({error, uploadFilename}, 'Failed to read media proxy sniff response');+ return null;+ }+ }+async extractFrames(request: MediaProxyFrameRequest): Promise<MediaProxyFrameResponse> {const response = await this.makeRequest('/_frames', request);if (!response) {@@ -183,7 +223,11 @@ export class MediaService extends IMediaService {return data;}- private async makeRequest(endpoint: string, body: MediaProxyRequestBody): Promise<Response | null> {+ private async makeRequest(+ endpoint: string,+ body: MediaProxyRequestBody,+ timeoutMs: number = MEDIA_PROXY_REQUEST_TIMEOUT_MS,+ ): Promise<Response | null> {try {const url = `http://${Config.mediaProxy.host}:${Config.mediaProxy.port}${endpoint}`;const response = await fetch(url, {@@ -193,7 +237,7 @@ export class MediaService extends IMediaService {'Content-Type': 'application/json',Authorization: `Bearer ${Config.mediaProxy.secretKey}`,},- signal: AbortSignal.timeout(MEDIA_PROXY_REQUEST_TIMEOUT_MS),+ signal: AbortSignal.timeout(timeoutMs),});if (!response.ok) {const errorText = await FetchUtils.streamToStringWithLimit(response.body, {diff --git a/fluxer_api/src/api/openapi/openapi.json b/fluxer_api/src/api/openapi/openapi.jsonindex 0aa798912..8301ac723 100644--- a/fluxer_api/src/api/openapi/openapi.json+++ b/fluxer_api/src/api/openapi/openapi.json@@ -107,6 +107,63 @@"security": [{"botToken": []}]}},+ "/attachments/refresh-urls": {+ "post": {+ "operationId": "refresh_attachment_urls",+ "summary": "Refresh attachment URLs",+ "tags": ["Messages"],+ "responses": {+ "200": {+ "description": "Success",+ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/RefreshAttachmentUrlsResponse"}}}+ },+ "400": {+ "description": "Bad Request - The request was malformed or contained invalid data",+ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}+ },+ "401": {+ "description": "Unauthorized - Authentication is required or the token is invalid",+ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}+ },+ "403": {+ "description": "Forbidden - You do not have permission to perform this action",+ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}+ },+ "429": {+ "description": "Too Many Requests - You are being rate limited",+ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ThrottledError"}}},+ "headers": {+ "Retry-After": {+ "description": "Number of seconds to wait before retrying (only on 429)",+ "schema": {"type": "integer"}+ },+ "X-RateLimit-Limit": {+ "description": "The number of requests that can be made in the current window",+ "schema": {"type": "integer"}+ },+ "X-RateLimit-Remaining": {+ "description": "The number of remaining requests that can be made",+ "schema": {"type": "integer"}+ },+ "X-RateLimit-Reset": {+ "description": "Unix timestamp when the rate limit resets",+ "schema": {"type": "integer"}+ }+ }+ },+ "500": {+ "description": "Internal Server Error - An unexpected error occurred",+ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Error"}}}+ }+ },+ "description": "Reissues the expiring signature on attachment URLs. Returns one entry per requested URL, in the order they were requested, each pairing the URL exactly as it was sent with a freshly signed copy. A URL that is not an attachment URL of this instance is returned unchanged. No membership or existence check is performed.",+ "security": [{"botToken": []}, {"sessionToken": []}],+ "requestBody": {+ "required": true,+ "content": {"application/json": {"schema": {"$ref": "#/components/schemas/RefreshAttachmentUrlsRequest"}}}+ }+ }+ },"/auth/authorize-ip": {"post": {"operationId": "authorize_ip_address",@@ -28845,6 +28902,31 @@"properties": {"token": {"description": "The IP authorization token from email", "type": "string"}},"required": ["token"]},+ "RefreshAttachmentUrlsRequest": {+ "type": "object",+ "properties": {+ "attachment_urls": {+ "minItems": 1,+ "maxItems": 50,+ "type": "array",+ "items": {"type": "string", "maxLength": 2048},+ "description": "Attachment URLs to refresh (1-50 entries, each at most 2048 characters)"+ }+ },+ "required": ["attachment_urls"]+ },+ "RefreshAttachmentUrlsResponse": {+ "type": "object",+ "properties": {+ "refreshed_urls": {+ "type": "array",+ "items": {"$ref": "#/components/schemas/RefreshedAttachmentUrl"},+ "description": "One entry per requested URL, in the order they were requested"+ }+ },+ "required": ["refreshed_urls"],+ "additionalProperties": false+ },"ApplicationsMeResponse": {"type": "object","properties": {@@ -29351,6 +29433,18 @@{"name": "SPAMMER", "value": "64", "description": "User is flagged as a spammer"}]},+ "RefreshedAttachmentUrl": {+ "type": "object",+ "properties": {+ "original": {"type": "string", "description": "The requested URL, echoed back unchanged"},+ "refreshed": {+ "type": "string",+ "description": "The same URL carrying a fresh signature, or the original when it is not an attachment URL of ours"+ }+ },+ "required": ["original", "refreshed"],+ "additionalProperties": false+ },"PasswordType": {"type": "string"},"EmailType": {"type": "string"},"AuthSessionLocation": {@@ -34217,8 +34311,8 @@{"name": "Billing", "description": "Subscription and payment management via Stripe"},{"name": "Premium", "description": "Premium subscription features and benefits"},{"name": "Gifts", "description": "Gift codes and redemption"},- {"name": "Connections"},{"name": "Messages"},+ {"name": "Connections"},{"name": "Donations"},{"name": "Experiments"},{"name": "Geolocation"},diff --git a/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts b/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.tsindex 0304a40a2..b9824196b 100644--- a/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts+++ b/fluxer_api/src/api/rate_limit_configs/ChannelRateLimitConfig.ts@@ -68,6 +68,10 @@ export const ChannelRateLimitConfigs = {bucket: 'attachment:delete',config: {limit: 40, windowMs: ms('10 seconds')},} as RouteRateLimitConfig,+ ATTACHMENT_URLS_REFRESH: {+ bucket: 'attachment:refresh_urls',+ config: {limit: 20, windowMs: ms('10 seconds')},+ } as RouteRateLimitConfig,CHANNEL_TYPING: {bucket: 'channel:typing::channel_id',config: {limit: 20, windowMs: ms('10 seconds')},diff --git a/fluxer_api/src/api/test/Setup.ts b/fluxer_api/src/api/test/Setup.tsindex ee0ea4a5d..1b80b99ee 100644--- a/fluxer_api/src/api/test/Setup.ts+++ b/fluxer_api/src/api/test/Setup.ts@@ -62,6 +62,7 @@ function setDefaultTestEnv(): void {FLUXER_S3_SECRET_ACCESS_KEY: 'test',FLUXER_API_PRESIGNED_ATTACHMENT_UPLOADS_ENABLED: 'false',FLUXER_MEDIA_PROXY_SECRET_KEY: 'test-media-secret',+ FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64: 'AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=',FLUXER_ADMIN_SECRET_KEY_BASE: 'test-admin-secret',FLUXER_ADMIN_OAUTH_CLIENT_SECRET: 'test-admin-oauth-secret',FLUXER_APP_PROXY_PORT: '8773',diff --git a/fluxer_api/src/api/test/TestMediaService.ts b/fluxer_api/src/api/test/TestMediaService.tsindex 1bf91b95b..af05aebd7 100644--- a/fluxer_api/src/api/test/TestMediaService.ts+++ b/fluxer_api/src/api/test/TestMediaService.ts@@ -8,6 +8,7 @@ import {type MediaProxyFrameResponse,type MediaProxyMetadataRequest,type MediaProxyMetadataResponse,+ type MediaProxySniffResponse,} from '@app/api/infrastructure/IMediaService';import type {IStorageService} from '@app/api/infrastructure/IStorageService';@@ -87,6 +88,10 @@ export class TestMediaService extends IMediaService {return Buffer.alloc(1024);}+ async sniffUpload(_uploadFilename: string): Promise<MediaProxySniffResponse | null> {+ return {content_type: null};+ }+async extractFrames(_request: MediaProxyFrameRequest): Promise<MediaProxyFrameResponse> {return {frames: [diff --git a/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts b/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.tsindex f8e023974..690386173 100644--- a/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts+++ b/fluxer_api/src/api/test/mocks/RepositoryBackedMessageResponseDataService.ts@@ -2,9 +2,9 @@import {AttachmentDecayRepository} from '@app/api/attachment/AttachmentDecayRepository';import {AttachmentDecayService} from '@app/api/attachment/AttachmentDecayService';+import {makeSignedAttachmentCdnUrl, signAttachmentUrl} from '@app/api/attachment/AttachmentUrls';import type {ChannelID, GuildID, MessageID, UserID} from '@app/api/BrandedTypes';import {createUserID} from '@app/api/BrandedTypes';-import {Config} from '@app/api/Config';import {type MessageResponseAccessContext,MessageResponseDataService,@@ -58,6 +58,14 @@ class NoopNatsConnectionManager implements INatsConnectionManager {}}+function signOwnUrl(url: string | null | undefined): string | null {+ return url == null ? null : signAttachmentUrl(url);+}++function mediaProxyUrl(url: string | null | undefined): string | null {+ return url?.startsWith('http') ? signAttachmentUrl(url) : null;+}+export class RepositoryBackedMessageResponseDataService extends MessageResponseDataService {private readonly attachmentDecayRepository = new AttachmentDecayRepository();private readonly attachmentDecayService = new AttachmentDecayService(this.attachmentDecayRepository);@@ -343,8 +351,7 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseD}private mapAttachmentUrl(message: Message, attachment: Attachment): string {- const filename = encodeURIComponent(attachment.filename);- return `${Config.endpoints.media}/attachments/${message.channelId.toString()}/${message.id.toString()}/${attachment.id.toString()}/${filename}`;+ return makeSignedAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename);}private async mapAttachments(@@ -399,7 +406,7 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseDprivate mapEmbed(embed: Embed, message: Message): MessageEmbedResponse {return {type: embed.type ?? 'rich',- url: embed.url,+ url: signOwnUrl(embed.url),title: embed.title,color: embed.color,timestamp: embed.timestamp?.toISOString() ?? null,@@ -425,9 +432,9 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseDconst iconUrl = 'iconUrl' in author ? author.iconUrl : null;return {name: author.name,- url: author.url,- icon_url: iconUrl,- proxy_icon_url: iconUrl,+ url: signOwnUrl(author.url),+ icon_url: signOwnUrl(iconUrl),+ proxy_icon_url: mediaProxyUrl(iconUrl),};}@@ -435,8 +442,8 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseDif (!footer?.text) return null;return {text: footer.text,- icon_url: footer.iconUrl,- proxy_icon_url: footer.iconUrl,+ icon_url: signOwnUrl(footer.iconUrl),+ proxy_icon_url: mediaProxyUrl(footer.iconUrl),};}@@ -450,10 +457,10 @@ export class RepositoryBackedMessageResponseDataService extends MessageResponseDprivate mapEmbedMedia(media: EmbedMedia | null, message: Message) {if (!media?.url) return null;- const url = this.resolveAttachmentUrl(media.url, message);+ const resolved = this.resolveAttachmentUrl(media.url, message);return {- url,- proxy_url: url.startsWith('http') ? url : null,+ url: signAttachmentUrl(resolved),+ proxy_url: mediaProxyUrl(resolved),content_type: media.contentType,content_hash: media.contentHash,width: media.width,diff --git a/fluxer_api/src/api/utils/UnfurlerUtils.ts b/fluxer_api/src/api/utils/UnfurlerUtils.tsindex 89d865aea..357e1f245 100644--- a/fluxer_api/src/api/utils/UnfurlerUtils.ts+++ b/fluxer_api/src/api/utils/UnfurlerUtils.ts@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport {createHash} from 'node:crypto';+import {stripOwnAttachmentSignature} from '@app/api/attachment/AttachmentUrls';import {Config} from '@app/api/Config';import {Logger} from '@app/api/Logger';import * as InviteUtils from '@app/api/utils/InviteUtils';@@ -103,9 +104,10 @@ export function extractURLs(inputText: string) {if (isFluxerAppExcludedURL(url)) continue;const encoded = idnaEncodeURL(url);if (!encoded) continue;- if (!seen.has(encoded)) {- seen.add(encoded);- result.push(encoded);+ const canonical = stripOwnAttachmentSignature(encoded);+ if (!seen.has(canonical)) {+ seen.add(canonical);+ result.push(canonical);if (result.length >= 5) break;}}diff --git a/fluxer_api/src/api/webhook/WebhookService.ts b/fluxer_api/src/api/webhook/WebhookService.tsindex 5f79e5f97..f748a95ec 100644--- a/fluxer_api/src/api/webhook/WebhookService.ts+++ b/fluxer_api/src/api/webhook/WebhookService.ts@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport fs from 'node:fs/promises';+import {stripOwnAttachmentSignature} from '@app/api/attachment/AttachmentUrls';import type {ChannelID, GuildID, MessageID, UserID, WebhookID, WebhookToken} from '@app/api/BrandedTypes';import {createChannelID, createGuildID, createWebhookID, createWebhookToken} from '@app/api/BrandedTypes';import type {IChannelRepository} from '@app/api/channel/IChannelRepository';@@ -571,12 +572,13 @@ export class WebhookService {private async getWebhookAvatar({webhookId,- avatarUrl,+ avatarUrl: requestedAvatarUrl,}: {webhookId: WebhookID;avatarUrl: string | null;}): Promise<string | null> {- if (!avatarUrl) return null;+ if (!requestedAvatarUrl) return null;+ const avatarUrl = stripOwnAttachmentSignature(requestedAvatarUrl);try {const cacheKey = `webhook:${webhookId}:avatar:${avatarUrl}`;const avatarCache = await this.cacheService.get<string>(cacheKey);diff --git a/fluxer_api/src/api/worker/tasks/HarvestGuildData.ts b/fluxer_api/src/api/worker/tasks/HarvestGuildData.tsindex 52c0f515a..7f7e73fb6 100644--- a/fluxer_api/src/api/worker/tasks/HarvestGuildData.ts+++ b/fluxer_api/src/api/worker/tasks/HarvestGuildData.ts@@ -11,9 +11,10 @@ import {createArchiveTask,throwIfArchiveTerminallyFailed,} from '@app/api/archive/ArchiveTask';+import {makeDataPackageAttachmentCdnUrl} from '@app/api/attachment/AttachmentUrls';import {type AttachmentID, type ChannelID, createGuildID, type MessageID} from '@app/api/BrandedTypes';import {Config} from '@app/api/Config';-import {makeAttachmentCdnKey, makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers';+import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';import type {IStorageService} from '@app/api/infrastructure/IStorageService';import {Logger} from '@app/api/Logger';import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils';@@ -54,6 +55,32 @@ interface PendingAttachmentDownload {filename: string;}+export interface GuildHarvestAttachment {+ id: AttachmentID;+ filename: string;+ size: bigint;+ contentType: string;+ width: number | null;+ height: number | null;+}++export function buildGuildHarvestAttachment(+ channelId: ChannelID,+ attachment: GuildHarvestAttachment,+ includeAttachments: boolean,+): Record<string, unknown> {+ return {+ attachment_id: attachment.id.toString(),+ filename: attachment.filename,+ size: attachment.size.toString(),+ content_type: attachment.contentType,+ archive_path: includeAttachments ? `attachments/${channelId}/${attachment.id}/${attachment.filename}` : null,+ cdn_url: makeDataPackageAttachmentCdnUrl(channelId, attachment.id, attachment.filename),+ width: attachment.width,+ height: attachment.height,+ };+}+function cdnBucket(): string {return Config.s3.buckets.cdn;}@@ -182,16 +209,7 @@ const harvestGuildData: ArchiveTaskHandler = async (payload, helpers, attempt) =if (msg.authorId == null) continue;const attachments: Array<object> = [];for (const att of msg.attachments) {- attachments.push({- attachment_id: att.id.toString(),- filename: att.filename,- size: att.size.toString(),- content_type: att.contentType,- archive_path: validated.includeAttachments ? `attachments/${channel.id}/${att.id}/${att.filename}` : null,- cdn_url: makeAttachmentCdnUrl(channel.id, att.id, att.filename),- width: att.width,- height: att.height,- });+ attachments.push(buildGuildHarvestAttachment(channel.id, att, validated.includeAttachments));if (validated.includeAttachments) {channelDownloads.push({channelId: channel.id,diff --git a/fluxer_api/src/api/worker/tasks/HarvestUserData.ts b/fluxer_api/src/api/worker/tasks/HarvestUserData.tsindex 91a54668c..aebfa0b5d 100644--- a/fluxer_api/src/api/worker/tasks/HarvestUserData.ts+++ b/fluxer_api/src/api/worker/tasks/HarvestUserData.ts@@ -11,6 +11,7 @@ import {createArchiveTask,throwIfArchiveTerminallyFailed,} from '@app/api/archive/ArchiveTask';+import {makeDataPackageAttachmentCdnUrl} from '@app/api/attachment/AttachmentUrls';import {type ChannelID,createAttachmentID,@@ -21,7 +22,6 @@ import {type UserID,} from '@app/api/BrandedTypes';import {Config} from '@app/api/Config';-import {makeAttachmentCdnUrl} from '@app/api/channel/services/message/MessageHelpers';import {isChannelEligible,isTimestampInWindow,@@ -340,7 +340,7 @@ export async function harvestMessages(content_type: attachment.contentType,content_hash: null,archive_path: null,- cdn_url: makeAttachmentCdnUrl(channelId, attachment.id, attachment.filename),+ cdn_url: makeDataPackageAttachmentCdnUrl(channelId, attachment.id, attachment.filename),width: attachment.width,height: attachment.height,})),diff --git a/fluxer_app/src/features/app/constants/Endpoints.ts b/fluxer_app/src/features/app/constants/Endpoints.tsindex c866ceb0c..712f2cb35 100644--- a/fluxer_app/src/features/app/constants/Endpoints.ts+++ b/fluxer_app/src/features/app/constants/Endpoints.ts@@ -4,6 +4,7 @@ import {ME} from '@fluxer/constants/src/AppConstants';export const Endpoints = {INSTANCE: '/instance',+ ATTACHMENTS_REFRESH_URLS: '/attachments/refresh-urls',AUTH_LOGIN: '/auth/login',AUTH_LOGIN_MFA_TOTP: '/auth/login/mfa/totp',AUTH_LOGIN_MFA_WEBAUTHN_OPTIONS: '/auth/login/mfa/webauthn/authentication-options',diff --git a/fluxer_app/src/features/channel/components/AutocompleteMemePreview.tsx b/fluxer_app/src/features/channel/components/AutocompleteMemePreview.tsxindex 58e12f29e..fc7f90f45 100644--- a/fluxer_app/src/features/channel/components/AutocompleteMemePreview.tsx+++ b/fluxer_app/src/features/channel/components/AutocompleteMemePreview.tsx@@ -4,6 +4,7 @@ import {useAnimatedMediaVideoPlayback} from '@app/features/app/hooks/useAnimatedimport {useShouldAnimate} from '@app/features/app/hooks/useShouldAnimate';import styles from '@app/features/channel/components/AutocompleteEmoji.module.css';import type {FavoriteMeme} from '@app/features/expressions/models/FavoriteMeme';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import {buildStaticGifPreviewURL} from '@app/features/messaging/utils/MediaProxyUtils';import {MusicNoteIcon} from '@phosphor-icons/react';import {observer} from 'mobx-react-lite';@@ -32,6 +33,7 @@ export const AutocompleteMemePreview = observer(({meme}: {meme: FavoriteMeme}) =const isVideoMeme = contentType.startsWith('video/');const isAnimatedImageMeme = !isVideoMeme && !isAudioMeme && contentType.includes('gif');const motionAllowed = useShouldAnimate({kind: 'gif', isAnimated: isVideoMeme || isAnimatedImageMeme});+ const memeUrl = AttachmentUrlRefresher.fresh(meme.url, {refreshUnsigned: true});if (isAudioMeme) {return (<div className={styles.audioIconWrapper} data-flx="channel.autocomplete-meme-preview.audio-icon-wrapper">@@ -44,16 +46,14 @@ export const AutocompleteMemePreview = observer(({meme}: {meme: FavoriteMeme}) =);}if (isVideoMeme && motionAllowed) {- return (- <AutocompleteMemeVideo src={meme.url} data-flx="channel.autocomplete-meme-preview.autocomplete-meme-video" />- );+ return <AutocompleteMemeVideo src={memeUrl} data-flx="channel.autocomplete-meme-preview.autocomplete-meme-video" />;}const needsStillFrame = (isVideoMeme || isAnimatedImageMeme) && !motionAllowed;return (<imgdraggable={false}className={styles.memeIcon}- src={needsStillFrame ? buildStaticGifPreviewURL(meme.url) : meme.url}+ src={needsStillFrame ? buildStaticGifPreviewURL(memeUrl) : memeUrl}alt={meme.name}aria-hidden={true}data-flx="channel.autocomplete-meme-preview.meme-icon"diff --git a/fluxer_app/src/features/channel/components/MessageAttachmentStateUtils.ts b/fluxer_app/src/features/channel/components/MessageAttachmentStateUtils.tsindex e95ff7723..0102d1c82 100644--- a/fluxer_app/src/features/channel/components/MessageAttachmentStateUtils.ts+++ b/fluxer_app/src/features/channel/components/MessageAttachmentStateUtils.ts@@ -2,6 +2,7 @@import {splitMediaAndFileAttachments} from '@app/features/channel/components/MessageAttachmentUtils';import DeveloperOptions from '@app/features/devtools/state/DeveloperOptions';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import {mapAttachmentsWithExpiry} from '@app/features/messaging/utils/AttachmentExpiryUtils';import UserSettings from '@app/features/user/state/UserSettings';import type {MessageAttachment} from '@fluxer/schema/src/domains/message/MessageResponseSchemas';@@ -13,11 +14,25 @@ interface AttachmentRenderingState {shouldUseMosaic: boolean;}+function freshAttachmentUrl(url: string | null): string | null {+ return url === null ? null : AttachmentUrlRefresher.fresh(url);+}++export function withFreshAttachmentUrls(attachment: MessageAttachment): MessageAttachment {+ const url = freshAttachmentUrl(attachment.url);+ const proxyUrl = freshAttachmentUrl(attachment.proxy_url);+ if (url === attachment.url && proxyUrl === attachment.proxy_url) return attachment;+ return {...attachment, url, proxy_url: proxyUrl};+}+export const getAttachmentRenderingState = (snapshotAttachments?: ReadonlyArray<MessageAttachment> | null,): AttachmentRenderingState => {const attachments = snapshotAttachments ?? [];- const expiryApplied = mapAttachmentsWithExpiry(attachments, DeveloperOptions.mockAttachmentStates);+ const expiryApplied = mapAttachmentsWithExpiry(attachments, DeveloperOptions.mockAttachmentStates).map((entry) => ({+ isExpired: entry.isExpired,+ attachment: withFreshAttachmentUrls(entry.attachment),+ }));const enrichedAttachments = expiryApplied.map((entry) => entry.attachment);const activeAttachments = expiryApplied.filter((entry) => !entry.isExpired).map((entry) => entry.attachment);const {mediaAttachments} = splitMediaAndFileAttachments(activeAttachments);diff --git a/fluxer_app/src/features/channel/components/MobileMemesPicker.tsx b/fluxer_app/src/features/channel/components/MobileMemesPicker.tsxindex 00dc6854a..b00d76d3b 100644--- a/fluxer_app/src/features/channel/components/MobileMemesPicker.tsx+++ b/fluxer_app/src/features/channel/components/MobileMemesPicker.tsx@@ -20,6 +20,7 @@ import FavoriteMemes from '@app/features/expressions/state/FavoriteMemes';import {AUDIO_DESCRIPTOR, GIFS_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors';import {isKeyboardActivationKey} from '@app/features/input/utils/KeyboardUtils';import {useNearViewport} from '@app/features/messaging/hooks/useNearViewport';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import {buildStaticGifPreviewURL} from '@app/features/messaging/utils/MediaProxyUtils';import {ComponentBus} from '@app/features/platform/utils/ComponentBus';import {DeleteIcon, EditIcon} from '@app/features/ui/action_menu/ContextMenuIcons';@@ -428,7 +429,8 @@ const GridItem = observer(const isAudio = contentType.startsWith('audio/');const isGifImage = !isVideo && contentType.toLowerCase().includes('gif');const shouldAnimateGif = useShouldAnimate({kind: 'gif', isAnimated: !isAudio && (isVideo || isGifImage)});- const thumbnailSrc = isGifImage && !shouldAnimateGif ? buildStaticGifPreviewURL(url) : url;+ const memeUrl = AttachmentUrlRefresher.fresh(url, {refreshUnsigned: true});+ const thumbnailSrc = isGifImage && !shouldAnimateGif ? buildStaticGifPreviewURL(memeUrl) : memeUrl;const videoPlaybackAllowed = useAnimatedMediaVideoPlayback(videoRef, {enabled: isVisible && !isAudio && isVideo,shouldPlay: shouldAnimateGif,@@ -471,7 +473,7 @@ const GridItem = observer(disablePictureInPicture={true}disableRemotePlayback={true}preload={shouldAnimateGif ? 'auto' : 'metadata'}- src={url}+ src={memeUrl}data-flx="channel.mobile-memes-picker.grid-item.gif"/>)}diff --git a/fluxer_app/src/features/channel/components/embeds/ChannelEmbed.tsx b/fluxer_app/src/features/channel/components/embeds/ChannelEmbed.tsxindex 07772a6b0..e643e58b4 100644--- a/fluxer_app/src/features/channel/components/embeds/ChannelEmbed.tsx+++ b/fluxer_app/src/features/channel/components/embeds/ChannelEmbed.tsx@@ -26,6 +26,7 @@ import Channels from '@app/features/channel/state/Channels';import Guilds from '@app/features/guild/state/Guilds';import {SUPPRESS_EMBEDS_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors';import * as MessageCommands from '@app/features/messaging/commands/MessageCommands';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import {getEmbedMediaDimensions} from '@app/features/messaging/utils/MediaDimensionConfig';import {buildAnimatedImageProxyURL} from '@app/features/messaging/utils/MediaProxyUtils';import {buildMessageEmbedCopyText} from '@app/features/messaging/utils/MessageCopyTextUtils';@@ -39,6 +40,7 @@ import FocusRing from '@app/features/ui/focus_ring/FocusRing';import MobileLayout from '@app/features/ui/state/MobileLayout';import {MessageAttachmentFlags, MessageEmbedTypes, Permissions} from '@fluxer/constants/src/ChannelConstants';import {GuildOperations} from '@fluxer/constants/src/GuildConstants';+import type {EmbedMedia, MessageEmbed} from '@fluxer/schema/src/domains/message/EmbedSchemas';import {useLingui} from '@lingui/react/macro';import {XIcon} from '@phosphor-icons/react';import {clsx} from 'clsx';@@ -47,7 +49,29 @@ import type React from 'react';import {type FC, useCallback, useMemo} from 'react';const mediaFocusRingClass = messageStyles.mediaFocusRing;-export const Embed: FC<EmbedProps> = observer(({embed, message, embedIndex, contextualEmbeds, onDelete, isPreview}) => {++function withFreshEmbedMediaUrls(media: EmbedMedia | undefined): EmbedMedia | undefined {+ if (!media) return media;+ const url = AttachmentUrlRefresher.fresh(media.url);+ const proxyUrl = media.proxy_url === undefined ? undefined : AttachmentUrlRefresher.fresh(media.proxy_url);+ if (url === media.url && proxyUrl === media.proxy_url) return media;+ return {...media, url, proxy_url: proxyUrl};+}++function withFreshEmbedUrls(embed: MessageEmbed): MessageEmbed {+ const image = withFreshEmbedMediaUrls(embed.image);+ const thumbnail = withFreshEmbedMediaUrls(embed.thumbnail);+ const video = withFreshEmbedMediaUrls(embed.video);+ const audio = withFreshEmbedMediaUrls(embed.audio);+ if (image === embed.image && thumbnail === embed.thumbnail && video === embed.video && audio === embed.audio) {+ return embed;+ }+ return {...embed, image, thumbnail, video, audio};+}++export const Embed: FC<EmbedProps> = observer((props: EmbedProps) => {+ const {message, embedIndex, contextualEmbeds, onDelete, isPreview} = props;+ const embed = withFreshEmbedUrls(props.embed);const {i18n} = useLingui();const {enabled: isMobile} = MobileLayout;const channel = Channels.getChannel(message.channelId);diff --git a/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreview.tsx b/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreview.tsxindex cf29ddae6..d1e49a091 100644--- a/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreview.tsx+++ b/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreview.tsx@@ -9,6 +9,10 @@ import {CsvAttachmentTablePanel} from '@app/features/channel/components/embeds/aimport {TextualAttachmentCodePanel} from '@app/features/channel/components/embeds/attachments/TextualAttachmentCodePanel';import styles from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreview.module.css';import {TextualAttachmentPreviewBottomSheet} from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreviewBottomSheet';+import {+ fetchTextualPreviewText,+ PreviewSizeLimitError,+} from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreviewFetch';import {TextualAttachmentPreviewFooter} from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreviewFooter';import {TextualAttachmentPreviewModal} from '@app/features/channel/components/embeds/attachments/TextualAttachmentPreviewModal';import {@@ -45,56 +49,6 @@ import {type MouseEvent, useCallback, useEffect, useMemo, useState} from 'react'const logger = new Logger('TextualAttachmentPreview');-class PreviewSizeLimitError extends Error {- constructor() {- super('Attachment preview exceeds the size limit');- this.name = 'PreviewSizeLimitError';- }-}--async function readPreviewText(response: Response): Promise<string> {- const contentLength = response.headers.get('content-length');- if (contentLength !== null) {- const parsedContentLength = Number(contentLength);- if (Number.isFinite(parsedContentLength) && parsedContentLength > TEXT_PREVIEW_MAX_BYTES) {- throw new PreviewSizeLimitError();- }- }- const body = response.body;- if (!body) {- throw new Error('Attachment preview response has no readable body');- }- const reader = body.getReader();- const chunks: Array<Uint8Array> = [];- let totalBytes = 0;- try {- while (true) {- const {done, value} = await reader.read();- if (done) {- break;- }- if (!value) {- continue;- }- totalBytes += value.byteLength;- if (totalBytes > TEXT_PREVIEW_MAX_BYTES) {- await reader.cancel().catch(() => undefined);- throw new PreviewSizeLimitError();- }- chunks.push(value);- }- } finally {- reader.releaseLock();- }- const bytes = new Uint8Array(totalBytes);- let offset = 0;- for (const chunk of chunks) {- bytes.set(chunk, offset);- offset += chunk.byteLength;- }- return new TextDecoder().decode(bytes);-}-export const TextualAttachmentPreview = observer(function TextualAttachmentPreview({attachment,spoilerHidden = false,@@ -146,13 +100,7 @@ export const TextualAttachmentPreview = observer(function TextualAttachmentPrevisetStatus('loading');setPreviewError(null);const controller = new AbortController();- fetch(attachment.url, {signal: controller.signal})- .then((response) => {- if (!response.ok) {- throw new Error(`Attachment preview request failed: ${response.status} ${response.statusText}`);- }- return readPreviewText(response);- })+ fetchTextualPreviewText(attachment.url, controller.signal).then((value) => {if (controller.signal.aborted) {return;diff --git a/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreviewFetch.ts b/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreviewFetch.tsnew file mode 100644index 000000000..27f93621d--- /dev/null+++ b/fluxer_app/src/features/channel/components/embeds/attachments/TextualAttachmentPreviewFetch.ts@@ -0,0 +1,84 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';+import {TEXT_PREVIEW_MAX_BYTES} from '@app/features/messaging/utils/AttachmentPreviewUtils';++const SIGNATURE_REFUSAL_STATUSES = new Set([401, 403, 404, 410]);++export class PreviewSizeLimitError extends Error {+ constructor() {+ super('Attachment preview exceeds the size limit');+ this.name = 'PreviewSizeLimitError';+ }+}++export function isAttachmentSignatureRefusal(status: number): boolean {+ return SIGNATURE_REFUSAL_STATUSES.has(status);+}++export async function readPreviewText(response: Response): Promise<string> {+ const contentLength = response.headers.get('content-length');+ if (contentLength !== null) {+ const parsedContentLength = Number(contentLength);+ if (Number.isFinite(parsedContentLength) && parsedContentLength > TEXT_PREVIEW_MAX_BYTES) {+ throw new PreviewSizeLimitError();+ }+ }+ const body = response.body;+ if (!body) {+ throw new Error('Attachment preview response has no readable body');+ }+ const reader = body.getReader();+ const chunks: Array<Uint8Array> = [];+ let totalBytes = 0;+ try {+ while (true) {+ const {done, value} = await reader.read();+ if (done) {+ break;+ }+ if (!value) {+ continue;+ }+ totalBytes += value.byteLength;+ if (totalBytes > TEXT_PREVIEW_MAX_BYTES) {+ await reader.cancel().catch(() => undefined);+ throw new PreviewSizeLimitError();+ }+ chunks.push(value);+ }+ } finally {+ reader.releaseLock();+ }+ const bytes = new Uint8Array(totalBytes);+ let offset = 0;+ for (const chunk of chunks) {+ bytes.set(chunk, offset);+ offset += chunk.byteLength;+ }+ return new TextDecoder().decode(bytes);+}++function requestFailed(response: Response): Error {+ return new Error(`Attachment preview request failed: ${response.status} ${response.statusText}`);+}++export async function fetchTextualPreviewText(url: string, signal: AbortSignal): Promise<string> {+ const target = await AttachmentUrlRefresher.refresh(url);+ const response = await fetch(target, {signal});+ if (response.ok) {+ return readPreviewText(response);+ }+ if (!isAttachmentSignatureRefusal(response.status)) {+ throw requestFailed(response);+ }+ const refreshed = await AttachmentUrlRefresher.refresh(url, {force: true});+ if (refreshed === target) {+ throw requestFailed(response);+ }+ const retried = await fetch(refreshed, {signal});+ if (!retried.ok) {+ throw requestFailed(retried);+ }+ return readPreviewText(retried);+}diff --git a/fluxer_app/src/features/channel/components/embeds/media/EmbedGifv.tsx b/fluxer_app/src/features/channel/components/embeds/media/EmbedGifv.tsxindex a3d684eb2..a5de76ce2 100644--- a/fluxer_app/src/features/channel/components/embeds/media/EmbedGifv.tsx+++ b/fluxer_app/src/features/channel/components/embeds/media/EmbedGifv.tsx@@ -19,6 +19,7 @@ import {safePause, safePlay} from '@app/features/channel/components/GifVideoPoolimport {useMaybeMessageViewContext} from '@app/features/channel/components/MessageViewContext';import type {Channel} from '@app/features/channel/models/Channel';import {isKeyboardActivationKey} from '@app/features/input/utils/KeyboardUtils';+import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError';import {useDeleteAttachment} from '@app/features/messaging/hooks/useDeleteAttachment';import {useMatureMedia} from '@app/features/messaging/hooks/useMatureMedia';import {useMediaFavorite} from '@app/features/messaging/hooks/useMediaFavorite';@@ -389,6 +390,7 @@ export const EmbedGifv: FC<const messageViewContext = useMaybeMessageViewContext();const mediaCalculator = useEmbedMediaCalculator();const videoRef = useRef<HTMLVideoElement>(null);+ const handleVideoError = useAttachmentRefreshOnError(videoProxyURL);const [containerElement, setContainerElement] = useState<HTMLDivElement | null>(null);const {isPointerInside, hasFocusInside} = useMediaSurfaceInteraction(containerElement);const {shouldBlur, gateReason, canReveal, reveal: revealSensitiveMedia} = useMatureMedia(nsfw, channelId);@@ -632,6 +634,7 @@ export const EmbedGifv: FC<height={dimensions.height}tabIndex={-1}aria-label={i18n._(ANIMATED_GIF_VIDEO_DESCRIPTOR)}+ onError={handleVideoError}data-embed-media="gifv"data-flx="channel.embeds.media.embed-gifv.video-element"/>diff --git a/fluxer_app/src/features/channel/components/embeds/media/VoiceMessagePlayer.tsx b/fluxer_app/src/features/channel/components/embeds/media/VoiceMessagePlayer.tsxindex 0247e7450..0187888f8 100644--- a/fluxer_app/src/features/channel/components/embeds/media/VoiceMessagePlayer.tsx+++ b/fluxer_app/src/features/channel/components/embeds/media/VoiceMessagePlayer.tsx@@ -6,6 +6,7 @@ import styles from '@app/features/channel/components/embeds/media/VoiceMessagePlimport {useMaybeMessageViewContext} from '@app/features/channel/components/MessageViewContext';import {PAUSE_DESCRIPTOR, PLAY_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors';import {getCachedNumberFormat} from '@app/features/i18n/utils/IntlCache';+import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError';import {buildMediaProxyURL} from '@app/features/messaging/utils/MediaProxyUtils';import {Logger} from '@app/features/platform/utils/AppLogger';import {remFromPx} from '@app/features/theme/layout/RemFromPx';@@ -119,6 +120,7 @@ const VoiceMessagePlayer: React.FC<VoiceMessagePlayerProps> = observer(const {i18n} = useLingui();const messageViewContext = useMaybeMessageViewContext();const effectiveSrc = buildMediaProxyURL(src);+ const handleMediaError = useAttachmentRefreshOnError(effectiveSrc);const [hasStarted, setHasStarted] = useState(false);const [wantsMetadata, setWantsMetadata] = useState(false);const [prePlayCurrentTime, setPrePlayCurrentTime] = useState(0);@@ -324,6 +326,7 @@ const VoiceMessagePlayer: React.FC<VoiceMessagePlayerProps> = observer(ref={mediaRef as React.RefObject<HTMLAudioElement>}src={hasStarted || wantsMetadata ? effectiveSrc : undefined}preload={wantsMetadata ? 'metadata' : 'none'}+ onError={handleMediaError}data-flx="channel.embeds.media.voice-message-player.audio"><track kind="captions" data-flx="channel.embeds.media.voice-message-player.track" />diff --git a/fluxer_app/src/features/channel/components/pickers/gif/FavoriteGifTypes.ts b/fluxer_app/src/features/channel/components/pickers/gif/FavoriteGifTypes.tsindex 286cf6e5c..da3af0610 100644--- a/fluxer_app/src/features/channel/components/pickers/gif/FavoriteGifTypes.ts+++ b/fluxer_app/src/features/channel/components/pickers/gif/FavoriteGifTypes.ts@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {stripAttachmentSignature} from '@app/features/messaging/utils/AttachmentCdnUrl';import {inferFormatContentType, PREVIEW_FORMAT_PRIORITY} from '@fluxer/schema/src/domains/gif/GifMediaFormatKeys';export interface FavoriteGifMediaFormat {@@ -107,16 +108,36 @@ export function pickCanonicalPreviewFormat(export const STORED_PREVIEW_DEVICE_PIXEL_RATIO = 2;+export function stripFavoriteGifEntrySignatures(entry: FavoriteGifEntry): FavoriteGifEntry {+ const url = stripAttachmentSignature(entry.url);+ const proxyUrl = stripAttachmentSignature(entry.proxy_url);+ const media: Record<string, FavoriteGifMediaFormat> = {};+ let mediaChanged = false;+ for (const [key, format] of Object.entries(entry.media)) {+ const src = stripAttachmentSignature(format.src);+ const proxySrc = stripAttachmentSignature(format.proxy_src);+ if (src === format.src && proxySrc === format.proxy_src) {+ media[key] = format;+ continue;+ }+ mediaChanged = true;+ media[key] = {...format, src, proxy_src: proxySrc};+ }+ if (url === entry.url && proxyUrl === entry.proxy_url && !mediaChanged) return entry;+ return {...entry, url, proxy_url: proxyUrl, media};+}+export function slimFavoriteGifEntry(entry: FavoriteGifEntry): FavoriteGifEntry {- const best = pickBestPreviewFormat(entry.media, 'any', {+ const stripped = stripFavoriteGifEntrySignatures(entry);+ const best = pickBestPreviewFormat(stripped.media, 'any', {cssWidth: PREVIEW_TILE_CSS_WIDTH,devicePixelRatio: STORED_PREVIEW_DEVICE_PIXEL_RATIO,});if (best == null) {- return Object.keys(entry.media).length === 0 ? entry : {...entry, media: {}};+ return Object.keys(stripped.media).length === 0 ? stripped : {...stripped, media: {}};}return {- ...entry,+ ...stripped,proxy_url: best.format.proxy_src,width: best.format.width,height: best.format.height,diff --git a/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGrid.tsx b/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGrid.tsxindex 697969ae9..e3d64cfd8 100644--- a/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGrid.tsx+++ b/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGrid.tsx@@ -16,6 +16,7 @@ import FavoriteGif from '@app/features/expressions/state/FavoriteGif';import FavoriteMemes from '@app/features/expressions/state/FavoriteMemes';import * as GifSlugUtils from '@app/features/expressions/utils/GifSlugUtils';import {FAVORITES_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import QuickSwitcher from '@app/features/search/state/QuickSwitcher';import {msg} from '@lingui/core/macro';import {useLingui} from '@lingui/react/macro';@@ -57,6 +58,7 @@ export const GifPickerGrid = observer(const favoriteGifs = FavoriteGif.favoriteGifs;const favoriteGifsVersion = favoriteGifs.length;const useSavedMediaForGifFavorites = FavoriteGif.saveGifFavoritesAsSavedMedia;+ const attachmentUrlRevision = AttachmentUrlRefresher.revision;const data: Array<GifPickerGridItemData> = useMemo(() => {return buildGifPickerGridData({surface: store.isShowingFavorites ? 'favorites' : store.isShowingFeatured ? 'featured' : 'results',@@ -88,6 +90,7 @@ export const GifPickerGrid = observer(selectGif,store.featuredFavoritePreviewSeed,i18n.locale,+ attachmentUrlRevision,]);const itemKeys = useMemo(() => data.filter((item) => item.type !== 'skeleton').map((item) => item.key), [data]);const itemByKey = useMemo(() => new Map(data.map((item) => [item.key, item])), [data]);diff --git a/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGridData.ts b/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGridData.tsindex bca5a047e..ba3306214 100644--- a/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGridData.ts+++ b/fluxer_app/src/features/channel/components/pickers/gif/GifPickerGridData.ts@@ -8,6 +8,7 @@ import {import type {GifPickerGridItemData} from '@app/features/channel/components/pickers/gif/GifPickerTypes';import type {Gif, GifFeatured} from '@app/features/expressions/commands/GifCommands';import * as GifSlugUtils from '@app/features/expressions/utils/GifSlugUtils';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';const CATEGORY_TILE_WIDTH = 200;const CATEGORY_TILE_HEIGHT = 96;@@ -34,6 +35,10 @@ export interface BuildGifPickerGridDataInput {trendingTitle: string;}+function freshStoredUrl(url: string): string {+ return AttachmentUrlRefresher.fresh(url, {refreshUnsigned: true});+}+export function buildSkeletonGifPickerItems(count: number): Array<GifPickerGridItemData> {return Array.from({length: count}, (_, i) => ({type: 'skeleton',@@ -68,9 +73,9 @@ function buildFavoriteGifItems(for (let index = favoriteGifs.length - 1; index >= 0; index -= 1) {const entry = favoriteGifs[index];const best = pickBestPreviewFormat(entry.media);- const fallbackSrc = GifSlugUtils.isUsableMediaSource(entry.proxy_url) ? entry.proxy_url : '';- const previewSrc = best?.format.src ?? fallbackSrc;- const previewProxySrc = best?.format.proxy_src ?? fallbackSrc;+ const fallbackSrc = GifSlugUtils.isUsableMediaSource(entry.proxy_url) ? freshStoredUrl(entry.proxy_url) : '';+ const previewSrc = best ? freshStoredUrl(best.format.src) : fallbackSrc;+ const previewProxySrc = best ? freshStoredUrl(best.format.proxy_src) : fallbackSrc;const previewWidth = best?.format.width ?? entry.width;const previewHeight = best?.format.height ?? entry.height;const previewContentType = best ? inferFormatContentType(best.key) : entry.content_type;@@ -107,14 +112,21 @@ function buildFeaturedItems(input: BuildGifPickerGridDataInput): Array<GifPickerconst favoriteGifPreview = pickBestPreviewFormat(favoriteGifPreviewEntry?.media);const favoriteMemeCandidate =gifvMemes.length > 0 ? (gifvMemes[Math.floor(input.featuredFavoritePreviewSeed * gifvMemes.length)] ?? null) : null;- const favoriteMemePreview = favoriteMemeCandidate?.url ?? '';+ const favoriteMemePreview = favoriteMemeCandidate ? freshStoredUrl(favoriteMemeCandidate.url) : '';const usesFavoriteMemePreview = input.useSavedMediaForGifFavorites && input.favoriteGifs.length === 0;const favoriteTilePreview = usesFavoriteMemePreview? favoriteMemePreview- : favoriteGifPreview?.format.src || favoriteGifPreviewEntry?.proxy_url || favoriteGifPreviewEntry?.url || '';+ : freshStoredUrl(+ favoriteGifPreview?.format.src || favoriteGifPreviewEntry?.proxy_url || favoriteGifPreviewEntry?.url || '',+ );const favoriteTileProxyPreview = usesFavoriteMemePreview? favoriteTilePreview- : favoriteGifPreview?.format.proxy_src || favoriteGifPreviewEntry?.proxy_url || favoriteGifPreviewEntry?.url || '';+ : freshStoredUrl(+ favoriteGifPreview?.format.proxy_src ||+ favoriteGifPreviewEntry?.proxy_url ||+ favoriteGifPreviewEntry?.url ||+ '',+ );const favoriteTileContentType = (() => {if (usesFavoriteMemePreview) return favoriteMemeCandidate?.contentType ?? '';if (favoriteGifPreview) return inferFormatContentType(favoriteGifPreview.key);diff --git a/fluxer_app/src/features/channel/components/pickers/memes/MemeGridItem.tsx b/fluxer_app/src/features/channel/components/pickers/memes/MemeGridItem.tsxindex 0f6e14c93..ad06630d0 100644--- a/fluxer_app/src/features/channel/components/pickers/memes/MemeGridItem.tsx+++ b/fluxer_app/src/features/channel/components/pickers/memes/MemeGridItem.tsx@@ -13,6 +13,7 @@ import * as FavoriteMemeCommands from '@app/features/expressions/commands/Favoriimport {EditFavoriteMemeModal} from '@app/features/expressions/components/modals/EditFavoriteMemeModal';import type {FavoriteMeme} from '@app/features/expressions/models/FavoriteMeme';import {isKeyboardActivationKey} from '@app/features/input/utils/KeyboardUtils';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import {buildMediaProxyURL, buildStaticGifPreviewURL} from '@app/features/messaging/utils/MediaProxyUtils';import {ComponentBus} from '@app/features/platform/utils/ComponentBus';import * as ModalCommands from '@app/features/ui/commands/ModalCommands';@@ -113,10 +114,14 @@ export const MemeGridItem = observer(isHovering: isVideoPreviewActive,});const shouldRenderVideoPreview = !isAudio && isVideo && isVideoPreviewActive && shouldAnimateGif;- const thumbnailSrc = isGifImage && !shouldAnimateGif ? buildStaticGifPreviewURL(meme.url) : meme.url;+ const memeUrl = AttachmentUrlRefresher.fresh(meme.url, {refreshUnsigned: true});+ const thumbnailSrc =+ isGifImage && !shouldAnimateGif+ ? AttachmentUrlRefresher.fresh(buildStaticGifPreviewURL(meme.url), {refreshUnsigned: true})+ : memeUrl;const videoPreviewStartTime = meme.duration && meme.duration > 0 ? meme.duration / 2 : null;usePooledVideo({- src: shouldRenderVideoPreview ? meme.url : null,+ src: shouldRenderVideoPreview ? memeUrl : null,containerRef: videoContainerRef,videoPool,autoPlay: shouldRenderVideoPreview,@@ -219,7 +224,7 @@ export const MemeGridItem = observer(data-flx="channel.pickers.memes.meme-grid-item.div--2"><VideoPoster- src={meme.url}+ src={memeUrl}placeholder={meme.placeholder}data-flx="channel.pickers.memes.meme-grid-item.video-poster"/>diff --git a/fluxer_app/src/features/expressions/state/FavoriteGif.ts b/fluxer_app/src/features/expressions/state/FavoriteGif.tsindex 4a6366252..4c77f310d 100644--- a/fluxer_app/src/features/expressions/state/FavoriteGif.ts+++ b/fluxer_app/src/features/expressions/state/FavoriteGif.ts@@ -4,7 +4,9 @@ import {type FavoriteGifEntry,type FavoriteGifMediaFormat,slimFavoriteGifEntry,+ stripFavoriteGifEntrySignatures,} from '@app/features/channel/components/pickers/gif/FavoriteGifTypes';+import {stripAttachmentSignature} from '@app/features/messaging/utils/AttachmentCdnUrl';import {makeSyncedField} from '@app/features/user/state/SyncedField';import {FAVORITE_GIF_MAX_ENCODED_BYTES} from '@app/features/user/state/SyncedFieldBudget';import type {FavoriteGifMediaFormat as FavoriteGifMediaFormatProto} from '@fluxer/schema/src/gen/fluxer/user/preferences/v1/pickers_pb';@@ -91,25 +93,29 @@ class FavoriteGif {}hasUrl(url: string): boolean {- return this.favoriteGifs.some((entry) => entry.url === url);+ const target = stripAttachmentSignature(url);+ return this.favoriteGifs.some((entry) => stripAttachmentSignature(entry.url) === target);}findByUrl(url: string): FavoriteGifEntry | null {- return this.favoriteGifs.find((entry) => entry.url === url) ?? null;+ const target = stripAttachmentSignature(url);+ return this.favoriteGifs.find((entry) => stripAttachmentSignature(entry.url) === target) ?? null;}addEntry(entry: FavoriteGifEntry): void {- if (this.hasUrl(entry.url)) return;- this.favoriteGifs = [...this.favoriteGifs, entry];+ const stored = stripFavoriteGifEntrySignatures(entry);+ if (this.hasUrl(stored.url)) return;+ this.favoriteGifs = [...this.favoriteGifs, stored];}removeByUrl(url: string): void {if (!this.hasUrl(url)) return;- this.favoriteGifs = this.favoriteGifs.filter((entry) => entry.url !== url);+ const target = stripAttachmentSignature(url);+ this.favoriteGifs = this.favoriteGifs.filter((entry) => stripAttachmentSignature(entry.url) !== target);}replaceAll(entries: ReadonlyArray<FavoriteGifEntry>): void {- this.favoriteGifs = [...entries];+ this.favoriteGifs = entries.map(stripFavoriteGifEntrySignatures);}setSaveGifFavoritesAsSavedMedia(value: boolean): void {diff --git a/fluxer_app/src/features/gateway/transport/GatewayConnection.ts b/fluxer_app/src/features/gateway/transport/GatewayConnection.tsindex 39ab06352..1d701f49b 100644--- a/fluxer_app/src/features/gateway/transport/GatewayConnection.ts+++ b/fluxer_app/src/features/gateway/transport/GatewayConnection.ts@@ -28,6 +28,7 @@ import {selectGuildActivationTarget} from '@app/features/gateway/transport/Guildimport GuildMatureContentAgree from '@app/features/guild/state/GuildMatureContentAgree';import GuildMembers from '@app/features/member/state/GuildMembers';import MemberSearch from '@app/features/member/state/MemberSearch';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import Messages from '@app/features/messaging/state/MessagingMessages';import Navigation from '@app/features/navigation/state/Navigation';import SelectedGuild from '@app/features/navigation/state/SelectedGuild';@@ -587,6 +588,7 @@ class GatewayConnection {Presence.handleSessionInvalidated();Messages.handleSessionInvalidated();FavoriteMemes.reset();+ AttachmentUrlRefresher.reset();GuildMatureContentAgree.reset();Initialization.reset();MemberSearch.handleLogout();diff --git a/fluxer_app/src/features/messaging/components/markdown/renderers/LinkRenderer.tsx b/fluxer_app/src/features/messaging/components/markdown/renderers/LinkRenderer.tsxindex 39c880d23..b19da2e9b 100644--- a/fluxer_app/src/features/messaging/components/markdown/renderers/LinkRenderer.tsx+++ b/fluxer_app/src/features/messaging/components/markdown/renderers/LinkRenderer.tsx@@ -28,6 +28,7 @@ import {type RendererProps,} from '@app/features/messaging/components/markdown/renderers/RendererTypes';import {ExternalLinkWarningModal} from '@app/features/messaging/components/modals/ExternalLinkWarningModal';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import {openExternalUrlWithWarning} from '@app/features/messaging/utils/ExternalLinkUtils';import {goToMessage} from '@app/features/messaging/utils/MessageNavigator';import type {LinkNode} from '@app/features/messaging/utils/markdown/parser/Nodes';@@ -889,12 +890,16 @@ export const LinkRenderer = observer(function LinkRenderer({logger.warn('Invalid URL in link:', url);}}+ const href = AttachmentUrlRefresher.fresh(url);+ const warmAttachmentUrl = () => AttachmentUrlRefresher.warm(url);return (<FocusRing key={id} offset={-2} data-flx="messaging.markdown.renderers.link-renderer.focus-ring--2"><a- href={url}+ href={href}target={isInternal ? undefined : '_blank'}rel={isInternal ? undefined : 'noopener noreferrer'}+ onPointerEnter={warmAttachmentUrl}+ onFocus={warmAttachmentUrl}onClick={(e) => {e.stopPropagation();if (handleClick) {diff --git a/fluxer_app/src/features/messaging/components/modals/MediaViewerModal.tsx b/fluxer_app/src/features/messaging/components/modals/MediaViewerModal.tsxindex b25e9cd35..b9ff4d294 100644--- a/fluxer_app/src/features/messaging/components/modals/MediaViewerModal.tsx+++ b/fluxer_app/src/features/messaging/components/modals/MediaViewerModal.tsx@@ -18,6 +18,7 @@ import {getMediaViewerPortalRoot} from '@app/features/messaging/components/modalimport {useDeleteAttachment} from '@app/features/messaging/hooks/useDeleteAttachment';import {useMediaFavorite} from '@app/features/messaging/hooks/useMediaFavorite';import type {Message} from '@app/features/messaging/models/MessagingMessage';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import {formatAttachmentDate} from '@app/features/messaging/utils/AttachmentExpiryUtils';import {openExternalUrlWithWarning} from '@app/features/messaging/utils/ExternalLinkUtils';import {createDownloadHandler} from '@app/features/messaging/utils/FileDownloadUtils';@@ -273,13 +274,20 @@ const MobileMediaOptionsSheet: FC<MobileMediaOptionsSheetProps> = observer(funct/>);});+function withFreshViewerItemUrls(item: MediaViewerItem): MediaViewerItem {+ if (!item) return item;+ const src = AttachmentUrlRefresher.fresh(item.src);+ const originalSrc = AttachmentUrlRefresher.fresh(item.originalSrc);+ if (src === item.src && originalSrc === item.originalSrc) return item;+ return {...item, src, originalSrc};+}const MediaViewerModalComponent: FC = observer(() => {const {i18n} = useLingui();const {isOpen, items, currentIndex, channelId, messageId, message, sourceChannel, allowAttachmentDelete} =MediaViewer;const {enabled: isMobile} = MobileLayout;const [isMediaMenuOpen, setIsMediaMenuOpen] = useState(false);- const currentItem = items[currentIndex];+ const currentItem = withFreshViewerItemUrls(items[currentIndex]);const currentGifvIsActualGif = currentItem != null && isGifvRenderedAsImage(currentItem);useBottomSheetBackHandler(isOpen, MediaViewerCommands.closeMediaViewer);useEffect(() => {@@ -296,7 +304,10 @@ const MediaViewerModalComponent: FC = observer(() => {count === 2 ? [(currentIndex + 1) % count] : [(currentIndex - 1 + count) % count, (currentIndex + 1) % count];const unpinCallbacks = adjacentIndices.map((index) => {const item = items[index];- if (!item || !isViewerImageItem(item) || item.src.startsWith('blob:')) return () => {};+ if (!item) return () => {};+ AttachmentUrlRefresher.warm(item.src);+ AttachmentUrlRefresher.warm(item.originalSrc);+ if (!isViewerImageItem(item) || item.src.startsWith('blob:')) return () => {};return ImageCacheUtils.pinImage(buildViewerMediaURL(item));});return () => {diff --git a/fluxer_app/src/features/messaging/hooks/useAttachmentRefreshOnError.ts b/fluxer_app/src/features/messaging/hooks/useAttachmentRefreshOnError.tsnew file mode 100644index 000000000..7d84622b4--- /dev/null+++ b/fluxer_app/src/features/messaging/hooks/useAttachmentRefreshOnError.ts@@ -0,0 +1,14 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';+import {useCallback, useRef} from 'react';++export function useAttachmentRefreshOnError(src: string | undefined): () => void {+ const refreshedSourceRef = useRef<string | null>(null);+ return useCallback(() => {+ if (src === undefined || src.length === 0) return;+ if (refreshedSourceRef.current === src) return;+ refreshedSourceRef.current = src;+ void AttachmentUrlRefresher.refresh(src, {force: true});+ }, [src]);+}diff --git a/fluxer_app/src/features/messaging/hooks/useMediaLoading.ts b/fluxer_app/src/features/messaging/hooks/useMediaLoading.tsindex e31bcf2eb..a2837bbed 100644--- a/fluxer_app/src/features/messaging/hooks/useMediaLoading.ts+++ b/fluxer_app/src/features/messaging/hooks/useMediaLoading.ts@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport DeveloperOptions from '@app/features/devtools/state/DeveloperOptions';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import * as ImageCacheUtils from '@app/features/messaging/utils/ImageCacheUtils';import {decodeThumbHashDataURL} from '@app/features/messaging/utils/ThumbHashUtils';import {type SyntheticEvent, useCallback, useLayoutEffect, useMemo, useRef, useState} from 'react';@@ -123,6 +124,7 @@ export function useMediaLoading(const {enabled = true} = options;const shouldForcePlaceholder = DeveloperOptions.forceRenderPlaceholders || DeveloperOptions.forceMediaLoading;const mediaElementRef = useRef<MediaLoadingElement | null>(null);+ const refreshedSourceRef = useRef<string | null>(null);const loadEnabled = enabled && src.length > 0 && !shouldForcePlaceholder;const sourceIdentity = useMemo<MediaLoadingSourceIdentity>(() => ({src, loadEnabled}), [loadEnabled, src]);const committedSourceIdentityRef = useRef(sourceIdentity);@@ -184,6 +186,10 @@ export function useMediaLoading(if (mediaElementRef.current !== element) return;if (!mediaElementMatchesSource(element, src)) return;ImageCacheUtils.forgetImage(src);+ if (refreshedSourceRef.current !== src) {+ refreshedSourceRef.current = src;+ void AttachmentUrlRefresher.refresh(src, {force: true});+ }setSourceState((currentState) => {if (currentState.src === src && currentState.loadEnabled === loadEnabled && currentState.loaded) {return {...currentState, cached: false};diff --git a/fluxer_app/src/features/messaging/hooks/useNearViewport.ts b/fluxer_app/src/features/messaging/hooks/useNearViewport.tsindex ec352d879..505ce64c5 100644--- a/fluxer_app/src/features/messaging/hooks/useNearViewport.ts+++ b/fluxer_app/src/features/messaging/hooks/useNearViewport.ts@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {stripAttachmentSignature} from '@app/features/messaging/utils/AttachmentCdnUrl';import {observeIntersection} from '@app/features/platform/utils/SharedIntersectionObserver';import {LRUCache} from 'lru-cache';import {createContext, useCallback, useContext, useEffect, useState} from 'react';@@ -23,6 +24,11 @@ export function resolveObserverRoot(resolve: ScrollSurfaceResolver, element: Elereturn surface;}+export function resolveViewportKey(rememberKey: string | null | undefined): string | null {+ if (!rememberKey) return null;+ return stripAttachmentSignature(rememberKey);+}+interface UseNearViewportOptions {disabled?: boolean;rememberKey?: string | null;@@ -37,8 +43,9 @@ export function useNearViewport<T extends Element>({threshold = 0,}: UseNearViewportOptions = {}): {ref: (node: T | null) => void; isNearViewport: boolean} {const resolveScrollSurface = useContext(NearViewportSurfaceContext);+ const viewportKey = resolveViewportKey(rememberKey);const loadImmediately = disabled || typeof IntersectionObserver === 'undefined';- const wasRemembered = rememberKey ? rememberedViewportKeys.has(rememberKey) : false;+ const wasRemembered = viewportKey ? rememberedViewportKeys.has(viewportKey) : false;const [element, setElement] = useState<T | null>(null);const [isNearViewport, setIsNearViewport] = useState(loadImmediately || wasRemembered);const ref = useCallback((node: T | null) => {@@ -50,13 +57,13 @@ export function useNearViewport<T extends Element>({}}, [disabled]);useEffect(() => {- if (!isNearViewport || !rememberKey) return;+ if (!isNearViewport || !viewportKey) return;if (disabled || typeof IntersectionObserver === 'undefined') return;- rememberedViewportKeys.set(rememberKey, true);- }, [disabled, isNearViewport, rememberKey]);+ rememberedViewportKeys.set(viewportKey, true);+ }, [disabled, isNearViewport, viewportKey]);useEffect(() => {if (disabled || isNearViewport || !element) return undefined;- if (rememberKey && rememberedViewportKeys.has(rememberKey)) {+ if (viewportKey && rememberedViewportKeys.has(viewportKey)) {setIsNearViewport(true);return undefined;}@@ -70,14 +77,14 @@ export function useNearViewport<T extends Element>({(entry) => {if (!entry.isIntersecting && entry.intersectionRatio <= 0) return;stopObserving();- if (rememberKey) {- rememberedViewportKeys.set(rememberKey, true);+ if (viewportKey) {+ rememberedViewportKeys.set(viewportKey, true);}setIsNearViewport(true);},{root: resolveObserverRoot(resolveScrollSurface, element), rootMargin, threshold},);return stopObserving;- }, [disabled, element, isNearViewport, rememberKey, resolveScrollSurface, rootMargin, threshold]);+ }, [disabled, element, isNearViewport, viewportKey, resolveScrollSurface, rootMargin, threshold]);return {ref, isNearViewport};}diff --git a/fluxer_app/src/features/messaging/state/AttachmentUrlRefresher.ts b/fluxer_app/src/features/messaging/state/AttachmentUrlRefresher.tsnew file mode 100644index 000000000..239a4b861--- /dev/null+++ b/fluxer_app/src/features/messaging/state/AttachmentUrlRefresher.ts@@ -0,0 +1,242 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++import {Endpoints} from '@app/features/app/constants/Endpoints';+import RuntimeConfig from '@app/features/app/state/RuntimeConfig';+import {+ ATTACHMENT_URL_REFRESH_MARGIN_MS,+ applyAttachmentSignature,+ attachmentCacheKey,+ attachmentUrlNeedsRefresh,+ isAttachmentCdnUrl,+ readAttachmentSignatureFields,+ readAttachmentUrlSignature,+ stripAttachmentSignature,+} from '@app/features/messaging/utils/AttachmentCdnUrl';+import {resolveRetryAfterMs} from '@app/features/messaging/utils/RetryAfterUtils';+import {http} from '@app/features/platform/transport/RestTransport';+import {HttpError} from '@app/features/platform/types/EndpointError';+import {Logger} from '@app/features/platform/utils/AppLogger';+import type {RefreshAttachmentUrlsResponse} from '@fluxer/schema/src/domains/message/AttachmentSchemas';+import {makeAutoObservable, runInAction, untracked} from 'mobx';++const logger = new Logger('AttachmentUrlRefresher');++const FLUSH_WINDOW_MS = 16;+const MAX_URLS_PER_REQUEST = 50;+const UNSIGNED_RESULT_COOLDOWN_MS = 600_000;+const TRANSPORT_ERROR_COOLDOWN_MS = 30_000;+const RATE_LIMIT_FALLBACK_COOLDOWN_MS = 10_000;++type CacheEntry = {kind: 'signature'; fields: string; goodUntilMs: number} | {kind: 'blocked'; untilMs: number};++interface QueuedRequest {+ url: string;+ resolvers: Array<(fields: string | null) => void>;+}++class AttachmentUrlRefresher {+ entries = new Map<string, CacheEntry>();+ readonly queued = new Map<string, QueuedRequest>();+ readonly inFlight = new Map<string, Promise<string | null>>();+ flushTimer: ReturnType<typeof setTimeout> | null = null;+ cooldownUntilMs = 0;+ generation = 0;+ revision = 0;++ constructor() {+ makeAutoObservable(+ this,+ {+ queued: false,+ inFlight: false,+ flushTimer: false,+ cooldownUntilMs: false,+ generation: false,+ },+ {autoBind: true},+ );+ }++ fresh(url: string, options: {refreshUnsigned?: boolean} = {}): string {+ if (!url || !isAttachmentCdnUrl(url, RuntimeConfig.mediaEndpoint)) return url;+ const signature = readAttachmentUrlSignature(url);+ const nowMs = Date.now();+ if (signature !== null && signature.expiresAtMs > nowMs + ATTACHMENT_URL_REFRESH_MARGIN_MS) return url;+ const key = attachmentCacheKey(url);+ if (key === null) return url;+ const entry = this.entries.get(key);+ if (entry?.kind === 'signature' && entry.goodUntilMs > nowMs) {+ return applyAttachmentSignature(url, entry.fields);+ }+ if (signature === null && options.refreshUnsigned !== true) return url;+ if (entry?.kind === 'blocked' && entry.untilMs > nowMs) return url;+ this.enqueue(key, url, nowMs);+ return url;+ }++ warm(url: string): void {+ if (!url || !isAttachmentCdnUrl(url, RuntimeConfig.mediaEndpoint)) return;+ const nowMs = Date.now();+ if (!attachmentUrlNeedsRefresh(url, nowMs)) return;+ const key = attachmentCacheKey(url);+ if (key === null) return;+ const entry = untracked(() => this.entries.get(key));+ if (entry?.kind === 'signature' && entry.goodUntilMs > nowMs) return;+ if (entry?.kind === 'blocked' && entry.untilMs > nowMs) return;+ this.enqueue(key, url, nowMs);+ }++ async refresh(url: string, options: {force?: boolean} = {}): Promise<string> {+ if (!url || !isAttachmentCdnUrl(url, RuntimeConfig.mediaEndpoint)) return url;+ const key = attachmentCacheKey(url);+ if (key === null) return url;+ const nowMs = Date.now();+ const entry = untracked(() => this.entries.get(key));+ if (entry?.kind === 'signature' && entry.goodUntilMs > nowMs) {+ return applyAttachmentSignature(url, entry.fields);+ }+ if (options.force !== true) {+ if (entry?.kind === 'blocked' && entry.untilMs > nowMs) return url;+ if (!attachmentUrlNeedsRefresh(url, nowMs)) return url;+ }+ const fields = await this.request(key, url, nowMs);+ return fields === null ? url : applyAttachmentSignature(url, fields);+ }++ reset(): void {+ this.generation += 1;+ this.revision += 1;+ this.entries = new Map();+ for (const request of this.queued.values()) {+ for (const resolve of request.resolvers) resolve(null);+ }+ this.queued.clear();+ this.inFlight.clear();+ this.cooldownUntilMs = 0;+ if (this.flushTimer !== null) {+ clearTimeout(this.flushTimer);+ this.flushTimer = null;+ }+ }++ private forgetExpired(nowMs: number): void {+ for (const [key, entry] of this.entries) {+ const untilMs = entry.kind === 'signature' ? entry.goodUntilMs : entry.untilMs;+ if (untilMs <= nowMs) this.entries.delete(key);+ }+ }++ private enqueue(key: string, url: string, nowMs: number): void {+ if (this.inFlight.has(key)) return;+ if (nowMs < this.cooldownUntilMs) return;+ if (this.queued.has(key)) return;+ this.queued.set(key, {url: stripAttachmentSignature(url), resolvers: []});+ this.scheduleFlush();+ }++ private request(key: string, url: string, nowMs: number): Promise<string | null> {+ const existing = this.inFlight.get(key);+ if (existing) return existing;+ if (nowMs < this.cooldownUntilMs) return Promise.resolve(null);+ const queued = this.queued.get(key) ?? {url: stripAttachmentSignature(url), resolvers: []};+ this.queued.set(key, queued);+ this.scheduleFlush();+ return new Promise<string | null>((resolve) => {+ queued.resolvers.push(resolve);+ });+ }++ private scheduleFlush(): void {+ if (this.flushTimer !== null) return;+ this.flushTimer = setTimeout(() => {+ this.flushTimer = null;+ void this.flush();+ }, FLUSH_WINDOW_MS);+ }++ private async flush(): Promise<void> {+ if (this.queued.size === 0) return;+ const nowMs = Date.now();+ if (nowMs < this.cooldownUntilMs) {+ this.drainQueue();+ return;+ }+ const batch = Array.from(this.queued.entries()).slice(0, MAX_URLS_PER_REQUEST);+ for (const [key] of batch) this.queued.delete(key);+ if (this.queued.size > 0) this.scheduleFlush();+ const generation = this.generation;+ const send = this.send(batch, generation);+ for (const [key] of batch) {+ this.inFlight.set(+ key,+ send.then((outcome) => outcome.get(key) ?? null),+ );+ }+ const outcome = await send;+ for (const [key, request] of batch) {+ if (generation === this.generation) this.inFlight.delete(key);+ const fields = outcome.get(key) ?? null;+ for (const resolve of request.resolvers) resolve(fields);+ }+ }++ private async send(batch: Array<[string, QueuedRequest]>, generation: number): Promise<Map<string, string>> {+ const resolved = new Map<string, string>();+ try {+ const response = await http.post<RefreshAttachmentUrlsResponse>(Endpoints.ATTACHMENTS_REFRESH_URLS, {+ body: {attachment_urls: batch.map(([, request]) => request.url)},+ });+ if (generation !== this.generation) return resolved;+ const byOriginal = new Map(+ (response.body?.refreshed_urls ?? []).map((entry) => [entry.original, entry.refreshed]),+ );+ const nowMs = Date.now();+ const updates: Array<[string, CacheEntry]> = [];+ for (const [key, request] of batch) {+ const refreshed = byOriginal.get(request.url);+ const fields = refreshed === undefined ? null : readAttachmentSignatureFields(refreshed);+ const expiry = refreshed === undefined ? null : readAttachmentUrlSignature(refreshed);+ if (fields === null || expiry === null) {+ updates.push([key, {kind: 'blocked', untilMs: nowMs + UNSIGNED_RESULT_COOLDOWN_MS}]);+ continue;+ }+ resolved.set(key, fields);+ updates.push([+ key,+ {kind: 'signature', fields, goodUntilMs: expiry.expiresAtMs - ATTACHMENT_URL_REFRESH_MARGIN_MS},+ ]);+ }+ runInAction(() => {+ this.forgetExpired(nowMs);+ for (const [key, entry] of updates) this.entries.set(key, entry);+ this.revision += 1;+ });+ return resolved;+ } catch (error) {+ if (generation !== this.generation) return resolved;+ const nowMs = Date.now();+ if (error instanceof HttpError && error.status === 429) {+ this.cooldownUntilMs = nowMs + (resolveRetryAfterMs(error) ?? RATE_LIMIT_FALLBACK_COOLDOWN_MS);+ return resolved;+ }+ logger.warn('Failed to refresh attachment URLs', error);+ runInAction(() => {+ this.forgetExpired(nowMs);+ for (const [key] of batch) {+ this.entries.set(key, {kind: 'blocked', untilMs: nowMs + TRANSPORT_ERROR_COOLDOWN_MS});+ }+ this.revision += 1;+ });+ return resolved;+ }+ }++ private drainQueue(): void {+ for (const request of this.queued.values()) {+ for (const resolve of request.resolvers) resolve(null);+ }+ this.queued.clear();+ }+}++export default new AttachmentUrlRefresher();diff --git a/fluxer_app/src/features/messaging/utils/AttachmentCdnUrl.ts b/fluxer_app/src/features/messaging/utils/AttachmentCdnUrl.tsnew file mode 100644index 000000000..901a6eed0--- /dev/null+++ b/fluxer_app/src/features/messaging/utils/AttachmentCdnUrl.ts@@ -0,0 +1,198 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++export const ATTACHMENT_URL_REFRESH_MARGIN_MS = 3_600_000;++export interface EndpointInfo {+ basePath: string;+ origin: string;+}++export interface AttachmentUrlSignature {+ expiresAtMs: number;+}++const ATTACHMENT_PATH_SEGMENT = '/attachments/';+const WINDOW_HEX_PATTERN = /^[0-9a-f]{8}$/u;+const MAC_HEX_PATTERN = /^[0-9a-f]{64}$/u;+const DIGITS_PATTERN = /^[0-9]+$/u;+const PLUS_PATTERN = /\+/gu;+const DATA_PACKAGE_USE_CASE = 'dp';+const DATA_PACKAGE_EXPIRY = '0';++export function parseEndpoint(endpoint: string): EndpointInfo | null {+ if (!endpoint) return null;+ try {+ const parsedEndpoint = new URL(endpoint);+ const basePath =+ parsedEndpoint.pathname.length > 1 && parsedEndpoint.pathname.endsWith('/')+ ? parsedEndpoint.pathname.slice(0, -1)+ : parsedEndpoint.pathname || '/';+ return {+ basePath,+ origin: parsedEndpoint.origin,+ };+ } catch {+ return null;+ }+}++export function isUrlOnEndpoint(targetUrl: URL, endpoint: EndpointInfo): boolean {+ if (targetUrl.origin !== endpoint.origin) return false;+ if (endpoint.basePath === '/') return true;+ return targetUrl.pathname === endpoint.basePath || targetUrl.pathname.startsWith(`${endpoint.basePath}/`);+}++export function isAttachmentCdnUrl(url: string, mediaEndpoint: string): boolean {+ if (!url.includes(ATTACHMENT_PATH_SEGMENT)) return false;+ const endpoint = parseEndpoint(mediaEndpoint);+ if (endpoint === null) return false;+ let parsed: URL;+ try {+ parsed = new URL(url);+ } catch {+ return false;+ }+ if (parsed.username !== '' || parsed.password !== '') return false;+ if (parsed.origin !== endpoint.origin) return false;+ const prefix = endpoint.basePath === '/' ? ATTACHMENT_PATH_SEGMENT : `${endpoint.basePath}${ATTACHMENT_PATH_SEGMENT}`;+ if (!parsed.pathname.startsWith(prefix)) return false;+ const segments = parsed.pathname.slice(prefix.length).split('/');+ if (segments.length < 3) return false;+ const [channelId, attachmentId] = segments;+ return (+ DIGITS_PATTERN.test(channelId) &&+ DIGITS_PATTERN.test(attachmentId) &&+ segments.slice(2).every((segment) => segment.length > 0)+ );+}++export function attachmentCacheKey(url: string): string | null {+ try {+ const parsed = new URL(url);+ return `${parsed.origin}${parsed.pathname}`;+ } catch {+ return null;+ }+}++interface SplitUrl {+ base: string;+ query: string;+ fragment: string;+}++function splitUrl(url: string): SplitUrl {+ const fragmentIndex = url.indexOf('#');+ const head = fragmentIndex < 0 ? url : url.slice(0, fragmentIndex);+ const fragment = fragmentIndex < 0 ? '' : url.slice(fragmentIndex);+ const queryIndex = head.indexOf('?');+ if (queryIndex < 0) return {base: head, query: '', fragment};+ return {base: head.slice(0, queryIndex), query: head.slice(queryIndex + 1), fragment};+}++function formDecodeName(name: string): string {+ try {+ return decodeURIComponent(name.replace(PLUS_PATTERN, ' '));+ } catch {+ return name;+ }+}++function fieldName(field: string): string {+ const separator = field.indexOf('=');+ return formDecodeName(separator < 0 ? field : field.slice(0, separator));+}++function fieldValue(field: string): string {+ const separator = field.indexOf('=');+ return separator < 0 ? '' : field.slice(separator + 1);+}++type SignatureFieldName = 'ex' | 'is' | 'hm' | 'uc';++function isSignatureFieldName(name: string): name is SignatureFieldName {+ return name === 'ex' || name === 'is' || name === 'hm' || name === 'uc';+}++function preservedFields(query: string): Array<string> {+ if (query.length === 0) return [];+ return query.split('&').filter((field) => {+ if (field.length === 0 || field === '=') return false;+ return !isSignatureFieldName(fieldName(field));+ });+}++interface ScannedSignature {+ ex: Array<string>;+ is: Array<string>;+ hm: Array<string>;+ uc: Array<string>;+}++function scanSignatureFields(url: string): ScannedSignature {+ const found: ScannedSignature = {ex: [], is: [], hm: [], uc: []};+ const {query} = splitUrl(url);+ if (query.length === 0) return found;+ for (const field of query.split('&')) {+ if (field.length === 0) continue;+ const name = fieldName(field);+ if (!isSignatureFieldName(name)) continue;+ found[name].push(fieldValue(field));+ }+ return found;+}++interface AttachmentSignatureFields {+ ex: string;+ is: string;+ hm: string;+ uc: string;+ expiresAtMs: number;+}++function parseAttachmentSignature(url: string): AttachmentSignatureFields | null {+ const found = scanSignatureFields(url);+ if (found.ex.length !== 1 || found.is.length !== 1 || found.hm.length !== 1 || found.uc.length > 1) return null;+ const ex = found.ex[0];+ const is = found.is[0];+ const hm = found.hm[0];+ if (!WINDOW_HEX_PATTERN.test(is) || !MAC_HEX_PATTERN.test(hm)) return null;+ if (found.uc.length === 1) {+ if (found.uc[0] !== DATA_PACKAGE_USE_CASE || ex !== DATA_PACKAGE_EXPIRY) return null;+ return {ex, is, hm, uc: DATA_PACKAGE_USE_CASE, expiresAtMs: Number.POSITIVE_INFINITY};+ }+ if (!WINDOW_HEX_PATTERN.test(ex)) return null;+ return {ex, is, hm, uc: '', expiresAtMs: Number.parseInt(ex, 16) * 1000};+}++export function readAttachmentUrlSignature(url: string): AttachmentUrlSignature | null {+ const signature = parseAttachmentSignature(url);+ return signature === null ? null : {expiresAtMs: signature.expiresAtMs};+}++export function readAttachmentSignatureFields(url: string): string | null {+ const signature = parseAttachmentSignature(url);+ if (signature === null) return null;+ const fields = `ex=${signature.ex}&is=${signature.is}&hm=${signature.hm}`;+ return signature.uc === '' ? fields : `${fields}&uc=${signature.uc}`;+}++export function stripAttachmentSignature(url: string): string {+ const {base, query, fragment} = splitUrl(url);+ const preserved = preservedFields(query);+ if (preserved.length === 0) return `${base}${fragment}`;+ return `${base}?${preserved.join('&')}${fragment}`;+}++export function applyAttachmentSignature(url: string, signatureFields: string): string {+ const {base, query, fragment} = splitUrl(url);+ const preserved = preservedFields(query);+ if (preserved.length === 0) return `${base}?${signatureFields}${fragment}`;+ return `${base}?${signatureFields}&${preserved.join('&')}${fragment}`;+}++export function attachmentUrlNeedsRefresh(url: string, nowMs: number): boolean {+ const signature = readAttachmentUrlSignature(url);+ if (signature === null) return true;+ return signature.expiresAtMs <= nowMs + ATTACHMENT_URL_REFRESH_MARGIN_MS;+}diff --git a/fluxer_app/src/features/messaging/utils/FileDownloadUtils.ts b/fluxer_app/src/features/messaging/utils/FileDownloadUtils.tsindex d803e920c..efdcaccf0 100644--- a/fluxer_app/src/features/messaging/utils/FileDownloadUtils.ts+++ b/fluxer_app/src/features/messaging/utils/FileDownloadUtils.ts@@ -1,6 +1,8 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport RuntimeConfig from '@app/features/app/state/RuntimeConfig';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';+import {isUrlOnEndpoint, parseEndpoint} from '@app/features/messaging/utils/AttachmentCdnUrl';import {isMobileOrTabletUserAgent} from '@app/features/platform/notifications/NotificationAlertOptions';import {supportsShowSaveFilePicker} from '@app/features/platform/types/Browser';import {Logger} from '@app/features/platform/utils/AppLogger';@@ -10,34 +12,6 @@ const logger = new Logger('FileDownloadUtils');type MediaType = 'image' | 'gif' | 'video' | 'audio' | 'file';-interface EndpointInfo {- basePath: string;- origin: string;-}--function parseEndpoint(endpoint: string): EndpointInfo | null {- if (!endpoint) return null;- try {- const parsedEndpoint = new URL(endpoint);- const basePath =- parsedEndpoint.pathname.length > 1 && parsedEndpoint.pathname.endsWith('/')- ? parsedEndpoint.pathname.slice(0, -1)- : parsedEndpoint.pathname || '/';- return {- basePath,- origin: parsedEndpoint.origin,- };- } catch {- return null;- }-}--function isUrlOnEndpoint(targetUrl: URL, endpoint: EndpointInfo): boolean {- if (targetUrl.origin !== endpoint.origin) return false;- if (endpoint.basePath === '/') return true;- return targetUrl.pathname === endpoint.basePath || targetUrl.pathname.startsWith(`${endpoint.basePath}/`);-}-function appendMediaProxyDownloadParam(src: string): string {let parsedSrc: URL;try {@@ -221,23 +195,21 @@ function downloadViaAnchor(src: string, suggestedName: string, options?: {appendexport async function downloadFile(src: string, type: MediaType, providedFilename?: string): Promise<void> {if (!src) return;+ const suggestedName = deriveSuggestedName(src, type, providedFilename);+ const target = await AttachmentUrlRefresher.refresh(src);if (isElectron()) {try {- const outcome = await downloadWithNative({- url: src,- suggestedName: deriveSuggestedName(src, type, providedFilename),- });+ const outcome = await downloadWithNative({url: target, suggestedName});if (outcome !== 'unavailable') return;} catch (error) {logger.warn('Native download failed', error);return;}}- const suggestedName = deriveSuggestedName(src, type, providedFilename);- if (await downloadViaFileSystemAccess(src, suggestedName, type)) return;- if (await downloadViaFetchBlob(src, suggestedName)) return;- if (downloadViaAnchor(src, suggestedName)) return;- await openExternalUrl(appendMediaProxyDownloadParam(src));+ if (await downloadViaFileSystemAccess(target, suggestedName, type)) return;+ if (await downloadViaFetchBlob(target, suggestedName)) return;+ if (downloadViaAnchor(target, suggestedName)) return;+ await openExternalUrl(appendMediaProxyDownloadParam(target));}export function createDownloadHandler(src: string, type: MediaType, providedFilename?: string) {diff --git a/fluxer_app/src/features/messaging/utils/SpoilerUtils.ts b/fluxer_app/src/features/messaging/utils/SpoilerUtils.tsindex a7bc6cc39..aa3df2337 100644--- a/fluxer_app/src/features/messaging/utils/SpoilerUtils.ts+++ b/fluxer_app/src/features/messaging/utils/SpoilerUtils.ts@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+import {stripAttachmentSignature} from '@app/features/messaging/utils/AttachmentCdnUrl';import Permission from '@app/features/permissions/state/Permission';import UserSettings from '@app/features/user/state/UserSettings';import {Permissions} from '@fluxer/constants/src/ChannelConstants';@@ -49,7 +50,7 @@ export function canonicalizeMediaUrl(url: string | null | undefined): string | nif (youtubeId) {return `youtube:${youtubeId}`;}- return parsed.href.replace(/\/$/, '');+ return stripAttachmentSignature(parsed.href).replace(/\/$/, '');} catch {return null;}diff --git a/fluxer_app/src/features/ui/action_menu/items/MediaMenuData.tsx b/fluxer_app/src/features/ui/action_menu/items/MediaMenuData.tsxindex 4458266c3..88f451c5d 100644--- a/fluxer_app/src/features/ui/action_menu/items/MediaMenuData.tsx+++ b/fluxer_app/src/features/ui/action_menu/items/MediaMenuData.tsx@@ -18,6 +18,7 @@ import {} from '@app/features/i18n/utils/CommonMessageDescriptors';import {EditAltTextModal} from '@app/features/messaging/components/modals/EditAltTextModal';import type {Message} from '@app/features/messaging/models/MessagingMessage';+import AttachmentUrlRefresher from '@app/features/messaging/state/AttachmentUrlRefresher';import {createDownloadHandler} from '@app/features/messaging/utils/FileDownloadUtils';import {buildMediaProxyURL, stripMediaProxyParams} from '@app/features/messaging/utils/MediaProxyUtils';import Permission from '@app/features/permissions/state/Permission';@@ -429,17 +430,18 @@ export async function copyMediaToClipboard({);return;}+ const freshSrc = await AttachmentUrlRefresher.refresh(originalSrc);if (type === 'file') {- await TextCopyCommands.copy(i18n, originalSrc, true);+ await TextCopyCommands.copy(i18n, freshSrc, true);ToastCommands.createToast({type: 'success', children: i18n._(LINK_COPIED_TO_CLIPBOARD_DESCRIPTOR)});return;}- const baseProxyURL = proxyURL ? stripMediaProxyParams(proxyURL) : null;+ const baseProxyURL = proxyURL ? await AttachmentUrlRefresher.refresh(stripMediaProxyParams(proxyURL)) : null;const clipboardFileMediaType = getClipboardFileMediaType(type);if (clipboardFileMediaType) {const electronApi = getElectronAPI();if (!electronApi?.clipboardWriteFile) {- await TextCopyCommands.copy(i18n, originalSrc, true);+ await TextCopyCommands.copy(i18n, freshSrc, true);ToastCommands.createToast({type: 'success', children: i18n._(URL_COPIED_TO_CLIPBOARD_DESCRIPTOR)});return;}@@ -451,7 +453,7 @@ export async function copyMediaToClipboard({timeout: 0,});const result = await electronApi.clipboardWriteFile({- url: baseProxyURL || originalSrc,+ url: baseProxyURL || freshSrc,suggestedName: defaultName,mediaType: clipboardFileMediaType,});@@ -470,7 +472,7 @@ export async function copyMediaToClipboard({const urlsToTry: Array<string> = [];if (baseProxyURL) urlsToTry.push(buildMediaProxyURL(baseProxyURL, {format: 'png'}));if (baseProxyURL) urlsToTry.push(baseProxyURL);- urlsToTry.push(originalSrc);+ urlsToTry.push(freshSrc);let toastId: string | null = null;try {toastId = ToastCommands.createToast({@@ -494,7 +496,7 @@ export async function copyMediaToClipboard({} catch (error) {logger.error('Failed to copy image to clipboard:', error);if (toastId) ToastCommands.destroyToast(toastId);- await TextCopyCommands.copy(i18n, originalSrc, true);+ await TextCopyCommands.copy(i18n, freshSrc, true);ToastCommands.createToast({type: 'success',children: i18n._(URL_COPIED_TO_CLIPBOARD_DESCRIPTOR),@@ -511,7 +513,7 @@ export async function copyMediaLinkToClipboard({i18n, originalSrc}: {i18n: I18n;);return;}- await TextCopyCommands.copy(i18n, originalSrc, true);+ await TextCopyCommands.copy(i18n, await AttachmentUrlRefresher.refresh(originalSrc), true);ToastCommands.createToast({type: 'success',children: i18n._(LINK_COPIED_TO_CLIPBOARD_DESCRIPTOR),diff --git a/fluxer_app/src/features/ui/utils/NativeUtils.ts b/fluxer_app/src/features/ui/utils/NativeUtils.tsindex b4e1d61c8..c1be6bc6a 100644--- a/fluxer_app/src/features/ui/utils/NativeUtils.ts+++ b/fluxer_app/src/features/ui/utils/NativeUtils.ts@@ -196,20 +196,26 @@ const getSafeExternalUrl = (href: string | null): string | null => {}};+async function refreshAttachmentUrl(url: string): Promise<string> {+ const {default: AttachmentUrlRefresher} = await import('@app/features/messaging/state/AttachmentUrlRefresher');+ return AttachmentUrlRefresher.refresh(url);+}+export async function openExternalUrl(url: string, target: string = '_blank') {const safeUrl = getSafeExternalUrl(url);if (!safeUrl) return;+ const refreshedUrl = await refreshAttachmentUrl(safeUrl);const electronApi = getElectronAPI();if (electronApi) {try {- await electronApi.openExternal(safeUrl);+ await electronApi.openExternal(refreshedUrl);return;} catch (error) {logger.error(' Failed to open external URL via Electron', error);return;}}- window.open(safeUrl, target, 'noopener,noreferrer');+ window.open(refreshedUrl, target, 'noopener,noreferrer');}interface ExternalLinkClickEvent {diff --git a/fluxer_app/src/features/voice/components/media_player/components/AudioPlayer.tsx b/fluxer_app/src/features/voice/components/media_player/components/AudioPlayer.tsxindex de8713082..5ff507785 100644--- a/fluxer_app/src/features/voice/components/media_player/components/AudioPlayer.tsx+++ b/fluxer_app/src/features/voice/components/media_player/components/AudioPlayer.tsx@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-laterimport {PAUSE_DESCRIPTOR, PLAY_DESCRIPTOR} from '@app/features/i18n/utils/CommonMessageDescriptors';+import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError';import FocusRing from '@app/features/ui/focus_ring/FocusRing';import {Tooltip} from '@app/features/ui/tooltip/Tooltip';import styles from '@app/features/voice/components/media_player/AudioPlayer.module.css';@@ -77,6 +78,7 @@ export function AudioPlayer({});const {volume, isMuted, setVolume, toggleMute} = useMediaVolume({mediaRef});const {escalateToMetadata, sourceAttribute, preloadAttribute} = useMetadataPreload(src, hasStarted);+ const handleMediaError = useAttachmentRefreshOnError(src);useEffect(() => {if (hasStarted && pendingPlayRef.current) {const timer = setTimeout(() => {@@ -210,6 +212,7 @@ export function AudioPlayer({ref={mediaRef as React.RefObject<HTMLAudioElement>}src={sourceAttribute}preload={preloadAttribute}+ onError={handleMediaError}data-flx="voice.media-player.audio-player.audio"/>{title && (diff --git a/fluxer_app/src/features/voice/components/media_player/components/InlineAudioPlayer.tsx b/fluxer_app/src/features/voice/components/media_player/components/InlineAudioPlayer.tsxindex 3fa6137b6..146519950 100644--- a/fluxer_app/src/features/voice/components/media_player/components/InlineAudioPlayer.tsx+++ b/fluxer_app/src/features/voice/components/media_player/components/InlineAudioPlayer.tsx@@ -8,6 +8,7 @@ import {PLAY_DESCRIPTOR,REMOVE_FROM_FAVORITES_DESCRIPTOR,} from '@app/features/i18n/utils/CommonMessageDescriptors';+import {useAttachmentRefreshOnError} from '@app/features/messaging/hooks/useAttachmentRefreshOnError';import {formatFileSize} from '@app/features/messaging/utils/FileUtils';import {remFromPx} from '@app/features/theme/layout/RemFromPx';import FocusRing from '@app/features/ui/focus_ring/FocusRing';@@ -111,6 +112,7 @@ export function InlineAudioPlayer({mediaRef,});const {escalateToMetadata, sourceAttribute, preloadAttribute} = useMetadataPreload(src, hasStarted);+ const handleMediaError = useAttachmentRefreshOnError(src);const displayDuration = duration > 0 ? duration : (initialDuration ?? 0);useLayoutEffect(() => {const media = mediaRef.current;@@ -205,6 +207,7 @@ export function InlineAudioPlayer({ref={mediaRef as React.RefObject<HTMLAudioElement>}src={sourceAttribute}preload={preloadAttribute}+ onError={handleMediaError}data-flx="voice.media-player.inline-audio-player.audio"><track kind="captions" data-flx="voice.media-player.inline-audio-player.track" />@@ -332,6 +335,7 @@ export function InlineAudioPlayer({ref={mediaRef as React.RefObject<HTMLAudioElement>}src={sourceAttribute}preload={preloadAttribute}+ onError={handleMediaError}data-flx="voice.media-player.inline-audio-player.audio--2"><track kind="captions" data-flx="voice.media-player.inline-audio-player.track--2" />diff --git a/fluxer_common/Cargo.toml b/fluxer_common/Cargo.tomlindex eb0f9f7e5..eacbcd6ae 100644--- a/fluxer_common/Cargo.toml+++ b/fluxer_common/Cargo.toml@@ -17,6 +17,7 @@ tempfile = "3.27.0"time = { version = "0.3.55", features = ["formatting", "macros", "parsing"] }tracing = "0.1.44"base64 = "0.23"+hex = "0.4.3"hmac = "0.13.0"sha2 = "0.11.0"thiserror = "2"diff --git a/fluxer_common/src/attachment_url_signature.rs b/fluxer_common/src/attachment_url_signature.rsnew file mode 100644index 000000000..6d5b3e092--- /dev/null+++ b/fluxer_common/src/attachment_url_signature.rs@@ -0,0 +1,1127 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use crate::external_media_path::percent_decode;+use hmac::{KeyInit, Mac};++pub const ATTACHMENT_URL_TTL_SECS: u64 = 86_400;+pub const ATTACHMENT_URL_BUCKET_SECS: u64 = 43_200;+pub const SIGNATURE_PARAMETER_NAMES: [&str; 4] = ["ex", "is", "hm", "uc"];++const DOMAIN: &str = "fluxer-attachment-url-v1";+const DATA_PACKAGE_USAGE: &str = "dp";+const DATA_PACKAGE_EXPIRES: &str = "0";+const WINDOW_HEX_LEN: usize = 8;+const WINDOW_MAX_SECS: u64 = 0xffff_ffff;+const SIGNATURE_HEX_LEN: usize = 64;++type HmacSha256 = hmac::Hmac<sha2::Sha256>;++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum Verdict {+ Valid,+ Missing,+ Malformed,+ Mismatch,+ Expired,+}++impl Verdict {+ pub fn label(self) -> &'static str {+ match self {+ Self::Valid => "valid",+ Self::Missing => "missing",+ Self::Malformed => "malformed",+ Self::Mismatch => "mismatch",+ Self::Expired => "expired",+ }+ }+}++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub struct Verification {+ pub verdict: Verdict,+ pub remaining_secs: Option<u64>,+}++impl Verification {+ fn without_expiry(verdict: Verdict) -> Self {+ Self {+ verdict,+ remaining_secs: None,+ }+ }+}++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum UrlKind {+ Ordinary,+ DataPackage,+}++impl UrlKind {+ pub fn usage(self) -> &'static str {+ match self {+ Self::Ordinary => "",+ Self::DataPackage => DATA_PACKAGE_USAGE,+ }+ }+}++pub fn issue_window(anchor_secs: u64, now_secs: u64) -> (u64, u64) {+ let elapsed = now_secs.saturating_sub(anchor_secs);+ let issued = anchor_secs+ .saturating_add((elapsed / ATTACHMENT_URL_BUCKET_SECS) * ATTACHMENT_URL_BUCKET_SECS);+ (issued, issued.saturating_add(ATTACHMENT_URL_TTL_SECS))+}++pub fn canonical_input(storage_key: &str, ex: u64, is: u64, kind: UrlKind) -> String {+ format!(+ "{DOMAIN}\n{ex:08x}\n{is:08x}\n{}\n{storage_key}",+ kind.usage()+ )+}++pub fn sign(storage_key: &str, ex: u64, is: u64, kind: UrlKind, secret: &[u8]) -> String {+ let input = canonical_input(storage_key, ex, is, kind);+ hex::encode(keyed_mac(secret, &input).finalize().into_bytes())+}++pub fn is_signature_parameter_name(name: &str) -> bool {+ let decoded = percent_decode(name, true);+ SIGNATURE_PARAMETER_NAMES+ .iter()+ .any(|candidate| candidate.as_bytes() == decoded.as_slice())+}++pub fn strip_signature(url: &str) -> String {+ let (head, fragment) = split_fragment(url);+ let (base, query) = split_query(head);+ let preserved = preserved_fields(query);+ if preserved.is_empty() {+ return format!("{base}{fragment}");+ }+ format!("{base}?{}{fragment}", preserved.join("&"))+}++pub fn with_signature(+ url: &str,+ storage_key: &str,+ anchor_secs: u64,+ now_secs: u64,+ secret: &[u8],+) -> String {+ let (issued, expires) = issue_window(anchor_secs, now_secs);+ if expires > WINDOW_MAX_SECS {+ return url.to_owned();+ }+ let signature = sign(storage_key, expires, issued, UrlKind::Ordinary, secret);+ replace_signature(+ url,+ &format!("ex={expires:08x}&is={issued:08x}&hm={signature}"),+ )+}++pub fn with_data_package_signature(+ url: &str,+ storage_key: &str,+ anchor_secs: u64,+ now_secs: u64,+ secret: &[u8],+) -> String {+ let (issued, expires) = issue_window(anchor_secs, now_secs);+ if expires > WINDOW_MAX_SECS {+ return url.to_owned();+ }+ let signature = sign(storage_key, 0, issued, UrlKind::DataPackage, secret);+ replace_signature(+ url,+ &format!(+ "ex={DATA_PACKAGE_EXPIRES}&is={issued:08x}&hm={signature}&uc={DATA_PACKAGE_USAGE}"+ ),+ )+}++pub fn decode_key(path: &str) -> Option<String> {+ let decoded = percent_decode(path.trim_start_matches('/'), false);+ std::str::from_utf8(&decoded).ok().map(ToOwned::to_owned)+}++pub fn verify(+ storage_key: &str,+ raw_query: Option<&str>,+ secrets: &[&[u8]],+ now_secs: u64,+) -> Verification {+ let Some(raw_query) = raw_query else {+ return Verification::without_expiry(Verdict::Missing);+ };+ let Some(fields) = signature_fields(raw_query) else {+ return Verification::without_expiry(Verdict::Malformed);+ };+ let (expires, issued, signature) = match (+ fields.expires,+ fields.issued,+ fields.signature,+ fields.usage,+ ) {+ (None, None, None, None) => return Verification::without_expiry(Verdict::Missing),+ (Some(expires), Some(issued), Some(signature), _) => (expires, issued, signature),+ _ => return Verification::without_expiry(Verdict::Malformed),+ };+ let kind = match fields.usage {+ None => UrlKind::Ordinary,+ Some(DATA_PACKAGE_USAGE) => UrlKind::DataPackage,+ Some(_) => return Verification::without_expiry(Verdict::Malformed),+ };+ let expires = match kind {+ UrlKind::Ordinary => parse_window(expires),+ UrlKind::DataPackage => (expires == DATA_PACKAGE_EXPIRES).then_some(0),+ };+ let (Some(expires), Some(issued)) = (expires, parse_window(issued)) else {+ return Verification::without_expiry(Verdict::Malformed);+ };+ if !is_lowercase_hex(signature, SIGNATURE_HEX_LEN) {+ return Verification::without_expiry(Verdict::Malformed);+ }+ let Ok(provided) = hex::decode(signature) else {+ return Verification::without_expiry(Verdict::Malformed);+ };+ if kind == UrlKind::Ordinary && issued > expires {+ return Verification::without_expiry(Verdict::Malformed);+ }+ let input = canonical_input(storage_key, expires, issued, kind);+ let matched = secrets+ .iter()+ .any(|secret| keyed_mac(secret, &input).verify_slice(&provided).is_ok());+ if !matched {+ return Verification::without_expiry(Verdict::Mismatch);+ }+ if kind == UrlKind::DataPackage {+ return Verification::without_expiry(Verdict::Valid);+ }+ if now_secs >= expires {+ return Verification::without_expiry(Verdict::Expired);+ }+ Verification {+ verdict: Verdict::Valid,+ remaining_secs: Some(expires - now_secs),+ }+}++#[derive(Default)]+struct SignatureFields<'a> {+ expires: Option<&'a str>,+ issued: Option<&'a str>,+ signature: Option<&'a str>,+ usage: Option<&'a str>,+}++fn signature_fields(raw_query: &str) -> Option<SignatureFields<'_>> {+ let mut fields = SignatureFields::default();+ for field in raw_query.split('&').filter(|field| !field.is_empty()) {+ let (name, value) = field.split_once('=').unwrap_or((field, ""));+ let slot = match percent_decode(name, true).as_slice() {+ b"ex" => &mut fields.expires,+ b"is" => &mut fields.issued,+ b"hm" => &mut fields.signature,+ b"uc" => &mut fields.usage,+ _ => continue,+ };+ if slot.replace(value).is_some() {+ return None;+ }+ }+ Some(fields)+}++fn keyed_mac(secret: &[u8], input: &str) -> HmacSha256 {+ let mut mac = HmacSha256::new_from_slice(secret).expect("hmac accepts any key length");+ mac.update(input.as_bytes());+ mac+}++fn parse_window(value: &str) -> Option<u64> {+ if !is_lowercase_hex(value, WINDOW_HEX_LEN) {+ return None;+ }+ u64::from_str_radix(value, 16).ok()+}++fn is_lowercase_hex(value: &str, len: usize) -> bool {+ value.len() == len+ && value+ .bytes()+ .all(|byte| matches!(byte, b'0'..=b'9' | b'a'..=b'f'))+}++fn replace_signature(url: &str, signature_fields: &str) -> String {+ let (head, fragment) = split_fragment(url);+ let (base, query) = split_query(head);+ let preserved = preserved_fields(query);+ if preserved.is_empty() {+ return format!("{base}?{signature_fields}{fragment}");+ }+ format!(+ "{base}?{signature_fields}&{}{fragment}",+ preserved.join("&")+ )+}++fn split_fragment(url: &str) -> (&str, &str) {+ match url.find('#') {+ Some(index) => (&url[..index], &url[index..]),+ None => (url, ""),+ }+}++fn split_query(head: &str) -> (&str, &str) {+ head.split_once('?').unwrap_or((head, ""))+}++fn preserved_fields(query: &str) -> Vec<&str> {+ query+ .split('&')+ .filter(|field| {+ if field.is_empty() || *field == "=" {+ return false;+ }+ let (name, _) = field.split_once('=').unwrap_or((*field, ""));+ !is_signature_parameter_name(name)+ })+ .collect()+}++#[cfg(test)]+mod tests {+ use super::*;+ use base64::prelude::*;+ use serde_json::Value;++ const VECTORS: &str = include_str!("testdata/attachment_url_signature_vectors.json");+ const KEY: &str = "attachments/1544725486800732163/1544971349200470016/cat.gif";+ const ANCHOR: u64 = 1_788_420_273;++ type Signer = fn(&str, &str, u64, u64, &[u8]) -> String;++ fn fixture() -> Value {+ serde_json::from_str(VECTORS).expect("the signature vectors parse as json")+ }++ fn fixture_secrets(fixture: &Value) -> Vec<Vec<u8>> {+ fixture["secrets_base64"]+ .as_array()+ .expect("the fixture carries a secret list")+ .iter()+ .map(|entry| {+ BASE64_STANDARD+ .decode(entry.as_str().expect("a secret is a string"))+ .expect("a secret is standard base64")+ })+ .collect()+ }++ fn cases<'a>(fixture: &'a Value, name: &str) -> &'a [Value] {+ fixture[name]+ .as_array()+ .expect("the fixture carries the case list")+ .as_slice()+ }++ fn text<'a>(case: &'a Value, field: &str) -> &'a str {+ case[field]+ .as_str()+ .unwrap_or_else(|| panic!("case carries {field}"))+ }++ fn number(case: &Value, field: &str) -> u64 {+ case[field]+ .as_u64()+ .unwrap_or_else(|| panic!("case carries {field}"))+ }++ fn secret_bytes() -> Vec<u8> {+ (0u8..32).collect()+ }++ fn other_secret_bytes() -> Vec<u8> {+ (32u8..64).collect()+ }++ fn verdict_of(+ storage_key: &str,+ raw_query: Option<&str>,+ secrets: &[&[u8]],+ now_secs: u64,+ ) -> Verdict {+ verify(storage_key, raw_query, secrets, now_secs).verdict+ }++ #[test]+ fn a_valid_ordinary_signature_reports_the_seconds_left_and_a_data_package_reports_none() {+ let secret = secret_bytes();+ let now = ANCHOR + 10;+ let (issued, expires) = issue_window(ANCHOR, now);+ let query = ordinary_query(KEY, issued, expires, &secret);+ for probe in [issued, now, expires - 1] {+ assert_eq!(+ Some(expires - probe),+ verify(KEY, Some(&query), &[&secret], probe).remaining_secs,+ "{probe}"+ );+ }+ for refused in [+ verify(KEY, Some(&query), &[&secret], expires),+ verify(KEY, None, &[&secret], now),+ verify(KEY, Some(&query), &[&other_secret_bytes()], now),+ ] {+ assert_ne!(Verdict::Valid, refused.verdict);+ assert_eq!(None, refused.remaining_secs);+ }+ let package = verify(+ KEY,+ Some(&data_package_query(KEY, issued, &secret)),+ &[&secret],+ now,+ );+ assert_eq!(Verdict::Valid, package.verdict);+ assert_eq!(None, package.remaining_secs);+ }++ fn ordinary_query(key: &str, issued: u64, expires: u64, secret: &[u8]) -> String {+ format!(+ "ex={expires:08x}&is={issued:08x}&hm={}",+ sign(key, expires, issued, UrlKind::Ordinary, secret)+ )+ }++ fn data_package_query(key: &str, issued: u64, secret: &[u8]) -> String {+ format!(+ "ex=0&is={issued:08x}&hm={}&uc=dp",+ sign(key, 0, issued, UrlKind::DataPackage, secret)+ )+ }++ #[test]+ fn the_fixture_declares_the_constants_the_code_uses() {+ let fixture = fixture();+ assert_eq!(3, fixture["version"].as_u64().expect("version"));+ assert_eq!(+ ATTACHMENT_URL_TTL_SECS,+ fixture["ttl_secs"].as_u64().expect("ttl")+ );+ assert_eq!(+ ATTACHMENT_URL_BUCKET_SECS,+ fixture["bucket_secs"].as_u64().expect("bucket")+ );+ assert_eq!(+ 1_420_070_400_000,+ fixture["fluxer_epoch_ms"].as_u64().expect("epoch")+ );+ }++ #[test]+ fn every_sign_vector_reproduces_byte_for_byte() {+ let fixture = fixture();+ let secrets = fixture_secrets(&fixture);+ let parsed = cases(&fixture, "sign");+ let mut run = 0;+ let mut data_packages = 0;+ for case in parsed {+ let name = text(case, "name");+ let storage_key = text(case, "storage_key");+ let anchor = number(case, "anchor");+ let now = number(case, "now");+ let (issued, expires) = issue_window(anchor, now);+ assert_eq!(format!("{issued:08x}"), text(case, "is"), "{name} issued");+ let (kind, signed_expires, signer): (_, _, Signer) = match text(case, "uc") {+ "" => {+ assert_eq!(format!("{expires:08x}"), text(case, "ex"), "{name} expires");+ (UrlKind::Ordinary, expires, with_signature)+ }+ "dp" => {+ assert_eq!("0", text(case, "ex"), "{name} expires");+ data_packages += 1;+ (UrlKind::DataPackage, 0, with_data_package_signature)+ }+ other => panic!("{name} carries an unknown uc {other}"),+ };+ assert_eq!(+ text(case, "signature_input"),+ canonical_input(storage_key, signed_expires, issued, kind),+ "{name} input"+ );+ assert_eq!(+ text(case, "hm"),+ sign(storage_key, signed_expires, issued, kind, &secrets[0]),+ "{name} signature"+ );+ assert_eq!(+ text(case, "signed"),+ signer(text(case, "url"), storage_key, anchor, now, &secrets[0]),+ "{name} signed url"+ );+ assert_eq!(+ text(case, "url"),+ strip_signature(text(case, "signed")),+ "{name} strips back"+ );+ let signed = text(case, "signed");+ assert!(!signed.ends_with('&'), "{name} trailing separator");+ assert!(!signed.contains("&&"), "{name} doubled separator");+ run += 1;+ }+ assert_eq!(parsed.len(), run);+ assert!(run >= 8);+ assert!(data_packages >= 3);+ }++ #[test]+ fn every_strip_vector_reproduces_byte_for_byte() {+ let fixture = fixture();+ let parsed = cases(&fixture, "strip");+ let mut run = 0;+ for case in parsed {+ assert_eq!(+ text(case, "stripped"),+ strip_signature(text(case, "url")),+ "{}",+ text(case, "name")+ );+ run += 1;+ }+ assert_eq!(parsed.len(), run);+ assert!(run >= 10);+ }++ #[test]+ fn every_verify_vector_reaches_its_declared_verdict() {+ let fixture = fixture();+ let secrets = fixture_secrets(&fixture);+ let parsed = cases(&fixture, "verify");+ let mut run = 0;+ let mut rotations = 0;+ let mut trailing_separators = 0;+ let mut doubled_separators = 0;+ for case in parsed {+ let name = text(case, "name");+ let held: Vec<&[u8]> = case["verifier_secret_indices"]+ .as_array()+ .expect("case names the secrets the verifier holds")+ .iter()+ .map(|index| {+ let index = index.as_u64().expect("a secret index is a number") as usize;+ secrets[index].as_slice()+ })+ .collect();+ assert!(!held.is_empty(), "{name}");+ if held.len() > 1 {+ rotations += 1;+ }+ let now = number(case, "now");+ let query = case["query"].as_str();+ let verdict = match decode_key(text(case, "path")) {+ Some(key) => verdict_of(&key, query, &held, now),+ None => Verdict::Malformed,+ };+ assert_eq!(text(case, "verdict"), verdict.label(), "{name}");+ if verdict == Verdict::Valid {+ if query.is_some_and(|query| query.ends_with('&')) {+ trailing_separators += 1;+ }+ if query.is_some_and(|query| query.contains("&&")) {+ doubled_separators += 1;+ }+ }+ run += 1;+ }+ assert_eq!(parsed.len(), run);+ assert!(run >= 50);+ assert!(rotations >= 3);+ assert!(trailing_separators >= 2);+ assert!(doubled_separators >= 2);+ }++ #[test]+ fn a_freshly_signed_url_verifies_at_every_second_of_its_life() {+ let secret = secret_bytes();+ let now = ANCHOR + 10;+ let signed = with_signature("https://media.test/x.gif", KEY, ANCHOR, now, &secret);+ let query = signed+ .split_once('?')+ .expect("a signed url carries a query")+ .1;+ let (issued, expires) = issue_window(ANCHOR, now);+ assert!(issued <= now && now < expires);+ for probe in [issued, now, expires - 1] {+ assert_eq!(+ Verdict::Valid,+ verdict_of(KEY, Some(query), &[&secret], probe),+ "{probe}"+ );+ }+ for probe in [expires, expires + 1] {+ assert_eq!(+ Verdict::Expired,+ verdict_of(KEY, Some(query), &[&secret], probe),+ "{probe}"+ );+ }+ }++ #[test]+ fn a_data_package_url_never_expires() {+ let secret = secret_bytes();+ let now = ANCHOR + 3 * ATTACHMENT_URL_BUCKET_SECS + 7;+ let signed =+ with_data_package_signature("https://media.test/x.gif", KEY, ANCHOR, now, &secret);+ let query = signed+ .split_once('?')+ .expect("a signed url carries a query")+ .1;+ let (issued, expires) = issue_window(ANCHOR, now);+ assert!(query.starts_with(&format!("ex=0&is={issued:08x}&hm=")));+ assert!(query.ends_with("&uc=dp"));+ for probe in [+ 0,+ issued,+ now,+ expires,+ expires + 1,+ u64::from(u32::MAX),+ u64::MAX,+ ] {+ assert_eq!(+ Verdict::Valid,+ verdict_of(KEY, Some(query), &[&secret], probe),+ "{probe}"+ );+ }+ assert_eq!(+ Verdict::Mismatch,+ verdict_of(KEY, Some(query), &[&other_secret_bytes()], now)+ );+ }++ #[test]+ fn the_issue_window_stays_on_the_anchor_grid() {+ for offset in [0, 1, ATTACHMENT_URL_BUCKET_SECS - 1] {+ for bucket in 0..8u64 {+ let now = ANCHOR + bucket * ATTACHMENT_URL_BUCKET_SECS + offset;+ let (issued, expires) = issue_window(ANCHOR, now);+ assert_eq!(ANCHOR + bucket * ATTACHMENT_URL_BUCKET_SECS, issued);+ assert_eq!(issued + ATTACHMENT_URL_TTL_SECS, expires);+ assert!(expires - now >= ATTACHMENT_URL_TTL_SECS - ATTACHMENT_URL_BUCKET_SECS);+ }+ }+ }++ #[test]+ fn a_clock_behind_the_anchor_issues_the_first_bucket() {+ let (issued, expires) = issue_window(ANCHOR, ANCHOR - 5_000);+ assert_eq!(ANCHOR, issued);+ assert_eq!(ANCHOR + ATTACHMENT_URL_TTL_SECS, expires);+ }++ #[test]+ fn a_window_beyond_eight_hex_digits_leaves_the_url_unchanged() {+ let secret = secret_bytes();+ let signers: [Signer; 2] = [with_signature, with_data_package_signature];+ let last_fitting_anchor = WINDOW_MAX_SECS - ATTACHMENT_URL_TTL_SECS;+ for signer in signers {+ for url in [+ "https://media.test/attachments/1/2/cat.gif",+ "https://media.test/attachments/1/2/cat.gif?width=64&ex=1&&is=2&hm=3&uc=4#top",+ ] {+ for anchor in [last_fitting_anchor + 1, WINDOW_MAX_SECS, u64::MAX] {+ assert_eq!(url, signer(url, KEY, anchor, 0, &secret), "{anchor} {url}");+ }+ assert_eq!(+ url,+ signer(+ url,+ KEY,+ last_fitting_anchor,+ last_fitting_anchor + ATTACHMENT_URL_BUCKET_SECS,+ &secret+ ),+ "{url}"+ );+ let signed = signer(url, KEY, last_fitting_anchor, 0, &secret);+ assert_ne!(url, signed, "{url}");+ assert!(signed.contains(&format!("is={last_fitting_anchor:08x}&")));+ }+ }+ let signed = with_signature(+ "https://media.test/x.gif",+ KEY,+ last_fitting_anchor,+ 0,+ &secret,+ );+ assert!(signed.contains("ex=ffffffff&"), "{signed}");+ }++ #[test]+ fn verification_tries_every_configured_secret() {+ let first = secret_bytes();+ let second = other_secret_bytes();+ let now = ANCHOR;+ let (issued, expires) = issue_window(ANCHOR, now);+ for query in [+ ordinary_query(KEY, issued, expires, &second),+ data_package_query(KEY, issued, &second),+ ] {+ assert_eq!(+ Verdict::Mismatch,+ verdict_of(KEY, Some(&query), &[&first], now)+ );+ assert_eq!(+ Verdict::Valid,+ verdict_of(KEY, Some(&query), &[&first, &second], now)+ );+ assert_eq!(+ Verdict::Valid,+ verdict_of(KEY, Some(&query), &[&second, &first], now)+ );+ assert_eq!(Verdict::Mismatch, verdict_of(KEY, Some(&query), &[], now));+ }+ }++ #[test]+ fn a_signature_never_carries_across_keys_or_windows() {+ let secret = secret_bytes();+ let now = ANCHOR;+ let (issued, expires) = issue_window(ANCHOR, now);+ let signature = sign(KEY, expires, issued, UrlKind::Ordinary, &secret);+ let query = ordinary_query(KEY, issued, expires, &secret);+ let package = data_package_query(KEY, issued, &secret);+ assert_eq!(+ Verdict::Valid,+ verdict_of(KEY, Some(&query), &[&secret], now)+ );+ assert_eq!(+ Verdict::Valid,+ verdict_of(KEY, Some(&package), &[&secret], now)+ );+ for other in [+ "attachments/1544725486800732163/1544971349200470016/cat.gi",+ "attachments/1544725486800732163/1544971349200470017/cat.gif",+ "attachments/1544725486800732163/1544971349200470016/cat.gif ",+ "attachments/1544725486800732163/1544971349200470016/sub/cat.gif",+ ] {+ for query in [&query, &package] {+ assert_eq!(+ Verdict::Mismatch,+ verdict_of(other, Some(query), &[&secret], now),+ "{other} {query}"+ );+ }+ }+ let shifted = format!(+ "ex={:08x}&is={issued:08x}&hm={signature}",+ expires + ATTACHMENT_URL_BUCKET_SECS+ );+ assert_eq!(+ Verdict::Mismatch,+ verdict_of(KEY, Some(&shifted), &[&secret], now)+ );+ let shifted_package = format!(+ "ex=0&is={:08x}&hm={}&uc=dp",+ issued + 1,+ sign(KEY, 0, issued, UrlKind::DataPackage, &secret)+ );+ assert_eq!(+ Verdict::Mismatch,+ verdict_of(KEY, Some(&shifted_package), &[&secret], now)+ );+ }++ #[test]+ fn a_signature_is_bound_to_its_usage() {+ let secret = secret_bytes();+ let now = ANCHOR;+ let (issued, expires) = issue_window(ANCHOR, now);+ assert_ne!(+ canonical_input(KEY, 0, issued, UrlKind::Ordinary),+ canonical_input(KEY, 0, issued, UrlKind::DataPackage)+ );+ let ordinary = sign(KEY, expires, issued, UrlKind::Ordinary, &secret);+ let package = sign(KEY, 0, issued, UrlKind::DataPackage, &secret);+ for (label, query) in [+ (+ "ordinary signature relabelled as a data package",+ format!("ex=0&is={issued:08x}&hm={ordinary}&uc=dp"),+ ),+ (+ "data package signature relabelled as ordinary",+ format!("ex={expires:08x}&is={issued:08x}&hm={package}"),+ ),+ (+ "ordinary input with a zero expiry signed under the data package",+ format!(+ "ex=0&is={issued:08x}&hm={}&uc=dp",+ sign(KEY, 0, issued, UrlKind::Ordinary, &secret)+ ),+ ),+ ] {+ assert_eq!(+ Verdict::Mismatch,+ verdict_of(KEY, Some(&query), &[&secret], now),+ "{label}"+ );+ }+ }++ #[test]+ fn every_parameter_shape_outside_the_grammar_is_refused() {+ let secret = secret_bytes();+ let now = ANCHOR;+ let (issued, expires) = issue_window(ANCHOR, now);+ let signature = sign(KEY, expires, issued, UrlKind::Ordinary, &secret);+ let package = sign(KEY, 0, issued, UrlKind::DataPackage, &secret);+ let valid = ordinary_query(KEY, issued, expires, &secret);+ let valid_package = data_package_query(KEY, issued, &secret);+ assert_eq!(+ Verdict::Valid,+ verdict_of(KEY, Some(&valid), &[&secret], now)+ );+ assert_eq!(+ Verdict::Valid,+ verdict_of(KEY, Some(&valid_package), &[&secret], now)+ );+ for (label, query, expected) in [+ ("no query", None, Verdict::Missing),+ ("empty query", Some(String::new()), Verdict::Missing),+ (+ "only transform parameters",+ Some("width=64&format=webp".to_owned()),+ Verdict::Missing,+ ),+ (+ "only the signature",+ Some(format!("hm={signature}")),+ Verdict::Malformed,+ ),+ (+ "only the window",+ Some(format!("ex={expires:08x}&is={issued:08x}")),+ Verdict::Malformed,+ ),+ (+ "repeated signature",+ Some(format!("{valid}&hm={signature}")),+ Verdict::Malformed,+ ),+ (+ "percent escaped second name",+ Some(format!("%65x={expires:08x}&{valid}")),+ Verdict::Malformed,+ ),+ (+ "uppercase signature",+ Some(format!(+ "ex={expires:08x}&is={issued:08x}&hm={}",+ signature.to_ascii_uppercase()+ )),+ Verdict::Malformed,+ ),+ (+ "percent escaped value",+ Some(format!(+ "ex={expires:08x}&is={issued:08x}&hm=%{}",+ &signature[2..]+ )),+ Verdict::Malformed,+ ),+ (+ "short window field",+ Some(format!("ex={:07x}&is={issued:08x}&hm={signature}", 1)),+ Verdict::Malformed,+ ),+ (+ "issued after expiry",+ Some(format!("ex={issued:08x}&is={expires:08x}&hm={signature}")),+ Verdict::Malformed,+ ),+ (+ "name without a value",+ Some(format!("ex&is={issued:08x}&hm={signature}")),+ Verdict::Malformed,+ ),+ (+ "zero expiry without uc",+ Some(format!("ex=0&is={issued:08x}&hm={package}")),+ Verdict::Malformed,+ ),+ (+ "padded zero expiry with uc",+ Some(format!("ex=00000000&is={issued:08x}&hm={package}&uc=dp")),+ Verdict::Malformed,+ ),+ (+ "eight digit expiry with uc",+ Some(format!("{valid}&uc=dp")),+ Verdict::Malformed,+ ),+ (+ "double zero expiry with uc",+ Some(format!("ex=00&is={issued:08x}&hm={package}&uc=dp")),+ Verdict::Malformed,+ ),+ (+ "another uc value",+ Some(format!("ex=0&is={issued:08x}&hm={package}&uc=dq")),+ Verdict::Malformed,+ ),+ (+ "uppercase uc",+ Some(format!("ex=0&is={issued:08x}&hm={package}&uc=DP")),+ Verdict::Malformed,+ ),+ (+ "empty uc",+ Some(format!("ex=0&is={issued:08x}&hm={package}&uc=")),+ Verdict::Malformed,+ ),+ (+ "uc without a value",+ Some(format!("ex=0&is={issued:08x}&hm={package}&uc")),+ Verdict::Malformed,+ ),+ (+ "percent escaped uc value",+ Some(format!("ex=0&is={issued:08x}&hm={package}&uc=%64p")),+ Verdict::Malformed,+ ),+ ("uc alone", Some("uc=dp".to_owned()), Verdict::Malformed),+ (+ "uc among transform parameters",+ Some("width=64&uc=dp&format=webp".to_owned()),+ Verdict::Malformed,+ ),+ (+ "data package without its signature",+ Some(format!("ex=0&is={issued:08x}&uc=dp")),+ Verdict::Malformed,+ ),+ (+ "data package without its issued field",+ Some(format!("ex=0&hm={package}&uc=dp")),+ Verdict::Malformed,+ ),+ (+ "repeated uc",+ Some(format!("{valid_package}&uc=dp")),+ Verdict::Malformed,+ ),+ (+ "percent escaped repeated uc",+ Some(format!("{valid_package}&%75c=dp")),+ Verdict::Malformed,+ ),+ (+ "repeated zero expiry",+ Some(format!("ex=0&{valid_package}")),+ Verdict::Malformed,+ ),+ (+ "short data package issued field",+ Some(format!("ex=0&is={:07x}&hm={package}&uc=dp", 1)),+ Verdict::Malformed,+ ),+ (+ "uppercase data package signature",+ Some(format!(+ "ex=0&is={issued:08x}&hm={}&uc=dp",+ package.to_ascii_uppercase()+ )),+ Verdict::Malformed,+ ),+ ] {+ assert_eq!(+ expected,+ verdict_of(KEY, query.as_deref(), &[&secret], now),+ "{label}"+ );+ }+ }++ #[test]+ fn tolerated_input_shapes_still_verify() {+ let secret = secret_bytes();+ let now = ANCHOR;+ let (issued, expires) = issue_window(ANCHOR, now);+ let signature = sign(KEY, expires, issued, UrlKind::Ordinary, &secret);+ let package = sign(KEY, 0, issued, UrlKind::DataPackage, &secret);+ for (label, query) in [+ (+ "a trailing separator",+ format!("ex={expires:08x}&is={issued:08x}&hm={signature}&"),+ ),+ (+ "a doubled separator between the fields",+ format!("ex={expires:08x}&&is={issued:08x}&hm={signature}"),+ ),+ (+ "empty pairs around the signature",+ format!("&=&ex={expires:08x}&&is={issued:08x}&hm={signature}&&"),+ ),+ (+ "signature after other parameters",+ format!("width=64&ex={expires:08x}&is={issued:08x}&hm={signature}"),+ ),+ (+ "unknown parameters between the signature fields",+ format!("ex={expires:08x}&width=64&is={issued:08x}&format=webp&hm={signature}"),+ ),+ (+ "a data package with a trailing separator",+ format!("ex=0&is={issued:08x}&hm={package}&uc=dp&"),+ ),+ (+ "a doubled separator between the data package fields",+ format!("ex=0&is={issued:08x}&&hm={package}&uc=dp"),+ ),+ (+ "data package fields in another order",+ format!("uc=dp&width=64&hm={package}&is={issued:08x}&ex=0"),+ ),+ (+ "percent escaped uc name",+ format!("ex=0&is={issued:08x}&hm={package}&%75c=dp"),+ ),+ (+ "empty pairs around a data package",+ format!("&=&ex=0&&is={issued:08x}&hm={package}&=&uc=dp&&"),+ ),+ ] {+ assert_eq!(+ Verdict::Valid,+ verdict_of(KEY, Some(&query), &[&secret], now),+ "{label}"+ );+ }+ }++ #[test]+ fn every_signature_parameter_name_is_recognised_after_form_decoding() {+ for name in ["ex", "is", "hm", "uc", "%65x", "%75c", "u%63", "%68%6D"] {+ assert!(is_signature_parameter_name(name), "{name}");+ }+ for name in [+ "", "e", "exp", "UC", "u+c", "u%2Bc", "%2575c", "width", "dp",+ ] {+ assert!(!is_signature_parameter_name(name), "{name}");+ }+ }++ #[test]+ fn a_decoded_key_names_one_object_across_every_path_spelling() {+ for (path, expected) in [+ ("/attachments/1/2/cat.gif", "attachments/1/2/cat.gif"),+ ("///attachments/1/2/cat.gif", "attachments/1/2/cat.gif"),+ ("attachments/1/2/cat.gif", "attachments/1/2/cat.gif"),+ ("/attachments/1/2/caf%C3%A9.gif", "attachments/1/2/café.gif"),+ ("/attachments/1/2/café.gif", "attachments/1/2/café.gif"),+ ("/attachments/1/2/a%2Fb.gif", "attachments/1/2/a/b.gif"),+ ("/attachments/1/2/a+b.gif", "attachments/1/2/a+b.gif"),+ ("/attachments/1/2/a%25b.gif", "attachments/1/2/a%b.gif"),+ ("/attachments/1/2/a%2.gif", "attachments/1/2/a%2.gif"),+ ] {+ assert_eq!(Some(expected.to_owned()), decode_key(path), "{path}");+ }+ for path in [+ "/attachments/1/2/caf%C3%28.gif",+ "/attachments/1/2/%FF.gif",+ "/attachments/1/2/%ED%A0%80.gif",+ ] {+ assert_eq!(None, decode_key(path), "{path}");+ }+ }++ #[test]+ fn signing_replaces_an_existing_signature_and_keeps_everything_else() {+ let secret = secret_bytes();+ let now = ANCHOR;+ let canonical = "https://media.test/attachments/1/2/cat.gif?width=64&height=64#anchor";+ let key = "attachments/1/2/cat.gif";+ let signed = with_signature(canonical, key, ANCHOR, now, &secret);+ assert!(signed.contains("?ex="));+ assert!(signed.ends_with("&width=64&height=64#anchor"));+ let resigned = with_signature(+ &signed,+ key,+ ANCHOR,+ now + ATTACHMENT_URL_BUCKET_SECS,+ &secret,+ );+ let package = with_data_package_signature(&resigned, key, ANCHOR, now, &secret);+ let back = with_signature(&package, key, ANCHOR, now, &secret);+ assert_eq!(signed, back);+ for url in [&resigned, &package, &back] {+ assert_eq!(1, url.matches("ex=").count(), "{url}");+ assert_eq!(1, url.matches("is=").count(), "{url}");+ assert_eq!(1, url.matches("hm=").count(), "{url}");+ assert!(url.ends_with("&width=64&height=64#anchor"), "{url}");+ assert_eq!(canonical, strip_signature(url), "{url}");+ }+ assert_eq!(1, package.matches("uc=dp&").count(), "{package}");+ assert!(!resigned.contains("uc="), "{resigned}");+ assert!(!back.contains("uc="), "{back}");+ }++ #[test]+ fn a_bare_signed_url_ends_with_its_last_signature_field() {+ let secret = secret_bytes();+ let url = "https://media.test/attachments/1/2/cat.gif";+ let key = "attachments/1/2/cat.gif";+ for (signed, names) in [+ (+ with_signature(url, key, ANCHOR, ANCHOR, &secret),+ &["ex=", "is=", "hm="][..],+ ),+ (+ with_data_package_signature(url, key, ANCHOR, ANCHOR, &secret),+ &["ex=0", "is=", "hm=", "uc=dp"][..],+ ),+ ] {+ assert!(!signed.ends_with('&'), "{signed}");+ assert!(!signed.contains("&&"), "{signed}");+ let query = signed.split_once('?').expect("a signed url has a query").1;+ let fields: Vec<&str> = query.split('&').collect();+ assert_eq!(names.len(), fields.len(), "{signed}");+ for (field, name) in fields.iter().zip(names) {+ assert!(field.starts_with(name), "{signed}");+ }+ }+ }++ #[test]+ fn a_preserved_parameter_follows_one_separator() {+ let secret = secret_bytes();+ let key = "attachments/1/2/cat.gif";+ let signers: [Signer; 2] = [with_signature, with_data_package_signature];+ for signer in signers {+ for (url, tail) in [+ (+ "https://media.test/attachments/1/2/cat.gif?width=100",+ "&width=100",+ ),+ (+ "https://media.test/attachments/1/2/cat.gif?width=100&height=64#top",+ "&width=100&height=64#top",+ ),+ ("https://media.test/attachments/1/2/cat.gif#top", "#top"),+ ("https://media.test/attachments/1/2/cat.gif?&=&", ""),+ ] {+ let signed = signer(url, key, ANCHOR, ANCHOR, &secret);+ assert!(signed.ends_with(tail), "{signed}");+ assert!(!signed.contains("&&"), "{signed}");+ let head = signed+ .split_once('#')+ .map_or(signed.as_str(), |(head, _)| head);+ assert!(!head.ends_with('&'), "{signed}");+ }+ }+ }+}diff --git a/fluxer_common/src/lib.rs b/fluxer_common/src/lib.rsindex 6582122b0..9c821ffa8 100644--- a/fluxer_common/src/lib.rs+++ b/fluxer_common/src/lib.rs@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+pub mod attachment_url_signature;pub mod config;pub mod external_media_path;pub mod geoip;diff --git a/fluxer_common/src/testdata/attachment_url_signature_vectors.json b/fluxer_common/src/testdata/attachment_url_signature_vectors.jsonnew file mode 100644index 000000000..8abed9876--- /dev/null+++ b/fluxer_common/src/testdata/attachment_url_signature_vectors.json@@ -0,0 +1,793 @@+{+ "version": 3,+ "ttl_secs": 86400,+ "bucket_secs": 43200,+ "fluxer_epoch_ms": 1420070400000,+ "secrets_base64": ["AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", "ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8="],+ "sign": [+ {+ "name": "first issue at the snowflake second",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "cat.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif",+ "anchor": 1788420273,+ "now": 1788420273,+ "ex": "6a9a7231",+ "is": "6a9920b1",+ "hm": "2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "uc": "",+ "signature_input": "fluxer-attachment-url-v1\n6a9a7231\n6a9920b1\n\nattachments/1544725486800732163/1544971349200470016/cat.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27"+ },+ {+ "name": "first issue with a slash inside the filename",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "a/b.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/a/b.gif",+ "anchor": 1788420273,+ "now": 1788420273,+ "ex": "6a9a7231",+ "is": "6a9920b1",+ "hm": "5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258",+ "uc": "",+ "signature_input": "fluxer-attachment-url-v1\n6a9a7231\n6a9920b1\n\nattachments/1544725486800732163/1544971349200470016/a/b.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/a/b.gif",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/a/b.gif?ex=6a9a7231&is=6a9920b1&hm=5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258"+ },+ {+ "name": "first issue with a unicode filename",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "café.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/café.gif",+ "anchor": 1788420273,+ "now": 1788420273,+ "ex": "6a9a7231",+ "is": "6a9920b1",+ "hm": "37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c",+ "uc": "",+ "signature_input": "fluxer-attachment-url-v1\n6a9a7231\n6a9920b1\n\nattachments/1544725486800732163/1544971349200470016/café.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/café.gif",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/café.gif?ex=6a9a7231&is=6a9920b1&hm=37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c"+ },+ {+ "name": "second bucket",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "cat.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif",+ "anchor": 1788420273,+ "now": 1788463473,+ "ex": "6a9b1af1",+ "is": "6a99c971",+ "hm": "d5f9d09526e0be7825b32ee9522927e0786d3fcb576e566ed93d642b6fc05bf5",+ "uc": "",+ "signature_input": "fluxer-attachment-url-v1\n6a9b1af1\n6a99c971\n\nattachments/1544725486800732163/1544971349200470016/cat.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9b1af1&is=6a99c971&hm=d5f9d09526e0be7825b32ee9522927e0786d3fcb576e566ed93d642b6fc05bf5"+ },+ {+ "name": "far future issue stays on the bucket grid",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "cat.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif",+ "anchor": 1788420273,+ "now": 1792740280,+ "ex": "6adc5d31",+ "is": "6adb0bb1",+ "hm": "efbfba51976929974135d10c7760fc1dfa1a2e21bfd8e005493fb568a4a532b1",+ "uc": "",+ "signature_input": "fluxer-attachment-url-v1\n6adc5d31\n6adb0bb1\n\nattachments/1544725486800732163/1544971349200470016/cat.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6adc5d31&is=6adb0bb1&hm=efbfba51976929974135d10c7760fc1dfa1a2e21bfd8e005493fb568a4a532b1"+ },+ {+ "name": "transform parameters and a fragment follow the signature",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "cat.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif",+ "anchor": 1788420273,+ "now": 1788420273,+ "ex": "6a9a7231",+ "is": "6a9920b1",+ "hm": "2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "uc": "",+ "signature_input": "fluxer-attachment-url-v1\n6a9a7231\n6a9920b1\n\nattachments/1544725486800732163/1544971349200470016/cat.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64&height=64#top",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&width=64&height=64#top"+ },+ {+ "name": "data package at the snowflake second",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "cat.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif",+ "anchor": 1788420273,+ "now": 1788420273,+ "ex": "0",+ "is": "6a9920b1",+ "hm": "0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a",+ "uc": "dp",+ "signature_input": "fluxer-attachment-url-v1\n00000000\n6a9920b1\ndp\nattachments/1544725486800732163/1544971349200470016/cat.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp"+ },+ {+ "name": "data package with a unicode filename",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "café.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/café.gif",+ "anchor": 1788420273,+ "now": 1788420273,+ "ex": "0",+ "is": "6a9920b1",+ "hm": "89e63377486a58781d1f168c80a438e99a1c87f71f8bb5a7401f2512770286c2",+ "uc": "dp",+ "signature_input": "fluxer-attachment-url-v1\n00000000\n6a9920b1\ndp\nattachments/1544725486800732163/1544971349200470016/café.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/café.gif",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/café.gif?ex=0&is=6a9920b1&hm=89e63377486a58781d1f168c80a438e99a1c87f71f8bb5a7401f2512770286c2&uc=dp"+ },+ {+ "name": "data package issued in a later bucket",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "cat.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif",+ "anchor": 1788420273,+ "now": 1792740280,+ "ex": "0",+ "is": "6adb0bb1",+ "hm": "34bdd1bc2fdb906efa5ff6bb2db8be256318be7457b60dfa593ee0507eec0881",+ "uc": "dp",+ "signature_input": "fluxer-attachment-url-v1\n00000000\n6adb0bb1\ndp\nattachments/1544725486800732163/1544971349200470016/cat.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6adb0bb1&hm=34bdd1bc2fdb906efa5ff6bb2db8be256318be7457b60dfa593ee0507eec0881&uc=dp"+ },+ {+ "name": "data package keeps transform parameters after uc",+ "channel_id": "1544725486800732163",+ "attachment_id": "1544971349200470016",+ "filename": "cat.gif",+ "storage_key": "attachments/1544725486800732163/1544971349200470016/cat.gif",+ "anchor": 1788420273,+ "now": 1788420273,+ "ex": "0",+ "is": "6a9920b1",+ "hm": "0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a",+ "uc": "dp",+ "signature_input": "fluxer-attachment-url-v1\n00000000\n6a9920b1\ndp\nattachments/1544725486800732163/1544971349200470016/cat.gif",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64#top",+ "signed": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&width=64#top"+ }+ ],+ "strip": [+ {+ "name": "a signed url strips back to its canonical form",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif"+ },+ {+ "name": "a trailing separator on a signed url is tolerated",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif"+ },+ {+ "name": "transform parameters survive the strip",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&width=64&height=64",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64&height=64"+ },+ {+ "name": "a signature after other parameters is still removed",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64&ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64"+ },+ {+ "name": "empty pairs are dropped",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?&=&ex=6a9a7231&&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif"+ },+ {+ "name": "an unsigned url is unchanged",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64"+ },+ {+ "name": "a fragment survives the strip",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27#top",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif#top"+ },+ {+ "name": "a percent escaped signature name is removed too",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?%65x=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&width=64",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64"+ },+ {+ "name": "a data package url strips back to its canonical form",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif"+ },+ {+ "name": "transform parameters survive a data package strip",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&width=64#top",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64#top"+ },+ {+ "name": "uc is removed whatever its value",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?uc=zz&width=64&uc",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64"+ },+ {+ "name": "a percent escaped uc name is removed too",+ "url": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&%75c=dp&width=64",+ "stripped": "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif?width=64"+ }+ ],+ "verify": [+ {+ "name": "the emitted form verifies",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "valid one second before expiry",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788506672,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "expired at the expiry second",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788506673,+ "verifier_secret_indices": [0],+ "verdict": "expired"+ },+ {+ "name": "expired long after the window",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788549873,+ "verifier_secret_indices": [0],+ "verdict": "expired"+ },+ {+ "name": "a trailing separator is tolerated",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a doubled separator between the fields is tolerated",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "empty pairs are tolerated",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "&=&ex=6a9a7231&&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&&",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "the signature may follow other parameters",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "width=64&ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "transform parameters are not signed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&width=64&format=webp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "no query at all is missing",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": null,+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "missing"+ },+ {+ "name": "an empty query is missing",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "missing"+ },+ {+ "name": "only transform parameters is missing",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "width=64",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "missing"+ },+ {+ "name": "a partial set is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a repeated signature is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a percent escaped duplicate name is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "%65x=6a9a7231&ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "an uppercase signature is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2F82FAFD058D10847AFFCCC91735BC5B95753F4081F4DA68AE054C221D674F27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a percent escaped value is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=%82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a seven digit window field is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "issued after expiry is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9920b1&is=6a9a7231&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a path whose bytes are not utf-8 is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/caf%C3%28.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a signature over another object mismatches",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a signature from another key mismatches",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [1],+ "verdict": "mismatch"+ },+ {+ "name": "an altered expiry mismatches",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7232&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "the rotation key verifies its own signature",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=f1963bb94826bb42d68224ea35c317a07fd3cfd7853cd5346ca85ec47611d04d",+ "now": 1788420273,+ "verifier_secret_indices": [1],+ "verdict": "valid"+ },+ {+ "name": "a signature from the second key mismatches when only the first is held",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=f1963bb94826bb42d68224ea35c317a07fd3cfd7853cd5346ca85ec47611d04d",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a signature from the second key verifies while both are held",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=f1963bb94826bb42d68224ea35c317a07fd3cfd7853cd5346ca85ec47611d04d",+ "now": 1788420273,+ "verifier_secret_indices": [0, 1],+ "verdict": "valid"+ },+ {+ "name": "a signature from the first key verifies whatever order the keys are held in",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [1, 0],+ "verdict": "valid"+ },+ {+ "name": "a percent encoded unicode filename verifies",+ "path": "/attachments/1544725486800732163/1544971349200470016/caf%C3%A9.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a raw unicode filename verifies",+ "path": "/attachments/1544725486800732163/1544971349200470016/café.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=37efc4faf535ba29ecf6580cb47a88d80b14cd6319e29931c06ddffa761abd9c",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a leading double slash names the same object",+ "path": "//attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "an encoded slash names the decoded key",+ "path": "/attachments/1544725486800732163/1544971349200470016/a%2Fb.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "the decoded key is what is signed",+ "path": "/attachments/1544725486800732163/1544971349200470016/a/b.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a plus is not a space in the key",+ "path": "/attachments/1544725486800732163/1544971349200470016/a+b.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=5d594e9ce78bc3c82e735dd29bcfdf7cee9ebb4391304b1df90d454207feb258",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "the emitted data package form verifies",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a data package verifies long after an ordinary url would expire",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp",+ "now": 4102444800,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a data package tolerates a trailing separator",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a doubled separator between the data package fields is tolerated",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a data package tolerates its fields in any position",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "uc=dp&width=64&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&is=6a9920b1&ex=0",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "transform parameters after a data package are not signed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&width=64&format=webp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a percent escaped uc name counts as uc",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&%75c=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a data package with a percent encoded unicode filename verifies",+ "path": "/attachments/1544725486800732163/1544971349200470016/caf%C3%A9.gif",+ "query": "ex=0&is=6a9920b1&hm=89e63377486a58781d1f168c80a438e99a1c87f71f8bb5a7401f2512770286c2&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a data package from the second key verifies while both are held",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0a442f8c3a83e6075764ada1d3b833fa9c1977354861c12e55ba04502af77cdf&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0, 1],+ "verdict": "valid"+ },+ {+ "name": "a data package from a key removed from rotation mismatches",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0a442f8c3a83e6075764ada1d3b833fa9c1977354861c12e55ba04502af77cdf&uc=dp",+ "now": 4102444800,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a data package from another key mismatches",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [1],+ "verdict": "mismatch"+ },+ {+ "name": "a data package over another object mismatches",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=89e63377486a58781d1f168c80a438e99a1c87f71f8bb5a7401f2512770286c2&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "an altered issued field on a data package mismatches",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b2&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "an ordinary signature relabelled as a data package mismatches",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a data package signature relabelled as an ordinary url mismatches",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a data package with uc removed is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a tampered uc value is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dq",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "an uppercase uc value is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=DP",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "an empty uc value is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a percent escaped uc value is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=%64p",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a zero padded expiry with uc is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=00000000&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "uc on an ordinary url is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "uc alone is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "uc among transform parameters is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "width=64&uc=dp&format=webp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a data package without its signature is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a repeated uc is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a percent escaped repeated uc is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp&%75c=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a repeated zero expiry is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&ex=0&is=6a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a seven digit data package issued field is malformed",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=0&is=a9920b1&hm=0f888cfdbf17060cead41b6e9a05b172c8c24bf95c133624be14b34b4c69af5a&uc=dp",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "malformed"+ },+ {+ "name": "a leading triple slash names the same object",+ "path": "///attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "percent encoded slashes in the prefix name the same object",+ "path": "/attachments%2F1544725486800732163%2F1544971349200470016%2Fcat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "valid"+ },+ {+ "name": "a dot segment after the prefix is part of the key",+ "path": "/attachments/./1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a percent encoded dot segment after the prefix is part of the key",+ "path": "/attachments/%2E/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a dot segment between the id components is part of the key",+ "path": "/attachments/1544725486800732163/./1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a doubled slash inside the path is part of the key",+ "path": "/attachments/1544725486800732163//1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a parent segment before the prefix is part of the key",+ "path": "/x/../attachments/1544725486800732163/1544971349200470016/cat.gif",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ },+ {+ "name": "a trailing dot segment is part of the key",+ "path": "/attachments/1544725486800732163/1544971349200470016/cat.gif/.",+ "query": "ex=6a9a7231&is=6a9920b1&hm=2f82fafd058d10847affccc91735bc5b95753f4081f4da68ae054c221d674f27",+ "now": 1788420273,+ "verifier_secret_indices": [0],+ "verdict": "mismatch"+ }+ ]+}diff --git a/fluxer_docs/scripts/VerifyDocsCoverage.ts b/fluxer_docs/scripts/VerifyDocsCoverage.tsindex dbedbd22a..48ce40e8b 100644--- a/fluxer_docs/scripts/VerifyDocsCoverage.ts+++ b/fluxer_docs/scripts/VerifyDocsCoverage.ts@@ -192,6 +192,7 @@ const MEDIA_PROXY_ROUTES = new Map([['HEAD /_health', '.route("/_health", get(routes::ops::health))'],['GET /_metrics', '.route("/_metrics", get(routes::ops::metrics_handler))'],['POST /_metadata', '.route("/_metadata", post(routes::internal::metadata_handler))'],+ ['POST /_sniff', '.route("/_sniff", post(routes::internal::sniff_handler))'],['POST /_thumbnail', '.route("/_thumbnail", post(routes::internal::thumbnail_handler))'],['POST /_frames', '.route("/_frames", post(routes::internal::frames_handler))'],[@@ -439,6 +440,25 @@ for (const [route, {documentedIn}] of OUT_OF_BAND_CREDENTIAL) {}failures += section('stale anchors (the code moved, update this script)', staleAnchors);+const mediaProxyRouterSource = await readFile(path.join(MEDIA_PROXY_SERVER_DIR, 'runtime.rs'), 'utf8');+const mediaProxyRegisteredPaths = [...mediaProxyRouterSource.matchAll(/\.route\(\s*"([^"]+)"/gu)].map((match) =>+ match[1].replace(/\{[^}]*\}/gu, '{}'),+);+const mediaProxyListedPaths = new Set([...MEDIA_PROXY_ROUTES.keys()].map((shape) => shape.split(' ')[1]));+const unlistedMediaProxyPaths = [...new Set(mediaProxyRegisteredPaths)]+ .filter((routePath) => !mediaProxyListedPaths.has(routePath))+ .map((routePath) => `${routePath}: build_router registers it and MEDIA_PROXY_ROUTES does not name it`)+ .sort();+if (mediaProxyRegisteredPaths.length === 0) {+ unlistedMediaProxyPaths.push(+ 'fluxer_media_proxy/src/server/runtime.rs registers no route, so this check has gone blind',+ );+}+failures += section('registered by fluxer_media_proxy but absent from this script', unlistedMediaProxyPaths);+console.log(+ ` routes registered in fluxer_media_proxy/src/server/runtime.rs: ${mediaProxyRegisteredPaths.length.toString()}`,+);+const mediaProxyDocumented = documented.filter((route) => route.file.startsWith('media-proxy/'));const adminDocumented = documented.filter((route) => !route.file.startsWith('media-proxy/') && stripVersionPrefix(route.path).startsWith('/admin'),diff --git a/fluxer_docs/src/content/docs/http-api/memes.mdx b/fluxer_docs/src/content/docs/http-api/memes.mdxindex 1e20a7692..0813c7512 100644--- a/fluxer_docs/src/content/docs/http-api/memes.mdx+++ b/fluxer_docs/src/content/docs/http-api/memes.mdx@@ -54,7 +54,7 @@ One media asset owned by one account. Fluxer resolves every media field at creat<sup>2</sup> [Save meme from message](#save-meme-from-message) falls back to the selected attachment's own description when the body supplies no non-empty alt text-<sup>3</sup> `url` is the media endpoint followed by `attachments/{user_id}/{attachment_id}/{filename}`+<sup>3</sup> `url` is the media endpoint followed by `attachments/{user_id}/{attachment_id}/{filename}`, and Fluxer signs it on every read as a [signed attachment URL](/http-api/messages/#signed-attachment-urls)<sup>4</sup> Set when the source attachment or embed was already animated, and when the Media Proxy reports the stored copy as animateddiff --git a/fluxer_docs/src/content/docs/http-api/messages.mdx b/fluxer_docs/src/content/docs/http-api/messages.mdxindex 24f96c80a..97dd6a95a 100644--- a/fluxer_docs/src/content/docs/http-api/messages.mdx+++ b/fluxer_docs/src/content/docs/http-api/messages.mdx@@ -150,8 +150,8 @@ An attachment is one file stored against a message and served from media deliver| content_type<sup>1</sup> | string | The media type detected for the attachment || content_hash | ?string | The hash of the attachment's stored bytes, or null when they were never hashed || size | integer | The stored size of the attachment in bytes |-| url<sup>2</sup> | ?string | Public attachment URL, or null once the attachment has expired |-| proxy_url<sup>2</sup> | ?string | Media Proxy URL, or null once the attachment has expired |+| url<sup>2</sup> | ?string | Signed public attachment URL, or null once the attachment has expired |+| proxy_url<sup>2</sup> | ?string | Signed Media Proxy URL, or null once the attachment has expired || width?<sup>3</sup> | integer | Pixel width for visual media || height?<sup>3</sup> | integer | Pixel height for visual media || placeholder? | string | Encoded media placeholder |@@ -164,14 +164,47 @@ An attachment is one file stored against a message and served from media deliver<sup>1</sup> Always present. It falls back to a type guessed from the stored filename when the attachment has no recorded media type, and to `application/octet-stream` when the filename yields no guess-<sup>2</sup> Both fields are the same canonical media endpoint URL, and both become null together once the attachment has expired+<sup>2</sup> Both fields are the same media endpoint URL with the same [signature](#signed-attachment-urls), and both become null together once the attachment has expired<sup>3</sup> Omitted for an audio media type even when a stored dimension exists-<sup>4</sup> The key is always present. Its value is the recorded decay deadline, or null when the attachment has none+<sup>4</sup> The key is always present. Its value is the recorded decay deadline, or null when the attachment has none. It is unrelated to the `ex` time of a signed URL<sup>5</sup> Present and true only when the attachment has already expired, so an absent key must be read as false+A new attachment starts with the media type derived from the filename given when its upload was requested. For a pre-uploaded attachment that is the `filename` of its [attachment upload request item](#attachment-upload-request-item-object), which can differ from the `filename` the message sends. When the starting type is not a supported image, video, or audio type, Fluxer checks the first 8 KiB of the file for one. For an image, video, or audio file the Media Proxy can read, `content_type` is then the type detected from its bytes. When the starting type is not a media type, SVG bytes keep it. So does a file Fluxer cannot check or the Media Proxy cannot read.++### Signed attachment URLs++An instance configured with attachment URL secrets signs `url` and `proxy_url`. Fluxer adds the signature parameters to the attachment path, before any other parameter. An instance without those secrets returns the URL with none of them.++| Field | Type | Description |+| --- | --- | --- |+| ex | string | The Unix time in seconds at which the URL expires, as 8 lowercase hexadecimal digits, or `0` on a data package URL |+| is | string | The Unix time in seconds at which the URL was issued, as 8 lowercase hexadecimal digits |+| hm | string | The signature over `ex`, `is`, `uc`, and the attachment path, as 64 lowercase hexadecimal digits |+| uc | string | Present only on a data package URL, where the value is always `dp` |++An ordinary URL has `ex`, `is`, and `hm`, and no `uc`. It is valid for 24 hours from `is`. `is` starts at the attachment's creation time and moves forward in steps of 12 hours, so a URL Fluxer returns has more than 12 and at most 24 hours left. Fluxer signs the URL each time it returns the attachment, so a later read of the message returns a URL for the current step. [Refresh attachment URLs](#refresh-attachment-urls) signs a URL again without reading the message.++A client MUST keep `ex`, `is`, `hm`, and `uc` exactly as issued. It MAY add any other query parameter, such as a [transformation](/media-proxy/transformations/), because the signature covers no other parameter. A client SHOULD refresh a URL whose `ex` is absent or less than one hour away. Once the operator enforces signatures, a read with a missing, altered, or expired signature returns 404 from the Media Proxy, as [Signed attachment URLs](/media-proxy/overview/#signed-attachment-urls) defines.++Fluxer never rewrites message `content` or embed text. An attachment URL pasted into either keeps the parameters it was sent with, and it stops working when its own `ex` passes.++### Data package attachment URLs++A [data harvest](/http-api/users/data-harvest/) archive records a data package URL for each attachment instead of an ordinary one. It has `uc=dp`, and its `ex` is the single character `0`:++```text+?ex=0&is={issued}&hm={signature}&uc=dp+```++A data package URL does not expire. The Media Proxy checks its signature against every configured secret and skips the expiry check, so it reads its object for as long as the secret that signed it stays in the instance's secret list. Removing that secret is the only thing that ends it, and it ends every data package URL that secret signed at once.++`is` is set the way it is on an ordinary URL, and the signature covers `uc`, so an ordinary URL relabelled `uc=dp` does not verify, and neither does a data package URL with `uc` removed.++A client treats a data package URL as never expiring and does not refresh it before use. A refused read still triggers the same forced refresh as any other attachment URL, and that returns an ordinary URL.+### Example```json@@ -183,8 +216,8 @@ An attachment is one file stored against a message and served from media deliver"content_type": "image/png","content_hash": "9f86d081884c7d659a2feaa0c55ad015","size": 18422,- "url": "https://media.example.com/attachments/1501314428688998182/1501320000000000001/diagram.png",- "proxy_url": "https://media.example.com/attachments/1501314428688998182/1501320000000000001/diagram.png",+ "url": "https://media.example.com/attachments/1501314428688998182/1501320000000000001/diagram.png?ex=69fba4b5&is=69fa5335&hm=5586be3ae70411fe6582431649e2f28abc6c201e4f12a4f43a3256b72b90d3de",+ "proxy_url": "https://media.example.com/attachments/1501314428688998182/1501320000000000001/diagram.png?ex=69fba4b5&is=69fa5335&hm=5586be3ae70411fe6582431649e2f28abc6c201e4f12a4f43a3256b72b90d3de","width": 1280,"height": 720,"flags": 0,@@ -440,6 +473,10 @@ Every optional field above is omitted entirely.<sup>1</sup> Always derived from `url`, so every serialised media object has it+When `url` is an attachment URL of this instance, Fluxer signs both fields, so `url` and `proxy_url` are the same URL with the same fresh [signature](#signed-attachment-urls). A rich embed can name such a URL with or without `ex`, `is`, `hm`, and `uc`. Fluxer removes them before it stores the URL, and signs both fields again each time it returns the embed.++Fluxer signs every other embed field that can hold an attachment URL of this instance the same way. That is `url` on the embed itself, `url`, `icon_url`, and `proxy_icon_url` on an [embed author](#embed-author-object), `icon_url` and `proxy_icon_url` on an [embed footer](#embed-footer-object), and `url` on an [embed provider](#embed-provider-object). An icon URL and its proxy twin are then the same signed URL. A field whose URL is not an attachment URL of this instance is returned as it was stored, and a proxy field of an external URL is the [signed external media](/media-proxy/routes/#get-signed-external-media) path instead.+## Embed footer object### Structure@@ -660,7 +697,7 @@ One declared upload inside a [Request attachment upload URLs](#request-attachmen<sup>1</sup> A decimal string is converted to the integer-<sup>2</sup> The issued upload URL and the stored attachment both use the media type derived from `filename`+<sup>2</sup> The issued upload URL uses the media type derived from `filename`. The [message attachment object](#message-attachment-object) states how the stored attachment is typed### Singlepart attachment upload object@@ -726,6 +763,28 @@ One planned upload named in a [Complete attachment upload](#complete-attachment-| upload_filename | string | Opaque upload key returned when the upload was planned || upload_id | string | Multipart upload identifier returned when the upload was planned |+## Refreshed attachment URL object++One result of [Refresh attachment URLs](#refresh-attachment-urls).++### Structure++| Field | Type | Description |+| --- | --- | --- |+| original | string | The URL exactly as it was sent |+| refreshed<sup>1</sup> | string | The URL with a fresh [signature](#signed-attachment-urls), or `original` unchanged |++<sup>1</sup> Equal to `original` when the URL is not an attachment URL of this instance, or when the instance has no attachment URL secrets++A URL is an attachment URL of this instance when all of these hold:++- It parses as an absolute URL whose scheme, host, and port equal those of the media endpoint. The host is compared without regard to case, and a default port equals no port.+- Its path, as written in the string, starts with the media endpoint path followed by `/attachments/`.+- The rest of the path, percent-decoded, is valid UTF-8 and a safe storage key with no empty, `.`, or `..` segment.+- That key is `attachments/{channel_id}/{attachment_id}/{filename}`, where both IDs are decimal digits and the filename is not empty.++To sign a URL, Fluxer removes every `ex`, `is`, `hm`, and `uc` parameter and every empty pair, then places the fresh `ex`, `is`, and `hm` first. Every other parameter follows in its original order and bytes, and a fragment is kept. A [data package URL](#data-package-attachment-urls) loses its `uc` this way, so the result is an ordinary URL valid for 24 hours.+## Channel pin objectA channel pin pairs one pinned message with the time it was pinned.@@ -1019,6 +1078,51 @@ The operation makes each completed upload available for a message. It creates no10 requests per 10 seconds for each authenticated user and channel ID, on the `channel:attachment:upload::channel_id` bucket, which is shared with [Request attachment upload URLs](#request-attachment-upload-urls).+## Refresh attachment URLs++<RouteHeader method="POST" path="/v1/attachments/refresh-urls" bot />++Signs from 1 through 50 attachment URLs again. Returns one [refreshed attachment URL](#refreshed-attachment-url-object) object for each URL sent. A [data package URL](#data-package-attachment-urls) is accepted like any other, and its result is an ordinary URL valid for 24 hours.++### Limitations++- An OAuth2 bearer token is refused with 403 `ACCESS_DENIED`.+- Fluxer checks no channel membership, no permission, and no attachment existence, and reads no stored data. A URL is signed whether or not its object exists or the caller can see it.++### JSON body++| Field | Type | Description |+| --- | --- | --- |+| attachment_urls<sup>1</sup> | array[string] | URLs to sign again (1-50 entries, each at most 2,048 characters) |++<sup>1</sup> Each entry is read exactly as sent, with no trimming or normalisation++### Response body++| Field | Type | Description |+| --- | --- | --- |+| refreshed_urls<sup>1</sup> | array[[refreshed attachment URL](#refreshed-attachment-url-object) object] | Results in request order |++<sup>1</sup> Contains exactly one entry for each requested URL, in the order the entries were sent, and a duplicated URL therefore produces a duplicated entry++### Response++| Status | Body | Condition |+| --- | --- | --- |+| 200 | response body | Every URL was processed |+| 400 | [error response](/http-api/#error-response) | Body, URL count, or a URL length is invalid and the request returns `INVALID_FORM_BODY` |+| 401 | [error response](/http-api/#error-response) | The credential is missing or invalid and the request returns `UNAUTHORIZED` |+| 403 | [error response](/http-api/#error-response) | Caller presents a bearer credential and the request returns `ACCESS_DENIED` |+| 403 | [error response](/http-api/#error-response) | The account has an outstanding required action and the request returns `ACCOUNT_SUSPICIOUS_ACTIVITY` |++:::caution[A refreshed URL is not an access check]+Anyone who can authenticate can sign any attachment URL of the instance whose path they know. A signature limits how long a copied URL works. It does not decide who can read the file.+:::++### Rate limit++20 requests per 10 seconds for each authenticated user, on the `attachment:refresh_urls` bucket.+## Create message<RouteHeader method="POST" path="/v1/channels/{channel_id}/messages" bot />diff --git a/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx b/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdxindex 5e701be88..248f12c8a 100644--- a/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx+++ b/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx@@ -189,7 +189,7 @@ Fluxer prepares the archive in the background. It contains:- `account/security.json`- the account avatar and banner under `assets/user/` when present-Attachment metadata appears with its message and has the attachment ID, filename, size, content type, CDN URL, and pixel dimensions. The attachment files themselves are never included.+Attachment metadata appears with its message and has the attachment ID, filename, size, content type, CDN URL, and pixel dimensions. The CDN URL is a [data package URL](/http-api/messages/#data-package-attachment-urls), so it does not expire and still reads its attachment after the archive's own seven day deadline passes. Removing the secret that signed it is the only thing that ends it. The attachment files themselves are never included.Every authored message is collected, with no ceiling on the count. Messages that no longer exist are omitted. Other read failures fail the attempt. After completion, Fluxer attempts to email a download URL if the account has an email address and the instance has email delivery enabled. That URL expires seven days after it was issued. Email failure does not prevent downloading a completed archive.diff --git a/fluxer_docs/src/content/docs/media-proxy/overview.md b/fluxer_docs/src/content/docs/media-proxy/overview.mdindex bb5f1731b..554843835 100644--- a/fluxer_docs/src/content/docs/media-proxy/overview.md+++ b/fluxer_docs/src/content/docs/media-proxy/overview.md@@ -17,7 +17,7 @@ The Media Proxy serves Fluxer attachments, image assets, themes, entrance sound| Image asset<sup>1</sup> | `/avatars`, `/icons`, `/branding`, `/banners`, `/splashes`, `/embed-splashes`, `/emojis`, `/stickers` | [Image asset contract](/media-proxy/routes/#image-asset-contract) || Static object<sup>2</sup> | `/{key}` | [Get static object](/media-proxy/routes/#get-static-object) || Upload relay | `/v1/relay/{key}` | [Upload relay](/media-proxy/upload-relay/) |-| Operator and internal | `/_health`, `/_metrics`, `/_metadata`, `/_thumbnail`, `/_frames` | [Operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) |+| Operator and internal | `/_health`, `/_metrics`, `/_metadata`, `/_sniff`, `/_thumbnail`, `/_frames` | [Operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) |<sup>1</sup> Guild member avatars and banners are image assets too, at `/guilds/{guild_id}/users/{user_id}/avatars` and `/guilds/{guild_id}/users/{user_id}/banners`@@ -33,10 +33,10 @@ The upload relay is the exception in both respects. It is served below `/v1/rela## Authorisation-No public Media Proxy route reads the HTTP API `Authorization` header. A stored object is authorised by its path, signed external media by its path signature, and an upload by the capability in its URL. Only the internal `POST` endpoints read `Authorization`, and they require the exact value `Bearer {secret}` built from the deployment secret.+No public Media Proxy route reads the HTTP API `Authorization` header. An attachment is authorised by its path and, under the [signed attachment URL policy](#signed-attachment-urls), by the signature in its query string. Any other stored object is authorised by its path, signed external media by its path signature, and an upload by the capability in its URL. Only the internal `POST` endpoints read `Authorization`, and they require the exact value `Bearer {secret}` built from the deployment secret.:::caution[A media URL is a bearer capability]-An attachment path has no signature and no expiry, so anyone holding the URL reads the object for as long as it exists. Leaving the channel does not revoke a URL already issued.+Anyone holding a URL reads the object without an account. Under `enforce`, an ordinary signed attachment URL works until its `ex` time, which is at most one day after it was signed, and a [data package URL](/http-api/messages/#data-package-attachment-urls) works until the secret that signed it is removed. Under `off` or `report`, an attachment path works for as long as the object exists. Leaving the channel does not revoke a URL already issued.:::The Media Proxy has no request-count rate limit, so no response has the [rate limit headers](/topics/rate-limits/#rate-limit-headers) of the versioned HTTP API. A failure returns a short `text/plain` body, and [media error response](/media-proxy/responses-and-limits/#media-error-response) defines that shape.@@ -50,10 +50,11 @@ One Media Proxy process serves exactly one mode. The mode is fixed at startup an| `mp` | Attachments, signed external media, themes, entrance sounds, and every image asset route || `static` | Every read route as a raw object read from the static bucket, with no transformation and no SVG rasterisation<sup>1</sup> || `upload` | The [upload relay](/media-proxy/upload-relay/), plus every `mp` read route from the same buckets |+| `relay` | The [upload relay](/media-proxy/upload-relay/) alone. Every read path returns 404 |<sup>1</sup> A `static` mode endpoint also strips `X-Robots-Tag` from every response and sends no `Content-Disposition`-The relay `PUT` is the only route with a mode gate, and it returns 404 outside `upload` mode. On a read that requests no transformation, only an `mp` endpoint rasterises SVG, so an `upload` endpoint returns the original SVG bytes. The [operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) behave the same in every mode.+The relay `PUT` returns 404 outside `upload` and `relay` mode. A `relay` endpoint returns 404 for a `GET` or `HEAD` of any read path and reads no object for it. On a read that requests no transformation, only an `mp` endpoint rasterises SVG, so an `upload` endpoint returns the original SVG bytes. The [operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) behave the same in every mode.The operator chooses which mode serves each published base URL. The reference self-hosted deployment serves `endpoints.media` from an `upload` mode process.@@ -67,32 +68,102 @@ On the [signed external route](/media-proxy/routes/#get-signed-external-media),## Request headers-Only `Range` affects the representation a public read returns. `X-Forwarded-For` decides which address the [media access allowlist](#access-restrictions) evaluates, so it can turn a 200 into a 403 without changing the representation. Every other request header, including `Authorization`, `Cookie`, `Accept`, `Origin`, `If-Range`, `If-None-Match`, `If-Modified-Since`, `If-Match`, and `If-Unmodified-Since`, is ignored on a public read route.+Only `Range` affects the representation a public read returns. The [cross-origin read policy](#cross-origin-reads) reads `Origin`, and under `enforce` that header decides `Access-Control-Allow-Origin` and can turn a 200 into a 403. Every other request header, including `Authorization`, `Cookie`, `Accept`, `If-Range`, `If-None-Match`, `If-Modified-Since`, `If-Match`, and `If-Unmodified-Since`, is ignored on a public read route.### Common request headers| Field | Type | Description || --- | --- | --- || Range?<sup>1</sup> | string | One byte range under the [byte-range contract](#byte-ranges) |-| X-Forwarded-For?<sup>2</sup> | string | Client address chain read by the [media access allowlist](#access-restrictions) |+| Origin?<sup>2</sup> | string | Serialised request origin read by the [cross-origin read policy](#cross-origin-reads) |<sup>1</sup> A value whose unit is not the exact lowercase `bytes`, or that names more than one range, is ignored, and the response has the complete representation-<sup>2</sup> Read only when the allowlist gate is enabled and the peer address is a configured trusted proxy. A value from any other peer is ignored+<sup>2</sup> Read only when the cross-origin read policy is `report` or `enforce`-## Access restrictions+## Cross-origin reads-An operator can restrict public reads to a CDN edge address allowlist. This restriction is disabled by default.+An operator can limit which web origins read media through CORS. `FLUXER_MEDIA_PROXY_CORS_MODE` sets this cross-origin read policy, and it defaults to `off`.-A request from an address outside the list returns 403. `/_health`, `/_metrics`, `/_metadata`, `/_thumbnail`, `/_frames`, and every path below `/v1/relay/` are exempt.+| Value | Description |+| --- | --- |+| `off` | A media representation has `Access-Control-Allow-Origin: *`, and no `Origin` is read |+| `report` | Every response is the same as under `off`, and each read `enforce` would refuse is logged |+| `enforce` | A read from an origin outside the allowlist returns 403 |++Under `enforce`, an `Origin` is allowed when it equals the serialised form of an entry in `FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS` byte for byte. Anything else is refused, including `null`, a repeated `Origin` header, and a value that is not ASCII. A refused read returns 403 on every read path, and the Media Proxy reads no object and fetches no external URL for it. [Media Proxy settings](/operator/configuration/#media-proxy-settings) defines how an entry is parsed and serialised.++A read with no `Origin` is served without `Access-Control-Allow-Origin`. A browser sends no `Origin` for an `<img>` or a `<video>` without `crossorigin`, so those elements still load. An allowed read gets its own origin back in `Access-Control-Allow-Origin` on a media representation and on a 416. No error has CORS headers.++Under `enforce`, every response to a `GET` or `HEAD` on a read path has `Vary: Accept-Encoding, Origin`, the 403 for a refused `Origin` included.++`report` changes no response. The Media Proxy logs a warning with the reason `cors_origin_would_deny` for each read `enforce` would refuse.++The policy applies to a response the Media Proxy generates, and a shared cache in front of it answers a hit on its own. An entry stored while the policy was `off` holds `Access-Control-Allow-Origin: *` and a year of `Cache-Control`, and a foreign origin reading that URL gets both back until the entry goes. Purge `/attachments/` when you move the policy to `enforce`. [Caching in front of Fluxer](/operator/reverse-proxy/#caching-in-front-of-fluxer) states how.++A `static` or `relay` endpoint ignores the policy. The [upload relay](/media-proxy/upload-relay/) and the [operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) are exempt from it. A method other than `GET` or `HEAD` on a read path returns 405 whatever its `Origin`.++The reference self-hosted deployment leaves the policy `off`. An operator who turns it on gets the instance public origin as the only entry, and [Media Proxy settings](/operator/configuration/#media-proxy-settings) states how to allow `https://web.fluxer.app` as well.++## Signed attachment URLs++An operator can require a signature on every attachment read. `FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE` sets this signed attachment URL policy, and it defaults to `off`. The reference self-hosted deployment leaves it `off`, and turning it on takes a secret and the mode. The Media Proxy reads the mode once at start, and [Turning the signature policy on](/operator/reverse-proxy/#turning-the-signature-policy-on) gives the order to move it and a cache in front of it, and what each pairing of the two serves.++| Value | Description |+| --- | --- |+| `off` | No signature is read, and an attachment read is served whatever its query string |+| `report` | Every attachment read is served, and each read `enforce` would refuse is logged |+| `enforce` | An attachment read without a valid signature returns 404 |++An instance with attachment URL secrets signs every attachment URL it returns, and [Refresh attachment URLs](/http-api/messages/#refresh-attachment-urls) signs one again. An ordinary signed URL has three query parameters, and Fluxer places them first in this order:++```text+?ex={expires}&is={issued}&hm={signature}+```++`ex` and `is` are each exactly 8 lowercase hexadecimal digits, the Unix time in seconds at which the URL expires and at which it was issued. `hm` is exactly 64 lowercase hexadecimal digits. Any other parameter follows them after a single `&`. A client MUST keep them exactly as issued and MAY add or change any other parameter.-When the peer address is a configured trusted proxy, the client address is the rightmost `X-Forwarded-For` hop that is not itself a trusted proxy. Otherwise the Media Proxy uses the peer address and ignores `X-Forwarded-For`. It does the same when there is no `X-Forwarded-For`, or when every hop in it is unparseable or is itself a trusted proxy.+A [data package URL](/http-api/messages/#data-package-attachment-urls) has a fourth parameter, `uc`, and its `ex` is the single character `0`:++```text+?ex=0&is={issued}&hm={signature}&uc=dp+```++`uc` is exactly `dp` and nothing else. `is` and `hm` have the same shape as on an ordinary URL. The 8-digit rule for `ex`, and the rule that `is` is no later than `ex`, hold for an ordinary URL alone. A data package URL never expires.++The Media Proxy reads the raw query string. It splits it on `&`, skips empty fields, and splits each field at its first `=`. It form-decodes each name and takes each value exactly as sent. A read is refused when any of these holds:++- There is no query string, or none of `ex`, `is`, `hm`, and `uc` is present.+- One or two of `ex`, `is`, and `hm` are present, so `uc` alone is refused too.+- Any of the four occurs more than once.+- `uc` is present with a value other than `dp`.+- `ex` is not 8 lowercase hexadecimal digits on an ordinary URL, or not the single character `0` on a data package URL.+- `is` is not 8 lowercase hexadecimal digits, or `hm` is not 64 lowercase hexadecimal digits.+- `is` is later than `ex` on an ordinary URL.+- `hm` does not verify against any configured secret.+- The current time is at or after `ex` on an ordinary URL.++A shared cache in front of the Media Proxy must leave the signature parameters `ex`, `is`, `hm`, and `uc` out of its cache key, and must decode each parameter name before it compares it. A cache that keeps them stores a new entry every time the same object is signed again. [Caching in front of Fluxer](/operator/reverse-proxy/#caching-in-front-of-fluxer) states the rest of the rules.++Every other parameter is ignored, and so is the order of the parameters. The signature is over `ex`, `is`, `uc`, and the percent-decoded storage key, and never over the host, the scheme, the port, or another parameter. Because `uc` is signed, an ordinary URL relabelled `uc=dp` does not verify, and neither does a data package URL with `uc` removed. One signed URL therefore reads every [transformation](/media-proxy/transformations/) and `download` variant of its object. Every percent-encoded spelling of the same key verifies the same way, and a key whose decoded bytes are not valid UTF-8 is refused.++Under `enforce`, a refused read returns 404 with the body `This content is no longer available.`, `Cache-Control: no-store`, and no `Access-Control-Allow-Origin`. The Media Proxy reads no object for it. The response is the same for every reason, for `GET` and `HEAD`, with or without a `Range`, and whatever transformation or `download` the request names. A read whose object does not exist returns that same 404 under `enforce`, even with a valid signature, so a response never tells a missing object apart from a refused signature. `off` and `report` return the ordinary `Not found` body for a missing object.++The Media Proxy checks the method first, then the `relay` mode, then the [cross-origin read policy](#cross-origin-reads), then the storage key, and only then the signature. A refused `Origin` returns 403 and an invalid storage key returns 400, and neither reads a signature.++`report` refuses nothing. Every status and body under `report` is the one `off` returns, and so is every header apart from the cache policy. A read whose signature is valid gets the same capped `Cache-Control` and `CDN-Cache-Control` that `enforce` sets, so the entries a cache stores during the dry run do not outlive the URLs that made them. A read `report` would refuse keeps the one-year policy `off` returns. The Media Proxy logs a warning with the reason `attachment_signature_would_deny` for each read `enforce` would refuse. The warning has the verdict `missing`, `malformed`, `mismatch`, or `expired`, the request identifier, and the user agent. Under every mode, the request log writes the values of `ex`, `is`, `hm`, and `uc` as `[redacted]`.++A `static` or `relay` endpoint ignores the policy. The policy covers `/attachments/` paths alone, so signed external media, image assets, themes, and entrance sounds need no signature. The [operator and internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) are exempt, and they read an attachment URL of this instance without a signature.++:::caution[A signature is not access control]+Refresh attachment URLs signs any well-formed attachment URL of the instance for any signed-in user or bot, with no membership check. A signature limits how long a copied URL works without a refresh. It does not decide who reads the object.+:::## Selector parsingA read route names its representation in its query string. The attachment, signed external, and image asset routes accept selectors. The theme and entrance sound routes accept none and ignore any query string.-Query names and values use URL form decoding, so `+` decodes to a space and a percent escape decodes to its byte. When a name occurs more than once, the final value wins. An unknown name is ignored.+Query names and values use URL form decoding, so `+` decodes to a space and a percent escape decodes to its byte. When a name occurs more than once, the final value wins. An unknown name is ignored. The [signed attachment URL policy](#signed-attachment-urls) reads `ex`, `is`, `hm`, and `uc` under its own rules.The Media Proxy does not redirect alternative spellings to a canonical URL.@@ -126,18 +197,18 @@ Content disposition follows the media type the origin declared, so SVG mislabell| Field | Type | Description || --- | --- | --- |-| Content-Type<sup>1</sup> | string | The detected or selected media type |+| Content-Type<sup>1</sup> | string | The detected or selected media type, or `text/plain; charset=utf-8` for a JavaScript type outside `static` mode || Content-Length?<sup>2</sup> | integer | The selected body length, including on HEAD || Content-Range? | string | Present on 206 as `bytes {start}-{end}/{size}` and on 416 as `bytes */{size}` || Content-Disposition?<sup>3</sup> | string | The route-selected inline or attachment disposition || Accept-Ranges | string | The literal `bytes` on every media representation |-| Access-Control-Allow-Origin | string | The literal `*` on media responses |+| Access-Control-Allow-Origin? | string | The literal `*` on media responses. Under `enforce` it is the request origin, or absent with no `Origin` || Cache-Control<sup>4</sup> | string | The browser cache policy for the representation || CDN-Cache-Control | string | The matching shared cache policy |-| Vary | string | The literal `Accept-Encoding` |+| Vary | string | The literal `Accept-Encoding`, or `Accept-Encoding, Origin` under `enforce` || X-Content-Type-Options | string | The literal `nosniff` || X-Robots-Tag<sup>5</sup> | string | The indexing policy |-| X-Fluxer-Version<sup>6</sup> | string | The serving Fluxer build identifier, or `dev` when none is configured |+| X-Fluxer-Version | string | The serving Fluxer build identifier, or `dev` when none is configured |<sup>1</sup> Fixed at `text/css; charset=utf-8` on the theme route. A 416 response has none@@ -149,8 +220,6 @@ Content disposition follows the media type the origin declared, so SVG mislabell<sup>5</sup> The literal `noindex, nofollow, nosnippet, noimageindex, notranslate, max-snippet:0, max-image-preview:none, max-video-preview:0`. Present in `mp` and `upload` mode, and a `static` mode endpoint removes it-<sup>6</sup> Absent from a media access allowlist rejection-Every successful media representation uses `Cache-Control: public, max-age=31536000` and `CDN-Cache-Control: public, max-age=31536000`. An audio or video representation adds `no-transform` to `Cache-Control` only. No response repeats its policy in an `Expires` header. [Cache policies](/media-proxy/responses-and-limits/#cache-policies) lists every cache policy this surface sets, including the responses that set none.The upload relay is the only route that returns an `ETag`, and it relays the object storage value for the stored object. No read route sends an `ETag` or a `Last-Modified`, so a cache revalidates a representation by fetching it again.@@ -159,12 +228,18 @@ The upload relay is the only route that returns an `ETag`, and it relays the objThe signed path is derived from the target URL, so the bytes behind one unchanged target are cached for a year by browsers under `Cache-Control` and by shared caches under `CDN-Cache-Control`.:::-A 416 response has `Accept-Ranges`, `Content-Range`, `Access-Control-Allow-Origin`, `Vary`, and `X-Robots-Tag`, with no `Content-Type` and no cache policy. Its body is empty.+A 416 response has `Accept-Ranges`, `Content-Range`, `Access-Control-Allow-Origin`, `Vary`, and `X-Robots-Tag`, with no `Content-Type` and no cache policy. Its body is empty. The [cross-origin read policy](#cross-origin-reads) sets its `Access-Control-Allow-Origin` and `Vary` the same way it does on a 200.## Content detectionUse the response's `Content-Type`. It can differ from the filename extension or the origin's declared type.+An endpoint in any mode other than `static` replaces a `Content-Type` that names a JavaScript media type with `text/plain; charset=utf-8`, at any status. It splits the value at every comma and reads each part on its own. In each part it skips leading spaces and tabs, then reads the type up to the first space, tab, semicolon, or opening parenthesis. The header is replaced when that type in any part equals one of these, ignoring ASCII case: `application/ecmascript`, `application/javascript`, `application/x-ecmascript`, `application/x-javascript`, `text/ecmascript`, `text/javascript`, `text/javascript1.0` through `text/javascript1.5`, `text/jscript`, `text/livescript`, `text/x-ecmascript`, and `text/x-javascript`. So `text/javascript x`, `TEXT/JAVASCRIPT (x)`, and `video/mp4, text/javascript` are replaced, and `text/javascript1.6` and `text/javascriptx` are left as they are.++The endpoint also replaces a `Content-Type` that is not visible ASCII, at any status. A 200 or 206 with no `Content-Type`, or with one that is empty or only spaces and tabs, gets `text/plain; charset=utf-8` too. A response with any other status and no `Content-Type` keeps none, so a 416 and the empty-body method rejection of a registered path still have no `Content-Type`. `Content-Disposition` is unchanged, so it still names the original filename.++A `static` endpoint is the one endpoint that changes no `Content-Type`. An `upload` or `relay` endpoint applies the same replacement an `mp` endpoint does, which is what makes the reference self-hosted deployment safe, because it serves `endpoints.media` from an `upload` endpoint.+## Content dispositionDisposition follows the resolved media type. An image other than SVG and a video are served inline. Every other type, including SVG and PDF, is served as an attachment. An explicit `download` request forces attachment disposition on every route that accepts the parameter.diff --git a/fluxer_docs/src/content/docs/media-proxy/responses-and-limits.md b/fluxer_docs/src/content/docs/media-proxy/responses-and-limits.mdindex 90b923297..334b8d854 100644--- a/fluxer_docs/src/content/docs/media-proxy/responses-and-limits.md+++ b/fluxer_docs/src/content/docs/media-proxy/responses-and-limits.md@@ -14,9 +14,9 @@ An unsuccessful response body is an English reason phrase under the content type| --- | --- | --- || 400 | Bad request | An invalid storage key, dimension, format, or external path, an unparsable relay `partNumber`, a failed attachment or external transformation, or a relay upload body the endpoint failed to read from the client<sup>1</sup> || 401 | Unauthorized | An invalid signed external path signature, a missing, malformed, or expired relay capability, or a missing or invalid internal bearer token |-| 403 | Media access denied | The media access allowlist rejected the client address |-| 403 | Forbidden | A relay capability presented for another bucket, key, method, `uploadId`, or `partNumber`, or `/_metrics` requested from a non-loopback address |-| 404 | Not found | A `GET` or `HEAD` of an unrouted path, an object that does not exist, or a relay `PUT` outside `upload` mode |+| 403 | Forbidden | A relay capability presented for another bucket, key, method, `uploadId`, or `partNumber`, `/_metrics` requested from a non-loopback address, or a public read whose `Origin` the [cross-origin read policy](/media-proxy/overview/#cross-origin-reads) refuses |+| 404 | Not found | A `GET` or `HEAD` of an unrouted path or of any read path on a `relay` endpoint, an object that does not exist, or a relay `PUT` outside `upload` and `relay` mode |+| 404 | This content is no longer available.<sup>4</sup> | An attachment read `enforce` refuses, or a missing attachment object under `enforce` || 405 | Method not allowed<sup>2</sup> | A method other than `GET` or `HEAD` on a read route or on an unrouted path || 413 | Payload too large | A stored object, external body, upload body, or internal request body beyond its bound || 500 | Transcode failed | An image asset transcode failed and the source is not directly displayable |@@ -27,13 +27,15 @@ An unsuccessful response body is an English reason phrase under the content type<sup>1</sup> The relay also answers 400 when the client connection fails part way through the body-<sup>2</sup> The registered paths, `/_health`, `/_metrics`, `/_metadata`, `/_thumbnail`, `/_frames`, and `/v1/relay/{key}`, answer an unaccepted method with an empty body, no `Content-Type`, and an `Allow` header+<sup>2</sup> The registered paths, `/_health`, `/_metrics`, `/_metadata`, `/_sniff`, `/_thumbnail`, `/_frames`, and `/v1/relay/{key}`, answer an unaccepted method with an empty body, no `Content-Type`, and an `Allow` header. All seven are registered in every mode, so the method rejection is the same in every mode, even where the path itself answers 404<sup>3</sup> `/_metadata` is the only endpoint that remaps an origin 429. The signed external read route retains 429 as 429+<sup>4</sup> Sent under the `enforce` [signed attachment URL policy](/media-proxy/overview/#signed-attachment-urls) alone, where it also replaces `Not found` for an attachment object that does not exist. Under `report` every response is the one `off` returns, so a missing object keeps the `Not found` body there+When the Media Proxy passes an external origin status through to the client, the body is `Upstream fetch failed` on the signed external read route. On `/_metadata`, the body is the canonical reason phrase of that status. An object store error that maps to no case above uses the canonical reason phrase of its status.-Every error a route produces uses `Cache-Control: no-store` and the standard [security headers](/media-proxy/overview/#representation-headers). A 416 response, a media access allowlist rejection, and the empty-body method rejection of a registered path have the security headers and set no cache policy, and [Cache policies](#cache-policies) names them. No plain-text error has CORS headers unless it came from the [upload relay](/media-proxy/upload-relay/), and no error has `Retry-After` or a request identifier.+Every error a route produces uses `Cache-Control: no-store` and the standard [security headers](/media-proxy/overview/#representation-headers). The exceptions have the security headers and set no cache policy, and [Cache policies](#cache-policies) names them. No plain-text error has CORS headers unless it came from the [upload relay](/media-proxy/upload-relay/), and no error has `Retry-After` or a request identifier. Under the `enforce` [cross-origin read policy](/media-proxy/overview/#cross-origin-reads), every `GET` or `HEAD` error on a read path has `Vary: Accept-Encoding, Origin`. That includes the 403 for a refused `Origin`.### Handling contract@@ -49,8 +51,8 @@ Every `HEAD` response has an empty body, so the Body column describes `GET`, `PU| 206 | Selected media bytes for `GET` and empty for `HEAD` | One byte range is satisfiable || 400 | [Media error response](#media-error-response) | A path, query, target, transformation, or relay request is invalid || 401 | [Media error response](#media-error-response) | An external path signature, a relay capability, or an internal token is invalid |-| 403 | [Media error response](#media-error-response) | The media access policy denied the request, a relay capability does not match the request, or an external origin returned 403 |-| 404 | [Media error response](#media-error-response) | The route or object does not exist, the relay is not served here, or an external origin returned 404 |+| 403 | [Media error response](#media-error-response) | The cross-origin read policy denied the request, a relay capability does not match the request, or an external origin returned 403 |+| 404 | [Media error response](#media-error-response) | The route or object does not exist, a `relay` endpoint received a read, the relay is not served here, an attachment signature was refused, or an external origin returned 404 || 405<sup>2</sup> | [Media error response](#media-error-response) | The route rejects the method, or an external origin returned 405 || 406, 408, 409, 410, 411, 412, 414, 415, 428, 429<sup>3</sup> | [Media error response](#media-error-response) | An external origin returned that status || 413 | [Media error response](#media-error-response) | A source, external response, or upload exceeds its applicable bound |@@ -60,7 +62,7 @@ Every `HEAD` response has an empty body, so the Body column describes `GET`, `PU| 503 | [Media error response](#media-error-response) | Upload spool capacity is exhausted, an external buffer reservation or allocation failed, or an external origin answered `/_metadata` with 429<sup>5</sup> || 504 | [Media error response](#media-error-response) | Transformation capacity was unavailable or a transformation deadline expired |-<sup>1</sup> The [internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) `/_metadata` and `/_frames` answer 200 with JSON, `/_health` with plain text, and `/_metrics` with the Prometheus text exposition+<sup>1</sup> The [internal endpoints](/media-proxy/routes/#operator-and-internal-endpoints) `/_metadata`, `/_sniff`, and `/_frames` answer 200 with JSON, `/_health` with plain text, and `/_metrics` with the Prometheus text exposition<sup>2</sup> A read route path, or any other path outside the registered paths, answers with the [media error response](#media-error-response). Each registered path answers with an empty body, no `Content-Type`, and an `Allow` header@@ -88,7 +90,7 @@ Decoded images are limited to 16,384 pixels on either edge and 268,435,456 pixelThe upload relay limits a body to the smaller of the authorised upload size and the endpoint body limit. That endpoint limit defaults to 500 MiB and can be configured from 1 byte through 5 GiB.-An internal `/_metadata`, `/_thumbnail`, or `/_frames` request body is limited to the base64 expansion of the 500 MiB media bound plus 1 MiB. All answer a larger body with 413.+An internal `/_metadata`, `/_sniff`, `/_thumbnail`, or `/_frames` request body is limited to the base64 expansion of the 500 MiB media bound plus 1 MiB. All answer a larger body with 413.## Work admission@@ -118,13 +120,17 @@ A status and its headers are chosen before the body is sent. A streamed object sEvery successful media representation uses `Cache-Control: public, max-age=31536000` and `CDN-Cache-Control: public, max-age=31536000`. An audio or video representation appends `no-transform` to `Cache-Control` only. Signed external media uses the same policy as stored media.-No route sets `immutable`, `Expires`, `ETag`, or `Last-Modified` on a read response, and no route selects a shorter cache policy. A cache revalidates a representation by fetching it again.+No route sets `immutable`, `Expires`, `ETag`, or `Last-Modified` on a read response. A cache revalidates a representation by fetching it again.++One read selects a shorter policy. Under the `report` or `enforce` [signed attachment URL policy](/media-proxy/overview/#signed-attachment-urls), an attachment read whose signature is valid gets `public, max-age=` the seconds left on that signature in both headers, so no entry outlives the URL that made it. A data package URL never expires, so it gets the one-day ceiling instead. Every other read, and every attachment read under `off`, keeps the year.++A one-year `max-age` is only safe where a purge path reaches every cache that stores media. With no `ETag` and no `Last-Modified` there is nothing to revalidate against, so a deleted or replaced object is served until its entry is purged or drops out. A cache with no purge path must set a shorter lifetime of its own instead. [Caching in front of Fluxer](/operator/reverse-proxy/#caching-in-front-of-fluxer) states how.-Every route-produced error response uses `Cache-Control: no-store`, and a successful `/_metrics` read uses it too. A 416 response, a successful upload relay response, a media access allowlist rejection, and the empty-body method rejection of a registered path set no cache policy at all.+Every route-produced error response uses `Cache-Control: no-store`, and a successful `/_metrics` read uses it too. A 416 response, a successful upload relay response, and the empty-body method rejection of a registered path set no cache policy at all.## Range response headers-A complete media response has `Accept-Ranges: bytes`, the representation `Content-Type`, and an exact `Content-Length`<sup>1</sup>. A 206 also sets `Content-Range` to the selected interval over the complete size and `Content-Length` to the selected byte count. A 416 has `Content-Range: bytes */{size}` and `Accept-Ranges: bytes` with an empty body.+A complete media response has `Accept-Ranges: bytes`, the representation `Content-Type`, and an exact `Content-Length`<sup>1</sup>. Outside `static` mode that `Content-Type` is `text/plain; charset=utf-8` when it names a JavaScript media type, and a 200 or 206 with no `Content-Type` gets the same value. [Content detection](/media-proxy/overview/#content-detection) defines both rules. A 206 also sets `Content-Range` to the selected interval over the complete size and `Content-Length` to the selected byte count. A 416 has `Content-Range: bytes */{size}` and `Accept-Ranges: bytes` with an empty body.<sup>1</sup> A streamed signed external response omits `Content-Length` when the origin declared nonediff --git a/fluxer_docs/src/content/docs/media-proxy/routes.mdx b/fluxer_docs/src/content/docs/media-proxy/routes.mdxindex 322fb1f63..9a1486cb5 100644--- a/fluxer_docs/src/content/docs/media-proxy/routes.mdx+++ b/fluxer_docs/src/content/docs/media-proxy/routes.mdx@@ -20,8 +20,8 @@ Every media read route can produce these statuses in addition to the ones in its| Status | Body | Condition || --- | --- | --- |-| 403 | Media access denied | The request address is not permitted by the instance media access policy |-| 404 | Not found | The path is not a media route or the object does not exist |+| 403 | Forbidden | The [cross-origin read policy](/media-proxy/overview/#cross-origin-reads) refuses the request `Origin` on an `mp` or `upload` endpoint |+| 404 | Not found | The path is not a media route, the endpoint runs in `relay` mode, or the object does not exist || 405 | Method not allowed | The method is neither `GET` nor `HEAD` || 413 | Payload too large | The stored object exceeds the 500 MiB media bound || 502 | Bad gateway | The object store could not be read |@@ -30,10 +30,10 @@ Every media read route can produce these statuses in addition to the ones in its<RouteHeader method="GET" path="/attachments/{path}" unauthenticated />-Returns the attachment identified by the path, optionally transformed. The path comes from the attachment URL the HTTP API returned.+Returns the attachment identified by the path, optionally transformed. The path and its signature come from the attachment URL the HTTP API returned. Request that URL as issued. A transformation parameter can be added to its query string.:::caution[An attachment URL is a bearer capability]-The path has no signature and no expiry, so anyone holding the URL reads the object from any origin for as long as it exists. Leaving the guild does not revoke a URL a member already holds.+Anyone holding the URL reads the object from any origin the [cross-origin read policy](/media-proxy/overview/#cross-origin-reads) allows. Under the `enforce` [signed attachment URL policy](/media-proxy/overview/#signed-attachment-urls) it works until `ex`, and otherwise for as long as the object exists. A [data package URL](/http-api/messages/#data-package-attachment-urls) has no `ex` and works until the secret that signed it is removed. Leaving the guild does not revoke a URL a member already holds.:::### Path parameters@@ -55,9 +55,15 @@ The path has no signature and no expiry, so anyone holding the URL reads the obj| animated? | boolean | Whether animated output is requested || effort?<sup>1</sup> | integer | The WebP encoder effort, clamped to 9 || download? | boolean | Whether attachment disposition is requested |+| ex?<sup>2</sup> | string | The expiry of the signed URL, as 8 lowercase hexadecimal digits of Unix seconds, or `0` on a data package URL |+| is?<sup>2</sup> | string | The issue time of the signed URL, as 8 lowercase hexadecimal digits of Unix seconds |+| hm?<sup>2</sup> | string | The signature, as 64 lowercase hexadecimal digits |+| uc?<sup>2</sup> | string | The data package marker, always `dp` when present |<sup>1</sup> Read only on this route, and an empty or unparsable value is ignored. Only lossless animated WebP output uses a value above 6+<sup>2</sup> Read under the `report` or `enforce` [signed attachment URL policy](/media-proxy/overview/#signed-attachment-urls), which also defines how they are parsed. `enforce` requires `ex`, `is`, and `hm`, and reads `uc` on every URL, where its presence selects the [data package URL](/http-api/messages/#data-package-attachment-urls) grammar+[When a transformation runs](/media-proxy/transformations/#when-a-transformation-runs) defines the trigger.When a transformation is requested, `Range` selects bytes from the transformed result.@@ -73,7 +79,8 @@ A video source requires an explicit image `format`. A video request with `width`| 200 | Complete media representation | The object and selected representation are available || 206 | Selected media bytes | One range is satisfiable || 400 | Bad request | The decoded key, a dimension, the format, or the transformation request is invalid, or the transformation failed |-| 404 | Not found | The attachment object does not exist |+| 404 | Not found | The attachment object does not exist and the signed attachment URL policy is `off` or `report` |+| 404 | This content is no longer available. | The signed attachment URL policy refuses the signature, or the object does not exist under `enforce` || 416 | empty | The range is unsatisfiable || 504 | Gateway timeout | Transformation capacity was unavailable or the transformation exceeded its deadline |@@ -93,7 +100,7 @@ Fluxer constructs the signed path and exposes it on these fields:- `proxy_url` of a [resolved GIF entry](/http-api/memes/#resolved-gif-entry-object)- `proxy_src` of a [GIF media format object](/http-api/memes/#gif-media-format-object)-A field holds a signed path only when its source URL is external. A URL already under the Media Proxy endpoint, and any URL the deployment cannot sign, is returned verbatim, so a client MUST request the value exactly as issued and MUST NOT assume it has the signed shape.+A field holds a signed external path only when its source URL is external. A URL already under the Media Proxy endpoint keeps that endpoint, and gets an [attachment URL signature](/http-api/messages/#signed-attachment-urls) when it names an attachment of this instance. Any URL the deployment cannot sign is returned verbatim, so a client MUST request the value exactly as issued and MUST NOT assume it has the signed shape.### Path parameters@@ -488,14 +495,17 @@ The complete request path is the storage key after percent-decoding. The route a## Operator and internal endpoints-These paths are not part of the public API surface. They are exempt from the media access allowlist and are not published by instance discovery. A client MUST NOT depend on them.+These paths are not part of the public API surface and are not published by instance discovery. A client MUST NOT depend on them.| Path | Method | Description || --- | --- | --- || `/_health` | GET | Returns 200 while the process is running || `/_metrics` | GET | Returns the Prometheus text exposition of the process. A request from a non-loopback address returns 403 || `/_metadata` | POST | Extracts media metadata, a placeholder, and an optional NSFW verdict for the HTTP API |+| `/_sniff` | POST | Returns the media type detected in the first 8 KiB of a staged upload for the HTTP API || `/_thumbnail` | POST | Produces a WebP thumbnail of a staged upload for the HTTP API || `/_frames` | POST | Extracts one JPEG video frame for the HTTP API |-`/_metadata`, `/_thumbnail`, and `/_frames` require `Authorization: Bearer {deployment secret key}` and return 401 without it. Their JSON request bodies are bounded, and a body beyond the bound returns 413.+`/_metadata`, `/_sniff`, `/_thumbnail`, and `/_frames` require `Authorization: Bearer {deployment secret key}` and return 401 without it. Their JSON request bodies are bounded, and a body beyond the bound returns 413.++`/_sniff` returns 200 with a JSON `content_type`. It is the image, video, or audio media type the Media Proxy detects in the upload, or null when the bytes are SVG or a type it cannot process. An upload key that names no object returns 404.diff --git a/fluxer_docs/src/content/docs/media-proxy/upload-relay.mdx b/fluxer_docs/src/content/docs/media-proxy/upload-relay.mdxindex dcbc3e31a..672f397dd 100644--- a/fluxer_docs/src/content/docs/media-proxy/upload-relay.mdx+++ b/fluxer_docs/src/content/docs/media-proxy/upload-relay.mdx@@ -12,7 +12,7 @@ The upload relay is where a client sends the bytes of a file. Each upload URL haPaths here are relative to the relay base in the issued upload URL. That base is a deployment setting.-The capability is the complete authorisation. The relay reads no HTTP API `Authorization` header and has no request-count rate limit. Every path below `/v1/relay/` is exempt from the [media access policy](/media-proxy/overview/#access-restrictions). The `PUT` has a [deployment mode](/media-proxy/overview/#deployment-modes) gate and returns 404 outside `upload` mode.+The capability is the complete authorisation. The relay reads no HTTP API `Authorization` header and has no request-count rate limit. The `PUT` has a [deployment mode](/media-proxy/overview/#deployment-modes) gate and returns 404 outside `upload` and `relay` mode.:::note[Some upload URLs address object storage directly]An upload URL points directly to object storage when the caller's IP address geolocates to a country the deployment lists in `keep_direct_countries`. Use the URL returned by the HTTP API. This page describes relay URLs only.diff --git a/fluxer_docs/src/content/docs/operator/configuration.mdx b/fluxer_docs/src/content/docs/operator/configuration.mdxindex bdd9da078..787c6a957 100644--- a/fluxer_docs/src/content/docs/operator/configuration.mdx+++ b/fluxer_docs/src/content/docs/operator/configuration.mdx@@ -58,7 +58,7 @@ Default `http`. The scheme for internal service URLs. Must be `http` or `https`,## Secrets you must generate-Every value below ships as `CHANGE_ME`. Replace them all before the first start. Compose refuses to start when one is unset or empty, and names the variable. Generate each with `openssl rand -hex 32`, except the relay secret and the VAPID pair, whose own entries name the command. The installer in [Get started](/operator/get-started/) generates them all, so come here to rotate one value later or to set them up by hand.+Every value below ships as `CHANGE_ME`. Replace them all before the first start. Compose refuses to start when one is unset or empty, and names the variable. Generate each with `openssl rand -hex 32`, except the relay secret, the attachment URL secrets, and the VAPID pair, whose own entries name the command. The installer in [Get started](/operator/get-started/) generates them all, so come here to rotate one value later or to set them up by hand.#### `POSTGRES_PASSWORD`@@ -90,7 +90,21 @@ Media Proxy URLs. Must match across `api`, `worker`, `media-proxy`, `gifs`, and#### `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64`-Generate with `openssl rand -base64 32`. [Upload relay](/media-proxy/upload-relay/) capability tokens. Must be standard base64 decoding to at least 32 bytes. Hex fails. Must match on `api`, `worker`, and `media-proxy`. The API and the Media Proxy both check it at boot and refuse to start without it in `upload` mode.+Generate with `openssl rand -base64 32`. [Upload relay](/media-proxy/upload-relay/) capability tokens. Must be standard base64 decoding to at least 32 bytes. Must match on `api`, `worker`, and `media-proxy`. `api` and `worker` refuse to start without it in every mode. `media-proxy` refuses to start without it when `FLUXER_MEDIA_PROXY_MODE` is `upload` or `relay`.++#### `FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64`++Generate each entry with `openssl rand -base64 32`. Signs the [attachment URLs](/media-proxy/overview/#signed-attachment-urls) Fluxer returns. A comma-separated list, where each entry must be standard base64 decoding to at least 32 bytes. Spaces around an entry and empty entries are ignored. The first entry signs and every entry verifies. Must match on `api`, `worker`, and `media-proxy`, and must differ from `FLUXER_MEDIA_PROXY_SECRET_KEY`. `api` and `worker` refuse to start when an entry is invalid, and with no entry they return unsigned URLs. `media-proxy` reads it only when `FLUXER_MEDIA_PROXY_MODE` is `mp` or `upload`, and then refuses to start with an invalid entry, or with no entry when `FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE` is `report` or `enforce`.++The Compose stack leaves it empty, so a self-hosted instance returns unsigned URLs until an operator sets one. Setting it alone changes nothing on the read side, because `FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE` stays `off` until it is set too.++Rotate it in three steps, and apply each one with `docker compose up -d` before starting the next:++1. Add the new secret as the second entry. Every `media-proxy` then accepts it before any service signs with it.+2. Move the new secret to the first entry, so it signs.+3. Remove the old secret no sooner than one day after step 2, because an ordinary URL it signed stays valid for up to a day.++A [data package URL](/http-api/messages/#data-package-attachment-urls) inside a harvest export never expires, so removing a secret ends every data package URL it signed at once, including the links inside exports already delivered, and those exports have to be rebuilt. Step 3 is safe for ordinary URLs alone.#### `FLUXER_ADMIN_SECRET_KEY_BASE`@@ -1309,7 +1323,7 @@ The Media Proxy takes uploads, transforms images, and serves media back. All are#### `FLUXER_MEDIA_PROXY_MODE`-Default `mp`. Which routes are served. `mp`, `static`, or `upload`. Anything else is a startup error. Compose sets `upload`.+Default `mp`. Which routes are served. `mp`, `static`, `upload`, or `relay`. `relay` serves the upload relay and the operator and internal endpoints, and returns 404 for every read. Anything else is a startup error. Also settable with `--mode`. Compose sets `upload`.#### `FLUXER_MEDIA_PROXY_HOST`@@ -1411,6 +1425,22 @@ Default empty. An external NSFW classifier. No such service ships with the stackDefault `0.85`. The media-side NSFW cutoff. Accepts 0.0 to 1.0 and finite. Distinct from the API threshold.+#### `FLUXER_MEDIA_PROXY_CORS_MODE`++Default `off`. The [cross-origin read policy](/media-proxy/overview/#cross-origin-reads). `off`, `report`, or `enforce`. Anything else is a startup error. `report` logs each read whose origin would be refused and changes no response. `enforce` refuses those reads with 403 and sets `Access-Control-Allow-Origin` to the request origin. When `FLUXER_MEDIA_PROXY_MODE` is `static` or `relay`, the Media Proxy ignores it, so a bad value is no startup error. Compose leaves it `off`.++#### `FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS`++Default empty. Comma-separated origins that may read media through CORS, such as `https://web.fluxer.app`. Spaces around an entry and empty entries are ignored. Each entry is parsed as a URL and must be `http` or `https` with no credentials, query, fragment, or path beyond `/`. Its host must be an IPv4 address, a bracketed IPv6 address, or a domain name whose labels are ASCII letters, digits, and hyphens after parsing. So a wildcard such as `https://*.fluxer.app`, a brace pattern, an underscore, a trailing dot on a domain name, or any non-ASCII character fails. Write an internationalised domain name in its punycode form. Any failing entry makes the process exit. An entry is compared in the form a browser sends. `https://Web.Fluxer.App:443/` matches `https://web.fluxer.app`. Required when `FLUXER_MEDIA_PROXY_CORS_MODE` is `report` or `enforce`. When the policy is `report` or `enforce`, the Media Proxy logs the parsed list once at startup. When `FLUXER_MEDIA_PROXY_MODE` is `static` or `relay`, the Media Proxy ignores it.++Compose sets it to the public origin, built from `FLUXER_PUBLIC_ORIGIN`, or from `FLUXER_PUBLIC_SCHEME`, `FLUXER_DOMAIN`, and `FLUXER_PUBLIC_PORT`. That origin is the only entry, so the web app the instance serves reads its own media. The hosted web client and the desktop app run on `https://web.fluxer.app`. To let them read this instance's media through CORS, set the list in `.env` to the public origin and `https://web.fluxer.app`, such as `https://chat.example.com,https://web.fluxer.app`. A value set in `.env` replaces the Compose value, so it must name the public origin too.++#### `FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE`++Default `off`. The [signed attachment URL policy](/media-proxy/overview/#signed-attachment-urls). `off`, `report`, or `enforce`. Anything else is a startup error, an empty value included. `report` logs each attachment read `enforce` would refuse and serves it. `enforce` refuses those reads with 404. Under both, a read whose signature is valid gets a `Cache-Control` that ends with the signature instead of the usual year. `report` and `enforce` require `FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64`, and the Media Proxy logs the mode and the number of secrets once at startup. When `FLUXER_MEDIA_PROXY_MODE` is `static` or `relay`, the Media Proxy ignores it and the secrets, so a bad value is no startup error. Compose leaves it `off`, so a self-hosted instance signs nothing until an operator turns it on.++This variable is the whole switch, and moving between the three states needs nothing else changed. The Media Proxy reads it once at start, so apply a change with `docker compose up -d media-proxy`, which replaces the container. `docker compose restart media-proxy` reuses the old environment. A cache or proxy in front of the instance has a switch of its own, and [Turning the signature policy on](/operator/reverse-proxy/#turning-the-signature-policy-on) gives the order to move the two in and what each pairing serves.+`media-proxy` range-checks these values at startup and then reads them nowhere: `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_BYTES` and `FLUXER_MEDIA_PROXY_UPLOAD_RELAY_BUFFERED_RETRY_TOTAL_BYTES`. A bad value still fails the boot.## Gateway settingsdiff --git a/fluxer_docs/src/content/docs/operator/reverse-proxy.mdx b/fluxer_docs/src/content/docs/operator/reverse-proxy.mdxindex 77852b066..4eec2f65d 100644--- a/fluxer_docs/src/content/docs/operator/reverse-proxy.mdx+++ b/fluxer_docs/src/content/docs/operator/reverse-proxy.mdx@@ -436,6 +436,82 @@ The edge reads the variable at container start, so apply a change with `docker c`private_ranges` trusts the whole LAN. With `FLUXER_EDGE_BIND=0.0.0.0:8080`, any host on that network can impersonate any client address. Firewall the port to the proxy host.:::+## Caching in front of Fluxer++Media reads are the only traffic worth caching, and the instance expects no cache in front of it. Every rule below is a correctness rule rather than a tuning choice, and a cache that cannot follow one should not store media at all.++#### Leave the signature parameters out of the cache key++`ex`, `is`, `hm`, and `uc` are the [signed attachment URL](/media-proxy/overview/#signed-attachment-urls) parameters. A key that keeps them stores a new entry every time the same object is signed again, which empties the cache of anything useful. Drop all four and keep every other parameter, because the rest select the representation.++Only drop them if your proxy also meets the next two rules. A key without them is the same key for a signed request and an unsigned one, so the signature check is the only thing between an unsigned request and the object.++#### Decode parameter names, and take the last value++Fluxer parses a query string with form URL decoding, so `%73ize` is the name `size` and `+` is a space. When a name repeats, the final value wins. A cache that compares raw bytes, or that keeps the first value, can be handed `%73ize=64` or `size=64&size=4096` and made to store one rendering under the key for another. Match both rules, or refuse to cache a request whose query string you cannot parse that way.++#### Check the signature before the cache lookup++A cache hit never reaches the instance, so the Media Proxy's own signature check never runs on one. Under the `enforce` [signed attachment URL policy](/media-proxy/overview/#signed-attachment-urls), verify the signature in the proxy before it looks in the cache. An nginx `access_by_lua_block` runs in the access phase, which is before the cache lookup. A proxy that cannot do that must leave `/attachments/` uncached, or it will serve an expired or forged URL from a warm entry.++#### Verify the same path the cache key is built from++Running before the cache lookup is not enough on its own. The check and the key must read the same spelling of the path, or a request the check skips still reaches the entry the key names.++In nginx that spelling is `$uri`, which nginx has already percent decoded, merged repeated slashes in, and resolved `.` and `..` out of. `$request_uri` is the raw bytes the client sent. A check that scopes itself on `$request_uri` never sees `//attachments/...`, `/attachments%2F...`, `/attachments/./...`, or `/x/../attachments/...` as an attachment read, while `$uri` and the cache key make all four the same object. Read `$uri` for the prefix test and for the storage key, take the query from `$request_uri`, and refuse a path whose `$uri` is not a usable storage key rather than passing it through.++The signature covers the decoded storage key, which is `$uri` without its leading slash.++#### Do not cache a range request without slice support++nginx removes `Range` from a request it may cache unless the slice module is built in, so a seek into a video pulls the whole object. Set `proxy_cache_max_range_offset 0`. A range request then reaches the instance untouched and is not stored, while an entry that is already warm still answers ranges from cache.++#### Wire up purge, and bound the entry lifetime anyway++Every media 200 is `public, max-age=31536000` with no `ETag` and no `Last-Modified`, so a cache never revalidates and a deleted object stays until its entry goes. Point `FLUXER_CACHE_PURGE_HTTP_ENDPOINT` at a receiver that clears your cache, and read the contract it must meet under [Cache purge](/operator/configuration/#feature-flags-and-development-switches). Give stored media a shorter lifetime of your own as well, because a purge that is lost is never sent again.++## Turning the signature policy on++`FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE` is the instance switch, and a proxy that verifies signatures has one of its own. Both sides name the same three states, `off`, `report`, and `enforce`. Both read their setting once at start, so apply an instance change with `docker compose up -d media-proxy`, which replaces the container. `docker compose restart media-proxy` reuses the old environment and changes nothing.++Turn the policy on in this order, and stop wherever a refusal appears that you did not expect:++1. Give `api`, `worker`, and `media-proxy` the same `FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64`, and give the proxy the same list. Every URL the instance returns is signed from here on, and nothing verifies yet.+2. Leave it there until the clients holding the unsigned URLs issued before step 1 have replaced them. A client refreshes a URL whose `ex` is absent or under an hour away, and one that has been open since before step 1 keeps its unsigned URLs until it reloads.+3. Set both sides to `report` and read the logs. Under `report` nothing is refused, and each read `enforce` would refuse is logged with its reason on the side that saw it.+4. Move the proxy to `enforce` once `report` is quiet.+5. Move the instance to `enforce`.++The proxy goes first because a cache hit never reaches the instance. Reversing steps 4 and 5 leaves every warm entry readable without a signature for as long as it lives.++From step 3 the instance caps `Cache-Control` on a read whose signature is valid at the time that signature has left, where `off` returns a year. `report` caps it too, so the entries stored during the dry run do not outlive the URLs that made them. A read `report` would refuse still returns a year, because there is no signature to cap it against. Entries stored before step 3 keep the year they were given.++### When the two sides disagree++| Proxy | Instance | What an attachment read gets |+| --- | --- | --- |+| `off` | `off` | Served. Neither side reads a signature |+| `off` | `report` | Served. The instance logs what `enforce` would refuse |+| `off` | `enforce` | Refused, unless a warm entry answers it without reaching the instance |+| `report` | `off` | Served. The proxy logs what its own `enforce` would refuse |+| `report` | `report` | Served. Both sides log |+| `report` | `enforce` | Refused, unless a warm entry answers it without reaching the instance |+| `enforce` | `off` | Refused at the proxy, before the cache lookup |+| `enforce` | `report` | Refused at the proxy. The instance logs nothing, because it is never asked |+| `enforce` | `enforce` | Refused at the proxy |++The two rows that serve a read the instance would refuse are the ones where the instance enforces alone. The instance is right and the cache is stale, and the entry answers until it goes, so pair an instance-first change with a purge of `/attachments/`.++A change to `FLUXER_MEDIA_PROXY_CORS_MODE` has the same problem and no step order that avoids it. An entry stored while the policy was `off` holds `Access-Control-Allow-Origin: *` and a year of `Cache-Control`, and the proxy replays both on every hit without asking the instance. The proxy's own signature check does not help, because it reads the signature and never reads `Origin`, so a foreign page holding one valid signed URL still reads the bytes cross-origin from a warm entry. Purge `/attachments/` when you move the CORS policy to `enforce`, the same purge step 5 needs.++`enforce` at the proxy alone covers every request that passes through it. Anything that can reach the instance another way is unchecked until the instance enforces too: a second cache, a health probe from outside the proxy, an internal service that reads an attachment URL directly, and a deployment running no proxy at all. Those are what step 5 covers, and what it can break.++Step 4 also ends the instance's own evidence. A read the proxy refuses never reaches the instance, so once the proxy enforces the instance logs nothing under `report` and its stream stays empty whatever it would have refused. Take step 5 on what step 3 showed, while both sides were still in `report`.++Both sides must hold the same secret list. A side missing a key refuses the URLs signed with it, so add a new key to both sides before the instance starts signing with it, and keep a retired key on both until the last URL it signed has expired. An ordinary URL expires within a day. A [data package URL](/http-api/messages/#data-package-attachment-urls) never does, and removing the key that signed it is what ends it.++With no secret list, `api` and `worker` return unsigned URLs, and a side in `enforce` refuses every attachment read. Never move a side to `enforce` before signing is on.+## Voice media does not use the proxyLiveKit signalling goes through `/livekit/*` like everything else. WebRTC media does not touch the proxy at all:diff --git a/fluxer_media_proxy/src/cli.rs b/fluxer_media_proxy/src/cli.rsindex e99a9e313..1ab635bac 100644--- a/fluxer_media_proxy/src/cli.rs+++ b/fluxer_media_proxy/src/cli.rs@@ -1,6 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use crate::config::{Config, DeploymentMode, StorageBackend};+use crate::config::{Config, StorageBackend};use clap::{ArgAction, Parser, Subcommand, ValueEnum};#[derive(Debug, Parser)]@@ -38,6 +38,7 @@ pub enum ModeArg {Mp,Static,Upload,+ Relay,}#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)]@@ -56,6 +57,16 @@ whereK: Into<String>,V: Into<String>,{+ let mode_override = args.mode.map(|mode| {+ (+ "FLUXER_MEDIA_PROXY_MODE".to_owned(),+ mode.env_value().to_owned(),+ )+ });+ let vars = mode_override.into_iter().chain(+ vars.into_iter()+ .map(|(key, value)| (key.into(), value.into())),+ );let mut cfg = Config::load_from_iter(vars)?;apply_overrides(args, &mut cfg)?;Ok(cfg)@@ -69,9 +80,6 @@ fn apply_overrides(args: &Args, cfg: &mut Config) -> anyhow::Result<()> {if let Some(port) = args.port {cfg.port = port;}- if let Some(mode) = args.mode {- cfg.mode = mode.into();- }if let Some(storage_backend) = args.storage_backend {cfg.storage.backend = storage_backend.into();}@@ -88,12 +96,13 @@ fn apply_overrides(args: &Args, cfg: &mut Config) -> anyhow::Result<()> {Ok(())}-impl From<ModeArg> for DeploymentMode {- fn from(value: ModeArg) -> Self {- match value {- ModeArg::Mp => Self::Mp,- ModeArg::Static => Self::Static,- ModeArg::Upload => Self::Upload,+impl ModeArg {+ fn env_value(self) -> &'static str {+ match self {+ Self::Mp => "mp",+ Self::Static => "static",+ Self::Upload => "upload",+ Self::Relay => "relay",}}}@@ -110,6 +119,9 @@ impl From<StorageBackendArg> for StorageBackend {#[cfg(test)]mod tests {use super::*;+ use crate::config::{DeploymentMode, PolicyMode};++ const RELAY_SECRET: &str = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";#[test]fn cli_overrides_env_config() {@@ -145,6 +157,59 @@ mod tests {assert!(cfg.read_only);}+ #[test]+ fn cli_mode_relay_selects_relay_mode() {+ let args = Args::try_parse_from(["fluxer-media-proxy", "--mode", "relay"]).unwrap();+ let cfg = load_config_from_iter(+ &args,+ [+ ("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret"),+ (+ "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64",+ RELAY_SECRET,+ ),+ ],+ )+ .unwrap();+ assert_eq!(DeploymentMode::Relay, cfg.mode);+ }++ #[test]+ fn cli_mode_decides_the_mode_dependent_config() {+ let relay = Args::try_parse_from(["fluxer-media-proxy", "--mode", "relay"]).unwrap();+ let cfg = load_config_from_iter(+ &relay,+ [+ ("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret"),+ ("FLUXER_MEDIA_PROXY_MODE", "mp"),+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "enforce"),+ ("FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS", "not an origin"),+ (+ "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64",+ RELAY_SECRET,+ ),+ ],+ )+ .unwrap();+ assert_eq!(DeploymentMode::Relay, cfg.mode);+ assert_eq!(PolicyMode::Off, cfg.cors.mode);+ assert!(cfg.cors.allowed_origins.is_empty());++ let upload = Args::try_parse_from(["fluxer-media-proxy", "--mode", "upload"]).unwrap();+ let err = load_config_from_iter(+ &upload,+ [+ ("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret"),+ ("FLUXER_MEDIA_PROXY_MODE", "static"),+ ],+ )+ .unwrap_err();+ assert_eq!(+ "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload and relay modes",+ err.to_string()+ );+ }+#[test]fn cli_parses_healthcheck_subcommand() {assert_eq!(diff --git a/fluxer_media_proxy/src/config/mod.rs b/fluxer_media_proxy/src/config/mod.rsindex 301adc2dc..1b40109cb 100644--- a/fluxer_media_proxy/src/config/mod.rs+++ b/fluxer_media_proxy/src/config/mod.rs@@ -6,10 +6,12 @@ mod tests;use crate::constants;use crate::secret::{SecretBytes, SecretString};+use http::HeaderValue;use parse::{EnvMap, decode_upload_relay_secret, default_native_transform_concurrency, non_empty,- parse_bool, parse_bucket_style, parse_f32, parse_mode_env, parse_storage_backend, parse_u16,- parse_u64, parse_usize, validate_read_endpoint,+ parse_allowed_origins, parse_attachment_url_secrets, parse_bool, parse_bucket_style, parse_f32,+ parse_mode_env, parse_policy_mode, parse_storage_backend, parse_u16, parse_u64, parse_usize,+ validate_read_endpoint,};use std::{env, path::PathBuf};@@ -31,6 +33,52 @@ pub enum DeploymentMode {Mp,Static,Upload,+ Relay,+}++impl DeploymentMode {+ pub fn serves_upload_relay(self) -> bool {+ matches!(self, Self::Upload | Self::Relay)+ }+}++#[derive(Clone, Copy, Debug, Eq, PartialEq)]+pub enum PolicyMode {+ Off,+ Report,+ Enforce,+}++impl PolicyMode {+ pub fn label(self) -> &'static str {+ match self {+ Self::Off => "off",+ Self::Report => "report",+ Self::Enforce => "enforce",+ }+ }+}++#[derive(Clone, Debug)]+pub struct CorsConfig {+ pub mode: PolicyMode,+ pub allowed_origins: Vec<HeaderValue>,+}++#[derive(Clone, Debug)]+pub struct AttachmentSignatureConfig {+ pub mode: PolicyMode,+ pub(crate) secrets: Vec<SecretBytes>,+}++impl AttachmentSignatureConfig {+ pub(crate) fn secret_count(&self) -> usize {+ self.secrets.len()+ }++ pub(crate) fn secrets(&self) -> Vec<&[u8]> {+ self.secrets.iter().map(SecretBytes::expose).collect()+ }}#[derive(Clone, Debug)]@@ -92,6 +140,8 @@ pub struct Config {pub storage: StorageConfig,pub media: MediaServingConfig,pub upload_relay: UploadRelayConfig,+ pub cors: CorsConfig,+ pub attachment_signature: AttachmentSignatureConfig,}impl Config {@@ -167,6 +217,8 @@ impl Config {storage: StorageConfig::load(&env)?,media: MediaServingConfig::load(&env)?,upload_relay: UploadRelayConfig::load(&env, mode)?,+ cors: CorsConfig::load(&env, mode)?,+ attachment_signature: AttachmentSignatureConfig::load(&env, mode)?,})}}@@ -380,3 +432,57 @@ impl UploadRelayConfig {})}}++impl CorsConfig {+ fn load(env: &EnvMap, mode: DeploymentMode) -> anyhow::Result<Self> {+ if matches!(mode, DeploymentMode::Static | DeploymentMode::Relay) {+ return Ok(Self {+ mode: PolicyMode::Off,+ allowed_origins: Vec::new(),+ });+ }+ let cors_mode = parse_policy_mode(+ "FLUXER_MEDIA_PROXY_CORS_MODE",+ env.get("FLUXER_MEDIA_PROXY_CORS_MODE"),+ )?;+ let allowed_origins = parse_allowed_origins(+ "FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS",+ env.get("FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS"),+ )?;+ anyhow::ensure!(+ cors_mode == PolicyMode::Off || !allowed_origins.is_empty(),+ "FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS is required when FLUXER_MEDIA_PROXY_CORS_MODE is report or enforce"+ );+ Ok(Self {+ mode: cors_mode,+ allowed_origins,+ })+ }+}++impl AttachmentSignatureConfig {+ fn load(env: &EnvMap, mode: DeploymentMode) -> anyhow::Result<Self> {+ if matches!(mode, DeploymentMode::Static | DeploymentMode::Relay) {+ return Ok(Self {+ mode: PolicyMode::Off,+ secrets: Vec::new(),+ });+ }+ let signature_mode = parse_policy_mode(+ "FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE",+ env.get("FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE"),+ )?;+ let secrets = parse_attachment_url_secrets(+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64",+ env.get("FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64"),+ )?;+ anyhow::ensure!(+ signature_mode == PolicyMode::Off || !secrets.is_empty(),+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64 is required when FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE is report or enforce"+ );+ Ok(Self {+ mode: signature_mode,+ secrets,+ })+ }+}diff --git a/fluxer_media_proxy/src/config/parse.rs b/fluxer_media_proxy/src/config/parse.rsindex 59c375860..40527de0e 100644--- a/fluxer_media_proxy/src/config/parse.rs+++ b/fluxer_media_proxy/src/config/parse.rs@@ -1,8 +1,9 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use super::{BucketStyle, DeploymentMode, StorageBackend};+use super::{BucketStyle, DeploymentMode, PolicyMode, StorageBackend};use crate::secret::SecretBytes;use base64::{Engine as _, engine::general_purpose};+use http::HeaderValue;#[derive(Debug, Default)]pub(super) struct EnvMap(Vec<(String, String)>);@@ -33,6 +34,7 @@ fn parse_mode(raw: &str) -> Option<DeploymentMode> {"mp" => Some(DeploymentMode::Mp),"static" => Some(DeploymentMode::Static),"upload" => Some(DeploymentMode::Upload),+ "relay" => Some(DeploymentMode::Relay),_ => None,}}@@ -43,10 +45,111 @@ pub(super) fn parse_mode_env(raw: Option<&str>) -> anyhow::Result<Option<Deploym};let raw = raw.trim();parse_mode(raw).map(Some).ok_or_else(|| {- anyhow::anyhow!("FLUXER_MEDIA_PROXY_MODE must be one of: mp, static, upload")+ anyhow::anyhow!("FLUXER_MEDIA_PROXY_MODE must be one of: mp, static, upload, relay")})}+pub(super) fn parse_policy_mode(var_name: &str, raw: Option<&str>) -> anyhow::Result<PolicyMode> {+ let Some(raw) = raw else {+ return Ok(PolicyMode::Off);+ };+ match raw.trim().to_ascii_lowercase().as_str() {+ "off" => Ok(PolicyMode::Off),+ "report" => Ok(PolicyMode::Report),+ "enforce" => Ok(PolicyMode::Enforce),+ _ => Err(anyhow::anyhow!(+ "{var_name} must be one of: off, report, enforce"+ )),+ }+}++pub(super) fn parse_attachment_url_secrets(+ var_name: &str,+ raw: Option<&str>,+) -> anyhow::Result<Vec<SecretBytes>> {+ let Some(raw) = raw else {+ return Ok(Vec::new());+ };+ let mut out: Vec<SecretBytes> = Vec::new();+ for entry in raw.split(',') {+ let entry = entry.trim();+ if entry.is_empty() {+ continue;+ }+ let decoded = general_purpose::STANDARD+ .decode(entry)+ .map_err(|_| anyhow::anyhow!("{var_name} entries must be standard base64"))?;+ anyhow::ensure!(+ decoded.len() >= 32,+ "{var_name} entries must decode to at least 32 bytes"+ );+ let candidate = SecretBytes::new(decoded);+ if !out+ .iter()+ .any(|existing| existing.expose() == candidate.expose())+ {+ out.push(candidate);+ }+ }+ Ok(out)+}++pub(super) fn parse_allowed_origins(+ var_name: &str,+ raw: Option<&str>,+) -> anyhow::Result<Vec<HeaderValue>> {+ let Some(raw) = raw else {+ return Ok(Vec::new());+ };+ let mut out: Vec<HeaderValue> = Vec::new();+ for entry in raw.split(',') {+ let entry = entry.trim();+ if entry.is_empty() {+ continue;+ }+ let origin = parse_origin(entry)+ .ok_or_else(|| anyhow::anyhow!("{var_name} contains an invalid origin: {entry}"))?;+ if !out.contains(&origin) {+ out.push(origin);+ }+ }+ Ok(out)+}++fn parse_origin(entry: &str) -> Option<HeaderValue> {+ if !entry.is_ascii() {+ return None;+ }+ let url = url::Url::parse(entry).ok()?;+ let bare = matches!(url.scheme(), "http" | "https")+ && url.host().is_some_and(is_origin_host)+ && url.username().is_empty()+ && url.password().is_none()+ && url.path() == "/"+ && url.query().is_none()+ && url.fragment().is_none();+ if !bare {+ return None;+ }+ let serialised = url.origin().ascii_serialization();+ if serialised == "null" {+ return None;+ }+ HeaderValue::from_str(&serialised).ok()+}++fn is_origin_host(host: url::Host<&str>) -> bool {+ match host {+ url::Host::Ipv4(_) | url::Host::Ipv6(_) => true,+ url::Host::Domain(domain) => domain.split('.').all(|label| {+ !label.is_empty()+ && label+ .bytes()+ .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')+ }),+ }+}+pub(super) fn non_empty(raw: Option<&str>) -> Option<String> {raw.map(str::trim).filter(|value| !value.is_empty())@@ -109,8 +212,8 @@ pub(super) fn decode_upload_relay_secret() -> anyhow::Result<SecretBytes> {let Some(raw) = raw.map(str::trim).filter(|s| !s.is_empty()) else {anyhow::ensure!(- mode != DeploymentMode::Upload,- "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload mode"+ !mode.serves_upload_relay(),+ "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload and relay modes");return Ok(SecretBytes::new(Vec::new()));};diff --git a/fluxer_media_proxy/src/config/tests/attachment_signature.rs b/fluxer_media_proxy/src/config/tests/attachment_signature.rsnew file mode 100644index 000000000..47fd4e6b4--- /dev/null+++ b/fluxer_media_proxy/src/config/tests/attachment_signature.rs@@ -0,0 +1,233 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use super::{base_env, env_with};+use crate::config::{Config, DeploymentMode, PolicyMode};++const MODE_KEY: &str = "FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE";+const SECRETS_KEY: &str = "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64";+const RELAY_SECRET_KEY: &str = "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64";+const FIRST: &str = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE=";+const SECOND: &str = "AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI=";+const TOO_SHORT: &str = "AwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAw==";+const OVERSIZED: &str =+ "BAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBA==";++fn secrets(cfg: &Config) -> Vec<Vec<u8>> {+ cfg.attachment_signature+ .secrets()+ .iter()+ .map(|secret| secret.to_vec())+ .collect()+}++fn enforce_with_secrets(raw: &str) -> anyhow::Result<Config> {+ Config::load_from_iter(env_with(&[(MODE_KEY, "enforce"), (SECRETS_KEY, raw)]))+}++#[test]+fn the_signature_policy_defaults_to_off_with_no_secrets() {+ let cfg = Config::load_from_iter(base_env()).unwrap();+ assert_eq!(PolicyMode::Off, cfg.attachment_signature.mode);+ assert_eq!(0, cfg.attachment_signature.secret_count());+ assert!(secrets(&cfg).is_empty());+}++#[test]+fn the_signature_mode_parses_every_variant_case_insensitively() {+ for raw in ["off", "OFF", " off "] {+ let cfg = Config::load_from_iter(env_with(&[(MODE_KEY, raw)])).unwrap();+ assert_eq!(PolicyMode::Off, cfg.attachment_signature.mode, "{raw:?}");+ }+ for (raw, expected) in [+ ("report", PolicyMode::Report),+ (" REPORT ", PolicyMode::Report),+ ("enforce", PolicyMode::Enforce),+ ("Enforce", PolicyMode::Enforce),+ ] {+ let cfg =+ Config::load_from_iter(env_with(&[(MODE_KEY, raw), (SECRETS_KEY, FIRST)])).unwrap();+ assert_eq!(expected, cfg.attachment_signature.mode, "{raw:?}");+ assert_eq!(vec![vec![1u8; 32]], secrets(&cfg), "{raw:?}");+ }+}++#[test]+fn rejects_an_unknown_signature_mode() {+ for raw in ["strict", "true", "1", "", "deny"] {+ let err =+ Config::load_from_iter(env_with(&[(MODE_KEY, raw), (SECRETS_KEY, FIRST)])).unwrap_err();+ assert_eq!(+ format!("{MODE_KEY} must be one of: off, report, enforce"),+ err.to_string(),+ "{raw:?}"+ );+ }+}++#[test]+fn the_cors_mode_error_still_names_its_own_variable() {+ let err = Config::load_from_iter(env_with(&[("FLUXER_MEDIA_PROXY_CORS_MODE", "strict")]))+ .unwrap_err();+ assert_eq!(+ "FLUXER_MEDIA_PROXY_CORS_MODE must be one of: off, report, enforce",+ err.to_string()+ );+}++#[test]+fn report_and_enforce_require_a_secret() {+ for mode in ["report", "enforce"] {+ let unset = Config::load_from_iter(env_with(&[(MODE_KEY, mode)])).unwrap_err();+ let empty =+ Config::load_from_iter(env_with(&[(MODE_KEY, mode), (SECRETS_KEY, "")])).unwrap_err();+ let separators_only =+ Config::load_from_iter(env_with(&[(MODE_KEY, mode), (SECRETS_KEY, " , , ")]))+ .unwrap_err();+ for err in [unset, empty, separators_only] {+ assert_eq!(+ format!("{SECRETS_KEY} is required when {MODE_KEY} is report or enforce"),+ err.to_string(),+ "{mode}"+ );+ }+ }+}++#[test]+fn rejects_an_entry_that_is_not_standard_base64() {+ for raw in [+ "not base64".to_owned(),+ "AQEB_AQE".to_owned(),+ format!("{FIRST},not base64"),+ ] {+ let err = enforce_with_secrets(&raw).unwrap_err();+ assert_eq!(+ format!("{SECRETS_KEY} entries must be standard base64"),+ err.to_string(),+ "{raw:?}"+ );+ }+}++#[test]+fn rejects_an_entry_shorter_than_thirty_two_bytes() {+ for raw in [TOO_SHORT, "AQ==", "AQEB"] {+ let err = enforce_with_secrets(raw).unwrap_err();+ assert_eq!(+ format!("{SECRETS_KEY} entries must decode to at least 32 bytes"),+ err.to_string(),+ "{raw:?}"+ );+ }+}++#[test]+fn an_error_never_echoes_an_entry() {+ for raw in [TOO_SHORT, "aHVudGVyMg=="] {+ let err = enforce_with_secrets(raw).unwrap_err().to_string();+ assert!(!err.contains(raw), "{err}");+ }+}++#[test]+fn empty_entries_are_skipped_and_duplicates_are_dropped() {+ let cfg = enforce_with_secrets(&format!(" {FIRST} , ,{SECOND},{FIRST}, ")).unwrap();+ assert_eq!(vec![vec![1u8; 32], vec![2u8; 32]], secrets(&cfg));+ assert_eq!(2, cfg.attachment_signature.secret_count());+}++#[test]+fn the_first_entry_stays_first_and_a_longer_key_is_accepted() {+ let cfg = enforce_with_secrets(&format!("{SECOND},{OVERSIZED}")).unwrap();+ assert_eq!(vec![vec![2u8; 32], vec![4u8; 64]], secrets(&cfg));+}++#[test]+fn off_mode_keeps_a_staged_secret_and_still_rejects_an_invalid_one() {+ let cfg = Config::load_from_iter(env_with(&[(MODE_KEY, "off"), (SECRETS_KEY, FIRST)])).unwrap();+ assert_eq!(PolicyMode::Off, cfg.attachment_signature.mode);+ assert_eq!(vec![vec![1u8; 32]], secrets(&cfg));++ let cfg = Config::load_from_iter(env_with(&[(SECRETS_KEY, FIRST)])).unwrap();+ assert_eq!(PolicyMode::Off, cfg.attachment_signature.mode);+ assert_eq!(vec![vec![1u8; 32]], secrets(&cfg));++ let err = Config::load_from_iter(env_with(&[(MODE_KEY, "off"), (SECRETS_KEY, TOO_SHORT)]))+ .unwrap_err();+ assert_eq!(+ format!("{SECRETS_KEY} entries must decode to at least 32 bytes"),+ err.to_string()+ );+}++#[test]+fn static_mode_ignores_both_signature_keys() {+ let cfg = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_MODE", "static"),+ (MODE_KEY, "enforce"),+ (SECRETS_KEY, "not base64"),+ ]))+ .unwrap();+ assert_eq!(DeploymentMode::Static, cfg.mode);+ assert_eq!(PolicyMode::Off, cfg.attachment_signature.mode);+ assert!(secrets(&cfg).is_empty());+}++#[test]+fn relay_mode_ignores_both_signature_keys() {+ for (mode, raw) in [+ ("enforce", "not base64"),+ ("enforce", TOO_SHORT),+ ("strict", FIRST),+ ] {+ let cfg = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_MODE", "relay"),+ (RELAY_SECRET_KEY, FIRST),+ (MODE_KEY, mode),+ (SECRETS_KEY, raw),+ ]))+ .unwrap();+ assert_eq!(DeploymentMode::Relay, cfg.mode);+ assert_eq!(+ PolicyMode::Off,+ cfg.attachment_signature.mode,+ "{mode} {raw}"+ );+ assert!(secrets(&cfg).is_empty(), "{mode} {raw}");+ }+}++#[test]+fn upload_mode_still_applies_both_signature_keys() {+ let cfg = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_MODE", "upload"),+ (RELAY_SECRET_KEY, FIRST),+ (MODE_KEY, "enforce"),+ (SECRETS_KEY, SECOND),+ ]))+ .unwrap();+ assert_eq!(DeploymentMode::Upload, cfg.mode);+ assert_eq!(PolicyMode::Enforce, cfg.attachment_signature.mode);+ assert_eq!(vec![vec![2u8; 32]], secrets(&cfg));++ let err = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_MODE", "upload"),+ (RELAY_SECRET_KEY, FIRST),+ (MODE_KEY, "enforce"),+ ]))+ .unwrap_err();+ assert_eq!(+ format!("{SECRETS_KEY} is required when {MODE_KEY} is report or enforce"),+ err.to_string()+ );+}++#[test]+fn debug_output_never_reveals_a_signing_secret() {+ let cfg = enforce_with_secrets(&format!("{FIRST},{SECOND}")).unwrap();+ let rendered = format!("{cfg:?}");+ assert!(!rendered.contains(FIRST));+ assert!(!rendered.contains(SECOND));+ assert!(!rendered.contains("\u{1}\u{1}"));+ assert_eq!(4, rendered.matches("[REDACTED]").count());+}diff --git a/fluxer_media_proxy/src/config/tests/cors.rs b/fluxer_media_proxy/src/config/tests/cors.rsnew file mode 100644index 000000000..34e735851--- /dev/null+++ b/fluxer_media_proxy/src/config/tests/cors.rs@@ -0,0 +1,258 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use super::{allowed_origins, base_env, env_with};+use crate::config::{Config, DeploymentMode, PolicyMode};+use base64::{Engine as _, engine::general_purpose};++const ORIGINS_KEY: &str = "FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS";++fn enforce_with_origins(raw: &'static str) -> anyhow::Result<Config> {+ Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "enforce"),+ (ORIGINS_KEY, raw),+ ]))+}++#[test]+fn cors_defaults_to_off_with_an_empty_allowlist() {+ let cfg = Config::load_from_iter(base_env()).unwrap();+ assert_eq!(PolicyMode::Off, cfg.cors.mode);+ assert!(cfg.cors.allowed_origins.is_empty());+}++#[test]+fn cors_mode_parses_every_variant_case_insensitively() {+ for (raw, expected) in [("off", PolicyMode::Off), ("OFF", PolicyMode::Off)] {+ let cfg =+ Config::load_from_iter(env_with(&[("FLUXER_MEDIA_PROXY_CORS_MODE", raw)])).unwrap();+ assert_eq!(expected, cfg.cors.mode, "{raw:?}");+ assert!(cfg.cors.allowed_origins.is_empty());+ }++ for (raw, expected) in [+ (" report ", PolicyMode::Report),+ ("Enforce", PolicyMode::Enforce),+ ] {+ let cfg = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", raw),+ (ORIGINS_KEY, "https://web.fluxer.app"),+ ]))+ .unwrap();+ assert_eq!(expected, cfg.cors.mode, "{raw:?}");+ assert_eq!(vec!["https://web.fluxer.app"], allowed_origins(&cfg));+ }+}++#[test]+fn rejects_an_unknown_cors_mode() {+ for raw in ["strict", "true", "1", ""] {+ let err = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", raw),+ (ORIGINS_KEY, "https://web.fluxer.app"),+ ]))+ .unwrap_err();+ assert_eq!(+ "FLUXER_MEDIA_PROXY_CORS_MODE must be one of: off, report, enforce",+ err.to_string(),+ "{raw:?}"+ );+ }+}++#[test]+fn allowed_origins_are_normalised_to_their_browser_serialisation() {+ for (raw, expected) in [+ ("https://Web.Fluxer.App", vec!["https://web.fluxer.app"]),+ ("https://web.fluxer.app/", vec!["https://web.fluxer.app"]),+ ("https://web.fluxer.app:443", vec!["https://web.fluxer.app"]),+ ("http://localhost:8088", vec!["http://localhost:8088"]),+ ("http://127.0.0.1:8088", vec!["http://127.0.0.1:8088"]),+ ("http://[::1]:8088", vec!["http://[::1]:8088"]),+ (+ "https://xn--bcher-kva.example",+ vec!["https://xn--bcher-kva.example"],+ ),+ (+ " https://a.example , ,https://b.example,",+ vec!["https://a.example", "https://b.example"],+ ),+ (+ "https://a.example,https://A.example",+ vec!["https://a.example"],+ ),+ ] {+ let cfg = enforce_with_origins(raw).unwrap();+ assert_eq!(expected, allowed_origins(&cfg), "{raw:?}");+ }+}++#[test]+fn rejects_an_allowed_origin_that_is_not_a_bare_http_origin() {+ for entry in [+ "*",+ "null",+ "web.fluxer.app",+ "ftp://web.fluxer.app",+ "file:///tmp/x",+ "fluxer-app://app",+ "https://[email protected]",+ "https://user:[email protected]",+ "https://web.fluxer.app/app",+ "https://web.fluxer.app?x=1",+ "https://web.fluxer.app#f",+ "https://*.fluxer.app",+ "https://{web}.fluxer.app",+ "https://*",+ "https://web.*.fluxer.app",+ "https://~",+ "https://$",+ "https://a..b",+ "https://.fluxer.app",+ "https://web.fluxer.app.",+ "https://bücher.example",+ "https://w\u{435}b.fluxer.app",+ ] {+ let err = enforce_with_origins(entry).unwrap_err();+ assert_eq!(+ format!("{ORIGINS_KEY} contains an invalid origin: {entry}"),+ err.to_string()+ );+ }++ let err = enforce_with_origins("https://{web,canary}.fluxer.app").unwrap_err();+ assert_eq!(+ format!("{ORIGINS_KEY} contains an invalid origin: https://{{web"),+ err.to_string()+ );+}++#[test]+fn an_invalid_entry_fails_even_beside_a_valid_one() {+ let err =+ enforce_with_origins("https://web.fluxer.app, https://web.fluxer.app/app").unwrap_err();+ assert_eq!(+ format!("{ORIGINS_KEY} contains an invalid origin: https://web.fluxer.app/app"),+ err.to_string()+ );+}++#[test]+fn report_and_enforce_require_an_allowed_origin() {+ for mode in ["report", "enforce"] {+ let unset = Config::load_from_iter(env_with(&[("FLUXER_MEDIA_PROXY_CORS_MODE", mode)]))+ .unwrap_err();+ let empty = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", mode),+ (ORIGINS_KEY, ""),+ ]))+ .unwrap_err();+ let separators_only = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", mode),+ (ORIGINS_KEY, " , "),+ ]))+ .unwrap_err();+ for err in [unset, empty, separators_only] {+ assert_eq!(+ "FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS is required when FLUXER_MEDIA_PROXY_CORS_MODE is report or enforce",+ err.to_string(),+ "{mode}"+ );+ }+ }+}++#[test]+fn off_mode_keeps_a_staged_allowlist_and_still_rejects_an_invalid_one() {+ let cfg = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "off"),+ (+ ORIGINS_KEY,+ "https://web.fluxer.app,https://web.canary.fluxer.app",+ ),+ ]))+ .unwrap();+ assert_eq!(PolicyMode::Off, cfg.cors.mode);+ assert_eq!(+ vec!["https://web.fluxer.app", "https://web.canary.fluxer.app"],+ allowed_origins(&cfg)+ );++ let cfg = Config::load_from_iter(env_with(&[(ORIGINS_KEY, "https://web.fluxer.app")])).unwrap();+ assert_eq!(PolicyMode::Off, cfg.cors.mode);+ assert_eq!(vec!["https://web.fluxer.app"], allowed_origins(&cfg));++ let err = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "off"),+ (ORIGINS_KEY, "not an origin"),+ ]))+ .unwrap_err();+ assert_eq!(+ format!("{ORIGINS_KEY} contains an invalid origin: not an origin"),+ err.to_string()+ );+}++#[test]+fn static_mode_ignores_both_cors_keys() {+ let cfg = Config::load_from_iter(env_with(&[+ ("FLUXER_MEDIA_PROXY_MODE", "static"),+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "enforce"),+ (ORIGINS_KEY, "not an origin"),+ ]))+ .unwrap();+ assert_eq!(DeploymentMode::Static, cfg.mode);+ assert_eq!(PolicyMode::Off, cfg.cors.mode);+ assert!(cfg.cors.allowed_origins.is_empty());+}++#[test]+fn relay_mode_ignores_both_cors_keys() {+ let relay_secret = general_purpose::STANDARD.encode([5u8; 32]);+ for (cors_mode, origins) in [+ ("enforce", "not an origin"),+ ("enforce", "https://*.fluxer.app"),+ ("strict", "https://web.fluxer.app"),+ ] {+ let mut env: Vec<(&str, &str)> = env_with(&[+ ("FLUXER_MEDIA_PROXY_MODE", "relay"),+ ("FLUXER_MEDIA_PROXY_CORS_MODE", cors_mode),+ (ORIGINS_KEY, origins),+ ]);+ env.push((+ "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64",+ relay_secret.as_str(),+ ));+ let cfg = Config::load_from_iter(env).unwrap();+ assert_eq!(DeploymentMode::Relay, cfg.mode);+ assert_eq!(PolicyMode::Off, cfg.cors.mode, "{cors_mode} {origins}");+ assert!(cfg.cors.allowed_origins.is_empty(), "{cors_mode} {origins}");+ }+}++#[test]+fn upload_mode_still_applies_the_cors_keys() {+ let relay_secret = general_purpose::STANDARD.encode([5u8; 32]);+ let upload_env = |origins: &'static str| {+ let mut env: Vec<(&str, &str)> = env_with(&[+ ("FLUXER_MEDIA_PROXY_MODE", "upload"),+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "enforce"),+ (ORIGINS_KEY, origins),+ ]);+ env.push((+ "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64",+ relay_secret.as_str(),+ ));+ env+ };++ let cfg = Config::load_from_iter(upload_env("https://web.fluxer.app")).unwrap();+ assert_eq!(DeploymentMode::Upload, cfg.mode);+ assert_eq!(PolicyMode::Enforce, cfg.cors.mode);+ assert_eq!(vec!["https://web.fluxer.app"], allowed_origins(&cfg));++ let err = Config::load_from_iter(upload_env("not an origin")).unwrap_err();+ assert_eq!(+ format!("{ORIGINS_KEY} contains an invalid origin: not an origin"),+ err.to_string()+ );+}diff --git a/fluxer_media_proxy/src/config/tests/mod.rs b/fluxer_media_proxy/src/config/tests/mod.rsindex 7f368c168..d155d6ae1 100644--- a/fluxer_media_proxy/src/config/tests/mod.rs+++ b/fluxer_media_proxy/src/config/tests/mod.rs@@ -1,5 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-later+mod attachment_signature;+mod cors;mod public_endpoint;mod s3_read;@@ -10,8 +12,16 @@ fn base_env() -> Vec<(&'static str, &'static str)> {vec![("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret")]}-fn env_with(extra: &[(&'static str, &'static str)]) -> Vec<(&'static str, &'static str)> {- let mut env = base_env();+fn allowed_origins(cfg: &Config) -> Vec<&str> {+ cfg.cors+ .allowed_origins+ .iter()+ .map(|origin| origin.to_str().unwrap())+ .collect()+}++fn env_with<'a>(extra: &[(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> {+ let mut env: Vec<(&'a str, &'a str)> = base_env();env.extend_from_slice(extra);env}@@ -100,6 +110,27 @@ fn upload_mode_requires_relay_secret() {);}+#[test]+fn relay_mode_requires_the_relay_secret() {+ let err = Config::load_from_iter([+ ("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret"),+ ("FLUXER_MEDIA_PROXY_MODE", "relay"),+ ])+ .unwrap_err();+ assert_eq!(+ "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64 is required in upload and relay modes",+ err.to_string()+ );+}++#[test]+fn serves_upload_relay_is_true_only_for_upload_and_relay() {+ assert!(!DeploymentMode::Mp.serves_upload_relay());+ assert!(!DeploymentMode::Static.serves_upload_relay());+ assert!(DeploymentMode::Upload.serves_upload_relay());+ assert!(DeploymentMode::Relay.serves_upload_relay());+}+#[test]fn parses_upload_relay_secret() {let secret = general_purpose::STANDARD.encode([7u8; 32]);@@ -164,7 +195,10 @@ fn rejects_invalid_mode_env() {("FLUXER_MEDIA_PROXY_MODE", "worker"),]).unwrap_err();- assert!(err.to_string().contains("FLUXER_MEDIA_PROXY_MODE"));+ assert_eq!(+ "FLUXER_MEDIA_PROXY_MODE must be one of: mp, static, upload, relay",+ err.to_string()+ );}#[test]@@ -240,11 +274,21 @@ fn production_media_proxy_release_env_loads() {),("FLUXER_MEDIA_PROXY_TRANSFORM_CACHE_TTL_MS", "1800000"),("FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS", "30000"),+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "enforce"),+ (+ "FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS",+ "https://web.fluxer.app,https://web.canary.fluxer.app",+ ),])).unwrap();assert_eq!("production", cfg.node_env);assert_eq!(DeploymentMode::Mp, cfg.mode);+ assert_eq!(PolicyMode::Enforce, cfg.cors.mode);+ assert_eq!(+ vec!["https://web.fluxer.app", "https://web.canary.fluxer.app"],+ allowed_origins(&cfg)+ );assert!(cfg.read_only);assert_eq!(StorageBackend::S3, cfg.storage.backend);assert_eq!("ewr1", cfg.storage.s3_region);@@ -272,10 +316,17 @@ fn production_static_proxy_release_env_loads() {("FLUXER_MEDIA_PROXY_STORAGE_BACKEND", "s3"),("FLUXER_MEDIA_PROXY_READ_ONLY", "true"),("FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS", "30000"),+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "enforce"),+ (+ "FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS",+ "https://web.fluxer.app,https://web.canary.fluxer.app",+ ),])).unwrap();assert_eq!(DeploymentMode::Static, cfg.mode);+ assert_eq!(PolicyMode::Off, cfg.cors.mode);+ assert!(cfg.cors.allowed_origins.is_empty());assert!(cfg.read_only);assert_eq!(StorageBackend::S3, cfg.storage.backend);assert_eq!("static", cfg.storage.bucket_static);@@ -288,7 +339,7 @@ fn production_uploads_release_env_loads() {let relay_secret = general_purpose::STANDARD.encode([9u8; 48]);let cfg = Config::load_from_iter(with_shared_runtime_env(&[("RELEASE_CHANNEL", "stable"),- ("FLUXER_MEDIA_PROXY_MODE", "upload"),+ ("FLUXER_MEDIA_PROXY_MODE", "relay"),("FLUXER_MEDIA_PROXY_STORAGE_BACKEND", "s3"),("FLUXER_MEDIA_PROXY_READ_ONLY", "false"),("FLUXER_MEDIA_PROXY_SOCKET_IO_TIMEOUT_MS", "300000"),@@ -308,7 +359,7 @@ fn production_uploads_release_env_loads() {])).unwrap();- assert_eq!(DeploymentMode::Upload, cfg.mode);+ assert_eq!(DeploymentMode::Relay, cfg.mode);assert!(!cfg.read_only);assert_eq!(300_000, cfg.socket_io_timeout_ms);assert_eq!(900_000, cfg.upload_relay.s3_timeout_ms);@@ -381,6 +432,8 @@ fn every_deployment_mode_and_storage_backend_variant_parses() {("Static", DeploymentMode::Static),("upload", DeploymentMode::Upload),(" UPLOAD ", DeploymentMode::Upload),+ ("relay", DeploymentMode::Relay),+ (" RELAY ", DeploymentMode::Relay),] {let cfg = Config::load_from_iter([("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret"),diff --git a/fluxer_media_proxy/src/http_headers.rs b/fluxer_media_proxy/src/http_headers.rsindex d4c95deda..6c42cde4e 100644--- a/fluxer_media_proxy/src/http_headers.rs+++ b/fluxer_media_proxy/src/http_headers.rs@@ -1,7 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse crate::range::ByteRange;-use http::{HeaderMap, HeaderName, HeaderValue, header};+use http::{HeaderMap, HeaderName, HeaderValue, StatusCode, header};pub const ROBOTS: &str = "noindex, nofollow, nosnippet, noimageindex, notranslate, max-snippet:0, max-image-preview:none, max-video-preview:0";pub const MEDIA_CSP: &str = "default-src 'none'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; object-src 'none'; script-src 'none'; script-src-attr 'none'; script-src-elem 'none'; style-src 'unsafe-inline'; img-src 'self' blob: data:; media-src 'self' blob:; sandbox allow-same-origin";@@ -40,6 +40,28 @@ pub fn add_security_headers(headers: &mut HeaderMap) {set_static_header(headers, header::CONTENT_SECURITY_POLICY, MEDIA_CSP);}+pub fn neutralize_script_content_type(status: StatusCode, headers: &mut HeaderMap) {+ let values = headers.get_all(header::CONTENT_TYPE);+ let scriptable = values.iter().any(|value| {+ value+ .to_str()+ .map_or(true, crate::mime::is_javascript_content_type)+ });+ let untyped = values.iter().all(|value| {+ value+ .as_bytes()+ .iter()+ .all(|byte| matches!(byte, b' ' | b'\t'))+ });+ let serves_body = matches!(status, StatusCode::OK | StatusCode::PARTIAL_CONTENT);+ if scriptable || (untyped && serves_body) {+ headers.insert(+ header::CONTENT_TYPE,+ HeaderValue::from_static(crate::mime::INERT_CONTENT_TYPE),+ );+ }+}+fn set_static_header(headers: &mut HeaderMap, name: HeaderName, value: &'static str) {headers.entry(name)@@ -281,6 +303,85 @@ mod tests {assert!(headers.get("CDN-Cache-Control").is_none());}+ #[test]+ fn neutralizing_rewrites_only_javascript_or_opaque_content_types() {+ let mut script = HeaderMap::new();+ script.insert(+ header::CONTENT_TYPE,+ HeaderValue::from_static("text/javascript"),+ );+ neutralize_script_content_type(StatusCode::OK, &mut script);+ assert_eq!(value(&script, "content-type"), "text/plain; charset=utf-8");++ let mut image = HeaderMap::new();+ image.insert(header::CONTENT_TYPE, HeaderValue::from_static("image/png"));+ neutralize_script_content_type(StatusCode::OK, &mut image);+ assert_eq!(value(&image, "content-type"), "image/png");++ let mut opaque = HeaderMap::new();+ opaque.insert(+ header::CONTENT_TYPE,+ HeaderValue::from_bytes(&[0xC3, 0x28]).expect("opaque header value"),+ );+ neutralize_script_content_type(StatusCode::OK, &mut opaque);+ assert_eq!(value(&opaque, "content-type"), "text/plain; charset=utf-8");++ let mut duplicated = HeaderMap::new();+ duplicated.insert(header::CONTENT_TYPE, HeaderValue::from_static("image/png"));+ duplicated.append(+ header::CONTENT_TYPE,+ HeaderValue::from_static("text/javascript"),+ );+ neutralize_script_content_type(StatusCode::OK, &mut duplicated);+ assert_eq!(duplicated.get_all(header::CONTENT_TYPE).iter().count(), 1);+ assert_eq!(+ value(&duplicated, "content-type"),+ "text/plain; charset=utf-8"+ );++ let mut junk = HeaderMap::new();+ junk.insert(+ header::CONTENT_TYPE,+ HeaderValue::from_static("video/mp4, text/javascript x"),+ );+ neutralize_script_content_type(StatusCode::NOT_FOUND, &mut junk);+ assert_eq!(value(&junk, "content-type"), "text/plain; charset=utf-8");++ let mut absent = HeaderMap::new();+ neutralize_script_content_type(StatusCode::RANGE_NOT_SATISFIABLE, &mut absent);+ assert!(absent.is_empty());+ }++ #[test]+ fn neutralizing_types_only_an_untyped_full_or_partial_response() {+ for status in [StatusCode::OK, StatusCode::PARTIAL_CONTENT] {+ for blank in [None, Some(""), Some(" \t ")] {+ let mut headers = HeaderMap::new();+ if let Some(blank) = blank {+ headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(blank));+ }+ neutralize_script_content_type(status, &mut headers);+ assert_eq!(headers.get_all(header::CONTENT_TYPE).iter().count(), 1);+ assert_eq!(+ value(&headers, "content-type"),+ "text/plain; charset=utf-8",+ "{status} {blank:?}"+ );+ }+ }+ for status in [+ StatusCode::NO_CONTENT,+ StatusCode::NOT_MODIFIED,+ StatusCode::NOT_FOUND,+ StatusCode::METHOD_NOT_ALLOWED,+ StatusCode::RANGE_NOT_SATISFIABLE,+ ] {+ let mut headers = HeaderMap::new();+ neutralize_script_content_type(status, &mut headers);+ assert!(headers.is_empty(), "{status}");+ }+ }+#[test]fn content_length_parser_requires_one_canonical_decimal_value() {let mut headers = HeaderMap::new();diff --git a/fluxer_media_proxy/src/metrics/attachment_signature.rs b/fluxer_media_proxy/src/metrics/attachment_signature.rsnew file mode 100644index 000000000..37fd60c87--- /dev/null+++ b/fluxer_media_proxy/src/metrics/attachment_signature.rs@@ -0,0 +1,40 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use fluxer_common::attachment_url_signature::Verdict;+use std::sync::atomic::{AtomicU64, Ordering};++pub(super) const VERDICTS: [Verdict; 5] = [+ Verdict::Valid,+ Verdict::Missing,+ Verdict::Malformed,+ Verdict::Mismatch,+ Verdict::Expired,+];++pub(super) const VERDICT_COUNT: usize = VERDICTS.len();++pub struct AttachmentSignatureMetrics {+ pub(super) verdicts: [AtomicU64; VERDICT_COUNT],+}++impl AttachmentSignatureMetrics {+ pub(crate) fn new() -> Self {+ Self {+ verdicts: [const { AtomicU64::new(0) }; VERDICT_COUNT],+ }+ }++ pub fn record(&self, verdict: Verdict) {+ self.verdicts[verdict_index(verdict)].fetch_add(1, Ordering::Relaxed);+ }+}++pub(super) fn verdict_index(verdict: Verdict) -> usize {+ match verdict {+ Verdict::Valid => 0,+ Verdict::Missing => 1,+ Verdict::Malformed => 2,+ Verdict::Mismatch => 3,+ Verdict::Expired => 4,+ }+}diff --git a/fluxer_media_proxy/src/metrics/mod.rs b/fluxer_media_proxy/src/metrics/mod.rsindex 6e513f29a..31dc39d8b 100644--- a/fluxer_media_proxy/src/metrics/mod.rs+++ b/fluxer_media_proxy/src/metrics/mod.rs@@ -1,5 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-later+pub mod attachment_signature;pub mod cache;pub mod external;pub mod histogram;@@ -15,6 +16,7 @@ pub mod transform;mod tests;use self::{+ attachment_signature::AttachmentSignatureMetrics,cache::{CoalescerMetrics, TransformCacheMetrics},external::ExternalMetrics,http_client::HTTPClientMetrics,@@ -58,6 +60,7 @@ pub struct Metrics {external: Arc<ExternalMetrics>,relay: Arc<RelayMetrics>,http_client: Arc<HTTPClientMetrics>,+ attachment_signature: Arc<AttachmentSignatureMetrics>,start_ms: i64,}@@ -74,6 +77,7 @@ impl Metrics {external: Arc::new(ExternalMetrics::new()),relay: Arc::new(RelayMetrics::new()),http_client: Arc::new(HTTPClientMetrics::new()),+ attachment_signature: Arc::new(AttachmentSignatureMetrics::new()),start_ms: now_ms(),}}@@ -118,6 +122,10 @@ impl Metrics {Arc::clone(&self.http_client)}+ pub fn attachment_signature(&self) -> Arc<AttachmentSignatureMetrics> {+ Arc::clone(&self.attachment_signature)+ }+pub fn render(&self) -> String {let mut out = String::new();render_metrics(&mut out, self)diff --git a/fluxer_media_proxy/src/metrics/rendering.rs b/fluxer_media_proxy/src/metrics/rendering.rsindex b7c84553b..610e96e19 100644--- a/fluxer_media_proxy/src/metrics/rendering.rs+++ b/fluxer_media_proxy/src/metrics/rendering.rs@@ -1,7 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse super::{- Metrics,+ Metrics, attachment_signature,histogram::Histogram,now_ms,request::{REQUEST_KIND_COUNT, RequestKind},@@ -272,6 +272,19 @@ pub(super) fn render_metrics(out: &mut String, metrics: &Metrics) -> fmt::Result"fluxer_media_proxy_avif_libheif_decode_failures_total",&metrics.transform.avif_libheif_decode_failures,)?;+ writeln!(+ out,+ "# TYPE fluxer_media_proxy_attachment_signature_verdicts_total counter"+ )?;+ for verdict in attachment_signature::VERDICTS {+ writeln!(+ out,+ "fluxer_media_proxy_attachment_signature_verdicts_total{{verdict=\"{}\"}} {}",+ verdict.label(),+ metrics.attachment_signature.verdicts[attachment_signature::verdict_index(verdict)]+ .load(Ordering::Relaxed)+ )?;+ }let uptime_ms = now_ms() - metrics.start_ms;writeln!(out,diff --git a/fluxer_media_proxy/src/metrics/request.rs b/fluxer_media_proxy/src/metrics/request.rsindex aa7ee9365..61e4daf59 100644--- a/fluxer_media_proxy/src/metrics/request.rs+++ b/fluxer_media_proxy/src/metrics/request.rs@@ -8,6 +8,7 @@ use std::sync::atomic::{AtomicU64, Ordering};pub enum RequestKind {Health,Metadata,+ Sniff,Thumbnail,Frames,AssetImage,@@ -21,9 +22,10 @@ pub enum RequestKind {}impl RequestKind {- pub const ALL: [RequestKind; 12] = [+ pub const ALL: [RequestKind; 13] = [Self::Health,Self::Metadata,+ Self::Sniff,Self::Thumbnail,Self::Frames,Self::AssetImage,@@ -40,6 +42,7 @@ impl RequestKind {match self {Self::Health => "health",Self::Metadata => "metadata",+ Self::Sniff => "sniff",Self::Thumbnail => "thumbnail",Self::Frames => "frames",Self::AssetImage => "asset_image",diff --git a/fluxer_media_proxy/src/metrics/tests.rs b/fluxer_media_proxy/src/metrics/tests.rsindex 16505a437..a4f64971c 100644--- a/fluxer_media_proxy/src/metrics/tests.rs+++ b/fluxer_media_proxy/src/metrics/tests.rs@@ -1,6 +1,9 @@// SPDX-License-Identifier: AGPL-3.0-or-later-use super::{Metrics, duration_millis, histogram::Histogram, request::RequestKind};+use super::{+ Metrics, attachment_signature, duration_millis, histogram::Histogram, request::RequestKind,+};+use fluxer_common::attachment_url_signature::Verdict;use std::collections::HashSet;use std::sync::atomic::Ordering;use std::time::Duration;@@ -72,6 +75,29 @@ const FROZEN_SERIES: &[(&str, &str)] = &["counter",),("fluxer_media_proxy_process_uptime_seconds", "counter"),+ (+ "fluxer_media_proxy_attachment_signature_verdicts_total",+ "counter",+ ),+];++const FROZEN_SIGNATURE_VERDICT_LABELS: &[&str] =+ &["valid", "missing", "malformed", "mismatch", "expired"];++const FROZEN_REQUEST_KIND_LABELS: &[&str] = &[+ "health",+ "metadata",+ "sniff",+ "thumbnail",+ "frames",+ "asset_image",+ "guild_member_image",+ "attachment",+ "external",+ "static",+ "themes",+ "upload",+ "other",];const GOLDEN_IMAGE_DURATION_SERIES: &str = r#"# TYPE fluxer_media_proxy_transform_image_duration_ms histogram@@ -151,6 +177,12 @@ fn request_kind_labels_are_unique() {}}+#[test]+fn request_kind_labels_match_the_frozen_label_list() {+ let labels: Vec<&str> = RequestKind::ALL.iter().map(|kind| kind.label()).collect();+ assert_eq!(FROZEN_REQUEST_KIND_LABELS, labels.as_slice());+}+#[test]fn render_keeps_every_frozen_series_name_and_shape() {let metrics = Metrics::new();@@ -166,11 +198,13 @@ fn render_keeps_every_frozen_series_name_and_shape() {"missing frozen series {name}");}- for kind in RequestKind::ALL {- assert!(text.contains(&format!(- "fluxer_media_proxy_requests_4xx_total{{kind=\"{}\"}} 0\n",- kind.label()- )));+ for label in FROZEN_REQUEST_KIND_LABELS {+ assert!(+ text.contains(&format!(+ "fluxer_media_proxy_requests_4xx_total{{kind=\"{label}\"}} 0\n"+ )),+ "missing request series for kind {label}"+ );}assert!(text.contains(GOLDEN_IMAGE_DURATION_SERIES));assert!(text.contains(GOLDEN_ROUTE_DURATION_SERIES));@@ -185,6 +219,58 @@ fn render_keeps_every_frozen_series_name_and_shape() {));}+#[test]+fn render_carries_a_zero_series_for_every_signature_verdict() {+ let text = Metrics::new().render();+ for label in FROZEN_SIGNATURE_VERDICT_LABELS {+ assert!(+ text.contains(&format!(+ "fluxer_media_proxy_attachment_signature_verdicts_total{{verdict=\"{label}\"}} 0\n"+ )),+ "missing signature verdict series for {label}"+ );+ }+}++#[test]+fn every_verdict_indexes_its_own_series() {+ for verdict in attachment_signature::VERDICTS {+ assert_eq!(+ verdict,+ attachment_signature::VERDICTS[attachment_signature::verdict_index(verdict)]+ );+ }+ let labels: Vec<&str> = attachment_signature::VERDICTS+ .iter()+ .map(|verdict| verdict.label())+ .collect();+ assert_eq!(FROZEN_SIGNATURE_VERDICT_LABELS, labels.as_slice());+}++#[test]+fn recording_a_verdict_moves_only_its_own_series() {+ let metrics = Metrics::new();+ let signature = metrics.attachment_signature();+ signature.record(Verdict::Mismatch);+ signature.record(Verdict::Mismatch);+ signature.record(Verdict::Valid);+ let text = metrics.render();+ for (label, count) in [+ ("valid", 1),+ ("missing", 0),+ ("malformed", 0),+ ("mismatch", 2),+ ("expired", 0),+ ] {+ assert!(+ text.contains(&format!(+ "fluxer_media_proxy_attachment_signature_verdicts_total{{verdict=\"{label}\"}} {count}\n"+ )),+ "{label} must read {count}\n{text}"+ );+ }+}+#[test]fn informational_and_unknown_statuses_stay_out_of_the_client_error_series() {let metrics = Metrics::new();diff --git a/fluxer_media_proxy/src/mime/mod.rs b/fluxer_media_proxy/src/mime/mod.rsindex aee00c55c..7c198f1ed 100644--- a/fluxer_media_proxy/src/mime/mod.rs+++ b/fluxer_media_proxy/src/mime/mod.rs@@ -15,7 +15,10 @@ use self::iso_bmff::iso_bmff_sniff;use self::stream_containers::{is_adts, looks_like_svg, matroska_sniff, mpeg_ts_sniff, ogg_sniff};use crate::media_type::MediaType;-pub use self::registry::{category, extension_mime, normalize, passthrough_mime};+pub use self::registry::{+ INERT_CONTENT_TYPE, category, extension_mime, is_javascript_content_type, normalize,+ passthrough_mime,+};#[derive(Clone, Copy, Debug, Eq, PartialEq)]pub enum Category {diff --git a/fluxer_media_proxy/src/mime/registry.rs b/fluxer_media_proxy/src/mime/registry.rsindex 2e82daa83..5a5cd2f75 100644--- a/fluxer_media_proxy/src/mime/registry.rs+++ b/fluxer_media_proxy/src/mime/registry.rs@@ -4,6 +4,41 @@ use super::Category;use crate::media_type::MediaType;use http::HeaderValue;+pub const INERT_CONTENT_TYPE: &str = "text/plain; charset=utf-8";++const JAVASCRIPT_ESSENCES: [&str; 16] = [+ "application/ecmascript",+ "application/javascript",+ "application/x-ecmascript",+ "application/x-javascript",+ "text/ecmascript",+ "text/javascript",+ "text/javascript1.0",+ "text/javascript1.1",+ "text/javascript1.2",+ "text/javascript1.3",+ "text/javascript1.4",+ "text/javascript1.5",+ "text/jscript",+ "text/livescript",+ "text/x-ecmascript",+ "text/x-javascript",+];++pub fn is_javascript_content_type(content_type: &str) -> bool {+ content_type.split(',').any(|member| {+ member+ .trim_start_matches([' ', '\t'])+ .split([' ', '\t', ';', '('])+ .next()+ .is_some_and(|essence| {+ JAVASCRIPT_ESSENCES+ .iter()+ .any(|javascript| essence.eq_ignore_ascii_case(javascript))+ })+ })+}+pub fn normalize(raw: Option<&str>) -> Option<&str> {let value = raw?;let semi = value.find(';').unwrap_or(value.len());diff --git a/fluxer_media_proxy/src/mime/tests.rs b/fluxer_media_proxy/src/mime/tests.rsindex 7dd264bba..00afad681 100644--- a/fluxer_media_proxy/src/mime/tests.rs+++ b/fluxer_media_proxy/src/mime/tests.rs@@ -2,9 +2,138 @@use super::*;use crate::test_fixtures::{- adversarial_media_bytes, animated_gif_fixture, apng_header, minimal_gif,+ ADVERSARIAL_TEXT_INPUTS, adversarial_media_bytes, animated_gif_fixture, apng_header,+ minimal_gif,};+const WHATWG_JAVASCRIPT_ESSENCES: [&str; 16] = [+ "application/ecmascript",+ "application/javascript",+ "application/x-ecmascript",+ "application/x-javascript",+ "text/ecmascript",+ "text/javascript",+ "text/javascript1.0",+ "text/javascript1.1",+ "text/javascript1.2",+ "text/javascript1.3",+ "text/javascript1.4",+ "text/javascript1.5",+ "text/jscript",+ "text/livescript",+ "text/x-ecmascript",+ "text/x-javascript",+];++#[test]+fn javascript_content_type_matches_every_whatwg_essence() {+ for essence in WHATWG_JAVASCRIPT_ESSENCES {+ assert!(is_javascript_content_type(essence), "{essence}");+ assert!(+ is_javascript_content_type(&essence.to_ascii_uppercase()),+ "{essence} upper case"+ );+ }+}++#[test]+fn javascript_content_type_ignores_case_whitespace_and_parameters() {+ for content_type in [+ "TEXT/JavaScript",+ " \ttext/javascript ; charset=utf-8",+ "text/javascript;goal=module",+ ] {+ assert!(is_javascript_content_type(content_type), "{content_type:?}");+ }+}++#[test]+fn javascript_content_type_checks_every_comma_separated_member() {+ assert!(is_javascript_content_type("image/png, text/javascript"));+ assert!(is_javascript_content_type("text/javascript, image/png"));+ assert!(!is_javascript_content_type(+ "video/mp4; codecs=\"avc1.42E01E, mp4a.40.2\""+ ));+}++#[test]+fn javascript_content_type_ends_the_essence_where_browsers_do() {+ for content_type in [+ "text/javascript x",+ "text/javascript\tx",+ "text/javascript(x)",+ "TEXT/JAVASCRIPT (x)",+ " text/javascript ;charset=x",+ "\ttext/javascript\t;goal=module",+ "application/x-javascript(",+ "text/jscript x;charset=utf-8",+ ] {+ assert!(is_javascript_content_type(content_type), "{content_type:?}");+ }+}++#[test]+fn javascript_content_type_checks_a_comma_member_that_has_junk() {+ for content_type in [+ "video/mp4, text/javascript x",+ "video/mp4,text/javascript(x)",+ "image/png, \ttext/javascript\tx",+ "text/plain; a=\",\", text/javascript x",+ "text/javascript x, video/mp4",+ ] {+ assert!(is_javascript_content_type(content_type), "{content_type:?}");+ }+}++#[test]+fn javascript_content_type_leaves_every_other_type_alone() {+ for content_type in [+ "text/css; charset=utf-8",+ "text/css",+ "application/wasm",+ "text/html",+ "application/xhtml+xml",+ "image/svg+xml",+ "image/png",+ "video/mp2t",+ "video/mp4",+ "application/json",+ "text/plain",+ "application/octet-stream",+ "text/javascript1.6",+ "text/javascriptx",+ "text/javascriptx (x)",+ "text/javascript1.6 x",+ "xtext/javascript",+ "x text/javascript",+ "text /javascript",+ "text/java script",+ "text/javascript\"x",+ "video/mp4, image/png x",+ "",+ " ",+ ",",+ ] {+ assert!(+ !is_javascript_content_type(content_type),+ "{content_type:?}"+ );+ }+}++#[test]+fn the_inert_content_type_is_not_javascript() {+ assert_eq!("text/plain; charset=utf-8", INERT_CONTENT_TYPE);+ assert!(!is_javascript_content_type(INERT_CONTENT_TYPE));+}++#[test]+fn javascript_content_type_never_panics_on_adversarial_text() {+ for text in ADVERSARIAL_TEXT_INPUTS {+ assert!(!is_javascript_content_type(text), "{text:?}");+ }+}+fn vp8x_webp(flags: u8, width_minus_one: u32, height_minus_one: u32) -> Vec<u8> {let mut webp = vec![0_u8; 30];webp[0..4].copy_from_slice(b"RIFF");diff --git a/fluxer_media_proxy/src/request_log/mod.rs b/fluxer_media_proxy/src/request_log/mod.rsindex 895094080..4c4d5af60 100644--- a/fluxer_media_proxy/src/request_log/mod.rs+++ b/fluxer_media_proxy/src/request_log/mod.rs@@ -19,6 +19,7 @@ use axum::{middleware::Next,response::Response,};+use fluxer_common::attachment_url_signature::is_signature_parameter_name;use rand::RngExt;use stage::{StageTimingSnapshot, StageTimings};use std::{future::Future, sync::Arc, time::Instant};@@ -27,7 +28,8 @@ use tracing::{Level, event};const ID_ALPHABET: &[u8] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ";const ID_LEN: usize = 12;const TARGET_LOG_BYTES_MAX: usize = 512;-const HEADER_LOG_BYTES_MAX: usize = 256;+pub(crate) const HEADER_LOG_BYTES_MAX: usize = 256;+const REDACTED_VALUE: &str = "[redacted]";#[derive(Clone, Debug)]pub struct RequestId(pub String);@@ -70,7 +72,7 @@ impl RequestObservation {kind: classify_route(path),path: clip(path, TARGET_LOG_BYTES_MAX),query: query- .map(|query| clip(query, TARGET_LOG_BYTES_MAX))+ .map(|query| clip(&redact_signature_values(query), TARGET_LOG_BYTES_MAX)).unwrap_or_default(),referer: header_str(headers, header::REFERER).map(|value| clip(&external_url_for_log(value), HEADER_LOG_BYTES_MAX)),@@ -87,6 +89,7 @@ pub fn classify_route(path: &str) -> RequestKind {"/_health" => Some(RequestKind::Health),"/_metrics" => Some(RequestKind::Other),"/_metadata" => Some(RequestKind::Metadata),+ "/_sniff" => Some(RequestKind::Sniff),"/_thumbnail" => Some(RequestKind::Thumbnail),"/_frames" => Some(RequestKind::Frames),_ => None,@@ -271,7 +274,28 @@ fn log_reason(reason: Option<ErrorReason>, status: StatusCode) -> (&'static str,}}-fn clip(value: &str, max: usize) -> String {+fn redact_signature_values(query: &str) -> String {+ if !query.contains('=') {+ return query.to_owned();+ }+ let mut out = String::with_capacity(query.len());+ for (index, field) in query.split('&').enumerate() {+ if index > 0 {+ out.push('&');+ }+ match field.split_once('=') {+ Some((name, _)) if is_signature_parameter_name(name) => {+ out.push_str(name);+ out.push('=');+ out.push_str(REDACTED_VALUE);+ }+ _ => out.push_str(field),+ }+ }+ out+}++pub(crate) fn clip(value: &str, max: usize) -> String {if value.len() <= max {return value.to_owned();}@@ -292,8 +316,48 @@ fn default_reason(status: StatusCode) -> &'static str {mod tests {use super::*;use crate::metrics::Metrics;- use axum::{Router, body::Body, middleware, response::IntoResponse, routing::get};+ use axum::{+ Router,+ body::Body,+ middleware,+ response::IntoResponse,+ routing::{get, post},+ };+ use std::sync::Mutex;use tower::ServiceExt;+ use tracing_subscriber::fmt::MakeWriter;++ #[derive(Clone, Default)]+ struct CapturedLog(Arc<Mutex<Vec<u8>>>);++ impl CapturedLog {+ fn text(&self) -> String {+ String::from_utf8(self.0.lock().expect("captured log is not poisoned").clone())+ .expect("captured log is utf-8")+ }+ }++ impl std::io::Write for CapturedLog {+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {+ self.0+ .lock()+ .expect("captured log is not poisoned")+ .extend_from_slice(buf);+ Ok(buf.len())+ }++ fn flush(&mut self) -> std::io::Result<()> {+ Ok(())+ }+ }++ impl<'a> MakeWriter<'a> for CapturedLog {+ type Writer = Self;++ fn make_writer(&'a self) -> Self::Writer {+ self.clone()+ }+ }fn req_id_is_alphabet_only(id: &str) -> bool {id.len() == ID_LEN && id.bytes().all(|c| ID_ALPHABET.contains(&c))@@ -324,6 +388,8 @@ mod tests {fn classify_route_buckets_known_prefixes() {assert_eq!(RequestKind::Health, classify_route("/_health"));assert_eq!(RequestKind::Metadata, classify_route("/_metadata"));+ assert_eq!(RequestKind::Sniff, classify_route("/_sniff"));+ assert_eq!(RequestKind::Other, classify_route("/_metrics"));assert_eq!(RequestKind::Upload, classify_route("/v1/relay/abc"));assert_eq!(RequestKind::External, classify_route("/external/x/y"));assert_eq!(RequestKind::Attachment, classify_route("/attachments/a/b"));@@ -366,6 +432,69 @@ mod tests {);}+ #[test]+ fn a_live_signature_never_reaches_the_log() {+ let signature = "f84dea2dbe168641c69753d838f270806583fedfe5c86f3f7a9a80f93af7ad28";+ for query in [+ format!("ex=6a9a7231&is=6a9920b1&hm={signature}"),+ format!("ex=6a9a7231&is=6a9920b1&hm={signature}&"),+ format!("width=64&ex=6a9a7231&is=6a9920b1&hm={signature}"),+ format!("%65x=6a9a7231&is=6a9920b1&hm={signature}&"),+ format!("hm={signature}&hm={signature}"),+ format!("ex=0&is=6a9920b1&hm={signature}&uc=dp"),+ format!("ex=0&is=6a9920b1&hm={signature}&uc=dp&"),+ format!("width=64&%75c=dp&hm={signature}&ex=6a9a7231&is=6a9920b1"),+ ] {+ let observation = RequestObservation::new(+ RequestId::generate(),+ Method::GET,+ "/attachments/1/2/a.png",+ Some(&query),+ &HeaderMap::new(),+ );+ assert!(!observation.query.contains(signature), "{query}");+ assert!(!observation.query.contains("6a9a7231"), "{query}");+ assert!(!observation.query.contains("6a9920b1"), "{query}");+ assert_eq!(+ query.matches('&').count(),+ observation.query.matches('&').count(),+ "{query}"+ );+ }+ }++ #[test]+ fn redaction_leaves_every_other_parameter_byte_for_byte() {+ for query in [+ "",+ "size=128",+ "width=64&height=64&format=webp",+ "download=1",+ "hm",+ "uc",+ "a=b&&c=d",+ "ucx=dp&dp=1",+ ] {+ assert_eq!(query, redact_signature_values(query), "{query}");+ }+ assert_eq!(+ "width=64&ex=[redacted]&is=[redacted]&hm=[redacted]&download=1",+ redact_signature_values("width=64&ex=6a9a7231&is=6a9920b1&hm=abc&download=1")+ );+ assert_eq!(+ "ex=[redacted]&is=[redacted]&hm=[redacted]&uc=[redacted]&&width=64",+ redact_signature_values("ex=0&is=6a9920b1&hm=abc&uc=dp&&width=64")+ );+ assert_eq!(+ "ex=[redacted]&is=[redacted]&hm=[redacted]",+ redact_signature_values("ex=6a9a7231&is=6a9920b1&hm=abc")+ );+ assert_eq!(+ "ex=[redacted]&is=[redacted]&hm=[redacted]&uc=[redacted]",+ redact_signature_values("ex=0&is=6a9920b1&hm=abc&uc=dp")+ );+ }+#[test]fn clip_truncates_with_marker() {assert_eq!("abc", clip("abc", 8));@@ -463,6 +592,56 @@ mod tests {assert!(rendered.contains("fluxer_media_proxy_request_duration_ms_count 1\n"));}+ #[tokio::test]+ async fn a_successful_sniff_writes_its_own_request_line_and_series() {+ let metrics = Metrics::new();+ let app = Router::new()+ .route("/_sniff", post(|| async { "{\"content_type\":null}" }))+ .route("/_metrics", get(|| async { "" }))+ .layer(middleware::from_fn_with_state(metrics.request(), trace));+ let captured = CapturedLog::default();+ let subscriber = tracing_subscriber::fmt()+ .with_writer(captured.clone())+ .with_ansi(false)+ .with_max_level(tracing::Level::TRACE)+ .finish();+ {+ let _guard = tracing::subscriber::set_default(subscriber);+ for (method, uri) in [(Method::POST, "/_sniff"), (Method::GET, "/_metrics")] {+ let response = app+ .clone()+ .oneshot(+ axum::http::Request::builder()+ .method(method)+ .uri(uri)+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();+ assert_eq!(StatusCode::OK, response.status(), "{uri}");+ }+ }+ let log = captured.text();+ let request_lines: Vec<&str> = log+ .lines()+ .filter(|line| line.contains(" request "))+ .collect();+ assert_eq!(1, request_lines.len(), "{log}");+ assert!(request_lines[0].contains("kind=\"sniff\""), "{log}");+ assert!(request_lines[0].contains("method=POST"), "{log}");+ assert!(request_lines[0].contains("path=/_sniff"), "{log}");+ assert!(request_lines[0].contains("status=200"), "{log}");+ let rendered = metrics.render();+ assert!(rendered.contains("fluxer_media_proxy_requests_2xx_total{kind=\"sniff\"} 1\n"));+ assert!(rendered.contains("fluxer_media_proxy_requests_2xx_total{kind=\"other\"} 1\n"));+ assert!(+ rendered.contains(+ "fluxer_media_proxy_request_duration_by_route_ms_count{kind=\"sniff\"} 1\n"+ )+ );+ }+#[tokio::test]async fn trace_public_request_observes_a_non_axum_request() {let metrics = Metrics::new();diff --git a/fluxer_media_proxy/src/server/attachment_signature.rs b/fluxer_media_proxy/src/server/attachment_signature.rsnew file mode 100644index 000000000..c6e5b2dc9--- /dev/null+++ b/fluxer_media_proxy/src/server/attachment_signature.rs@@ -0,0 +1,432 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use crate::{+ config::AttachmentSignatureConfig,+ metrics::now_ms,+ request_log::{self, HEADER_LOG_BYTES_MAX, RequestId, clip},+ server::response::error::text_with_source,+};+use axum::{+ http::{HeaderMap, HeaderValue, Method, StatusCode, header},+ response::Response,+};+use fluxer_common::attachment_url_signature::{+ ATTACHMENT_URL_TTL_SECS, Verdict, Verification, decode_key, verify,+};+use std::{+ sync::atomic::{AtomicI64, AtomicU64, Ordering},+ time::{SystemTime, UNIX_EPOCH},+};++const UNAVAILABLE_BODY: &str = "This content is no longer available.";+const ATTACHMENT_PATH: &str = "/attachments/";+const WOULD_DENY_LOG_WINDOW_MS: i64 = 1_000;+const WOULD_DENY_LOG_LINES_PER_WINDOW: u64 = 20;++pub(in crate::server) struct WouldDenyLog {+ window_start_ms: AtomicI64,+ logged_in_window: AtomicU64,+ suppressed: AtomicU64,+}++impl WouldDenyLog {+ pub(in crate::server) fn new() -> Self {+ Self {+ window_start_ms: AtomicI64::new(i64::MIN),+ logged_in_window: AtomicU64::new(0),+ suppressed: AtomicU64::new(0),+ }+ }++ fn admit(&self, now_ms: i64) -> Option<u64> {+ if now_ms.saturating_sub(self.window_start_ms.load(Ordering::Relaxed))+ >= WOULD_DENY_LOG_WINDOW_MS+ {+ self.window_start_ms.store(now_ms, Ordering::Relaxed);+ self.logged_in_window.store(0, Ordering::Relaxed);+ }+ if self.logged_in_window.fetch_add(1, Ordering::Relaxed) >= WOULD_DENY_LOG_LINES_PER_WINDOW+ {+ self.suppressed.fetch_add(1, Ordering::Relaxed);+ return None;+ }+ Some(self.suppressed.swap(0, Ordering::Relaxed))+ }+}++pub(in crate::server) fn check(+ cfg: &AttachmentSignatureConfig,+ path: &str,+ raw_query: Option<&str>,+) -> Verification {+ let Some(key) = decode_key(path) else {+ return Verification {+ verdict: Verdict::Malformed,+ remaining_secs: None,+ };+ };+ verify(&key, raw_query, &cfg.secrets(), now_secs())+}++pub(in crate::server) fn limit_cache_lifetime(response: &mut Response, verification: Verification) {+ if verification.verdict != Verdict::Valid+ || !matches!(+ response.status(),+ StatusCode::OK | StatusCode::PARTIAL_CONTENT+ )+ {+ return;+ }+ let max_age = verification+ .remaining_secs+ .unwrap_or(ATTACHMENT_URL_TTL_SECS);+ let headers = response.headers_mut();+ let no_transform = headers+ .get(header::CACHE_CONTROL)+ .and_then(|value| value.to_str().ok())+ .is_some_and(|value| value.contains("no-transform"));+ let shared = format!("public, max-age={max_age}");+ let browser = if no_transform {+ format!("{shared}, no-transform")+ } else {+ shared.clone()+ };+ if let (Ok(browser), Ok(shared)) = (+ HeaderValue::from_str(&browser),+ HeaderValue::from_str(&shared),+ ) {+ headers.insert(header::CACHE_CONTROL, browser);+ headers.insert("CDN-Cache-Control", shared);+ }+}++pub(in crate::server) fn denied_response(verdict: Verdict) -> Response {+ text_with_source(+ StatusCode::NOT_FOUND,+ UNAVAILABLE_BODY,+ refusal_code(verdict),+ verdict,+ )+}++pub(in crate::server) fn masked_not_found_response(key: &str) -> Response {+ text_with_source(+ StatusCode::NOT_FOUND,+ UNAVAILABLE_BODY,+ "attachment_not_found",+ format!("key={key}"),+ )+}++pub(in crate::server) fn log_would_deny(+ budget: &WouldDenyLog,+ method: &Method,+ verdict: Verdict,+ headers: &HeaderMap,+ request_id: Option<&RequestId>,+) {+ let Some(suppressed) = budget.admit(now_ms()) else {+ return;+ };+ tracing::warn!(+ reason = "attachment_signature_would_deny",+ kind = request_log::classify_route(ATTACHMENT_PATH).label(),+ method = %method,+ verdict = verdict.label(),+ req = request_id.map(RequestId::as_str).unwrap_or(""),+ suppressed,+ user_agent = %user_agent_for_log(headers),+ "attachment read would be refused"+ );+}++fn refusal_code(verdict: Verdict) -> &'static str {+ match verdict {+ Verdict::Missing => "attachment_signature_missing",+ Verdict::Malformed => "attachment_signature_malformed",+ Verdict::Mismatch => "attachment_signature_mismatch",+ Verdict::Expired => "attachment_signature_expired",+ Verdict::Valid => "attachment_signature_valid",+ }+}++fn user_agent_for_log(headers: &HeaderMap) -> String {+ headers+ .get(header::USER_AGENT)+ .and_then(|value| value.to_str().ok())+ .map(|value| clip(value, HEADER_LOG_BYTES_MAX))+ .unwrap_or_default()+}++fn now_secs() -> u64 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap_or_default()+ .as_secs()+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::{config::PolicyMode, http_headers, request_log::ErrorReason};+ use axum::body::to_bytes;+ use base64::{Engine as _, engine::general_purpose};+ use fluxer_common::attachment_url_signature::{UrlKind, issue_window, sign};+ use std::sync::{Arc, Mutex};+ use tracing_subscriber::fmt::MakeWriter;++ const SECRET: [u8; 32] = [3u8; 32];+ const KEY: &str = "attachments/1/2/cat.gif";+ const PATH: &str = "/attachments/1/2/cat.gif";++ #[derive(Clone, Default)]+ struct CapturedLog(Arc<Mutex<Vec<u8>>>);++ impl CapturedLog {+ fn text(&self) -> String {+ String::from_utf8(self.0.lock().expect("captured log is not poisoned").clone())+ .expect("captured log is utf-8")+ }+ }++ impl std::io::Write for CapturedLog {+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {+ self.0+ .lock()+ .expect("captured log is not poisoned")+ .extend_from_slice(buf);+ Ok(buf.len())+ }++ fn flush(&mut self) -> std::io::Result<()> {+ Ok(())+ }+ }++ impl<'a> MakeWriter<'a> for CapturedLog {+ type Writer = Self;++ fn make_writer(&'a self) -> Self::Writer {+ self.clone()+ }+ }++ fn enforcing_config() -> AttachmentSignatureConfig {+ crate::config::Config::load_from_iter([+ ("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret"),+ ("FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE", "enforce"),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64",+ general_purpose::STANDARD.encode(SECRET).as_str(),+ ),+ ])+ .expect("signature config")+ .attachment_signature+ }++ fn check_verdict(+ cfg: &AttachmentSignatureConfig,+ path: &str,+ raw_query: Option<&str>,+ ) -> Verdict {+ check(cfg, path, raw_query).verdict+ }++ fn signed_query(key: &str, now: u64) -> String {+ let (issued, expires) = issue_window(now, now);+ format!(+ "ex={expires:08x}&is={issued:08x}&hm={}",+ sign(key, expires, issued, UrlKind::Ordinary, &SECRET)+ )+ }++ #[test]+ fn the_check_reads_the_raw_query_against_the_decoded_key() {+ let cfg = enforcing_config();+ assert_eq!(PolicyMode::Enforce, cfg.mode);+ let now = now_secs();+ let query = signed_query(KEY, now);+ assert_eq!(Verdict::Valid, check_verdict(&cfg, PATH, Some(&query)));+ assert_eq!(Verdict::Missing, check_verdict(&cfg, PATH, None));+ assert_eq!(+ Verdict::Mismatch,+ check_verdict(&cfg, "/attachments/1/2/dog.gif", Some(&query))+ );+ assert_eq!(+ Verdict::Malformed,+ check_verdict(&cfg, "/attachments/1/2/%FF.gif", Some(&query))+ );+ assert_eq!(+ Verdict::Valid,+ check_verdict(&cfg, PATH, Some(&format!("{query}&")))+ );+ let (issued, _) = issue_window(1, 1);+ let package = format!(+ "ex=0&is={issued:08x}&hm={}&uc=dp",+ sign(KEY, 0, issued, UrlKind::DataPackage, &SECRET)+ );+ assert_eq!(Verdict::Valid, check_verdict(&cfg, PATH, Some(&package)));+ assert_eq!(+ Verdict::Valid,+ check_verdict(&cfg, PATH, Some(&format!("{package}&")))+ );+ assert_eq!(+ Verdict::Malformed,+ check_verdict(&cfg, PATH, Some(&package.replace("&uc=dp", "")))+ );+ assert_eq!(+ Verdict::Mismatch,+ check_verdict(&cfg, "/attachments/1/2/dog.gif", Some(&package))+ );+ }++ #[tokio::test]+ async fn every_refusal_is_the_same_thirty_six_byte_not_found() {+ let mut bodies = Vec::new();+ for verdict in [+ Verdict::Missing,+ Verdict::Malformed,+ Verdict::Mismatch,+ Verdict::Expired,+ ] {+ let response = denied_response(verdict);+ assert_eq!(StatusCode::NOT_FOUND, response.status());+ let headers = response.headers().clone();+ assert_eq!(+ "text/plain; charset=utf-8",+ headers.get(header::CONTENT_TYPE).expect("content type")+ );+ assert_eq!(+ "no-store",+ headers.get(header::CACHE_CONTROL).expect("cache control")+ );+ assert_eq!(+ "nosniff",+ headers+ .get(header::X_CONTENT_TYPE_OPTIONS)+ .expect("nosniff")+ );+ assert_eq!(+ http_headers::MEDIA_CSP,+ headers.get(header::CONTENT_SECURITY_POLICY).expect("csp")+ );+ assert_eq!(+ http_headers::STRICT_TRANSPORT_SECURITY,+ headers+ .get(header::STRICT_TRANSPORT_SECURITY)+ .expect("hsts")+ );+ assert!(!headers.contains_key(header::ACCESS_CONTROL_ALLOW_ORIGIN));+ let reason = response+ .extensions()+ .get::<ErrorReason>()+ .expect("a refusal carries an error reason")+ .clone();+ assert_eq!(refusal_code(verdict), reason.code);+ assert!(reason.code.starts_with("attachment_signature_"));+ bodies.push(+ to_bytes(response.into_body(), usize::MAX)+ .await+ .expect("body"),+ );+ }+ bodies.push(+ to_bytes(masked_not_found_response(KEY).into_body(), usize::MAX)+ .await+ .expect("body"),+ );+ for body in &bodies {+ assert_eq!(36, body.len());+ assert_eq!(UNAVAILABLE_BODY.as_bytes(), body.as_ref());+ assert_eq!(bodies[0], *body);+ }+ }++ #[test]+ fn the_would_deny_line_carries_the_verdict_and_a_clipped_user_agent() {+ let mut headers = HeaderMap::new();+ headers.insert(+ header::USER_AGENT,+ "u".repeat(4096).parse().expect("ascii user agent"),+ );+ let captured = CapturedLog::default();+ let subscriber = tracing_subscriber::fmt()+ .with_writer(captured.clone())+ .with_ansi(false)+ .with_max_level(tracing::Level::TRACE)+ .finish();+ {+ let _guard = tracing::subscriber::set_default(subscriber);+ log_would_deny(+ &WouldDenyLog::new(),+ &Method::GET,+ Verdict::Expired,+ &headers,+ Some(&RequestId("ABCDEFGHJKMN".to_owned())),+ );+ }+ let line = captured.text();+ assert!(+ line.contains("reason=\"attachment_signature_would_deny\""),+ "{line}"+ );+ assert!(line.contains("kind=\"attachment\""), "{line}");+ assert!(line.contains("method=GET"), "{line}");+ assert!(line.contains("verdict=\"expired\""), "{line}");+ assert!(line.contains("req=\"ABCDEFGHJKMN\""), "{line}");+ assert!(line.contains(&"u".repeat(HEADER_LOG_BYTES_MAX)), "{line}");+ assert!(+ !line.contains(&"u".repeat(HEADER_LOG_BYTES_MAX + 1)),+ "{line}"+ );+ }++ #[test]+ fn a_request_without_an_id_or_user_agent_still_logs_one_line() {+ let captured = CapturedLog::default();+ let subscriber = tracing_subscriber::fmt()+ .with_writer(captured.clone())+ .with_ansi(false)+ .with_max_level(tracing::Level::TRACE)+ .finish();+ {+ let _guard = tracing::subscriber::set_default(subscriber);+ log_would_deny(+ &WouldDenyLog::new(),+ &Method::HEAD,+ Verdict::Missing,+ &HeaderMap::new(),+ None,+ );+ }+ let line = captured.text();+ assert_eq!(1, line.lines().count(), "{line}");+ assert!(line.contains("verdict=\"missing\""), "{line}");+ assert!(line.contains("req=\"\""), "{line}");+ assert!(line.contains("suppressed=0"), "{line}");+ }++ #[test]+ fn the_would_deny_budget_bounds_a_window_and_reports_what_it_dropped() {+ let budget = WouldDenyLog::new();+ let window = 10_000;+ for line in 0..WOULD_DENY_LOG_LINES_PER_WINDOW {+ assert_eq!(Some(0), budget.admit(window), "line {line}");+ }+ for line in 0..5 {+ assert_eq!(None, budget.admit(window + 500), "line {line}");+ }+ let next = window + WOULD_DENY_LOG_WINDOW_MS;+ assert_eq!(Some(5), budget.admit(next));+ assert_eq!(Some(0), budget.admit(next));+ }++ #[test]+ fn the_would_deny_budget_never_wedges_on_a_clock_that_goes_backwards() {+ let budget = WouldDenyLog::new();+ assert_eq!(Some(0), budget.admit(10_000));+ assert_eq!(Some(0), budget.admit(9_000));+ assert_eq!(Some(0), budget.admit(i64::MIN));+ assert_eq!(Some(0), budget.admit(i64::MAX));+ }+}diff --git a/fluxer_media_proxy/src/server/cors.rs b/fluxer_media_proxy/src/server/cors.rsnew file mode 100644index 000000000..650150cb5--- /dev/null+++ b/fluxer_media_proxy/src/server/cors.rs@@ -0,0 +1,337 @@+// SPDX-License-Identifier: AGPL-3.0-or-later++use crate::{request_log, server::response::error::text_with_source};+use axum::{+ http::{HeaderMap, HeaderValue, Method, StatusCode, header},+ response::Response,+};++const VARY_ON_ORIGIN: &str = "Accept-Encoding, Origin";+const ORIGIN_LOG_BYTES_MAX: usize = 256;++#[derive(Debug, Eq, PartialEq)]+pub(in crate::server) enum OriginCheck {+ Absent,+ Allowed(HeaderValue),+ Refused,+}++pub(in crate::server) fn check_origin(allowed: &[HeaderValue], request: &HeaderMap) -> OriginCheck {+ let mut origins = request.get_all(header::ORIGIN).iter();+ let Some(origin) = origins.next() else {+ return OriginCheck::Absent;+ };+ if origins.next().is_some() {+ return OriginCheck::Refused;+ }+ allowed+ .iter()+ .find(|candidate| candidate.as_bytes() == origin.as_bytes())+ .map_or(OriginCheck::Refused, |candidate| {+ OriginCheck::Allowed(candidate.clone())+ })+}++pub(in crate::server) fn refused_response(request: &HeaderMap) -> Response {+ let mut response = text_with_source(+ StatusCode::FORBIDDEN,+ "Forbidden",+ "cors_origin_denied",+ format!("origin={}", origin_for_log(request)),+ );+ response+ .headers_mut()+ .insert(header::VARY, HeaderValue::from_static(VARY_ON_ORIGIN));+ response+}++pub(in crate::server) fn log_would_refuse(method: &Method, path: &str, request: &HeaderMap) {+ tracing::warn!(+ reason = "cors_origin_would_deny",+ kind = request_log::classify_route(path).label(),+ method = %method,+ origin = %origin_for_log(request),+ "public read origin would be refused"+ );+}++pub(in crate::server) fn narrow_response(check: &OriginCheck, headers: &mut HeaderMap) {+ headers.insert(header::VARY, HeaderValue::from_static(VARY_ON_ORIGIN));+ if !headers.contains_key(header::ACCESS_CONTROL_ALLOW_ORIGIN) {+ return;+ }+ match check {+ OriginCheck::Allowed(origin) => {+ headers.insert(header::ACCESS_CONTROL_ALLOW_ORIGIN, origin.clone());+ }+ OriginCheck::Absent | OriginCheck::Refused => {+ headers.remove(header::ACCESS_CONTROL_ALLOW_ORIGIN);+ }+ }+}++fn origin_for_log(request: &HeaderMap) -> &str {+ let Some(Ok(origin)) = request.get(header::ORIGIN).map(HeaderValue::to_str) else {+ return "";+ };+ &origin[..origin.len().min(ORIGIN_LOG_BYTES_MAX)]+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::{+ http_headers::{self, add_media_headers, add_unsatisfiable_headers},+ request_log::ErrorReason,+ server::response::error::text,+ test_fixtures::ADVERSARIAL_TEXT_INPUTS,+ };+ use axum::body::to_bytes;+ use std::sync::{Arc, Mutex};+ use tracing_subscriber::fmt::MakeWriter;++ const WEB: &str = "https://web.fluxer.app";+ const CANARY: &str = "https://web.canary.fluxer.app";++ #[derive(Clone, Default)]+ struct CapturedLog(Arc<Mutex<Vec<u8>>>);++ impl CapturedLog {+ fn text(&self) -> String {+ String::from_utf8(self.0.lock().expect("captured log is not poisoned").clone())+ .expect("captured log is utf-8")+ }+ }++ impl std::io::Write for CapturedLog {+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {+ self.0+ .lock()+ .expect("captured log is not poisoned")+ .extend_from_slice(buf);+ Ok(buf.len())+ }++ fn flush(&mut self) -> std::io::Result<()> {+ Ok(())+ }+ }++ impl<'a> MakeWriter<'a> for CapturedLog {+ type Writer = Self;++ fn make_writer(&'a self) -> Self::Writer {+ self.clone()+ }+ }++ fn allowlist() -> Vec<HeaderValue> {+ vec![+ HeaderValue::from_static(WEB),+ HeaderValue::from_static(CANARY),+ ]+ }++ fn origin_headers(origins: &[HeaderValue]) -> HeaderMap {+ let mut headers = HeaderMap::new();+ for origin in origins {+ headers.append(header::ORIGIN, origin.clone());+ }+ headers+ }++ fn check(origins: &[HeaderValue]) -> OriginCheck {+ check_origin(&allowlist(), &origin_headers(origins))+ }++ fn header_text(headers: &HeaderMap, name: header::HeaderName) -> Option<&str> {+ headers+ .get(name)+ .map(|value| value.to_str().expect("header is ascii"))+ }++ fn vary_values(headers: &HeaderMap) -> Vec<&str> {+ headers+ .get_all(header::VARY)+ .iter()+ .map(|value| value.to_str().expect("vary is ascii"))+ .collect()+ }++ #[test]+ fn origin_check_classifies_every_origin_shape() {+ assert_eq!(OriginCheck::Absent, check(&[]));+ for allowed in [WEB, CANARY] {+ assert_eq!(+ OriginCheck::Allowed(HeaderValue::from_static(allowed)),+ check(&[HeaderValue::from_static(allowed)]),+ "{allowed}"+ );+ }+ for refused in [+ "null",+ "https://evil.example",+ "https://web.fluxer.app/",+ "HTTPS://WEB.FLUXER.APP",+ "https://web.fluxer.app:443",+ "http://web.fluxer.app",+ "https://web.fluxer.app.evil.example",+ "",+ ] {+ assert_eq!(+ OriginCheck::Refused,+ check(&[HeaderValue::from_static(refused)]),+ "{refused:?}"+ );+ }+ assert_eq!(+ OriginCheck::Refused,+ check(&[HeaderValue::from_bytes(&[0xC3, 0x28]).expect("opaque origin")])+ );+ assert_eq!(+ OriginCheck::Refused,+ check(&[HeaderValue::from_static(WEB), HeaderValue::from_static(WEB)])+ );+ assert_eq!(+ OriginCheck::Refused,+ check(&[+ HeaderValue::from_static(WEB),+ HeaderValue::from_static(CANARY)+ ])+ );+ }++ #[test]+ fn adversarial_text_is_never_an_allowed_origin() {+ let mut checked = 0;+ for text in ADVERSARIAL_TEXT_INPUTS {+ let Ok(origin) = HeaderValue::from_str(text) else {+ continue;+ };+ assert_eq!(OriginCheck::Refused, check(&[origin]), "{text:?}");+ checked += 1;+ }+ assert!(+ checked > 0,+ "at least one adversarial input is a header value"+ );+ }++ #[test]+ fn narrowing_rewrites_only_an_allow_origin_a_builder_set() {+ let allowed = OriginCheck::Allowed(HeaderValue::from_static(WEB));++ let mut media_allowed = HeaderMap::new();+ add_media_headers(&mut media_allowed, 16, "image/png", None);+ narrow_response(&allowed, &mut media_allowed);+ assert_eq!(+ Some(WEB),+ header_text(&media_allowed, header::ACCESS_CONTROL_ALLOW_ORIGIN)+ );++ let mut media_absent = HeaderMap::new();+ add_media_headers(&mut media_absent, 16, "image/png", None);+ narrow_response(&OriginCheck::Absent, &mut media_absent);+ assert!(!media_absent.contains_key(header::ACCESS_CONTROL_ALLOW_ORIGIN));++ let mut media_refused = HeaderMap::new();+ add_media_headers(&mut media_refused, 16, "image/png", None);+ narrow_response(&OriginCheck::Refused, &mut media_refused);+ assert!(!media_refused.contains_key(header::ACCESS_CONTROL_ALLOW_ORIGIN));++ let mut unsatisfiable = HeaderMap::new();+ add_unsatisfiable_headers(&mut unsatisfiable, 16);+ narrow_response(&allowed, &mut unsatisfiable);+ assert_eq!(+ Some(WEB),+ header_text(&unsatisfiable, header::ACCESS_CONTROL_ALLOW_ORIGIN)+ );++ let mut not_found = text(StatusCode::NOT_FOUND, "Not Found").headers().clone();+ narrow_response(&allowed, &mut not_found);+ assert!(!not_found.contains_key(header::ACCESS_CONTROL_ALLOW_ORIGIN));++ for headers in [+ &media_allowed,+ &media_absent,+ &media_refused,+ &unsatisfiable,+ ¬_found,+ ] {+ assert_eq!(vec!["Accept-Encoding, Origin"], vary_values(headers));+ }+ }++ #[tokio::test]+ async fn refused_response_is_a_no_store_forbidden_without_cors() {+ let response = refused_response(&origin_headers(&[HeaderValue::from_static(+ "https://evil.example",+ )]));+ assert_eq!(StatusCode::FORBIDDEN, response.status());+ let headers = response.headers();+ assert_eq!(+ Some("text/plain; charset=utf-8"),+ header_text(headers, header::CONTENT_TYPE)+ );+ assert_eq!(+ Some("no-store"),+ header_text(headers, header::CACHE_CONTROL)+ );+ assert_eq!(vec!["Accept-Encoding, Origin"], vary_values(headers));+ assert!(!headers.contains_key(header::ACCESS_CONTROL_ALLOW_ORIGIN));+ assert_eq!(+ Some(http_headers::MEDIA_CSP),+ header_text(headers, header::CONTENT_SECURITY_POLICY)+ );+ assert_eq!(+ Some(http_headers::STRICT_TRANSPORT_SECURITY),+ header_text(headers, header::STRICT_TRANSPORT_SECURITY)+ );+ let reason = response+ .extensions()+ .get::<ErrorReason>()+ .expect("a refusal carries an error reason")+ .clone();+ assert_eq!("cors_origin_denied", reason.code);+ assert_eq!(+ Some(format!("{:?}", "origin=https://evil.example")),+ reason.source+ );+ assert_eq!(+ b"Forbidden".as_slice(),+ to_bytes(response.into_body(), usize::MAX)+ .await+ .expect("refusal body")+ );+ }++ #[test]+ fn a_long_origin_is_clipped_for_logging() {+ let long = "a".repeat(4096);+ let headers = origin_headers(&[HeaderValue::from_str(&long).expect("ascii origin")]);+ let clipped = "a".repeat(ORIGIN_LOG_BYTES_MAX);+ assert_eq!(clipped, origin_for_log(&headers));++ let captured = CapturedLog::default();+ let subscriber = tracing_subscriber::fmt()+ .with_writer(captured.clone())+ .with_ansi(false)+ .finish();+ {+ let _guard = tracing::subscriber::set_default(subscriber);+ log_would_refuse(&Method::GET, "/avatars/1/hash.png", &headers);+ }+ let line = captured.text();+ assert!(line.contains("reason=\"cors_origin_would_deny\""), "{line}");+ assert!(line.contains("kind=\"asset_image\""), "{line}");+ assert!(line.contains("method=GET"), "{line}");+ assert!(+ line.trim_end().ends_with(&format!("origin={clipped}")),+ "{line}"+ );+ assert!(+ !line.contains(&"a".repeat(ORIGIN_LOG_BYTES_MAX + 1)),+ "{line}"+ );+ }+}diff --git a/fluxer_media_proxy/src/server/middleware.rs b/fluxer_media_proxy/src/server/middleware.rsindex a2cc96353..1acf3bf3e 100644--- a/fluxer_media_proxy/src/server/middleware.rs+++ b/fluxer_media_proxy/src/server/middleware.rs@@ -150,10 +150,13 @@ pub(in crate::server) async fn add_security_header_middleware(next: axum::middleware::Next,) -> Response {let mut response = next.run(request).await;+ let status = response.status();let headers = response.headers_mut();http_headers::add_security_headers(headers);if mode == DeploymentMode::Static {headers.remove("X-Robots-Tag");+ } else {+ http_headers::neutralize_script_content_type(status, headers);}response}@@ -161,7 +164,386 @@ pub(in crate::server) async fn add_security_header_middleware(#[cfg(test)]mod tests {use super::*;- use axum::{Router, routing::get};+ use crate::{+ server::{routes::dispatch::catch_all, state::AppState},+ storage::tests::{FakeObject, fake_s3},+ };+ use axum::{+ Router,+ http::{HeaderMap, HeaderName, Method, StatusCode, header},+ routing::get,+ };++ const INERT: &str = "text/plain; charset=utf-8";++ const BROWSER_PARSED_JAVASCRIPT: [&str; 7] = [+ "text/javascript x",+ "text/javascript\tx",+ "text/javascript(x)",+ "TEXT/JAVASCRIPT (x)",+ "video/mp4, text/javascript x",+ " text/javascript ;charset=x",+ "image/png,text/javascript(x)",+ ];++ async fn headers_for(mode: DeploymentMode, content_type: Option<&'static str>) -> HeaderMap {+ headers_with_status(mode, StatusCode::OK, content_type).await+ }++ async fn headers_with_status(+ mode: DeploymentMode,+ status: StatusCode,+ content_type: Option<&'static str>,+ ) -> HeaderMap {+ let router = Router::new()+ .route(+ "/probe",+ get(move || async move {+ let mut response = Response::new(Body::empty());+ *response.status_mut() = status;+ if let Some(content_type) = content_type {+ response+ .headers_mut()+ .insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type));+ }+ response+ }),+ )+ .layer(axum::middleware::from_fn_with_state(+ mode,+ add_security_header_middleware,+ ));+ let response = tower::ServiceExt::oneshot(+ router,+ Request::builder()+ .uri("/probe")+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();+ response.headers().clone()+ }++ fn header_text(headers: &HeaderMap, name: HeaderName) -> Option<&str> {+ headers.get(name).map(|value| value.to_str().unwrap())+ }++ #[tokio::test]+ async fn mp_mode_serves_javascript_as_plain_text() {+ for content_type in [+ "text/javascript",+ "application/javascript; charset=utf-8",+ "image/png, text/javascript",+ ] {+ let headers = headers_for(DeploymentMode::Mp, Some(content_type)).await;+ assert_eq!(+ Some(INERT),+ header_text(&headers, header::CONTENT_TYPE),+ "{content_type}"+ );+ assert_eq!(+ Some("nosniff"),+ header_text(&headers, header::X_CONTENT_TYPE_OPTIONS),+ "{content_type}"+ );+ }+ }++ #[tokio::test]+ async fn mp_mode_serves_javascript_that_browsers_parse_past_junk_as_plain_text() {+ for content_type in BROWSER_PARSED_JAVASCRIPT {+ for status in [StatusCode::OK, StatusCode::PARTIAL_CONTENT] {+ let headers =+ headers_with_status(DeploymentMode::Mp, status, Some(content_type)).await;+ assert_eq!(+ Some(INERT),+ header_text(&headers, header::CONTENT_TYPE),+ "{status} {content_type:?}"+ );+ assert_eq!(1, headers.get_all(header::CONTENT_TYPE).iter().count());+ }+ }+ }++ #[tokio::test]+ async fn static_mode_serves_javascript_unchanged() {+ for content_type in ["text/javascript", "application/javascript; charset=utf-8"]+ .into_iter()+ .chain(BROWSER_PARSED_JAVASCRIPT)+ {+ let headers = headers_for(DeploymentMode::Static, Some(content_type)).await;+ assert_eq!(+ Some(content_type),+ header_text(&headers, header::CONTENT_TYPE),+ "{content_type:?}"+ );+ }+ }++ #[tokio::test]+ async fn every_mode_that_serves_stored_bytes_neutralizes_javascript() {+ for mode in [+ DeploymentMode::Mp,+ DeploymentMode::Upload,+ DeploymentMode::Relay,+ ] {+ for content_type in ["text/javascript", "application/javascript; charset=utf-8"]+ .into_iter()+ .chain(BROWSER_PARSED_JAVASCRIPT)+ {+ let headers = headers_for(mode, Some(content_type)).await;+ assert_eq!(+ Some(INERT),+ header_text(&headers, header::CONTENT_TYPE),+ "{mode:?} {content_type:?}"+ );+ }+ }+ }++ #[tokio::test]+ async fn non_javascript_content_types_pass_through_every_mode() {+ for mode in [+ DeploymentMode::Mp,+ DeploymentMode::Static,+ DeploymentMode::Upload,+ DeploymentMode::Relay,+ ] {+ for content_type in [+ "text/css; charset=utf-8",+ "text/css",+ "video/mp2t",+ "video/mp4",+ "image/png",+ "image/svg+xml",+ "application/json",+ "text/plain",+ "text/javascriptx",+ "text/javascript1.6",+ ] {+ let headers = headers_for(mode, Some(content_type)).await;+ assert_eq!(+ Some(content_type),+ header_text(&headers, header::CONTENT_TYPE),+ "{mode:?} {content_type}"+ );+ }+ }+ }++ #[tokio::test]+ async fn mp_mode_types_an_untyped_full_or_partial_response_as_plain_text() {+ for status in [StatusCode::OK, StatusCode::PARTIAL_CONTENT] {+ for content_type in [None, Some(""), Some(" ")] {+ let headers = headers_with_status(DeploymentMode::Mp, status, content_type).await;+ assert_eq!(+ Some(INERT),+ header_text(&headers, header::CONTENT_TYPE),+ "{status} {content_type:?}"+ );+ assert_eq!(+ Some("nosniff"),+ header_text(&headers, header::X_CONTENT_TYPE_OPTIONS),+ "{status} {content_type:?}"+ );+ }+ }+ }++ #[tokio::test]+ async fn untyped_responses_of_other_statuses_or_modes_stay_untyped() {+ for status in [+ StatusCode::NO_CONTENT,+ StatusCode::NOT_MODIFIED,+ StatusCode::NOT_FOUND,+ StatusCode::METHOD_NOT_ALLOWED,+ StatusCode::RANGE_NOT_SATISFIABLE,+ ] {+ let headers = headers_with_status(DeploymentMode::Mp, status, None).await;+ assert_eq!(None, headers.get(header::CONTENT_TYPE), "{status}");+ }+ for status in [+ StatusCode::OK,+ StatusCode::PARTIAL_CONTENT,+ StatusCode::RANGE_NOT_SATISFIABLE,+ ] {+ let headers = headers_with_status(DeploymentMode::Static, status, None).await;+ assert_eq!(None, headers.get(header::CONTENT_TYPE), "{status}");+ }+ for mode in [DeploymentMode::Upload, DeploymentMode::Relay] {+ let headers = headers_with_status(mode, StatusCode::RANGE_NOT_SATISFIABLE, None).await;+ assert_eq!(None, headers.get(header::CONTENT_TYPE), "{mode:?}");+ for status in [StatusCode::OK, StatusCode::PARTIAL_CONTENT] {+ let headers = headers_with_status(mode, status, None).await;+ assert_eq!(+ Some(INERT),+ header_text(&headers, header::CONTENT_TYPE),+ "{mode:?} {status}"+ );+ }+ }+ }++ fn transport_stream_object(content_type: &str) -> FakeObject {+ let mut body = vec![0_u8; 376];+ body[0] = 0x47;+ body[188] = 0x47;+ FakeObject {+ body,+ head_length: Some(376),+ content_type: Some(content_type.to_owned()),+ ..FakeObject::default()+ }+ }++ fn security_layered_router(app: &Arc<AppState>, mode: DeploymentMode) -> Router {+ Router::new()+ .fallback(axum::routing::any(catch_all))+ .layer(axum::middleware::from_fn_with_state(+ mode,+ add_security_header_middleware,+ ))+ .with_state(Arc::clone(app))+ }++ async fn send(+ router: &Router,+ method: Method,+ uri: &str,+ range: Option<&'static str>,+ ) -> Response {+ let mut request = Request::builder().method(method).uri(uri);+ if let Some(range) = range {+ request = request.header(header::RANGE, range);+ }+ tower::ServiceExt::oneshot(router.clone(), request.body(Body::empty()).unwrap())+ .await+ .unwrap()+ }++ fn assert_attachment_named(headers: &HeaderMap, filename: &str) {+ let disposition =+ header_text(headers, header::CONTENT_DISPOSITION).expect("content disposition");+ assert!(+ disposition.starts_with("attachment") && disposition.contains(filename),+ "{disposition}"+ );+ }++ #[tokio::test]+ async fn a_stored_javascript_attachment_is_plain_text_through_the_security_layer() {+ let fake = fake_s3().await;+ let tmp = tempfile::tempdir().unwrap();+ let mut cfg = fake.config(tmp.path());+ cfg.mode = DeploymentMode::Mp;+ let app = Arc::new(AppState::for_tests(cfg));+ let bucket = app.cfg.storage.bucket_cdn.clone();+ let segment = transport_stream_object("text/javascript");+ fake.put_object(+ &format!("{bucket}/attachments/1/2/segment.js"),+ segment.clone(),+ );+ fake.put_object(+ &format!("{bucket}/attachments/1/2/seg.ts"),+ transport_stream_object("video/mp2t"),+ );+ fake.put_object(+ &format!("{bucket}/themes/abc123.css"),+ FakeObject {+ body: b"body{color:#fff}".to_vec(),+ content_type: Some("text/css".to_owned()),+ ..FakeObject::default()+ },+ );+ let mp = security_layered_router(&app, DeploymentMode::Mp);++ let response = send(&mp, Method::GET, "/attachments/1/2/segment.js", None).await;+ assert_eq!(StatusCode::OK, response.status());+ assert_eq!(+ Some(INERT),+ header_text(response.headers(), header::CONTENT_TYPE)+ );+ assert_eq!(+ Some("nosniff"),+ header_text(response.headers(), header::X_CONTENT_TYPE_OPTIONS)+ );+ assert_attachment_named(response.headers(), "segment.js");+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .unwrap();+ assert_eq!(segment.body.as_slice(), &body[..]);++ let head = send(&mp, Method::HEAD, "/attachments/1/2/segment.js", None).await;+ assert_eq!(StatusCode::OK, head.status());+ assert_eq!(+ Some(INERT),+ header_text(head.headers(), header::CONTENT_TYPE)+ );++ let ranged = send(+ &mp,+ Method::GET,+ "/attachments/1/2/segment.js",+ Some("bytes=0-0"),+ )+ .await;+ assert_eq!(StatusCode::PARTIAL_CONTENT, ranged.status());+ assert_eq!(+ Some(INERT),+ header_text(ranged.headers(), header::CONTENT_TYPE)+ );++ let unsatisfiable = send(+ &mp,+ Method::HEAD,+ "/attachments/1/2/segment.js",+ Some("bytes=999-"),+ )+ .await;+ assert_eq!(StatusCode::RANGE_NOT_SATISFIABLE, unsatisfiable.status());+ assert_eq!(None, unsatisfiable.headers().get(header::CONTENT_TYPE));+ assert_eq!(+ Some("nosniff"),+ header_text(unsatisfiable.headers(), header::X_CONTENT_TYPE_OPTIONS)+ );++ let download = send(+ &mp,+ Method::GET,+ "/attachments/1/2/segment.js?download=1",+ None,+ )+ .await;+ assert_eq!(StatusCode::OK, download.status());+ assert_eq!(+ Some(INERT),+ header_text(download.headers(), header::CONTENT_TYPE)+ );+ assert_attachment_named(download.headers(), "segment.js");++ let upload = security_layered_router(&app, DeploymentMode::Upload);+ let neutralized = send(&upload, Method::GET, "/attachments/1/2/segment.js", None).await;+ assert_eq!(StatusCode::OK, neutralized.status());+ assert_eq!(+ Some(INERT),+ header_text(neutralized.headers(), header::CONTENT_TYPE)+ );++ let transport_stream = send(&mp, Method::GET, "/attachments/1/2/seg.ts", None).await;+ assert_eq!(StatusCode::OK, transport_stream.status());+ assert_eq!(+ Some("video/mp2t"),+ header_text(transport_stream.headers(), header::CONTENT_TYPE)+ );++ let theme = send(&mp, Method::GET, "/themes/abc123.css", None).await;+ assert_eq!(StatusCode::OK, theme.status());+ assert_eq!(+ Some("text/css; charset=utf-8"),+ header_text(theme.headers(), header::CONTENT_TYPE)+ );+ }async fn robots_header_for(mode: DeploymentMode) -> Option<String> {let router = Router::new()diff --git a/fluxer_media_proxy/src/server/mod.rs b/fluxer_media_proxy/src/server/mod.rsindex 5bd5888fe..540584cc6 100644--- a/fluxer_media_proxy/src/server/mod.rs+++ b/fluxer_media_proxy/src/server/mod.rs@@ -1,6 +1,8 @@// SPDX-License-Identifier: AGPL-3.0-or-latermod asset_path;+mod attachment_signature;+mod cors;mod download_stream;mod external;mod format_policy;diff --git a/fluxer_media_proxy/src/server/relay/tests.rs b/fluxer_media_proxy/src/server/relay/tests.rsindex 0639550d2..1f53de5f3 100644--- a/fluxer_media_proxy/src/server/relay/tests.rs+++ b/fluxer_media_proxy/src/server/relay/tests.rs@@ -150,6 +150,7 @@ async fn relay_body_stream_waits_out_a_client_stall_inside_the_total_budget() {}fn upload_relay_test_config(+ mode: &str,storage_root: &std::path::Path,spool_dir: &std::path::Path,relay_secret: &[u8],@@ -159,7 +160,7 @@ fn upload_relay_test_config("FLUXER_MEDIA_PROXY_SECRET_KEY".to_owned(),"secret".to_owned(),),- ("FLUXER_MEDIA_PROXY_MODE".to_owned(), "upload".to_owned()),+ ("FLUXER_MEDIA_PROXY_MODE".to_owned(), mode.to_owned()),("FLUXER_MEDIA_PROXY_STORAGE_BACKEND".to_owned(),"local".to_owned(),@@ -222,7 +223,7 @@ async fn relay_put_accepts_unknown_content_length_body() {let spool_dir = tmp_root.join("spool");tokio::fs::create_dir_all(&spool_dir).await.unwrap();let relay_secret = [7u8; 32];- let cfg = upload_relay_test_config(&storage_root, &spool_dir, &relay_secret);+ let cfg = upload_relay_test_config("upload", &storage_root, &spool_dir, &relay_secret);let key = "guild/diagnostics.txt";let token = encode_token(&TokenPayload {@@ -316,7 +317,7 @@ async fn relay_put_rejects_a_capability_issued_for_another_key() {let spool_dir = tmp_root.join("spool");tokio::fs::create_dir_all(&spool_dir).await.unwrap();let relay_secret = [7u8; 32];- let cfg = upload_relay_test_config(&storage_root, &spool_dir, &relay_secret);+ let cfg = upload_relay_test_config("upload", &storage_root, &spool_dir, &relay_secret);let token = relay_test_token("guild/a.bin", &relay_secret);let requested_key = "guild/b.bin";@@ -352,7 +353,7 @@ async fn relay_put_answers_500_when_the_spool_write_fails() {let storage_root = tmp_root.join("storage");let spool_dir = tmp_root.join("missing-spool");let relay_secret = [7u8; 32];- let cfg = upload_relay_test_config(&storage_root, &spool_dir, &relay_secret);+ let cfg = upload_relay_test_config("upload", &storage_root, &spool_dir, &relay_secret);let key = "guild/unspoolable.bin";let token = relay_test_token(key, &relay_secret);let request = Request::builder()@@ -392,7 +393,7 @@ async fn relay_put_streams_known_length_body_without_spooling() {let spool_dir = tmp_root.join("spool");tokio::fs::create_dir_all(&spool_dir).await.unwrap();let relay_secret = [7u8; 32];- let cfg = upload_relay_test_config(&storage_root, &spool_dir, &relay_secret);+ let cfg = upload_relay_test_config("upload", &storage_root, &spool_dir, &relay_secret);let key = "guild/streamed.bin";let token = relay_test_token(key, &relay_secret);let body = Bytes::from_static(b"streamed straight through");@@ -426,7 +427,7 @@ async fn relay_put_rejects_streaming_body_longer_than_declared() {let spool_dir = tmp_root.join("spool");tokio::fs::create_dir_all(&spool_dir).await.unwrap();let relay_secret = [7u8; 32];- let cfg = upload_relay_test_config(&storage_root, &spool_dir, &relay_secret);+ let cfg = upload_relay_test_config("upload", &storage_root, &spool_dir, &relay_secret);let key = "guild/overrun.bin";let token = relay_test_token(key, &relay_secret);@@ -459,7 +460,7 @@ async fn relay_put_rejects_streaming_body_shorter_than_declared() {let spool_dir = tmp_root.join("spool");tokio::fs::create_dir_all(&spool_dir).await.unwrap();let relay_secret = [7u8; 32];- let cfg = upload_relay_test_config(&storage_root, &spool_dir, &relay_secret);+ let cfg = upload_relay_test_config("upload", &storage_root, &spool_dir, &relay_secret);let key = "guild/short.bin";let token = relay_test_token(key, &relay_secret);@@ -584,6 +585,84 @@ async fn relay_put_returns_ok_for_a_malformed_upstream_etag() {}}+#[tokio::test]+async fn relay_put_stores_the_object_in_upload_and_relay_modes() {+ for mode in ["upload", "relay"] {+ let tmp = tempfile::tempdir().unwrap();+ let tmp_root = tmp.path().canonicalize().unwrap();+ let storage_root = tmp_root.join("storage");+ let spool_dir = tmp_root.join("spool");+ tokio::fs::create_dir_all(&spool_dir).await.unwrap();+ let relay_secret = [7u8; 32];+ let cfg = upload_relay_test_config(mode, &storage_root, &spool_dir, &relay_secret);+ let key = "guild/relayed.bin";+ let token = relay_test_token(key, &relay_secret);+ let body = Bytes::from_static(b"relayed upload bytes");++ let response = relay_put(+ State(test_app_state(cfg)),+ Path(key.to_owned()),+ Query(HashMap::from([("t".to_owned(), token)])),+ content_length_headers(body.len() as u64),+ Request::builder()+ .method(Method::PUT)+ .body(Body::from(body.clone()))+ .unwrap(),+ )+ .await;++ assert_eq!(StatusCode::OK, response.status(), "{mode}");+ let stored = tokio::fs::read(storage_root.join("uploads").join(key))+ .await+ .unwrap();+ assert_eq!(body.as_ref(), stored.as_slice(), "{mode}");+ }+}++#[tokio::test]+async fn relay_put_returns_404_outside_upload_and_relay_modes() {+ for mode in ["mp", "static"] {+ let tmp = tempfile::tempdir().unwrap();+ let tmp_root = tmp.path().canonicalize().unwrap();+ let storage_root = tmp_root.join("storage");+ let spool_dir = tmp_root.join("spool");+ tokio::fs::create_dir_all(&spool_dir).await.unwrap();+ let relay_secret = [7u8; 32];+ let cfg = upload_relay_test_config(mode, &storage_root, &spool_dir, &relay_secret);+ let key = "guild/unserved.bin";+ let token = relay_test_token(key, &relay_secret);+ let body = Bytes::from_static(b"no relay in this mode");++ let response = relay_put(+ State(test_app_state(cfg)),+ Path(key.to_owned()),+ Query(HashMap::from([("t".to_owned(), token)])),+ content_length_headers(body.len() as u64),+ Request::builder()+ .method(Method::PUT)+ .body(Body::from(body))+ .unwrap(),+ )+ .await;++ assert_eq!(StatusCode::NOT_FOUND, response.status(), "{mode}");+ assert!(+ !response+ .headers()+ .contains_key(header::ACCESS_CONTROL_ALLOW_ORIGIN),+ "{mode}"+ );+ let response_body = to_bytes(response.into_body(), 64).await.unwrap();+ assert_eq!(b"Not Found", response_body.as_ref(), "{mode}");+ assert!(+ !tokio::fs::try_exists(storage_root.join("uploads").join(key))+ .await+ .unwrap_or(false),+ "{mode}"+ );+ }+}+#[test]fn relay_etag_degrades_an_unrepresentable_upstream_value_to_an_empty_header() {assert_eq!("", relay_etag("\"broken\u{7f}tag\"").to_str().unwrap());@@ -712,7 +791,7 @@ async fn relay_put_omits_the_etag_when_the_store_returns_none() {let spool_dir = tmp_root.join("spool");tokio::fs::create_dir_all(&spool_dir).await.unwrap();let relay_secret = [7u8; 32];- let cfg = upload_relay_test_config(&storage_root, &spool_dir, &relay_secret);+ let cfg = upload_relay_test_config("upload", &storage_root, &spool_dir, &relay_secret);let key = "guild/no-upstream-etag.bin";let token = relay_test_token(key, &relay_secret);let body = Bytes::from_static(b"local backend bytes");diff --git a/fluxer_media_proxy/src/server/routes/dispatch.rs b/fluxer_media_proxy/src/server/routes/dispatch.rsindex a5f7c2d1a..865332b99 100644--- a/fluxer_media_proxy/src/server/routes/dispatch.rs+++ b/fluxer_media_proxy/src/server/routes/dispatch.rs@@ -1,13 +1,16 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse crate::{- config::DeploymentMode,+ config::{DeploymentMode, PolicyMode},constants::AssetKind,+ request_log::RequestId,server::{asset_path::{StorageKeyDecodeError, decode_storage_key, parse_entrance_sound_path,parse_guild_member_asset_path, parse_simple_asset_path, parse_standard_asset_path,},+ attachment_signature,+ cors::{self, OriginCheck},external::serve_external,response::error::{text, text_with_source},state::AppState,@@ -17,11 +20,21 @@ use crate::{use axum::{body::Body,extract::{Query, State},- http::{Method, Request, StatusCode},+ http::{HeaderMap, Method, Request, StatusCode},response::Response,};+use fluxer_common::attachment_url_signature::{SIGNATURE_PARAMETER_NAMES, Verdict};use std::{collections::HashMap, sync::Arc};+struct PublicRead<'a> {+ method: Method,+ path: &'a str,+ query: Option<&'a str>,+ params: &'a HashMap<String, String>,+ headers: &'a HeaderMap,+ request_id: Option<&'a RequestId>,+}+pub(in crate::server) async fn catch_all(State(app): State<Arc<AppState>>,Query(params): Query<HashMap<String, String>>,@@ -31,6 +44,9 @@ pub(in crate::server) async fn catch_all(if method != Method::GET && method != Method::HEAD {return text(StatusCode::METHOD_NOT_ALLOWED, "Method Not Allowed");}+ if app.cfg.mode == DeploymentMode::Relay {+ return text(StatusCode::NOT_FOUND, "Not Found");+ }let path = request.uri().path().to_owned();if app.cfg.mode == DeploymentMode::Static {let key = match decode_storage_key(&path) {@@ -46,52 +62,109 @@ pub(in crate::server) async fn catch_all().await;}+ let query = request.uri().query().map(ToOwned::to_owned);+ let request_id = request.extensions().get::<RequestId>().cloned();+ let read = PublicRead {+ method: method.clone(),+ path: &path,+ query: query.as_deref(),+ params: ¶ms,+ headers: request.headers(),+ request_id: request_id.as_ref(),+ };+ if app.cfg.cors.mode == PolicyMode::Off {+ return serve_public_read(&app, read).await;+ }+ let check = cors::check_origin(&app.cfg.cors.allowed_origins, request.headers());+ if check == OriginCheck::Refused {+ if app.cfg.cors.mode == PolicyMode::Enforce {+ return cors::refused_response(request.headers());+ }+ cors::log_would_refuse(&method, &path, request.headers());+ }+ let mut response = serve_public_read(&app, read).await;+ if app.cfg.cors.mode == PolicyMode::Enforce {+ cors::narrow_response(&check, response.headers_mut());+ }+ response+}++async fn serve_public_read(app: &Arc<AppState>, read: PublicRead<'_>) -> Response {+ let PublicRead {+ method,+ path,+ query,+ params,+ headers,+ request_id,+ } = read;if let Some(rest) = path.strip_prefix("/external/") {- return serve_external(&app, method, rest, ¶ms, request.headers()).await;+ return serve_external(app, method, rest, params, headers).await;}if path.starts_with("/attachments/") {- let key = match decode_storage_key(&path) {+ let key = match decode_storage_key(path) {Ok(key) => key,Err(err) => return storage_key_decode_response(err),};- return stored::serve_attachment(&app, method, &key, ¶ms, request.headers()).await;+ let signature_mode = app.cfg.attachment_signature.mode;+ let mut checked = None;+ if signature_mode != PolicyMode::Off {+ let verification =+ attachment_signature::check(&app.cfg.attachment_signature, path, query);+ app.metrics+ .attachment_signature()+ .record(verification.verdict);+ if verification.verdict != Verdict::Valid {+ if signature_mode == PolicyMode::Enforce {+ return attachment_signature::denied_response(verification.verdict);+ }+ attachment_signature::log_would_deny(+ &app.signature_log,+ &method,+ verification.verdict,+ headers,+ request_id,+ );+ }+ checked = Some(verification);+ }+ let params = without_signature_parameters(params);+ let mut response = stored::serve_attachment(app, method, &key, ¶ms, headers).await;+ if let Some(verification) = checked {+ attachment_signature::limit_cache_lifetime(&mut response, verification);+ }+ return response;}if path.starts_with("/themes/") && path.ends_with(".css") {- let key = match decode_storage_key(&path) {+ let key = match decode_storage_key(path) {Ok(key) => key,Err(err) => return storage_key_decode_response(err),};return stored::serve_stored_with_override(- &app,+ app,method,&app.cfg.storage.bucket_cdn,&key,"text/css; charset=utf-8",- request.headers(),+ headers,).await;}- if let Some(key) = parse_entrance_sound_path(&path) {- return stored::serve_stored_raw(- &app,- method,- &app.cfg.storage.bucket_cdn,- &key,- request.headers(),- )- .await;+ if let Some(key) = parse_entrance_sound_path(path) {+ return stored::serve_stored_raw(app, method, &app.cfg.storage.bucket_cdn, &key, headers)+ .await;}- if let Some(asset) = parse_guild_member_asset_path(&path) {- return stored::serve_asset_image(&app, method, asset, ¶ms, request.headers()).await;+ if let Some(asset) = parse_guild_member_asset_path(path) {+ return stored::serve_asset_image(app, method, asset, params, headers).await;}- if let Some(asset) = parse_simple_asset_path(&path, AssetKind::Emoji) {- return stored::serve_asset_image(&app, method, asset, ¶ms, request.headers()).await;+ if let Some(asset) = parse_simple_asset_path(path, AssetKind::Emoji) {+ return stored::serve_asset_image(app, method, asset, params, headers).await;}- if let Some(asset) = parse_simple_asset_path(&path, AssetKind::Sticker) {- return stored::serve_asset_image(&app, method, asset, ¶ms, request.headers()).await;+ if let Some(asset) = parse_simple_asset_path(path, AssetKind::Sticker) {+ return stored::serve_asset_image(app, method, asset, params, headers).await;}- if let Some(asset) = parse_standard_asset_path(&path) {- return stored::serve_asset_image(&app, method, asset, ¶ms, request.headers()).await;+ if let Some(asset) = parse_standard_asset_path(path) {+ return stored::serve_asset_image(app, method, asset, params, headers).await;}text(StatusCode::NOT_FOUND, "Not Found")}@@ -105,18 +178,89 @@ fn storage_key_decode_response(err: StorageKeyDecodeError) -> Response {)}+fn without_signature_parameters(params: &HashMap<String, String>) -> HashMap<String, String> {+ params+ .iter()+ .filter(|(name, _)| !SIGNATURE_PARAMETER_NAMES.contains(&name.as_str()))+ .map(|(name, value)| (name.clone(), value.clone()))+ .collect()+}+#[cfg(test)]mod tests {use super::*;use crate::config::Config;- use axum::{body::to_bytes, http::header};+ use axum::{+ body::to_bytes,+ http::{HeaderValue, header},+ };+ use base64::{Engine as _, engine::general_purpose};use bytes::Bytes;- use std::path::Path;+ use fluxer_common::attachment_url_signature::{+ ATTACHMENT_URL_TTL_SECS, UrlKind, issue_window, sign, with_data_package_signature,+ with_signature,+ };+ use std::{+ path::Path,+ sync::Mutex,+ time::{SystemTime, UNIX_EPOCH},+ };+ use tracing_subscriber::fmt::MakeWriter;++ #[derive(Clone, Default)]+ struct CapturedLog(Arc<Mutex<Vec<u8>>>);++ impl CapturedLog {+ fn text(&self) -> String {+ String::from_utf8(self.0.lock().expect("captured log is not poisoned").clone())+ .expect("captured log is utf-8")+ }+ }++ impl std::io::Write for CapturedLog {+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {+ self.0+ .lock()+ .expect("captured log is not poisoned")+ .extend_from_slice(buf);+ Ok(buf.len())+ }++ fn flush(&mut self) -> std::io::Result<()> {+ Ok(())+ }+ }++ impl<'a> MakeWriter<'a> for CapturedLog {+ type Writer = Self;++ fn make_writer(&'a self) -> Self::Writer {+ self.clone()+ }+ }const STORED_BYTES: &[u8] = b"raw-stored-bytes";+ const THEME_BYTES: &[u8] = b"body{color:#fff}";+ const WEB: &str = "https://web.fluxer.app";+ const EVIL: &str = "https://evil.example";+ const OPAQUE_ORIGIN: &[u8] = &[0xC3, 0x28];+ const ORIGIN_VARY: &str = "Accept-Encoding, Origin";+ const ATTACHMENT_PATH: &str = "/attachments/1/2/file.bin";+ const THEME_PATH: &str = "/themes/dark.css";+ const ENTRANCE_SOUND_PATH: &str = "/entrance-sounds/42/abc123def456.wav";+ const ASSET_PATH: &str = "/avatars/123456789012345678/a1b2c3d4e5f6.png";+ const BAD_SIGNATURE_PATH: &str = "/external/badsig/https/example.test/a.png";+ const ENFORCE: &[(&str, &str)] = &[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "enforce"),+ ("FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS", WEB),+ ];+ const REPORT: &[(&str, &str)] = &[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "report"),+ ("FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS", WEB),+ ];- fn dispatch_config(mode: &str, storage_root: &Path) -> Config {- Config::load_from_iter([+ fn dispatch_config(mode: &str, storage_root: &Path, extra: &[(&str, &str)]) -> Config {+ let mut env = vec![("FLUXER_MEDIA_PROXY_SECRET_KEY".to_owned(),"secret".to_owned(),@@ -130,8 +274,25 @@ mod tests {"FLUXER_MEDIA_PROXY_STORAGE_ROOT".to_owned(),storage_root.display().to_string(),),- ])- .expect("dispatch test config")+ (+ "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64".to_owned(),+ general_purpose::STANDARD.encode([7u8; 32]),+ ),+ ];+ env.extend(+ extra+ .iter()+ .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())),+ );+ Config::load_from_iter(env).expect("dispatch test config")+ }++ fn dispatch_app(mode: &str, storage_root: &Path, extra: &[(&str, &str)]) -> Arc<AppState> {+ Arc::new(AppState::for_tests(dispatch_config(+ mode,+ storage_root,+ extra,+ )))}fn write_object(storage_root: &Path, bucket: &str, key: &str, body: &[u8]) {@@ -140,15 +301,43 @@ mod tests {std::fs::write(path, body).expect("write object");}- async fn dispatch(app: &Arc<AppState>, method: Method, path: &str) -> Response {- let request = Request::builder()- .method(method)- .uri(path)- .body(Body::empty())- .expect("dispatch request");+ fn write_stored_representations(storage_root: &Path) {+ write_object(+ storage_root,+ "cdn",+ "attachments/1/2/file.bin",+ STORED_BYTES,+ );+ write_object(storage_root, "cdn", "themes/dark.css", THEME_BYTES);+ }++ fn read_request(+ method: Method,+ path: &str,+ origins: &[&[u8]],+ range: Option<&str>,+ ) -> Request<Body> {+ let mut builder = Request::builder().method(method).uri(path);+ for origin in origins {+ builder = builder.header(+ header::ORIGIN,+ HeaderValue::from_bytes(origin).expect("origin header value"),+ );+ }+ if let Some(range) = range {+ builder = builder.header(header::RANGE, range);+ }+ builder.body(Body::empty()).expect("dispatch request")+ }++ async fn dispatch_request(app: &Arc<AppState>, request: Request<Body>) -> Response {catch_all(State(Arc::clone(app)), Query(HashMap::new()), request).await}+ async fn dispatch(app: &Arc<AppState>, method: Method, path: &str) -> Response {+ dispatch_request(app, read_request(method, path, &[], None)).await+ }+async fn dispatch_body(app: &Arc<AppState>, path: &str) -> Bytes {let response = dispatch(app, Method::GET, path).await;assert_eq!(StatusCode::OK, response.status(), "{path} must be served");@@ -157,6 +346,63 @@ mod tests {.expect("dispatch body")}+ async fn body_of(response: Response) -> Bytes {+ to_bytes(response.into_body(), usize::MAX)+ .await+ .expect("response body")+ }++ fn vary_values(response: &Response) -> Vec<String> {+ response+ .headers()+ .get_all(header::VARY)+ .iter()+ .map(|value| value.to_str().expect("vary is ascii").to_owned())+ .collect()+ }++ fn refused_origin_shapes() -> [(&'static str, Vec<&'static [u8]>); 4] {+ [+ ("foreign", vec![EVIL.as_bytes()]),+ ("null", vec![b"null".as_slice()]),+ ("duplicated", vec![WEB.as_bytes(), WEB.as_bytes()]),+ ("malformed", vec![OPAQUE_ORIGIN]),+ ]+ }++ fn stored_representation_cases() -> [(Method, &'static str, Option<&'static str>, StatusCode); 7]+ {+ [+ (Method::GET, ATTACHMENT_PATH, None, StatusCode::OK),+ (+ Method::GET,+ ATTACHMENT_PATH,+ Some("bytes=0-3"),+ StatusCode::PARTIAL_CONTENT,+ ),+ (+ Method::GET,+ ATTACHMENT_PATH,+ Some("bytes=100-200"),+ StatusCode::RANGE_NOT_SATISFIABLE,+ ),+ (Method::HEAD, ATTACHMENT_PATH, None, StatusCode::OK),+ (+ Method::HEAD,+ ATTACHMENT_PATH,+ Some("bytes=0-3"),+ StatusCode::PARTIAL_CONTENT,+ ),+ (Method::GET, THEME_PATH, None, StatusCode::OK),+ (+ Method::GET,+ THEME_PATH,+ Some("bytes=999-"),+ StatusCode::RANGE_NOT_SATISFIABLE,+ ),+ ]+ }+#[tokio::test]async fn static_mode_short_circuits_every_path_to_the_static_bucket() {let tmp = tempfile::tempdir().expect("storage root");@@ -170,7 +416,7 @@ mod tests {STORED_BYTES,);write_object(root, "cdn", "themes/only-in-cdn.css", STORED_BYTES);- let app = Arc::new(AppState::for_tests(dispatch_config("static", root)));+ let app = dispatch_app("static", root, &[]);assert_eq!(STORED_BYTES,@@ -204,11 +450,11 @@ mod tests {"web/NOTICE.md",];- fn static_mode_app(root: &Path) -> Arc<AppState> {+ fn static_mode_app(root: &Path, extra: &[(&str, &str)]) -> Arc<AppState> {for key in STATIC_KEYS {write_object(root, "static", key, STORED_BYTES);}- Arc::new(AppState::for_tests(dispatch_config("static", root)))+ dispatch_app("static", root, extra)}fn header_value(response: &Response, name: impl header::AsHeaderName) -> Option<String> {@@ -222,7 +468,7 @@ mod tests {async fn static_mode_caches_every_asset_forever() {let tmp = tempfile::tempdir().expect("storage root");let root = tmp.path().canonicalize().expect("canonical storage root");- let app = static_mode_app(root.as_path());+ let app = static_mode_app(root.as_path(), &[]);for key in STATIC_KEYS {let response = dispatch(&app, Method::GET, &format!("/{key}")).await;assert_eq!(StatusCode::OK, response.status(), "key={key}");@@ -238,7 +484,7 @@ mod tests {async fn static_mode_omits_expires_and_relies_on_cache_control() {let tmp = tempfile::tempdir().expect("storage root");let root = tmp.path().canonicalize().expect("canonical storage root");- let app = static_mode_app(root.as_path());+ let app = static_mode_app(root.as_path(), &[]);let response = dispatch(&app, Method::GET, "/avatars/0.png").await;assert_eq!(StatusCode::OK, response.status());assert_eq!(None, header_value(&response, header::EXPIRES));@@ -253,7 +499,7 @@ mod tests {let tmp = tempfile::tempdir().expect("storage root");let root = tmp.path().canonicalize().expect("canonical storage root");let root = root.as_path();- write_object(root, "cdn", "themes/dark.css", b"body{color:#fff}");+ write_object(root, "cdn", "themes/dark.css", THEME_BYTES);write_object(root,"cdn",@@ -261,7 +507,7 @@ mod tests {STORED_BYTES,);write_object(root, "static", "themes/dark.css", STORED_BYTES);- let app = Arc::new(AppState::for_tests(dispatch_config("mp", root)));+ let app = dispatch_app("mp", root, &[]);let response = dispatch(&app, Method::GET, "/themes/dark.css").await;assert_eq!(StatusCode::OK, response.status());@@ -275,7 +521,7 @@ mod tests {.expect("ascii content type"));assert_eq!(- b"body{color:#fff}".as_slice(),+ THEME_BYTES,to_bytes(response.into_body(), usize::MAX).await.expect("theme body")@@ -303,4 +549,1548 @@ mod tests {.status());}++ #[tokio::test]+ async fn relay_mode_answers_every_read_path_with_404() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_object(root, "cdn", "attachments/1/2/stream.js", STORED_BYTES);+ write_object(root, "cdn", "themes/dark.css", THEME_BYTES);+ write_object(+ root,+ "cdn",+ "entrance-sounds/42/abc123def456.wav",+ STORED_BYTES,+ );+ let app = dispatch_app("relay", root, &[]);+ for path in [+ "/attachments/1/2/stream.js",+ "/external/sig/https/example.com/a.png",+ THEME_PATH,+ ENTRANCE_SOUND_PATH,+ ASSET_PATH,+ "/emojis/1.png",+ "/stickers/1.webp",+ "/",+ "/nothing/here",+ ] {+ for method in [Method::GET, Method::HEAD] {+ let response = dispatch(&app, method.clone(), path).await;+ assert_eq!(StatusCode::NOT_FOUND, response.status(), "{method} {path}");+ assert_eq!(+ Some("text/plain; charset=utf-8".to_owned()),+ header_value(&response, header::CONTENT_TYPE),+ "{method} {path}"+ );+ assert_eq!(+ Some("no-store".to_owned()),+ header_value(&response, header::CACHE_CONTROL),+ "{method} {path}"+ );+ assert_eq!(+ None,+ header_value(&response, header::ACCESS_CONTROL_ALLOW_ORIGIN),+ "{method} {path}"+ );+ }+ }+ assert!(+ app.metrics+ .render()+ .contains("fluxer_media_proxy_storage_hits_total 0\n")+ );+ }++ #[tokio::test]+ async fn upload_mode_serves_what_relay_mode_refuses() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_object(root, "cdn", "attachments/1/2/stream.js", STORED_BYTES);+ let upload = dispatch_app("upload", root, &[]);+ let relay = dispatch_app("relay", root, &[]);++ assert_eq!(+ STORED_BYTES,+ dispatch_body(&upload, "/attachments/1/2/stream.js").await+ );+ assert_eq!(+ StatusCode::NOT_FOUND,+ dispatch(&relay, Method::GET, "/attachments/1/2/stream.js")+ .await+ .status()+ );+ }++ #[tokio::test]+ async fn relay_mode_rejects_a_bad_storage_key_as_404_not_400() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ let upload = dispatch_app("upload", root, &[]);+ let relay = dispatch_app("relay", root, &[]);++ assert_eq!(+ StatusCode::BAD_REQUEST,+ dispatch(&upload, Method::GET, "/attachments/%2e%2e/x")+ .await+ .status()+ );+ assert_eq!(+ StatusCode::NOT_FOUND,+ dispatch(&relay, Method::GET, "/attachments/%2e%2e/x")+ .await+ .status()+ );+ }++ #[tokio::test]+ async fn relay_mode_keeps_405_for_other_methods() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let app = dispatch_app("relay", root.as_path(), &[]);+ for path in ["/attachments/1/2/stream.js", "/nothing/here"] {+ for method in [+ Method::POST,+ Method::PUT,+ Method::DELETE,+ Method::PATCH,+ Method::OPTIONS,+ ] {+ assert_eq!(+ StatusCode::METHOD_NOT_ALLOWED,+ dispatch(&app, method.clone(), path).await.status(),+ "{method} {path}"+ );+ }+ }+ }++ #[tokio::test]+ async fn enforce_echoes_an_allowed_origin_on_every_stored_representation() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_stored_representations(root);+ let app = dispatch_app("mp", root, ENFORCE);+ for (method, path, range, status) in stored_representation_cases() {+ let label = format!("{method} {path} range={range:?}");+ let response =+ dispatch_request(&app, read_request(method, path, &[WEB.as_bytes()], range)).await;+ assert_eq!(status, response.status(), "{label}");+ assert_eq!(+ Some(WEB.to_owned()),+ header_value(&response, header::ACCESS_CONTROL_ALLOW_ORIGIN),+ "{label}"+ );+ assert_eq!(+ vec![ORIGIN_VARY.to_owned()],+ vary_values(&response),+ "{label}"+ );+ }+ }++ #[tokio::test]+ async fn enforce_serves_origin_less_reads_without_allow_origin() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_stored_representations(root);+ let off = dispatch_app("mp", root, &[]);+ let enforce = dispatch_app("mp", root, ENFORCE);+ for (method, path, range, status) in stored_representation_cases() {+ let label = format!("{method} {path} range={range:?}");+ let expected =+ dispatch_request(&off, read_request(method.clone(), path, &[], range)).await;+ let actual = dispatch_request(&enforce, read_request(method, path, &[], range)).await;+ assert_eq!(status, expected.status(), "{label}");+ assert_eq!(status, actual.status(), "{label}");+ assert_eq!(+ None,+ header_value(&actual, header::ACCESS_CONTROL_ALLOW_ORIGIN),+ "{label}"+ );+ assert_eq!(+ vec![ORIGIN_VARY.to_owned()],+ vary_values(&actual),+ "{label}"+ );+ assert_eq!(body_of(expected).await, body_of(actual).await, "{label}");+ }+ }++ #[tokio::test]+ async fn enforce_refuses_foreign_null_and_malformed_origins_before_any_read() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_stored_representations(root);+ write_object(+ root,+ "cdn",+ "entrance-sounds/42/abc123def456.wav",+ STORED_BYTES,+ );+ let asset = parse_standard_asset_path(ASSET_PATH).expect("asset path");+ write_object(+ root,+ "cdn",+ &asset.storage_key,+ &crate::test_fixtures::synthetic_png(64, 64),+ );+ let app = dispatch_app("mp", root, ENFORCE);+ for (shape, origins) in refused_origin_shapes() {+ for path in [+ ATTACHMENT_PATH,+ THEME_PATH,+ ENTRANCE_SOUND_PATH,+ ASSET_PATH,+ BAD_SIGNATURE_PATH,+ "/nothing/here",+ ] {+ for method in [Method::GET, Method::HEAD] {+ let label = format!("{shape} {method} {path}");+ let response =+ dispatch_request(&app, read_request(method, path, &origins, None)).await;+ assert_eq!(StatusCode::FORBIDDEN, response.status(), "{label}");+ assert_eq!(+ None,+ header_value(&response, header::ACCESS_CONTROL_ALLOW_ORIGIN),+ "{label}"+ );+ }+ }+ }+ assert!(+ app.metrics+ .render()+ .contains("fluxer_media_proxy_storage_hits_total 0\n")+ );+ }++ #[tokio::test]+ async fn enforce_leaves_read_errors_without_cors_but_varies_them_on_origin() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let app = dispatch_app("mp", root.as_path(), ENFORCE);+ let with_origin: &[&[u8]] = &[WEB.as_bytes()];+ let without_origin: &[&[u8]] = &[];+ for origins in [with_origin, without_origin] {+ for (path, status) in [+ ("/attachments/1/2/missing.bin", StatusCode::NOT_FOUND),+ ("/attachments/%2e%2e/x", StatusCode::BAD_REQUEST),+ ("/external/nosignature", StatusCode::BAD_REQUEST),+ (BAD_SIGNATURE_PATH, StatusCode::UNAUTHORIZED),+ ] {+ for method in [Method::GET, Method::HEAD] {+ let label = format!("{method} {path} origins={}", origins.len());+ let response =+ dispatch_request(&app, read_request(method, path, origins, None)).await;+ assert_eq!(status, response.status(), "{label}");+ assert_eq!(+ None,+ header_value(&response, header::ACCESS_CONTROL_ALLOW_ORIGIN),+ "{label}"+ );+ assert_eq!(+ vec![ORIGIN_VARY.to_owned()],+ vary_values(&response),+ "{label}"+ );+ }+ }+ }+ }++ #[tokio::test]+ async fn enforce_refuses_a_foreign_origin_on_a_transform_cache_hit() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let app = dispatch_app("mp", root.as_path(), ENFORCE);+ let asset = parse_standard_asset_path(ASSET_PATH).expect("asset path");+ app.store+ .write_object(+ &app.cfg.storage.bucket_cdn,+ &asset.storage_key,+ &crate::test_fixtures::synthetic_png(512, 512),+ "image/png",+ )+ .await+ .expect("write asset");+ for _ in 0..2 {+ let response = dispatch_request(+ &app,+ read_request(Method::GET, ASSET_PATH, &[WEB.as_bytes()], None),+ )+ .await;+ assert_eq!(StatusCode::OK, response.status());+ assert_eq!(+ Some(WEB.to_owned()),+ header_value(&response, header::ACCESS_CONTROL_ALLOW_ORIGIN)+ );+ assert_eq!(vec![ORIGIN_VARY.to_owned()], vary_values(&response));+ assert!(!body_of(response).await.is_empty());+ }+ assert!(+ app.metrics+ .render()+ .contains("fluxer_media_proxy_transform_cache_hits_total 1\n")+ );+ let response = dispatch_request(+ &app,+ read_request(Method::GET, ASSET_PATH, &[EVIL.as_bytes()], None),+ )+ .await;+ assert_eq!(StatusCode::FORBIDDEN, response.status());+ assert!(+ app.metrics+ .render()+ .contains("fluxer_media_proxy_transform_cache_hits_total 1\n"),+ "a refused origin never reaches the transform cache"+ );+ }++ #[tokio::test]+ async fn enforce_applies_to_upload_mode_reads() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_stored_representations(root);+ let app = dispatch_app("upload", root, ENFORCE);++ let allowed = dispatch_request(+ &app,+ read_request(Method::GET, ATTACHMENT_PATH, &[WEB.as_bytes()], None),+ )+ .await;+ assert_eq!(StatusCode::OK, allowed.status());+ assert_eq!(+ Some(WEB.to_owned()),+ header_value(&allowed, header::ACCESS_CONTROL_ALLOW_ORIGIN)+ );++ let foreign = dispatch_request(+ &app,+ read_request(Method::GET, ATTACHMENT_PATH, &[EVIL.as_bytes()], None),+ )+ .await;+ assert_eq!(StatusCode::FORBIDDEN, foreign.status());++ let origin_less = dispatch(&app, Method::GET, ATTACHMENT_PATH).await;+ assert_eq!(StatusCode::OK, origin_less.status());+ assert_eq!(+ None,+ header_value(&origin_less, header::ACCESS_CONTROL_ALLOW_ORIGIN)+ );+ assert_eq!(STORED_BYTES, body_of(origin_less).await);+ }++ #[tokio::test]+ async fn report_mode_changes_no_response_byte() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_stored_representations(root);+ let off = dispatch_app("mp", root, &[]);+ let report = dispatch_app("mp", root, REPORT);+ let origin_shapes: [&[&[u8]]; 4] = [+ &[EVIL.as_bytes()],+ &[b"null".as_slice()],+ &[WEB.as_bytes()],+ &[],+ ];+ for origins in origin_shapes {+ for (method, path, range, status) in [+ (Method::GET, ATTACHMENT_PATH, None, StatusCode::OK),+ (+ Method::GET,+ ATTACHMENT_PATH,+ Some("bytes=0-3"),+ StatusCode::PARTIAL_CONTENT,+ ),+ (+ Method::GET,+ ATTACHMENT_PATH,+ Some("bytes=100-200"),+ StatusCode::RANGE_NOT_SATISFIABLE,+ ),+ (Method::HEAD, ATTACHMENT_PATH, None, StatusCode::OK),+ (+ Method::GET,+ "/attachments/1/2/missing.bin",+ None,+ StatusCode::NOT_FOUND,+ ),+ ] {+ let label = format!("{method} {path} range={range:?} origins={origins:?}");+ let expected =+ dispatch_request(&off, read_request(method.clone(), path, origins, range))+ .await;+ let actual =+ dispatch_request(&report, read_request(method, path, origins, range)).await;+ assert_eq!(status, expected.status(), "{label}");+ assert_eq!(expected.status(), actual.status(), "{label}");+ assert_eq!(expected.headers(), actual.headers(), "{label}");+ assert_eq!(body_of(expected).await, body_of(actual).await, "{label}");+ }+ }+ }++ #[tokio::test]+ async fn report_mode_logs_one_would_deny_line_for_each_refused_origin_only() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_stored_representations(root);+ let report = dispatch_app("mp", root, REPORT);+ let cases: [(&[&[u8]], Option<&str>); 4] = [+ (&[EVIL.as_bytes()], Some("origin=https://evil.example")),+ (&[b"null".as_slice()], Some("origin=null")),+ (&[WEB.as_bytes()], None),+ (&[], None),+ ];+ for (origins, expected_origin) in cases {+ let label = format!("origins={origins:?}");+ let captured = CapturedLog::default();+ let subscriber = tracing_subscriber::fmt()+ .with_writer(captured.clone())+ .with_ansi(false)+ .with_max_level(tracing::Level::TRACE)+ .finish();+ let response = {+ let _guard = tracing::subscriber::set_default(subscriber);+ dispatch_request(+ &report,+ read_request(Method::GET, ATTACHMENT_PATH, origins, None),+ )+ .await+ };+ assert_eq!(StatusCode::OK, response.status(), "{label}");+ let log = captured.text();+ let would_deny: Vec<&str> = log+ .lines()+ .filter(|line| line.contains("reason=\"cors_origin_would_deny\""))+ .collect();+ match expected_origin {+ Some(origin) => {+ assert_eq!(1, would_deny.len(), "{label}\n{log}");+ assert!(would_deny[0].contains(" WARN "), "{label}\n{log}");+ assert!(would_deny[0].trim_end().ends_with(origin), "{label}\n{log}");+ }+ None => assert!(would_deny.is_empty(), "{label}\n{log}"),+ }+ }+ }++ #[tokio::test]+ async fn off_mode_never_inspects_origin() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_stored_representations(root);+ let app = dispatch_app("mp", root, &[]);+ for (shape, origins) in refused_origin_shapes() {+ let response = dispatch_request(+ &app,+ read_request(Method::GET, ATTACHMENT_PATH, &origins, None),+ )+ .await;+ assert_eq!(StatusCode::OK, response.status(), "{shape}");+ assert_eq!(+ Some("*".to_owned()),+ header_value(&response, header::ACCESS_CONTROL_ALLOW_ORIGIN),+ "{shape}"+ );+ assert_eq!(+ vec!["Accept-Encoding".to_owned()],+ vary_values(&response),+ "{shape}"+ );+ }+ }++ #[tokio::test]+ async fn static_mode_is_unchanged_with_every_cors_key_set() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let app = static_mode_app(root.as_path(), ENFORCE);+ let origin_shapes: [&[&[u8]]; 4] = [+ &[EVIL.as_bytes()],+ &[b"null".as_slice()],+ &[WEB.as_bytes()],+ &[],+ ];+ for origins in origin_shapes {+ let label = format!("origins={origins:?}");+ let response = dispatch_request(+ &app,+ read_request(Method::GET, "/avatars/0.png", origins, None),+ )+ .await;+ assert_eq!(StatusCode::OK, response.status(), "{label}");+ assert_eq!(+ Some("*".to_owned()),+ header_value(&response, header::ACCESS_CONTROL_ALLOW_ORIGIN),+ "{label}"+ );+ assert_eq!(+ vec!["Accept-Encoding".to_owned()],+ vary_values(&response),+ "{label}"+ );+ assert_eq!(+ Some(STATIC_CACHE_CONTROL.to_owned()),+ header_value(&response, header::CACHE_CONTROL),+ "{label}"+ );+ }+ }++ const SIGNATURE_SECRET: [u8; 32] = [11u8; 32];+ const OTHER_SIGNATURE_SECRET: [u8; 32] = [12u8; 32];+ const SIGNATURE_ENFORCE: &[(&str, &str)] = &[+ ("FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE", "enforce"),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64",+ "CwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCws=",+ ),+ ];+ const SIGNATURE_REPORT: &[(&str, &str)] = &[+ ("FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE", "report"),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64",+ "CwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCws=",+ ),+ ];+ const SIGNATURE_OFF_WITH_KEYS: &[(&str, &str)] = &[+ ("FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE", "off"),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64",+ "CwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCws=",+ ),+ ];+ const UNAVAILABLE: &[u8] = b"This content is no longer available.";+ const ATTACHMENT_KEY: &str = "attachments/1/2/file.bin";+ const IMAGE_PATH: &str = "/attachments/1/2/pic.png";+ const IMAGE_KEY: &str = "attachments/1/2/pic.png";++ fn now_secs() -> u64 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .expect("the clock is after the unix epoch")+ .as_secs()+ }++ fn params_from(query: Option<&str>) -> HashMap<String, String> {+ query+ .map(|query| {+ query+ .split('&')+ .filter(|field| !field.is_empty())+ .filter_map(|field| field.split_once('='))+ .map(|(name, value)| (name.to_owned(), value.to_owned()))+ .collect()+ })+ .unwrap_or_default()+ }++ fn with_extra(signed: String, extra: &str) -> String {+ if extra.is_empty() {+ return signed;+ }+ format!("{signed}&{extra}")+ }++ fn signed_uri(path: &str, key: &str, extra: &str) -> String {+ let now = now_secs();+ with_extra(+ with_signature(path, key, now, now, &SIGNATURE_SECRET),+ extra,+ )+ }++ fn data_package_uri(path: &str, key: &str, extra: &str) -> String {+ with_extra(+ with_data_package_signature(path, key, 1, 1, &SIGNATURE_SECRET),+ extra,+ )+ }++ fn foreign_secret_data_package_uri(path: &str, key: &str) -> String {+ with_data_package_signature(path, key, 1, 1, &OTHER_SIGNATURE_SECRET)+ }++ fn expired_uri(path: &str, key: &str) -> String {+ let expires = now_secs() - 1;+ let issued = expires - ATTACHMENT_URL_TTL_SECS;+ format!(+ "{path}?ex={expires:08x}&is={issued:08x}&hm={}",+ sign(key, expires, issued, UrlKind::Ordinary, &SIGNATURE_SECRET)+ )+ }++ fn foreign_key_uri(path: &str) -> String {+ let now = now_secs();+ let (issued, expires) = issue_window(now, now);+ format!(+ "{path}?ex={expires:08x}&is={issued:08x}&hm={}",+ sign(+ "attachments/9/9/other.bin",+ expires,+ issued,+ UrlKind::Ordinary,+ &SIGNATURE_SECRET+ )+ )+ }++ fn foreign_secret_uri(path: &str, key: &str) -> String {+ let now = now_secs();+ let (issued, expires) = issue_window(now, now);+ format!(+ "{path}?ex={expires:08x}&is={issued:08x}&hm={}",+ sign(+ key,+ expires,+ issued,+ UrlKind::Ordinary,+ &OTHER_SIGNATURE_SECRET+ )+ )+ }++ fn malformed_uri(path: &str, key: &str) -> String {+ let now = now_secs();+ let (issued, expires) = issue_window(now, now);+ format!(+ "{path}?ex={expires:08x}&is={issued:08x}&hm={0}&hm={0}",+ sign(key, expires, issued, UrlKind::Ordinary, &SIGNATURE_SECRET)+ )+ }++ async fn dispatch_uri(+ app: &Arc<AppState>,+ method: Method,+ uri: &str,+ origins: &[&[u8]],+ range: Option<&str>,+ ) -> Response {+ let request = read_request(method, uri, origins, range);+ let params = params_from(request.uri().query());+ catch_all(State(Arc::clone(app)), Query(params), request).await+ }++ fn counter(app: &Arc<AppState>, name: &str) -> u64 {+ app.metrics+ .render()+ .lines()+ .find_map(|line| line.strip_prefix(name)?.trim().parse().ok())+ .expect("the counter is rendered")+ }++ fn write_signed_representations(root: &Path) {+ write_stored_representations(root);+ write_object(+ root,+ "cdn",+ IMAGE_KEY,+ &crate::test_fixtures::synthetic_png(128, 128),+ );+ }++ struct SignedReadCase {+ method: Method,+ path: &'static str,+ key: &'static str,+ range: Option<&'static str>,+ extra: &'static str,+ status: StatusCode,+ }++ fn signed_read_case(+ method: Method,+ path: &'static str,+ key: &'static str,+ range: Option<&'static str>,+ extra: &'static str,+ status: StatusCode,+ ) -> SignedReadCase {+ SignedReadCase {+ method,+ path,+ key,+ range,+ extra,+ status,+ }+ }++ fn signed_read_cases() -> [SignedReadCase; 7] {+ [+ signed_read_case(+ Method::GET,+ ATTACHMENT_PATH,+ ATTACHMENT_KEY,+ None,+ "",+ StatusCode::OK,+ ),+ signed_read_case(+ Method::GET,+ ATTACHMENT_PATH,+ ATTACHMENT_KEY,+ Some("bytes=0-3"),+ "",+ StatusCode::PARTIAL_CONTENT,+ ),+ signed_read_case(+ Method::GET,+ ATTACHMENT_PATH,+ ATTACHMENT_KEY,+ Some("bytes=100-200"),+ "",+ StatusCode::RANGE_NOT_SATISFIABLE,+ ),+ signed_read_case(+ Method::HEAD,+ ATTACHMENT_PATH,+ ATTACHMENT_KEY,+ None,+ "",+ StatusCode::OK,+ ),+ signed_read_case(+ Method::HEAD,+ ATTACHMENT_PATH,+ ATTACHMENT_KEY,+ Some("bytes=0-3"),+ "",+ StatusCode::PARTIAL_CONTENT,+ ),+ signed_read_case(+ Method::GET,+ ATTACHMENT_PATH,+ ATTACHMENT_KEY,+ None,+ "download=1",+ StatusCode::OK,+ ),+ signed_read_case(+ Method::GET,+ IMAGE_PATH,+ IMAGE_KEY,+ None,+ "width=64",+ StatusCode::OK,+ ),+ ]+ }++ fn without_cache_policy(headers: &HeaderMap) -> HeaderMap {+ let mut headers = headers.clone();+ headers.remove(header::CACHE_CONTROL);+ headers.remove("cdn-cache-control");+ headers+ }++ #[tokio::test]+ async fn enforce_serves_a_signed_attachment_on_every_read_shape() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let off = dispatch_app("mp", root, &[]);+ let enforce = dispatch_app("mp", root, SIGNATURE_ENFORCE);++ for case in signed_read_cases() {+ let SignedReadCase {+ method,+ path,+ key,+ range,+ extra,+ status,+ } = case;+ let unsigned = if extra.is_empty() {+ path.to_owned()+ } else {+ format!("{path}?{extra}")+ };+ for signed in [+ signed_uri(path, key, extra),+ data_package_uri(path, key, extra),+ ] {+ let label = format!("{method} {signed} range={range:?}");+ let expected = dispatch_uri(&off, method.clone(), &unsigned, &[], range).await;+ let actual = dispatch_uri(&enforce, method.clone(), &signed, &[], range).await;+ assert_eq!(status, expected.status(), "{label}");+ assert_eq!(expected.status(), actual.status(), "{label}");+ assert_eq!(+ without_cache_policy(expected.headers()),+ without_cache_policy(actual.headers()),+ "{label}"+ );+ match (+ header_value(&expected, header::CACHE_CONTROL),+ header_value(&actual, header::CACHE_CONTROL),+ ) {+ (None, None) => {}+ (Some(unsigned_policy), Some(signed_policy)) => {+ assert!(unsigned_policy.contains("max-age=31536000"), "{label}");+ assert!(signed_policy.starts_with("public, max-age="), "{label}");+ assert!(!signed_policy.contains("max-age=31536000"), "{label}");+ assert!(+ !header_value(&actual, "cdn-cache-control")+ .expect("a signed read pairs the cdn cache policy")+ .contains("max-age=31536000"),+ "{label}"+ );+ }+ mismatch => panic!("{label} {mismatch:?}"),+ }+ assert_eq!(body_of(expected).await, body_of(actual).await, "{label}");+ }+ }+ }++ #[tokio::test]+ async fn enforce_answers_every_refusal_with_the_same_bytes() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let app = dispatch_app("mp", root, SIGNATURE_ENFORCE);+ let missing_object = signed_uri(+ "/attachments/1/2/absent.bin",+ "attachments/1/2/absent.bin",+ "",+ );+ let package = data_package_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, "");+ let refusals: [(&str, String); 13] = [+ (+ "data package from another secret",+ foreign_secret_data_package_uri(ATTACHMENT_PATH, ATTACHMENT_KEY),+ ),+ (+ "data package with uc removed",+ package.replace("&uc=dp", ""),+ ),+ (+ "data package with a tampered uc",+ package.replace("uc=dp", "uc=dq"),+ ),+ (+ "data package with a padded expiry",+ package.replace("ex=0&", "ex=00000000&"),+ ),+ (+ "data package signature for another key",+ data_package_uri(+ "/attachments/9/9/other.bin",+ "attachments/9/9/other.bin",+ "",+ )+ .replace("/attachments/9/9/other.bin", ATTACHMENT_PATH),+ ),+ ("no signature", ATTACHMENT_PATH.to_owned()),+ (+ "other parameters only",+ format!("{ATTACHMENT_PATH}?width=64"),+ ),+ (+ "partial set",+ format!("{ATTACHMENT_PATH}?ex=6a9a7231&is=6a9920b1"),+ ),+ (+ "repeated parameter",+ malformed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY),+ ),+ ("expired", expired_uri(ATTACHMENT_PATH, ATTACHMENT_KEY)),+ (+ "signature for another key",+ foreign_key_uri(ATTACHMENT_PATH),+ ),+ (+ "signature from another secret",+ foreign_secret_uri(ATTACHMENT_PATH, ATTACHMENT_KEY),+ ),+ ("valid signature over a missing object", missing_object),+ ];+ let mut seen: Vec<(StatusCode, HeaderMap, Bytes)> = Vec::new();+ for (label, uri) in refusals {+ for method in [Method::GET, Method::HEAD] {+ for range in [None, Some("bytes=0-3"), Some("bytes=100-200")] {+ for origins in [&[][..], &[WEB.as_bytes()][..]] {+ let response =+ dispatch_uri(&app, method.clone(), &uri, origins, range).await;+ let status = response.status();+ let headers = response.headers().clone();+ let body = body_of(response).await;+ assert_eq!(StatusCode::NOT_FOUND, status, "{label} {method}");+ assert_eq!(UNAVAILABLE, body.as_ref(), "{label} {method}");+ assert_eq!(36, body.len(), "{label} {method}");+ assert_eq!(+ Some("no-store"),+ headers+ .get(header::CACHE_CONTROL)+ .map(|value| value.to_str().expect("ascii")),+ "{label} {method}"+ );+ assert!(+ !headers.contains_key(header::ACCESS_CONTROL_ALLOW_ORIGIN),+ "{label} {method}"+ );+ seen.push((status, headers, body));+ }+ }+ }+ }+ let first = seen.first().expect("at least one refusal").clone();+ for (status, headers, body) in seen {+ assert_eq!(first.0, status);+ assert_eq!(first.1, headers);+ assert_eq!(first.2, body);+ }+ }++ #[tokio::test]+ async fn enforce_keeps_a_bad_storage_key_a_bad_request() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let app = dispatch_app("mp", root.as_path(), SIGNATURE_ENFORCE);+ let response = dispatch_uri(&app, Method::GET, "/attachments/%2e%2e/x", &[], None).await;+ assert_eq!(StatusCode::BAD_REQUEST, response.status());+ assert_eq!(b"Bad Request".as_slice(), body_of(response).await);+ }++ #[tokio::test]+ async fn enforce_refuses_before_any_object_read() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let app = dispatch_app("mp", root, SIGNATURE_ENFORCE);+ for uri in [+ ATTACHMENT_PATH.to_owned(),+ expired_uri(ATTACHMENT_PATH, ATTACHMENT_KEY),+ foreign_key_uri(ATTACHMENT_PATH),+ format!("{IMAGE_PATH}?width=64"),+ ] {+ let response = dispatch_uri(&app, Method::GET, &uri, &[], None).await;+ assert_eq!(StatusCode::NOT_FOUND, response.status(), "{uri}");+ }+ assert_eq!(0, counter(&app, "fluxer_media_proxy_storage_hits_total"));+ assert_eq!(+ 0,+ counter(&app, "fluxer_media_proxy_transform_cache_misses_total")+ );+ }++ #[tokio::test]+ async fn a_refusal_never_reaches_a_warmed_transform_cache() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let app = dispatch_app("mp", root, SIGNATURE_ENFORCE);+ for _ in 0..2 {+ let uri = signed_uri(IMAGE_PATH, IMAGE_KEY, "width=64");+ let response = dispatch_uri(&app, Method::GET, &uri, &[], None).await;+ assert_eq!(StatusCode::OK, response.status());+ assert!(!body_of(response).await.is_empty());+ }+ assert_eq!(+ 1,+ counter(&app, "fluxer_media_proxy_transform_cache_hits_total")+ );+ let hits = counter(&app, "fluxer_media_proxy_storage_hits_total");+ assert!(hits > 0);++ let expired = format!("{}&width=64", expired_uri(IMAGE_PATH, IMAGE_KEY));+ let response = dispatch_uri(&app, Method::GET, &expired, &[], None).await;+ assert_eq!(StatusCode::NOT_FOUND, response.status());+ assert_eq!(UNAVAILABLE, body_of(response).await.as_ref());+ assert_eq!(+ 1,+ counter(&app, "fluxer_media_proxy_transform_cache_hits_total"),+ "an expired url never reaches the transform cache"+ );+ assert_eq!(+ hits,+ counter(&app, "fluxer_media_proxy_storage_hits_total"),+ "an expired url never reaches storage"+ );+ }++ #[tokio::test]+ async fn enforce_accepts_every_spelling_of_the_signed_key() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_object(root, "cdn", "attachments/1/2/café.gif", STORED_BYTES);+ write_object(root, "cdn", "attachments/1/2/a/b.gif", STORED_BYTES);+ let app = dispatch_app("mp", root, SIGNATURE_ENFORCE);+ for (path, key) in [+ ("/attachments/1/2/caf%C3%A9.gif", "attachments/1/2/café.gif"),+ ("/attachments/1/2/café.gif", "attachments/1/2/café.gif"),+ ("/attachments/1/2/a%2Fb.gif", "attachments/1/2/a/b.gif"),+ ("/attachments/1/2/a/b.gif", "attachments/1/2/a/b.gif"),+ ] {+ let now = now_secs();+ let (issued, expires) = issue_window(now, now);+ let uri = format!(+ "{path}?ex={expires:08x}&is={issued:08x}&hm={}",+ sign(key, expires, issued, UrlKind::Ordinary, &SIGNATURE_SECRET)+ );+ for spelling in [uri.clone(), format!("{uri}&")] {+ let response = dispatch_uri(&app, Method::GET, &spelling, &[], None).await;+ assert_eq!(StatusCode::OK, response.status(), "{spelling}");+ assert_eq!(STORED_BYTES, body_of(response).await, "{spelling}");+ }+ }+ }++ #[tokio::test]+ async fn enforce_leaves_every_other_read_route_unsigned() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ write_object(+ root,+ "cdn",+ "entrance-sounds/42/abc123def456.wav",+ STORED_BYTES,+ );+ let asset = parse_standard_asset_path(ASSET_PATH).expect("asset path");+ write_object(+ root,+ "cdn",+ &asset.storage_key,+ &crate::test_fixtures::synthetic_png(64, 64),+ );+ let off = dispatch_app("mp", root, &[]);+ let enforce = dispatch_app("mp", root, SIGNATURE_ENFORCE);+ for path in [THEME_PATH, ENTRANCE_SOUND_PATH, ASSET_PATH, "/nothing/here"] {+ for method in [Method::GET, Method::HEAD] {+ let label = format!("{method} {path}");+ let expected = dispatch_uri(&off, method.clone(), path, &[], None).await;+ let actual = dispatch_uri(&enforce, method, path, &[], None).await;+ assert_eq!(expected.status(), actual.status(), "{label}");+ assert_eq!(expected.headers(), actual.headers(), "{label}");+ assert_eq!(body_of(expected).await, body_of(actual).await, "{label}");+ }+ }+ }++ #[tokio::test]+ async fn static_and_relay_modes_are_unchanged_with_the_signature_keys_set() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_object(root, "static", "avatars/0.png", STORED_BYTES);+ write_object(root, "cdn", ATTACHMENT_KEY, STORED_BYTES);++ let plain_static = dispatch_app("static", root, &[]);+ let signed_static = dispatch_app("static", root, SIGNATURE_ENFORCE);+ let expected = dispatch_uri(&plain_static, Method::GET, "/avatars/0.png", &[], None).await;+ let actual = dispatch_uri(&signed_static, Method::GET, "/avatars/0.png", &[], None).await;+ assert_eq!(StatusCode::OK, expected.status());+ assert_eq!(expected.status(), actual.status());+ assert_eq!(expected.headers(), actual.headers());+ assert_eq!(body_of(expected).await, body_of(actual).await);++ let relay = dispatch_app("relay", root, SIGNATURE_ENFORCE);+ let response = dispatch_uri(&relay, Method::GET, ATTACHMENT_PATH, &[], None).await;+ assert_eq!(StatusCode::NOT_FOUND, response.status());+ assert_eq!(b"Not Found".as_slice(), body_of(response).await);++ let upload = dispatch_app("upload", root, SIGNATURE_ENFORCE);+ let refused = dispatch_uri(&upload, Method::GET, ATTACHMENT_PATH, &[], None).await;+ assert_eq!(StatusCode::NOT_FOUND, refused.status());+ assert_eq!(UNAVAILABLE, body_of(refused).await.as_ref());+ let allowed = dispatch_uri(+ &upload,+ Method::GET,+ &signed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, ""),+ &[],+ None,+ )+ .await;+ assert_eq!(StatusCode::OK, allowed.status());+ assert_eq!(STORED_BYTES, body_of(allowed).await);+ }++ #[tokio::test]+ async fn a_cors_refusal_precedes_the_signature_check() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let mut env: Vec<(&str, &str)> = ENFORCE.to_vec();+ env.extend_from_slice(SIGNATURE_ENFORCE);+ let app = dispatch_app("mp", root, &env);+ for uri in [+ ATTACHMENT_PATH.to_owned(),+ signed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, ""),+ expired_uri(ATTACHMENT_PATH, ATTACHMENT_KEY),+ ] {+ let response = dispatch_uri(&app, Method::GET, &uri, &[EVIL.as_bytes()], None).await;+ assert_eq!(StatusCode::FORBIDDEN, response.status(), "{uri}");+ assert_eq!(b"Forbidden".as_slice(), body_of(response).await, "{uri}");+ }+ assert_eq!(0, counter(&app, "fluxer_media_proxy_storage_hits_total"));+ }++ #[tokio::test]+ async fn report_mode_changes_no_response_byte_for_a_stored_object() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let off = dispatch_app("mp", root, &[]);+ let report = dispatch_app("mp", root, SIGNATURE_REPORT);+ for uri in [+ ATTACHMENT_PATH.to_owned(),+ format!("{ATTACHMENT_PATH}?download=1"),+ expired_uri(ATTACHMENT_PATH, ATTACHMENT_KEY),+ foreign_key_uri(ATTACHMENT_PATH),+ format!("{IMAGE_PATH}?width=64"),+ ] {+ for method in [Method::GET, Method::HEAD] {+ for range in [None, Some("bytes=0-3"), Some("bytes=100-200")] {+ let label = format!("{method} {uri} range={range:?}");+ let expected = dispatch_uri(&off, method.clone(), &uri, &[], range).await;+ let actual = dispatch_uri(&report, method.clone(), &uri, &[], range).await;+ assert_eq!(expected.status(), actual.status(), "{label}");+ assert_eq!(expected.headers(), actual.headers(), "{label}");+ assert_eq!(body_of(expected).await, body_of(actual).await, "{label}");+ }+ }+ }+ }++ #[tokio::test]+ async fn report_mode_changes_only_the_cache_lifetime_of_a_valid_signature() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let off = dispatch_app("mp", root, &[]);+ let report = dispatch_app("mp", root, SIGNATURE_REPORT);+ for uri in [+ signed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, ""),+ data_package_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, ""),+ signed_uri(IMAGE_PATH, IMAGE_KEY, "width=64"),+ ] {+ for method in [Method::GET, Method::HEAD] {+ for range in [None, Some("bytes=0-3")] {+ let label = format!("{method} {uri} range={range:?}");+ let expected = dispatch_uri(&off, method.clone(), &uri, &[], range).await;+ let actual = dispatch_uri(&report, method.clone(), &uri, &[], range).await;+ assert_eq!(expected.status(), actual.status(), "{label}");+ assert_eq!(+ without_cache_policy(expected.headers()),+ without_cache_policy(actual.headers()),+ "{label}"+ );+ assert_eq!(+ Some(STATIC_CACHE_CONTROL.to_owned()),+ header_value(&expected, header::CACHE_CONTROL),+ "{label}"+ );+ let capped = header_value(&actual, header::CACHE_CONTROL)+ .expect("a report read of a valid signature states its own max age");+ let max_age: u64 = capped+ .strip_prefix("public, max-age=")+ .expect("the capped policy is a plain max age")+ .parse()+ .expect("a max age is a number");+ assert!(max_age <= ATTACHMENT_URL_TTL_SECS, "{label} {capped}");+ assert_eq!(+ Some(capped),+ header_value(&actual, "cdn-cache-control"),+ "{label}"+ );+ assert_eq!(body_of(expected).await, body_of(actual).await, "{label}");+ }+ }+ }+ }++ #[tokio::test]+ async fn report_leaves_a_refusable_read_caching_forever() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let report = dispatch_app("mp", root, SIGNATURE_REPORT);+ for uri in [+ ATTACHMENT_PATH.to_owned(),+ expired_uri(ATTACHMENT_PATH, ATTACHMENT_KEY),+ foreign_key_uri(ATTACHMENT_PATH),+ ] {+ let response = dispatch_uri(&report, Method::GET, &uri, &[], None).await;+ assert_eq!(StatusCode::OK, response.status(), "{uri}");+ assert_eq!(+ Some(STATIC_CACHE_CONTROL.to_owned()),+ header_value(&response, header::CACHE_CONTROL),+ "{uri}"+ );+ }+ }++ #[tokio::test]+ async fn only_enforce_masks_a_missing_object_so_it_matches_a_refusal() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let off = dispatch_app("mp", root, &[]);+ let report = dispatch_app("mp", root, SIGNATURE_REPORT);+ let enforce = dispatch_app("mp", root, SIGNATURE_ENFORCE);+ let uri = "/attachments/1/2/absent.bin";+ let plain = dispatch_uri(&off, Method::GET, uri, &[], None).await;+ assert_eq!(StatusCode::NOT_FOUND, plain.status());+ assert_eq!(b"Not Found".as_slice(), body_of(plain).await);++ let dry_run = dispatch_uri(&report, Method::GET, uri, &[], None).await;+ assert_eq!(StatusCode::NOT_FOUND, dry_run.status());+ assert_eq!(b"Not Found".as_slice(), body_of(dry_run).await);++ let signed = signed_uri(uri, "attachments/1/2/absent.bin", "");+ let masked = dispatch_uri(&enforce, Method::GET, &signed, &[], None).await;+ assert_eq!(StatusCode::NOT_FOUND, masked.status());+ assert_eq!(UNAVAILABLE, body_of(masked).await.as_ref());+ }++ #[tokio::test]+ async fn an_enforced_attachment_is_never_cached_past_its_signature() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let app = dispatch_app("mp", root, SIGNATURE_ENFORCE);++ let response = dispatch_uri(+ &app,+ Method::GET,+ &signed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, ""),+ &[],+ None,+ )+ .await;+ assert_eq!(StatusCode::OK, response.status());+ let policy = header_value(&response, header::CACHE_CONTROL).expect("cache control");+ let max_age: u64 = policy+ .strip_prefix("public, max-age=")+ .expect("a signed attachment states its own max age")+ .parse()+ .expect("a max age is a number");+ assert!(max_age <= ATTACHMENT_URL_TTL_SECS, "{policy}");+ assert!(max_age + 60 >= ATTACHMENT_URL_TTL_SECS, "{policy}");+ assert_eq!(Some(policy), header_value(&response, "cdn-cache-control"));++ let package = dispatch_uri(+ &app,+ Method::GET,+ &data_package_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, ""),+ &[],+ None,+ )+ .await;+ assert_eq!(StatusCode::OK, package.status());+ assert_eq!(+ Some(format!("public, max-age={ATTACHMENT_URL_TTL_SECS}")),+ header_value(&package, header::CACHE_CONTROL)+ );+ }++ #[tokio::test]+ async fn an_unenforced_read_and_every_other_route_cache_forever() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ for env in [&[][..], SIGNATURE_OFF_WITH_KEYS] {+ let app = dispatch_app("mp", root, env);+ let response = dispatch_uri(+ &app,+ Method::GET,+ &signed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, ""),+ &[],+ None,+ )+ .await;+ assert_eq!(StatusCode::OK, response.status());+ assert_eq!(+ Some(STATIC_CACHE_CONTROL.to_owned()),+ header_value(&response, header::CACHE_CONTROL)+ );+ }+ let enforcing = dispatch_app("mp", root, SIGNATURE_ENFORCE);+ let theme = dispatch_uri(&enforcing, Method::GET, THEME_PATH, &[], None).await;+ assert_eq!(StatusCode::OK, theme.status());+ assert_eq!(+ Some(STATIC_CACHE_CONTROL.to_owned()),+ header_value(&theme, header::CACHE_CONTROL)+ );+ }++ #[tokio::test]+ async fn report_mode_logs_one_line_for_each_refused_verdict_only() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let report = dispatch_app("mp", root, SIGNATURE_REPORT);+ let cases: [(&str, String); 8] = [+ ("missing", ATTACHMENT_PATH.to_owned()),+ ("malformed", malformed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY)),+ ("mismatch", foreign_key_uri(ATTACHMENT_PATH)),+ ("expired", expired_uri(ATTACHMENT_PATH, ATTACHMENT_KEY)),+ ("", signed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, "")),+ ("", data_package_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, "")),+ (+ "malformed",+ data_package_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, "").replace("&uc=dp", ""),+ ),+ (+ "mismatch",+ foreign_secret_data_package_uri(ATTACHMENT_PATH, ATTACHMENT_KEY),+ ),+ ];+ for (verdict, uri) in cases {+ let captured = CapturedLog::default();+ let subscriber = tracing_subscriber::fmt()+ .with_writer(captured.clone())+ .with_ansi(false)+ .with_max_level(tracing::Level::TRACE)+ .finish();+ let response = {+ let _guard = tracing::subscriber::set_default(subscriber);+ dispatch_uri(&report, Method::GET, &uri, &[], None).await+ };+ assert_eq!(StatusCode::OK, response.status(), "{verdict} {uri}");+ let log = captured.text();+ let lines: Vec<&str> = log+ .lines()+ .filter(|line| line.contains("reason=\"attachment_signature_would_deny\""))+ .collect();+ if verdict.is_empty() {+ assert!(lines.is_empty(), "{uri}\n{log}");+ continue;+ }+ assert_eq!(1, lines.len(), "{verdict} {uri}\n{log}");+ assert!(lines[0].contains(" WARN "), "{verdict}\n{log}");+ assert!(+ lines[0].contains(&format!("verdict=\"{verdict}\"")),+ "{verdict}\n{log}"+ );+ }+ }++ #[tokio::test]+ async fn off_mode_serves_a_signature_carrying_request_unchanged() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let plain = dispatch_app("mp", root, &[]);+ let staged = dispatch_app("mp", root, SIGNATURE_OFF_WITH_KEYS);+ for uri in [+ ATTACHMENT_PATH.to_owned(),+ expired_uri(ATTACHMENT_PATH, ATTACHMENT_KEY),+ foreign_key_uri(ATTACHMENT_PATH),+ format!("{ATTACHMENT_PATH}?ex=zzz&is=&hm="),+ "/attachments/1/2/absent.bin".to_owned(),+ ] {+ let expected = dispatch_uri(&plain, Method::GET, &uri, &[], None).await;+ let actual = dispatch_uri(&staged, Method::GET, &uri, &[], None).await;+ assert_eq!(expected.status(), actual.status(), "{uri}");+ assert_eq!(expected.headers(), actual.headers(), "{uri}");+ assert_eq!(body_of(expected).await, body_of(actual).await, "{uri}");+ }+ }++ #[tokio::test]+ async fn off_mode_serves_a_signed_read_byte_identical_to_an_unsigned_read() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ for env in [&[][..], SIGNATURE_OFF_WITH_KEYS] {+ let unsigned_app = dispatch_app("mp", root, &[]);+ let signed_app = dispatch_app("mp", root, env);+ for case in signed_read_cases() {+ let SignedReadCase {+ method,+ path,+ key,+ range,+ extra,+ status,+ } = case;+ let separator = if extra.is_empty() { "" } else { "&" };+ let unsigned = if extra.is_empty() {+ path.to_owned()+ } else {+ format!("{path}?{extra}")+ };+ for signed in [+ signed_uri(path, key, extra),+ data_package_uri(path, key, extra),+ with_extra(expired_uri(path, key), extra),+ format!("{path}?ex=zzz&is=&hm=&uc=xx{separator}{extra}"),+ format!("{path}?uc=dp{separator}{extra}"),+ ] {+ let label = format!("{method} {signed} range={range:?} env={env:?}");+ let expected =+ dispatch_uri(&unsigned_app, method.clone(), &unsigned, &[], range).await;+ let actual =+ dispatch_uri(&signed_app, method.clone(), &signed, &[], range).await;+ assert_eq!(status, expected.status(), "{label}");+ assert_eq!(expected.status(), actual.status(), "{label}");+ assert_eq!(expected.headers(), actual.headers(), "{label}");+ assert_eq!(body_of(expected).await, body_of(actual).await, "{label}");+ }+ }+ }+ }++ #[test]+ fn signature_parameters_never_reach_the_attachment_reader() {+ let params: HashMap<String, String> = [+ ("ex", "0"),+ ("is", "6a9920b1"),+ ("hm", "abc"),+ ("uc", "dp"),+ ("width", "64"),+ ("download", "1"),+ ("EX", "1"),+ ("%75c", "dp"),+ ("ucx", "1"),+ ]+ .into_iter()+ .map(|(name, value)| (name.to_owned(), value.to_owned()))+ .collect();+ let filtered = without_signature_parameters(¶ms);+ let mut names: Vec<&str> = filtered.keys().map(String::as_str).collect();+ names.sort_unstable();+ assert_eq!(vec!["%75c", "EX", "download", "ucx", "width"], names);+ assert_eq!(Some("64"), filtered.get("width").map(String::as_str));+ assert_eq!(Some("1"), filtered.get("download").map(String::as_str));+ }++ const SIGNATURE_VERDICTS: &str = "fluxer_media_proxy_attachment_signature_verdicts_total";++ fn verdict_counter(app: &Arc<AppState>, verdict: &str) -> u64 {+ counter(+ app,+ &format!("{SIGNATURE_VERDICTS}{{verdict=\"{verdict}\"}}"),+ )+ }++ fn verdict_cases() -> [(&'static str, String); 5] {+ [+ ("missing", ATTACHMENT_PATH.to_owned()),+ ("malformed", malformed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY)),+ ("mismatch", foreign_key_uri(ATTACHMENT_PATH)),+ ("expired", expired_uri(ATTACHMENT_PATH, ATTACHMENT_KEY)),+ ("valid", signed_uri(ATTACHMENT_PATH, ATTACHMENT_KEY, "")),+ ]+ }++ #[tokio::test]+ async fn report_and_enforce_count_the_verdict_of_every_attachment_read() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ for env in [SIGNATURE_REPORT, SIGNATURE_ENFORCE] {+ let app = dispatch_app("mp", root, env);+ for (verdict, uri) in verdict_cases() {+ dispatch_uri(&app, Method::GET, &uri, &[], None).await;+ assert_eq!(1, verdict_counter(&app, verdict), "{verdict} {uri} {env:?}");+ }+ for (verdict, _) in verdict_cases() {+ assert_eq!(1, verdict_counter(&app, verdict), "{verdict} {env:?}");+ }+ }+ }++ #[tokio::test]+ async fn off_mode_counts_no_verdict_at_all() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ for env in [&[][..], SIGNATURE_OFF_WITH_KEYS] {+ let app = dispatch_app("mp", root, env);+ for (_, uri) in verdict_cases() {+ dispatch_uri(&app, Method::GET, &uri, &[], None).await;+ }+ for (verdict, _) in verdict_cases() {+ assert_eq!(0, verdict_counter(&app, verdict), "{verdict} {env:?}");+ }+ }+ }++ #[tokio::test]+ async fn a_report_burst_bounds_its_log_lines_and_still_counts_every_read() {+ const BURST: usize = 50;+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let root = root.as_path();+ write_signed_representations(root);+ let report = dispatch_app("mp", root, SIGNATURE_REPORT);+ let captured = CapturedLog::default();+ let subscriber = tracing_subscriber::fmt()+ .with_writer(captured.clone())+ .with_ansi(false)+ .with_max_level(tracing::Level::TRACE)+ .finish();+ {+ let _guard = tracing::subscriber::set_default(subscriber);+ for _ in 0..BURST {+ let response = dispatch_uri(&report, Method::GET, ATTACHMENT_PATH, &[], None).await;+ assert_eq!(StatusCode::OK, response.status());+ }+ tokio::time::sleep(std::time::Duration::from_millis(1_100)).await;+ dispatch_uri(&report, Method::GET, ATTACHMENT_PATH, &[], None).await;+ }+ let log = captured.text();+ let lines: Vec<&str> = log+ .lines()+ .filter(|line| line.contains("reason=\"attachment_signature_would_deny\""))+ .collect();+ assert!(+ lines.len() < BURST,+ "{} lines for {BURST} refusable reads\n{log}",+ lines.len()+ );+ assert_eq!(+ (BURST + 1) as u64,+ verdict_counter(&report, "missing"),+ "every refusable read is counted"+ );+ let suppressed: u64 = lines+ .last()+ .expect("the burst logs at least one line")+ .split("suppressed=")+ .nth(1)+ .expect("an emitted line reports the lines it stands in for")+ .split_whitespace()+ .next()+ .expect("a suppressed count")+ .parse()+ .expect("a suppressed count is a number");+ assert!(suppressed > 0, "{log}");+ assert!(+ lines.len() as u64 + suppressed <= (BURST + 1) as u64,+ "{log}"+ );+ }}diff --git a/fluxer_media_proxy/src/server/routes/internal.rs b/fluxer_media_proxy/src/server/routes/internal.rsindex c7c5f3215..a1a872333 100644--- a/fluxer_media_proxy/src/server/routes/internal.rs+++ b/fluxer_media_proxy/src/server/routes/internal.rs@@ -7,6 +7,7 @@ use crate::{media_process, mime,output_format::OutputFormat,server::{+ format_policy::is_svg_content_type,media_operations::{MediaFailure, MediaInput, MediaInputLimit, MetadataOutput, load_media_input,resolve_metadata,@@ -30,9 +31,12 @@ use axum::{};use base64::{Engine as _, engine::general_purpose};use bytes::Bytes;+use futures_util::StreamExt as _;use serde::Deserialize;use std::sync::Arc;+const UPLOAD_SNIFF_PREFIX_BYTES: usize = 8192;+#[derive(Debug, Deserialize)]struct MetadataRequest {version: Option<i64>,@@ -267,6 +271,69 @@ pub(in crate::server) async fn frames_handler(}}+pub(in crate::server) async fn sniff_handler(+ State(app): State<Arc<AppState>>,+ headers: HeaderMap,+ request: Request<Body>,+) -> Response {+ if !check_internal_auth(&headers, app.cfg.secret_key.expose()) {+ return text(StatusCode::UNAUTHORIZED, "Unauthorized");+ }+ let body = match read_limited_body(request).await {+ Ok(body) => body,+ Err(status) => return text(status, canonical_reason_str(status)),+ };+ let req: UploadFileRequest = match serde_json::from_slice(&body) {+ Ok(req) => req,+ Err(_) => return text(StatusCode::BAD_REQUEST, "Bad Request"),+ };+ let range = format!("bytes=0-{}", UPLOAD_SNIFF_PREFIX_BYTES - 1);+ let object = match app+ .store+ .stream_object(+ &app.cfg.storage.bucket_uploads,+ &req.upload_filename,+ Some(&range),+ )+ .await+ {+ Ok(object) => object,+ Err(err) => return storage_error_response(&req.upload_filename, err),+ };+ let prefix = match read_sniff_prefix(object.body).await {+ Ok(prefix) => prefix,+ Err(err) => {+ return text_with_source(+ StatusCode::BAD_GATEWAY,+ "Bad Gateway",+ "sniff_read_failed",+ err,+ );+ }+ };+ let sniffed = mime::sniff(&prefix).mime;+ let content_type = (mime::is_supported_media_mime(sniffed) && !is_svg_content_type(sniffed))+ .then_some(sniffed);+ json_response(+ StatusCode::OK,+ serde_json::json!({ "content_type": content_type }).to_string(),+ )+}++async fn read_sniff_prefix(body: Body) -> Result<Vec<u8>, axum::Error> {+ let mut prefix = Vec::with_capacity(UPLOAD_SNIFF_PREFIX_BYTES);+ let mut chunks = body.into_data_stream();+ while prefix.len() < UPLOAD_SNIFF_PREFIX_BYTES {+ let Some(chunk) = chunks.next().await else {+ break;+ };+ let chunk = chunk?;+ let wanted = chunk.len().min(UPLOAD_SNIFF_PREFIX_BYTES - prefix.len());+ prefix.extend_from_slice(&chunk[..wanted]);+ }+ Ok(prefix)+}+fn check_internal_auth(headers: &HeaderMap, secret: &str) -> bool {let Some(auth) = headers.get(header::AUTHORIZATION)@@ -303,9 +370,14 @@ async fn read_limited_body(request: Request<Body>) -> Result<Bytes, StatusCode>#[cfg(test)]mod tests {use super::*;- use crate::{config::Config, test_fixtures::synthetic_png};+ use crate::{+ config::Config,+ storage::tests::{FakeObject, fake_s3},+ test_fixtures::synthetic_png,+ };use axum::http::HeaderValue;use http_body_util::BodyExt as _;+ use std::path::Path;use tokio::sync::mpsc;fn test_app_state() -> Arc<AppState> {@@ -314,6 +386,58 @@ mod tests {Arc::new(AppState::for_tests(cfg))}+ fn local_storage_config(storage_root: &Path) -> Config {+ Config::load_from_iter([+ (+ "FLUXER_MEDIA_PROXY_SECRET_KEY".to_owned(),+ "secret".to_owned(),+ ),+ (+ "FLUXER_MEDIA_PROXY_STORAGE_ROOT".to_owned(),+ storage_root.display().to_string(),+ ),+ ])+ .expect("test config")+ }++ fn local_storage_app() -> (tempfile::TempDir, Arc<AppState>) {+ let tmp = tempfile::tempdir().expect("temp storage root");+ let storage_root = tmp.path().canonicalize().expect("canonical storage root");+ let app = Arc::new(AppState::for_tests(local_storage_config(&storage_root)));+ (tmp, app)+ }++ async fn store_upload(app: &AppState, key: &str, data: &[u8], content_type: &str) {+ app.store+ .write_object(&app.cfg.storage.bucket_uploads, key, data, content_type)+ .await+ .expect("stored upload");+ }++ async fn sniff_upload(app: &Arc<AppState>, key: &str) -> Response {+ let body = serde_json::json!({ "type": "upload", "upload_filename": key }).to_string();+ sniff_handler(+ State(Arc::clone(app)),+ authorized_headers(),+ json_request(body),+ )+ .await+ }++ fn mpeg_ts_segment() -> Vec<u8> {+ let mut segment = vec![0u8; 564];+ for offset in [0, 188, 376] {+ segment[offset] = 0x47;+ }+ segment+ }++ fn png_followed_by_zeros() -> Vec<u8> {+ let mut upload = synthetic_png(8, 8);+ upload.extend(std::iter::repeat_n(0u8, 64 * 1024));+ upload+ }+fn authorized_headers() -> HeaderMap {let mut headers = HeaderMap::new();headers.insert(@@ -341,18 +465,8 @@ mod tests {.expect("oversized request")}- fn saturated_transform_app(storage_root: &std::path::Path) -> Arc<AppState> {- let mut cfg = Config::load_from_iter([- (- "FLUXER_MEDIA_PROXY_SECRET_KEY".to_owned(),- "secret".to_owned(),- ),- (- "FLUXER_MEDIA_PROXY_STORAGE_ROOT".to_owned(),- storage_root.display().to_string(),- ),- ])- .expect("test config");+ fn saturated_transform_app(storage_root: &Path) -> Arc<AppState> {+ let mut cfg = local_storage_config(storage_root);cfg.media.max_native_transforms = 1;cfg.media.worker_queue_capacity = 0;Arc::new(AppState::for_tests(cfg))@@ -401,6 +515,88 @@ mod tests {assert_eq!("{\"frames\":[]}", response_body(response).await);}+ #[tokio::test]+ async fn metadata_reads_an_own_attachment_url_whatever_its_signature() {+ use fluxer_common::attachment_url_signature::{+ ATTACHMENT_URL_TTL_SECS, UrlKind, sign, with_signature,+ };+ use std::time::{SystemTime, UNIX_EPOCH};++ const SECRET: [u8; 32] = [5u8; 32];+ const ENDPOINT: &str = "https://media.test";+ let tmp = tempfile::tempdir().expect("temp storage root");+ let storage_root = tmp.path().canonicalize().expect("canonical storage root");+ let app = Arc::new(AppState::for_tests(+ Config::load_from_iter([+ (+ "FLUXER_MEDIA_PROXY_SECRET_KEY".to_owned(),+ "secret".to_owned(),+ ),+ (+ "FLUXER_MEDIA_PROXY_STORAGE_ROOT".to_owned(),+ storage_root.display().to_string(),+ ),+ (+ "FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT".to_owned(),+ ENDPOINT.to_owned(),+ ),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE".to_owned(),+ "enforce".to_owned(),+ ),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64".to_owned(),+ general_purpose::STANDARD.encode(SECRET),+ ),+ ])+ .expect("metadata signature config"),+ ));+ let key = "attachments/1/2/cat.png";+ app.store+ .write_object(+ &app.cfg.storage.bucket_cdn,+ key,+ &synthetic_png(4, 4),+ "image/png",+ )+ .await+ .expect("stored attachment");+ let unsigned = format!("{ENDPOINT}/{key}");+ let now = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .expect("the clock is after the unix epoch")+ .as_secs();+ let expires = now - 1;+ let issued = expires - ATTACHMENT_URL_TTL_SECS;+ let expired = format!(+ "{unsigned}?ex={expires:08x}&is={issued:08x}&hm={}",+ sign(key, expires, issued, UrlKind::Ordinary, &SECRET)+ );++ for spelling in [+ unsigned.clone(),+ expired,+ with_signature(&unsigned, key, now, now, &SECRET),+ ] {+ let body = serde_json::json!({+ "version": 2,+ "type": "external",+ "nsfw": "allow",+ "with_base64": true,+ "url": spelling,+ })+ .to_string();+ let response = metadata_handler(+ State(Arc::clone(&app)),+ authorized_headers(),+ json_request(body),+ )+ .await;+ assert_eq!(StatusCode::OK, response.status(), "{spelling}");+ assert!(response_body(response).await.contains("\"width\":4"));+ }+ }+#[tokio::test]async fn metadata_requires_version_two_exactly() {let encoded = general_purpose::STANDARD.encode(synthetic_png(4, 4));@@ -511,4 +707,210 @@ mod tests {drop(release);held.await.expect("held task").expect("held work");}++ #[tokio::test]+ async fn sniff_requires_the_internal_bearer_secret() {+ let (_tmp, app) = local_storage_app();+ store_upload(&app, "x.js", &mpeg_ts_segment(), "text/javascript").await;+ let body = r#"{"type":"upload","upload_filename":"x.js"}"#;++ let response = sniff_handler(+ State(Arc::clone(&app)),+ HeaderMap::new(),+ json_request(body.to_owned()),+ )+ .await;+ assert_eq!(StatusCode::UNAUTHORIZED, response.status());+ assert_eq!("Unauthorized", response_body(response).await);++ let mut wrong_secret = HeaderMap::new();+ wrong_secret.insert(+ header::AUTHORIZATION,+ HeaderValue::from_static("Bearer other"),+ );+ let response = sniff_handler(State(app), wrong_secret, json_request(body.to_owned())).await;+ assert_eq!(StatusCode::UNAUTHORIZED, response.status());+ }++ #[tokio::test]+ async fn sniff_reports_mpeg_ts_packets_stored_under_a_javascript_name_as_video_mp2t() {+ let (_tmp, app) = local_storage_app();+ store_upload(&app, "x.js", &mpeg_ts_segment(), "text/javascript").await;++ let response = sniff_upload(&app, "x.js").await;++ assert_eq!(StatusCode::OK, response.status());+ assert_eq!(+ Some("application/json"),+ response+ .headers()+ .get(header::CONTENT_TYPE)+ .and_then(|value| value.to_str().ok())+ );+ assert_eq!(+ r#"{"content_type":"video/mp2t"}"#,+ response_body(response).await+ );+ }++ #[tokio::test]+ async fn sniff_reports_null_for_javascript_svg_markup_and_pdf() {+ let (_tmp, app) = local_storage_app();+ let svg = br#"<svg xmlns="http://www.w3.org/2000/svg"></svg>"#.as_slice();+ assert_eq!("image/svg+xml", mime::sniff(svg).mime);++ for (key, data) in [+ ("a.js", b"export const a = 1;\n".as_slice()),+ ("b.tsx", svg),+ ("c.pdf", b"%PDF-1.7".as_slice()),+ ] {+ store_upload(&app, key, data, "application/octet-stream").await;+ let response = sniff_upload(&app, key).await;+ assert_eq!(StatusCode::OK, response.status(), "{key}");+ assert_eq!(+ r#"{"content_type":null}"#,+ response_body(response).await,+ "{key}"+ );+ }+ }++ #[tokio::test]+ async fn sniff_reports_null_for_an_empty_upload() {+ let (_tmp, app) = local_storage_app();+ store_upload(&app, "empty.js", b"", "text/javascript").await;++ let response = sniff_upload(&app, "empty.js").await;++ assert_eq!(StatusCode::OK, response.status());+ assert_eq!(r#"{"content_type":null}"#, response_body(response).await);+ }++ #[tokio::test]+ async fn sniff_reports_null_when_object_storage_answers_the_prefix_range_with_416() {+ let fake = fake_s3().await;+ fake.put_object(+ "uploads/empty.js",+ FakeObject {+ content_type: Some("text/javascript".to_owned()),+ read_status: Some(416),+ ..FakeObject::default()+ },+ );+ let tmp = tempfile::tempdir().expect("temp storage root");+ let app = Arc::new(AppState::for_tests(fake.config(tmp.path())));++ let response = sniff_upload(&app, "empty.js").await;++ assert_eq!(StatusCode::OK, response.status());+ assert_eq!(r#"{"content_type":null}"#, response_body(response).await);+ }++ #[tokio::test]+ async fn sniff_reports_the_prefix_verdict_for_an_upload_larger_than_the_prefix() {+ let (_tmp, app) = local_storage_app();+ store_upload(&app, "large.bin", &png_followed_by_zeros(), "text/plain").await;++ let response = sniff_upload(&app, "large.bin").await;++ assert_eq!(StatusCode::OK, response.status());+ assert_eq!(+ r#"{"content_type":"image/png"}"#,+ response_body(response).await+ );+ }++ #[tokio::test]+ async fn sniff_asks_object_storage_for_only_the_prefix_range() {+ let fake = fake_s3().await;+ fake.put_object(+ "uploads/large.bin",+ FakeObject {+ body: png_followed_by_zeros(),+ content_type: Some("text/plain".to_owned()),+ ..FakeObject::default()+ },+ );+ let tmp = tempfile::tempdir().expect("temp storage root");+ let app = Arc::new(AppState::for_tests(fake.config(tmp.path())));++ let response = sniff_upload(&app, "large.bin").await;++ assert_eq!(StatusCode::OK, response.status());+ assert_eq!(+ r#"{"content_type":"image/png"}"#,+ response_body(response).await+ );+ let reads: Vec<_> = fake+ .requests()+ .into_iter()+ .filter(|(method, _, _, _)| *method == Method::GET)+ .collect();+ assert_eq!(1, reads.len());+ let (_, uri, headers, _) = &reads[0];+ assert_eq!("/uploads/large.bin", uri.path());+ assert_eq!(+ Some("bytes=0-8191"),+ headers+ .get(header::RANGE)+ .and_then(|value| value.to_str().ok())+ );+ }++ #[tokio::test]+ async fn sniff_prefix_stops_at_the_prefix_when_storage_ignores_the_range() {+ let upload = png_followed_by_zeros();+ let mut yielded = 0;+ let stream_source = upload.clone();+ let body = Body::from_stream(futures_util::stream::poll_fn(move |_| {+ assert!(+ yielded < UPLOAD_SNIFF_PREFIX_BYTES,+ "the body was polled again after {yielded} bytes were yielded"+ );+ let end = (yielded + 1000).min(stream_source.len());+ let chunk = Bytes::copy_from_slice(&stream_source[yielded..end]);+ yielded = end;+ std::task::Poll::Ready(Some(Ok::<_, std::io::Error>(chunk)))+ }));++ let prefix = read_sniff_prefix(body).await.expect("prefix read");++ assert_eq!(&upload[..UPLOAD_SNIFF_PREFIX_BYTES], prefix.as_slice());+ }++ #[tokio::test]+ async fn sniff_answers_a_missing_upload_with_404() {+ let (_tmp, app) = local_storage_app();++ let response = sniff_upload(&app, "missing.js").await;++ assert_eq!(StatusCode::NOT_FOUND, response.status());+ assert_eq!("Not Found", response_body(response).await);+ }++ #[tokio::test]+ async fn sniff_answers_a_body_without_an_upload_filename_with_400() {+ let response = sniff_handler(+ State(test_app_state()),+ authorized_headers(),+ json_request(r#"{"type":"upload"}"#.to_owned()),+ )+ .await;++ assert_eq!(StatusCode::BAD_REQUEST, response.status());+ assert_eq!("Bad Request", response_body(response).await);+ }++ #[tokio::test]+ async fn sniff_answers_an_oversized_body_with_413_payload_too_large() {+ let response = sniff_handler(+ State(test_app_state()),+ authorized_headers(),+ oversized_request(),+ )+ .await;++ assert_eq!(StatusCode::PAYLOAD_TOO_LARGE, response.status());+ assert_eq!("Payload Too Large", response_body(response).await);+ }}diff --git a/fluxer_media_proxy/src/server/routes/ops.rs b/fluxer_media_proxy/src/server/routes/ops.rsindex 080fb895f..2e2dfc4ef 100644--- a/fluxer_media_proxy/src/server/routes/ops.rs+++ b/fluxer_media_proxy/src/server/routes/ops.rs@@ -12,8 +12,11 @@ use axum::{};use std::{net::SocketAddr, sync::Arc};-pub(in crate::server) async fn health() -> &'static str {- "OK"+pub(in crate::server) async fn health(State(app): State<Arc<AppState>>) -> String {+ format!(+ "OK attachment_signature={}",+ app.cfg.attachment_signature.mode.label()+ )}fn is_loopback_peer(peer: &SocketAddr) -> bool {diff --git a/fluxer_media_proxy/src/server/routes/relay.rs b/fluxer_media_proxy/src/server/routes/relay.rsindex 77306554b..035a763d4 100644--- a/fluxer_media_proxy/src/server/routes/relay.rs+++ b/fluxer_media_proxy/src/server/routes/relay.rs@@ -1,7 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-or-lateruse crate::{- config::DeploymentMode,external_path, http_client, http_headers,server::{relay::body::{@@ -54,7 +53,7 @@ pub(in crate::server) async fn relay_put(headers: HeaderMap,request: Request<Body>,) -> Response {- if app.cfg.mode != DeploymentMode::Upload {+ if !app.cfg.mode.serves_upload_relay() {return text(StatusCode::NOT_FOUND, "Not Found");}let key = external_path::percent_decode_string(&key, false);diff --git a/fluxer_media_proxy/src/server/runtime.rs b/fluxer_media_proxy/src/server/runtime.rsindex 3c35fa629..1ef6cc2e7 100644--- a/fluxer_media_proxy/src/server/runtime.rs+++ b/fluxer_media_proxy/src/server/runtime.rs@@ -7,7 +7,11 @@ use super::{routes,state::AppState,};-use crate::{aggregate_error::aggregate_results, config::Config, media_process, request_log};+use crate::{+ aggregate_error::aggregate_results,+ config::{Config, PolicyMode},+ media_process, request_log,+};use axum::{Router, middleware,routing::{any, get, post, put},@@ -29,6 +33,28 @@ pub async fn run(cfg: Config) -> anyhow::Result<()> {"object body reads served from the S3 read endpoint");}+ if state.cfg.cors.mode != PolicyMode::Off {+ let allowed_origins = state+ .cfg+ .cors+ .allowed_origins+ .iter()+ .filter_map(|origin| origin.to_str().ok())+ .collect::<Vec<_>>()+ .join(",");+ info!(+ mode = ?state.cfg.cors.mode,+ allowed_origins = %allowed_origins,+ "cross-origin read policy active"+ );+ }+ if state.cfg.attachment_signature.mode != PolicyMode::Off {+ info!(+ mode = ?state.cfg.attachment_signature.mode,+ secrets = state.cfg.attachment_signature.secret_count(),+ "attachment url signature policy active"+ );+ }let drain = HttpRequestDrain::new();let app = build_router(Arc::clone(&state), drain.clone());let listener = TcpListener::bind(addr).await?;@@ -47,6 +73,7 @@ fn build_router(state: Arc<AppState>, drain: HttpRequestDrain) -> Router {.route("/_health", get(routes::ops::health)).route("/_metrics", get(routes::ops::metrics_handler)).route("/_metadata", post(routes::internal::metadata_handler))+ .route("/_sniff", post(routes::internal::sniff_handler)).route("/_thumbnail", post(routes::internal::thumbnail_handler)).route("/_frames", post(routes::internal::frames_handler)).route(@@ -115,15 +142,28 @@ async fn shutdown_signal() {#[cfg(test)]mod tests {use super::*;+ use crate::http_headers;use axum::{body::Body,- http::{Method, Request, StatusCode},+ extract::ConnectInfo,+ http::{Method, Request, StatusCode, header},+ response::Response,};+ use base64::{Engine as _, engine::general_purpose};++ const ENFORCE: &[(&str, &str)] = &[+ ("FLUXER_MEDIA_PROXY_CORS_MODE", "enforce"),+ (+ "FLUXER_MEDIA_PROXY_CORS_ALLOWED_ORIGINS",+ "https://web.fluxer.app",+ ),+ ];const DECLARED_ROUTES: &[(&str, &str, &str)] = &[("/_health", "/_health", "GET,HEAD"),("/_metrics", "/_metrics", "GET,HEAD"),("/_metadata", "/_metadata", "POST"),+ ("/_sniff", "/_sniff", "POST"),("/_thumbnail", "/_thumbnail", "POST"),("/_frames", "/_frames", "POST"),(@@ -133,15 +173,39 @@ mod tests {),];- fn test_router() -> Router {- let cfg = Config::load_from_iter([("FLUXER_MEDIA_PROXY_SECRET_KEY", "secret")])- .expect("test config");+ fn test_config(mode: &str, extra: &[(&str, &str)]) -> Config {+ let mut env = vec![+ (+ "FLUXER_MEDIA_PROXY_SECRET_KEY".to_owned(),+ "secret".to_owned(),+ ),+ ("FLUXER_MEDIA_PROXY_MODE".to_owned(), mode.to_owned()),+ (+ "FLUXER_MEDIA_PROXY_UPLOAD_RELAY_SECRET_BASE64".to_owned(),+ general_purpose::STANDARD.encode([7u8; 32]),+ ),+ ];+ env.extend(+ extra+ .iter()+ .map(|(key, value)| ((*key).to_owned(), (*value).to_owned())),+ );+ Config::load_from_iter(env).expect("test config")+ }++ fn test_router_for(cfg: Config) -> Router {build_router(Arc::new(AppState::for_tests(cfg)), HttpRequestDrain::new())}- async fn probe(path: &str, method: Method) -> axum::response::Response {- tower::ServiceExt::oneshot(- test_router(),+ async fn send(router: Router, request: Request<Body>) -> Response {+ tower::ServiceExt::oneshot(router, request)+ .await+ .expect("router response")+ }++ async fn probe(router: Router, path: &str, method: Method) -> Response {+ send(+ router,Request::builder().method(method).uri(path)@@ -149,37 +213,248 @@ mod tests {.expect("probe request"),).await- .expect("router response")+ }++ fn header_text(response: &Response, name: impl header::AsHeaderName) -> Option<&str> {+ response+ .headers()+ .get(name)+ .map(|value| value.to_str().expect("header is ascii"))+ }++ const SIGNATURE_SECRET: [u8; 32] = [11u8; 32];+ const SIGNATURE_REPORT: &[(&str, &str)] = &[+ ("FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE", "report"),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64",+ "CwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCws=",+ ),+ ];+ const SIGNATURE_ENFORCE: &[(&str, &str)] = &[+ ("FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE", "enforce"),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64",+ "CwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCws=",+ ),+ ];+ const ATTACHMENT_PATH: &str = "/attachments/1/2/file.bin";+ const ATTACHMENT_KEY: &str = "attachments/1/2/file.bin";++ async fn body_text(response: Response) -> String {+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)+ .await+ .expect("response body");+ String::from_utf8(body.to_vec()).expect("body is utf-8")}#[tokio::test]- async fn router_exposes_exactly_the_declared_route_and_method_table() {- for (declared, path, allowed) in DECLARED_ROUTES {- let response = probe(path, Method::DELETE).await;+ async fn health_names_the_attachment_signature_mode() {+ for (extra, expected) in [+ (&[][..], "off"),+ (SIGNATURE_REPORT, "report"),+ (SIGNATURE_ENFORCE, "enforce"),+ ] {+ let router = test_router_for(test_config("mp", extra));+ let response = probe(router, "/_health", Method::GET).await;+ assert_eq!(StatusCode::OK, response.status(), "{expected}");assert_eq!(- StatusCode::METHOD_NOT_ALLOWED,- response.status(),- "{declared} must reject an undeclared method"- );- assert_eq!(- *allowed,- response- .headers()- .get("allow")- .expect("method router sets allow")- .to_str()- .expect("ascii allow header"),- "{declared} method table"+ format!("OK attachment_signature={expected}"),+ body_text(response).await,+ "{expected}");}}+ #[tokio::test]+ async fn an_enforced_refusal_is_countable_apart_from_a_genuine_missing_object() {+ let tmp = tempfile::tempdir().expect("storage root");+ let root = tmp.path().canonicalize().expect("canonical storage root");+ let mut env: Vec<(String, String)> = SIGNATURE_ENFORCE+ .iter()+ .map(|(key, value)| ((*key).to_owned(), (*value).to_owned()))+ .collect();+ env.push((+ "FLUXER_MEDIA_PROXY_SECRET_KEY".to_owned(),+ "secret".to_owned(),+ ));+ env.push((+ "FLUXER_MEDIA_PROXY_STORAGE_BACKEND".to_owned(),+ "local".to_owned(),+ ));+ env.push((+ "FLUXER_MEDIA_PROXY_STORAGE_ROOT".to_owned(),+ root.display().to_string(),+ ));+ let cfg = Config::load_from_iter(env).expect("signature router config");+ let state = Arc::new(AppState::for_tests(cfg));+ let router = build_router(Arc::clone(&state), HttpRequestDrain::new());++ let refused = probe(router.clone(), ATTACHMENT_PATH, Method::GET).await;+ assert_eq!(StatusCode::NOT_FOUND, refused.status());++ let now = std::time::SystemTime::now()+ .duration_since(std::time::UNIX_EPOCH)+ .expect("the clock is after the unix epoch")+ .as_secs();+ let signed = fluxer_common::attachment_url_signature::with_signature(+ ATTACHMENT_PATH,+ ATTACHMENT_KEY,+ now,+ now,+ &SIGNATURE_SECRET,+ );+ let missing = probe(router, &signed, Method::GET).await;+ assert_eq!(StatusCode::NOT_FOUND, missing.status());++ let text = state.metrics.render();+ assert!(+ text.contains("fluxer_media_proxy_requests_4xx_total{kind=\"attachment\"} 2\n"),+ "{text}"+ );+ assert!(+ text.contains(+ "fluxer_media_proxy_attachment_signature_verdicts_total{verdict=\"missing\"} 1\n"+ ),+ "{text}"+ );+ assert!(+ text.contains(+ "fluxer_media_proxy_attachment_signature_verdicts_total{verdict=\"valid\"} 1\n"+ ),+ "{text}"+ );+ }++ #[tokio::test]+ async fn router_exposes_exactly_the_declared_route_and_method_table() {+ for mode in ["mp", "static", "upload", "relay"] {+ let router = test_router_for(test_config(mode, &[]));+ for (declared, path, allowed) in DECLARED_ROUTES {+ let response = probe(router.clone(), path, Method::DELETE).await;+ assert_eq!(+ StatusCode::METHOD_NOT_ALLOWED,+ response.status(),+ "{mode} {declared} must reject an undeclared method"+ );+ assert_eq!(+ Some(*allowed),+ header_text(&response, header::ALLOW),+ "{mode} {declared} method table"+ );+ }+ }+ }+#[tokio::test]async fn unknown_paths_reach_the_catch_all_fallback() {- let response = probe("/avatars/1/hash.png", Method::DELETE).await;+ let router = test_router_for(test_config("mp", &[]));+ let response = probe(router.clone(), "/avatars/1/hash.png", Method::DELETE).await;assert_eq!(StatusCode::METHOD_NOT_ALLOWED, response.status());assert!(response.headers().get("allow").is_none());- let response = probe("/nope", Method::GET).await;+ let response = probe(router, "/nope", Method::GET).await;assert_eq!(StatusCode::NOT_FOUND, response.status());}++ #[tokio::test]+ async fn relay_mode_router_serves_health_relay_and_internal_but_not_reads() {+ let router = test_router_for(test_config("relay", &[]));++ let health = probe(router.clone(), "/_health", Method::GET).await;+ assert_eq!(StatusCode::OK, health.status());++ let options = probe(+ router.clone(),+ "/v1/relay/uploads/file.png",+ Method::OPTIONS,+ )+ .await;+ assert_eq!(StatusCode::NO_CONTENT, options.status());+ assert_eq!(+ Some("*"),+ header_text(&options, header::ACCESS_CONTROL_ALLOW_ORIGIN)+ );+ assert_eq!(+ Some("PUT, OPTIONS"),+ header_text(&options, header::ACCESS_CONTROL_ALLOW_METHODS)+ );++ let put = probe(router.clone(), "/v1/relay/uploads/file.png", Method::PUT).await;+ assert_eq!(StatusCode::UNAUTHORIZED, put.status());++ let metadata = probe(router.clone(), "/_metadata", Method::POST).await;+ assert_eq!(StatusCode::UNAUTHORIZED, metadata.status());++ let sniff = probe(router.clone(), "/_sniff", Method::POST).await;+ assert_eq!(StatusCode::UNAUTHORIZED, sniff.status());++ let read = probe(router, "/attachments/1/2/stream.js", Method::GET).await;+ assert_eq!(StatusCode::NOT_FOUND, read.status());+ assert_eq!(+ None,+ header_text(&read, header::ACCESS_CONTROL_ALLOW_ORIGIN)+ );+ }++ #[tokio::test]+ async fn internal_routes_and_the_relay_ignore_the_cors_policy() {+ let off = test_router_for(test_config("upload", &[]));+ let enforce = test_router_for(test_config("upload", ENFORCE));+ let request = |method: Method, path: &str| {+ Request::builder()+ .method(method)+ .uri(path)+ .header(header::ORIGIN, "https://evil.example")+ .extension(ConnectInfo(SocketAddr::from(([127, 0, 0, 1], 40000))))+ .body(Body::empty())+ .expect("cors probe request")+ };+ for (method, path) in [+ (Method::GET, "/_health"),+ (Method::POST, "/_metadata"),+ (Method::POST, "/_sniff"),+ (Method::OPTIONS, "/v1/relay/uploads/file.png"),+ (Method::PUT, "/v1/relay/uploads/file.png"),+ (Method::GET, "/_metrics"),+ ] {+ let label = format!("{method} {path}");+ let expected = send(off.clone(), request(method.clone(), path)).await;+ let actual = send(enforce.clone(), request(method, path)).await;+ assert_eq!(expected.status(), actual.status(), "{label}");+ assert_eq!(expected.headers(), actual.headers(), "{label}");+ for vary in actual.headers().get_all(header::VARY) {+ assert!(+ !vary.to_str().expect("ascii vary").contains("Origin"),+ "{label}"+ );+ }+ }+ }++ #[tokio::test]+ async fn a_refused_read_through_the_router_has_security_and_version_headers() {+ let state = Arc::new(AppState::for_tests(test_config("mp", ENFORCE)));+ let router = build_router(Arc::clone(&state), HttpRequestDrain::new());+ let response = send(+ router,+ Request::builder()+ .method(Method::GET)+ .uri("/avatars/1/hash.png")+ .header(header::ORIGIN, "null".to_owned())+ .body(Body::empty())+ .expect("refused read request"),+ )+ .await;+ assert_eq!(StatusCode::FORBIDDEN, response.status());+ assert!(response.headers().contains_key("x-fluxer-version"));+ assert_eq!(+ Some(http_headers::STRICT_TRANSPORT_SECURITY),+ header_text(&response, header::STRICT_TRANSPORT_SECURITY)+ );+ assert!(+ state+ .metrics+ .render()+ .contains("fluxer_media_proxy_requests_4xx_total{kind=\"asset_image\"} 1\n")+ );+ }}diff --git a/fluxer_media_proxy/src/server/self_origin.rs b/fluxer_media_proxy/src/server/self_origin.rsindex 1367b18e2..095e1a576 100644--- a/fluxer_media_proxy/src/server/self_origin.rs+++ b/fluxer_media_proxy/src/server/self_origin.rs@@ -46,8 +46,10 @@ pub(in crate::server) fn resolve(app: &AppState, url: &str) -> Option<SelfOrigin}return Some(SelfOrigin::External { url: target });}- if path.starts_with("/attachments/") || (path.starts_with("/themes/") && path.ends_with(".css"))- {+ if path.starts_with("/attachments/") {+ return stored(cdn, decode_storage_key(path).ok()?);+ }+ if path.starts_with("/themes/") && path.ends_with(".css") {return stored(cdn, decode_storage_key(path).ok()?);}if let Some(key) = parse_entrance_sound_path(path) {@@ -307,4 +309,56 @@ mod tests {Some("/a.png"));}++ #[test]+ fn an_enforcing_instance_still_resolves_its_own_attachment_for_internal_reads() {+ use base64::Engine as _;+ use fluxer_common::attachment_url_signature::with_signature;+ use std::time::{SystemTime, UNIX_EPOCH};++ const SECRET: [u8; 32] = [3u8; 32];+ let app = AppState::for_tests(+ Config::load_from_iter([+ (+ "FLUXER_MEDIA_PROXY_SECRET_KEY".to_owned(),+ "secret".to_owned(),+ ),+ (+ "FLUXER_MEDIA_PROXY_PUBLIC_ENDPOINT".to_owned(),+ format!("{ENDPOINT}/"),+ ),+ (+ "FLUXER_MEDIA_PROXY_STORAGE_BACKEND".to_owned(),+ "local".to_owned(),+ ),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_SIGNATURE_MODE".to_owned(),+ "enforce".to_owned(),+ ),+ (+ "FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64".to_owned(),+ base64::engine::general_purpose::STANDARD.encode(SECRET),+ ),+ ])+ .expect("self origin signature config"),+ );+ let key = "attachments/1544725486800732163/1544971349200470016/cat.gif";+ let unsigned = format!("{ENDPOINT}/{key}");+ let now = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .expect("the clock is after the unix epoch")+ .as_secs();+ let signed = with_signature(&unsigned, key, now, now, &SECRET);++ for spelling in [+ unsigned.clone(),+ signed.clone(),+ format!("{signed}&width=64#frame"),+ ] {+ match resolve(&app, &spelling) {+ Some(SelfOrigin::Stored { key: resolved, .. }) => assert_eq!(key, resolved),+ _ => panic!("an own attachment resolves to a stored object: {spelling}"),+ }+ }+ }}diff --git a/fluxer_media_proxy/src/server/state.rs b/fluxer_media_proxy/src/server/state.rsindex eff5338cc..87d7e638c 100644--- a/fluxer_media_proxy/src/server/state.rs+++ b/fluxer_media_proxy/src/server/state.rs@@ -11,6 +11,7 @@ use crate::{nsfw::NSFWClient,response_body_limit::RESPONSE_BODY_TRANSPORT_CHUNK_BYTES_MAX,server::{+ attachment_signature::WouldDenyLog,external::{ExternalHintCache, new_external_hint_cache},transform::TransformRuntime,},@@ -23,6 +24,7 @@ pub(in crate::server) struct AppState {pub(in crate::server) metrics: Arc<metrics::Metrics>,pub(in crate::server) store: Store,pub(in crate::server) media: MediaRuntime,+ pub(in crate::server) signature_log: WouldDenyLog,}pub(in crate::server) struct MediaRuntime {@@ -54,6 +56,7 @@ impl AppState {metrics,store,media,+ signature_log: WouldDenyLog::new(),})}@@ -69,6 +72,7 @@ impl AppState {metrics,store,media,+ signature_log: WouldDenyLog::new(),}}}diff --git a/fluxer_media_proxy/src/server/stored/mod.rs b/fluxer_media_proxy/src/server/stored/mod.rsindex f7a689a7f..46c26e654 100644--- a/fluxer_media_proxy/src/server/stored/mod.rs+++ b/fluxer_media_proxy/src/server/stored/mod.rs@@ -8,12 +8,14 @@ mod tests;use crate::{asset_size,coalescer::CoalescerError,+ config::PolicyMode,constants::{self, AssetExtension, AssetKind},image_quality::ImageQuality,image_transform::{AnimationMode, ResizeMode},media_process, mime, output_format,server::{asset_path::{ParsedAssetPath, asset_filename_hint},+ attachment_signature,format_policy::{OriginalImageRequest, default_transform_quality,effective_animated_image_output_format, extension_from_mime, is_svg_content_type,@@ -293,6 +295,22 @@ pub(in crate::server) async fn serve_attachment(key: &str,params: &HashMap<String, String>,headers: &HeaderMap,+) -> Response {+ let response = read_attachment(app, method, key, params, headers).await;+ if app.cfg.attachment_signature.mode == PolicyMode::Enforce+ && response.status() == StatusCode::NOT_FOUND+ {+ return attachment_signature::masked_not_found_response(key);+ }+ response+}++async fn read_attachment(+ app: &Arc<AppState>,+ method: Method,+ key: &str,+ params: &HashMap<String, String>,+ headers: &HeaderMap,) -> Response {let filename = filename_from_storage_key(key);let animated = animated_param(params, false);diff --git a/fluxer_media_proxy/src/storage/tests/mod.rs b/fluxer_media_proxy/src/storage/tests/mod.rsindex 82a085d31..aa3e3a781 100644--- a/fluxer_media_proxy/src/storage/tests/mod.rs+++ b/fluxer_media_proxy/src/storage/tests/mod.rs@@ -9,8 +9,8 @@ mod s3_read_endpoint;use crate::{config::{- BucketStyle, Config, DeploymentMode, MediaServingConfig, StorageBackend, StorageConfig,- UploadRelayConfig,+ AttachmentSignatureConfig, BucketStyle, Config, CorsConfig, DeploymentMode,+ MediaServingConfig, PolicyMode, StorageBackend, StorageConfig, UploadRelayConfig,},constants,metrics::{Metrics, http_client::HTTPClientMetrics, storage::StorageMetrics},@@ -84,6 +84,14 @@ fn test_config(root: &Path) -> Config {spool_chunk_bytes: 64 * 1024,spool_max_total_bytes: 1 << 30,},+ cors: CorsConfig {+ mode: PolicyMode::Off,+ allowed_origins: Vec::new(),+ },+ attachment_signature: AttachmentSignatureConfig {+ mode: PolicyMode::Off,+ secrets: Vec::new(),+ },}}diff --git a/fluxer_messages/Cargo.toml b/fluxer_messages/Cargo.tomlindex 022a3715f..b45ef4d7f 100644--- a/fluxer_messages/Cargo.toml+++ b/fluxer_messages/Cargo.toml@@ -8,6 +8,7 @@ publish = false[dependencies]fluxer_common = { path = "../fluxer_common" }anyhow = "1.0.104"+base64 = "0.23"chrono = { version = "0.4.45", default-features = false, features = ["serde"] }fluxer-svc = { path = "../fluxer_svc" }fluxer_markdown_parser = { path = "../packages/markdown_parser/rust" }diff --git a/fluxer_messages/src/shard_impl.rs b/fluxer_messages/src/shard_impl.rsindex 4bd578274..b03ad1b1a 100644--- a/fluxer_messages/src/shard_impl.rs+++ b/fluxer_messages/src/shard_impl.rs@@ -15,6 +15,7 @@ use crate::types::{MessageSnapshot, MessageStickerItem,};use crate::udt;+use base64::prelude::{BASE64_STANDARD, Engine};use chrono::{DateTime, Utc};use fluxer_common::user_flags::{USER_FLAG_STAFF, visible_user_flags};use fluxer_svc::shard::ShardService;@@ -254,6 +255,7 @@ struct ResponseBuildOptions {can_read_message_history: bool,media_endpoint: String,media_proxy_secret_key: String,+ attachment_url_secret: Vec<u8>,include_reactions: bool,nonce: Option<String>,tts: bool,@@ -1199,15 +1201,18 @@ impl<T: Transport> MessagesShard<T> {) -> Option<ApiMessageAttachmentResponse> {let attachment_id = attachment.attachment_id?;let filename = attachment.filename.clone().unwrap_or_default();- let url = make_attachment_cdn_url(- &options.media_endpoint,- channel_id,- attachment_id,- &filename,- );let decay = context.attachment_decay.get(&attachment_id);let expired =decay.is_some_and(|expires_at| expires_at.timestamp_millis() <= now_epoch_millis());+ let url = (!expired).then(|| {+ attachment_cdn_url(+ channel_id,+ attachment_id,+ &filename,+ options,+ now_epoch_secs(),+ )+ });let content_type = attachment.content_type.clone().unwrap_or_else(|| {mime_guess::from_path(&filename).first_or_octet_stream()@@ -1223,8 +1228,8 @@ impl<T: Transport> MessagesShard<T> {content_type: Some(content_type),content_hash: attachment.content_hash.clone(),size: assert_safe_byte_size(attachment.size.unwrap_or_default()),- url: (!expired).then_some(url.clone()),- proxy_url: (!expired).then_some(url),+ url: url.clone(),+ proxy_url: url,width: (!is_audio).then_some(attachment.width).flatten(),height: (!is_audio).then_some(attachment.height).flatten(),placeholder: attachment.placeholder.clone(),@@ -1324,24 +1329,24 @@ impl<T: Transport> MessagesShard<T> {embed_type: embed_type.unwrap_or_else(|| "rich".to_owned()),title,description,- url,+ url: url.map(|url| sign_own_url(&url, options)),timestamp: timestamp.map(epoch_millis_to_iso),color,author: author.and_then(|author| {author.name.map(|name| ApiEmbedAuthorResponse {name,- url: author.url,+ url: author.url.map(|url| sign_own_url(&url, options)),proxy_icon_url: author.icon_url.as_ref().map(|url| external_media_proxy_url(url, options)),- icon_url: author.icon_url,+ icon_url: author.icon_url.map(|url| sign_own_url(&url, options)),})}),provider: provider.and_then(|provider| {provider.name.map(|name| ApiEmbedProviderResponse {name,- url: provider.url,+ url: provider.url.map(|url| sign_own_url(&url, options)),icon_url: None,proxy_icon_url: None,})@@ -1357,7 +1362,7 @@ impl<T: Transport> MessagesShard<T> {.icon_url.as_ref().map(|url| external_media_proxy_url(url, options)),- icon_url: footer.icon_url,+ icon_url: footer.icon_url.map(|url| sign_own_url(&url, options)),})}),fields: fields.map(|fields| fields.into_iter().map(map_embed_field_response).collect()),@@ -1376,7 +1381,7 @@ impl<T: Transport> MessagesShard<T> {let url = media.url?;Some(ApiEmbedMediaResponse {proxy_url: external_media_proxy_url(&url, options),- url,+ url: sign_own_url(&url, options),width: media.width,height: media.height,duration: media.duration,@@ -2151,6 +2156,7 @@ impl<T: Transport> ShardService for MessagesShard<T> {can_read_message_history,media_endpoint,media_proxy_secret_key,+ attachment_url_secret_base64,include_reactions,nonce,tts,@@ -2173,6 +2179,9 @@ impl<T: Transport> ShardService for MessagesShard<T> {can_read_message_history,media_endpoint,media_proxy_secret_key,+ attachment_url_secret: decode_attachment_url_secret(+ attachment_url_secret_base64.as_deref(),+ ),include_reactions: include_reactions.unwrap_or(true),nonce,tts: tts.unwrap_or(false),@@ -2192,6 +2201,7 @@ impl<T: Transport> ShardService for MessagesShard<T> {can_read_message_history,media_endpoint,media_proxy_secret_key,+ attachment_url_secret_base64,include_reactions,nonce,tts,@@ -2211,6 +2221,9 @@ impl<T: Transport> ShardService for MessagesShard<T> {can_read_message_history,media_endpoint,media_proxy_secret_key,+ attachment_url_secret: decode_attachment_url_secret(+ attachment_url_secret_base64.as_deref(),+ ),include_reactions: include_reactions.unwrap_or(true),nonce,tts: tts.unwrap_or(false),@@ -2230,6 +2243,7 @@ impl<T: Transport> ShardService for MessagesShard<T> {can_read_message_history,media_endpoint,media_proxy_secret_key,+ attachment_url_secret_base64,include_reactions,} => {let viewer_user_id = parse_i64(&viewer_user_id, "viewer_user_id")?;@@ -2247,6 +2261,9 @@ impl<T: Transport> ShardService for MessagesShard<T> {can_read_message_history,media_endpoint,media_proxy_secret_key,+ attachment_url_secret: decode_attachment_url_secret(+ attachment_url_secret_base64.as_deref(),+ ),include_reactions: include_reactions.unwrap_or(true),nonce: None,tts: false,@@ -2267,6 +2284,7 @@ impl<T: Transport> ShardService for MessagesShard<T> {can_read_message_history,media_endpoint,media_proxy_secret_key,+ attachment_url_secret_base64,include_reactions,} => {let channel_id = parse_i64(&channel_id, "channel_id")?;@@ -2301,6 +2319,9 @@ impl<T: Transport> ShardService for MessagesShard<T> {can_read_message_history,media_endpoint,media_proxy_secret_key,+ attachment_url_secret: decode_attachment_url_secret(+ attachment_url_secret_base64.as_deref(),+ ),include_reactions: include_reactions.unwrap_or(true),nonce: None,tts: false,@@ -2331,6 +2352,13 @@ fn now_epoch_millis() -> i64 {.as_millis() as i64}+fn now_epoch_secs() -> u64 {+ SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .unwrap_or_default()+ .as_secs()+}+fn message_row_key(channel_id: i64, bucket: i32, message_id: i64) -> anyhow::Result<String> {postgres::kv_key(&[KeyPart::BigInt(channel_id),@@ -2816,6 +2844,10 @@ fn map_call(call: &MessageCall) -> ApiMessageCallResponse {}}+fn make_attachment_cdn_key(channel_id: i64, attachment_id: i64, filename: &str) -> String {+ format!("attachments/{channel_id}/{attachment_id}/{filename}")+}+fn make_attachment_cdn_url(media_endpoint: &str,channel_id: i64,@@ -2823,27 +2855,173 @@ fn make_attachment_cdn_url(filename: &str,) -> String {format!(- "{}/attachments/{}/{}/{}",+ "{}/{}",media_endpoint.trim_end_matches('/'),- channel_id,+ make_attachment_cdn_key(channel_id, attachment_id, filename)+ )+}++fn decode_attachment_url_secret(encoded: Option<&str>) -> Vec<u8> {+ let Some(encoded) = encoded.filter(|value| !value.is_empty()) else {+ return Vec::new();+ };+ match BASE64_STANDARD.decode(encoded) {+ Ok(secret) => secret,+ Err(error) => {+ tracing::warn!(%error, "attachment url secret is not standard base64");+ Vec::new()+ }+ }+}++fn attachment_anchor_secs(attachment_id: i64) -> u64 {+ snowflake_to_epoch_millis(attachment_id).max(0) as u64 / 1_000+}++fn sign_attachment_url(+ url: &str,+ storage_key: &str,+ attachment_id: i64,+ now_secs: u64,+ secret: &[u8],+) -> String {+ if secret.is_empty() {+ return url.to_owned();+ }+ fluxer_common::attachment_url_signature::with_signature(+ url,+ storage_key,+ attachment_anchor_secs(attachment_id),+ now_secs,+ secret,+ )+}++fn attachment_cdn_url(+ channel_id: i64,+ attachment_id: i64,+ filename: &str,+ options: &ResponseBuildOptions,+ now_secs: u64,+) -> String {+ let storage_key = make_attachment_cdn_key(channel_id, attachment_id, filename);+ let url = make_attachment_cdn_url(&options.media_endpoint, channel_id, attachment_id, filename);+ sign_attachment_url(+ &url,+ &storage_key,attachment_id,- filename+ now_secs,+ &options.attachment_url_secret,)}-fn external_media_proxy_url(input_url: &str, options: &ResponseBuildOptions) -> String {- if input_url.starts_with(options.media_endpoint.trim_end_matches('/')) {- return input_url.to_owned();+fn is_own_endpoint(media_endpoint: &str, target: &Url) -> bool {+ let Ok(base) = Url::parse(media_endpoint) else {+ return false;+ };+ let same_host = match (base.host_str(), target.host_str()) {+ (Some(base_host), Some(target_host)) => base_host.eq_ignore_ascii_case(target_host),+ _ => false,+ };+ same_host+ && base.scheme().eq_ignore_ascii_case(target.scheme())+ && base.port_or_known_default() == target.port_or_known_default()+ && target+ .path()+ .strip_prefix(base.path().trim_end_matches('/'))+ .is_some_and(|rest| rest.starts_with('/'))+}++fn raw_url_path(url: &str) -> &str {+ let Some((_, after_scheme)) = url.split_once("://") else {+ return "";+ };+ let Some(start) = after_scheme.find(['/', '?', '#']) else {+ return "";+ };+ let path = &after_scheme[start..];+ if !path.starts_with('/') {+ return "";}- if options.media_proxy_secret_key.is_empty() {- return input_url.to_owned();+ path.find(['?', '#']).map_or(path, |end| &path[..end])+}++fn own_attachment_key(input_url: &str, media_endpoint: &str) -> Option<(String, i64)> {+ let endpoint_path = raw_url_path(media_endpoint).trim_end_matches('/');+ let path = raw_url_path(input_url).strip_prefix(endpoint_path)?;+ if !path.starts_with("/attachments/") {+ return None;+ }+ let storage_key = fluxer_common::attachment_url_signature::decode_key(path)?;+ let attachment_id = attachment_key_id(&storage_key)?;+ Some((storage_key, attachment_id))+}++fn attachment_key_id(storage_key: &str) -> Option<i64> {+ let segments: Vec<&str> = storage_key.split('/').collect();+ let ["attachments", channel_id, attachment_id, filename @ ..] = segments.as_slice() else {+ return None;+ };+ if !is_snowflake_segment(channel_id)+ || !is_snowflake_segment(attachment_id)+ || filename.is_empty()+ || filename+ .iter()+ .any(|part| matches!(*part, "" | "." | "..") || part.contains('\0'))+ {+ return None;}+ attachment_id.parse().ok()+}++fn is_snowflake_segment(segment: &str) -> bool {+ (1..=20).contains(&segment.len()) && segment.bytes().all(|byte| byte.is_ascii_digit())+}++fn sign_own_media_url(+ input_url: &str,+ media_endpoint: &str,+ now_secs: u64,+ secret: &[u8],+) -> String {+ let Some((storage_key, attachment_id)) = own_attachment_key(input_url, media_endpoint) else {+ return input_url.to_owned();+ };+ sign_attachment_url(input_url, &storage_key, attachment_id, now_secs, secret)+}++fn sign_own_url(input_url: &str, options: &ResponseBuildOptions) -> String {+ sign_own_media_url(+ input_url,+ options.media_endpoint.trim_end_matches('/'),+ now_epoch_secs(),+ &options.attachment_url_secret,+ )+}++fn external_media_proxy_url(input_url: &str, options: &ResponseBuildOptions) -> String {+ media_proxy_url_at(input_url, options, now_epoch_secs())+}++fn media_proxy_url_at(input_url: &str, options: &ResponseBuildOptions, now_secs: u64) -> String {+ let media_endpoint = options.media_endpoint.trim_end_matches('/');let parsed_url = match Url::parse(input_url) {Ok(url) => url,Err(_) => return input_url.to_owned(),};+ if is_own_endpoint(media_endpoint, &parsed_url) {+ return sign_own_media_url(+ input_url,+ media_endpoint,+ now_secs,+ &options.attachment_url_secret,+ );+ }+ if options.media_proxy_secret_key.is_empty() {+ return input_url.to_owned();+ }fluxer_common::external_media_path::build_external_media_proxy_url(- options.media_endpoint.trim_end_matches('/'),+ media_endpoint,parsed_url.as_str(),options.media_proxy_secret_key.as_bytes(),)@@ -3536,6 +3714,7 @@ mod tests {can_read_message_history: true,media_endpoint: "https://media.example.com".to_owned(),media_proxy_secret_key: "secret".to_owned(),+ attachment_url_secret: Vec::new(),include_reactions: false,nonce: None,tts: false,@@ -3774,4 +3953,580 @@ mod tests {assert!(response.is_none());assert!(recorded_deletions(&deleted).is_empty());}++ const SIGNATURE_VECTORS: &str =+ include_str!("../../fluxer_common/src/testdata/attachment_url_signature_vectors.json");+ const SIGNED_CHANNEL_ID: i64 = 1_544_725_486_800_732_163;+ const SIGNED_ATTACHMENT_ID: i64 = 1_544_971_349_200_470_016;+ const SIGNED_FILENAME: &str = "cat.gif";+ const SIGNED_ANCHOR_SECS: u64 = 1_788_420_273;++ fn signing_options() -> ResponseBuildOptions {+ ResponseBuildOptions {+ media_endpoint: "https://media.test".to_owned(),+ attachment_url_secret: (0u8..32).collect(),+ ..build_options()+ }+ }++ fn signed_attachment() -> MessageAttachment {+ serde_json::from_value(json!({+ "attachment_id": SIGNED_ATTACHMENT_ID.to_string(),+ "filename": SIGNED_FILENAME,+ "size": 1024,+ }))+ .unwrap()+ }++ fn signed_storage_key() -> String {+ make_attachment_cdn_key(SIGNED_CHANNEL_ID, SIGNED_ATTACHMENT_ID, SIGNED_FILENAME)+ }++ fn unsigned_attachment_url(options: &ResponseBuildOptions) -> String {+ make_attachment_cdn_url(+ &options.media_endpoint,+ SIGNED_CHANNEL_ID,+ SIGNED_ATTACHMENT_ID,+ SIGNED_FILENAME,+ )+ }++ fn assert_signs(url: &str, unsigned_prefix: &str, options: &ResponseBuildOptions, now: u64) {+ assert_eq!(+ Some(format!("{unsigned_prefix}?")),+ url.split_once("ex=").map(|(head, _)| head.to_owned()),+ "{url}"+ );+ assert!(!url.contains("/external/"), "{url}");+ assert!(!url.ends_with('&'), "{url}");+ assert!(!url.contains("&&"), "{url}");+ let query = url.split_once('?').expect("a signed url carries a query").1;+ assert_eq!(+ fluxer_common::attachment_url_signature::Verdict::Valid,+ fluxer_common::attachment_url_signature::verify(+ &signed_storage_key(),+ Some(query),+ &[&options.attachment_url_secret],+ now,+ )+ .verdict,+ "{url}"+ );+ }++ #[test]+ fn the_signing_anchor_is_the_attachment_snowflake_second() {+ assert_eq!(+ SIGNED_ANCHOR_SECS,+ attachment_anchor_secs(SIGNED_ATTACHMENT_ID)+ );+ assert_eq!(0, attachment_anchor_secs(i64::MIN));+ }++ #[test]+ fn attachment_urls_match_the_shared_signature_fixture() {+ let fixture: serde_json::Value = serde_json::from_str(SIGNATURE_VECTORS).unwrap();+ let secret = BASE64_STANDARD+ .decode(fixture["secrets_base64"][0].as_str().unwrap())+ .unwrap();+ let options = ResponseBuildOptions {+ attachment_url_secret: secret,+ ..signing_options()+ };+ let parsed = fixture["sign"].as_array().unwrap();+ let mut run = 0;+ let mut data_packages = 0;+ for case in parsed {+ let name = case["name"].as_str().unwrap();+ if case["uc"].as_str().unwrap() == "dp" {+ data_packages += 1;+ continue;+ }+ assert_eq!("", case["uc"].as_str().unwrap(), "{name} uc");+ let channel_id: i64 = case["channel_id"].as_str().unwrap().parse().unwrap();+ let attachment_id: i64 = case["attachment_id"].as_str().unwrap().parse().unwrap();+ let filename = case["filename"].as_str().unwrap();+ let now = case["now"].as_u64().unwrap();+ let url = case["url"].as_str().unwrap();+ let signed = case["signed"].as_str().unwrap();+ assert_eq!(+ case["anchor"].as_u64().unwrap(),+ attachment_anchor_secs(attachment_id),+ "{name} anchor"+ );+ assert_eq!(+ case["storage_key"].as_str().unwrap(),+ make_attachment_cdn_key(channel_id, attachment_id, filename),+ "{name} key"+ );+ let unsigned = make_attachment_cdn_url(+ &options.media_endpoint,+ channel_id,+ attachment_id,+ filename,+ );+ assert_eq!(+ Some(unsigned.as_str()),+ url.split(['?', '#']).next(),+ "{name} unsigned url"+ );+ if url == unsigned {+ assert_eq!(+ signed,+ attachment_cdn_url(channel_id, attachment_id, filename, &options, now),+ "{name} attachment url"+ );+ }+ assert_eq!(+ signed,+ media_proxy_url_at(url, &options, now),+ "{name} embed url"+ );+ run += 1;+ }+ assert_eq!(parsed.len(), run + data_packages);+ assert!(run >= 5);+ assert!(data_packages >= 1);+ }++ #[test]+ fn an_attachment_signs_url_and_proxy_url_identically() {+ let deleted = DeletedMessageKeys::default();+ let shard = recording_shard(&deleted);+ let options = signing_options();+ let mapped = shard+ .map_attachment(+ SIGNED_CHANNEL_ID,+ &signed_attachment(),+ &options,+ &ResponseContext::default(),+ )+ .expect("an attachment carrying an id maps");++ let url = mapped.url.expect("a live attachment carries a url");+ assert_eq!(Some(url.clone()), mapped.proxy_url);+ assert_signs(+ &url,+ &unsigned_attachment_url(&options),+ &options,+ now_epoch_secs(),+ );+ }++ #[test]+ fn an_own_url_whose_filename_carries_a_slash_is_signed() {+ let options = signing_options();+ let now = SIGNED_ANCHOR_SECS + 10;+ let key = "attachments/1544725486800732163/1544971349200470016/a/b.gif";++ for url in [+ format!("{}/{key}", options.media_endpoint),+ format!(+ "{}/attachments/1544725486800732163/1544971349200470016/a%2Fb.gif",+ options.media_endpoint+ ),+ ] {+ let signed = media_proxy_url_at(&url, &options, now);+ assert_eq!(+ Some(format!("{url}?")),+ signed.split_once("ex=").map(|(head, _)| head.to_owned()),+ "{url}"+ );+ let query = signed+ .split_once('?')+ .expect("a signed url carries a query")+ .1;+ assert_eq!(+ fluxer_common::attachment_url_signature::Verdict::Valid,+ fluxer_common::attachment_url_signature::verify(+ key,+ Some(query),+ &[&options.attachment_url_secret],+ now,+ )+ .verdict,+ "{url}"+ );+ }++ for refused in [+ format!(+ "{}/attachments/1544725486800732163/1544971349200470016/a//b.gif",+ options.media_endpoint+ ),+ format!(+ "{}/attachments/1544725486800732163/1544971349200470016/a/../b.gif",+ options.media_endpoint+ ),+ format!(+ "{}/attachments/1544725486800732163/1544971349200470016/a/",+ options.media_endpoint+ ),+ ] {+ assert_eq!(+ refused,+ media_proxy_url_at(&refused, &options, now),+ "{refused}"+ );+ }+ }++ #[test]+ fn a_decayed_attachment_is_never_signed() {+ let deleted = DeletedMessageKeys::default();+ let shard = recording_shard(&deleted);+ let options = signing_options();+ let context = ResponseContext {+ attachment_decay: [(+ SIGNED_ATTACHMENT_ID,+ DateTime::<Utc>::from_timestamp_millis(now_epoch_millis() - 1_000).unwrap(),+ )]+ .into_iter()+ .collect(),+ ..ResponseContext::default()+ };++ let mapped = shard+ .map_attachment(SIGNED_CHANNEL_ID, &signed_attachment(), &options, &context)+ .expect("a decayed attachment still maps");++ assert_eq!(None, mapped.url);+ assert_eq!(None, mapped.proxy_url);+ assert_eq!(Some(true), mapped.expired);+ }++ #[test]+ fn an_attachment_backed_embed_signs_every_own_url_field() {+ let deleted = DeletedMessageKeys::default();+ let shard = recording_shard(&deleted);+ let options = signing_options();+ let unsigned = unsigned_attachment_url(&options);+ let embed: MessageEmbed = serde_json::from_value(json!({+ "type": "rich",+ "url": unsigned,+ "author": {"name": "author", "url": unsigned, "icon_url": unsigned},+ "provider": {"name": "provider", "url": unsigned},+ "footer": {"text": "footer", "icon_url": unsigned},+ "image": {"url": unsigned},+ "thumbnail": {"url": unsigned},+ }))+ .unwrap();++ let mapped = shard.map_embed(&embed, &options);++ let now = now_epoch_secs();+ let base = mapped.base;+ let author = base.author.expect("the embed carries an author");+ let provider = base.provider.expect("the embed carries a provider");+ let footer = base.footer.expect("the embed carries a footer");+ let image = base.image.expect("the embed carries an image");+ let thumbnail = base.thumbnail.expect("the embed carries a thumbnail");+ for signed in [+ base.url.expect("the embed carries a url"),+ author.url.expect("the author carries a url"),+ author.icon_url.expect("the author carries an icon url"),+ author+ .proxy_icon_url+ .expect("the author carries a proxy icon url"),+ provider.url.expect("the provider carries a url"),+ footer.icon_url.expect("the footer carries an icon url"),+ footer+ .proxy_icon_url+ .expect("the footer carries a proxy icon url"),+ image.url.clone(),+ image.proxy_url.clone(),+ thumbnail.url,+ thumbnail.proxy_url,+ ] {+ assert_signs(&signed, &unsigned, &options, now);+ }+ assert_eq!(image.url, image.proxy_url);+ }++ #[test]+ fn a_foreign_embed_url_is_never_rewritten() {+ let deleted = DeletedMessageKeys::default();+ let shard = recording_shard(&deleted);+ let options = signing_options();+ let embed: MessageEmbed = serde_json::from_value(json!({+ "type": "link",+ "url": "https://example.com/article",+ "author": {"name": "author", "url": "https://example.com/author"},+ "provider": {"name": "provider", "url": "https://example.com"},+ }))+ .unwrap();++ let mapped = shard.map_embed(&embed, &options);++ let base = mapped.base;+ assert_eq!(+ Some("https://example.com/article".to_owned()),+ base.url,+ "a page link never reaches the media proxy"+ );+ assert_eq!(+ Some("https://example.com/author".to_owned()),+ base.author.expect("the embed carries an author").url+ );+ assert_eq!(+ Some("https://example.com".to_owned()),+ base.provider.expect("the embed carries a provider").url+ );+ }++ #[test]+ fn an_own_attachment_embed_url_is_signed_for_every_origin_spelling() {+ let options = signing_options();+ let now = SIGNED_ANCHOR_SECS;+ for prefix in [+ "https://media.test",+ "https://MEDIA.test",+ "https://Media.Test",+ "https://media.test:443",+ ] {+ let input = format!("{prefix}/{}", signed_storage_key());+ let signed = media_proxy_url_at(&input, &options, now);+ assert_signs(&signed, &input, &options, now);+ }+ }++ #[test]+ fn an_own_attachment_embed_url_keeps_its_transform_parameters_and_fragment() {+ let options = signing_options();+ let now = SIGNED_ANCHOR_SECS;+ let input = format!(+ "https://media.test/{}?width=64&format=webp#frame",+ signed_storage_key()+ );++ let signed = media_proxy_url_at(&input, &options, now);++ assert!(signed.ends_with("&width=64&format=webp#frame"), "{signed}");+ assert!(!signed.contains("/external/"), "{signed}");+ let query = signed+ .split_once('?')+ .expect("a signed url carries a query")+ .1+ .split_once('#')+ .expect("the fragment is kept")+ .0;+ assert_eq!(+ fluxer_common::attachment_url_signature::Verdict::Valid,+ fluxer_common::attachment_url_signature::verify(+ &signed_storage_key(),+ Some(query),+ &[&options.attachment_url_secret],+ now,+ )+ .verdict+ );+ }++ #[test]+ fn an_own_endpoint_url_outside_attachments_passes_through_unsigned() {+ let options = signing_options();+ let now = SIGNED_ANCHOR_SECS;+ for input in [+ "https://media.test/emojis/1544725486800732163.webp",+ "https://media.test/avatars/1/abc.png",+ "https://media.test/stickers/1/abc.png",+ "https://media.test/attachments/1544725486800732163",+ "https://media.test/attachments/1544725486800732163/1544971349200470016",+ "https://media.test/attachments/1544725486800732163/1544971349200470016/",+ "https://media.test/attachments/1544725486800732163/not-a-snowflake/cat.gif",+ ] {+ assert_eq!(input, media_proxy_url_at(input, &options, now), "{input}");+ }+ }++ #[test]+ fn a_foreign_url_is_still_wrapped_into_the_external_path() {+ let options = signing_options();+ let now = SIGNED_ANCHOR_SECS;+ for input in [+ "https://example.com/attachments/1544725486800732163/1544971349200470016/cat.gif",+ "https://media.test.example.com/attachments/1/2/cat.gif",+ "http://media.test/attachments/1/2/cat.gif",+ "https://media.test:8443/attachments/1/2/cat.gif",+ ] {+ let wrapped = media_proxy_url_at(input, &options, now);+ assert!(+ wrapped.starts_with("https://media.test/external/"),+ "{input}"+ );+ assert!(!wrapped.contains("ex="), "{input}");+ }+ }++ #[test]+ fn an_absent_attachment_secret_leaves_every_url_unsigned() {+ let deleted = DeletedMessageKeys::default();+ let shard = recording_shard(&deleted);+ let options = ResponseBuildOptions {+ attachment_url_secret: Vec::new(),+ ..signing_options()+ };+ let unsigned = unsigned_attachment_url(&options);++ let mapped = shard+ .map_attachment(+ SIGNED_CHANNEL_ID,+ &signed_attachment(),+ &options,+ &ResponseContext::default(),+ )+ .expect("an attachment carrying an id maps");++ assert_eq!(Some(unsigned.clone()), mapped.url);+ assert_eq!(Some(unsigned.clone()), mapped.proxy_url);+ assert_eq!(+ unsigned,+ media_proxy_url_at(&unsigned, &options, SIGNED_ANCHOR_SECS)+ );+ assert_eq!(+ unsigned,+ attachment_cdn_url(+ SIGNED_CHANNEL_ID,+ SIGNED_ATTACHMENT_ID,+ SIGNED_FILENAME,+ &options,+ SIGNED_ANCHOR_SECS+ )+ );+ }++ #[test]+ fn a_self_hosted_path_prefix_only_matches_whole_segments() {+ let options = ResponseBuildOptions {+ media_endpoint: "https://self.test/media".to_owned(),+ ..signing_options()+ };+ let now = SIGNED_ANCHOR_SECS;+ let own = format!("https://self.test/media/{}", signed_storage_key());++ let signed = media_proxy_url_at(&own, &options, now);++ assert_signs(&signed, &own, &options, now);+ let foreign = format!("https://self.test/mediaxyz/{}", signed_storage_key());+ assert!(+ media_proxy_url_at(&foreign, &options, now)+ .starts_with("https://self.test/media/external/")+ );+ }++ #[test]+ fn a_percent_encoded_filename_signs_the_decoded_key() {+ let options = signing_options();+ let now = SIGNED_ANCHOR_SECS;+ let encoded =+ "https://media.test/attachments/1544725486800732163/1544971349200470016/caf%C3%A9.gif";+ let decoded =+ "https://media.test/attachments/1544725486800732163/1544971349200470016/café.gif";+ let key = "attachments/1544725486800732163/1544971349200470016/café.gif";++ for input in [encoded, decoded] {+ let signed = media_proxy_url_at(input, &options, now);+ let query = signed+ .split_once('?')+ .expect("a signed url carries a query")+ .1;+ assert_eq!(+ fluxer_common::attachment_url_signature::Verdict::Valid,+ fluxer_common::attachment_url_signature::verify(+ key,+ Some(query),+ &[&options.attachment_url_secret],+ now,+ )+ .verdict,+ "{input}"+ );+ }+ }++ #[test]+ fn a_percent_encoded_slash_signs_the_key_the_media_proxy_decodes() {+ let options = signing_options();+ let now = SIGNED_ANCHOR_SECS;+ for input in [+ "https://media.test/attachments/1544725486800732163%2F1544971349200470016/cat.gif",+ "https://media.test/attachments/1544725486800732163%2f1544971349200470016%2Fcat.gif",+ "https://media.test/attachments/%31544725486800732163/1544971349200470016/cat.gif",+ ] {+ let raw_path = input.strip_prefix("https://media.test").unwrap();+ assert_eq!(+ Some(signed_storage_key()),+ fluxer_common::attachment_url_signature::decode_key(raw_path),+ "{input}"+ );+ let signed = media_proxy_url_at(input, &options, now);+ assert_signs(&signed, input, &options, now);+ }+ for input in [+ "https://media.test/attachments/1544725486800732163/1544971349200470016/%2F",+ "https://media.test/attachments/1544725486800732163/1544971349200470016/cat.gif%00",+ ] {+ assert_eq!(input, media_proxy_url_at(input, &options, now), "{input}");+ }+ }++ #[test]+ fn a_dot_segment_is_never_signed_under_its_normalised_key() {+ let options = signing_options();+ let now = SIGNED_ANCHOR_SECS;+ for input in [+ "https://media.test/attachments/1544725486800732163/9/../1544971349200470016/cat.gif",+ "https://media.test/attachments/1544725486800732163/1544971349200470016/./cat.gif",+ "https://media.test/attachments/1544725486800732163/1544971349200470016/%2E/cat.gif",+ "https://media.test/attachments/1544725486800732163/9/%2e%2e/1544971349200470016/cat.gif",+ ] {+ assert_eq!(+ format!("/{}", signed_storage_key()),+ Url::parse(input).unwrap().path(),+ "{input}"+ );+ assert_eq!(input, media_proxy_url_at(input, &options, now), "{input}");+ }+ for input in [+ "https://media.test/attachments/1544725486800732163/1544971349200470016/..",+ "https://media.test/attachments/+1544725486800732163/1544971349200470016/cat.gif",+ "https://media.test/attachments/1544725486800732163/-1544971349200470016/cat.gif",+ "https://media.test/attachments/1544725486800732163/1544971349200470016/caf%C3%28.gif",+ ] {+ assert_eq!(input, media_proxy_url_at(input, &options, now), "{input}");+ }+ }++ #[test]+ fn a_request_without_the_secret_field_still_decodes() {+ let request: MessageRequest = serde_json::from_value(json!({+ "op": "ListResponses",+ "channel_id": "10",+ "viewer_user_id": "1",+ "limit": 50,+ "can_read_message_history": true,+ "media_endpoint": "https://media.test",+ "media_proxy_secret_key": "secret",+ }))+ .expect("an older caller omits the attachment secret");++ match request {+ MessageRequest::ListResponses {+ attachment_url_secret_base64,+ ..+ } => assert_eq!(None, attachment_url_secret_base64),+ other => panic!("unexpected request {other:?}"),+ }+ }++ #[test]+ fn a_secret_that_is_not_base64_disables_signing() {+ assert!(decode_attachment_url_secret(None).is_empty());+ assert!(decode_attachment_url_secret(Some("")).is_empty());+ assert!(decode_attachment_url_secret(Some("not base64!")).is_empty());+ assert_eq!(+ (0u8..32).collect::<Vec<u8>>(),+ decode_attachment_url_secret(Some("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="))+ );+ }}diff --git a/fluxer_messages/src/types.rs b/fluxer_messages/src/types.rsindex bb50a4bc1..3360e445c 100644--- a/fluxer_messages/src/types.rs+++ b/fluxer_messages/src/types.rs@@ -91,6 +91,8 @@ pub enum MessageRequest {can_read_message_history: bool,media_endpoint: String,media_proxy_secret_key: String,+ #[serde(default)]+ attachment_url_secret_base64: Option<String>,include_reactions: Option<bool>,nonce: Option<String>,tts: Option<bool>,@@ -103,6 +105,8 @@ pub enum MessageRequest {can_read_message_history: bool,media_endpoint: String,media_proxy_secret_key: String,+ #[serde(default)]+ attachment_url_secret_base64: Option<String>,include_reactions: Option<bool>,nonce: Option<String>,tts: Option<bool>,@@ -115,6 +119,8 @@ pub enum MessageRequest {can_read_message_history: bool,media_endpoint: String,media_proxy_secret_key: String,+ #[serde(default)]+ attachment_url_secret_base64: Option<String>,include_reactions: Option<bool>,},ListResponses {@@ -129,6 +135,8 @@ pub enum MessageRequest {can_read_message_history: bool,media_endpoint: String,media_proxy_secret_key: String,+ #[serde(default)]+ attachment_url_secret_base64: Option<String>,include_reactions: Option<bool>,},ExtractMentions {diff --git a/packages/config/src/ConfigLoader.ts b/packages/config/src/ConfigLoader.tsindex 2a564a42c..2c56010af 100644--- a/packages/config/src/ConfigLoader.ts+++ b/packages/config/src/ConfigLoader.ts@@ -144,6 +144,9 @@ function defaultConfig(): MasterConfig {token_ttl_secs: 900,keep_direct_countries: [],},+ attachment_urls: {+ secrets_base64: [],+ },},gateway: {port: 8771,@@ -354,6 +357,28 @@ function validateUploadRelaySecret(value: string, mode: string): void {}}+function isCanonicalStandardBase64(value: string): boolean {+ if (!/^[A-Za-z0-9+/]+={0,2}$/u.test(value)) {+ return false;+ }+ return Buffer.from(value, 'base64').toString('base64') === value;+}++function validateAttachmentUrlSecrets(values: Array<string>): void {+ for (const value of values) {+ const trimmed = value.trim();+ if (trimmed.length === 0) {+ continue;+ }+ if (!isCanonicalStandardBase64(trimmed)) {+ throw new Error('FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64 entries must be standard base64');+ }+ if (Buffer.from(trimmed, 'base64').length < 32) {+ throw new Error('FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64 entries must decode to at least 32 bytes');+ }+ }+}+function assertBoolean(value: unknown, envName: string): asserts value is boolean {if (typeof value !== 'boolean') {throw new Error(`${envName} must be true or false`);@@ -575,6 +600,7 @@ function normalizeConfig(config: MasterConfig): MasterConfig {requireString(config.s3?.secret_access_key, 'FLUXER_S3_SECRET_ACCESS_KEY');requireString(config.services.media_proxy.secret_key, 'FLUXER_MEDIA_PROXY_SECRET_KEY');validateUploadRelaySecret(config.services.media_proxy.upload_relay.secret_base64, config.services.media_proxy.mode);+ validateAttachmentUrlSecrets(config.services.media_proxy.attachment_urls.secrets_base64);requireString(config.services.admin.secret_key_base, 'FLUXER_ADMIN_SECRET_KEY_BASE');requireString(config.services.admin.oauth_client_secret, 'FLUXER_ADMIN_OAUTH_CLIENT_SECRET');requireString(config.services.gateway.rpc_auth_token, 'FLUXER_GATEWAY_RPC_AUTH_TOKEN');diff --git a/packages/config/src/MasterConfig.ts b/packages/config/src/MasterConfig.tsindex ffe6a332a..8e6c04998 100644--- a/packages/config/src/MasterConfig.ts+++ b/packages/config/src/MasterConfig.ts@@ -152,6 +152,9 @@ export interface MasterConfig {token_ttl_secs: number;keep_direct_countries: Array<string>;};+ attachment_urls: {+ secrets_base64: Array<string>;+ };};gateway: {port: number;diff --git a/packages/config/src/config_loader/EnvironmentOverrides.ts b/packages/config/src/config_loader/EnvironmentOverrides.tsindex cb0b29f04..11fa34e24 100644--- a/packages/config/src/config_loader/EnvironmentOverrides.ts+++ b/packages/config/src/config_loader/EnvironmentOverrides.ts@@ -172,6 +172,10 @@ const NAMED_FLUXER_ENV_OVERRIDES: Record<string, NamedEnvOverride> = {path: ['services', 'media_proxy', 'upload_relay', 'keep_direct_countries'],parse: parseCsv,},+ FLUXER_MEDIA_PROXY_ATTACHMENT_URL_SECRETS_BASE64: {+ path: ['services', 'media_proxy', 'attachment_urls', 'secrets_base64'],+ parse: parseCsv,+ },FLUXER_ADMIN_PORT: {path: ['services', 'admin', 'port'], parse: parseInteger},FLUXER_ADMIN_BASE_PATH: {path: ['services', 'admin', 'base_path']},FLUXER_ADMIN_SECRET_KEY_BASE: {path: ['services', 'admin', 'secret_key_base']},diff --git a/packages/schema/src/domains/message/AttachmentSchemas.ts b/packages/schema/src/domains/message/AttachmentSchemas.tsindex 82580e959..4923cb148 100644--- a/packages/schema/src/domains/message/AttachmentSchemas.ts+++ b/packages/schema/src/domains/message/AttachmentSchemas.ts@@ -54,3 +54,30 @@ export const ClientAttachmentReferenceRequest = ClientAttachmentBase.extend({});export type ClientAttachmentReferenceRequest = z.infer<typeof ClientAttachmentReferenceRequest>;++export const RefreshAttachmentUrlsRequest = z.object({+ attachment_urls: z+ .array(z.string().max(2048))+ .min(1)+ .max(50)+ .describe('Attachment URLs to refresh (1-50 entries, each at most 2048 characters)'),+});++export type RefreshAttachmentUrlsRequest = z.infer<typeof RefreshAttachmentUrlsRequest>;++export const RefreshedAttachmentUrl = z.object({+ original: z.string().describe('The requested URL, echoed back unchanged'),+ refreshed: z+ .string()+ .describe('The same URL carrying a fresh signature, or the original when it is not an attachment URL of ours'),+});++export type RefreshedAttachmentUrl = z.infer<typeof RefreshedAttachmentUrl>;++export const RefreshAttachmentUrlsResponse = z.object({+ refreshed_urls: z+ .array(RefreshedAttachmentUrl)+ .describe('One entry per requested URL, in the order they were requested'),+});++export type RefreshAttachmentUrlsResponse = z.infer<typeof RefreshAttachmentUrlsResponse>;