Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
19.0 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import type {Readable} from 'node:stream';
import type {AttachmentID, ChannelID, MessageID, UserID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import type {UploadedAttachment} from '@app/api/channel/AttachmentDTOs';
import type {ChannelRepository} from '@app/api/channel/ChannelRepository';
import type {
	AttachmentUploadMode,
	AttachmentUploadTraceRepository,
} from '@app/api/channel/repositories/message/AttachmentUploadTraceRepository';
import {acquireFormUploadSlot, type FormUploadHolder} from '@app/api/channel/services/FormUploadSlots';
import type {MessageInteractionService} from '@app/api/channel/services/MessageInteractionService';
import type {MessageService} from '@app/api/channel/services/MessageService';
import {dispatchMessageUpdateBroadcast} from '@app/api/channel/services/message/MessageGatewayDispatch';
import {
	assertAttachmentFileSizesWithinLimit,
	isCrosspostCopy,
	isMessageEmpty,
	isOperationDisabled,
	makeAttachmentCdnKey,
	makeAttachmentCdnUrl,
	purgeMessageAttachments as purgeMessageAttachmentsHelper,
} from '@app/api/channel/services/message/MessageHelpers';
import {applyUploadRelayDecision, resolveUploadRelayDecision} from '@app/api/channel/services/UploadRelay';
import {SYSTEM_USER_ID} from '@app/api/constants/Core';
import type {ThreadViewer} from '@app/api/experiment/ChannelThreadsGate';
import type {IPurgeQueue} from '@app/api/infrastructure/CachePurgeQueue';
import type {IGatewayService} from '@app/api/infrastructure/IGatewayService';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {resolveAttachmentMaxFileSize} from '@app/api/limits/AttachmentFileSizeLimit';
import type {LimitConfigService} from '@app/api/limits/LimitConfigService';
import type {RequestCache} from '@app/api/middleware/RequestCacheMiddleware';
import type {Attachment} from '@app/api/models/Attachment';
import type {Channel} from '@app/api/models/Channel';
import type {Message} from '@app/api/models/Message';
import type {UserRepository} from '@app/api/user/repositories/UserRepository';
import {assertGuildMemberCanCommunicate} from '@app/api/utils/GuildCommunicationUtils';
import {Permissions, TEXT_BASED_CHANNEL_TYPES} from '@fluxer/constants/src/ChannelConstants';
import {GuildOperations} from '@fluxer/constants/src/GuildConstants';
import {
	ATTACHMENT_UPLOAD_CHUNK_THRESHOLD,
	ATTACHMENT_UPLOAD_MAX_CHUNKS,
	resolveAttachmentUploadPartSize,
} from '@fluxer/constants/src/LimitConstants';
import {THREAD_FEATURE_CHANNEL_TYPES} from '@fluxer/constants/src/ThreadConstants';
import {ValidationErrorCodes} from '@fluxer/constants/src/ValidationErrorCodes';
import {CannotSendMessageToNonTextChannelError} from '@fluxer/errors/src/domains/channel/CannotSendMessageToNonTextChannelError';
import {UnknownChannelError} from '@fluxer/errors/src/domains/channel/UnknownChannelError';
import {UnknownMessageError} from '@fluxer/errors/src/domains/channel/UnknownMessageError';
import {FeatureTemporarilyDisabledError} from '@fluxer/errors/src/domains/core/FeatureTemporarilyDisabledError';
import {FileSizeTooLargeError} from '@fluxer/errors/src/domains/core/FileSizeTooLargeError';
import {InputValidationError} from '@fluxer/errors/src/domains/core/InputValidationError';
import {MissingPermissionsError} from '@fluxer/errors/src/domains/core/MissingPermissionsError';
import {ServiceUnavailableError} from '@fluxer/errors/src/domains/core/ServiceUnavailableError';
import {UnknownUserError} from '@fluxer/errors/src/domains/user/UnknownUserError';
import type {GuildResponse} from '@fluxer/schema/src/domains/guild/GuildResponseSchemas';
import type {
	CompleteMultipartAttachmentUploadItem,
	CompleteMultipartAttachmentUploadResult,
	PresignedAttachmentUploadRequestItem,
	PresignedAttachmentUploadResponseItem,
} from '@fluxer/schema/src/domains/message/AttachmentUploadSchemas';
import {getContentTypeFromFilename} from '@pkgs/mime_utils/src/ContentTypeUtils';

interface DeleteAttachmentParams {
	userId: UserID;
	viewer: ThreadViewer;
	channelId: ChannelID;
	messageId: MessageID;
	attachmentId: AttachmentID;
	requestCache: RequestCache;
}

type UploadActor = 'member' | 'webhook';

interface FormDataUploadTarget {
	userId: UserID;
	viewer: ThreadViewer;
	channelId: ChannelID;
	actor?: UploadActor;
}

export interface FormDataUpload {
	maxFileSize: number;
	assertCanAttachFiles: () => Promise<void>;
}

interface StoredFormDataFile {
	index: number;
	uploadKey: string;
	size: number;
}

interface UploadFormDataAttachmentsParams {
	userId: UserID;
	channelId: ChannelID;
	clientIp: string;
	files: Array<StoredFormDataFile>;
	attachmentMetadata: Array<{
		id: number;
		filename: string;
	}>;
}

interface RequestPresignedAttachmentUploadUrlsParams {
	userId: UserID;
	viewer: ThreadViewer;
	channelId: ChannelID;
	clientIp: string;
	attachments: Array<PresignedAttachmentUploadRequestItem>;
}

interface CompleteMultipartAttachmentUploadsParams {
	userId: UserID;
	viewer: ThreadViewer;
	channelId: ChannelID;
	clientIp: string;
	uploads: Array<CompleteMultipartAttachmentUploadItem>;
}

interface UploadChannel {
	channel: Channel;
	guild: GuildResponse | null;
	assertCanAttachFiles: () => Promise<void>;
}

export class AttachmentUploadService {
	constructor(
		private channelRepository: ChannelRepository,
		private userRepository: UserRepository,
		private storageService: IStorageService,
		private attachmentUploadTraceRepository: AttachmentUploadTraceRepository,
		private purgeQueue: IPurgeQueue,
		private messageInteractionService: MessageInteractionService,
		private messageService: MessageService,
		private limitConfigService: LimitConfigService,
		private gatewayService: IGatewayService,
	) {}

	async storeFormDataFile({
		body,
		filename,
		holder,
	}: {
		body: Readable;
		filename: string;
		holder: FormUploadHolder;
	}): Promise<string> {
		const uploadKey = crypto.randomUUID();
		const bucket = Config.s3.buckets.uploads;
		const contentType = getContentTypeFromFilename(filename);
		const releaseSlot = await acquireFormUploadSlot(holder);
		try {
			await runAttachmentStorageOperation(() =>
				this.storageService.uploadObject({
					bucket,
					key: uploadKey,
					body,
					contentType,
				}),
			);
		} catch (error) {
			await this.storageService.deleteObject(bucket, uploadKey).catch(() => undefined);
			throw body.errored ?? error;
		} finally {
			releaseSlot();
		}
		return uploadKey;
	}

	async deleteFormDataFiles(uploadKeys: Array<string>): Promise<void> {
		await Promise.all(
			uploadKeys.map((uploadKey) =>
				this.storageService.deleteObject(Config.s3.buckets.uploads, uploadKey).catch(() => undefined),
			),
		);
	}

	async uploadFormDataAttachments({
		userId,
		channelId,
		clientIp,
		files,
		attachmentMetadata,
	}: UploadFormDataAttachmentsParams): Promise<Array<UploadedAttachment>> {
		const metadataMap = new Map(attachmentMetadata.map((m) => [m.id, m]));
		return Promise.all(
			files.map(async ({index, uploadKey, size}) => {
				const metadata = metadataMap.get(index);
				if (!metadata) {
					throw new Error(`Internal error: metadata not found for file index ${index}`);
				}
				const filename = metadata.filename;
				const contentType = getContentTypeFromFilename(filename);
				await this.attachmentUploadTraceRepository.recordRequestedUpload({
					uploadKey,
					userId,
					channelId,
					filename,
					contentType,
					uploadMode: 'form_data',
					requestIp: clientIp,
				});
				const uploaded: UploadedAttachment = {
					id: index,
					upload_filename: uploadKey,
					filename,
					file_size: size,
					content_type: contentType,
				};
				return uploaded;
			}),
		);
	}

	async requestPresignedAttachmentUploadUrls({
		userId,
		viewer,
		channelId,
		clientIp,
		attachments,
	}: RequestPresignedAttachmentUploadUrlsParams): Promise<Array<PresignedAttachmentUploadResponseItem>> {
		if (!Config.presignedAttachmentUploadsEnabled) {
			throw new FeatureTemporarilyDisabledError();
		}
		const {maxFileSize, assertCanAttachFiles} = await this.prepareFormDataUpload({userId, viewer, channelId});
		await assertCanAttachFiles();
		assertAttachmentFileSizesWithinLimit(
			attachments.map(({file_size}) => file_size),
			maxFileSize,
		);
		const uploadRelayDecision = await resolveUploadRelayDecision(clientIp);
		return Promise.all(
			attachments.map(async (attachment) => {
				const uploadKey = crypto.randomUUID();
				const derivedContentType = getContentTypeFromFilename(attachment.filename);
				const bucket = Config.s3.buckets.uploads;
				const uploadMode: AttachmentUploadMode =
					attachment.file_size <= ATTACHMENT_UPLOAD_CHUNK_THRESHOLD ? 'presigned_singlepart' : 'presigned_multipart';
				await this.attachmentUploadTraceRepository.recordRequestedUpload({
					uploadKey,
					userId,
					channelId,
					filename: attachment.filename,
					contentType: derivedContentType,
					uploadMode,
					requestIp: clientIp,
				});
				if (attachment.file_size <= ATTACHMENT_UPLOAD_CHUNK_THRESHOLD) {
					const presigned_upload_url = await this.storageService.getPresignedUploadURL({
						bucket,
						key: uploadKey,
						contentType: derivedContentType,
						contentLength: attachment.file_size,
					});
					const upload_url = applyUploadRelayDecision({
						presignedUrl: presigned_upload_url,
						bucket,
						key: uploadKey,
						relayDecision: uploadRelayDecision,
						contentType: derivedContentType,
						maxBytes: attachment.file_size,
					});
					return {
						upload_mode: 'singlepart',
						id: attachment.id,
						filename: attachment.filename,
						upload_filename: uploadKey,
						upload_url,
						file_size: attachment.file_size,
						content_type: derivedContentType,
					};
				}
				const partSize = resolveAttachmentUploadPartSize(attachment.file_size);
				const partCount = Math.ceil(attachment.file_size / partSize);
				if (partCount > ATTACHMENT_UPLOAD_MAX_CHUNKS) {
					throw new FileSizeTooLargeError(maxFileSize);
				}
				const {uploadId} = await runAttachmentStorageOperation(() =>
					this.storageService.createMultipartUpload({
						bucket,
						key: uploadKey,
						contentType: derivedContentType,
					}),
				);
				const parts = await Promise.all(
					Array.from({length: partCount}, async (_, index) => {
						const partNumber = index + 1;
						const partContentLength =
							partNumber < partCount ? partSize : attachment.file_size - partSize * (partCount - 1);
						const presigned_upload_url = await this.storageService.getPresignedUploadPartURL({
							bucket,
							key: uploadKey,
							uploadId,
							partNumber,
							contentLength: partContentLength,
						});
						const upload_url = applyUploadRelayDecision({
							presignedUrl: presigned_upload_url,
							bucket,
							key: uploadKey,
							relayDecision: uploadRelayDecision,
							uploadId,
							partNumber,
							maxBytes: partContentLength,
						});
						return {part_number: partNumber, upload_url};
					}),
				);
				return {
					upload_mode: 'multipart',
					id: attachment.id,
					filename: attachment.filename,
					upload_filename: uploadKey,
					file_size: attachment.file_size,
					content_type: derivedContentType,
					upload_id: uploadId,
					part_size: partSize,
					parts,
				};
			}),
		);
	}

	async completeMultipartAttachmentUploads({
		userId,
		viewer,
		channelId,
		clientIp,
		uploads,
	}: CompleteMultipartAttachmentUploadsParams): Promise<Array<CompleteMultipartAttachmentUploadResult>> {
		if (!Config.presignedAttachmentUploadsEnabled) {
			throw new FeatureTemporarilyDisabledError();
		}
		const {maxFileSize, assertCanAttachFiles} = await this.prepareFormDataUpload({userId, viewer, channelId});
		await assertCanAttachFiles();
		const bucket = Config.s3.buckets.uploads;
		return Promise.all(
			uploads.map(async ({upload_filename, upload_id}, index) => {
				const pendingUpload = await this.attachmentUploadTraceRepository.getPendingUpload({
					uploadKey: upload_filename,
					userId,
					channelId,
					uploadMode: 'presigned_multipart',
				});
				if (!pendingUpload) {
					throw InputValidationError.fromCode(
						`uploads.${index}.upload_filename`,
						ValidationErrorCodes.UPLOADED_ATTACHMENT_NOT_FOUND,
						{filename: upload_filename},
					);
				}
				const parts = await runAttachmentStorageOperation(() =>
					this.storageService.listParts({
						bucket,
						key: upload_filename,
						uploadId: upload_id,
					}),
				);
				if (parts.length === 0) {
					await this.storageService
						.abortMultipartUpload({bucket, key: upload_filename, uploadId: upload_id})
						.catch(() => undefined);
					throw InputValidationError.fromCode('parts', ValidationErrorCodes.NO_UPLOADED_PARTS_TO_FINALIZE);
				}
				const totalUploadedBytes = parts.reduce((sum, part) => sum + (part.size ?? 0), 0);
				if (totalUploadedBytes > maxFileSize) {
					await this.storageService
						.abortMultipartUpload({bucket, key: upload_filename, uploadId: upload_id})
						.catch(() => undefined);
					throw new FileSizeTooLargeError(maxFileSize);
				}
				try {
					await runAttachmentStorageOperation(() =>
						this.storageService.completeMultipartUpload({
							bucket,
							key: upload_filename,
							uploadId: upload_id,
							parts,
						}),
					);
					await this.attachmentUploadTraceRepository.markUploadCompleted({
						uploadKey: upload_filename,
						completionIp: clientIp,
					});
				} catch (error) {
					await this.storageService
						.abortMultipartUpload({bucket, key: upload_filename, uploadId: upload_id})
						.catch(() => undefined);
					throw error;
				}
				return {upload_filename};
			}),
		);
	}

	async deleteAttachment({
		userId,
		viewer,
		channelId,
		messageId,
		attachmentId,
		requestCache,
	}: DeleteAttachmentParams): Promise<void> {
		const {channel, guild} = await this.messageInteractionService.authService.getChannelAuthenticated({
			userId,
			channelId,
			viewer,
		});
		if (isOperationDisabled(guild, GuildOperations.SEND_MESSAGE)) {
			throw new FeatureTemporarilyDisabledError();
		}
		const message = await this.channelRepository.messages.getMessage(channelId, messageId);
		if (!message) {
			throw new UnknownMessageError();
		}
		if (message.authorId !== userId) {
			throw new MissingPermissionsError();
		}
		if (!message.attachments || message.attachments.length === 0) {
			throw new UnknownMessageError();
		}
		const attachment = message.attachments.find((a: Attachment) => a.id === attachmentId);
		if (!attachment) {
			throw new UnknownMessageError();
		}
		const isLastAttachment = message.attachments.length === 1;
		const willBeEmpty = isLastAttachment && isMessageEmpty(message, true);
		if (willBeEmpty) {
			await this.messageService.deletion.deleteMessage({
				userId,
				viewer,
				channelId,
				messageId,
				requestCache,
			});
			return;
		}
		const updatedMessage = await this.messageService.writeLock.withFreshMessage(channelId, messageId, async (fresh) => {
			if (!fresh || fresh.authorId !== userId) {
				throw new UnknownMessageError();
			}
			const freshAttachment = fresh.attachments.find((a: Attachment) => a.id === attachmentId);
			if (!freshAttachment) {
				throw new UnknownMessageError();
			}
			const updatedAttachments = fresh.attachments.filter((a: Attachment) => a.id !== attachmentId);
			if (updatedAttachments.length === 0 && isMessageEmpty(fresh, true)) {
				return null;
			}
			const updatedRowData = {
				...fresh.toRow(),
				edited_timestamp: new Date(),
				attachments:
					updatedAttachments.length > 0 ? updatedAttachments.map((a: Attachment) => a.toMessageAttachment()) : null,
			};
			return this.messageService.crosspostPropagation.withPublishedEditBudget({fresh, actor: 'author'}, () =>
				this.channelRepository.messages.upsertMessage(updatedRowData, fresh.toRow()),
			);
		});
		if (!updatedMessage) {
			await this.messageService.deletion.deleteMessage({
				userId,
				viewer,
				channelId,
				messageId,
				requestCache,
			});
			return;
		}
		if (!isCrosspostCopy(updatedMessage)) {
			const cdnKey = makeAttachmentCdnKey(message.channelId, attachment.id, attachment.filename);
			await this.storageService.deleteObject(Config.s3.buckets.cdn, cdnKey);
			const cdnUrl = makeAttachmentCdnUrl(message.channelId, attachment.id, attachment.filename);
			await this.purgeQueue.addUrls([cdnUrl]);
		}
		await dispatchMessageUpdateBroadcast({gatewayService: this.gatewayService, channel, message: updatedMessage});
		await this.messageService.crosspostPropagation.propagateEdit(updatedMessage);
	}

	async purgeChannelAttachments(channel: Channel): Promise<void> {
		const batchSize = 100;
		let beforeMessageId: MessageID | undefined;
		while (true) {
			const messages = await this.channelRepository.messages.listMessages(channel.id, beforeMessageId, batchSize);
			if (messages.length === 0) {
				return;
			}
			await Promise.all(
				messages.map((message: Message) =>
					purgeMessageAttachmentsHelper(message, this.storageService, this.purgeQueue),
				),
			);
			if (messages.length < batchSize) {
				return;
			}
			beforeMessageId = messages[messages.length - 1].id;
		}
	}

	async prepareFormDataUpload({
		userId,
		viewer,
		channelId,
		actor = 'member',
	}: FormDataUploadTarget): Promise<FormDataUpload> {
		const {channel, guild, assertCanAttachFiles} =
			actor === 'webhook'
				? await this.getWebhookUploadChannel(channelId)
				: await this.getMemberUploadChannel({userId, viewer, channelId});
		if (!TEXT_BASED_CHANNEL_TYPES.has(channel.type) && !THREAD_FEATURE_CHANNEL_TYPES.has(channel.type)) {
			throw new CannotSendMessageToNonTextChannelError();
		}
		const user = await this.userRepository.findUnique(userId);
		if (!user) {
			throw new UnknownUserError();
		}
		const maxFileSize = resolveAttachmentMaxFileSize(this.limitConfigService.getConfigSnapshot(), {
			user,
			guildFeatures: guild?.features ?? null,
		});
		return {maxFileSize, assertCanAttachFiles};
	}

	private async getMemberUploadChannel({
		userId,
		viewer,
		channelId,
	}: {
		userId: UserID;
		viewer: ThreadViewer;
		channelId: ChannelID;
	}): Promise<UploadChannel> {
		const {channel, guild, checkPermission, member} =
			await this.messageInteractionService.authService.getChannelAuthenticated({
				userId,
				channelId,
				viewer,
			});
		const assertCanAttachFiles = async () => {
			if (guild) {
				await checkPermission(Permissions.SEND_MESSAGES | Permissions.ATTACH_FILES);
				assertGuildMemberCanCommunicate(member);
			}
		};
		return {channel, guild, assertCanAttachFiles};
	}

	private async getWebhookUploadChannel(channelId: ChannelID): Promise<UploadChannel> {
		const channel = await this.channelRepository.channelData.findUnique(channelId);
		if (!channel) {
			throw new UnknownChannelError();
		}
		if (!channel.guildId) {
			return {channel, guild: null, assertCanAttachFiles: async () => {}};
		}
		const guild = await this.gatewayService.getGuildData({
			guildId: channel.guildId,
			userId: SYSTEM_USER_ID,
			skipMembershipCheck: true,
		});
		return {channel, guild, assertCanAttachFiles: async () => {}};
	}
}

async function runAttachmentStorageOperation<T>(operation: () => Promise<T>): Promise<T> {
	try {
		return await operation();
	} catch (error) {
		throw new ServiceUnavailableError({
			message: 'Attachment storage is temporarily unavailable',
			cause: error instanceof Error ? error : undefined,
		});
	}
}