Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
14.6 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import {ArchiveAttemptSupersededError} from '@app/api/archive/ArchiveAttemptSupersededError';
import {
	type ArchiveTaskHandler,
	ArchiveTerminalFailureError,
	createArchiveTask,
	throwIfArchiveTerminallyFailed,
} from '@app/api/archive/ArchiveTask';
import {makeDataPackageAttachmentCdnUrl} from '@app/api/attachment/AttachmentUrls';
import {type AttachmentID, type ChannelID, createGuildID, type MessageID} from '@app/api/BrandedTypes';
import {Config} from '@app/api/Config';
import {makeAttachmentCdnKey} from '@app/api/channel/services/message/MessageHelpers';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {Logger} from '@app/api/Logger';
import {mapWithConcurrency} from '@app/api/utils/ConcurrencyUtils';
import {writeZipArchive} from '@app/api/worker/utils/ArchiveFile';
import {createArchiveJsonBuffer} from '@app/api/worker/utils/ArchiveJson';
import {buildHashedAssetKey, getAnimatedAssetExtension} from '@app/api/worker/utils/AssetArchiveHelpers';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import {GUILD_TEXT_BASED_CHANNEL_TYPES} from '@fluxer/constants/src/ChannelConstants';
import {MAX_THREAD_MEMBERS, THREAD_CHANNEL_TYPES} from '@fluxer/constants/src/ThreadConstants';
import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';
import {z} from 'zod';

const CHANNEL_CONCURRENCY = 4;
const ASSET_CONCURRENCY = 8;
const ATTACHMENT_CONCURRENCY = 16;
const P_START = 5;
const P_META = 15;
const P_MESSAGES = 60;
const P_ASSETS = 68;
const P_ATTACHMENTS = 88;
const P_ZIP = 95;
const MESSAGE_BATCH_SIZE = 100;
const MESSAGE_LIMIT_PER_CHANNEL = 1000;
const PayloadSchema = z.object({
	guildId: z.string(),
	archiveId: z.string(),
	requestedBy: z.string(),
	includeAttachments: z.boolean().default(false),
});

interface PendingAttachmentDownload {
	channelId: ChannelID;
	attachmentId: AttachmentID;
	filename: string;
}

interface GuildHarvestAttachment {
	id: AttachmentID;
	filename: string;
	size: bigint;
	contentType: string;
	width: number | null;
	height: number | null;
}

const MAX_ARCHIVE_FILENAME_BYTES = 255;

function archiveAttachmentName(attachmentId: AttachmentID, filename: string): string {
	if (
		filename.length === 0 ||
		filename === '.' ||
		filename === '..' ||
		Buffer.byteLength(filename, 'utf8') > MAX_ARCHIVE_FILENAME_BYTES ||
		/[\0/\\]/.test(filename)
	) {
		return attachmentId.toString();
	}
	return filename;
}

function buildGuildHarvestAttachment(
	channelId: ChannelID,
	attachment: GuildHarvestAttachment,
	includeAttachments: boolean,
): Record<string, unknown> {
	return {
		attachment_id: attachment.id.toString(),
		filename: attachment.filename,
		size: attachment.size.toString(),
		content_type: attachment.contentType,
		archive_path: includeAttachments
			? `attachments/${channelId}/${attachment.id}/${archiveAttachmentName(attachment.id, attachment.filename)}`
			: null,
		cdn_url: makeDataPackageAttachmentCdnUrl(channelId, attachment.id, attachment.filename),
		width: attachment.width,
		height: attachment.height,
	};
}

function cdnBucket(): string {
	return Config.s3.buckets.cdn;
}

async function downloadToDisk(
	storageService: IStorageService,
	bucket: string,
	key: string,
	destPath: string,
): Promise<void> {
	try {
		await fs.promises.mkdir(path.dirname(destPath), {recursive: true});
		await storageService.writeObjectToDisk(bucket, key, destPath);
	} catch (error) {
		if (!(error instanceof Error) || (error.name !== 'NoSuchKey' && error.name !== 'NotFound')) throw error;
		Logger.warn({key}, 'Skipping missing S3 object during harvest');
	}
}

const harvestGuildData: ArchiveTaskHandler = async (payload, helpers, attempt) => {
	const validated = PayloadSchema.parse(payload);
	helpers.logger.debug({payload}, 'Processing harvestGuildData task');
	const guildId = createGuildID(BigInt(validated.guildId));
	const archiveId = BigInt(validated.archiveId);
	const guildIdStr = guildId.toString();
	const {guildRepository, channelRepository, adminArchiveRepository, storageService} = getWorkerDependencies();
	const existingArchive = await adminArchiveRepository.findBySubjectAndArchiveId('guild', guildId, archiveId);
	if (!existingArchive) throw new Error('Admin archive record not found for guild');
	throwIfArchiveTerminallyFailed(existingArchive);
	if (existingArchive.completedAt) {
		Logger.info({guildId, archiveId}, 'Guild archive already completed, skipping');
		return;
	}
	const adminArchive = await adminArchiveRepository.markAsStarted(existingArchive, 'Starting guild archive');
	const {attemptId, expiresAt} = adminArchive;
	assert(attemptId !== null && expiresAt !== null, 'Claimed guild archive is incomplete');
	let progressUpdate = Promise.resolve();
	const updateProgress = (percent: number, step: string): Promise<void> => {
		progressUpdate = progressUpdate.then(() => adminArchiveRepository.updateProgress(adminArchive, percent, step));
		return progressUpdate;
	};
	let storageKey: string;
	let fileSize: bigint;
	try {
		await using tmpDir = await fs.promises.mkdtempDisposable(path.join(os.tmpdir(), 'fluxer-guild-archive-'));
		const contentDir = path.join(tmpDir.path, 'content');
		const zipPath = path.join(tmpDir.path, 'archive.zip');
		await fs.promises.mkdir(contentDir);
		const guild = await guildRepository.findUnique(guildId);
		if (!guild) throw new Error(`Guild ${guildIdStr} not found`);
		await updateProgress(P_START, 'Collecting guild metadata');
		const [roles, members, channels, emojis, stickers] = await Promise.all([
			guildRepository.listRoles(guildId),
			guildRepository.listMembers(guildId),
			channelRepository.channelData
				.listGuildChannels(guildId, 'complete')
				.then(async (guildChannels) => [
					...guildChannels,
					...(await channelRepository.channelData.listChannels(
						await channelRepository.threads.listGuildThreadIds(guildId, {parents: guildChannels}),
					)),
				]),
			guildRepository.listEmojis(guildId),
			guildRepository.listStickers(guildId),
		]);
		await updateProgress(P_META, 'Writing guild metadata');
		const threadStates = await channelRepository.threads.getStates(
			channels.filter((c) => THREAD_CHANNEL_TYPES.has(c.type)).map((c) => c.id),
		);
		const threadMembers = await mapWithConcurrency(threadStates, CHANNEL_CONCURRENCY, (state) =>
			channelRepository.threads.listMembers(state.threadId, {limit: MAX_THREAD_MEMBERS}),
		);
		const guildJson = {
			guild: {
				id: guild.id.toString(),
				name: guild.name,
				owner_id: guild.ownerId.toString(),
				features: Array.from(guild.features),
				verification_level: guild.verificationLevel,
				default_message_notifications: guild.defaultMessageNotifications,
				explicit_content_filter: guild.explicitContentFilter,
				created_at: snowflakeToDate(guild.id).toISOString(),
			},
			roles: roles.map((r) => ({
				id: r.id.toString(),
				name: r.name,
				color: r.color,
				position: r.position,
				permissions: r.permissions.toString(),
				mentionable: r.isMentionable,
				hoist: r.isHoisted,
				icon: r.iconHash,
				unicode_emoji: r.unicodeEmoji,
			})),
			members: members.map((m) => ({
				user_id: m.userId.toString(),
				joined_at: m.joinedAt.toISOString(),
				nickname: m.nickname,
				role_ids: Array.from(m.roleIds).map((id) => id.toString()),
				avatar_hash: m.avatarHash,
				banner_hash: m.bannerHash,
			})),
			emojis: emojis.map((e) => ({
				id: e.id.toString(),
				name: e.name,
				animated: e.isAnimated,
				creator_id: e.creatorId.toString(),
			})),
			stickers: stickers.map((s) => ({
				id: s.id.toString(),
				name: s.name,
				description: s.description,
				animated: s.animated,
				tags: s.tags,
				creator_id: s.creatorId.toString(),
			})),
			channels: channels.map((c) => ({
				id: c.id.toString(),
				name: c.name,
				type: c.type,
				parent_id: c.parentId?.toString() ?? null,
				topic: c.topic,
				nsfw: c.isNsfw,
				position: c.position,
				last_message_id: c.lastMessageId?.toString() ?? null,
			})),
			...(threadStates.length > 0
				? {
						threads: threadStates.map((state, index) => ({
							id: state.threadId.toString(),
							parent_id: state.parentId.toString(),
							type: state.type,
							archived: state.archived,
							locked: state.locked,
							invitable: state.invitable,
							auto_archive_duration: state.autoArchiveDuration,
							archive_timestamp: state.archiveTimestamp?.toISOString() ?? null,
							created_at: state.createdAt.toISOString(),
							member_ids: threadMembers[index]!.map((member) => member.userId.toString()),
						})),
					}
				: {}),
		};
		await fs.promises.writeFile(path.join(contentDir, 'guild.json'), createArchiveJsonBuffer(guildJson));
		await updateProgress(P_META, `Harvesting messages from ${channels.length} channels`);
		const textChannels = channels.filter(
			(c) => GUILD_TEXT_BASED_CHANNEL_TYPES.has(c.type) || THREAD_CHANNEL_TYPES.has(c.type),
		);
		const pendingDownloads: Array<PendingAttachmentDownload> = [];
		let processedChannels = 0;
		await mapWithConcurrency(textChannels, CHANNEL_CONCURRENCY, async (channel) => {
			const messages: Array<object> = [];
			let beforeMessageId: MessageID | undefined;
			let channelDownloads: Array<PendingAttachmentDownload> = [];
			while (messages.length < MESSAGE_LIMIT_PER_CHANNEL) {
				const batch = await channelRepository.messages.listMessages(channel.id, beforeMessageId, MESSAGE_BATCH_SIZE);
				if (batch.length === 0) break;
				for (const msg of batch) {
					if (msg.authorId == null) continue;
					const attachments: Array<object> = [];
					for (const att of msg.attachments) {
						attachments.push(buildGuildHarvestAttachment(channel.id, att, validated.includeAttachments));
						if (validated.includeAttachments) {
							channelDownloads.push({
								channelId: channel.id,
								attachmentId: att.id,
								filename: att.filename,
							});
						}
					}
					messages.push({
						id: msg.id.toString(),
						author_id: msg.authorId.toString(),
						timestamp: snowflakeToDate(msg.id).toISOString(),
						content: msg.content ?? null,
						attachments,
					});
				}
				beforeMessageId = batch[batch.length - 1]!.id;
			}
			const chanDir = path.join(contentDir, 'channels', channel.id.toString());
			await fs.promises.mkdir(chanDir, {recursive: true});
			await fs.promises.writeFile(path.join(chanDir, 'messages.json'), createArchiveJsonBuffer(messages));
			pendingDownloads.push(...channelDownloads);
			channelDownloads = [];
			processedChannels++;
			const pct = P_META + Math.floor((processedChannels / Math.max(textChannels.length, 1)) * (P_MESSAGES - P_META));
			await updateProgress(pct, `Messages: ${processedChannels}/${textChannels.length} channels`);
		});
		await updateProgress(P_MESSAGES, 'Downloading guild assets');
		type AssetJob = {
			key: string;
			dest: string;
		};
		const assetJobs: Array<AssetJob> = [];
		for (const {hash, prefix, fileName} of [
			{hash: guild.iconHash, prefix: 'icons', fileName: 'icon'},
			{hash: guild.bannerHash, prefix: 'banners', fileName: 'banner'},
			{hash: guild.splashHash, prefix: 'splashes', fileName: 'splash'},
			{hash: guild.embedSplashHash, prefix: 'embed-splashes', fileName: 'embed-splash'},
		]) {
			if (!hash) continue;
			const ext = getAnimatedAssetExtension(hash);
			assetJobs.push({
				key: buildHashedAssetKey(prefix, guildIdStr, hash),
				dest: path.join(contentDir, 'assets', 'guild', `${fileName}.${ext}`),
			});
		}
		for (const role of roles) {
			if (!role.iconHash) continue;
			const id = role.id.toString();
			assetJobs.push({
				key: buildHashedAssetKey('role-icons', id, role.iconHash),
				dest: path.join(
					contentDir,
					'assets',
					'guild',
					'role-icons',
					`${id}.${getAnimatedAssetExtension(role.iconHash)}`,
				),
			});
		}
		for (const emoji of emojis) {
			const id = emoji.id.toString();
			assetJobs.push({
				key: `emojis/${id}`,
				dest: path.join(contentDir, 'assets', 'guild', 'emojis', `${id}.${emoji.isAnimated ? 'gif' : 'webp'}`),
			});
		}
		for (const sticker of stickers) {
			const id = sticker.id.toString();
			assetJobs.push({
				key: `stickers/${id}`,
				dest: path.join(contentDir, 'assets', 'guild', 'stickers', `${id}.${sticker.animated ? 'gif' : 'png'}`),
			});
		}
		await mapWithConcurrency(assetJobs, ASSET_CONCURRENCY, async ({key, dest}) => {
			await downloadToDisk(storageService, cdnBucket(), key, dest);
		});
		await updateProgress(P_ASSETS, `Downloading ${pendingDownloads.length} attachments`);
		if (pendingDownloads.length > 0) {
			let doneCount = 0;
			let lastPct = P_ASSETS;
			await mapWithConcurrency(pendingDownloads, ATTACHMENT_CONCURRENCY, async (dl) => {
				const storageKey = makeAttachmentCdnKey(dl.channelId, dl.attachmentId, dl.filename);
				const dest = path.join(
					contentDir,
					'attachments',
					dl.channelId.toString(),
					dl.attachmentId.toString(),
					archiveAttachmentName(dl.attachmentId, dl.filename),
				);
				await downloadToDisk(storageService, cdnBucket(), storageKey, dest);
				doneCount++;
				const newPct = P_ASSETS + Math.floor((doneCount / pendingDownloads.length) * (P_ATTACHMENTS - P_ASSETS));
				if (newPct > lastPct) {
					lastPct = newPct;
					await updateProgress(newPct, `Attachments: ${doneCount}/${pendingDownloads.length}`);
				}
			});
		}
		await updateProgress(P_ATTACHMENTS, 'Creating archive');
		await writeZipArchive(zipPath, (archive) => archive.directory(contentDir));
		await updateProgress(P_ZIP, 'Uploading archive');
		storageKey = `archives/guilds/${guildId}/${archiveId}/${attemptId}/guild-archive.zip`;
		const zipStat = await fs.promises.stat(zipPath);
		fileSize = BigInt(zipStat.size);
		await storageService.uploadObjectFromFile({
			bucket: Config.s3.buckets.harvests,
			key: storageKey,
			filePath: zipPath,
			contentLength: zipStat.size,
			contentType: 'application/zip',
			expiresAt,
		});
	} catch (error) {
		if (error instanceof ArchiveAttemptSupersededError) throw error;
		Logger.error({error, guildId, archiveId}, 'Failed to harvest guild data');
		const message = error instanceof Error ? error.message : String(error);
		try {
			if (attempt.isLastAttempt) {
				await adminArchiveRepository.markAsTerminallyFailed(adminArchive, message);
			} else {
				await adminArchiveRepository.markAsFailed(adminArchive, message);
			}
		} catch (failureError) {
			if (failureError instanceof ArchiveAttemptSupersededError) throw failureError;
			throw new AggregateError([error, failureError], 'Failed to prepare guild archive and record its failure');
		}
		if (attempt.isLastAttempt) throw new ArchiveTerminalFailureError(message);
		throw error;
	}
	await adminArchiveRepository.markAsCompleted(adminArchive, storageKey, fileSize, expiresAt);
};

export default createArchiveTask(harvestGuildData);