Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
3.0 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import {Config} from '@app/api/Config';
import {
	BANNED_FILE_SHAS_REFRESH_CHANNEL,
	ContentBlocklistCategory,
	isBlocklistFeedFileSha,
} from '@app/api/constants/ContentModeration';
import {EXTERNAL_RESPONSE_LIMITS} from '@app/api/utils/ExternalResponseLimits';
import * as FetchUtils from '@app/api/utils/FetchUtils';
import {getWorkerDependencies} from '@app/api/worker/WorkerContext';
import type {WorkerTaskHandler, WorkerTaskHelpers} from '@pkgs/worker/src/contracts/WorkerTask';

const MALWARE_BAZAAR_SHA256_URL = 'https://bazaar.abuse.ch/export/txt/sha256/recent/';
const SHA256_RE = /^[0-9a-fA-F]{64}$/;

async function removeFeedFileShas(helpers: WorkerTaskHelpers): Promise<void> {
	const {adminRepository, kvClient} = getWorkerDependencies();
	let removed = 0;
	for (const row of await adminRepository.loadAllBannedFileShas()) {
		if (!isBlocklistFeedFileSha(row)) continue;
		if (await adminRepository.unbanFeedFileSha(row.sha256_hex)) removed++;
	}
	if (removed > 0) {
		await kvClient.publish(BANNED_FILE_SHAS_REFRESH_CHANNEL, 'refresh');
	}
	helpers.logger.info({removed}, 'Removed file-SHA blocklist feed rows');
}

const syncFileShaBlocklists: WorkerTaskHandler = async (_payload, helpers) => {
	helpers.logger.info('Starting file-SHA blocklist sync');
	await helpers.setContextLink('/file-sha-bans');
	if (!Config.blocklistFeeds.enabled) {
		await removeFeedFileShas(helpers);
		return;
	}
	const {adminRepository, kvClient} = getWorkerDependencies();
	let added = 0;
	try {
		const res = await fetch(MALWARE_BAZAAR_SHA256_URL, {signal: AbortSignal.timeout(120000)});
		if (!res.ok) {
			FetchUtils.discardResponseBody(res.body, res.status);
			throw new Error(`HTTP ${res.status}`);
		}
		const text = await FetchUtils.streamToStringWithLimit(res.body, {
			maxBytes: EXTERNAL_RESPONSE_LIMITS.fileBlocklistBytes,
			headers: res.headers,
			url: res.url,
			description: 'MalwareBazaar SHA-256 feed',
		});
		const freshHashes = new Set<string>();
		for (const line of text.split('\n')) {
			const l = line.trim().toLowerCase();
			if (!l || l.startsWith('#') || l.startsWith('"')) continue;
			if (SHA256_RE.test(l)) freshHashes.add(l);
		}
		helpers.logger.info({count: freshHashes.size}, 'Fetched MalwareBazaar SHA-256 hashes');
		const existing = await adminRepository.loadAllBannedFileShas();
		const existingSet = new Set(existing.map((r) => r.sha256_hex));
		for (const sha of freshHashes) {
			if (existingSet.has(sha)) continue;
			await adminRepository.banFileSha({
				sha256_hex: sha,
				category: ContentBlocklistCategory.MALWARE_BAZAAR,
				severity: 2,
				content_type: null,
				source_url: MALWARE_BAZAAR_SHA256_URL,
				added_at: new Date(),
				added_by: null,
				notes: null,
			});
			added++;
		}
	} catch (err) {
		helpers.logger.warn({error: err}, 'Failed to fetch MalwareBazaar SHA-256 feed');
	}
	if (added > 0) {
		await kvClient.publish(BANNED_FILE_SHAS_REFRESH_CHANNEL, 'refresh');
	}
	helpers.logger.info({added}, 'File-SHA blocklist sync complete');
};

export default syncFileShaBlocklists;