A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
import {AdminRepository} from '@app/api/admin/AdminRepository';
import {
BLOCKLIST_FEED_CATEGORIES,
COMBINED_FEED_URLS_KEY,
feedUrlsKey,
readLinesFromS3,
} from '@app/api/blocklist/BlocklistS3';
import {Config} from '@app/api/Config';
import {BANNED_URL_DOMAINS_REFRESH_CHANNEL, BANNED_URLS_REFRESH_CHANNEL} from '@app/api/constants/ContentModeration';
import type {IStorageService} from '@app/api/infrastructure/IStorageService';
import {Logger} from '@app/api/Logger';
import {RefreshedCache} from '@app/api/utils/RefreshedCache';
import {UrlHostRuleSet} from '@app/api/utils/UrlHostRules';
import {canonicalizeUrl, extractLinkHosts, extractUrlCandidates} from '@app/api/utils/UrlNormalizer';
class UrlBlocklistCache extends RefreshedCache {
constructor() {
super({name: 'URL blocklist cache', channels: [BANNED_URLS_REFRESH_CHANNEL, BANNED_URL_DOMAINS_REFRESH_CHANNEL]});
}
private exactUrls: Set<string> = new Set();
private feedUrls: Set<string> = new Set();
private hostRules = new UrlHostRuleSet();
private adminRepository = new AdminRepository();
private storageService: IStorageService | null = null;
setStorageService(storageService: IStorageService | null): void {
this.storageService = storageService;
}
protected async load(): Promise<void> {
const [manualUrls, domains, feedUrls] = await Promise.all([
this.adminRepository.loadAllBannedUrls(),
this.adminRepository.loadAllBannedUrlDomains(),
this.loadFeedUrls(),
]);
const nextUrls = new Set(feedUrls);
for (const row of manualUrls) {
if (row.url_canonical) nextUrls.add(row.url_canonical.toLowerCase());
}
const nextHostRules = new UrlHostRuleSet();
for (const row of domains) {
nextHostRules.add(row.domain, row.match_subdomains ?? true);
}
this.exactUrls = nextUrls;
this.hostRules = nextHostRules;
Logger.debug(
{urls: nextUrls.size, ...nextHostRules.size, feedUrls: feedUrls.size},
'URL blocklist cache refreshed',
);
}
private async loadFeedUrls(): Promise<Set<string>> {
if (!this.storageService || !Config.blocklistFeeds.enabled) {
this.feedUrls = new Set();
return this.feedUrls;
}
const storageService = this.storageService;
const keys = [...BLOCKLIST_FEED_CATEGORIES.map(feedUrlsKey), COMBINED_FEED_URLS_KEY];
try {
const lineSets = await Promise.all(keys.map((key) => readLinesFromS3(storageService, key)));
this.feedUrls = new Set(lineSets.flat());
} catch (error) {
Logger.warn({error}, 'Failed to read URL blocklist feed files, keeping the previous feed URLs');
}
return this.feedUrls;
}
isUrlBanned(rawUrl: string): boolean {
const canonical = canonicalizeUrl(rawUrl);
if (!canonical) return false;
return this.exactUrls.has(canonical);
}
isUrlOrDomainBanned(rawUrl: string): boolean {
const canonical = canonicalizeUrl(rawUrl);
if (!canonical) return false;
if (this.exactUrls.has(canonical)) return true;
let host: string;
try {
host = new URL(canonical).hostname;
} catch {
return false;
}
return this.isHostnameBanned(host);
}
isHostnameBanned(host: string): boolean {
return this.hostRules.matches(host);
}
containsBannedLink(text: string): boolean {
for (const url of extractUrlCandidates(text)) {
if (this.isUrlOrDomainBanned(url)) return true;
}
for (const host of extractLinkHosts(text)) {
if (this.isHostnameBanned(host)) return true;
}
return false;
}
addExactUrl(canonical: string): void {
this.exactUrls.add(canonical.toLowerCase());
}
removeExactUrl(canonical: string): void {
this.exactUrls.delete(canonical.toLowerCase());
}
addDomain(domain: string, matchSubdomains = true): void {
this.hostRules.add(domain, matchSubdomains);
}
removeDomain(domain: string): void {
this.hostRules.remove(domain);
}
get size(): {
urls: number;
domains: number;
patterns: number;
} {
return {
urls: this.exactUrls.size,
...this.hostRules.size,
};
}
resetForTesting(): void {
this.resetRefreshState();
this.exactUrls = new Set();
this.feedUrls = new Set();
this.hostRules = new UrlHostRuleSet();
this.storageService = null;
}
}
export const urlBlocklistCache = new UrlBlocklistCache();