Node.js based forum software built for the modern web
feat: Third-party blocklists (#14115)
Julian Lam4b7be68d1bd7
diff --git a/install/package.json b/install/package.jsonindex 43d32072b5..bd522d6551 100644--- a/install/package.json+++ b/install/package.json@@ -65,6 +65,7 @@"cronstrue": "3.14.0","cropperjs": "1.6.2","csrf-sync": "4.2.1",+ "csv-parse": "6.2.1","daemon": "1.1.0","diff": "8.0.4","esbuild": "0.27.4",diff --git a/public/language/en-GB/admin/settings/activitypub.json b/public/language/en-GB/admin/settings/activitypub.jsonindex 677af75cd7..6c65323ce5 100644--- a/public/language/en-GB/admin/settings/activitypub.json+++ b/public/language/en-GB/admin/settings/activitypub.json@@ -41,6 +41,17 @@"relays.state-2": "Active","relays.errors.invalid-url": "Please enter a valid URL",+ "blocklists": "Third-party Blocklists",+ "blocklists-help": "For the safety of you and your users, maintaining a blocklist is an essential part of any trust & safety effort when working with content from outside the local moderation scope. NodeBB ships with some recommended defaults, and these can be customized here.",+ "blocklists-default": "NodeBB ships with two blocklists by default, the <a href=\"https://about.iftas.org/library/iftas-dni-list/\">IFTAS Do Not Interact Denylist</a> and the <a href=\"https://about.iftas.org/library/iftas-abandoned-and-unmanaged-domain-list/\">IFTAS Abandoned and Unmanaged Domain Denylist</a>.",+ "blocklists.url": "URL",+ "blocklists.count": "Domains",+ "blocklists.add": "Add New Blocklist",+ "blocklists.add-help": "Enter the URL of the blocklist you would like to add. NodeBB understands blocklists that match the format used by <a href=\"https://iftas.org\">IFTAS</a>.",+ "blocklists.refreshed": "Blocklist refreshed — it now contains %1 entries",+ "blocklists.view.title": "View Blocklist",+ "blocklists.view.intro": "These are the %1 domain(s) blocked by this blocklist:",+"server-filtering": "Filtering","count": "This NodeBB is currently aware of <strong>%1</strong> server(s)","server.filter-help": "Specify servers you would like to bar from federating with your NodeBB. Alternatively, you may opt to selectively <em>allow</em> federation with specific servers, instead. Both options are supported, although they are mutually exclusive.",diff --git a/public/openapi/components/schemas/admin/blocklists.yaml b/public/openapi/components/schemas/admin/blocklists.yamlnew file mode 100644index 0000000000..75c4555432--- /dev/null+++ b/public/openapi/components/schemas/admin/blocklists.yaml@@ -0,0 +1,14 @@+BlocklistObject:+ type: object+ properties:+ url:+ type: string+ description: The blocklist URL+ example: https://example.org/blocklist+ count:+ type: number+ description: The number of blocked domains on the list+BlocklistArray:+ type: array+ items:+ $ref: '#/BlocklistObject'\ No newline at end of filediff --git a/public/openapi/read/admin/federation/safety.yaml b/public/openapi/read/admin/federation/safety.yamlindex 73015d3f8c..2ddd916eb7 100644--- a/public/openapi/read/admin/federation/safety.yaml+++ b/public/openapi/read/admin/federation/safety.yaml@@ -13,6 +13,8 @@ get:properties:title:type: string+ blocklists:+ $ref: ../../../components/schemas/admin/blocklists.yaml#/BlocklistArrayinstanceCount:type: number- $ref: ../../../components/schemas/CommonProps.yaml#/CommonProps\ No newline at end of filediff --git a/public/openapi/write.yaml b/public/openapi/write.yamlindex 0741c799ed..e03ce4f122 100644--- a/public/openapi/write.yaml+++ b/public/openapi/write.yaml@@ -286,6 +286,12 @@ paths:$ref: 'write/admin/activitypub/relays.yaml'/admin/activitypub/relays/{url}:$ref: 'write/admin/activitypub/relays/url.yaml'+ /admin/activitypub/blocklists:+ $ref: 'write/admin/activitypub/blocklists.yaml'+ /admin/activitypub/blocklists/{url}:+ $ref: 'write/admin/activitypub/blocklists/url.yaml'+ /admin/activitypub/blocklists/{url}/refresh:+ $ref: 'write/admin/activitypub/blocklists/url/refresh.yaml'/files/:$ref: 'write/files.yaml'/files/folder:diff --git a/public/openapi/write/admin/activitypub/blocklists.yaml b/public/openapi/write/admin/activitypub/blocklists.yamlnew file mode 100644index 0000000000..00bee2d995--- /dev/null+++ b/public/openapi/write/admin/activitypub/blocklists.yaml@@ -0,0 +1,28 @@+post:+ tags:+ - admin+ summary: add blocklist+ description: This operation add a new blocklist+ requestBody:+ required: true+ content:+ application/json:+ schema:+ type: object+ properties:+ url:+ type: string+ description: The blocklist url+ example: https://example.org/blocklist+ responses:+ '200':+ description: blocklist successfully added+ content:+ application/json:+ schema:+ type: object+ properties:+ status:+ $ref: ../../../components/schemas/Status.yaml#/Status+ response:+ $ref: ../../../components/schemas/admin/blocklists.yaml#/BlocklistArraydiff --git a/public/openapi/write/admin/activitypub/blocklists/url.yaml b/public/openapi/write/admin/activitypub/blocklists/url.yamlnew file mode 100644index 0000000000..59b6bab90e--- /dev/null+++ b/public/openapi/write/admin/activitypub/blocklists/url.yaml@@ -0,0 +1,55 @@+get:+ tags:+ - admin+ summary: view blocklist+ description: This operation retrieves the domains in a specific blocklist+ parameters:+ - name: url+ in: path+ required: true+ schema:+ type: string+ example: https%3A%2F%2Fexample.org%2Fblocklist+ responses:+ '200':+ description: blocklist successfully retrieved+ content:+ application/json:+ schema:+ type: object+ properties:+ status:+ $ref: ../../../../components/schemas/Status.yaml#/Status+ response:+ type: object+ properties:+ domains:+ type: array+ items:+ type: string+ count:+ type: number+delete:+ tags:+ - admin+ summary: delete blocklist+ description: This operation removes a previously-added blocklist+ parameters:+ - in: path+ name: url+ schema:+ type: string+ required: true+ example: https%3A%2F%2Fexample.org%2Fblocklist+ responses:+ '200':+ description: blocklist successfully deleted+ content:+ application/json:+ schema:+ type: object+ properties:+ status:+ $ref: ../../../../components/schemas/Status.yaml#/Status+ response:+ $ref: ../../../../components/schemas/admin/blocklists.yaml#/BlocklistArray\ No newline at end of filediff --git a/public/openapi/write/admin/activitypub/blocklists/url/refresh.yaml b/public/openapi/write/admin/activitypub/blocklists/url/refresh.yamlnew file mode 100644index 0000000000..205fa5b3b1--- /dev/null+++ b/public/openapi/write/admin/activitypub/blocklists/url/refresh.yaml@@ -0,0 +1,30 @@+post:+ tags:+ - admin+ summary: refresh blocklist+ description: This operation refreshes the blocklist domains+ parameters:+ - name: url+ in: path+ required: true+ schema:+ type: string+ example: https%3A%2F%2Fexample.org%2Fblocklist+ responses:+ '200':+ description: blocklist successfully refreshed+ content:+ application/json:+ schema:+ type: object+ properties:+ status:+ $ref: ../../../../../components/schemas/Status.yaml#/Status+ response:+ type: object+ properties:+ blocklists:+ $ref: ../../../../../components/schemas/admin/blocklists.yaml#/BlocklistArray+ count:+ type: number+ description: The number of domains managed by this blocklistdiff --git a/public/src/admin/federation/safety.js b/public/src/admin/federation/safety.jsnew file mode 100644index 0000000000..f32da91904--- /dev/null+++ b/public/src/admin/federation/safety.js@@ -0,0 +1,113 @@+'use strict';++import { get, post, del } from 'api';+import { error } from 'alerts';+import { render } from 'benchpress';+import { alert, dialog } from 'bootbox';++export function init() {+ setupBlocklists();+};++function setupBlocklists() {+ const blocklistsEl = document.getElementById('blocklists');+ if (blocklistsEl) {+ blocklistsEl.addEventListener('click', (e) => {+ const subselector = e.target.closest('[data-action]');+ if (subselector) {+ const url = subselector.closest('tr').getAttribute('data-url');+ const action = subselector.getAttribute('data-action');+ switch (action) {+ case 'blocklists.add': {+ throwModal();+ break;+ }++ case 'blocklists.refresh': {+ refresh(url);+ break;+ }++ case 'blocklists.view': {+ view(url);+ break;+ }++ case 'blocklists.remove': {+ del(`/admin/activitypub/blocklists/${encodeURIComponent(url)}`, {})+ .then(renderList)+ .catch(error);+ }+ }+ }+ });+ }+}++async function renderList(blocklists) {+ const html = await app.parseAndTranslate('admin/federation/safety', 'blocklists', { blocklists });+ const tbodyEl = document.querySelector('#blocklists tbody');+ if (tbodyEl) {+ $(tbodyEl).html(html);+ }+}++function throwModal() {+ render('admin/partials/activitypub/blocklists', {}).then(function (html) {+ const submit = function () {+ const formEl = modal.find('form').get(0);+ if (!formEl.reportValidity()) {+ return false;+ }++ const payload = Object.fromEntries(new FormData(formEl));+ post('/admin/activitypub/blocklists', payload).then(async (data) => {+ const html = await app.parseAndTranslate('admin/federation/safety', 'blocklists', { blocklists: data });+ const tbodyEl = document.querySelector('#blocklists tbody');+ if (tbodyEl) {+ $(tbodyEl).html(html);+ }+ }).catch(error);+ };+ const modal = bootbox.dialog({+ title: '[[admin/settings/activitypub:blocklists.add]]',+ message: html,+ buttons: {+ save: {+ label: '[[global:save]]',+ className: 'btn-primary',+ callback: submit,+ },+ },+ });++ modal.on('shown.bs.modal', function () {+ modal.find('input').focus();+ });+ });+}++async function refresh(url) {+ try {+ const data = await post(`/admin/activitypub/blocklists/${encodeURIComponent(url)}/refresh`);+ alert(`[[admin/settings/activitypub:blocklists.refreshed, ${data.count}]]`);+ renderList(data.blocklists);+ } catch (e) {+ error(e);+ }+}++async function view(url) {+ console.log('ehjre');+ try {+ const { domains, count } = await get(`/admin/activitypub/blocklists/${encodeURIComponent(url)}`);+ dialog({+ title: '[[admin/settings/activitypub:blocklists.view.title]]',+ message: `\+ <p>[[admin/settings/activitypub:blocklists.view.intro, ${count}]]</p> \+ <ul>` + domains.map(domain => `<li>${domain}</li>`).join('\n') + '</ul>',+ });+ } catch (e) {+ error(e);+ }+}\ No newline at end of filediff --git a/src/activitypub/blocklists.js b/src/activitypub/blocklists.jsnew file mode 100644index 0000000000..86424d7856--- /dev/null+++ b/src/activitypub/blocklists.js@@ -0,0 +1,78 @@+'use strict';++const { parse } = require('csv-parse/sync');++const db = require('../database');+const request = require('../request');+const activitypub = module.parent.exports;++const Blocklists = module.exports;++Blocklists.list = async () => {+ const blocklists = await db.getSortedSetMembers('blocklists');+ const counts = await db.sortedSetsCard(blocklists.map(blocklist => `blocklist:${blocklist}`));++ return blocklists.map((url, idx) => {+ return { url, count: counts[idx] };+ });+};++Blocklists.get = async (url) => {+ const domains = await db.getSortedSetMembers(`blocklist:${url}`);++ return {+ domains,+ count: domains.length,+ };+};++Blocklists.add = async (url) => {+ const now = Date.now();++ await Promise.all([+ db.sortedSetAdd('blocklists', now, url),+ Blocklists.refresh(url),+ ]);+};++Blocklists.remove = async (url) => {+ await Promise.all([+ db.sortedSetRemove('blocklists', url),+ db.delete(`blocklist:${url}`),+ ]);+};++Blocklists.refresh = async (url) => {+ activitypub.helpers.log(`[blocklists/refresh] Processing ${url}`);++ const { body: csv } = await request.get(url);+ let records;+ try {+ records = parse(csv, {+ columns: true,+ skip_empty_lines: true,+ trim: true,+ });+ if (!records.length) {+ return 0;+ }+ } catch (e) {+ return 0;+ }++ await db.sortedSetAdd(+ `blocklist:${url}`,+ records.map(entry => entry['#severity'] === 'silence' ? 2 : 1),+ records.map(entry => entry['#domain'])+ );++ return records.length;+};++Blocklists.check = async (domain) => {+ const blocklists = await Blocklists.list();+ let present = await db.isMemberOfSortedSets(blocklists.map(({ url }) => `blocklist:${url}`), domain);+ present = present.reduce((memo, present) => memo || present, false);++ return !present;+};\ No newline at end of filediff --git a/src/activitypub/index.js b/src/activitypub/index.jsindex 37750664fe..d2f299ecfc 100644--- a/src/activitypub/index.js+++ b/src/activitypub/index.js@@ -64,6 +64,7 @@ ActivityPub.notes = require('./notes');ActivityPub.contexts = require('./contexts');ActivityPub.actors = require('./actors');ActivityPub.instances = require('./instances');+ActivityPub.blocklists = require('./blocklists');ActivityPub.feps = require('./feps');ActivityPub.rules = require('./rules');ActivityPub.relays = require('./relays');@@ -134,11 +135,11 @@ ActivityPub.resolveInboxes = async (ids) => {// Filter out blocked instancesconst blocked = [];- inboxArr = inboxArr.filter((inbox) => {+ const allowed = await Promise.all(inboxArr.map(async (inbox) => {let allowed = false;try {const { hostname } = new URL(inbox);- allowed = ActivityPub.instances.isAllowed(hostname);+ allowed = await ActivityPub.instances.isAllowed(hostname);if (!allowed) {blocked.push(inbox);}@@ -147,7 +148,8 @@ ActivityPub.resolveInboxes = async (ids) => {}return allowed;- });+ }));+ inboxArr = inboxArr.filter((_, idx) => allowed[idx]);if (blocked.length) {ActivityPub.helpers.log(`[activitypub/resolveInboxes] Not delivering to blocked instances: ${blocked.join(', ')}`);}@@ -306,7 +308,7 @@ ActivityPub.get = async (type, id, uri, options) => {}const { hostname } = new URL(uri);- const allowed = ActivityPub.instances.isAllowed(hostname);+ const allowed = await ActivityPub.instances.isAllowed(hostname);if (!allowed) {ActivityPub.helpers.log(`[activitypub/get] Not retrieving ${uri}, domain is blocked.`);const e = new Error(`[[error:activitypub.get-failed]]`);diff --git a/src/activitypub/instances.js b/src/activitypub/instances.jsindex 64502c0998..4302c17f48 100644--- a/src/activitypub/instances.js+++ b/src/activitypub/instances.js@@ -2,6 +2,7 @@const meta = require('../meta');const db = require('../database');+const activitypub = module.parent.exports;const Instances = module.exports;@@ -11,9 +12,15 @@ Instances.log = async (domain) => {Instances.getCount = async () => db.sortedSetCard('instances:lastSeen');-Instances.isAllowed = (domain) => {+Instances.isAllowed = async (domain) => {+ const allowed = await activitypub.blocklists.check(domain);let { activitypubFilter: type, activitypubFilterList: list } = meta.config;++ if (!allowed && !type) {+ return allowed;+ }+list = new Set(String(list).split('\n'));// eslint-disable-next-line no-bitwise- return list.has(domain) ^ !type;+ return allowed || (list.has(domain) ^ !type);};diff --git a/src/activitypub/jobs.js b/src/activitypub/jobs.jsindex ca403a8045..a8544f4c02 100644--- a/src/activitypub/jobs.js+++ b/src/activitypub/jobs.js@@ -50,6 +50,18 @@ Jobs.start = async () => {runOnInit: false,onTick: async () => await tryCronJob(backfill),});++ await cron.addJob({+ name: 'ap:blocklist:refresh',+ cronTime: '15 0 * * *',+ runOnInit: false,+ onTick: async () => {+ const lists = await activitypub.blocklists.list();+ await Promise.all(lists.map(({ url }) => {+ return activitypub.blocklists.refresh(url);+ }));+ },+ });};async function retryFailedMessages() {diff --git a/src/controllers/admin/federation.js b/src/controllers/admin/federation.jsindex 21ed25885f..d089d6d509 100644--- a/src/controllers/admin/federation.js+++ b/src/controllers/admin/federation.js@@ -44,9 +44,11 @@ federationController.pruning = function (req, res) {federationController.safety = async function (req, res) {const instanceCount = await activitypub.instances.getCount();+ const blocklists = await activitypub.blocklists.list();res.render(`admin/federation/safety`, {title: '[[admin/menu:federation/safety]]',+ blocklists,instanceCount,});};diff --git a/src/controllers/write/admin.js b/src/controllers/write/admin.jsindex 65621bc853..105b7f5247 100644--- a/src/controllers/write/admin.js+++ b/src/controllers/write/admin.js@@ -126,3 +126,36 @@ Admin.activitypub.removeRelay = async (req, res) => {await activitypub.relays.remove(url);helpers.formatApiResponse(200, res, await activitypub.relays.list());};+++Admin.activitypub.addBlocklist = async (req, res, next) => {+ const { url } = req.body;+ if (!url) {+ return next();+ }++ await activitypub.blocklists.add(url);+ helpers.formatApiResponse(200, res, await activitypub.blocklists.list());+};++Admin.activitypub.viewBlocklist = async (req, res) => {+ const { url } = req.params;++ helpers.formatApiResponse(200, res, await activitypub.blocklists.get(url));+};++Admin.activitypub.removeBlocklist = async (req, res) => {+ const { url } = req.params;++ await activitypub.blocklists.remove(url);+ helpers.formatApiResponse(200, res, await activitypub.blocklists.list());+};++Admin.activitypub.refreshBlocklist = async (req, res) => {+ const { url } = req.params;++ const count = await activitypub.blocklists.refresh(url);+ const blocklists = await activitypub.blocklists.list();++ helpers.formatApiResponse(200, res, { blocklists, count });+};diff --git a/src/install.js b/src/install.jsindex c360868521..9724063df4 100644--- a/src/install.js+++ b/src/install.js@@ -244,6 +244,12 @@ async function setupDefaultConfigs() {await meta.configs.setOnEmpty(defaults);await meta.configs.init();++ const db = require('./database');+ await db.sortedSetAdd('blocklists', [Date.now(), Date.now()], [+ 'https://about.iftas.org/wp-content/uploads/2025/10/iftas-dni-latest.csv',+ 'https://about.iftas.org/wp-content/uploads/2025/10/iftas-abandoned-unmanaged-latest.csv',+ ]);}async function enableDefaultTheme() {diff --git a/src/middleware/activitypub.js b/src/middleware/activitypub.jsindex 9ea81c0035..0a8a89324d 100644--- a/src/middleware/activitypub.js+++ b/src/middleware/activitypub.js@@ -105,7 +105,7 @@ middleware.assertPayload = helpers.try(async function (req, res, next) {// Domain checkconst { hostname } = new URL(actor);- const allowed = activitypub.instances.isAllowed(hostname);+ const allowed = await activitypub.instances.isAllowed(hostname);if (!allowed) {activitypub.helpers.log(`[middleware/activitypub] Blocked incoming activity from ${hostname}.`);return res.sendStatus(403);diff --git a/src/routes/write/admin.js b/src/routes/write/admin.jsindex 2121953efe..caac9e9b3e 100644--- a/src/routes/write/admin.js+++ b/src/routes/write/admin.js@@ -30,6 +30,10 @@ module.exports = function () {setupApiRoute(router, 'put', '/activitypub/rules/order', [...middlewares, middleware.checkRequired.bind(null, ['rids'])], controllers.write.admin.activitypub.reorderRules);setupApiRoute(router, 'post', '/activitypub/relays', [...middlewares, middleware.checkRequired.bind(null, ['url'])], controllers.write.admin.activitypub.addRelay);setupApiRoute(router, 'delete', '/activitypub/relays/:url', [...middlewares], controllers.write.admin.activitypub.removeRelay);+ setupApiRoute(router, 'post', '/activitypub/blocklists', [...middlewares, middleware.checkRequired.bind(null, ['url'])], controllers.write.admin.activitypub.addBlocklist);+ setupApiRoute(router, 'get', '/activitypub/blocklists/:url', [...middlewares], controllers.write.admin.activitypub.viewBlocklist);+ setupApiRoute(router, 'delete', '/activitypub/blocklists/:url', [...middlewares], controllers.write.admin.activitypub.removeBlocklist);+ setupApiRoute(router, 'post', '/activitypub/blocklists/:url/refresh', [...middlewares], controllers.write.admin.activitypub.refreshBlocklist);return router;};diff --git a/src/upgrades/4.11.0/subscribe-to-default-blocklists.js b/src/upgrades/4.11.0/subscribe-to-default-blocklists.jsnew file mode 100644index 0000000000..feb0af76c7--- /dev/null+++ b/src/upgrades/4.11.0/subscribe-to-default-blocklists.js@@ -0,0 +1,14 @@+'use strict';++const activitypub = require('../../activitypub');++module.exports = {+ name: 'Subscribe to IFTAS DNI and AUD denylists',+ timestamp: Date.UTC(2026, 2, 23),+ method: async () => {+ await Promise.all([+ activitypub.blocklists.add('https://about.iftas.org/wp-content/uploads/2025/10/iftas-dni-latest.csv'),+ activitypub.blocklists.add('https://about.iftas.org/wp-content/uploads/2025/10/iftas-abandoned-unmanaged-latest.csv'),+ ]);+ },+};diff --git a/src/views/admin/federation/safety.tpl b/src/views/admin/federation/safety.tplindex 9c92085732..b7fe38c49d 100644--- a/src/views/admin/federation/safety.tpl+++ b/src/views/admin/federation/safety.tpl@@ -3,6 +3,45 @@<div class="row settings m-0"><div id="spy-container" class="col-12 col-md-8 px-0 mb-4" tabindex="0">+ <div id="denylists" class="mb-4">+ <h5 class="fw-bold tracking-tight settings-header">[[admin/settings/activitypub:blocklists]]</h5>+ <div class="mb-3">+ <p>[[admin/settings/activitypub:blocklists-help]]</p>+ <p class="form-text">[[admin/settings/activitypub:blocklists-default]]</p>+ <div class="mb-3 table-responsive-md">+ <table class="table table-striped" id="blocklists">+ <thead>+ <th>[[admin/settings/activitypub:blocklists.url]]</th>+ <th>[[admin/settings/activitypub:blocklists.count]]</th>+ <th></th>+ </thead>+ <tbody>+ {{{ each blocklists }}}+ <tr data-url="{./url}">+ <td>{./url}</td>+ <td>{./count}</td>+ <td>+ <div class="d-flex gap-3">+ <a href="#" data-action="blocklists.view"><i class="fa fa-list"></i></a>+ <a href="#" data-action="blocklists.refresh"><i class="fa fa-refresh"></i></a>+ <a href="#" data-action="blocklists.remove"><i class="fa fa-trash link-danger"></i></a>+ </div>+ </td>+ </tr>+ {{{ end }}}+ </tbody>+ <tfoot>+ <tr>+ <td colspan="3">+ <button class="btn btn-sm btn-primary" data-action="blocklists.add">[[admin/settings/activitypub:blocklists.add]]</button>+ </td>+ </tr>+ </tfoot>+ </table>+ </div>+ </div>+ </div>+<div id="server-filtering" class="mb-4"><h5 class="fw-bold tracking-tight settings-header">[[admin/settings/activitypub:server-filtering]]</h5><form>diff --git a/src/views/admin/partials/activitypub/blocklists.tpl b/src/views/admin/partials/activitypub/blocklists.tplnew file mode 100644index 0000000000..76ad526397--- /dev/null+++ b/src/views/admin/partials/activitypub/blocklists.tpl@@ -0,0 +1,10 @@+<p>[[admin/settings/activitypub:blocklists.add-help]]</p>++<hr />++<form role="form">+ <div class="mb-3">+ <label class="form-label" for="url">Blocklist URL</label>+ <input type="url" id="url" name="url" title="Blocklist URL" class="form-control" placeholder="https://example.org/blocklist.csv" required>+ </div>+</form>\ No newline at end of file