Node.js based forum software built for the modern web
feat(activitypub): hostname-aware custom emoji cache and proxy
Julian Lamf7376eab0405
diff --git a/src/activitypub/emoji.js b/src/activitypub/emoji.jsnew file mode 100644index 0000000000..bf7bece273--- /dev/null+++ b/src/activitypub/emoji.js@@ -0,0 +1,211 @@+'use strict';++const path = require('path');+const fs = require('fs').promises;+const nconf = require('nconf');+const mime = require('mime').default;+const winston = require('winston');++const db = require('../database');+const validator = require('validator');+const { check } = require('../ssrf');++const emojiLookupKey = 'emoji:ap:lookup';++function getEmojiDir() {+ const uploadPath = nconf.get('upload_path') || nconf.get('base_dir') || '.';+ return path.join(uploadPath, 'emoji', 'ap');+}++/**+ * Extract hostname from an emoji's icon URL or id.+ */+function extractHostname(icon) {+ if (!icon || !icon.url) {+ return null;+ }+ try {+ return new URL(icon.url).hostname;+ } catch {+ return null;+ }+}++/**+ * Build the Redis field key for a shortcode+hostname pair.+ */+function buildFieldKey(shortcode, hostname) {+ // Strip leading/trailing colons from shortcode for the key+ const clean = shortcode.replace(/^:+|:+$/g, '');+ return `${clean}:${hostname}`;+}++/**+ * Normalize a shortcode to :name: format.+ */+function normalizeShortcode(name) {+ let short = name;+ if (!short.startsWith(':')) {+ short = `:${short}`;+ }+ if (!short.endsWith(':')) {+ short = `${short}:`;+ }+ return short;+}++/**+ * Lookup a cached emoji by shortcode and hostname.+ * Returns { name, remoteUrl, localPath, mediaType } or null.+ */+async function getEmoji(shortcode, hostname) {+ const fieldKey = buildFieldKey(shortcode, hostname);+ const raw = await db.getObjectField(emojiLookupKey, fieldKey);+ if (!raw) {+ return null;+ }+ try {+ const metadata = JSON.parse(raw);+ // Resolve relative path to absolute+ if (metadata.localPath) {+ metadata.localPath = path.join(nconf.get('upload_path'), metadata.localPath);+ }+ return metadata;+ } catch {+ return null;+ }+}++/**+ * Download a remote emoji image and persist metadata in Redis.+ * Returns { name, remoteUrl, localPath, mediaType } on success, or null on failure.+ */+async function cacheEmoji(emojiTag) {+ const { name, icon } = emojiTag;+ if (!icon || !icon.url) {+ return null;+ }++ const hostname = extractHostname(icon);+ if (!hostname) {+ return null;+ }++ const shortcode = normalizeShortcode(name);+ const fieldKey = buildFieldKey(name, hostname);++ // Check if already cached+ const existing = await getEmoji(name, hostname);+ if (existing) {+ return existing;+ }++ // Determine file extension from content or URL+ const ext = mime.getExtension(icon.mediaType || '') || path.extname(new URL(icon.url).pathname).slice(1) || 'png';+ const cleanName = shortcode.replace(/^:+|:+$/g, '');+ const filename = `${cleanName}.${ext}`;+ const dirPath = path.join(getEmojiDir(), hostname);+ const localPath = path.join(dirPath, filename);++ try {+ // Ensure directory exists+ await fs.mkdir(dirPath, { recursive: true });++ // SSRF check before fetching remote image+ const { ok } = await check(icon.url);+ if (!ok) {+ winston.warn(`[activitypub:emoji] Blocked SSRF attempt for emoji ${shortcode}: ${icon.url}`);+ return null;+ }++ // Fetch the remote image+ const fetchResponse = await fetch(icon.url, {+ signal: AbortSignal.timeout(5000),+ });++ if (!fetchResponse.ok) {+ winston.warn(`[activitypub:emoji] Failed to fetch emoji ${shortcode} from ${hostname}: HTTP ${fetchResponse.status}`);+ return null;+ }++ const buffer = Buffer.from(await fetchResponse.arrayBuffer());+ await fs.writeFile(localPath, buffer);++ const mediaType = icon.mediaType || mime.getType(ext) || 'image/png';+ // Store path relative to upload_path so it survives installation moves+ const relPath = path.relative(nconf.get('upload_path'), localPath);+ const metadata = {+ name: shortcode,+ remoteUrl: icon.url,+ localPath: relPath,+ mediaType,+ };++ await db.setObjectField(emojiLookupKey, fieldKey, JSON.stringify(metadata));++ return metadata;+ } catch (err) {+ winston.warn(`[activitypub:emoji] Error caching emoji ${shortcode} from ${hostname}:`, err.message);+ return null;+ }+}++/**+ * Get the proxy URL for a cached emoji.+ * Returns a URL path that routes through the emoji proxy controller.+ */+function getProxyUrl(shortcode, hostname) {+ const clean = shortcode.replace(/^:+|:+$/g, '');+ return `/emoji/ap/${encodeURIComponent(clean)}/${encodeURIComponent(hostname)}`;+}++/**+ * Process a single emoji tag: cache if needed, return img tag HTML.+ */+async function processEmojiTag(emojiTag) {+ const { name, icon } = emojiTag;+ if (!icon || !icon.url) {+ return null;+ }++ const hostname = extractHostname(icon);+ if (!hostname) {+ return icon.url; // fallback to remote+ }++ const isImage = !icon.mediaType || icon.mediaType.startsWith('image/');+ if (!isImage) {+ return icon.url; // not an image emoji+ }++ if (!validator.isURL(icon.url, { require_protocol: true, require_host: true })) {+ return null;+ }++ // Try to get cached or download+ const cached = await getEmoji(name, hostname);+ let metadata = cached;++ if (!cached) {+ metadata = await cacheEmoji(emojiTag);+ }++ if (metadata) {+ const shortcode = normalizeShortcode(name);+ const proxyUrl = getProxyUrl(shortcode, hostname);+ return `<img class="not-responsive emoji" src="${proxyUrl}" title="${shortcode}" />`;+ }++ // Fallback: use remote URL directly if caching failed+ return `<img class="not-responsive emoji" src="${icon.url}" title="${normalizeShortcode(name)}" />`;+}++module.exports = {+ extractHostname,+ buildFieldKey,+ normalizeShortcode,+ getEmoji,+ cacheEmoji,+ getProxyUrl,+ processEmojiTag,+};diff --git a/src/activitypub/helpers.js b/src/activitypub/helpers.jsindex 766a699eaa..ad471de6d0 100644--- a/src/activitypub/helpers.js+++ b/src/activitypub/helpers.js@@ -18,6 +18,7 @@ const db = require('../database');const ttl = require('../cache/ttl');const user = require('../user');const activitypub = require('.');+const emoji = require('./emoji');// \w only matches ASCII, so match unicode letters/numbers/marks explicitly to support non-ASCII handlesconst webfingerRegex = /^(@|acct:)?[\p{L}\p{N}\p{M}_.-]+@.+$/u;@@ -751,7 +752,7 @@ Helpers.addressed = (id, activity) => {return combined.has(id);};-Helpers.renderEmoji = (text, tags, strip = false) => {+Helpers.renderEmoji = async (text, tags, strip = false) => {if (!text || !tags) {return text;}@@ -760,19 +761,22 @@ Helpers.renderEmoji = (text, tags, strip = false) => {let result = text;const parsed = new Set();- tags.forEach((tag) => {+ const eligibleTags = [];++ // Collect eligible emoji tags+ for (const tag of tags) {const isEmoji = tag.type === 'Emoji';const hasUrl = tag.icon && tag.icon.url;const isImage = !tag.icon?.mediaType || tag.icon.mediaType.startsWith('image/');if (isEmoji && (strip || (hasUrl && isImage))) {- if (!Helpers.isUri(tag.icon.url)) {- return;+ if (hasUrl && !Helpers.isUri(tag.icon.url)) {+ continue;}let { name } = tag;if (parsed.has(name)) {- return;+ continue;}if (!name.startsWith(':')) {@@ -782,18 +786,33 @@ Helpers.renderEmoji = (text, tags, strip = false) => {name = `${name}:`;}- const imgTag = strip ?- '' :- `<img class="not-responsive emoji" src="${tag.icon.url}" title="${name}" />`;-- let index = result.indexOf(name);- while (index !== -1) {- result = result.substring(0, index) + imgTag + result.substring(index + name.length);- index = result.indexOf(name, index + imgTag.length);- }+ eligibleTags.push({ name, tag });parsed.add(name);}- });+ }++ // Process all emoji tags in parallel+ const replacements = await Promise.all(eligibleTags.map(async ({ name, tag }) => {+ let imgTag;+ if (strip) {+ imgTag = '';+ } else {+ imgTag = await emoji.processEmojiTag(tag);+ }+ return { name, imgTag };+ }));++ // Apply replacements+ for (const { name, imgTag } of replacements) {+ if (imgTag === null) {+ continue;+ }+ let index = result.indexOf(name);+ while (index !== -1) {+ result = result.substring(0, index) + imgTag + result.substring(index + name.length);+ index = result.indexOf(name, index + imgTag.length);+ }+ }return result;};diff --git a/src/activitypub/index.js b/src/activitypub/index.jsindex e61bc8505b..032b8dd3ab 100644--- a/src/activitypub/index.js+++ b/src/activitypub/index.js@@ -94,6 +94,7 @@ ActivityPub.out = require('./out');ActivityPub.jobs = require('./jobs');ActivityPub.analytics = require('./analytics');ActivityPub.signatures = require('./signatures');+ActivityPub.emoji = require('./emoji');ActivityPub.resolveId = async (uid, id) => {try {diff --git a/src/activitypub/mocks.js b/src/activitypub/mocks.jsindex c9c38de342..ac5f9fe021 100644--- a/src/activitypub/mocks.js+++ b/src/activitypub/mocks.js@@ -249,7 +249,7 @@ Mocks.profile = async (actors) => {const iconBackgrounds = await user.getIconBackgrounds();let bgColor = Array.prototype.reduce.call(preferredUsername, (cur, next) => cur + next.charCodeAt(), 0);bgColor = iconBackgrounds[bgColor % iconBackgrounds.length];- summary = activitypub.helpers.renderEmoji(summary || '', tag);+ summary = await activitypub.helpers.renderEmoji(summary || '', tag);// Add custom fields into user hashconst customFields = actor.attachment && Array.isArray(actor.attachment) && actor.attachment.length ?@@ -361,13 +361,15 @@ Mocks.category = async (actors) => {const backgroundImage = !icon || typeof icon === 'string' ? icon : icon.url;+ const descriptionParsed = posts.sanitize(await activitypub.helpers.renderEmoji(summary || '', tag));+const payload = {cid,name,handle: `${preferredUsername}@${canonicalHostname}`,slug: `${preferredUsername}@${canonicalHostname}`,description: summary,- descriptionParsed: posts.sanitize(activitypub.helpers.renderEmoji(summary || '', tag)),+ descriptionParsed,icon: backgroundImage ? 'fa-nbb-none' : 'fa-comments',color: '#fff',bgColor,@@ -459,7 +461,7 @@ Mocks.message = async (object) => {object = await Mocks._normalize(object);let content = object.sourceContent || object.content;- content = activitypub.helpers.renderEmoji(content, object.tag);+ content = await activitypub.helpers.renderEmoji(content, object.tag);const message = {mid: object.id,diff --git a/src/activitypub/notes.js b/src/activitypub/notes.jsindex 014c2d27e1..4e96bde8b6 100644--- a/src/activitypub/notes.js+++ b/src/activitypub/notes.js@@ -207,7 +207,7 @@ Notes.assert = async (uid, input, options = { skipChecks: false, queue: false })}// Remove any custom emoji from title- title = activitypub.helpers.renderEmoji(title, _activitypub.tag, true);+ title = await activitypub.helpers.renderEmoji(title, _activitypub.tag, true);}mainPid = utils.isNumber(mainPid) ? parseInt(mainPid, 10) : mainPid;diff --git a/src/controllers/activitypub/emoji.js b/src/controllers/activitypub/emoji.jsnew file mode 100644index 0000000000..ff9762f2ec--- /dev/null+++ b/src/controllers/activitypub/emoji.js@@ -0,0 +1,62 @@+'use strict';++const fs = require('fs').promises;+const path = require('path');+const nconf = require('nconf');+const winston = require('winston');++const emoji = require('../../activitypub/emoji');++const Controller = module.exports;++/**+ * Serve a cached ActivityPub custom emoji image.+ * Route: GET /emoji/ap/:shortcode/:hostname+ */+Controller.serve = async (req, res) => {+ const { shortcode, hostname } = req.params;++ try {+ const metadata = await emoji.getEmoji(shortcode, hostname);+ if (!metadata || !metadata.localPath) {+ return res.sendStatus(404);+ }++ // Path traversal check+ const uploadPath = nconf.get('upload_path');+ if (!path.resolve(metadata.localPath).startsWith(path.resolve(uploadPath))) {+ winston.warn(`[activitypub:emoji] Path traversal attempt blocked: ${metadata.localPath}`);+ return res.sendStatus(403);+ }++ // Check file exists+ try {+ await fs.access(metadata.localPath);+ } catch {+ // File may have been deleted from disk but still in Redis+ return res.sendStatus(404);+ }++ const mediaType = metadata.mediaType || 'image/png';+ res.setHeader('Content-Type', mediaType);+ res.setHeader('Cache-Control', 'public, max-age=31536000, immutable');+ res.setHeader('Expires', new Date(Date.now() + 31536000000).toUTCString());++ // Stream the file+ const { createReadStream } = require('fs');+ const stream = createReadStream(metadata.localPath);+ stream.pipe(res);++ stream.on('error', (err) => {+ winston.warn(`[activitypub:emoji] Error serving file: ${err.message}`);+ if (!res.headersSent) {+ res.sendStatus(500);+ }+ });+ } catch (err) {+ winston.warn(`[activitypub:emoji] Error in serve: ${err.message}`);+ if (!res.headersSent) {+ res.sendStatus(500);+ }+ }+};diff --git a/src/controllers/activitypub/index.js b/src/controllers/activitypub/index.jsindex 1cd8a20f20..1b487a500e 100644--- a/src/controllers/activitypub/index.js+++ b/src/controllers/activitypub/index.js@@ -19,6 +19,7 @@ const Controller = module.exports;Controller.actors = require('./actors');Controller.topics = require('./topics');+Controller.emoji = require('./emoji');Controller.fetch = async (req, res, next) => {// Given a `resource` query parameter, attempts to retrieve and parse itdiff --git a/src/posts/create.js b/src/posts/create.jsindex da4596e75d..2586d1656b 100644--- a/src/posts/create.js+++ b/src/posts/create.js@@ -49,7 +49,7 @@ module.exports = function (Posts) {// Rewrite emoji references to inline image assetsconst property = postData.sourceContent && !postData.content ? 'sourceContent' : 'content';- postData[property] = activitypub.helpers.renderEmoji(postData[property], _activitypub.tag);+ postData[property] = await activitypub.helpers.renderEmoji(postData[property], _activitypub.tag);hasAttachment = _activitypub && _activitypub.attachment && _activitypub.attachment.length;}diff --git a/src/routes/activitypub.js b/src/routes/activitypub.jsindex bf875b9bfa..85e702fa01 100644--- a/src/routes/activitypub.js+++ b/src/routes/activitypub.js@@ -58,4 +58,6 @@ module.exports = function (app, middleware, controllers) {app.get('/category/:cid/:slug?', [...middlewares, middleware.assert.category], helpers.tryRoute(controllers.activitypub.actors.category));app.get('/message/:mid', [...middlewares, middleware.assert.message], helpers.tryRoute(controllers.activitypub.actors.message));++ app.get('/emoji/ap/:shortcode/:hostname', helpers.tryRoute(controllers.activitypub.emoji.serve));};diff --git a/test/activitypub/emoji.js b/test/activitypub/emoji.jsnew file mode 100644index 0000000000..5eefdb03b2--- /dev/null+++ b/test/activitypub/emoji.js@@ -0,0 +1,235 @@+'use strict';++const assert = require('assert');++const db = require('../mocks/databasemock');+const activitypub = require('../../src/activitypub');+const helpers = require('./helpers');++describe('Emoji', () => {+ before(() => {+ // Prevent real outbound requests (serve objects from the AP cache)+ helpers.mocks.mockRequests();+ });++ after(() => {+ helpers.mocks.restoreRequests();+ });++ const emojiModule = activitypub.emoji;+ const emojiLookupKey = 'emoji:ap:lookup';++ beforeEach(async () => {+ await db.delete(emojiLookupKey);+ });++ describe('extractHostname', () => {+ it('should extract hostname from a valid URL', () => {+ const hostname = emojiModule.extractHostname({ url: 'https://example.com/emoji/test.png' });+ assert.strictEqual(hostname, 'example.com');+ });++ it('should return null for null icon', () => {+ const hostname = emojiModule.extractHostname(null);+ assert.strictEqual(hostname, null);+ });++ it('should return null for missing URL', () => {+ const hostname = emojiModule.extractHostname({});+ assert.strictEqual(hostname, null);+ });+ });++ describe('buildFieldKey', () => {+ it('should build a field key from shortcode and hostname', () => {+ const key = emojiModule.buildFieldKey(':poop:', 'mastodon.social');+ assert.strictEqual(key, 'poop:mastodon.social');+ });++ it('should handle shortcodes without colons', () => {+ const key = emojiModule.buildFieldKey('poop', 'mastodon.social');+ assert.strictEqual(key, 'poop:mastodon.social');+ });+ });++ describe('normalizeShortcode', () => {+ it('should add colons if missing', () => {+ assert.strictEqual(emojiModule.normalizeShortcode('poop'), ':poop:');+ assert.strictEqual(emojiModule.normalizeShortcode(':poop'), ':poop:');+ assert.strictEqual(emojiModule.normalizeShortcode('poop:'), ':poop:');+ assert.strictEqual(emojiModule.normalizeShortcode(':poop:'), ':poop:');+ });+ });++ describe('getProxyUrl', () => {+ it('should generate a proxy URL', () => {+ const url = emojiModule.getProxyUrl(':poop:', 'mastodon.social');+ assert.strictEqual(url, '/emoji/ap/poop/mastodon.social');+ });++ it('should URL-encode special characters', () => {+ const url = emojiModule.getProxyUrl(':my-emoji:', 'sub.example.com');+ assert.strictEqual(url, '/emoji/ap/my-emoji/sub.example.com');+ });+ });++ describe('getEmoji', () => {+ it('should return null when emoji is not cached', async () => {+ const result = await emojiModule.getEmoji(':poop:', 'mastodon.social');+ assert.strictEqual(result, null);+ });++ it('should return cached metadata when emoji exists', async () => {+ const metadata = {+ name: ':poop:',+ remoteUrl: 'https://mastodon.social/emojis/poop.png',+ localPath: '/fake/path/poop.png',+ mediaType: 'image/png',+ };+ await db.setObjectField(emojiLookupKey, 'poop:mastodon.social', JSON.stringify(metadata));++ const result = await emojiModule.getEmoji(':poop:', 'mastodon.social');+ assert.deepStrictEqual(result, metadata);+ });+ });++ describe('processEmojiTag', () => {+ it('should return null for tag with no icon', async () => {+ const tag = { type: 'Emoji', name: ':test:' };+ const result = await emojiModule.processEmojiTag(tag);+ assert.strictEqual(result, null);+ });++ it('should return null for tag with no icon.url', async () => {+ const tag = { type: 'Emoji', name: ':test:', icon: {} };+ const result = await emojiModule.processEmojiTag(tag);+ assert.strictEqual(result, null);+ });++ it('should return remote URL fallback for non-image media type', async () => {+ const tag = {+ type: 'Emoji',+ name: ':test:',+ icon: {+ url: 'https://example.com/test',+ mediaType: 'video/mp4',+ },+ };+ const result = await emojiModule.processEmojiTag(tag);+ assert.strictEqual(result, 'https://example.com/test');+ });+ });++ describe('renderEmoji', () => {+ describe('basic rendering', () => {+ it('should return text unchanged when no tags', async () => {+ const result = await activitypub.helpers.renderEmoji('Hello world', []);+ assert.strictEqual(result, 'Hello world');+ });++ it('should return text unchanged when no text', async () => {+ const result = await activitypub.helpers.renderEmoji('', [{ type: 'Emoji', name: ':test:' }]);+ assert.strictEqual(result, '');+ });++ it('should return text unchanged when tags is null', async () => {+ const result = await activitypub.helpers.renderEmoji('Hello world', null);+ assert.strictEqual(result, 'Hello world');+ });+ });++ describe('strip mode', () => {+ it('should strip emoji shortcodes', async () => {+ const tags = [{+ type: 'Emoji',+ name: ':test:',+ icon: {+ url: 'https://example.com/test.png',+ mediaType: 'image/png',+ },+ }];+ const result = await activitypub.helpers.renderEmoji('Hello :test: world', tags, true);+ assert.strictEqual(result, 'Hello world');+ });++ it('should strip all occurrences', async () => {+ const tags = [{+ type: 'Emoji',+ name: ':happy:',+ icon: {+ url: 'https://example.com/happy.png',+ mediaType: 'image/png',+ },+ }];+ const result = await activitypub.helpers.renderEmoji(':happy: Hello :happy: world :happy:', tags, true);+ assert.strictEqual(result, ' Hello world ');+ });+ });++ describe('non-Emoji tags', () => {+ it('should ignore non-Emoji tags', async () => {+ const tags = [{+ type: 'Mention',+ name: '@user',+ icon: {+ url: 'https://example.com/user.png',+ },+ }];+ const result = await activitypub.helpers.renderEmoji('Hello @user world', tags);+ assert.strictEqual(result, 'Hello @user world');+ });+ });++ describe('deduplication', () => {+ it('should not process the same emoji twice', async () => {+ // Monkey-patch the emoji module directly (used internally by helpers.renderEmoji)+ const emojiModuleDirect = require('../../src/activitypub/emoji');+ const processed = [];+ const originalProcess = emojiModuleDirect.processEmojiTag;+ emojiModuleDirect.processEmojiTag = async (tag) => {+ processed.push(tag.name);+ return originalProcess(tag);+ };++ const tags = [+ {+ type: 'Emoji',+ name: ':test:',+ icon: {+ url: 'https://example.com/test.png',+ mediaType: 'image/png',+ },+ },+ {+ type: 'Emoji',+ name: ':test:',+ icon: {+ url: 'https://example.com/test2.png',+ mediaType: 'image/png',+ },+ },+ ];++ await activitypub.helpers.renderEmoji(':test:', tags);+ assert.strictEqual(processed.length, 1);++ emojiModuleDirect.processEmojiTag = originalProcess;+ });+ });++ describe('shortcodes without colons', () => {+ it('should add colons to shortcodes missing them', async () => {+ const tags = [{+ type: 'Emoji',+ name: 'test',+ icon: {+ url: 'https://example.com/test.png',+ mediaType: 'image/png',+ },+ }];+ const result = await activitypub.helpers.renderEmoji(':test:', tags, true);+ assert.strictEqual(result, '');+ });+ });+ });+});