Skip to content

Node.js based forum software built for the modern web

refactor: ability to browse to remote categories, group actor assertion logic, etc. -- no logic to assign topics to remote categories yet

Julian Lam1f40995f79f3
diff --git a/public/src/client/category.js b/public/src/client/category.js
index a2a5d579ad..515a871262 100644
--- a/public/src/client/category.js
+++ b/public/src/client/category.js
@@ -118,7 +118,7 @@ define('forum/category', [
};
Category.toBottom = async () => {
- const { count } = await api.get(`/categories/${ajaxify.data.category.cid}/count`);
+ const { count } = await api.get(`/categories/${encodeURIComponent(ajaxify.data.category.cid)}/count`);
navigator.scrollBottom(count - 1);
};
@@ -127,7 +127,7 @@ define('forum/category', [
hooks.fire('action:topics.loading');
const params = utils.params();
- infinitescroll.loadMore(`/categories/${ajaxify.data.cid}/topics`, {
+ infinitescroll.loadMore(`/categories/${encodeURIComponent(ajaxify.data.cid)}/topics`, {
after: after,
direction: direction,
query: params,
diff --git a/src/activitypub/actors.js b/src/activitypub/actors.js
index 594e4e5da8..576fd241af 100644
--- a/src/activitypub/actors.js
+++ b/src/activitypub/actors.js
@@ -7,6 +7,7 @@ const _ = require('lodash');
const db = require('../database');
const meta = require('../meta');
const batch = require('../batch');
+const categories = require('../categories');
const user = require('../user');
const utils = require('../utils');
const TTLCache = require('../cache/ttl');
@@ -20,15 +21,12 @@ const activitypub = module.parent.exports;
const Actors = module.exports;
-Actors.assert = async (ids, options = {}) => {
+Actors.qualify = async (ids, options = {}) => {
/**
- * Ensures that the passed in ids or webfinger handles are stored in database.
- * Options:
- * - update: boolean, forces re-fetch/process of the resolved id
- * Return one of:
- * - An array of newly processed ids
- * - false: if input incorrect (or webfinger handle cannot resolve)
- * - true: no new IDs processed; all passed-in IDs present.
+ * Sanity-checks, cache handling, webfinger translations, so that only
+ * an array of actor uris are handled by assert/assertGroup.
+ *
+ * This method is only called by assert/assertGroup (at least in core.)
*/
// Handle single values
@@ -85,6 +83,21 @@ Actors.assert = async (ids, options = {}) => {
});
}
+ return ids;
+};
+
+Actors.assert = async (ids, options = {}) => {
+ /**
+ * Ensures that the passed in ids or webfinger handles are stored in database.
+ * Options:
+ * - update: boolean, forces re-fetch/process of the resolved id
+ * Return one of:
+ * - An array of newly processed ids
+ * - false: if input incorrect (or webfinger handle cannot resolve)
+ * - true: no new IDs processed; all passed-in IDs present.
+ */
+
+ ids = await Actors.qualify(ids, options);
if (!ids.length) {
return true;
}
@@ -96,6 +109,7 @@ Actors.assert = async (ids, options = {}) => {
const urlMap = new Map();
const followersUrlMap = new Map();
const pubKeysMap = new Map();
+ const categories = new Set();
let actors = await Promise.all(ids.map(async (id) => {
try {
activitypub.helpers.log(`[activitypub/actors] Processing ${id}`);
@@ -104,8 +118,14 @@ Actors.assert = async (ids, options = {}) => {
let typeOk = false;
if (Array.isArray(actor.type)) {
typeOk = actor.type.some(type => activitypub._constants.acceptableActorTypes.has(type));
+ if (!typeOk && actor.type.some(type => activitypub._constants.acceptableGroupTypes.has(type))) {
+ categories.add(actor.id);
+ }
} else {
typeOk = activitypub._constants.acceptableActorTypes.has(actor.type);
+ if (!typeOk && activitypub._constants.acceptableGroupTypes.has(actor.type)) {
+ categories.add(actor.id);
+ }
}
if (
@@ -220,6 +240,142 @@ Actors.assert = async (ids, options = {}) => {
return actors;
};
+Actors.assertGroup = async (ids, options = {}) => {
+ /**
+ * Ensures that the passed in ids or webfinger handles are stored in database.
+ * Options:
+ * - update: boolean, forces re-fetch/process of the resolved id
+ * Return one of:
+ * - An array of newly processed ids
+ * - false: if input incorrect (or webfinger handle cannot resolve)
+ * - true: no new IDs processed; all passed-in IDs present.
+ */
+
+ ids = await Actors.qualify(ids, options);
+ if (!ids.length) {
+ return true;
+ }
+
+ activitypub.helpers.log(`[activitypub/actors] Asserting ${ids.length} group(s)`);
+
+ // NOTE: MAKE SURE EVERY DB ADDITION HAS A CORRESPONDING REMOVAL IN ACTORS.REMOVE!
+
+ const urlMap = new Map();
+ const followersUrlMap = new Map();
+ const pubKeysMap = new Map();
+ const users = new Set();
+ let groups = await Promise.all(ids.map(async (id) => {
+ try {
+ activitypub.helpers.log(`[activitypub/actors] Processing group ${id}`);
+ const actor = (typeof id === 'object' && id.hasOwnProperty('id')) ? id : await activitypub.get('uid', 0, id, { cache: process.env.CI === 'true' });
+
+ let typeOk = false;
+ if (Array.isArray(actor.type)) {
+ typeOk = actor.type.some(type => activitypub._constants.acceptableGroupTypes.has(type));
+ if (!typeOk && actor.type.some(type => activitypub._constants.acceptableActorTypes.has(type))) {
+ users.add(actor.id);
+ }
+ } else {
+ typeOk = activitypub._constants.acceptableGroupTypes.has(actor.type);
+ if (!typeOk && activitypub._constants.acceptableActorTypes.has(actor.type)) {
+ users.add(actor.id);
+ }
+ }
+
+ if (
+ !typeOk ||
+ !activitypub._constants.requiredActorProps.every(prop => actor.hasOwnProperty(prop))
+ ) {
+ return null;
+ }
+
+ // Save url for backreference
+ const url = Array.isArray(actor.url) ? actor.url.shift() : actor.url;
+ if (url && url !== actor.id) {
+ urlMap.set(url, actor.id);
+ }
+
+ // Save followers url for backreference
+ if (actor.hasOwnProperty('followers') && activitypub.helpers.isUri(actor.followers)) {
+ followersUrlMap.set(actor.followers, actor.id);
+ }
+
+ // Public keys
+ pubKeysMap.set(actor.id, actor.publicKey);
+
+ return actor;
+ } catch (e) {
+ if (e.code === 'ap_get_410') {
+ // const exists = await user.exists(id);
+ // if (exists) {
+ // await user.deleteAccount(id);
+ // }
+ }
+
+ return null;
+ }
+ }));
+ groups = groups.filter(Boolean); // remove unresolvable actors
+
+ // Build userData object for storage
+ const categoryObjs = (await activitypub.mocks.category(groups)).filter(Boolean);
+ const now = Date.now();
+
+ const bulkSet = categoryObjs.reduce((memo, category) => {
+ const key = `categoryRemote:${category.cid}`;
+ memo.push([key, category], [`${key}:keys`, pubKeysMap.get(category.cid)]);
+ return memo;
+ }, []);
+ if (urlMap.size) {
+ bulkSet.push(['remoteUrl:cid', Object.fromEntries(urlMap)]);
+ }
+ if (followersUrlMap.size) {
+ bulkSet.push(['followersUrl:cid', Object.fromEntries(followersUrlMap)]);
+ }
+
+ const exists = await db.isSortedSetMembers('usersRemote:lastCrawled', categoryObjs.map(p => p.cid));
+ const cidsForCurrent = categoryObjs.map((p, idx) => (exists[idx] ? p.cid : 0));
+ const current = await categories.getCategoriesFields(cidsForCurrent, ['slug']);
+ const queries = categoryObjs.reduce((memo, profile, idx) => {
+ const { slug } = current[idx];
+
+ if (options.update || slug !== profile.slug) {
+ if (cidsForCurrent[idx] !== 0 && slug) {
+ // memo.searchRemove.push(['ap.preferredUsername:sorted', `${slug.toLowerCase()}:${profile.uid}`]);
+ memo.handleRemove.push(slug.toLowerCase());
+ }
+
+ // memo.searchAdd.push(['ap.preferredUsername:sorted', 0, `${profile.slug.toLowerCase()}:${profile.uid}`]);
+ memo.handleAdd[profile.slug.toLowerCase()] = profile.cid;
+ }
+
+ // if (options.update || (profile.fullname && fullname !== profile.fullname)) {
+ // if (fullname && cidsForCurrent[idx] !== 0) {
+ // memo.searchRemove.push(['ap.name:sorted', `${fullname.toLowerCase()}:${profile.uid}`]);
+ // }
+
+ // memo.searchAdd.push(['ap.name:sorted', 0, `${profile.fullname.toLowerCase()}:${profile.uid}`]);
+ // }
+
+ return memo;
+ }, { /* searchRemove: [], searchAdd: [], */ handleRemove: [], handleAdd: {} });
+
+ // Removals
+ await Promise.all([
+ // db.sortedSetRemoveBulk(queries.searchRemove),
+ db.deleteObjectFields('handle:cid', queries.handleRemove),
+ ]);
+
+ await Promise.all([
+ db.setObjectBulk(bulkSet),
+ db.sortedSetAdd('usersRemote:lastCrawled', groups.map(() => now), groups.map(p => p.uid)),
+ // db.sortedSetAddBulk(queries.searchAdd),
+ db.setObject('handle:cid', queries.handleAdd),
+ ]);
+
+ return categoryObjs;
+};
+
Actors.getLocalFollowers = async (id) => {
const response = {
uids: new Set(),
diff --git a/src/activitypub/index.js b/src/activitypub/index.js
index 96fcbeac47..273e443221 100644
--- a/src/activitypub/index.js
+++ b/src/activitypub/index.js
@@ -39,7 +39,8 @@ ActivityPub._constants = Object.freeze({
acceptedPostTypes: [
'Note', 'Page', 'Article', 'Question', 'Video',
],
- acceptableActorTypes: new Set(['Application', 'Group', 'Organization', 'Person', 'Service']),
+ acceptableActorTypes: new Set(['Application', 'Organization', 'Person', 'Service']),
+ acceptableGroupTypes: new Set(['Group']),
requiredActorProps: ['inbox', 'outbox'],
acceptedProtocols: ['https', ...(process.env.CI === 'true' ? ['http'] : [])],
acceptable: {
diff --git a/src/activitypub/mocks.js b/src/activitypub/mocks.js
index 7726768016..1108937eee 100644
--- a/src/activitypub/mocks.js
+++ b/src/activitypub/mocks.js
@@ -216,7 +216,7 @@ Mocks.profile = async (actors) => {
uploadedpicture: undefined,
'cover:url': !image || typeof image === 'string' ? image : image.url,
'cover:position': '50% 50%',
- aboutme: summary,
+ aboutme: posts.sanitize(summary),
followerCount,
followingCount,
@@ -233,6 +233,73 @@ Mocks.profile = async (actors) => {
return profiles;
};
+Mocks.category = async (actors) => {
+ const categories = await Promise.all(actors.map(async (actor) => {
+ if (!actor) {
+ return null;
+ }
+
+ const cid = actor.id;
+ let hostname;
+ let {
+ url, preferredUsername, /* icon, */ image,
+ name, summary, followers, inbox, endpoints, tag,
+ } = actor;
+ preferredUsername = slugify(preferredUsername || name);
+ // const { followers: followerCount, following: followingCount } = await activitypub.actors.getLocalFollowCounts(uid);
+
+ try {
+ ({ hostname } = new URL(actor.id));
+ } catch (e) {
+ return null;
+ }
+
+ // No support for category avatars yet ;(
+ // let picture;
+ // if (icon) {
+ // picture = typeof icon === 'string' ? icon : icon.url;
+ // }
+ const iconBackgrounds = await user.getIconBackgrounds();
+ let bgColor = Array.prototype.reduce.call(preferredUsername, (cur, next) => cur + next.charCodeAt(), 0);
+ bgColor = iconBackgrounds[bgColor % iconBackgrounds.length];
+
+ // Replace emoji in summary
+ if (tag && Array.isArray(tag)) {
+ tag
+ .filter(tag => tag.type === 'Emoji' &&
+ isEmojiShortcode.test(tag.name) &&
+ tag.icon && tag.icon.mediaType && tag.icon.mediaType.startsWith('image/'))
+ .forEach((tag) => {
+ summary = summary.replace(new RegExp(tag.name, 'g'), `<img class="not-responsive emoji" src="${tag.icon.url}" title="${tag.name}" />`);
+ });
+ }
+
+ const payload = {
+ cid,
+ name,
+ handle: preferredUsername,
+ slug: `${preferredUsername}@${hostname}`,
+ description: summary,
+ descriptionParsed: posts.sanitize(summary),
+ icon: 'fa-comments',
+ color: '#fff',
+ bgColor,
+ backgroundImage: !image || typeof image === 'string' ? image : image.url,
+ // followerCount,
+ // followingCount,
+
+ url,
+ inbox,
+ sharedInbox: endpoints ? endpoints.sharedInbox : null,
+ followersUrl: followers,
+ };
+
+ return payload;
+ }));
+
+ return categories;
+};
+
Mocks.post = async (objects) => {
let single = false;
if (!Array.isArray(objects)) {
diff --git a/src/categories/data.js b/src/categories/data.js
index 9ad2783203..9039672775 100644
--- a/src/categories/data.js
+++ b/src/categories/data.js
@@ -36,8 +36,8 @@ module.exports = function (Categories) {
return [];
}
- cids = cids.map(cid => parseInt(cid, 10));
- const keys = cids.map(cid => `category:${cid}`);
+ cids = cids.map(cid => (utils.isNumber(cid) ? parseInt(cid, 10) : cid));
+ const keys = cids.map(cid => (utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`));
const categories = await db.getObjects(keys, fields);
// Handle cid -1
diff --git a/src/categories/index.js b/src/categories/index.js
index cde2f02d6a..1a254f9b15 100644
--- a/src/categories/index.js
+++ b/src/categories/index.js
@@ -10,6 +10,7 @@ const plugins = require('../plugins');
const privileges = require('../privileges');
const cache = require('../cache');
const meta = require('../meta');
+const utils = require('../utils');
const Categories = module.exports;
@@ -26,9 +27,14 @@ require('./search')(Categories);
Categories.icons = require('./icon');
Categories.exists = async function (cids) {
- return await db.exists(
- Array.isArray(cids) ? cids.map(cid => `category:${cid}`) : `category:${cids}`
- );
+ let keys;
+ if (Array.isArray(cids)) {
+ keys = cids.map(cid => (utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`));
+ } else {
+ keys = utils.isNumber(cids) ? `category:${cids}` : `categoryRemote:${cids}`;
+ }
+
+ return await db.exists(keys);
};
Categories.existsByHandle = async function (handle) {
diff --git a/src/categories/topics.js b/src/categories/topics.js
index 64e9046614..cd13deaf4b 100644
--- a/src/categories/topics.js
+++ b/src/categories/topics.js
@@ -31,6 +31,7 @@ module.exports = function (Categories) {
Categories.getPinnedTids({ ...data, start: 0, stop: -1 }),
Categories.buildTopicsSortedSet(data),
]);
+ console.log(set);
const totalPinnedCount = pinnedTids.length;
const pinnedTidsOnPage = pinnedTids.slice(data.start, data.stop !== -1 ? data.stop + 1 : undefined);
diff --git a/src/controllers/category.js b/src/controllers/category.js
index 27b5c749f9..a822022392 100644
--- a/src/controllers/category.js
+++ b/src/controllers/category.js
@@ -26,14 +26,22 @@ const validSorts = [
];
categoryController.get = async function (req, res, next) {
- const cid = req.params.category_id;
+ let cid = req.params.category_id;
if (cid === '-1') {
return helpers.redirect(res, `${res.locals.isAPI ? '/api' : ''}/world?${qs.stringify(req.query)}`);
}
+ if (!utils.isNumber(cid)) {
+ const assertion = await activitypub.actors.assertGroup([cid]);
+ cid = await db.getObjectField('handle:cid', cid);
+ if (!assertion || !cid) {
+ return next();
+ }
+ }
+
let currentPage = parseInt(req.query.page, 10) || 1;
let topicIndex = utils.isNumber(req.params.topic_index) ? parseInt(req.params.topic_index, 10) - 1 : 0;
- if ((req.params.topic_index && !utils.isNumber(req.params.topic_index)) || !utils.isNumber(cid)) {
+ if ((req.params.topic_index && !utils.isNumber(req.params.topic_index))) {
return next();
}
@@ -58,7 +66,7 @@ categoryController.get = async function (req, res, next) {
return helpers.notAllowed(req, res);
}
- if (!res.locals.isAPI && !req.params.slug && (categoryFields.slug && categoryFields.slug !== `${cid}/`)) {
+ if (utils.isNumber(cid) && !res.locals.isAPI && !req.params.slug && (categoryFields.slug && categoryFields.slug !== `${cid}/`)) {
return helpers.redirect(res, `/category/${categoryFields.slug}?${qs.stringify(req.query)}`, true);
}