Node.js based forum software built for the modern web
refactor: ability to browse to remote categories, group actor assertion logic, etc. -- no logic to assign topics to remote categories yet
Julian Lam1f40995f79f3
diff --git a/public/src/client/category.js b/public/src/client/category.jsindex a2a5d579ad..515a871262 100644--- a/public/src/client/category.js+++ b/public/src/client/category.js@@ -118,7 +118,7 @@ define('forum/category', [};Category.toBottom = async () => {- const { count } = await api.get(`/categories/${ajaxify.data.category.cid}/count`);+ const { count } = await api.get(`/categories/${encodeURIComponent(ajaxify.data.category.cid)}/count`);navigator.scrollBottom(count - 1);};@@ -127,7 +127,7 @@ define('forum/category', [hooks.fire('action:topics.loading');const params = utils.params();- infinitescroll.loadMore(`/categories/${ajaxify.data.cid}/topics`, {+ infinitescroll.loadMore(`/categories/${encodeURIComponent(ajaxify.data.cid)}/topics`, {after: after,direction: direction,query: params,diff --git a/src/activitypub/actors.js b/src/activitypub/actors.jsindex 594e4e5da8..576fd241af 100644--- a/src/activitypub/actors.js+++ b/src/activitypub/actors.js@@ -7,6 +7,7 @@ const _ = require('lodash');const db = require('../database');const meta = require('../meta');const batch = require('../batch');+const categories = require('../categories');const user = require('../user');const utils = require('../utils');const TTLCache = require('../cache/ttl');@@ -20,15 +21,12 @@ const activitypub = module.parent.exports;const Actors = module.exports;-Actors.assert = async (ids, options = {}) => {+Actors.qualify = async (ids, options = {}) => {/**- * Ensures that the passed in ids or webfinger handles are stored in database.- * Options:- * - update: boolean, forces re-fetch/process of the resolved id- * Return one of:- * - An array of newly processed ids- * - false: if input incorrect (or webfinger handle cannot resolve)- * - true: no new IDs processed; all passed-in IDs present.+ * Sanity-checks, cache handling, webfinger translations, so that only+ * an array of actor uris are handled by assert/assertGroup.+ *+ * This method is only called by assert/assertGroup (at least in core.)*/// Handle single values@@ -85,6 +83,21 @@ Actors.assert = async (ids, options = {}) => {});}+ return ids;+};++Actors.assert = async (ids, options = {}) => {+ /**+ * Ensures that the passed in ids or webfinger handles are stored in database.+ * Options:+ * - update: boolean, forces re-fetch/process of the resolved id+ * Return one of:+ * - An array of newly processed ids+ * - false: if input incorrect (or webfinger handle cannot resolve)+ * - true: no new IDs processed; all passed-in IDs present.+ */++ ids = await Actors.qualify(ids, options);if (!ids.length) {return true;}@@ -96,6 +109,7 @@ Actors.assert = async (ids, options = {}) => {const urlMap = new Map();const followersUrlMap = new Map();const pubKeysMap = new Map();+ const categories = new Set();let actors = await Promise.all(ids.map(async (id) => {try {activitypub.helpers.log(`[activitypub/actors] Processing ${id}`);@@ -104,8 +118,14 @@ Actors.assert = async (ids, options = {}) => {let typeOk = false;if (Array.isArray(actor.type)) {typeOk = actor.type.some(type => activitypub._constants.acceptableActorTypes.has(type));+ if (!typeOk && actor.type.some(type => activitypub._constants.acceptableGroupTypes.has(type))) {+ categories.add(actor.id);+ }} else {typeOk = activitypub._constants.acceptableActorTypes.has(actor.type);+ if (!typeOk && activitypub._constants.acceptableGroupTypes.has(actor.type)) {+ categories.add(actor.id);+ }}if (@@ -220,6 +240,142 @@ Actors.assert = async (ids, options = {}) => {return actors;};+Actors.assertGroup = async (ids, options = {}) => {+ /**+ * Ensures that the passed in ids or webfinger handles are stored in database.+ * Options:+ * - update: boolean, forces re-fetch/process of the resolved id+ * Return one of:+ * - An array of newly processed ids+ * - false: if input incorrect (or webfinger handle cannot resolve)+ * - true: no new IDs processed; all passed-in IDs present.+ */++ ids = await Actors.qualify(ids, options);+ if (!ids.length) {+ return true;+ }++ activitypub.helpers.log(`[activitypub/actors] Asserting ${ids.length} group(s)`);++ // NOTE: MAKE SURE EVERY DB ADDITION HAS A CORRESPONDING REMOVAL IN ACTORS.REMOVE!++ const urlMap = new Map();+ const followersUrlMap = new Map();+ const pubKeysMap = new Map();+ const users = new Set();+ let groups = await Promise.all(ids.map(async (id) => {+ try {+ activitypub.helpers.log(`[activitypub/actors] Processing group ${id}`);+ const actor = (typeof id === 'object' && id.hasOwnProperty('id')) ? id : await activitypub.get('uid', 0, id, { cache: process.env.CI === 'true' });++ let typeOk = false;+ if (Array.isArray(actor.type)) {+ typeOk = actor.type.some(type => activitypub._constants.acceptableGroupTypes.has(type));+ if (!typeOk && actor.type.some(type => activitypub._constants.acceptableActorTypes.has(type))) {+ users.add(actor.id);+ }+ } else {+ typeOk = activitypub._constants.acceptableGroupTypes.has(actor.type);+ if (!typeOk && activitypub._constants.acceptableActorTypes.has(actor.type)) {+ users.add(actor.id);+ }+ }++ if (+ !typeOk ||+ !activitypub._constants.requiredActorProps.every(prop => actor.hasOwnProperty(prop))+ ) {+ return null;+ }++ // Save url for backreference+ const url = Array.isArray(actor.url) ? actor.url.shift() : actor.url;+ if (url && url !== actor.id) {+ urlMap.set(url, actor.id);+ }++ // Save followers url for backreference+ if (actor.hasOwnProperty('followers') && activitypub.helpers.isUri(actor.followers)) {+ followersUrlMap.set(actor.followers, actor.id);+ }++ // Public keys+ pubKeysMap.set(actor.id, actor.publicKey);++ return actor;+ } catch (e) {+ if (e.code === 'ap_get_410') {+ // const exists = await user.exists(id);+ // if (exists) {+ // await user.deleteAccount(id);+ // }+ }++ return null;+ }+ }));+ groups = groups.filter(Boolean); // remove unresolvable actors++ // Build userData object for storage+ const categoryObjs = (await activitypub.mocks.category(groups)).filter(Boolean);+ const now = Date.now();++ const bulkSet = categoryObjs.reduce((memo, category) => {+ const key = `categoryRemote:${category.cid}`;+ memo.push([key, category], [`${key}:keys`, pubKeysMap.get(category.cid)]);+ return memo;+ }, []);+ if (urlMap.size) {+ bulkSet.push(['remoteUrl:cid', Object.fromEntries(urlMap)]);+ }+ if (followersUrlMap.size) {+ bulkSet.push(['followersUrl:cid', Object.fromEntries(followersUrlMap)]);+ }++ const exists = await db.isSortedSetMembers('usersRemote:lastCrawled', categoryObjs.map(p => p.cid));+ const cidsForCurrent = categoryObjs.map((p, idx) => (exists[idx] ? p.cid : 0));+ const current = await categories.getCategoriesFields(cidsForCurrent, ['slug']);+ const queries = categoryObjs.reduce((memo, profile, idx) => {+ const { slug } = current[idx];++ if (options.update || slug !== profile.slug) {+ if (cidsForCurrent[idx] !== 0 && slug) {+ // memo.searchRemove.push(['ap.preferredUsername:sorted', `${slug.toLowerCase()}:${profile.uid}`]);+ memo.handleRemove.push(slug.toLowerCase());+ }++ // memo.searchAdd.push(['ap.preferredUsername:sorted', 0, `${profile.slug.toLowerCase()}:${profile.uid}`]);+ memo.handleAdd[profile.slug.toLowerCase()] = profile.cid;+ }++ // if (options.update || (profile.fullname && fullname !== profile.fullname)) {+ // if (fullname && cidsForCurrent[idx] !== 0) {+ // memo.searchRemove.push(['ap.name:sorted', `${fullname.toLowerCase()}:${profile.uid}`]);+ // }++ // memo.searchAdd.push(['ap.name:sorted', 0, `${profile.fullname.toLowerCase()}:${profile.uid}`]);+ // }++ return memo;+ }, { /* searchRemove: [], searchAdd: [], */ handleRemove: [], handleAdd: {} });++ // Removals+ await Promise.all([+ // db.sortedSetRemoveBulk(queries.searchRemove),+ db.deleteObjectFields('handle:cid', queries.handleRemove),+ ]);++ await Promise.all([+ db.setObjectBulk(bulkSet),+ db.sortedSetAdd('usersRemote:lastCrawled', groups.map(() => now), groups.map(p => p.uid)),+ // db.sortedSetAddBulk(queries.searchAdd),+ db.setObject('handle:cid', queries.handleAdd),+ ]);++ return categoryObjs;+};+Actors.getLocalFollowers = async (id) => {const response = {uids: new Set(),diff --git a/src/activitypub/index.js b/src/activitypub/index.jsindex 96fcbeac47..273e443221 100644--- a/src/activitypub/index.js+++ b/src/activitypub/index.js@@ -39,7 +39,8 @@ ActivityPub._constants = Object.freeze({acceptedPostTypes: ['Note', 'Page', 'Article', 'Question', 'Video',],- acceptableActorTypes: new Set(['Application', 'Group', 'Organization', 'Person', 'Service']),+ acceptableActorTypes: new Set(['Application', 'Organization', 'Person', 'Service']),+ acceptableGroupTypes: new Set(['Group']),requiredActorProps: ['inbox', 'outbox'],acceptedProtocols: ['https', ...(process.env.CI === 'true' ? ['http'] : [])],acceptable: {diff --git a/src/activitypub/mocks.js b/src/activitypub/mocks.jsindex 7726768016..1108937eee 100644--- a/src/activitypub/mocks.js+++ b/src/activitypub/mocks.js@@ -216,7 +216,7 @@ Mocks.profile = async (actors) => {uploadedpicture: undefined,'cover:url': !image || typeof image === 'string' ? image : image.url,'cover:position': '50% 50%',- aboutme: summary,+ aboutme: posts.sanitize(summary),followerCount,followingCount,@@ -233,6 +233,73 @@ Mocks.profile = async (actors) => {return profiles;};+Mocks.category = async (actors) => {+ const categories = await Promise.all(actors.map(async (actor) => {+ if (!actor) {+ return null;+ }++ const cid = actor.id;+ let hostname;+ let {+ url, preferredUsername, /* icon, */ image,+ name, summary, followers, inbox, endpoints, tag,+ } = actor;+ preferredUsername = slugify(preferredUsername || name);+ // const { followers: followerCount, following: followingCount } = await activitypub.actors.getLocalFollowCounts(uid);++ try {+ ({ hostname } = new URL(actor.id));+ } catch (e) {+ return null;+ }++ // No support for category avatars yet ;(+ // let picture;+ // if (icon) {+ // picture = typeof icon === 'string' ? icon : icon.url;+ // }+ const iconBackgrounds = await user.getIconBackgrounds();+ let bgColor = Array.prototype.reduce.call(preferredUsername, (cur, next) => cur + next.charCodeAt(), 0);+ bgColor = iconBackgrounds[bgColor % iconBackgrounds.length];++ // Replace emoji in summary+ if (tag && Array.isArray(tag)) {+ tag+ .filter(tag => tag.type === 'Emoji' &&+ isEmojiShortcode.test(tag.name) &&+ tag.icon && tag.icon.mediaType && tag.icon.mediaType.startsWith('image/'))+ .forEach((tag) => {+ summary = summary.replace(new RegExp(tag.name, 'g'), `<img class="not-responsive emoji" src="${tag.icon.url}" title="${tag.name}" />`);+ });+ }++ const payload = {+ cid,+ name,+ handle: preferredUsername,+ slug: `${preferredUsername}@${hostname}`,+ description: summary,+ descriptionParsed: posts.sanitize(summary),+ icon: 'fa-comments',+ color: '#fff',+ bgColor,+ backgroundImage: !image || typeof image === 'string' ? image : image.url,+ // followerCount,+ // followingCount,++ url,+ inbox,+ sharedInbox: endpoints ? endpoints.sharedInbox : null,+ followersUrl: followers,+ };++ return payload;+ }));++ return categories;+};+Mocks.post = async (objects) => {let single = false;if (!Array.isArray(objects)) {diff --git a/src/categories/data.js b/src/categories/data.jsindex 9ad2783203..9039672775 100644--- a/src/categories/data.js+++ b/src/categories/data.js@@ -36,8 +36,8 @@ module.exports = function (Categories) {return [];}- cids = cids.map(cid => parseInt(cid, 10));- const keys = cids.map(cid => `category:${cid}`);+ cids = cids.map(cid => (utils.isNumber(cid) ? parseInt(cid, 10) : cid));+ const keys = cids.map(cid => (utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`));const categories = await db.getObjects(keys, fields);// Handle cid -1diff --git a/src/categories/index.js b/src/categories/index.jsindex cde2f02d6a..1a254f9b15 100644--- a/src/categories/index.js+++ b/src/categories/index.js@@ -10,6 +10,7 @@ const plugins = require('../plugins');const privileges = require('../privileges');const cache = require('../cache');const meta = require('../meta');+const utils = require('../utils');const Categories = module.exports;@@ -26,9 +27,14 @@ require('./search')(Categories);Categories.icons = require('./icon');Categories.exists = async function (cids) {- return await db.exists(- Array.isArray(cids) ? cids.map(cid => `category:${cid}`) : `category:${cids}`- );+ let keys;+ if (Array.isArray(cids)) {+ keys = cids.map(cid => (utils.isNumber(cid) ? `category:${cid}` : `categoryRemote:${cid}`));+ } else {+ keys = utils.isNumber(cids) ? `category:${cids}` : `categoryRemote:${cids}`;+ }++ return await db.exists(keys);};Categories.existsByHandle = async function (handle) {diff --git a/src/categories/topics.js b/src/categories/topics.jsindex 64e9046614..cd13deaf4b 100644--- a/src/categories/topics.js+++ b/src/categories/topics.js@@ -31,6 +31,7 @@ module.exports = function (Categories) {Categories.getPinnedTids({ ...data, start: 0, stop: -1 }),Categories.buildTopicsSortedSet(data),]);+ console.log(set);const totalPinnedCount = pinnedTids.length;const pinnedTidsOnPage = pinnedTids.slice(data.start, data.stop !== -1 ? data.stop + 1 : undefined);diff --git a/src/controllers/category.js b/src/controllers/category.jsindex 27b5c749f9..a822022392 100644--- a/src/controllers/category.js+++ b/src/controllers/category.js@@ -26,14 +26,22 @@ const validSorts = [];categoryController.get = async function (req, res, next) {- const cid = req.params.category_id;+ let cid = req.params.category_id;if (cid === '-1') {return helpers.redirect(res, `${res.locals.isAPI ? '/api' : ''}/world?${qs.stringify(req.query)}`);}+ if (!utils.isNumber(cid)) {+ const assertion = await activitypub.actors.assertGroup([cid]);+ cid = await db.getObjectField('handle:cid', cid);+ if (!assertion || !cid) {+ return next();+ }+ }+let currentPage = parseInt(req.query.page, 10) || 1;let topicIndex = utils.isNumber(req.params.topic_index) ? parseInt(req.params.topic_index, 10) - 1 : 0;- if ((req.params.topic_index && !utils.isNumber(req.params.topic_index)) || !utils.isNumber(cid)) {+ if ((req.params.topic_index && !utils.isNumber(req.params.topic_index))) {return next();}@@ -58,7 +66,7 @@ categoryController.get = async function (req, res, next) {return helpers.notAllowed(req, res);}- if (!res.locals.isAPI && !req.params.slug && (categoryFields.slug && categoryFields.slug !== `${cid}/`)) {+ if (utils.isNumber(cid) && !res.locals.isAPI && !req.params.slug && (categoryFields.slug && categoryFields.slug !== `${cid}/`)) {return helpers.redirect(res, `/category/${categoryFields.slug}?${qs.stringify(req.query)}`, true);}