ActivityPub-federated video streaming platform using P2P directly in your web browser
Prevent federation ddos
Chocobozzz454cdfa61a1b
diff --git a/packages/tests/src/api/activitypub/security.ts b/packages/tests/src/api/activitypub/security.tsindex 89a02944f2..19dc39db6f 100644--- a/packages/tests/src/api/activitypub/security.ts+++ b/packages/tests/src/api/activitypub/security.ts@@ -228,6 +228,36 @@ describe('Test ActivityPub security', function () {expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204)})+ it('Should reject a Collection with too many activities', async function () {+ const items = []++ for (let i = 0; i <= ACTIVITY_PUB.MAX_ACTIVITIES_PER_REQUEST; i++) {+ items.push({+ id: `${servers[1].url}/activities/${i}`,+ type: 'Like',+ actor: servers[1].url + '/accounts/peertube',+ object: servers[1].url + '/accounts/peertube'+ })+ }++ const body = await activityPubContextify({+ type: 'OrderedCollection',+ // Matches the HTTP-signature actor so the forwarded (JSON-LD signature) codepath is not triggered+ actor: servers[1].url + '/accounts/peertube',+ totalItems: items.length,+ orderedItems: items+ }, 'Collection', fakeFilter())++ const headers = buildGlobalHTTPHeaders(body, buildDigest)++ try {+ await makePOSTAPRequest(url, body, baseHttpSignature(), headers)+ expect(true, 'Did not throw').to.be.false+ } catch (err) {+ expect(err.statusCode).to.equal(HttpStatusCode.BAD_REQUEST_400)+ }+ })+it('Should refresh the actor keys', async function () {this.timeout(20000)@@ -487,6 +517,28 @@ describe('Test ActivityPub security', function () {}})+ it('Should fail with a body bigger than the JSON-LD signature check limit', async function () {+ const activity: any = {+ '@context': 'https://www.w3.org/ns/activitystreams',+ 'id': 'https://alice.example/notes/43/activity',+ 'type': 'Create',+ 'actor': servers[2].url + '/accounts/peertube',+ 'cc': 'https://www.w3.org/ns/activitystreams#Public',+ 'object': {+ id: 'https://alice.example/notes/43',+ type: 'Note',+ attributedTo: 'https://alice.example/actors/1',+ cc: 'https://www.w3.org/ns/activitystreams#Public',+ // Stay under the express.json() 500kb body limit, but above the JSON-LD signature check limit+ content: 'a'.repeat(300 * 1024)+ }+ }++ const { fail, statusCode } = await postActivity(activity)+ expect(fail).to.be.true+ expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403)+ })+it('Should refresh the actor keys', async function () {this.timeout(20000)diff --git a/server/core/controllers/activitypub/inbox.ts b/server/core/controllers/activitypub/inbox.tsindex cb2625ec59..fafc038aa8 100644--- a/server/core/controllers/activitypub/inbox.ts+++ b/server/core/controllers/activitypub/inbox.ts@@ -1,5 +1,6 @@import { Activity, ActivityPubCollection, ActivityPubOrderedCollection, HttpStatusCode, RootActivity } from '@peertube/peertube-models'import { InboxManager } from '@server/lib/activitypub/inbox-manager.js'+import { ACTIVITY_PUB } from '../../initializers/constants.js'import express from 'express'import { isActivityValid } from '../../helpers/custom-validators/activitypub/activity.js'import { createLogger } from '../../helpers/logger.js'@@ -66,6 +67,20 @@ function inboxController (req: express.Request, res: express.Response) {activities = [ rootActivity as Activity ]}+ if (activities.length > ACTIVITY_PUB.MAX_ACTIVITIES_PER_REQUEST) {+ logger.warn(+ 'Rejecting inbox request with %d activities by %s (max: %d). No compliant server sends that many activities in one request.',+ activities.length,+ res.locals.signature.actor.url,+ ACTIVITY_PUB.MAX_ACTIVITIES_PER_REQUEST+ )++ return res.fail({+ status: HttpStatusCode.BAD_REQUEST_400,+ message: `Too many activities in a single inbox request (max: ${ACTIVITY_PUB.MAX_ACTIVITIES_PER_REQUEST})`+ })+ }+// Only keep activities we are able to processlogger.debug('Filtering %d activities...', activities.length, { activities })activities = activities.filter(a => isActivityValid(a))diff --git a/server/core/helpers/custom-validators/activitypub/videos.ts b/server/core/helpers/custom-validators/activitypub/videos.tsindex 1d63f094e4..21f7e2e89a 100644--- a/server/core/helpers/custom-validators/activitypub/videos.ts+++ b/server/core/helpers/custom-validators/activitypub/videos.ts@@ -12,7 +12,7 @@ import {import { createLogger } from '@server/helpers/logger.js'import { spdxToPeertubeLicence } from '@server/helpers/video.js'import validator from 'validator'-import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js'+import { CONSTRAINTS_FIELDS, ACTIVITY_PUB, MIMETYPES } from '../../../initializers/constants.js'import { peertubeTruncate } from '../../core-utils.js'import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc.js'import { isLiveDvrWindowValid, isLiveLatencyModeValid } from '../video-lives.js'@@ -169,6 +169,12 @@ export function setAPCaptionUrlObject (url: any): url is ActivityCaptionUrlObjecfunction setValidRemoteTags (video: VideoObject) {if (Array.isArray(video.tag) === false) video.tag = []+ if (video.tag.length > ACTIVITY_PUB.MAX_VIDEO_TAGS) {+ logger.debug('Truncating remote video tag array from %d to %d entries.', video.tag.length, ACTIVITY_PUB.MAX_VIDEO_TAGS)++ video.tag = video.tag.slice(0, ACTIVITY_PUB.MAX_VIDEO_TAGS)+ }+video.tag = video.tag.filter(t => {return (t.type === 'Hashtag' && isVideoTagValid(t.name)) ||(t.type === 'SensitiveTag' && !!t.name)@@ -220,8 +226,25 @@ function isRemoteVideoContentValid (mediaType: string, content: string) {function setValidRemoteVideoUrls (video: any) {if (Array.isArray(video.url) === false) return false+ if (video.url.length > ACTIVITY_PUB.MAX_VIDEO_URLS) {+ logger.debug('Truncating remote video url array from %d to %d entries.', video.url.length, ACTIVITY_PUB.MAX_VIDEO_URLS)++ video.url = video.url.slice(0, ACTIVITY_PUB.MAX_VIDEO_URLS)+ }+video.url = video.url.filter(u => isRemoteVideoUrlValid(u))+ // HLS playlist url objects carry their own nested `tag` array (video/torrent/magnet/metadata links + segment hashes)+ for (const u of video.url) {+ if (!Array.isArray(u.tag)) continue++ if (u.tag.length > ACTIVITY_PUB.MAX_VIDEO_URLS) {+ logger.debug('Truncating remote HLS playlist tag array from %d to %d entries.', u.tag.length, ACTIVITY_PUB.MAX_VIDEO_URLS)++ u.tag = u.tag.slice(0, ACTIVITY_PUB.MAX_VIDEO_URLS)+ }+ }+return true}diff --git a/server/core/helpers/peertube-jsonld.ts b/server/core/helpers/peertube-jsonld.tsindex 97a3c5b733..7e9419d4ad 100644--- a/server/core/helpers/peertube-jsonld.ts+++ b/server/core/helpers/peertube-jsonld.ts@@ -2,6 +2,7 @@ import { omit } from '@peertube/peertube-core-utils'import { sha256 } from '@peertube/peertube-node-utils'import { createSign, createVerify } from 'crypto'import cloneDeep from 'lodash-es/cloneDeep.js'+import { ACTIVITY_PUB } from '@server/initializers/constants.js'import { MActor } from '../types/models/index.js'import { getAllContext } from './activity-pub-utils.js'import { jsonld } from './custom-jsonld-signature.js'@@ -13,32 +14,58 @@ const logger = createLogger()type ExpressRequest = { body: any }-export function compactJSONLDAndCheckSignature (fromActor: MActor, req: ExpressRequest): Promise<boolean> {+export async function compactJSONLDAndCheckSignature (fromActor: MActor, req: ExpressRequest): Promise<boolean> {if (req.body.signature.type === 'RsaSignature2017') {+ // URDNA2015 canonicalization of the whole body is ~O(n²) in the number of blank nodes+ // It runs before the signature is verified, so bound the input size before canonicalizing+ const bodySize = Buffer.byteLength(JSON.stringify(req.body))+ if (bodySize > ACTIVITY_PUB.CHECK_JSON_LD_SIGNATURE.MAX_BODY_SIZE) {+ logger.warn(+ 'Refusing to check JSON-LD signature of a %d bytes body sent by %s (max: %d).',+ bodySize,+ fromActor.url,+ ACTIVITY_PUB.CHECK_JSON_LD_SIGNATURE.MAX_BODY_SIZE+ )++ return false+ }+return compactJSONLDAndCheckRSA2017Signature(fromActor, req)}logger.warn('Unknown JSON LD signature %s.', req.body.signature.type, req.body)- return Promise.resolve(false)+ return false}// Backward compatibility with "other" implementationsexport async function compactJSONLDAndCheckRSA2017Signature (fromActor: MActor, req: ExpressRequest) {- const compacted = await jsonldCompact(omit(req.body, [ 'signature' ]))+ const { verified, body } = await checkRSA2017Signature(fromActor.publicKey, req.body)++ req.body = body++ return verified+}++// Core signature verification logic (canonicalization + RSA check)+async function checkRSA2017Signature (publicKey: string, originalBody: any): Promise<{+ verified: boolean+ body: any+}> {+ const compacted = await jsonldCompact(omit(originalBody, [ 'signature' ]))- fixCompacted(req.body, compacted)+ fixCompacted(originalBody, compacted)- req.body = { ...compacted, signature: req.body.signature }+ const body = { ...compacted, signature: originalBody.signature }if (containInvalidJsonldKeys(compacted)) {logger.warn('JSON-LD @included, @graph or @reverse are not supported')- return false+ return { verified: false, body }}const [ documentHash, optionsHash ] = await Promise.all([hashObject(compacted),- createSignatureHash(req.body.signature)+ createSignatureHash(body.signature)])const toVerify = optionsHash + documentHash@@ -46,7 +73,9 @@ export async function compactJSONLDAndCheckRSA2017Signature (fromActor: MActor,const verify = createVerify('RSA-SHA256')verify.update(toVerify, 'utf8')- return verify.verify(fromActor.publicKey, req.body.signature.signatureValue, 'base64')+ const verified = verify.verify(publicKey, body.signature.signatureValue, 'base64')++ return { verified, body }}function fixCompacted (original: any, compacted: any, depth = 1) {@@ -130,8 +159,14 @@ function jsonldCompact (obj: any) {function jsonldNormalize (obj: any) {return (jsonld as any).promises.normalize(obj, {safe: true,- algorithm: 'URDNA2015',- format: 'application/n-quads'+ format: 'application/n-quads',+ canonizeOptions: {+ algorithm: 'URDNA2015',+ // Explicitly bound the work rdf-canonize does on indistinguishable blank nodes+ maxWorkFactor: 1,+ // Also bound canonicalization by wall-clock time, in case maxWorkFactor is not enough+ signal: AbortSignal.timeout(ACTIVITY_PUB.CHECK_JSON_LD_SIGNATURE.CANONIZE_TIMEOUT)+ }})}diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex b72627a836..270228b2cb 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -964,6 +964,15 @@ export const ACTIVITY_PUB = {COLLECTION_ITEMS_PER_PAGE: 10,FETCH_PAGE_LIMIT: 2000,MAX_RECURSION_COMMENTS: 100,+ MAX_ACTIVITIES_PER_REQUEST: 100,+ MAX_VIDEO_URLS: 100,+ MAX_VIDEO_TAGS: 100,+ MAX_PLAYLIST_ELEMENTS: 1000,+ CHECK_JSON_LD_SIGNATURE: {+ // Max body size (bytes) we accept to canonicalize for a LD signature check+ MAX_BODY_SIZE: 256 * 1024,+ CANONIZE_TIMEOUT: 1000 * 5 // 5 seconds+ },ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 daysVIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 daysVIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 daysdiff --git a/server/core/lib/activitypub/playlists/create-update.ts b/server/core/lib/activitypub/playlists/create-update.tsindex cb50651780..fb4481d9ae 100644--- a/server/core/lib/activitypub/playlists/create-update.ts+++ b/server/core/lib/activitypub/playlists/create-update.ts@@ -4,7 +4,7 @@ import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitimport { retryTransactionWrapper } from '@server/helpers/database-utils.js'import { createLogger } from '@server/helpers/logger.js'import { PeerTubeRequestError } from '@server/helpers/requests.js'-import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants.js'+import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants.js'import { sequelizeTypescript } from '@server/initializers/database.js'import { updateRemotePlaylistThumbnailFromUrl } from '@server/lib/thumbnail.js'import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'@@ -127,11 +127,35 @@ async function getRemotePlaylistChannel (playlistObject: PlaylistObject) {async function fetchElementUrls (playlistObject: PlaylistObject) {let accItems: string[] = []- await crawlCollectionPage<string>(playlistObject.id, items => {- accItems = accItems.concat(items)- return Promise.resolve()- })+ // Stop crawling remote pages entirely once we hit the element limit+ const abortController = new AbortController()++ try {+ await crawlCollectionPage<string>(+ playlistObject.id,+ items => {+ accItems = accItems.concat(items.slice(0, ACTIVITY_PUB.MAX_PLAYLIST_ELEMENTS - accItems.length))++ if (accItems.length >= ACTIVITY_PUB.MAX_PLAYLIST_ELEMENTS) {+ logger.warn(+ 'Playlist %s has more than %d elements, truncating and stopping the crawl.',+ getAPId(playlistObject),+ ACTIVITY_PUB.MAX_PLAYLIST_ELEMENTS+ )++ abortController.abort()+ }++ return Promise.resolve()+ },+ undefined,+ abortController.signal+ )+ } catch (err) {+ // We voluntarily aborted the crawl above once the element limit was reached, this is not an error+ if (abortController.signal.aborted !== true) throw err+ }return accItems.filter(i => isActivityPubUrlValid(i))}diff --git a/server/core/lib/activitypub/process/process.ts b/server/core/lib/activitypub/process/process.tsindex 2ecc747e16..48a218d45e 100644--- a/server/core/lib/activitypub/process/process.ts+++ b/server/core/lib/activitypub/process/process.ts@@ -1,5 +1,6 @@import { Activity, ActivityType } from '@peertube/peertube-models'import { StatsManager } from '@server/lib/stat-manager.js'+import { ACTIVITY_PUB } from '../../../initializers/constants.js'import { createLogger } from '../../../helpers/logger.js'import { APProcessorOptions } from '../../../types/activitypub-processor.model.js'import { MActorDefault, MActorSignature } from '../../../types/models/index.js'@@ -52,6 +53,12 @@ export async function processActivities () {const { outboxUrl, signatureActor, inboxActor, fromFetch = false } = options+ if (activities.length > ACTIVITY_PUB.MAX_ACTIVITIES_PER_REQUEST) {+ logger.warn('Truncating %d activities to process to %d.', activities.length, ACTIVITY_PUB.MAX_ACTIVITIES_PER_REQUEST)++ activities = activities.slice(0, ACTIVITY_PUB.MAX_ACTIVITIES_PER_REQUEST)+ }+const actorsCache: { [ url: string ]: MActorSignature } = {}for (const activity of activities) {diff --git a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.tsindex 71cfa5a36c..b155593232 100644--- a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts+++ b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts@@ -7,6 +7,7 @@ import {ActivitySensitiveTagObject,ActivityTagObject,ActivityUrlObject,+ ActivityVideoFileMetadataUrlObject,ActivityVideoUrlObject,NSFWFlag,stringToNSFWFlag,@@ -61,16 +62,14 @@ export function getFileAttributesFromUrl (const fileUrlObjects = urls.filter(u => isAPVideoUrlObject(u))if (fileUrlObjects.length === 0) return []+ const metadataIndex = buildMetadataIndex(urls)+ const magnetIndex = buildMagnetIndex(urls)+const result: { file: FilteredModelAttributes<VideoFileModel>, infoHash: string }[] = []for (const fileUrlObject of fileUrlObjects) {// Fetch associated metadata url, if any- const metadata = urls.filter(isAPVideoFileUrlMetadataObject)- .find(u => {- return u.height === fileUrlObject.height &&- u.fps === fileUrlObject.fps &&- u.rel.includes(fileUrlObject.mediaType)- })+ const metadata = findMetadata(metadataIndex, fileUrlObject)const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrlObject.mediaType)const resolution = Math.min(fileUrlObject.height ?? Infinity, fileUrlObject.width ?? Infinity)@@ -81,7 +80,7 @@ export function getFileAttributesFromUrl (const fileUrl = fileUrlObject.hrefconst existingFile = oldFiles.find(f => f.fileUrl === fileUrl)- const { torrentFilename, infoHash, torrentUrl } = getTorrentRelatedInfo({ videoOrPlaylist, urls, fileUrlObject, existingFile })+ const { torrentFilename, infoHash, torrentUrl } = getTorrentRelatedInfo({ videoOrPlaylist, magnetIndex, fileUrlObject, existingFile })const attribute: Partial<AttributesOnly<MVideoFile>> = {extname,@@ -340,6 +339,45 @@ export function getVideoAttributesFromObject (videoChannel: MChannelId, videoObj// Private// ---------------------------------------------------------------------------+// Pre-built indexes so getFileAttributesFromUrl stays O(n) in the number of attacker-controlled url entries+function buildMetadataIndex (urls: (ActivityTagObject | ActivityUrlObject)[]) {+ const index = new Map<string, ActivityVideoFileMetadataUrlObject[]>()++ for (const url of urls) {+ if (!isAPVideoFileUrlMetadataObject(url)) continue++ const key = `${url.height}-${url.fps}`++ const existing = index.get(key)+ if (existing) existing.push(url)+ else index.set(key, [ url ])+ }++ return index+}++function findMetadata (+ metadataIndex: Map<string, ActivityVideoFileMetadataUrlObject[]>,+ fileUrlObject: ActivityVideoUrlObject+) {+ const candidates = metadataIndex.get(`${fileUrlObject.height}-${fileUrlObject.fps}`)+ if (!candidates) return undefined++ return candidates.find(u => u.rel.includes(fileUrlObject.mediaType))+}++function buildMagnetIndex (urls: (ActivityTagObject | ActivityUrlObject)[]) {+ const index = new Map<number, ActivityMagnetUrlObject>()++ for (const url of urls) {+ if (!isAPMagnetUrlObject(url)) continue++ if (!index.has(url.height)) index.set(url.height, url)+ }++ return index+}+function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject {return !!MIMETYPES.AP_VIDEO.MIMETYPE_EXT[url.mediaType]}@@ -366,15 +404,14 @@ function isAPSensitiveTagObject (tag: any): tag is ActivitySensitiveTagObject {function getTorrentRelatedInfo (options: {videoOrPlaylist: MVideo | MStreamingPlaylistVideo- urls: (ActivityTagObject | ActivityUrlObject)[]+ magnetIndex: Map<number, ActivityMagnetUrlObject>fileUrlObject: ActivityVideoUrlObjectexistingFile?: MVideoFile}) {- const { urls, fileUrlObject, videoOrPlaylist, existingFile } = options+ const { magnetIndex, fileUrlObject, videoOrPlaylist, existingFile } = options// Fetch associated magnet uri- const magnet = urls.filter(isAPMagnetUrlObject)- .find(u => u.height === fileUrlObject.height)+ const magnet = magnetIndex.get(fileUrlObject.height)if (!magnet) {return {