ActivityPub-federated video streaming platform using P2P directly in your web browser
Refactors actor follow score handling and intervals
Chocobozzzaf2ffa61a9cc
diff --git a/packages/tests/src/api/server/handle-down.ts b/packages/tests/src/api/server/handle-down.tsindex 9e425325b9..a5ba8b5b29 100644--- a/packages/tests/src/api/server/handle-down.ts+++ b/packages/tests/src/api/server/handle-down.ts@@ -120,8 +120,9 @@ describe('Test handle downs', function () {await killallServers([ servers[1] ])// Remove server 2 follower- for (let i = 0; i < 10; i++) {+ for (let i = 0; i < 7; i++) {await servers[0].videos.upload({ attributes: videoAttributes })+ await wait(1000)}await waitJobs([ servers[0], servers[2] ])@@ -149,8 +150,6 @@ describe('Test handle downs', function () {}await waitJobs(servers[0])- // Wait scheduler- await wait(11000)// Only server 3 is still a follower of server 1const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })@@ -192,12 +191,10 @@ describe('Test handle downs', function () {})it('Should send an update to server 3, and automatically fetch the video', async function () {- this.timeout(15000)-{const { data } = await servers[2].videos.list()expect(data).to.be.an('array')- expect(data).to.have.lengthOf(11)+ expect(data).to.have.lengthOf(8)}await servers[0].videos.update({ id: missedVideo1.uuid })@@ -209,7 +206,7 @@ describe('Test handle downs', function () {const { data } = await servers[2].videos.list()expect(data).to.be.an('array')// 1 video is unlisted- expect(data).to.have.lengthOf(12)+ expect(data).to.have.lengthOf(9)}// Check unlisted video@@ -218,8 +215,6 @@ describe('Test handle downs', function () {})it('Should send comments on a video to server 3, and automatically fetch the video', async function () {- this.timeout(25000)-await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })await waitJobs(servers)@@ -295,12 +290,12 @@ describe('Test handle downs', function () {}await waitJobs(servers)- await sqlCommands[1].setActorFollowScores(20)+ await sqlCommands[1].setActorFollowScores(2000)// Wait video expirationawait wait(11000)- // Refresh video -> score + 10 = 30+ // Refresh video -> score + 1000 = 3000await servers[1].videos.get({ id: videoIdsServer1[0] })await waitJobs(servers)@@ -314,7 +309,7 @@ describe('Test handle downs', function () {// Wait video expirationawait wait(11000)- for (let i = 0; i < 5; i++) {+ for (let i = 0; i < 3; i++) {try {await servers[1].videos.get({ id: videoIdsServer1[i] })await waitJobs([ servers[1] ])diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex fba7ae7b19..cde932840f 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -191,9 +191,9 @@ export const ROUTE_CACHE_LIFETIME = {// Number of points we add/remove after a successful/bad requestexport const ACTOR_FOLLOW_SCORE = {- PENALTY: -10,- BONUS: 10,- BASE: 1000,+ PENALTY: -1000,+ BONUS: 1000,+ BASE: 5000,MAX: 10000}@@ -359,7 +359,7 @@ export const REQUEST_TIMEOUTS = {export const SCHEDULER_INTERVALS_MS = {RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE),- ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour+ ACTOR_FOLLOW_SCORES: 60000 * 60 * 20, // 20 hoursREMOVE_OLD_JOBS: 60000 * 60, // 1 hourUPDATE_VIDEOS: 60000, // 1 minuteUPDATE_TOKEN_SESSION: 60000, // 1 minute@@ -1279,8 +1279,6 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {if (isTestOrDevInstance()) {PRIVATE_RSA_KEY_SIZE = 1024- ACTOR_FOLLOW_SCORE.BASE = 20-REMOTE_SCHEME.HTTP = 'http'REMOTE_SCHEME.WS = 'ws'diff --git a/server/core/lib/actor-follow-health-cache.ts b/server/core/lib/actor-follow-health-cache.tsindex ec5b03d651..7d8c248897 100644--- a/server/core/lib/actor-follow-health-cache.ts+++ b/server/core/lib/actor-follow-health-cache.ts@@ -3,16 +3,16 @@ import { logger } from '../helpers/logger.js'// Cache follows scores, instead of writing them too often in database// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores-class ActorFollowHealthCache {-+export class ActorFollowHealthCache {private static instance: ActorFollowHealthCache- private pendingFollowsScore: { [ url: string ]: number } = {}+ private pendingFollowsScore: { [url: string]: number } = {}- private pendingBadServer = new Set<number>()- private pendingGoodServer = new Set<number>()+ private readonly pendingBadServer = new Set<number>()+ private readonly pendingGoodServer = new Set<number>()private readonly badInboxes = new Set<string>()+ private readonly goodInboxes = new Set<string>()private constructor () {}@@ -26,8 +26,10 @@ class ActorFollowHealthCache {if (goodInboxes.length === 0 && badInboxes.length === 0) returnlogger.info(- 'Updating %d good actor follows and %d bad actor follows scores in cache.',- goodInboxes.length, badInboxes.length, { badInboxes }+ 'Updating %d good actor follows and %d bad actor follows.',+ goodInboxes.length,+ badInboxes.length,+ { badInboxes })for (const goodInbox of goodInboxes) {@@ -48,39 +50,49 @@ class ActorFollowHealthCache {return this.badInboxes.has(inboxUrl)}+ getBadInboxes () {+ return new Set(this.badInboxes)+ }++ clearBadInboxes () {+ this.badInboxes.clear()+ }++ // ---------------------------------------------------------------------------++ getGoodInboxes () {+ return new Set(this.goodInboxes)+ }++ clearGoodInboxes () {+ this.goodInboxes.clear()+ }++ // ---------------------------------------------------------------------------+addBadServerId (serverId: number) {this.pendingBadServer.add(serverId)}getBadFollowingServerIds () {- return Array.from(this.pendingBadServer)+ return new Set(this.pendingBadServer)}clearBadFollowingServerIds () {- this.pendingBadServer = new Set<number>()+ this.pendingBadServer.clear()}+ // ---------------------------------------------------------------------------+addGoodServerId (serverId: number) {this.pendingGoodServer.add(serverId)}getGoodFollowingServerIds () {- return Array.from(this.pendingGoodServer)+ return new Set(this.pendingGoodServer)}clearGoodFollowingServerIds () {- this.pendingGoodServer = new Set<number>()- }-- getPendingFollowsScore () {- return this.pendingFollowsScore+ this.pendingGoodServer.clear()}-- clearPendingFollowsScore () {- this.pendingFollowsScore = {}- }-}--export {- ActorFollowHealthCache}diff --git a/server/core/lib/schedulers/actor-follow-scheduler.ts b/server/core/lib/schedulers/actor-follow-scheduler.tsindex c184fd4e58..25a6c538e8 100644--- a/server/core/lib/schedulers/actor-follow-scheduler.ts+++ b/server/core/lib/schedulers/actor-follow-scheduler.ts@@ -8,6 +8,7 @@ import { ActorFollowHealthCache } from '../actor-follow-health-cache.js'import { AbstractScheduler } from './abstract-scheduler.js'const lTags = loggerTagsFactory('schedulers')+const FOLLOW_RESEND_STALE_MS = 1000 * 60 * 60 * 24 * 7 // 7 daysconst FOLLOW_RESEND_BATCH_SIZE = 100@@ -23,7 +24,7 @@ export class ActorFollowScheduler extends AbstractScheduler {protected async internalExecute () {// Run too often in test/dev instancesif (isProdInstance()) {- logger.info('Processing pending actor follow scores.', lTags())+ logger.info('Processing actor follows scheduler.', lTags())}await this.processPendingScores()@@ -34,20 +35,30 @@ export class ActorFollowScheduler extends AbstractScheduler {}private async processPendingScores () {- const pendingScores = ActorFollowHealthCache.Instance.getPendingFollowsScore()+ const goodInboxes = ActorFollowHealthCache.Instance.getGoodInboxes()+ const badInboxes = ActorFollowHealthCache.Instance.getBadInboxes()const badServerIds = ActorFollowHealthCache.Instance.getBadFollowingServerIds()const goodServerIds = ActorFollowHealthCache.Instance.getGoodFollowingServerIds()- ActorFollowHealthCache.Instance.clearPendingFollowsScore()+ ActorFollowHealthCache.Instance.clearGoodInboxes()+ ActorFollowHealthCache.Instance.clearBadInboxes()ActorFollowHealthCache.Instance.clearBadFollowingServerIds()ActorFollowHealthCache.Instance.clearGoodFollowingServerIds()- for (const inbox of Object.keys(pendingScores)) {- await ActorFollowModel.updateScore(inbox, pendingScores[inbox])+ for (const goodInbox of goodInboxes) {+ if (badInboxes.has(goodInbox)) continue++ await ActorFollowModel.updateScore(goodInbox, ACTOR_FOLLOW_SCORE.BONUS)+ }++ for (const badInbox of badInboxes) {+ if (goodInboxes.has(badInbox)) continue++ await ActorFollowModel.updateScore(badInbox, ACTOR_FOLLOW_SCORE.PENALTY)}- await ActorFollowModel.updateScoreByFollowingServers(badServerIds, ACTOR_FOLLOW_SCORE.PENALTY)- await ActorFollowModel.updateScoreByFollowingServers(goodServerIds, ACTOR_FOLLOW_SCORE.BONUS)+ await ActorFollowModel.updateScoreByFollowingServers(Array.from(badServerIds), ACTOR_FOLLOW_SCORE.PENALTY)+ await ActorFollowModel.updateScoreByFollowingServers(Array.from(goodServerIds), ACTOR_FOLLOW_SCORE.BONUS)}private async removeBadActorFollows () {