Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Refactors actor follow score handling and intervals

Chocobozzzaf2ffa61a9cc
diff --git a/packages/tests/src/api/server/handle-down.ts b/packages/tests/src/api/server/handle-down.ts
index 9e425325b9..a5ba8b5b29 100644
--- a/packages/tests/src/api/server/handle-down.ts
+++ b/packages/tests/src/api/server/handle-down.ts
@@ -120,8 +120,9 @@ describe('Test handle downs', function () {
await killallServers([ servers[1] ])
// Remove server 2 follower
- for (let i = 0; i < 10; i++) {
+ for (let i = 0; i < 7; i++) {
await servers[0].videos.upload({ attributes: videoAttributes })
+ await wait(1000)
}
await waitJobs([ servers[0], servers[2] ])
@@ -149,8 +150,6 @@ describe('Test handle downs', function () {
}
await waitJobs(servers[0])
- // Wait scheduler
- await wait(11000)
// Only server 3 is still a follower of server 1
const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' })
@@ -192,12 +191,10 @@ describe('Test handle downs', function () {
})
it('Should send an update to server 3, and automatically fetch the video', async function () {
- this.timeout(15000)
-
{
const { data } = await servers[2].videos.list()
expect(data).to.be.an('array')
- expect(data).to.have.lengthOf(11)
+ expect(data).to.have.lengthOf(8)
}
await servers[0].videos.update({ id: missedVideo1.uuid })
@@ -209,7 +206,7 @@ describe('Test handle downs', function () {
const { data } = await servers[2].videos.list()
expect(data).to.be.an('array')
// 1 video is unlisted
- expect(data).to.have.lengthOf(12)
+ expect(data).to.have.lengthOf(9)
}
// Check unlisted video
@@ -218,8 +215,6 @@ describe('Test handle downs', function () {
})
it('Should send comments on a video to server 3, and automatically fetch the video', async function () {
- this.timeout(25000)
-
await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' })
await waitJobs(servers)
@@ -295,12 +290,12 @@ describe('Test handle downs', function () {
}
await waitJobs(servers)
- await sqlCommands[1].setActorFollowScores(20)
+ await sqlCommands[1].setActorFollowScores(2000)
// Wait video expiration
await wait(11000)
- // Refresh video -> score + 10 = 30
+ // Refresh video -> score + 1000 = 3000
await servers[1].videos.get({ id: videoIdsServer1[0] })
await waitJobs(servers)
@@ -314,7 +309,7 @@ describe('Test handle downs', function () {
// Wait video expiration
await wait(11000)
- for (let i = 0; i < 5; i++) {
+ for (let i = 0; i < 3; i++) {
try {
await servers[1].videos.get({ id: videoIdsServer1[i] })
await waitJobs([ servers[1] ])
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index fba7ae7b19..cde932840f 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -191,9 +191,9 @@ export const ROUTE_CACHE_LIFETIME = {
// Number of points we add/remove after a successful/bad request
export const ACTOR_FOLLOW_SCORE = {
- PENALTY: -10,
- BONUS: 10,
- BASE: 1000,
+ PENALTY: -1000,
+ BONUS: 1000,
+ BASE: 5000,
MAX: 10000
}
@@ -359,7 +359,7 @@ export const REQUEST_TIMEOUTS = {
export const SCHEDULER_INTERVALS_MS = {
RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE),
- ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour
+ ACTOR_FOLLOW_SCORES: 60000 * 60 * 20, // 20 hours
REMOVE_OLD_JOBS: 60000 * 60, // 1 hour
UPDATE_VIDEOS: 60000, // 1 minute
UPDATE_TOKEN_SESSION: 60000, // 1 minute
@@ -1279,8 +1279,6 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {
if (isTestOrDevInstance()) {
PRIVATE_RSA_KEY_SIZE = 1024
- ACTOR_FOLLOW_SCORE.BASE = 20
-
REMOTE_SCHEME.HTTP = 'http'
REMOTE_SCHEME.WS = 'ws'
diff --git a/server/core/lib/actor-follow-health-cache.ts b/server/core/lib/actor-follow-health-cache.ts
index ec5b03d651..7d8c248897 100644
--- a/server/core/lib/actor-follow-health-cache.ts
+++ b/server/core/lib/actor-follow-health-cache.ts
@@ -3,16 +3,16 @@ import { logger } from '../helpers/logger.js'
// Cache follows scores, instead of writing them too often in database
// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores
-class ActorFollowHealthCache {
-
+export class ActorFollowHealthCache {
private static instance: ActorFollowHealthCache
- private pendingFollowsScore: { [ url: string ]: number } = {}
+ private pendingFollowsScore: { [url: string]: number } = {}
- private pendingBadServer = new Set<number>()
- private pendingGoodServer = new Set<number>()
+ private readonly pendingBadServer = new Set<number>()
+ private readonly pendingGoodServer = new Set<number>()
private readonly badInboxes = new Set<string>()
+ private readonly goodInboxes = new Set<string>()
private constructor () {}
@@ -26,8 +26,10 @@ class ActorFollowHealthCache {
if (goodInboxes.length === 0 && badInboxes.length === 0) return
logger.info(
- 'Updating %d good actor follows and %d bad actor follows scores in cache.',
- goodInboxes.length, badInboxes.length, { badInboxes }
+ 'Updating %d good actor follows and %d bad actor follows.',
+ goodInboxes.length,
+ badInboxes.length,
+ { badInboxes }
)
for (const goodInbox of goodInboxes) {
@@ -48,39 +50,49 @@ class ActorFollowHealthCache {
return this.badInboxes.has(inboxUrl)
}
+ getBadInboxes () {
+ return new Set(this.badInboxes)
+ }
+
+ clearBadInboxes () {
+ this.badInboxes.clear()
+ }
+
+ // ---------------------------------------------------------------------------
+
+ getGoodInboxes () {
+ return new Set(this.goodInboxes)
+ }
+
+ clearGoodInboxes () {
+ this.goodInboxes.clear()
+ }
+
+ // ---------------------------------------------------------------------------
+
addBadServerId (serverId: number) {
this.pendingBadServer.add(serverId)
}
getBadFollowingServerIds () {
- return Array.from(this.pendingBadServer)
+ return new Set(this.pendingBadServer)
}
clearBadFollowingServerIds () {
- this.pendingBadServer = new Set<number>()
+ this.pendingBadServer.clear()
}
+ // ---------------------------------------------------------------------------
+
addGoodServerId (serverId: number) {
this.pendingGoodServer.add(serverId)
}
getGoodFollowingServerIds () {
- return Array.from(this.pendingGoodServer)
+ return new Set(this.pendingGoodServer)
}
clearGoodFollowingServerIds () {
- this.pendingGoodServer = new Set<number>()
- }
-
- getPendingFollowsScore () {
- return this.pendingFollowsScore
+ this.pendingGoodServer.clear()
}
-
- clearPendingFollowsScore () {
- this.pendingFollowsScore = {}
- }
-}
-
-export {
- ActorFollowHealthCache
}
diff --git a/server/core/lib/schedulers/actor-follow-scheduler.ts b/server/core/lib/schedulers/actor-follow-scheduler.ts
index c184fd4e58..25a6c538e8 100644
--- a/server/core/lib/schedulers/actor-follow-scheduler.ts
+++ b/server/core/lib/schedulers/actor-follow-scheduler.ts
@@ -8,6 +8,7 @@ import { ActorFollowHealthCache } from '../actor-follow-health-cache.js'
import { AbstractScheduler } from './abstract-scheduler.js'
const lTags = loggerTagsFactory('schedulers')
+
const FOLLOW_RESEND_STALE_MS = 1000 * 60 * 60 * 24 * 7 // 7 days
const FOLLOW_RESEND_BATCH_SIZE = 100
@@ -23,7 +24,7 @@ export class ActorFollowScheduler extends AbstractScheduler {
protected async internalExecute () {
// Run too often in test/dev instances
if (isProdInstance()) {
- logger.info('Processing pending actor follow scores.', lTags())
+ logger.info('Processing actor follows scheduler.', lTags())
}
await this.processPendingScores()
@@ -34,20 +35,30 @@ export class ActorFollowScheduler extends AbstractScheduler {
}
private async processPendingScores () {
- const pendingScores = ActorFollowHealthCache.Instance.getPendingFollowsScore()
+ const goodInboxes = ActorFollowHealthCache.Instance.getGoodInboxes()
+ const badInboxes = ActorFollowHealthCache.Instance.getBadInboxes()
const badServerIds = ActorFollowHealthCache.Instance.getBadFollowingServerIds()
const goodServerIds = ActorFollowHealthCache.Instance.getGoodFollowingServerIds()
- ActorFollowHealthCache.Instance.clearPendingFollowsScore()
+ ActorFollowHealthCache.Instance.clearGoodInboxes()
+ ActorFollowHealthCache.Instance.clearBadInboxes()
ActorFollowHealthCache.Instance.clearBadFollowingServerIds()
ActorFollowHealthCache.Instance.clearGoodFollowingServerIds()
- for (const inbox of Object.keys(pendingScores)) {
- await ActorFollowModel.updateScore(inbox, pendingScores[inbox])
+ for (const goodInbox of goodInboxes) {
+ if (badInboxes.has(goodInbox)) continue
+
+ await ActorFollowModel.updateScore(goodInbox, ACTOR_FOLLOW_SCORE.BONUS)
+ }
+
+ for (const badInbox of badInboxes) {
+ if (goodInboxes.has(badInbox)) continue
+
+ await ActorFollowModel.updateScore(badInbox, ACTOR_FOLLOW_SCORE.PENALTY)
}
- await ActorFollowModel.updateScoreByFollowingServers(badServerIds, ACTOR_FOLLOW_SCORE.PENALTY)
- await ActorFollowModel.updateScoreByFollowingServers(goodServerIds, ACTOR_FOLLOW_SCORE.BONUS)
+ await ActorFollowModel.updateScoreByFollowingServers(Array.from(badServerIds), ACTOR_FOLLOW_SCORE.PENALTY)
+ await ActorFollowModel.updateScoreByFollowingServers(Array.from(goodServerIds), ACTOR_FOLLOW_SCORE.BONUS)
}
private async removeBadActorFollows () {