Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Send periodically Follows

Chocobozzz01602019b060
diff --git a/packages/server-commands/src/server/servers.ts b/packages/server-commands/src/server/servers.ts
index 0a5a4a4cf5..19b8455b2e 100644
--- a/packages/server-commands/src/server/servers.ts
+++ b/packages/server-commands/src/server/servers.ts
@@ -2,7 +2,7 @@ import { ensureDir } from 'fs-extra/esm'
import { isGithubCI } from '@peertube/peertube-node-utils'
import { PeerTubeServer, RunServerOptions } from './server.js'
-async function createSingleServer (serverNumber: number, configOverride?: object, options: RunServerOptions = {}) {
+export async function createSingleServer (serverNumber: number, configOverride?: object, options: RunServerOptions = {}) {
const server = new PeerTubeServer({ serverNumber })
await server.flushAndRun(configOverride, options)
@@ -10,7 +10,7 @@ async function createSingleServer (serverNumber: number, configOverride?: object
return server
}
-function createMultipleServers (totalServers: number, configOverride?: object, options: RunServerOptions = {}) {
+export function createMultipleServers (totalServers: number, configOverride?: object, options: RunServerOptions = {}) {
const serverPromises: Promise<PeerTubeServer>[] = []
for (let i = 1; i <= totalServers; i++) {
@@ -20,11 +20,11 @@ function createMultipleServers (totalServers: number, configOverride?: object, o
return Promise.all(serverPromises)
}
-function killallServers (servers: PeerTubeServer[]) {
+export function killallServers (servers: PeerTubeServer[]) {
return Promise.all(servers.filter(s => !!s).map(s => s.kill()))
}
-async function cleanupTests (servers: PeerTubeServer[]) {
+export async function cleanupTests (servers: PeerTubeServer[]) {
await killallServers(servers)
if (isGithubCI()) {
@@ -42,7 +42,7 @@ async function cleanupTests (servers: PeerTubeServer[]) {
return Promise.all(p)
}
-function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {
+export function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {
return {
import: {
videos: {
@@ -59,13 +59,3 @@ function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {
}
}
}
-
-// ---------------------------------------------------------------------------
-
-export {
- createSingleServer,
- createMultipleServers,
- cleanupTests,
- killallServers,
- getServerImportConfig
-}
diff --git a/packages/tests/src/api/server/follows.ts b/packages/tests/src/api/server/follows.ts
index 0895147f47..ac422a55f1 100644
--- a/packages/tests/src/api/server/follows.ts
+++ b/packages/tests/src/api/server/follows.ts
@@ -1,10 +1,12 @@
/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import { Video, VideoCommentPolicy, VideoPrivacy } from '@peertube/peertube-models'
+import { wait } from '@peertube/peertube-core-utils'
import { PeerTubeServer, cleanupTests, createMultipleServers, setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands'
import { expectAccountFollows, expectChannelsFollows } from '@tests/shared/actors.js'
import { testCaptionFile } from '@tests/shared/captions.js'
import { dateIsValid } from '@tests/shared/checks.js'
+import { SQLCommand } from '@tests/shared/sql-command.js'
import { completeVideoCheck } from '@tests/shared/videos.js'
import { expect } from 'chai'
@@ -638,4 +640,70 @@ describe('Test follows', function () {
await cleanupTests(servers)
})
})
+
+ describe('Periodic stale follow resend', function () {
+ let servers: PeerTubeServer[] = []
+ let sqlCommands: SQLCommand[] = []
+
+ const DAY_IN_MS = 24 * 60 * 60 * 1000
+
+ before(async function () {
+ this.timeout(120000)
+
+ servers = await createMultipleServers(2)
+ await setAccessTokensToServers(servers)
+
+ sqlCommands = servers.map(s => new SQLCommand(s))
+ })
+
+ it('Should resend stale follows and set updatedAt to now', async function () {
+ this.timeout(60000)
+
+ await servers[0].follows.follow({ hosts: [ servers[1].url ] })
+
+ await waitJobs(servers)
+
+ await sqlCommands[0].setActorFollowUpdatedAt(new Date(Date.now() - 8 * DAY_IN_MS).toISOString())
+
+ {
+ await sqlCommands[1].deleteAll('actorFollow')
+
+ const { data } = await servers[1].follows.getFollowers()
+ expect(data).to.have.lengthOf(0)
+ }
+
+ await wait(2500)
+ await waitJobs(servers)
+
+ const updatedAt = await sqlCommands[0].getFirstActorFollowUpdatedAt()
+ expect(updatedAt).to.exist
+
+ expect(new Date(updatedAt).getTime()).to.be.greaterThan(Date.now() - 2 * 60 * 1000)
+
+ const { data } = await servers[1].follows.getFollowers()
+ expect(data).to.have.lengthOf(1)
+ })
+
+ it('Should not resend follows that are fresher than 7 days', async function () {
+ this.timeout(60000)
+
+ await sqlCommands[0].setActorFollowUpdatedAt(new Date(Date.now() - 1 * DAY_IN_MS).toISOString())
+
+ await wait(2500)
+ await waitJobs(servers)
+
+ const updatedAt = await sqlCommands[0].getFirstActorFollowUpdatedAt()
+ expect(updatedAt).to.exist
+
+ expect(new Date(updatedAt).getTime()).to.be.lessThan(Date.now() - 12 * 60 * 60 * 1000)
+ })
+
+ after(async function () {
+ await cleanupTests(servers)
+
+ for (const command of sqlCommands) {
+ await command.cleanup()
+ }
+ })
+ })
})
diff --git a/packages/tests/src/shared/sql-command.ts b/packages/tests/src/shared/sql-command.ts
index b0083b49b4..54d4ed27e0 100644
--- a/packages/tests/src/shared/sql-command.ts
+++ b/packages/tests/src/shared/sql-command.ts
@@ -167,6 +167,20 @@ export class SQLCommand {
return this.updateQuery(`UPDATE "actorFollow" SET "score" = :newScore`, { newScore })
}
+ setActorFollowUpdatedAt (updatedAt: string) {
+ return this.updateQuery(`UPDATE "actorFollow" SET "updatedAt" = :updatedAt`, { updatedAt })
+ }
+
+ async getFirstActorFollowUpdatedAt () {
+ const rows = await this.selectQuery<{ updatedAt: string }>(
+ `SELECT "updatedAt" FROM "actorFollow" ORDER BY "id" ASC LIMIT 1`
+ )
+
+ return rows[0]?.updatedAt
+ }
+
+ // ---------------------------------------------------------------------------
+
setTokenField (accessToken: string, field: string, value: string) {
return this.updateQuery(
`UPDATE "oAuthToken" SET ${this.escapeColumnName(field)} = :value WHERE "accessToken" = :accessToken`,
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index 3fa728e4c2..c335698cb2 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -1286,7 +1286,6 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {
STATIC_MAX_AGE.SERVER = '0'
- SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES = 1000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_JOBS = 10000
SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY = 5000
SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS = 5000
@@ -1320,6 +1319,8 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {
}
if (isTestInstance()) {
+ SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES = 1000
+
ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
diff --git a/server/core/lib/activitypub/send/send-follow.ts b/server/core/lib/activitypub/send/send-follow.ts
index 26ea030ce5..9e34ad94a3 100644
--- a/server/core/lib/activitypub/send/send-follow.ts
+++ b/server/core/lib/activitypub/send/send-follow.ts
@@ -3,8 +3,9 @@ import { ActivityFollow } from '@peertube/peertube-models'
import { logger } from '../../../helpers/logger.js'
import { MActor, MActorFollowActors } from '../../../types/models/index.js'
import { unicastTo } from './shared/send-utils.js'
+import { afterCommitIfTransaction } from '@server/helpers/database-utils.js'
-function sendFollow (actorFollow: MActorFollowActors, t: Transaction) {
+export function sendFollow (actorFollow: MActorFollowActors, t: Transaction | undefined) {
const me = actorFollow.ActorFollower
const following = actorFollow.ActorFollowing
@@ -15,12 +16,12 @@ function sendFollow (actorFollow: MActorFollowActors, t: Transaction) {
const data = buildFollowActivity(actorFollow.url, me, following)
- return t.afterCommit(() => {
+ afterCommitIfTransaction(t, () => {
return unicastTo({ data, byActor: me, toActorUrl: following.inboxUrl, contextType: 'Follow' })
})
}
-function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor): ActivityFollow {
+export function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor): ActivityFollow {
return {
type: 'Follow',
id: url,
@@ -28,10 +29,3 @@ function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor)
object: targetActor.url
}
}
-
-// ---------------------------------------------------------------------------
-
-export {
- sendFollow,
- buildFollowActivity
-}
diff --git a/server/core/lib/schedulers/actor-follow-scheduler.ts b/server/core/lib/schedulers/actor-follow-scheduler.ts
index 9ca42f817b..c184fd4e58 100644
--- a/server/core/lib/schedulers/actor-follow-scheduler.ts
+++ b/server/core/lib/schedulers/actor-follow-scheduler.ts
@@ -1,4 +1,6 @@
import { isProdInstance, isTestOrDevInstance } from '@peertube/peertube-node-utils'
+import { sendFollow } from '@server/lib/activitypub/send/send-follow.js'
+import { setAsUpdated } from '@server/models/shared/update.js'
import { logger, loggerTagsFactory } from '../../helpers/logger.js'
import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
import { ActorFollowModel } from '../../models/actor/actor-follow.js'
@@ -6,6 +8,8 @@ import { ActorFollowHealthCache } from '../actor-follow-health-cache.js'
import { AbstractScheduler } from './abstract-scheduler.js'
const lTags = loggerTagsFactory('schedulers')
+const FOLLOW_RESEND_STALE_MS = 1000 * 60 * 60 * 24 * 7 // 7 days
+const FOLLOW_RESEND_BATCH_SIZE = 100
export class ActorFollowScheduler extends AbstractScheduler {
private static instance: AbstractScheduler
@@ -25,6 +29,8 @@ export class ActorFollowScheduler extends AbstractScheduler {
await this.processPendingScores()
await this.removeBadActorFollows()
+
+ await this.resendStaleActorFollows()
}
private async processPendingScores () {
@@ -54,6 +60,31 @@ export class ActorFollowScheduler extends AbstractScheduler {
}
}
+ private async resendStaleActorFollows () {
+ const olderThan = new Date(Date.now() - FOLLOW_RESEND_STALE_MS)
+
+ try {
+ const actorFollows = await ActorFollowModel.listOutgoingStaleForResend({
+ olderThan,
+ limit: FOLLOW_RESEND_BATCH_SIZE
+ })
+
+ if (actorFollows.length === 0) return
+
+ for (const actorFollow of actorFollows) {
+ sendFollow(actorFollow, undefined)
+
+ await setAsUpdated({ sequelize: ActorFollowModel.sequelize, table: 'actorFollow', id: actorFollow.id })
+ }
+
+ if (!isTestOrDevInstance()) {
+ logger.info('Queued %d stale actor follows for resend.', actorFollows.length, lTags())
+ }
+ } catch (err) {
+ logger.error('Error in stale actor follow resend scheduler.', { err, ...lTags() })
+ }
+ }
+
static get Instance () {
return this.instance || (this.instance = new this())
}
diff --git a/server/core/models/actor/actor-follow.ts b/server/core/models/actor/actor-follow.ts
index 5a623ca54d..97a2a9edd0 100644
--- a/server/core/models/actor/actor-follow.ts
+++ b/server/core/models/actor/actor-follow.ts
@@ -47,6 +47,7 @@ import { VideoChannelModel } from '../video/video-channel.js'
import { ActorModel, unusedActorAttributesForAPI } from './actor.js'
import { InstanceListFollowersQueryBuilder, ListFollowersOptions } from './sql/instance-list-followers-query-builder.js'
import { InstanceListFollowingQueryBuilder, ListFollowingOptions } from './sql/instance-list-following-query-builder.js'
+import { isTestInstance } from '@peertube/peertube-node-utils'
@Table({
tableName: 'actorFollow',
@@ -504,6 +505,47 @@ export class ActorFollowModel extends SequelizeModel<ActorFollowModel> {
return difference(hosts, followedHosts)
}
+ static listOutgoingStaleForResend (options: {
+ olderThan: Date
+ limit: number
+ }) {
+ const { olderThan, limit } = options
+
+ return ActorFollowModel.findAll<MActorFollowActors>({
+ logging: !isTestInstance(),
+ where: {
+ state: {
+ [Op.in]: [ 'pending', 'accepted' ]
+ },
+ updatedAt: {
+ [Op.lt]: olderThan
+ }
+ },
+ include: [
+ {
+ model: ActorModel.unscoped(),
+ required: true,
+ as: 'ActorFollower',
+ where: {
+ serverId: null
+ }
+ },
+ {
+ model: ActorModel.unscoped(),
+ required: true,
+ as: 'ActorFollowing',
+ where: {
+ serverId: {
+ [Op.ne]: null
+ }
+ }
+ }
+ ],
+ order: [ [ 'updatedAt', 'ASC' ] ],
+ limit
+ })
+ }
+
// ---------------------------------------------------------------------------
static listAcceptedFollowerUrlsForAP (actorIds: number[], t: Transaction, start?: number, count?: number) {