ActivityPub-federated video streaming platform using P2P directly in your web browser
Send periodically Follows
Chocobozzz01602019b060
diff --git a/packages/server-commands/src/server/servers.ts b/packages/server-commands/src/server/servers.tsindex 0a5a4a4cf5..19b8455b2e 100644--- a/packages/server-commands/src/server/servers.ts+++ b/packages/server-commands/src/server/servers.ts@@ -2,7 +2,7 @@ import { ensureDir } from 'fs-extra/esm'import { isGithubCI } from '@peertube/peertube-node-utils'import { PeerTubeServer, RunServerOptions } from './server.js'-async function createSingleServer (serverNumber: number, configOverride?: object, options: RunServerOptions = {}) {+export async function createSingleServer (serverNumber: number, configOverride?: object, options: RunServerOptions = {}) {const server = new PeerTubeServer({ serverNumber })await server.flushAndRun(configOverride, options)@@ -10,7 +10,7 @@ async function createSingleServer (serverNumber: number, configOverride?: objectreturn server}-function createMultipleServers (totalServers: number, configOverride?: object, options: RunServerOptions = {}) {+export function createMultipleServers (totalServers: number, configOverride?: object, options: RunServerOptions = {}) {const serverPromises: Promise<PeerTubeServer>[] = []for (let i = 1; i <= totalServers; i++) {@@ -20,11 +20,11 @@ function createMultipleServers (totalServers: number, configOverride?: object, oreturn Promise.all(serverPromises)}-function killallServers (servers: PeerTubeServer[]) {+export function killallServers (servers: PeerTubeServer[]) {return Promise.all(servers.filter(s => !!s).map(s => s.kill()))}-async function cleanupTests (servers: PeerTubeServer[]) {+export async function cleanupTests (servers: PeerTubeServer[]) {await killallServers(servers)if (isGithubCI()) {@@ -42,7 +42,7 @@ async function cleanupTests (servers: PeerTubeServer[]) {return Promise.all(p)}-function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {+export function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {return {import: {videos: {@@ -59,13 +59,3 @@ function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') {}}}--// -----------------------------------------------------------------------------export {- createSingleServer,- createMultipleServers,- cleanupTests,- killallServers,- getServerImportConfig-}diff --git a/packages/tests/src/api/server/follows.ts b/packages/tests/src/api/server/follows.tsindex 0895147f47..ac422a55f1 100644--- a/packages/tests/src/api/server/follows.ts+++ b/packages/tests/src/api/server/follows.ts@@ -1,10 +1,12 @@/* oxlint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */import { Video, VideoCommentPolicy, VideoPrivacy } from '@peertube/peertube-models'+import { wait } from '@peertube/peertube-core-utils'import { PeerTubeServer, cleanupTests, createMultipleServers, setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands'import { expectAccountFollows, expectChannelsFollows } from '@tests/shared/actors.js'import { testCaptionFile } from '@tests/shared/captions.js'import { dateIsValid } from '@tests/shared/checks.js'+import { SQLCommand } from '@tests/shared/sql-command.js'import { completeVideoCheck } from '@tests/shared/videos.js'import { expect } from 'chai'@@ -638,4 +640,70 @@ describe('Test follows', function () {await cleanupTests(servers)})})++ describe('Periodic stale follow resend', function () {+ let servers: PeerTubeServer[] = []+ let sqlCommands: SQLCommand[] = []++ const DAY_IN_MS = 24 * 60 * 60 * 1000++ before(async function () {+ this.timeout(120000)++ servers = await createMultipleServers(2)+ await setAccessTokensToServers(servers)++ sqlCommands = servers.map(s => new SQLCommand(s))+ })++ it('Should resend stale follows and set updatedAt to now', async function () {+ this.timeout(60000)++ await servers[0].follows.follow({ hosts: [ servers[1].url ] })++ await waitJobs(servers)++ await sqlCommands[0].setActorFollowUpdatedAt(new Date(Date.now() - 8 * DAY_IN_MS).toISOString())++ {+ await sqlCommands[1].deleteAll('actorFollow')++ const { data } = await servers[1].follows.getFollowers()+ expect(data).to.have.lengthOf(0)+ }++ await wait(2500)+ await waitJobs(servers)++ const updatedAt = await sqlCommands[0].getFirstActorFollowUpdatedAt()+ expect(updatedAt).to.exist++ expect(new Date(updatedAt).getTime()).to.be.greaterThan(Date.now() - 2 * 60 * 1000)++ const { data } = await servers[1].follows.getFollowers()+ expect(data).to.have.lengthOf(1)+ })++ it('Should not resend follows that are fresher than 7 days', async function () {+ this.timeout(60000)++ await sqlCommands[0].setActorFollowUpdatedAt(new Date(Date.now() - 1 * DAY_IN_MS).toISOString())++ await wait(2500)+ await waitJobs(servers)++ const updatedAt = await sqlCommands[0].getFirstActorFollowUpdatedAt()+ expect(updatedAt).to.exist++ expect(new Date(updatedAt).getTime()).to.be.lessThan(Date.now() - 12 * 60 * 60 * 1000)+ })++ after(async function () {+ await cleanupTests(servers)++ for (const command of sqlCommands) {+ await command.cleanup()+ }+ })+ })})diff --git a/packages/tests/src/shared/sql-command.ts b/packages/tests/src/shared/sql-command.tsindex b0083b49b4..54d4ed27e0 100644--- a/packages/tests/src/shared/sql-command.ts+++ b/packages/tests/src/shared/sql-command.ts@@ -167,6 +167,20 @@ export class SQLCommand {return this.updateQuery(`UPDATE "actorFollow" SET "score" = :newScore`, { newScore })}+ setActorFollowUpdatedAt (updatedAt: string) {+ return this.updateQuery(`UPDATE "actorFollow" SET "updatedAt" = :updatedAt`, { updatedAt })+ }++ async getFirstActorFollowUpdatedAt () {+ const rows = await this.selectQuery<{ updatedAt: string }>(+ `SELECT "updatedAt" FROM "actorFollow" ORDER BY "id" ASC LIMIT 1`+ )++ return rows[0]?.updatedAt+ }++ // ---------------------------------------------------------------------------+setTokenField (accessToken: string, field: string, value: string) {return this.updateQuery(`UPDATE "oAuthToken" SET ${this.escapeColumnName(field)} = :value WHERE "accessToken" = :accessToken`,diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex 3fa728e4c2..c335698cb2 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -1286,7 +1286,6 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {STATIC_MAX_AGE.SERVER = '0'- SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES = 1000SCHEDULER_INTERVALS_MS.REMOVE_OLD_JOBS = 10000SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY = 5000SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS = 5000@@ -1320,6 +1319,8 @@ if (process.env.PRODUCTION_CONSTANTS !== 'true') {}if (isTestInstance()) {+ SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES = 1000+ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 secondsACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 secondsdiff --git a/server/core/lib/activitypub/send/send-follow.ts b/server/core/lib/activitypub/send/send-follow.tsindex 26ea030ce5..9e34ad94a3 100644--- a/server/core/lib/activitypub/send/send-follow.ts+++ b/server/core/lib/activitypub/send/send-follow.ts@@ -3,8 +3,9 @@ import { ActivityFollow } from '@peertube/peertube-models'import { logger } from '../../../helpers/logger.js'import { MActor, MActorFollowActors } from '../../../types/models/index.js'import { unicastTo } from './shared/send-utils.js'+import { afterCommitIfTransaction } from '@server/helpers/database-utils.js'-function sendFollow (actorFollow: MActorFollowActors, t: Transaction) {+export function sendFollow (actorFollow: MActorFollowActors, t: Transaction | undefined) {const me = actorFollow.ActorFollowerconst following = actorFollow.ActorFollowing@@ -15,12 +16,12 @@ function sendFollow (actorFollow: MActorFollowActors, t: Transaction) {const data = buildFollowActivity(actorFollow.url, me, following)- return t.afterCommit(() => {+ afterCommitIfTransaction(t, () => {return unicastTo({ data, byActor: me, toActorUrl: following.inboxUrl, contextType: 'Follow' })})}-function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor): ActivityFollow {+export function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor): ActivityFollow {return {type: 'Follow',id: url,@@ -28,10 +29,3 @@ function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor)object: targetActor.url}}--// -----------------------------------------------------------------------------export {- sendFollow,- buildFollowActivity-}diff --git a/server/core/lib/schedulers/actor-follow-scheduler.ts b/server/core/lib/schedulers/actor-follow-scheduler.tsindex 9ca42f817b..c184fd4e58 100644--- a/server/core/lib/schedulers/actor-follow-scheduler.ts+++ b/server/core/lib/schedulers/actor-follow-scheduler.ts@@ -1,4 +1,6 @@import { isProdInstance, isTestOrDevInstance } from '@peertube/peertube-node-utils'+import { sendFollow } from '@server/lib/activitypub/send/send-follow.js'+import { setAsUpdated } from '@server/models/shared/update.js'import { logger, loggerTagsFactory } from '../../helpers/logger.js'import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'import { ActorFollowModel } from '../../models/actor/actor-follow.js'@@ -6,6 +8,8 @@ import { ActorFollowHealthCache } from '../actor-follow-health-cache.js'import { AbstractScheduler } from './abstract-scheduler.js'const lTags = loggerTagsFactory('schedulers')+const FOLLOW_RESEND_STALE_MS = 1000 * 60 * 60 * 24 * 7 // 7 days+const FOLLOW_RESEND_BATCH_SIZE = 100export class ActorFollowScheduler extends AbstractScheduler {private static instance: AbstractScheduler@@ -25,6 +29,8 @@ export class ActorFollowScheduler extends AbstractScheduler {await this.processPendingScores()await this.removeBadActorFollows()++ await this.resendStaleActorFollows()}private async processPendingScores () {@@ -54,6 +60,31 @@ export class ActorFollowScheduler extends AbstractScheduler {}}+ private async resendStaleActorFollows () {+ const olderThan = new Date(Date.now() - FOLLOW_RESEND_STALE_MS)++ try {+ const actorFollows = await ActorFollowModel.listOutgoingStaleForResend({+ olderThan,+ limit: FOLLOW_RESEND_BATCH_SIZE+ })++ if (actorFollows.length === 0) return++ for (const actorFollow of actorFollows) {+ sendFollow(actorFollow, undefined)++ await setAsUpdated({ sequelize: ActorFollowModel.sequelize, table: 'actorFollow', id: actorFollow.id })+ }++ if (!isTestOrDevInstance()) {+ logger.info('Queued %d stale actor follows for resend.', actorFollows.length, lTags())+ }+ } catch (err) {+ logger.error('Error in stale actor follow resend scheduler.', { err, ...lTags() })+ }+ }+static get Instance () {return this.instance || (this.instance = new this())}diff --git a/server/core/models/actor/actor-follow.ts b/server/core/models/actor/actor-follow.tsindex 5a623ca54d..97a2a9edd0 100644--- a/server/core/models/actor/actor-follow.ts+++ b/server/core/models/actor/actor-follow.ts@@ -47,6 +47,7 @@ import { VideoChannelModel } from '../video/video-channel.js'import { ActorModel, unusedActorAttributesForAPI } from './actor.js'import { InstanceListFollowersQueryBuilder, ListFollowersOptions } from './sql/instance-list-followers-query-builder.js'import { InstanceListFollowingQueryBuilder, ListFollowingOptions } from './sql/instance-list-following-query-builder.js'+import { isTestInstance } from '@peertube/peertube-node-utils'@Table({tableName: 'actorFollow',@@ -504,6 +505,47 @@ export class ActorFollowModel extends SequelizeModel<ActorFollowModel> {return difference(hosts, followedHosts)}+ static listOutgoingStaleForResend (options: {+ olderThan: Date+ limit: number+ }) {+ const { olderThan, limit } = options++ return ActorFollowModel.findAll<MActorFollowActors>({+ logging: !isTestInstance(),+ where: {+ state: {+ [Op.in]: [ 'pending', 'accepted' ]+ },+ updatedAt: {+ [Op.lt]: olderThan+ }+ },+ include: [+ {+ model: ActorModel.unscoped(),+ required: true,+ as: 'ActorFollower',+ where: {+ serverId: null+ }+ },+ {+ model: ActorModel.unscoped(),+ required: true,+ as: 'ActorFollowing',+ where: {+ serverId: {+ [Op.ne]: null+ }+ }+ }+ ],+ order: [ [ 'updatedAt', 'ASC' ] ],+ limit+ })+ }+// ---------------------------------------------------------------------------static listAcceptedFollowerUrlsForAP (actorIds: number[], t: Transaction, start?: number, count?: number) {