Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Refresh actors/videos on lazy load not found

Chocobozzz7962077f081f
diff --git a/packages/tests/src/api/server/lazy-static.ts b/packages/tests/src/api/server/lazy-static.ts
index c785a4bad2..d47f70c21b 100644
--- a/packages/tests/src/api/server/lazy-static.ts
+++ b/packages/tests/src/api/server/lazy-static.ts
@@ -1,6 +1,6 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await,@typescript-eslint/no-floating-promises */
-import { HttpStatusCode } from '@peertube/peertube-models'
+import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models'
import {
cleanupTests,
createMultipleServers,
@@ -119,6 +119,37 @@ describe('Test lazy static endpoints', function () {
await checkCachedFiles({ populated: true })
})
+ it('Should miss an update, but re-fetch the files on 404 error', async function () {
+ this.timeout(60000)
+
+ const updateServer1Files = async () => {
+ await servers[0].videos.update({ id: videoId, attributes: { thumbnailfile: 'custom-thumbnail.png' } })
+ await servers[0].users.updateMyAvatar({ fixture: 'custom-thumbnail.png' })
+ await waitJobs([ servers[0] ])
+ }
+
+ const testLazyStatic = async (expectedStatus: HttpStatusCodeType) => {
+ const video = await servers[1].videos.get({ id: videoId })
+ await makeRawRequest({ url: video.thumbnails[0].fileUrl, expectedStatus })
+
+ const user = await servers[1].accounts.get({ accountName: 'root@' + servers[0].host })
+ await makeRawRequest({ url: user.avatars[0].fileUrl, expectedStatus })
+ }
+
+ // Invalidate server 2 cache
+ await updateServer1Files()
+ await servers[1].kill()
+
+ // Server 2 miss another update
+ await updateServer1Files()
+
+ await servers[1].run()
+
+ await testLazyStatic(HttpStatusCode.NOT_FOUND_404)
+ await waitJobs(servers)
+ await testLazyStatic(HttpStatusCode.OK_200)
+ })
+
it('Should still have files after a server restart', async function () {
this.timeout(60000)
diff --git a/server/core/helpers/promise-cache.ts b/server/core/helpers/promise-cache.ts
index 303bab9762..3f9da76f7b 100644
--- a/server/core/helpers/promise-cache.ts
+++ b/server/core/helpers/promise-cache.ts
@@ -1,4 +1,8 @@
-export class CachePromiseFactory <A, R> {
+/**
+ * A helper to avoid multiple concurrent calls to the same promise
+ * It is used in particular for refreshing remote actors, to avoid multiple refreshes at the same time.
+ */
+export class CachePromiseFactory<A, R> {
private readonly running = new Map<string, Promise<R>>()
constructor (
diff --git a/server/core/lib/activitypub/actors/get.ts b/server/core/lib/activitypub/actors/get.ts
index 5811109f15..7aa2606e35 100644
--- a/server/core/lib/activitypub/actors/get.ts
+++ b/server/core/lib/activitypub/actors/get.ts
@@ -67,7 +67,13 @@ async function getOrCreateAPActor (
if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor
if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor
- const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({ actor, fetchedType: fetchType })
+ const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({
+ actor,
+ fetchedType: fetchType === 'all'
+ ? 'all'
+ : 'partial'
+ })
+
if (!actorRefreshed) throw new Error(`Actor ${actor.url} does not exist anymore.`)
await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections)
diff --git a/server/core/lib/activitypub/actors/refresh.ts b/server/core/lib/activitypub/actors/refresh.ts
index 636e57b10d..75b566f893 100644
--- a/server/core/lib/activitypub/actors/refresh.ts
+++ b/server/core/lib/activitypub/actors/refresh.ts
@@ -1,10 +1,10 @@
+import { HttpStatusCode } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CachePromiseFactory } from '@server/helpers/promise-cache.js'
import { PeerTubeRequestError } from '@server/helpers/requests.js'
import { ActorLoadByUrlType } from '@server/lib/model-loaders/index.js'
import { ActorModel } from '@server/models/actor/actor.js'
-import { MActorAccountChannelId, MActorFull } from '@server/types/models/index.js'
-import { HttpStatusCode } from '@peertube/peertube-models'
+import { MActorFull, MActorOutdated } from '@server/types/models/index.js'
import { fetchRemoteActor } from './shared/index.js'
import { APActorUpdater } from './updater.js'
import { getUrlFromWebfinger } from './webfinger.js'
@@ -13,12 +13,15 @@ type RefreshResult<T> = Promise<{ actor: T | MActorFull, refreshed: boolean }>
type RefreshOptions<T> = {
actor: T
- fetchedType: ActorLoadByUrlType
+ fetchedType: Extract<ActorLoadByUrlType, 'all'> | 'partial'
}
-const promiseCache = new CachePromiseFactory(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url)
+const promiseCache = new CachePromiseFactory(
+ doRefresh,
+ (options: RefreshOptions<MActorFull | MActorOutdated>) => options.actor.id + ''
+)
-function refreshActorIfNeeded<T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult<T> {
+function refreshActorIfNeeded<T extends MActorFull | MActorOutdated> (options: RefreshOptions<T>): RefreshResult<T> {
const actorArg = options.actor
if (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false })
@@ -31,13 +34,13 @@ export {
// ---------------------------------------------------------------------------
-async function doRefresh<T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult<MActorFull> {
+async function doRefresh<T extends MActorFull | MActorOutdated> (options: RefreshOptions<T>): RefreshResult<MActorFull> {
const { actor: actorArg, fetchedType } = options
// We need more attributes
const actor = fetchedType === 'all'
? actorArg as MActorFull
- : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)
+ : await ActorModel.loadAndPopulateAccountAndChannel(actorArg.id)
const lTags = loggerTagsFactory('ap', 'actor', 'refresh', actor.url)
diff --git a/server/core/lib/files-cache/avatar-image-file-cache.ts b/server/core/lib/files-cache/avatar-image-file-cache.ts
index b0163c6c9c..1d34d87a82 100644
--- a/server/core/lib/files-cache/avatar-image-file-cache.ts
+++ b/server/core/lib/files-cache/avatar-image-file-cache.ts
@@ -1,7 +1,12 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { ActorImageModel } from '@server/models/actor/actor-image.js'
+import { ActorModel } from '@server/models/actor/actor.js'
import { MActorImage } from '@server/types/models/index.js'
+import { refreshActorIfNeeded } from '../activitypub/actors/refresh.js'
import { AbstractImageFileCache } from './shared/abstract-image-file-cache.js'
+const lTags = loggerTagsFactory('lazy-load', 'avatar-image')
+
export class AvatarImageFileCache extends AbstractImageFileCache<MActorImage> {
protected loadModel (filename: string) {
return ActorImageModel.loadByFilename(filename)
@@ -14,4 +19,14 @@ export class AvatarImageFileCache extends AbstractImageFileCache<MActorImage> {
protected getFSFileCachedPath (model: MActorImage) {
return model.getFSCachedPath()
}
+
+ protected async onLazyFetchNotFound (model: MActorImage) {
+ try {
+ const actor = await ActorModel.loadForOutdated(model.actorId)
+
+ await refreshActorIfNeeded({ actor, fetchedType: 'partial' })
+ } catch (err) {
+ logger.error('Error while refreshing actor for avatar image lazy fetch', { ...lTags(), err })
+ }
+ }
}
diff --git a/server/core/lib/files-cache/shared/abstract-file-cache.ts b/server/core/lib/files-cache/shared/abstract-file-cache.ts
index 0a9eef819a..7770b8247b 100644
--- a/server/core/lib/files-cache/shared/abstract-file-cache.ts
+++ b/server/core/lib/files-cache/shared/abstract-file-cache.ts
@@ -25,6 +25,7 @@ export abstract class AbstractFileCache<M extends FileModel> {
protected abstract loadModel (filename: string): Promise<M>
protected abstract getFSFilePath (model: M): string
protected abstract getFSFileCachedPath (model: M): string
+ protected abstract onLazyFetchNotFound (model: M): any
async lazyServe (options: {
filename: string
@@ -66,7 +67,11 @@ export abstract class AbstractFileCache<M extends FileModel> {
try {
await this.downloadRemoteFile(file)
} catch (err) {
- logger.warn('Cannot process remote image %s.', file.fileUrl, { err })
+ if (err.status === HttpStatusCode.NOT_FOUND_404) {
+ this.onLazyFetchNotFound(file)
+ } else {
+ logger.warn('Cannot process remote image %s.', file.fileUrl, { err })
+ }
return undefined
}
diff --git a/server/core/lib/files-cache/video-captions-file-cache.ts b/server/core/lib/files-cache/video-captions-file-cache.ts
index b3a51ec16d..2a7c1e6585 100644
--- a/server/core/lib/files-cache/video-captions-file-cache.ts
+++ b/server/core/lib/files-cache/video-captions-file-cache.ts
@@ -1,7 +1,12 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
+import { VideoModel } from '@server/models/video/video.js'
import { MVideoCaption } from '@server/types/models/index.js'
+import { JobQueue } from '../job-queue/job-queue.js'
import { AbstractFileCache } from './shared/abstract-file-cache.js'
+const lTags = loggerTagsFactory('lazy-load', 'video-captions')
+
export class VideoCaptionsFileCache extends AbstractFileCache<MVideoCaption> {
protected loadModel (filename: string) {
return VideoCaptionModel.loadByFilename(filename)
@@ -14,4 +19,14 @@ export class VideoCaptionsFileCache extends AbstractFileCache<MVideoCaption> {
protected getFSFileCachedPath (model: MVideoCaption) {
return model.getFSFileCachedPath()
}
+
+ protected async onLazyFetchNotFound (model: MVideoCaption) {
+ try {
+ const video = await VideoModel.load(model.videoId)
+
+ JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
+ } catch (err) {
+ logger.error('Error while refreshing video for lazy fetch', { ...lTags(), err })
+ }
+ }
}
diff --git a/server/core/lib/files-cache/video-storyboards-image-file-cache.ts b/server/core/lib/files-cache/video-storyboards-image-file-cache.ts
index a7a73fcd44..a51b8228aa 100644
--- a/server/core/lib/files-cache/video-storyboards-image-file-cache.ts
+++ b/server/core/lib/files-cache/video-storyboards-image-file-cache.ts
@@ -1,7 +1,12 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { StoryboardModel } from '@server/models/video/storyboard.js'
+import { VideoModel } from '@server/models/video/video.js'
import { MStoryboard } from '@server/types/models/index.js'
+import { JobQueue } from '../job-queue/job-queue.js'
import { AbstractImageFileCache } from './shared/abstract-image-file-cache.js'
+const lTags = loggerTagsFactory('lazy-load', 'video-storyboards')
+
export class VideoStoryboardsImageFileCache extends AbstractImageFileCache<MStoryboard> {
protected loadModel (filename: string) {
return StoryboardModel.loadByFilename(filename)
@@ -14,4 +19,14 @@ export class VideoStoryboardsImageFileCache extends AbstractImageFileCache<MStor
protected getFSFileCachedPath (model: MStoryboard) {
return model.getFSCachedPath()
}
+
+ protected async onLazyFetchNotFound (model: MStoryboard) {
+ try {
+ const video = await VideoModel.load(model.videoId)
+
+ JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
+ } catch (err) {
+ logger.error('Error while refreshing video for lazy fetch', { ...lTags(), err })
+ }
+ }
}
diff --git a/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts b/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts
index 5e2d9102d6..8fedd42347 100644
--- a/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts
+++ b/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts
@@ -1,7 +1,12 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { ThumbnailModel } from '@server/models/video/thumbnail.js'
+import { VideoModel } from '@server/models/video/video.js'
import { MThumbnail } from '@server/types/models/index.js'
+import { JobQueue } from '../job-queue/job-queue.js'
import { AbstractImageFileCache } from './shared/abstract-image-file-cache.js'
+const lTags = loggerTagsFactory('lazy-load', 'video-thumbnails')
+
export class VideoThumbnailsImageFileCache extends AbstractImageFileCache<MThumbnail> {
protected loadModel (filename: string) {
return ThumbnailModel.loadByFilename(filename)
@@ -14,4 +19,14 @@ export class VideoThumbnailsImageFileCache extends AbstractImageFileCache<MThumb
protected getFSFileCachedPath (model: MThumbnail) {
return model.getFSCachedPath()
}
+
+ protected async onLazyFetchNotFound (model: MThumbnail) {
+ try {
+ const video = await VideoModel.load(model.videoId)
+
+ JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })
+ } catch (err) {
+ logger.error('Error while refreshing video for lazy fetch', { ...lTags(), err })
+ }
+ }
}
diff --git a/server/core/models/actor/actor.ts b/server/core/models/actor/actor.ts
index 172a6c0fd6..8d4b5eb2af 100644
--- a/server/core/models/actor/actor.ts
+++ b/server/core/models/actor/actor.ts
@@ -41,6 +41,7 @@ import {
MActorHost,
MActorHostOnly,
MActorId,
+ MActorOutdated,
MActorSummaryFormattable,
MActorUrl,
MActorWithInboxes
@@ -398,6 +399,17 @@ export class ActorModel extends SequelizeModel<ActorModel> {
return ActorModel.unscoped().findByPk(id, { transaction })
}
+ static async loadForOutdated (id: number, transaction?: Transaction): Promise<MActorOutdated> {
+ const actorServer = await getServerActor()
+ if (id === actorServer.id) return actorServer
+
+ return ActorModel.unscoped().findOne({
+ attributes: [ 'id', 'createdAt', 'updatedAt', 'serverId' ],
+ where: { id },
+ transaction
+ })
+ }
+
static loadFull (id: number, transaction?: Transaction): Promise<MActorFull> {
return ActorModel.scope(ScopeNames.FULL).findByPk(id, { transaction })
}
@@ -530,6 +542,15 @@ export class ActorModel extends SequelizeModel<ActorModel> {
return ActorModel.unscoped().findOne(query)
}
+ static loadAndPopulateAccountAndChannel (id: number, transaction?: Transaction): Promise<MActorFull> {
+ const query = {
+ where: { id },
+ transaction
+ }
+
+ return ActorModel.scope(ScopeNames.FULL).findOne(query)
+ }
+
static loadByUrlAndPopulateAccountAndChannel (url: string, transaction?: Transaction): Promise<MActorFull> {
const query = {
where: {
@@ -758,7 +779,7 @@ export class ActorModel extends SequelizeModel<ActorModel> {
return this.url + '/playlists'
}
- isLocal () {
+ isLocal (this: Pick<MActor, 'serverId'>) {
return this.serverId === null
}
@@ -812,7 +833,7 @@ export class ActorModel extends SequelizeModel<ActorModel> {
return findAppropriateImage(images, width)
}
- isOutdated () {
+ isOutdated (this: MActorOutdated) {
if (this.isLocal()) return false
return isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL)
diff --git a/server/core/types/models/actor/actor.ts b/server/core/types/models/actor/actor.ts
index a68535c4f0..f5060cfbec 100644
--- a/server/core/types/models/actor/actor.ts
+++ b/server/core/types/models/actor/actor.ts
@@ -27,6 +27,7 @@ export type MActorAudience = MActorUrl & MActorFollowersUrl
export type MActorWithInboxes = Pick<ActorModel, 'sharedInboxUrl' | 'inboxUrl' | 'getSharedInbox'>
export type MActorSignature = MActorAccountChannelId
+export type MActorOutdated = Pick<MActor, 'id' | 'createdAt' | 'updatedAt' | 'isLocal' | 'serverId' | 'isOutdated'>
export type MActorLight = Omit<MActor, 'privateKey' | 'publicKey'>
// ############################################################################