ActivityPub-federated video streaming platform using P2P directly in your web browser
Refresh actors/videos on lazy load not found
Chocobozzz7962077f081f
diff --git a/packages/tests/src/api/server/lazy-static.ts b/packages/tests/src/api/server/lazy-static.tsindex c785a4bad2..d47f70c21b 100644--- a/packages/tests/src/api/server/lazy-static.ts+++ b/packages/tests/src/api/server/lazy-static.ts@@ -1,6 +1,6 @@/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await,@typescript-eslint/no-floating-promises */-import { HttpStatusCode } from '@peertube/peertube-models'+import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models'import {cleanupTests,createMultipleServers,@@ -119,6 +119,37 @@ describe('Test lazy static endpoints', function () {await checkCachedFiles({ populated: true })})+ it('Should miss an update, but re-fetch the files on 404 error', async function () {+ this.timeout(60000)++ const updateServer1Files = async () => {+ await servers[0].videos.update({ id: videoId, attributes: { thumbnailfile: 'custom-thumbnail.png' } })+ await servers[0].users.updateMyAvatar({ fixture: 'custom-thumbnail.png' })+ await waitJobs([ servers[0] ])+ }++ const testLazyStatic = async (expectedStatus: HttpStatusCodeType) => {+ const video = await servers[1].videos.get({ id: videoId })+ await makeRawRequest({ url: video.thumbnails[0].fileUrl, expectedStatus })++ const user = await servers[1].accounts.get({ accountName: 'root@' + servers[0].host })+ await makeRawRequest({ url: user.avatars[0].fileUrl, expectedStatus })+ }++ // Invalidate server 2 cache+ await updateServer1Files()+ await servers[1].kill()++ // Server 2 miss another update+ await updateServer1Files()++ await servers[1].run()++ await testLazyStatic(HttpStatusCode.NOT_FOUND_404)+ await waitJobs(servers)+ await testLazyStatic(HttpStatusCode.OK_200)+ })+it('Should still have files after a server restart', async function () {this.timeout(60000)diff --git a/server/core/helpers/promise-cache.ts b/server/core/helpers/promise-cache.tsindex 303bab9762..3f9da76f7b 100644--- a/server/core/helpers/promise-cache.ts+++ b/server/core/helpers/promise-cache.ts@@ -1,4 +1,8 @@-export class CachePromiseFactory <A, R> {+/**+ * A helper to avoid multiple concurrent calls to the same promise+ * It is used in particular for refreshing remote actors, to avoid multiple refreshes at the same time.+ */+export class CachePromiseFactory<A, R> {private readonly running = new Map<string, Promise<R>>()constructor (diff --git a/server/core/lib/activitypub/actors/get.ts b/server/core/lib/activitypub/actors/get.tsindex 5811109f15..7aa2606e35 100644--- a/server/core/lib/activitypub/actors/get.ts+++ b/server/core/lib/activitypub/actors/get.ts@@ -67,7 +67,13 @@ async function getOrCreateAPActor (if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actorif (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor- const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({ actor, fetchedType: fetchType })+ const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({+ actor,+ fetchedType: fetchType === 'all'+ ? 'all'+ : 'partial'+ })+if (!actorRefreshed) throw new Error(`Actor ${actor.url} does not exist anymore.`)await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections)diff --git a/server/core/lib/activitypub/actors/refresh.ts b/server/core/lib/activitypub/actors/refresh.tsindex 636e57b10d..75b566f893 100644--- a/server/core/lib/activitypub/actors/refresh.ts+++ b/server/core/lib/activitypub/actors/refresh.ts@@ -1,10 +1,10 @@+import { HttpStatusCode } from '@peertube/peertube-models'import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { CachePromiseFactory } from '@server/helpers/promise-cache.js'import { PeerTubeRequestError } from '@server/helpers/requests.js'import { ActorLoadByUrlType } from '@server/lib/model-loaders/index.js'import { ActorModel } from '@server/models/actor/actor.js'-import { MActorAccountChannelId, MActorFull } from '@server/types/models/index.js'-import { HttpStatusCode } from '@peertube/peertube-models'+import { MActorFull, MActorOutdated } from '@server/types/models/index.js'import { fetchRemoteActor } from './shared/index.js'import { APActorUpdater } from './updater.js'import { getUrlFromWebfinger } from './webfinger.js'@@ -13,12 +13,15 @@ type RefreshResult<T> = Promise<{ actor: T | MActorFull, refreshed: boolean }>type RefreshOptions<T> = {actor: T- fetchedType: ActorLoadByUrlType+ fetchedType: Extract<ActorLoadByUrlType, 'all'> | 'partial'}-const promiseCache = new CachePromiseFactory(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url)+const promiseCache = new CachePromiseFactory(+ doRefresh,+ (options: RefreshOptions<MActorFull | MActorOutdated>) => options.actor.id + ''+)-function refreshActorIfNeeded<T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult<T> {+function refreshActorIfNeeded<T extends MActorFull | MActorOutdated> (options: RefreshOptions<T>): RefreshResult<T> {const actorArg = options.actorif (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false })@@ -31,13 +34,13 @@ export {// ----------------------------------------------------------------------------async function doRefresh<T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult<MActorFull> {+async function doRefresh<T extends MActorFull | MActorOutdated> (options: RefreshOptions<T>): RefreshResult<MActorFull> {const { actor: actorArg, fetchedType } = options// We need more attributesconst actor = fetchedType === 'all'? actorArg as MActorFull- : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url)+ : await ActorModel.loadAndPopulateAccountAndChannel(actorArg.id)const lTags = loggerTagsFactory('ap', 'actor', 'refresh', actor.url)diff --git a/server/core/lib/files-cache/avatar-image-file-cache.ts b/server/core/lib/files-cache/avatar-image-file-cache.tsindex b0163c6c9c..1d34d87a82 100644--- a/server/core/lib/files-cache/avatar-image-file-cache.ts+++ b/server/core/lib/files-cache/avatar-image-file-cache.ts@@ -1,7 +1,12 @@+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { ActorImageModel } from '@server/models/actor/actor-image.js'+import { ActorModel } from '@server/models/actor/actor.js'import { MActorImage } from '@server/types/models/index.js'+import { refreshActorIfNeeded } from '../activitypub/actors/refresh.js'import { AbstractImageFileCache } from './shared/abstract-image-file-cache.js'+const lTags = loggerTagsFactory('lazy-load', 'avatar-image')+export class AvatarImageFileCache extends AbstractImageFileCache<MActorImage> {protected loadModel (filename: string) {return ActorImageModel.loadByFilename(filename)@@ -14,4 +19,14 @@ export class AvatarImageFileCache extends AbstractImageFileCache<MActorImage> {protected getFSFileCachedPath (model: MActorImage) {return model.getFSCachedPath()}++ protected async onLazyFetchNotFound (model: MActorImage) {+ try {+ const actor = await ActorModel.loadForOutdated(model.actorId)++ await refreshActorIfNeeded({ actor, fetchedType: 'partial' })+ } catch (err) {+ logger.error('Error while refreshing actor for avatar image lazy fetch', { ...lTags(), err })+ }+ }}diff --git a/server/core/lib/files-cache/shared/abstract-file-cache.ts b/server/core/lib/files-cache/shared/abstract-file-cache.tsindex 0a9eef819a..7770b8247b 100644--- a/server/core/lib/files-cache/shared/abstract-file-cache.ts+++ b/server/core/lib/files-cache/shared/abstract-file-cache.ts@@ -25,6 +25,7 @@ export abstract class AbstractFileCache<M extends FileModel> {protected abstract loadModel (filename: string): Promise<M>protected abstract getFSFilePath (model: M): stringprotected abstract getFSFileCachedPath (model: M): string+ protected abstract onLazyFetchNotFound (model: M): anyasync lazyServe (options: {filename: string@@ -66,7 +67,11 @@ export abstract class AbstractFileCache<M extends FileModel> {try {await this.downloadRemoteFile(file)} catch (err) {- logger.warn('Cannot process remote image %s.', file.fileUrl, { err })+ if (err.status === HttpStatusCode.NOT_FOUND_404) {+ this.onLazyFetchNotFound(file)+ } else {+ logger.warn('Cannot process remote image %s.', file.fileUrl, { err })+ }return undefined}diff --git a/server/core/lib/files-cache/video-captions-file-cache.ts b/server/core/lib/files-cache/video-captions-file-cache.tsindex b3a51ec16d..2a7c1e6585 100644--- a/server/core/lib/files-cache/video-captions-file-cache.ts+++ b/server/core/lib/files-cache/video-captions-file-cache.ts@@ -1,7 +1,12 @@+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { VideoCaptionModel } from '@server/models/video/video-caption.js'+import { VideoModel } from '@server/models/video/video.js'import { MVideoCaption } from '@server/types/models/index.js'+import { JobQueue } from '../job-queue/job-queue.js'import { AbstractFileCache } from './shared/abstract-file-cache.js'+const lTags = loggerTagsFactory('lazy-load', 'video-captions')+export class VideoCaptionsFileCache extends AbstractFileCache<MVideoCaption> {protected loadModel (filename: string) {return VideoCaptionModel.loadByFilename(filename)@@ -14,4 +19,14 @@ export class VideoCaptionsFileCache extends AbstractFileCache<MVideoCaption> {protected getFSFileCachedPath (model: MVideoCaption) {return model.getFSFileCachedPath()}++ protected async onLazyFetchNotFound (model: MVideoCaption) {+ try {+ const video = await VideoModel.load(model.videoId)++ JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })+ } catch (err) {+ logger.error('Error while refreshing video for lazy fetch', { ...lTags(), err })+ }+ }}diff --git a/server/core/lib/files-cache/video-storyboards-image-file-cache.ts b/server/core/lib/files-cache/video-storyboards-image-file-cache.tsindex a7a73fcd44..a51b8228aa 100644--- a/server/core/lib/files-cache/video-storyboards-image-file-cache.ts+++ b/server/core/lib/files-cache/video-storyboards-image-file-cache.ts@@ -1,7 +1,12 @@+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { StoryboardModel } from '@server/models/video/storyboard.js'+import { VideoModel } from '@server/models/video/video.js'import { MStoryboard } from '@server/types/models/index.js'+import { JobQueue } from '../job-queue/job-queue.js'import { AbstractImageFileCache } from './shared/abstract-image-file-cache.js'+const lTags = loggerTagsFactory('lazy-load', 'video-storyboards')+export class VideoStoryboardsImageFileCache extends AbstractImageFileCache<MStoryboard> {protected loadModel (filename: string) {return StoryboardModel.loadByFilename(filename)@@ -14,4 +19,14 @@ export class VideoStoryboardsImageFileCache extends AbstractImageFileCache<MStorprotected getFSFileCachedPath (model: MStoryboard) {return model.getFSCachedPath()}++ protected async onLazyFetchNotFound (model: MStoryboard) {+ try {+ const video = await VideoModel.load(model.videoId)++ JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })+ } catch (err) {+ logger.error('Error while refreshing video for lazy fetch', { ...lTags(), err })+ }+ }}diff --git a/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts b/server/core/lib/files-cache/video-thumbnails-image-file-cache.tsindex 5e2d9102d6..8fedd42347 100644--- a/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts+++ b/server/core/lib/files-cache/video-thumbnails-image-file-cache.ts@@ -1,7 +1,12 @@+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { ThumbnailModel } from '@server/models/video/thumbnail.js'+import { VideoModel } from '@server/models/video/video.js'import { MThumbnail } from '@server/types/models/index.js'+import { JobQueue } from '../job-queue/job-queue.js'import { AbstractImageFileCache } from './shared/abstract-image-file-cache.js'+const lTags = loggerTagsFactory('lazy-load', 'video-thumbnails')+export class VideoThumbnailsImageFileCache extends AbstractImageFileCache<MThumbnail> {protected loadModel (filename: string) {return ThumbnailModel.loadByFilename(filename)@@ -14,4 +19,14 @@ export class VideoThumbnailsImageFileCache extends AbstractImageFileCache<MThumbprotected getFSFileCachedPath (model: MThumbnail) {return model.getFSCachedPath()}++ protected async onLazyFetchNotFound (model: MThumbnail) {+ try {+ const video = await VideoModel.load(model.videoId)++ JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } })+ } catch (err) {+ logger.error('Error while refreshing video for lazy fetch', { ...lTags(), err })+ }+ }}diff --git a/server/core/models/actor/actor.ts b/server/core/models/actor/actor.tsindex 172a6c0fd6..8d4b5eb2af 100644--- a/server/core/models/actor/actor.ts+++ b/server/core/models/actor/actor.ts@@ -41,6 +41,7 @@ import {MActorHost,MActorHostOnly,MActorId,+ MActorOutdated,MActorSummaryFormattable,MActorUrl,MActorWithInboxes@@ -398,6 +399,17 @@ export class ActorModel extends SequelizeModel<ActorModel> {return ActorModel.unscoped().findByPk(id, { transaction })}+ static async loadForOutdated (id: number, transaction?: Transaction): Promise<MActorOutdated> {+ const actorServer = await getServerActor()+ if (id === actorServer.id) return actorServer++ return ActorModel.unscoped().findOne({+ attributes: [ 'id', 'createdAt', 'updatedAt', 'serverId' ],+ where: { id },+ transaction+ })+ }+static loadFull (id: number, transaction?: Transaction): Promise<MActorFull> {return ActorModel.scope(ScopeNames.FULL).findByPk(id, { transaction })}@@ -530,6 +542,15 @@ export class ActorModel extends SequelizeModel<ActorModel> {return ActorModel.unscoped().findOne(query)}+ static loadAndPopulateAccountAndChannel (id: number, transaction?: Transaction): Promise<MActorFull> {+ const query = {+ where: { id },+ transaction+ }++ return ActorModel.scope(ScopeNames.FULL).findOne(query)+ }+static loadByUrlAndPopulateAccountAndChannel (url: string, transaction?: Transaction): Promise<MActorFull> {const query = {where: {@@ -758,7 +779,7 @@ export class ActorModel extends SequelizeModel<ActorModel> {return this.url + '/playlists'}- isLocal () {+ isLocal (this: Pick<MActor, 'serverId'>) {return this.serverId === null}@@ -812,7 +833,7 @@ export class ActorModel extends SequelizeModel<ActorModel> {return findAppropriateImage(images, width)}- isOutdated () {+ isOutdated (this: MActorOutdated) {if (this.isLocal()) return falsereturn isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL)diff --git a/server/core/types/models/actor/actor.ts b/server/core/types/models/actor/actor.tsindex a68535c4f0..f5060cfbec 100644--- a/server/core/types/models/actor/actor.ts+++ b/server/core/types/models/actor/actor.ts@@ -27,6 +27,7 @@ export type MActorAudience = MActorUrl & MActorFollowersUrlexport type MActorWithInboxes = Pick<ActorModel, 'sharedInboxUrl' | 'inboxUrl' | 'getSharedInbox'>export type MActorSignature = MActorAccountChannelId+export type MActorOutdated = Pick<MActor, 'id' | 'createdAt' | 'updatedAt' | 'isLocal' | 'serverId' | 'isOutdated'>export type MActorLight = Omit<MActor, 'privateKey' | 'publicKey'>// ############################################################################