ActivityPub-federated video streaming platform using P2P directly in your web browser
Update infohash on base url change
Chocobozzz5c2e200526ba
diff --git a/packages/tests/src/api/object-storage/videos.ts b/packages/tests/src/api/object-storage/videos.tsindex 25d2a9eabe..c1ff9b8412 100644--- a/packages/tests/src/api/object-storage/videos.ts+++ b/packages/tests/src/api/object-storage/videos.ts@@ -1,6 +1,6 @@/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */-import { getAllFiles } from '@peertube/peertube-core-utils'+import { getAllFiles, getHLS } from '@peertube/peertube-core-utils'import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models'import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'import {@@ -161,18 +161,8 @@ function runTestSuite (options: {const uuidsToDelete: string[] = []let deletedUrls: string[] = []- before(async function () {- this.timeout(240000)-- const port = await mockObjectStorageProxy.initialize()- baseMockUrl = useMockBaseUrl- ? `http://127.0.0.1:${port}`- : undefined-- await objectStorage.createMockBucket(options.playlistBucket)- await objectStorage.createMockBucket(options.webVideoBucket)-- const config = {+ function getConfig () {+ return {object_storage: {enabled: true,endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),@@ -201,8 +191,20 @@ function runTestSuite (options: {}}}+ }- servers = await createMultipleServers(2, config)+ before(async function () {+ this.timeout(240000)++ const port = await mockObjectStorageProxy.initialize()+ baseMockUrl = useMockBaseUrl+ ? `http://127.0.0.1:${port}`+ : undefined++ await objectStorage.createMockBucket(options.playlistBucket)+ await objectStorage.createMockBucket(options.webVideoBucket)++ servers = await createMultipleServers(2, getConfig())await setAccessTokensToServers(servers)await doubleFollow(servers[0], servers[1])@@ -250,6 +252,29 @@ function runTestSuite (options: {}})+ if (options.useMockBaseUrl) {+ it('Should update streaming playlist infohash if object storage base url changed', async function () {+ const newBaseMockUrl = `http://example.com`++ await servers[1].kill()++ await servers[1].run(+ merge(getConfig(), {+ object_storage: {+ streaming_playlists: {+ base_url: `${newBaseMockUrl}/${options.playlistBucket}`+ }+ }+ })+ )++ const { uuid } = await servers[1].videos.find({ name: 'video 2' })+ const video = await servers[1].videos.get({ id: uuid })++ await checkPlaylistInfohash({ video, files: getHLS(video).files, sqlCommand: sqlCommands[1] })+ })+ }+it('Should fetch correctly all the files', async function () {for (const url of deletedUrls.concat(keptUrls)) {await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })diff --git a/server/core/lib/hls.ts b/server/core/lib/hls.tsindex c6a766d5c6..c9a855259a 100644--- a/server/core/lib/hls.ts+++ b/server/core/lib/hls.ts@@ -2,6 +2,7 @@ import { sortBy, uniqify, uuidRegex } from '@peertube/peertube-core-utils'import { ffprobePromise, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'import { FileStorage, VideoResolution } from '@peertube/peertube-models'import { sha256 } from '@peertube/peertube-node-utils'+import { ApplicationModel } from '@server/models/application/application.js'import { VideoCaptionModel } from '@server/models/video/video-caption.js'import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo, MVideoCaption } from '@server/types/models/index.js'import { ensureDir, move, outputJSON, remove } from 'fs-extra/esm'@@ -25,7 +26,21 @@ import { VideoPathManager } from './video-path-manager.js'const lTags = loggerTagsFactory('hls')export async function updateStreamingPlaylistsInfohashesIfNeeded () {- const playlistsToUpdateIds = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()+ let playlistsToUpdateIds = new Set(await VideoStreamingPlaylistModel.listIdsByIncorrectPeerVersion())++ if (playlistsToUpdateIds.size !== 0) {+ logger.info(`Will update ${playlistsToUpdateIds.size} streaming playlists infohash because of protocol version change.`, lTags())+ }++ if (await ApplicationModel.streamingPlaylistBaseUrlChanged()) {+ const localIds = await VideoStreamingPlaylistModel.listIdsLocals()++ if (localIds.length !== 0) {+ logger.info(`Will update ${localIds.length} local streaming playlists infohash because of object storage base URL change.`, lTags())++ playlistsToUpdateIds = new Set([ ...playlistsToUpdateIds, ...localIds ])+ }+ }// Use separate SQL queries, because we could have many videos to updatefor (const playlistId of playlistsToUpdateIds) {diff --git a/server/core/models/application/application.ts b/server/core/models/application/application.tsindex 2778664bfd..02da09a668 100644--- a/server/core/models/application/application.ts+++ b/server/core/models/application/application.ts@@ -1,6 +1,8 @@import { getNodeABIVersion } from '@server/helpers/version.js'+import { CONFIG } from '@server/initializers/config.js'import memoizee from 'memoizee'-import { AllowNull, Column, Default, DefaultScope, HasOne, IsInt, Table } from 'sequelize-typescript'+import { AllowNull, Column, DataType, Default, DefaultScope, HasOne, IsInt, Table } from 'sequelize-typescript'+import { PickDeep } from 'type-fest'import { AccountModel } from '../account/account.js'import { ActorImageModel } from '../actor/actor-image.js'import { SequelizeModel } from '../shared/index.js'@@ -23,6 +25,8 @@ export const getServerActor = memoizee(async function () {return actor}, { promise: true })+type ConfigPart = PickDeep<typeof CONFIG, 'OBJECT_STORAGE.STREAMING_PLAYLISTS'>+@DefaultScope(() => ({include: [{@@ -54,6 +58,10 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {@Columndeclare nodeABIVersion: string+ @AllowNull(true)+ @Column(DataType.JSONB)+ declare configPart: ConfigPart+@HasOne(() => AccountModel, {foreignKey: {allowNull: true@@ -62,6 +70,9 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {})declare Account: Awaited<AccountModel>+ private static lastRunConfigPart: ConfigPart+ private static lastRunNodeABIVersion: string+static countTotal () {return ApplicationModel.count()}@@ -73,15 +84,33 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {static async nodeABIChanged () {const application = await this.load()- return application.nodeABIVersion !== getNodeABIVersion()+ const nodeABIVersion = this.lastRunNodeABIVersion || application.nodeABIVersion++ return nodeABIVersion !== getNodeABIVersion()+ }++ static async streamingPlaylistBaseUrlChanged () {+ const application = await this.load()+ const configPart = this.lastRunConfigPart || application.configPart++ return configPart?.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL !== CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL}- static async updateNodeVersions () {+ static async updateNodeVersionsOrConfig () {const application = await this.load()+ this.lastRunNodeABIVersion = application.nodeABIVersion+ this.lastRunConfigPart = application.configPart+application.nodeABIVersion = getNodeABIVersion()application.nodeVersion = process.version+ application.configPart = {+ OBJECT_STORAGE: {+ STREAMING_PLAYLISTS: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS+ }+ }+await application.save()}}diff --git a/server/core/models/video/video-streaming-playlist.ts b/server/core/models/video/video-streaming-playlist.tsindex 29dec28b6a..6fb11533b3 100644--- a/server/core/models/video/video-streaming-playlist.ts+++ b/server/core/models/video/video-streaming-playlist.ts@@ -157,7 +157,7 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPlreturn hashes}- static async listByIncorrectPeerVersion () {+ static async listIdsByIncorrectPeerVersion () {const rows = await VideoStreamingPlaylistModel.unscoped().findAll({raw: true,attributes: [ 'id' ],@@ -171,6 +171,26 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPlreturn rows.map(r => r.id)}+ static async listIdsLocals () {+ const rows = await VideoStreamingPlaylistModel.unscoped().findAll({+ raw: true,+ attributes: [ 'id' ],+ include: [+ {+ model: VideoModel.unscoped(),+ required: true,+ where: {+ remote: false+ }+ }+ ]+ })++ return rows.map(r => r.id)+ }++ // ---------------------------------------------------------------------------+static loadWithVideoAndFiles (id: number) {const options = {include: [diff --git a/server/server.ts b/server/server.tsindex 62e3e4daa9..a625c3d74c 100644--- a/server/server.ts+++ b/server/server.ts@@ -357,7 +357,7 @@ async function startApplication () {}}- ApplicationModel.updateNodeVersions()+ ApplicationModel.updateNodeVersionsOrConfig().catch(err => logger.error('Cannot update node versions.', { err }))JobQueue.Instance.start()