Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Update infohash on base url change

Chocobozzz5c2e200526ba
diff --git a/packages/tests/src/api/object-storage/videos.ts b/packages/tests/src/api/object-storage/videos.ts
index 25d2a9eabe..c1ff9b8412 100644
--- a/packages/tests/src/api/object-storage/videos.ts
+++ b/packages/tests/src/api/object-storage/videos.ts
@@ -1,6 +1,6 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
-import { getAllFiles } from '@peertube/peertube-core-utils'
+import { getAllFiles, getHLS } from '@peertube/peertube-core-utils'
import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import {
@@ -161,18 +161,8 @@ function runTestSuite (options: {
const uuidsToDelete: string[] = []
let deletedUrls: string[] = []
- before(async function () {
- this.timeout(240000)
-
- const port = await mockObjectStorageProxy.initialize()
- baseMockUrl = useMockBaseUrl
- ? `http://127.0.0.1:${port}`
- : undefined
-
- await objectStorage.createMockBucket(options.playlistBucket)
- await objectStorage.createMockBucket(options.webVideoBucket)
-
- const config = {
+ function getConfig () {
+ return {
object_storage: {
enabled: true,
endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
@@ -201,8 +191,20 @@ function runTestSuite (options: {
}
}
}
+ }
- servers = await createMultipleServers(2, config)
+ before(async function () {
+ this.timeout(240000)
+
+ const port = await mockObjectStorageProxy.initialize()
+ baseMockUrl = useMockBaseUrl
+ ? `http://127.0.0.1:${port}`
+ : undefined
+
+ await objectStorage.createMockBucket(options.playlistBucket)
+ await objectStorage.createMockBucket(options.webVideoBucket)
+
+ servers = await createMultipleServers(2, getConfig())
await setAccessTokensToServers(servers)
await doubleFollow(servers[0], servers[1])
@@ -250,6 +252,29 @@ function runTestSuite (options: {
}
})
+ if (options.useMockBaseUrl) {
+ it('Should update streaming playlist infohash if object storage base url changed', async function () {
+ const newBaseMockUrl = `http://example.com`
+
+ await servers[1].kill()
+
+ await servers[1].run(
+ merge(getConfig(), {
+ object_storage: {
+ streaming_playlists: {
+ base_url: `${newBaseMockUrl}/${options.playlistBucket}`
+ }
+ }
+ })
+ )
+
+ const { uuid } = await servers[1].videos.find({ name: 'video 2' })
+ const video = await servers[1].videos.get({ id: uuid })
+
+ await checkPlaylistInfohash({ video, files: getHLS(video).files, sqlCommand: sqlCommands[1] })
+ })
+ }
+
it('Should fetch correctly all the files', async function () {
for (const url of deletedUrls.concat(keptUrls)) {
await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
diff --git a/server/core/lib/hls.ts b/server/core/lib/hls.ts
index c6a766d5c6..c9a855259a 100644
--- a/server/core/lib/hls.ts
+++ b/server/core/lib/hls.ts
@@ -2,6 +2,7 @@ import { sortBy, uniqify, uuidRegex } from '@peertube/peertube-core-utils'
import { ffprobePromise, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
import { FileStorage, VideoResolution } from '@peertube/peertube-models'
import { sha256 } from '@peertube/peertube-node-utils'
+import { ApplicationModel } from '@server/models/application/application.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo, MVideoCaption } from '@server/types/models/index.js'
import { ensureDir, move, outputJSON, remove } from 'fs-extra/esm'
@@ -25,7 +26,21 @@ import { VideoPathManager } from './video-path-manager.js'
const lTags = loggerTagsFactory('hls')
export async function updateStreamingPlaylistsInfohashesIfNeeded () {
- const playlistsToUpdateIds = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
+ let playlistsToUpdateIds = new Set(await VideoStreamingPlaylistModel.listIdsByIncorrectPeerVersion())
+
+ if (playlistsToUpdateIds.size !== 0) {
+ logger.info(`Will update ${playlistsToUpdateIds.size} streaming playlists infohash because of protocol version change.`, lTags())
+ }
+
+ if (await ApplicationModel.streamingPlaylistBaseUrlChanged()) {
+ const localIds = await VideoStreamingPlaylistModel.listIdsLocals()
+
+ if (localIds.length !== 0) {
+ logger.info(`Will update ${localIds.length} local streaming playlists infohash because of object storage base URL change.`, lTags())
+
+ playlistsToUpdateIds = new Set([ ...playlistsToUpdateIds, ...localIds ])
+ }
+ }
// Use separate SQL queries, because we could have many videos to update
for (const playlistId of playlistsToUpdateIds) {
diff --git a/server/core/models/application/application.ts b/server/core/models/application/application.ts
index 2778664bfd..02da09a668 100644
--- a/server/core/models/application/application.ts
+++ b/server/core/models/application/application.ts
@@ -1,6 +1,8 @@
import { getNodeABIVersion } from '@server/helpers/version.js'
+import { CONFIG } from '@server/initializers/config.js'
import memoizee from 'memoizee'
-import { AllowNull, Column, Default, DefaultScope, HasOne, IsInt, Table } from 'sequelize-typescript'
+import { AllowNull, Column, DataType, Default, DefaultScope, HasOne, IsInt, Table } from 'sequelize-typescript'
+import { PickDeep } from 'type-fest'
import { AccountModel } from '../account/account.js'
import { ActorImageModel } from '../actor/actor-image.js'
import { SequelizeModel } from '../shared/index.js'
@@ -23,6 +25,8 @@ export const getServerActor = memoizee(async function () {
return actor
}, { promise: true })
+type ConfigPart = PickDeep<typeof CONFIG, 'OBJECT_STORAGE.STREAMING_PLAYLISTS'>
+
@DefaultScope(() => ({
include: [
{
@@ -54,6 +58,10 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {
@Column
declare nodeABIVersion: string
+ @AllowNull(true)
+ @Column(DataType.JSONB)
+ declare configPart: ConfigPart
+
@HasOne(() => AccountModel, {
foreignKey: {
allowNull: true
@@ -62,6 +70,9 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {
})
declare Account: Awaited<AccountModel>
+ private static lastRunConfigPart: ConfigPart
+ private static lastRunNodeABIVersion: string
+
static countTotal () {
return ApplicationModel.count()
}
@@ -73,15 +84,33 @@ export class ApplicationModel extends SequelizeModel<ApplicationModel> {
static async nodeABIChanged () {
const application = await this.load()
- return application.nodeABIVersion !== getNodeABIVersion()
+ const nodeABIVersion = this.lastRunNodeABIVersion || application.nodeABIVersion
+
+ return nodeABIVersion !== getNodeABIVersion()
+ }
+
+ static async streamingPlaylistBaseUrlChanged () {
+ const application = await this.load()
+ const configPart = this.lastRunConfigPart || application.configPart
+
+ return configPart?.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL !== CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL
}
- static async updateNodeVersions () {
+ static async updateNodeVersionsOrConfig () {
const application = await this.load()
+ this.lastRunNodeABIVersion = application.nodeABIVersion
+ this.lastRunConfigPart = application.configPart
+
application.nodeABIVersion = getNodeABIVersion()
application.nodeVersion = process.version
+ application.configPart = {
+ OBJECT_STORAGE: {
+ STREAMING_PLAYLISTS: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
+ }
+ }
+
await application.save()
}
}
diff --git a/server/core/models/video/video-streaming-playlist.ts b/server/core/models/video/video-streaming-playlist.ts
index 29dec28b6a..6fb11533b3 100644
--- a/server/core/models/video/video-streaming-playlist.ts
+++ b/server/core/models/video/video-streaming-playlist.ts
@@ -157,7 +157,7 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
return hashes
}
- static async listByIncorrectPeerVersion () {
+ static async listIdsByIncorrectPeerVersion () {
const rows = await VideoStreamingPlaylistModel.unscoped().findAll({
raw: true,
attributes: [ 'id' ],
@@ -171,6 +171,26 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
return rows.map(r => r.id)
}
+ static async listIdsLocals () {
+ const rows = await VideoStreamingPlaylistModel.unscoped().findAll({
+ raw: true,
+ attributes: [ 'id' ],
+ include: [
+ {
+ model: VideoModel.unscoped(),
+ required: true,
+ where: {
+ remote: false
+ }
+ }
+ ]
+ })
+
+ return rows.map(r => r.id)
+ }
+
+ // ---------------------------------------------------------------------------
+
static loadWithVideoAndFiles (id: number) {
const options = {
include: [
diff --git a/server/server.ts b/server/server.ts
index 62e3e4daa9..a625c3d74c 100644
--- a/server/server.ts
+++ b/server/server.ts
@@ -357,7 +357,7 @@ async function startApplication () {
}
}
- ApplicationModel.updateNodeVersions()
+ ApplicationModel.updateNodeVersionsOrConfig()
.catch(err => logger.error('Cannot update node versions.', { err }))
JobQueue.Instance.start()