Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Move infohash in a dedicated table

Chocobozzza3107fab888c
diff --git a/packages/tests/src/shared/sql-command.ts b/packages/tests/src/shared/sql-command.ts
index de9d848ee3..e7166fe8fc 100644
--- a/packages/tests/src/shared/sql-command.ts
+++ b/packages/tests/src/shared/sql-command.ts
@@ -171,12 +171,13 @@ export class SQLCommand {
// ---------------------------------------------------------------------------
async getPlaylistInfohash (playlistId: number) {
- const query = 'SELECT "p2pMediaLoaderInfohashes" FROM "videoStreamingPlaylist" WHERE id = :playlistId'
+ const query = `SELECT CONVERT_FROM("infohash", 'SQL_ASCII') AS "infohash" ` +
+ `FROM "videoInfohash" WHERE "videoStreamingPlaylistId" = :playlistId`
- const result = await this.selectQuery<{ p2pMediaLoaderInfohashes: string }>(query, { playlistId })
+ const result = await this.selectQuery<{ infohash: string }>(query, { playlistId })
if (!result || result.length === 0) return []
- return result[0].p2pMediaLoaderInfohashes
+ return result.map(r => r.infohash)
}
// ---------------------------------------------------------------------------
diff --git a/packages/tests/src/shared/streaming-playlists.ts b/packages/tests/src/shared/streaming-playlists.ts
index 459c687bc6..4cb4dc598f 100644
--- a/packages/tests/src/shared/streaming-playlists.ts
+++ b/packages/tests/src/shared/streaming-playlists.ts
@@ -110,7 +110,7 @@ export async function checkPlaylistInfohash (options: {
expect(dbInfohashes).to.include(infohash)
- await checkTrackerInfohash(video.account.host, infohash)
+ await checkTrackerInfohash(video.account.host, Buffer.from(infohash, 'ascii').toString('hex'))
}
}
diff --git a/packages/tests/src/shared/tracker.ts b/packages/tests/src/shared/tracker.ts
index 38bcc2754a..34f7b22fe9 100644
--- a/packages/tests/src/shared/tracker.ts
+++ b/packages/tests/src/shared/tracker.ts
@@ -1,12 +1,12 @@
import { makeGetRequest } from '@peertube/peertube-server-commands'
import { expect } from 'chai'
-export async function checkTrackerInfohash (serverUrl: string, infohash: string) {
+export async function checkTrackerInfohash (serverUrl: string, infohashHex: string) {
const path = '/tracker/announce'
// From bittorrent-tracker
// oxlint-disable-next-line @typescript-eslint/no-deprecated
- const infohashBinary = escape(Buffer.from(infohash, 'hex').toString('binary')).replace(/[@*/+]/g, function (char) {
+ const infohashBinary = escape(Buffer.from(infohashHex, 'hex').toString('binary')).replace(/[@*/+]/g, function (char) {
return '%' + char.charCodeAt(0).toString(16).toUpperCase()
})
diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.ts
index 732f91e52a..57784385d5 100644
--- a/server/core/controllers/api/videos/source.ts
+++ b/server/core/controllers/api/videos/source.ts
@@ -14,7 +14,7 @@ import { buildNextVideoState } from '@server/lib/video-state.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { VideoChannelActivityModel } from '@server/models/video/video-channel-activity.js'
import { VideoModel } from '@server/models/video/video.js'
-import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFull } from '@server/types/models/index.js'
+import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFileInfoHash, MVideoFull } from '@server/types/models/index.js'
import express from 'express'
import { move } from 'fs-extra/esm'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
@@ -93,7 +93,12 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R
const videoPhysicalFile = res.locals.updateVideoFileResumable
const user = res.locals.oauth.token.User
- const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video', ffprobe: res.locals.ffprobe })
+ const videoFile = await buildNewFile({
+ path: videoPhysicalFile.path,
+ mode: 'web-video',
+ ffprobe: res.locals.ffprobe
+ }) as MVideoFileInfoHash
+
const originalFilename = videoPhysicalFile.originalname
const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoFull.uuid)
diff --git a/server/core/controllers/tracker.ts b/server/core/controllers/tracker.ts
index 47cf83bc47..7b7c00bdb2 100644
--- a/server/core/controllers/tracker.ts
+++ b/server/core/controllers/tracker.ts
@@ -7,8 +7,7 @@ import { WebSocketServer } from 'ws'
import { logger } from '../helpers/logger.js'
import { CONFIG } from '../initializers/config.js'
import { LRU_CACHE, TRACKER_RATE_LIMITS } from '../initializers/constants.js'
-import { VideoFileModel } from '../models/video/video-file.js'
-import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js'
+import { VideoInfohashModel } from '../models/video/video-infohash.js'
export const trackerRouter = express.Router()
@@ -30,32 +29,26 @@ const trackerServer = new TrackerServer({
return cb(new Error('Tracker is disabled on this instance.'))
}
- const infohash = Buffer.from(infohashHex, 'hex').toString('binary')
-
const ip = params.type === 'ws'
? params.ip
: params.httpReq.ip
- const key = ip + '-' + infohash
+ const key = ip + '-' + infohashHex
peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1
peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1
if (CONFIG.TRACKER.REJECT_TOO_MANY_ANNOUNCES && peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) {
- return cb(new Error(`Too many requests (${peersIpInfoHash[key]} of ip ${ip} for torrent ${infohash}`))
+ return cb(new Error(`Too many requests (${peersIpInfoHash[key]} of ip ${ip} for torrent ${infohashHex}`))
}
try {
if (CONFIG.TRACKER.PRIVATE === false) return cb()
- const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExistCached(infohash)
- if (playlistExists === true) return cb()
-
- // Classic infohash (not p2p-media-loader custom one), stored as hex, use arg directly
- const videoFileExists = await VideoFileModel.doesInfohashExistCached(infohashHex)
- if (videoFileExists === true) return cb()
+ const infohashExists = await VideoInfohashModel.doesInfohashExistCached(infohashHex)
+ if (infohashExists === true) return cb()
- cb(new Error(`Unknown infoHash ${infohash} requested by ip ${ip}`))
+ cb(new Error(`Unknown infoHash ${infohashHex} requested by ip ${ip}`))
// Close socket connection and block IP for a few time
if (params.type === 'ws') {
diff --git a/server/core/initializers/database.ts b/server/core/initializers/database.ts
index bc8febbeab..b9ab687ccf 100644
--- a/server/core/initializers/database.ts
+++ b/server/core/initializers/database.ts
@@ -74,6 +74,7 @@ import { VideoLiveModel } from '../models/video/video-live.js'
import { VideoPlaylistElementModel } from '../models/video/video-playlist-element.js'
import { VideoPlaylistModel } from '../models/video/video-playlist.js'
import { VideoShareModel } from '../models/video/video-share.js'
+import { VideoInfohashModel } from '../models/video/video-infohash.js'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js'
import { VideoTagModel } from '../models/video/video-tag.js'
import { VideoModel } from '../models/video/video.js'
@@ -189,6 +190,7 @@ export async function initDatabaseModels (silent: boolean) {
UserNotificationModel,
UserNotificationSettingModel,
VideoStreamingPlaylistModel,
+ VideoInfohashModel,
VideoPlaylistModel,
VideoPlaylistElementModel,
LocalVideoViewerModel,
diff --git a/server/core/initializers/migrations/1095-video-infohash.ts b/server/core/initializers/migrations/1095-video-infohash.ts
new file mode 100644
index 0000000000..4576a406af
--- /dev/null
+++ b/server/core/initializers/migrations/1095-video-infohash.ts
@@ -0,0 +1,72 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+ transaction: Sequelize.Transaction
+ queryInterface: Sequelize.QueryInterface
+ sequelize: Sequelize.Sequelize
+}): Promise<void> {
+ const { transaction } = utils
+
+ {
+ const query = `
+CREATE TABLE IF NOT EXISTS "videoInfohash" (
+ "id" SERIAL,
+ "infohash" BYTEA NOT NULL,
+ "videoStreamingPlaylistId" INTEGER REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+ "videoFileId" INTEGER REFERENCES "videoFile" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+ PRIMARY KEY ("id")
+);`
+
+ await utils.sequelize.query(query, { transaction })
+ }
+
+ // Don't re-hydrate video streaming playlists infohash, PeerTube will re-generate them on the next startup
+
+ // Classic infohash are stored as 40 char hex on the video file
+ {
+ await utils.sequelize.query(
+ `CREATE OR REPLACE FUNCTION safe_decode_hex(input text)
+RETURNS bytea AS $$
+BEGIN
+ RETURN decode(input, 'hex');
+EXCEPTION
+ WHEN others THEN
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;`,
+ { transaction }
+ )
+
+ await utils.sequelize.query(
+ `INSERT INTO "videoInfohash" ("infohash", "videoFileId") ` +
+ `SELECT safe_decode_hex("infoHash"), "id" FROM "videoFile" WHERE "infoHash" IS NOT NULL AND safe_decode_hex("infoHash") IS NOT NULL`,
+ { transaction }
+ )
+
+ await utils.sequelize.query(`DROP FUNCTION safe_decode_hex(text);`, { transaction })
+ }
+
+ // Dropping a column also drops indexes that only depend on it (the old GIN index, and "videoFile"."infoHash")
+ {
+ await utils.sequelize.query(
+ `ALTER TABLE "videoStreamingPlaylist" DROP COLUMN "p2pMediaLoaderInfohashes"`,
+ { transaction }
+ )
+ }
+
+ {
+ await utils.sequelize.query(
+ `ALTER TABLE "videoFile" DROP COLUMN "infoHash"`,
+ { transaction }
+ )
+ }
+}
+
+function down () {
+ throw new Error('Not implemented.')
+}
+
+export {
+ down,
+ up
+}
diff --git a/server/core/lib/activitypub/videos/shared/abstract-builder.ts b/server/core/lib/activitypub/videos/shared/abstract-builder.ts
index 5421942bda..aebcac6f32 100644
--- a/server/core/lib/activitypub/videos/shared/abstract-builder.ts
+++ b/server/core/lib/activitypub/videos/shared/abstract-builder.ts
@@ -13,12 +13,14 @@ import { getServerAccount } from '@server/models/application/application.js'
import { StoryboardModel } from '@server/models/video/storyboard.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'
import { VideoLiveScheduleModel } from '@server/models/video/video-live-schedule.js'
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
import {
MStreamingPlaylistFiles,
MStreamingPlaylistFilesVideo,
+ MStreamingPlaylistFormattable,
MVideo,
MVideoCaption,
MVideoFile,
@@ -140,14 +142,10 @@ export abstract class APVideoAbstractBuilder {
protected async setWebVideoFiles (video: MVideoFull, t: Transaction) {
const oldFiles = video.VideoFiles || []
- const newVideoFiles = getFileAttributesFromUrl(video, this.videoObject.url, oldFiles).map(a => new VideoFileModel(a))
+ const toCreate = getFileAttributesFromUrl(video, this.videoObject.url, oldFiles)
+ .map(({ file, infoHash }) => ({ file: new VideoFileModel(file), infoHash }))
- // Remove video files that do not exist anymore
- await deleteAllModels(filterNonExistingModels(oldFiles, newVideoFiles), t)
-
- // Update or add other one
- const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))
- video.VideoFiles = await Promise.all(upsertTasks)
+ video.VideoFiles = await this.saveFiles({ toCreate, oldFiles, t, mode: 'video' })
}
protected async updateChapters (video: MVideoFull) {
@@ -182,20 +180,27 @@ export abstract class APVideoAbstractBuilder {
}
protected async setStreamingPlaylists (video: MVideoFull, t: Transaction) {
- const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject)
- const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
+ const toCreate = getStreamingPlaylistAttributesFromObject(video, this.videoObject)
// Remove video playlists that do not exist anymore
- await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t)
+ await deleteAllModels(
+ filterNonExistingModels(
+ video.VideoStreamingPlaylists || [],
+ toCreate.map(({ playlist }) => new VideoStreamingPlaylistModel(playlist))
+ ),
+ t
+ )
const oldPlaylists = video.VideoStreamingPlaylists
video.VideoStreamingPlaylists = []
- for (const playlistAttributes of streamingPlaylistAttributes) {
- const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)
+ for (const { playlist, tags, infoHashes } of toCreate) {
+ const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlist, t)
streamingPlaylistModel.Video = video
- await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, playlistAttributes.tagAPObject, t)
+ await streamingPlaylistModel.setInfoHashes(infoHashes ?? [], t)
+
+ await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, tags ?? [], t)
video.VideoStreamingPlaylists.push(streamingPlaylistModel)
}
@@ -204,7 +209,7 @@ export abstract class APVideoAbstractBuilder {
private async insertOrReplaceStreamingPlaylist (attributes: CreationAttributes<VideoStreamingPlaylistModel>, t: Transaction) {
const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t })
- return streamingPlaylist as MStreamingPlaylistFilesVideo
+ return streamingPlaylist as MStreamingPlaylistFilesVideo & MStreamingPlaylistFormattable
}
private getStreamingPlaylistFiles (oldPlaylists: MStreamingPlaylistFiles[], type: VideoStreamingPlaylistType_Type) {
@@ -222,17 +227,13 @@ export abstract class APVideoAbstractBuilder {
) {
const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(oldPlaylists || [], playlistModel.type)
- const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(
+ const toCreate = getFileAttributesFromUrl(
playlistModel,
tagObjects,
oldStreamingPlaylistFiles
- ).map(a => new VideoFileModel(a))
-
- await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t)
+ ).map(({ file, infoHash }) => ({ file: new VideoFileModel(file), infoHash }))
- // Update or add other one
- const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))
- playlistModel.VideoFiles = await Promise.all(upsertTasks)
+ playlistModel.VideoFiles = await this.saveFiles({ toCreate, oldFiles: oldStreamingPlaylistFiles, t, mode: 'streaming-playlist' })
}
protected async setAutomaticTags (options: {
@@ -253,4 +254,27 @@ export abstract class APVideoAbstractBuilder {
return automaticTagsByAccount
}
+
+ protected async saveFiles (options: {
+ toCreate: { file: MVideoFile, infoHash: string }[]
+ oldFiles: MVideoFile[]
+ t: Transaction
+ mode: 'video' | 'streaming-playlist'
+ }) {
+ const { toCreate, oldFiles, t, mode } = options
+
+ // Remove video files that do not exist anymore
+ await deleteAllModels(filterNonExistingModels(oldFiles, toCreate.map(({ file }) => file)), t)
+
+ // Update or add other one
+ const upsertTasks = toCreate.map(async ({ file, infoHash }) => {
+ const newFile = await VideoFileModel.customUpsert(file, mode, t)
+
+ newFile.InfoHash = await VideoInfohashModel.replaceFileInfohash(newFile.id, infoHash, t)
+
+ return newFile
+ })
+
+ return Promise.all(upsertTasks)
+ }
}
diff --git a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts
index bde81627c4..0ee448ce94 100644
--- a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -61,7 +61,8 @@ export function getFileAttributesFromUrl (
const fileUrlObjects = urls.filter(u => isAPVideoUrlObject(u))
if (fileUrlObjects.length === 0) return []
- const attributes: FilteredModelAttributes<VideoFileModel>[] = []
+ const result: { file: FilteredModelAttributes<VideoFileModel>, infoHash: string }[] = []
+
for (const fileUrlObject of fileUrlObjects) {
// Fetch associated metadata url, if any
const metadata = urls.filter(isAPVideoFileUrlMetadataObject)
@@ -100,7 +101,6 @@ export function getFileAttributesFromUrl (
filename: basename(fileUrl),
fileUrl,
- infoHash,
torrentFilename,
torrentUrl,
@@ -112,10 +112,10 @@ export function getFileAttributesFromUrl (
videoStreamingPlaylistId
}
- attributes.push(attribute)
+ result.push({ file: attribute, infoHash })
}
- return attributes
+ return result
}
function buildFileFormatFlags (fileUrl: ActivityVideoUrlObject, isStreamingPlaylist: boolean) {
@@ -165,13 +165,16 @@ export function getStreamingPlaylistAttributesFromObject (video: MVideoId & MVid
const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u))
if (playlistUrls.length === 0) return []
- const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []
+ const result: {
+ playlist: FilteredModelAttributes<VideoStreamingPlaylistModel>
+ tags?: ActivityTagObject[]
+ infoHashes: string[]
+ }[] = []
+
for (const playlistUrlObject of playlistUrls) {
const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)
- const files = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u))
-
- const attribute = {
+ const playlist = {
type: VideoStreamingPlaylistType.HLS,
playlistFilename: basename(playlistUrlObject.href),
@@ -182,21 +185,26 @@ export function getStreamingPlaylistAttributesFromObject (video: MVideoId & MVid
: null,
segmentsSha256Url: segmentsSha256UrlObject?.href ?? null,
-
- p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(
- video.uuid,
- files.map(f => ({ resolution: Math.min(f.height ?? Infinity, f.width ?? Infinity) }))
- ),
p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
- videoId: video.id,
-
- tagAPObject: playlistUrlObject.tag
+ videoId: video.id
}
- attributes.push(attribute)
+ result.push({
+ playlist,
+
+ tags: playlistUrlObject.tag,
+
+ // Persisted into the dedicated infohash table once the playlist is upserted (see setStreamingPlaylists)
+ infoHashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(
+ video.uuid,
+ playlistUrlObject.tag
+ .filter(u => isAPVideoUrlObject(u))
+ .map(f => ({ resolution: Math.min(f.height ?? Infinity, f.width ?? Infinity) }))
+ )
+ })
}
- return attributes
+ return result
}
export function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) {
diff --git a/server/core/lib/hls.ts b/server/core/lib/hls.ts
index cf2f566c2c..14e38755c8 100644
--- a/server/core/lib/hls.ts
+++ b/server/core/lib/hls.ts
@@ -49,7 +49,7 @@ export async function updateStreamingPlaylistsInfohashesIfNeeded () {
const playlist = await VideoStreamingPlaylistModel.loadWithVideo(playlistId, t)
const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlistId, t)
- playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)
+ await playlist.buildAndSetInfoHashes(playlist.Video, videoFiles, t)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
await playlist.save({ transaction: t })
@@ -67,7 +67,7 @@ export async function updateM3U8AndShaPlaylist (video: MVideo, playlist: MStream
// Refresh playlist, operations can take some time
playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id)
- playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)
+ await playlistWithFiles.buildAndSetInfoHashes(video, playlistWithFiles.VideoFiles)
await playlistWithFiles.save()
video.setHLSPlaylist(playlistWithFiles)
diff --git a/server/core/lib/job-queue/handlers/manage-video-torrent.ts b/server/core/lib/job-queue/handlers/manage-video-torrent.ts
index e07468c919..465e4f8411 100644
--- a/server/core/lib/job-queue/handlers/manage-video-torrent.ts
+++ b/server/core/lib/job-queue/handlers/manage-video-torrent.ts
@@ -1,11 +1,12 @@
-import { Job } from 'bullmq'
+import { ManageVideoTorrentPayload } from '@peertube/peertube-models'
import { extractVideo } from '@server/helpers/video.js'
-import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/lib/webtorrent.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { VideoModel } from '@server/models/video/video.js'
+import { createTorrentForFile, updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
-import { ManageVideoTorrentPayload } from '@peertube/peertube-models'
+import { VideoModel } from '@server/models/video/video.js'
+import { Job } from 'bullmq'
import { logger } from '../../../helpers/logger.js'
async function processManageVideoTorrent (job: Job) {
@@ -38,17 +39,17 @@ async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'c
await video.reload()
await file.reload()
- await createTorrentAndSetInfoHash(video, file)
+ const { infoHash, torrentFilename } = await createTorrentForFile(video, file)
// Refresh videoFile because the createTorrentAndSetInfoHash could be long
const refreshedFile = await VideoFileModel.loadWithVideo(file.id)
// File does not exist anymore, remove the generated torrent
if (!refreshedFile) return file.removeTorrent()
- refreshedFile.infoHash = file.infoHash
- refreshedFile.torrentFilename = file.torrentFilename
-
+ refreshedFile.torrentFilename = torrentFilename
await refreshedFile.save()
+
+ await VideoInfohashModel.replaceFileInfohash(refreshedFile.id, infoHash)
} finally {
fileMutexReleaser()
}
@@ -67,9 +68,7 @@ async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { ac
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid)
try {
- await updateTorrentMetadata(video || streamingPlaylist, file)
-
- await file.save()
+ await updateTorrentForFileAndSave(video || streamingPlaylist, file)
} finally {
fileMutexReleaser()
}
diff --git a/server/core/lib/job-queue/handlers/video-file-import.ts b/server/core/lib/job-queue/handlers/video-file-import.ts
index 104d5ec2c4..b2506170ae 100644
--- a/server/core/lib/job-queue/handlers/video-file-import.ts
+++ b/server/core/lib/job-queue/handlers/video-file-import.ts
@@ -5,7 +5,8 @@ import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
import { buildNewFile } from '@server/lib/video-file.js'
import { buildMoveVideoJob } from '@server/lib/video-jobs.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'
+import { createTorrentForFile } from '@server/lib/webtorrent.js'
+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoFull } from '@server/types/models/index.js'
import { Job } from 'bullmq'
@@ -70,8 +71,12 @@ async function updateVideoFile (video: MVideoFull, inputFilePath: string) {
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
await copy(inputFilePath, outputPath)
- video.VideoFiles.push(newVideoFile)
- await createTorrentAndSetInfoHash(video, newVideoFile)
+ const { infoHash, torrentFilename } = await createTorrentForFile(video, newVideoFile)
+ newVideoFile.torrentFilename = torrentFilename
+
+ const infohashModel = await VideoInfohashModel.replaceFileInfohash(newVideoFile.id, infoHash)
await newVideoFile.save()
+
+ video.VideoFiles.push(Object.assign(newVideoFile, { InfoHash: infohashModel }))
}
diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.ts
index 947adeefce..b416f974a5 100644
--- a/server/core/lib/job-queue/handlers/video-import.ts
+++ b/server/core/lib/job-queue/handlers/video-import.ts
@@ -24,9 +24,10 @@ import { buildNewFile } from '@server/lib/video-file.js'
import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
-import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '@server/lib/webtorrent.js'
+import { createTorrentForFile, downloadWebTorrentVideo } from '@server/lib/webtorrent.js'
import { UserModel } from '@server/models/user/user.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'
import { MUserId, MVideoFile, MVideoFull } from '@server/types/models/index.js'
import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
import { Job } from 'bullmq'
@@ -215,17 +216,23 @@ async function processFile (options: {
const thumbnails = await generateThumbnails({ videoImportWithFiles, videoFile, ffprobe })
- // Create torrent
- await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
+ const { infoHash, torrentFilename } = await createTorrentForFile(videoImportWithFiles.Video, videoFile)
const { videoImportUpdated, video } = await retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async t => {
// Refresh video
const video = await VideoModel.load(videoImportWithFiles.videoId, t)
- if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')
+ if (!video) {
+ await videoFile.removeTorrent()
+ throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')
+ }
+
+ videoFile.torrentFilename = torrentFilename
await videoFile.save({ transaction: t })
+ await VideoInfohashModel.replaceFileInfohash(videoFile.id, infoHash, t)
+
// Update video DB object
video.duration = duration
video.state = buildNextVideoState(video.state)
diff --git a/server/core/lib/live/shared/muxing-session.ts b/server/core/lib/live/shared/muxing-session.ts
index 0c0a5f7745..feed6ecbb8 100644
--- a/server/core/lib/live/shared/muxing-session.ts
+++ b/server/core/lib/live/shared/muxing-session.ts
@@ -258,7 +258,10 @@ class MuxingSession extends EventEmitter implements MuxingSession {
hlsStreams.push(VideoResolution.H_NOVIDEO)
}
- this.streamingPlaylist.assignP2PMediaLoaderInfoHashes(this.videoLive.Video, Array.from(hlsStreams).map(r => ({ resolution: r })))
+ await this.streamingPlaylist.buildAndSetInfoHashes(
+ this.videoLive.Video,
+ Array.from(hlsStreams).map(r => ({ resolution: r }))
+ )
await this.streamingPlaylist.save()
} catch (err) {
@@ -338,7 +341,6 @@ class MuxingSession extends EventEmitter implements MuxingSession {
fps,
size: -1,
extname: '.ts',
- infoHash: null,
formatFlags: VideoFileFormatFlag.NONE,
streams: resolution === VideoResolution.H_NOVIDEO
? VideoFileStream.AUDIO
diff --git a/server/core/lib/local-video-creator.ts b/server/core/lib/local-video-creator.ts
index c7cd2901cb..bb5611676c 100644
--- a/server/core/lib/local-video-creator.ts
+++ b/server/core/lib/local-video-creator.ts
@@ -25,7 +25,7 @@ import { VideoLiveScheduleModel } from '@server/models/video/video-live-schedule
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { VideoPasswordModel } from '@server/models/video/video-password.js'
import { VideoModel } from '@server/models/video/video.js'
-import { MChannel, MChannelAccountLight, MUserAccountId, MVideoFile, MVideoFull } from '@server/types/models/index.js'
+import { MChannel, MChannelAccountLight, MUserAccountId, MVideoFileInfoHash, MVideoFull } from '@server/types/models/index.js'
import { FilteredModelAttributes } from '@server/types/sequelize.js'
import { FfprobeData } from 'fluent-ffmpeg'
import { move } from 'fs-extra/esm'
@@ -93,7 +93,7 @@ export class LocalVideoCreator {
private readonly videoAttributeResultHook: VideoAttributeHookFilter
private video: MVideoFull
- private videoFile: MVideoFile
+ private videoFile: MVideoFileInfoHash
private videoPath: string
constructor (
@@ -142,7 +142,11 @@ export class LocalVideoCreator {
this.video.url = getLocalVideoActivityPubUrl(this.video)
if (this.videoFilePath) {
- this.videoFile = await buildNewFile({ path: this.videoFilePath, mode: 'web-video', ffprobe: this.videoFileProbe })
+ this.videoFile = await buildNewFile({
+ path: this.videoFilePath,
+ mode: 'web-video',
+ ffprobe: this.videoFileProbe
+ }) as MVideoFileInfoHash
this.videoPath = VideoPathManager.Instance.getFSVideoFileOutputPath(this.video, this.videoFile)
await move(this.videoFilePath, this.videoPath)
diff --git a/server/core/lib/local-video-updater.ts b/server/core/lib/local-video-updater.ts
index de5689bf72..281de57a00 100644
--- a/server/core/lib/local-video-updater.ts
+++ b/server/core/lib/local-video-updater.ts
@@ -331,7 +331,7 @@ export class LocalVideoUpdater {
if (filePathChanged && hls) {
logger.debug('Updating HLS playlist file paths after privacy change', this.lTags())
- hls.assignP2PMediaLoaderInfoHashes(video, hls.VideoFiles)
+ await hls.buildAndSetInfoHashes(video, hls.VideoFiles)
await hls.save()
}
}
diff --git a/server/core/lib/move-storage/move-to-file-system.ts b/server/core/lib/move-storage/move-to-file-system.ts
index 2a802c9ff6..fc06e06ac7 100644
--- a/server/core/lib/move-storage/move-to-file-system.ts
+++ b/server/core/lib/move-storage/move-to-file-system.ts
@@ -21,7 +21,7 @@ import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/p
import { updateHLSMasterOnCaptionChange, upsertCaptionPlaylistOnFS } from '@server/lib/video-captions.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
+import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'
import { VideoModel } from '@server/models/video/video.js'
import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
@@ -169,7 +169,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
}
if (updatedFile === true) {
- playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
+ await playlist.buildAndSetInfoHashes(video, playlist.VideoFiles)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
await playlist.save()
@@ -191,8 +191,7 @@ async function onVideoFileMoved (options: {
file.storage = FileStorage.FILE_SYSTEM
- await updateTorrentMetadata(videoOrPlaylist, file)
- await file.save()
+ await updateTorrentForFileAndSave(videoOrPlaylist, file)
logger.debug('Removing web video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
await objetStorageRemover()
diff --git a/server/core/lib/move-storage/move-to-object-storage.ts b/server/core/lib/move-storage/move-to-object-storage.ts
index 78bf0e93c0..62ca4b71bc 100644
--- a/server/core/lib/move-storage/move-to-object-storage.ts
+++ b/server/core/lib/move-storage/move-to-object-storage.ts
@@ -13,7 +13,7 @@ import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/p
import { updateHLSMasterOnCaptionChange } from '@server/lib/video-captions.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
+import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { VideoModel } from '@server/models/video/video.js'
import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
@@ -199,7 +199,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {
}
if (updatedFile === true) {
- playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
+ await playlist.buildAndSetInfoHashes(video, playlist.VideoFiles)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
await playlist.save()
@@ -222,8 +222,7 @@ async function onVideoFileMoved (options: {
file.storage = FileStorage.OBJECT_STORAGE
- await updateTorrentMetadata(videoOrPlaylist, file)
- await file.save()
+ await updateTorrentForFileAndSave(videoOrPlaylist, file)
logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())
await remove(oldPath)
diff --git a/server/core/lib/transcoding/hls-transcoding.ts b/server/core/lib/transcoding/hls-transcoding.ts
index 6f3a50be75..b0738b1977 100644
--- a/server/core/lib/transcoding/hls-transcoding.ts
+++ b/server/core/lib/transcoding/hls-transcoding.ts
@@ -3,8 +3,9 @@ import { canCopyForHLS, getVideoStreamDuration, HLSFromTSTranscodeOptions, HLSTr
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { deleteFileAndCatch } from '@server/helpers/fs.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
-import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'
-import { MVideo } from '@server/types/models/index.js'
+import { createTorrentForFile } from '@server/lib/webtorrent.js'
+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'
+import { MVideo, MVideoFile } from '@server/types/models/index.js'
import { MutexInterface } from 'async-mutex'
import { Job } from 'bullmq'
import { ensureDir, move } from 'fs-extra/esm'
@@ -116,7 +117,8 @@ export async function onHLSVideoFileTranscoding (options: {
await video.save()
}
- await createTorrentAndSetInfoHash(playlist, newVideoFile)
+ const { infoHash, torrentFilename } = await createTorrentForFile(playlist, newVideoFile)
+ newVideoFile.torrentFilename = torrentFilename
const oldFile = await VideoFileModel.loadHLSFile({
playlistId: playlist.id,
@@ -129,7 +131,13 @@ export async function onHLSVideoFileTranscoding (options: {
await oldFile.destroy()
}
- const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)
+ const savedVideoFile = await sequelizeTypescript.transaction(async t => {
+ const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', t) as MVideoFile
+
+ await VideoInfohashModel.replaceFileInfohash(savedVideoFile.id, infoHash, t)
+
+ return savedVideoFile
+ })
if (playlistGenerated) {
await createAllCaptionPlaylistsOnFSIfNeeded(video)
diff --git a/server/core/lib/transcoding/web-transcoding.ts b/server/core/lib/transcoding/web-transcoding.ts
index 3524742401..485237ca43 100644
--- a/server/core/lib/transcoding/web-transcoding.ts
+++ b/server/core/lib/transcoding/web-transcoding.ts
@@ -8,7 +8,9 @@ import {
import { VideoFileStream } from '@peertube/peertube-models'
import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js'
import { deleteFileAndCatch } from '@server/helpers/fs.js'
-import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { createTorrentForFile } from '@server/lib/webtorrent.js'
+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoFile, MVideoFull } from '@server/types/models/index.js'
import { Job } from 'bullmq'
@@ -225,7 +227,8 @@ export async function onWebVideoFileTranscoding (options: {
await move(videoOutputPath, outputPath, { overwrite: true })
- await createTorrentAndSetInfoHash(video, videoFile)
+ const { infoHash, torrentFilename } = await createTorrentForFile(video, videoFile)
+ videoFile.torrentFilename = torrentFilename
if (deleteWebInputVideoFile) {
await saveNewOriginalFileIfNeeded(video, deleteWebInputVideoFile)
@@ -237,14 +240,18 @@ export async function onWebVideoFileTranscoding (options: {
const existingFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })
if (existingFile) await video.removeWebVideoFile(existingFile)
- await VideoFileModel.customUpsert(videoFile, 'video', undefined)
+ await sequelizeTypescript.transaction(async t => {
+ await VideoFileModel.customUpsert(videoFile, 'video', t)
+ await VideoInfohashModel.replaceFileInfohash(videoFile.id, infoHash, t)
+ })
+
video.VideoFiles = await video.$get('VideoFiles')
if (wasAudioFile) {
await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: false })
}
- return { video, videoFile }
+ return { videoFile }
} finally {
mutexReleaser()
}
diff --git a/server/core/lib/video-studio.ts b/server/core/lib/video-studio.ts
index d1dabe60c3..d6c6a7c88f 100644
--- a/server/core/lib/video-studio.ts
+++ b/server/core/lib/video-studio.ts
@@ -3,7 +3,9 @@ import { getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
-import { createTorrentAndSetInfoHashFromPath } from '@server/lib/webtorrent.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { createTorrentForFileFromPath } from '@server/lib/webtorrent.js'
+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'
import { VideoModel } from '@server/models/video/video.js'
import { MUser, MVideoFile, MVideoFull, MVideoWithAllFiles, MVideoWithFile } from '@server/types/models/index.js'
import { move, remove } from 'fs-extra/esm'
@@ -107,10 +109,15 @@ export async function onVideoStudioEnded (options: {
await safeCleanupStudioTMPFiles(tasks)
- await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath)
+ const { infoHash, torrentFilename } = await createTorrentForFileFromPath(video, newFile, outputPath)
await removeAllFiles(video, newFile)
- await newFile.save()
+ await sequelizeTypescript.transaction(async t => {
+ newFile.torrentFilename = torrentFilename
+ await newFile.save({ transaction: t })
+
+ await VideoInfohashModel.replaceFileInfohash(newFile.id, infoHash, t)
+ })
video.duration = await getVideoStreamDuration(outputPath)
video.aspectRatio = buildAspectRatio({ width: newFile.width, height: newFile.height })
diff --git a/server/core/lib/webtorrent.ts b/server/core/lib/webtorrent.ts
index c018ef3869..67dfac584a 100644
--- a/server/core/lib/webtorrent.ts
+++ b/server/core/lib/webtorrent.ts
@@ -3,7 +3,8 @@ import { WEBSERVER } from '@server/initializers/constants.js'
import { generateTorrentFileName } from '@server/lib/paths.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { createTorrentFromWorker } from '@server/lib/worker/parent-process.js'
-import { MVideoFile } from '@server/types/models/video/video-file.js'
+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'
+import { MVideoFile, MVideoFileInfoHash } from '@server/types/models/video/video-file.js'
import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist.js'
import { MVideo } from '@server/types/models/video/video.js'
import bencode from 'bencode'
@@ -111,13 +112,13 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentPat
})
}
-export function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
+export function createTorrentForFile (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), videoPath => {
- return createTorrentAndSetInfoHashFromPath(videoOrPlaylist, videoFile, videoPath)
+ return createTorrentForFileFromPath(videoOrPlaylist, videoFile, videoPath)
})
}
-export async function createTorrentAndSetInfoHashFromPath (
+export async function createTorrentForFileFromPath (
videoOrPlaylist: MVideo | MStreamingPlaylistVideo,
videoFile: MVideoFile,
filePath: string
@@ -147,11 +148,14 @@ export async function createTorrentAndSetInfoHashFromPath (
// FIXME: typings: parseTorrent now returns an async result
const parsedTorrent = await (parseTorrent(torrentContent) as unknown as Promise<parseTorrent.Instance>)
- videoFile.infoHash = parsedTorrent.infoHash
- videoFile.torrentFilename = torrentFilename
+
+ return {
+ infoHash: parsedTorrent.infoHash,
+ torrentFilename: torrentFilename
+ }
}
-export async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
+export async function updateTorrentForFileAndSave (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {
const video = extractVideo(videoOrPlaylist)
if (!videoFile.torrentFilename) {
@@ -186,12 +190,15 @@ export async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamin
await remove(oldTorrentPath)
videoFile.torrentFilename = newTorrentFilename
- videoFile.infoHash = sha1(bencode.encode(decoded.info))
+
+ await VideoInfohashModel.replaceFileInfohash(videoFile.id, sha1(bencode.encode(decoded.info)))
+
+ await videoFile.save()
}
export function generateMagnetUri (
video: MVideo,
- videoFile: MVideoFile,
+ videoFile: MVideoFileInfoHash,
trackerUrls: string[]
) {
const xs = videoFile.getTorrentUrl()
@@ -205,7 +212,7 @@ export function generateMagnetUri (
xs,
announce,
urlList,
- infoHash: videoFile.infoHash,
+ infoHash: videoFile.InfoHash.toHexInfohash(),
name: video.name
}
diff --git a/server/core/models/video/formatter/video-activity-pub-format.ts b/server/core/models/video/formatter/video-activity-pub-format.ts
index d9d90b89be..7375383381 100644
--- a/server/core/models/video/formatter/video-activity-pub-format.ts
+++ b/server/core/models/video/formatter/video-activity-pub-format.ts
@@ -26,7 +26,7 @@ import {
getLocalVideoPlayerSettingsActivityPubUrl,
getLocalVideoSharesActivityPubUrl
} from '../../../lib/activitypub/url.js'
-import { MStreamingPlaylistFiles, MUserId, MVideo, MVideoAP, MVideoFile } from '../../../types/models/index.js'
+import { MStreamingPlaylistFormattable, MUserId, MVideo, MVideoAP, MVideoFileInfoHash } from '../../../types/models/index.js'
import { sortByResolutionDesc } from './shared/index.js'
import { getCategoryLabel, getLanguageLabel, getLicenceLabel } from './video-api-format.js'
@@ -203,7 +203,7 @@ function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] {
function buildVideoFileUrls (options: {
video: MVideo
- files: MVideoFile[]
+ files: MVideoFileInfoHash[]
user?: MUserId
}): ActivityUrlObject[] {
const { video, files } = options
@@ -269,9 +269,9 @@ function buildStreamingPlaylistUrls (video: MVideoAP): ActivityPlaylistUrlObject
}))
}
-function buildStreamingPlaylistTags (video: MVideoAP, playlist: MStreamingPlaylistFiles) {
+function buildStreamingPlaylistTags (video: MVideoAP, playlist: MStreamingPlaylistFormattable) {
return [
- ...playlist.p2pMediaLoaderInfohashes.map(i => ({ type: 'Infohash' as 'Infohash', name: i })),
+ ...(playlist.InfoHashes ?? []).map(i => ({ type: 'Infohash' as 'Infohash', name: i.toP2PMediaLoaderInfohash() })),
{
type: 'Link',
diff --git a/server/core/models/video/formatter/video-api-format.ts b/server/core/models/video/formatter/video-api-format.ts
index 980dffb751..dd08126681 100644
--- a/server/core/models/video/formatter/video-api-format.ts
+++ b/server/core/models/video/formatter/video-api-format.ts
@@ -24,8 +24,14 @@ import {
VIDEO_PRIVACIES,
VIDEO_STATES
} from '../../../initializers/constants.js'
-import { MServer, MStreamingPlaylistRedundanciesOpt, MVideoFormattable, MVideoFormattableDetails } from '../../../types/models/index.js'
-import { MVideoFile } from '../../../types/models/video/video-file.js'
+import {
+ MServer,
+ MStreamingPlaylistFormattable,
+ MVideoFormattable,
+ MVideoFormattableAdditionalAttributes,
+ MVideoFormattableDetails
+} from '../../../types/models/index.js'
+import { MVideoFileInfoHash } from '../../../types/models/video/video-file.js'
import { sortByResolutionDesc } from './shared/index.js'
export type VideoFormattingJSONOptions = {
@@ -208,7 +214,7 @@ export function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetail
export function streamingPlaylistsModelToFormattedJSON (
video: MVideoFormattable,
- playlists: MStreamingPlaylistRedundanciesOpt[]
+ playlists: MStreamingPlaylistFormattable[]
): VideoStreamingPlaylist[] {
if (isArray(playlists) === false) return []
@@ -232,7 +238,7 @@ export function streamingPlaylistsModelToFormattedJSON (
export function videoFilesModelToFormattedJSON (
video: MVideoFormattable,
- videoFiles: MVideoFile[],
+ videoFiles: MVideoFileInfoHash[],
options?: {
includePlaylistUrl?: true
includeMagnet?: boolean
@@ -241,7 +247,7 @@ export function videoFilesModelToFormattedJSON (
export function videoFilesModelToFormattedJSON (
video: MVideoFormattable,
- videoFiles: MVideoFile[],
+ videoFiles: MVideoFileInfoHash[],
options: {
includePlaylistUrl?: boolean // default false
includeMagnet?: boolean // default true
@@ -332,7 +338,7 @@ export function getStateLabel (id: number) {
// Private
// ---------------------------------------------------------------------------
-function buildAdditionalAttributes (video: MVideoFormattable, options: VideoFormattingJSONOptions) {
+function buildAdditionalAttributes (video: MVideoFormattable & MVideoFormattableAdditionalAttributes, options: VideoFormattingJSONOptions) {
const add = options.additionalAttributes
const result: Partial<VideoAdditionalAttributes> = {}
diff --git a/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts b/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts
index 8fe4c46310..10dc1fe160 100644
--- a/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts
+++ b/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts
@@ -4,8 +4,8 @@ import { Sequelize } from 'sequelize'
import validator from 'validator'
import { AbstractRunQuery } from '../../../../shared/abstract-run-query.js'
import { createSafeIn } from '../../../../shared/index.js'
-import { VideoTableAttributes } from './video-table-attributes.js'
import { TableAttributeOptions } from './table-attributes-options.model.js'
+import { VideoTableAttributes } from './video-table-attributes.js'
/**
* Abstract builder to create SQL query and fetch video models
@@ -124,6 +124,8 @@ export class AbstractVideoQueryBuilder extends AbstractRunQuery {
...this.buildAttributesObject('VideoFiles', this.tables.getFileAttributes())
}
+
+ this.includeFileInfohashJSONJoin('VideoFiles')
}
protected includeStreamingPlaylistFiles () {
@@ -142,6 +144,42 @@ export class AbstractVideoQueryBuilder extends AbstractRunQuery {
...this.buildAttributesObject('VideoStreamingPlaylists', this.tables.getStreamingPlaylistAttributes()),
...this.buildAttributesObject('VideoStreamingPlaylists->VideoFiles', this.tables.getFileAttributes())
}
+
+ this.includeFileInfohashJSONJoin('VideoStreamingPlaylists->VideoFiles')
+ this.includePlaylistInfohashesJSON()
+ }
+
+ private includePlaylistInfohashesJSON () {
+ this.addJoin(
+ `LEFT JOIN LATERAL (` +
+ `SELECT json_agg(ENCODE("infohash", 'hex')) AS "infohashes" ` +
+ `FROM "videoInfohash" WHERE "videoStreamingPlaylistId" = "VideoStreamingPlaylists"."id"` +
+ `) AS "VideoStreamingPlaylists->InfohashesJSON" ON TRUE`
+ )
+
+ this.attributes = {
+ ...this.attributes,
+
+ '"VideoStreamingPlaylists->InfohashesJSON"."infohashes"': '"VideoStreamingPlaylists.InfohashesJSON"'
+ }
+ }
+
+ private includeFileInfohashJSONJoin (prefix: string) {
+ const alias = `${prefix}->InfohashJSON`
+ const flatPrefix = prefix.replace(/->/g, '.')
+
+ this.addJoin(
+ `LEFT JOIN LATERAL (` +
+ `SELECT json_agg(ENCODE("infohash", 'hex')) AS "infoHash" ` +
+ `FROM "videoInfohash" WHERE "videoFileId" = "${prefix}"."id"` +
+ `) AS "${alias}" ON TRUE`
+ )
+
+ this.attributes = {
+ ...this.attributes,
+
+ [`"${alias}"."infoHash"`]: `"${flatPrefix}.InfohashJSON"`
+ }
}
protected includeUserHistory (userId: number) {
diff --git a/server/core/models/video/sql/video/shared/video-model-builder.ts b/server/core/models/video/sql/video/shared/video-model-builder.ts
index 321d44971a..1042f40baa 100644
--- a/server/core/models/video/sql/video/shared/video-model-builder.ts
+++ b/server/core/models/video/sql/video/shared/video-model-builder.ts
@@ -1,12 +1,12 @@
import { VideoInclude, VideoIncludeType } from '@peertube/peertube-models'
-import { AccountBlocklistModel } from '@server/models/blocklist/account-blocklist.js'
import { AccountModel } from '@server/models/account/account.js'
import { ActorImageModel } from '@server/models/actor/actor-image.js'
import { ActorModel } from '@server/models/actor/actor.js'
import { AutomaticTagModel } from '@server/models/automatic-tag/automatic-tag.js'
import { VideoAutomaticTagModel } from '@server/models/automatic-tag/video-automatic-tag.js'
-import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy.js'
+import { AccountBlocklistModel } from '@server/models/blocklist/account-blocklist.js'
import { ServerBlocklistModel } from '@server/models/blocklist/server-blocklist.js'
+import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy.js'
import { ServerModel } from '@server/models/server/server.js'
import { TrackerModel } from '@server/models/server/tracker.js'
import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js'
@@ -19,11 +19,12 @@ import { ThumbnailModel } from '../../../thumbnail.js'
import { VideoBlacklistModel } from '../../../video-blacklist.js'
import { VideoChannelModel } from '../../../video-channel.js'
import { VideoFileModel } from '../../../video-file.js'
+import { VideoInfohashModel } from '../../../video-infohash.js'
import { VideoLiveModel } from '../../../video-live.js'
import { VideoStreamingPlaylistModel } from '../../../video-streaming-playlist.js'
import { VideoModel } from '../../../video.js'
-import { VideoTableAttributes } from './video-table-attributes.js'
import { TableAttributeOptions } from './table-attributes-options.model.js'
+import { VideoTableAttributes } from './video-table-attributes.js'
type SQLRow = { [id: string]: string | number }
@@ -327,6 +328,7 @@ export class VideoModelBuilder {
const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoFiles')
const videoFileModel = new VideoFileModel(attributes, this.buildOpts)
+ this.addFileInfohash(row, 'VideoFiles', videoFileModel)
videoModel.VideoFiles.push(videoFileModel)
this.videoFileMemo[id] = videoFileModel
@@ -340,6 +342,8 @@ export class VideoModelBuilder {
const streamingPlaylist = new VideoStreamingPlaylistModel(attributes, this.buildOpts)
streamingPlaylist.VideoFiles = []
+ this.addPlaylistInfohashes(row, streamingPlaylist)
+
videoModel.VideoStreamingPlaylists.push(streamingPlaylist)
this.videoStreamingPlaylistMemo[id] = streamingPlaylist
@@ -353,11 +357,28 @@ export class VideoModelBuilder {
const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoStreamingPlaylists.VideoFiles')
const videoFileModel = new VideoFileModel(attributes, this.buildOpts)
+
+ this.addFileInfohash(row, 'VideoStreamingPlaylists.VideoFiles', videoFileModel)
+
streamingPlaylist.VideoFiles.push(videoFileModel)
this.videoFileMemo[id] = videoFileModel
}
+ private addPlaylistInfohashes (row: SQLRow, streamingPlaylist: VideoStreamingPlaylistModel) {
+ const hashes = row['VideoStreamingPlaylists.InfohashesJSON'] as any as string[] || []
+
+ streamingPlaylist.InfoHashes = hashes.map(h => new VideoInfohashModel({ infohash: Buffer.from(h, 'hex') }, this.buildOpts))
+ }
+
+ private addFileInfohash (row: SQLRow, prefixKey: string, videoFile: VideoFileModel) {
+ const hashes = row[`${prefixKey}.InfohashJSON`] as any as string[] || []
+
+ videoFile.InfoHash = hashes.length !== 0
+ ? new VideoInfohashModel({ infohash: Buffer.from(hashes[0], 'hex') }, this.buildOpts)
+ : null
+ }
+
private addRedundancy (row: SQLRow, prefix: string, to: VideoStreamingPlaylistModel) {
if (!to.RedundancyVideos) to.RedundancyVideos = []
diff --git a/server/core/models/video/sql/video/shared/video-table-attributes.ts b/server/core/models/video/sql/video/shared/video-table-attributes.ts
index 93c7449cfb..6fc2d95087 100644
--- a/server/core/models/video/sql/video/shared/video-table-attributes.ts
+++ b/server/core/models/video/sql/video/shared/video-table-attributes.ts
@@ -81,7 +81,6 @@ export class VideoTableAttributes {
'fileUrl',
'torrentFilename',
'torrentUrl',
- 'infoHash',
'fps',
'metadataUrl',
'videoStreamingPlaylistId',
@@ -100,7 +99,6 @@ export class VideoTableAttributes {
'playlistUrl',
'playlistFilename',
'type',
- 'p2pMediaLoaderInfohashes',
'p2pMediaLoaderPeerVersion',
'segmentsSha256Filename',
'segmentsSha256Url',
diff --git a/server/core/models/video/video-file.ts b/server/core/models/video/video-file.ts
index 5810616a58..bd5a479d77 100644
--- a/server/core/models/video/video-file.ts
+++ b/server/core/models/video/video-file.ts
@@ -24,7 +24,6 @@ import { getVideoFileMimeType } from '@server/lib/video-file.js'
import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js'
import { MStreamingPlaylistVideo, MVideo, MVideoWithHost, isStreamingPlaylist } from '@server/types/models/index.js'
import { remove } from 'fs-extra/esm'
-import memoizee from 'memoizee'
import { join } from 'path'
import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'
import {
@@ -36,6 +35,7 @@ import {
Default,
DefaultScope,
ForeignKey,
+ HasOne,
Is,
Scopes,
Table,
@@ -45,13 +45,13 @@ import validator from 'validator'
import {
isVideoFPSResolutionValid,
isVideoFileExtnameValid,
- isVideoFileInfoHashValid,
isVideoFileResolutionValid,
isVideoFileSizeValid
} from '../../helpers/custom-validators/videos.js'
-import { DOWNLOAD_PATHS, LAZY_STATIC_PATHS, MEMOIZE_LENGTH, MEMOIZE_TTL, STATIC_PATHS, WEBSERVER } from '../../initializers/constants.js'
+import { DOWNLOAD_PATHS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants.js'
import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file.js'
import { SequelizeModel, doesExist, parseAggregateResult, throwIfNotValid } from '../shared/index.js'
+import { VideoInfohashModel } from './video-infohash.js'
import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js'
import { VideoModel } from './video.js'
@@ -106,10 +106,6 @@ export enum ScopeNames {
@Table({
tableName: 'videoFile',
indexes: [
- {
- fields: [ 'infoHash' ]
- },
-
{
fields: [ 'torrentFilename' ],
unique: true
@@ -170,11 +166,6 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {
@Column
declare extname: string
- @AllowNull(true)
- @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))
- @Column
- declare infoHash: string
-
@AllowNull(false)
@Default(-1)
@Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))
@@ -244,17 +235,13 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {
})
declare VideoStreamingPlaylist: Awaited<VideoStreamingPlaylistModel>
- static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist.bind(VideoFileModel), {
- promise: true,
- max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
- maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
+ @HasOne(() => VideoInfohashModel, {
+ foreignKey: {
+ allowNull: true
+ },
+ onDelete: 'CASCADE'
})
-
- static doesInfohashExist (infoHash: string) {
- const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'
-
- return doesExist({ sequelize: this.sequelize, query, bind: { infoHash } })
- }
+ declare InfoHash: Awaited<VideoInfohashModel>
static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {
const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)
@@ -467,7 +454,7 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {
}
hasTorrent () {
- return this.infoHash && this.torrentFilename
+ return !!this.torrentFilename
}
getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {
diff --git a/server/core/models/video/video-infohash.ts b/server/core/models/video/video-infohash.ts
new file mode 100644
index 0000000000..d9d12f0557
--- /dev/null
+++ b/server/core/models/video/video-infohash.ts
@@ -0,0 +1,118 @@
+import { logger } from '@server/helpers/logger.js'
+import { MInfohash } from '@server/types/models/video/video-infohash.js'
+import memoizee from 'memoizee'
+import { Op, Transaction } from 'sequelize'
+import { AllowNull, Column, DataType, ForeignKey, Table } from 'sequelize-typescript'
+import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '../../initializers/constants.js'
+import { SequelizeModel, doesExist } from '../shared/index.js'
+import { VideoFileModel } from './video-file.js'
+import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js'
+
+@Table({
+ tableName: 'videoInfohash',
+ timestamps: false,
+ indexes: [
+ {
+ fields: [ 'infohash' ]
+ },
+ {
+ fields: [ 'videoStreamingPlaylistId' ],
+ where: {
+ videoStreamingPlaylistId: {
+ [Op.ne]: null
+ }
+ }
+ },
+ {
+ fields: [ 'videoFileId' ],
+ unique: true,
+ where: {
+ videoFileId: {
+ [Op.ne]: null
+ }
+ }
+ }
+ ]
+})
+export class VideoInfohashModel extends SequelizeModel<VideoInfohashModel> {
+ @AllowNull(false)
+ @Column(DataType.BLOB)
+ declare infohash: Buffer
+
+ @ForeignKey(() => VideoStreamingPlaylistModel)
+ @AllowNull(true)
+ @Column
+ declare videoStreamingPlaylistId: number
+
+ @ForeignKey(() => VideoFileModel)
+ @AllowNull(true)
+ @Column
+ declare videoFileId: number
+
+ // The 20 ASCII characters p2p-media-loader announces for this row
+ toP2PMediaLoaderInfohash () {
+ return this.infohash.toString('binary')
+ }
+
+ // The 40 char hex classic BitTorrent clients announce for this row
+ toHexInfohash () {
+ return this.infohash.toString('hex')
+ }
+
+ static doesInfohashExistCached = memoizee(VideoInfohashModel.doesInfohashHexExist.bind(VideoInfohashModel), {
+ promise: true,
+ max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
+ maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
+ })
+
+ // infohashHex is the hex representation announced on the wire (40 chars for a 20 byte infohash)
+ static doesInfohashHexExist (infohashHex: string) {
+ const query = 'SELECT 1 FROM "videoInfohash" WHERE "infohash" = $infohash LIMIT 1'
+
+ return doesExist({ sequelize: this.sequelize, query, bind: { infohash: Buffer.from(infohashHex, 'hex') } })
+ }
+
+ // ---------------------------------------------------------------------------
+
+ // p2p-media-loader infohashes are 20 ASCII characters (base64), announced as those 20 bytes
+ static replacePlaylistInfohashes (playlistId: number, hashes: string[], transaction?: Transaction) {
+ logger.debug('Replacing infohash for streaming playlist %d', playlistId, { hashes })
+
+ return this.replaceInfohashes({
+ where: { videoStreamingPlaylistId: playlistId },
+ rows: hashes.map(h => ({ infohash: Buffer.from(h, 'binary'), videoStreamingPlaylistId: playlistId })),
+ transaction
+ })
+ }
+
+ // Classic BitTorrent infohash stored as its 40 char hex on the video file
+ static async replaceFileInfohash (fileId: number, infohashHex: string | null, transaction?: Transaction) {
+ logger.debug('Replacing infohash for file %d', fileId, { infohashHex })
+
+ const results = await this.replaceInfohashes({
+ where: { videoFileId: fileId },
+ rows: infohashHex
+ ? [ { infohash: Buffer.from(infohashHex, 'hex'), videoFileId: fileId } ]
+ : [],
+ transaction
+ })
+
+ if (results.length === 0) return undefined
+
+ return results[0]
+ }
+
+ private static async replaceInfohashes (options: {
+ where: { videoStreamingPlaylistId: number } | { videoFileId: number }
+ rows: { infohash: Buffer, videoStreamingPlaylistId?: number, videoFileId?: number }[]
+ transaction?: Transaction
+ }): Promise<MInfohash[]> {
+ const { where, rows, transaction } = options
+
+ await VideoInfohashModel.destroy({ where, transaction })
+
+ if (rows.length === 0) return []
+
+ return VideoInfohashModel.bulkCreate(rows, { transaction })
+ }
+}
diff --git a/server/core/models/video/video-streaming-playlist.ts b/server/core/models/video/video-streaming-playlist.ts
index b385364f0d..4d6e47a1aa 100644
--- a/server/core/models/video/video-streaming-playlist.ts
+++ b/server/core/models/video/video-streaming-playlist.ts
@@ -10,6 +10,7 @@ import {
import { generateP2PMediaLoaderHash } from '@peertube/peertube-node-utils'
import { logger } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
import {
buildObjectStorageHLSPrivateFileUrl,
buildObjectStoragePublicFileUrl,
@@ -27,22 +28,13 @@ import {
MVideoPrivacy,
MVideoUUID
} from '@server/types/models/index.js'
-import memoizee from 'memoizee'
import { join } from 'path'
import { Op, Transaction } from 'sequelize'
-import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, Is, Table, UpdatedAt } from 'sequelize-typescript'
-import { isArrayOf } from '../../helpers/custom-validators/misc.js'
-import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos.js'
-import {
- CONSTRAINTS_FIELDS,
- MEMOIZE_LENGTH,
- MEMOIZE_TTL,
- P2P_MEDIA_LOADER_PEER_VERSION,
- STATIC_PATHS,
- WEBSERVER
-} from '../../initializers/constants.js'
+import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, Table, UpdatedAt } from 'sequelize-typescript'
+import { CONSTRAINTS_FIELDS, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS, WEBSERVER } from '../../initializers/constants.js'
import { VideoRedundancyModel } from '../redundancy/video-redundancy.js'
-import { SequelizeModel, doesExist, throwIfNotValid } from '../shared/index.js'
+import { SequelizeModel, doesExist } from '../shared/index.js'
+import { VideoInfohashModel } from './video-infohash.js'
import { VideoModel } from './video.js'
@Table({
@@ -54,10 +46,6 @@ import { VideoModel } from './video.js'
{
fields: [ 'videoId', 'type' ],
unique: true
- },
- {
- fields: [ 'p2pMediaLoaderInfohashes' ],
- using: 'gin'
}
]
})
@@ -80,11 +68,6 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))
declare playlistUrl: string
- @AllowNull(false)
- @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes'))
- @Column(DataType.ARRAY(DataType.STRING))
- declare p2pMediaLoaderInfohashes: string[]
-
@AllowNull(false)
@Column
declare p2pMediaLoaderPeerVersion: number
@@ -131,18 +114,13 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
})
declare RedundancyVideos: Awaited<VideoRedundancyModel>[]
- static doesInfohashExistCached = memoizee(VideoStreamingPlaylistModel.doesInfohashExist.bind(VideoStreamingPlaylistModel), {
- promise: true,
- max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,
- maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS
+ @HasMany(() => VideoInfohashModel, {
+ foreignKey: {
+ allowNull: true
+ },
+ onDelete: 'CASCADE'
})
-
- static doesInfohashExist (infoHash: string) {
- // Don't add a LIMIT 1 here to prevent seq scan by PostgreSQL (not sure why id doesn't use the index when we add a LIMIT)
- const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE "p2pMediaLoaderInfohashes" @> $infoHash'
-
- return doesExist({ sequelize: this.sequelize, query, bind: { infoHash: `{${infoHash}}` } }) // Transform infoHash in a PG array
- }
+ declare InfoHashes: Awaited<VideoInfohashModel>[]
static buildP2PMediaLoaderInfoHashes (videoUUID: string, files: { resolution: number }[]) {
const hashes: string[] = []
@@ -278,7 +256,6 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,
type: VideoStreamingPlaylistType.HLS,
storage: FileStorage.FILE_SYSTEM,
- p2pMediaLoaderInfohashes: [],
playlistFilename: generateHLSMasterPlaylistFilename(video.isLive),
segmentsSha256Filename: generateHlsSha256SegmentsFilename(video.isLive),
videoId: video.id
@@ -299,8 +276,20 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl
return doesExist({ sequelize: this.sequelize, query, bind: { videoUUID, storage } })
}
- assignP2PMediaLoaderInfoHashes (video: MVideoUUID, files: { resolution: number }[]) {
- this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(video.uuid, files)
+ buildAndSetInfoHashes (video: MVideoUUID, files: { resolution: number }[], transaction?: Transaction) {
+ const hashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(video.uuid, files)
+
+ return this.setInfoHashes(hashes, transaction)
+ }
+
+ async setInfoHashes (hashes: string[], transaction?: Transaction) {
+ const replace = (t: Transaction) => VideoInfohashModel.replacePlaylistInfohashes(this.id, hashes, t)
+
+ // Keep the delete + insert atomic even when the caller has no transaction
+ if (transaction) await replace(transaction)
+ else await sequelizeTypescript.transaction(replace)
+
+ this.InfoHashes = hashes.map(h => new VideoInfohashModel({ infohash: Buffer.from(h, 'binary') }))
}
// ---------------------------------------------------------------------------
diff --git a/server/core/types/models/video/video-file.ts b/server/core/types/models/video/video-file.ts
index 9df420bebb..d22b81dc62 100644
--- a/server/core/types/models/video/video-file.ts
+++ b/server/core/types/models/video/video-file.ts
@@ -1,5 +1,6 @@
import { PickWith } from '@peertube/peertube-typescript-utils'
import { VideoFileModel } from '../../../models/video/video-file.js'
+import { MInfohash } from './video-infohash.js'
import { MStreamingPlaylist, MStreamingPlaylistVideo } from './video-streaming-playlist.js'
import { MVideo, MVideoUUID } from './video.js'
@@ -7,7 +8,11 @@ type Use<K extends keyof VideoFileModel, M> = PickWith<VideoFileModel, K, M>
// ############################################################################
-export type MVideoFile = Omit<VideoFileModel, 'Video' | 'VideoStreamingPlaylist'>
+export type MVideoFile = Omit<VideoFileModel, 'Video' | 'VideoStreamingPlaylist' | 'InfoHash'>
+
+export type MVideoFileInfoHash =
+ & MVideoFile
+ & Use<'InfoHash', MInfohash>
export type MVideoFileVideo =
& MVideoFile
diff --git a/server/core/types/models/video/video-infohash.ts b/server/core/types/models/video/video-infohash.ts
new file mode 100644
index 0000000000..c02387e646
--- /dev/null
+++ b/server/core/types/models/video/video-infohash.ts
@@ -0,0 +1,3 @@
+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'
+
+export type MInfohash = VideoInfohashModel
diff --git a/server/core/types/models/video/video-streaming-playlist.ts b/server/core/types/models/video/video-streaming-playlist.ts
index 340858877b..2b9a830fc8 100644
--- a/server/core/types/models/video/video-streaming-playlist.ts
+++ b/server/core/types/models/video/video-streaming-playlist.ts
@@ -1,14 +1,19 @@
import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils'
import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist.js'
-import { MVideo, MVideoUUID } from './video.js'
-import { MVideoFile } from './video-file.js'
+import { MVideoFile, MVideoFileInfoHash } from './video-file.js'
+import { MInfohash } from './video-infohash.js'
import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy.js'
+import { MVideo, MVideoUUID } from './video.js'
type Use<K extends keyof VideoStreamingPlaylistModel, M> = PickWith<VideoStreamingPlaylistModel, K, M>
// ############################################################################
-export type MStreamingPlaylist = Omit<VideoStreamingPlaylistModel, 'Video' | 'RedundancyVideos' | 'VideoFiles'>
+export type MStreamingPlaylist = Omit<VideoStreamingPlaylistModel, 'Video' | 'RedundancyVideos' | 'VideoFiles' | 'InfoHashes'>
+
+export type MStreamingPlaylistInfoHash =
+ & MStreamingPlaylist
+ & Use<'InfoHashes', MInfohash[]>
export type MStreamingPlaylistFiles =
& MStreamingPlaylist
@@ -37,9 +42,9 @@ export type MStreamingPlaylistRedundancies =
& Use<'VideoFiles', MVideoFile[]>
& Use<'RedundancyVideos', MVideoRedundancyFileUrl[]>
-export type MStreamingPlaylistRedundanciesOpt =
- & MStreamingPlaylist
- & Use<'VideoFiles', MVideoFile[]>
+export type MStreamingPlaylistFormattable =
+ & MStreamingPlaylistInfoHash
+ & Use<'VideoFiles', MVideoFileInfoHash[]>
& PickWithOpt<VideoStreamingPlaylistModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]>
export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo {
diff --git a/server/core/types/models/video/video.ts b/server/core/types/models/video/video.ts
index 34be96dbc2..45ebf4d106 100644
--- a/server/core/types/models/video/video.ts
+++ b/server/core/types/models/video/video.ts
@@ -19,15 +19,10 @@ import {
MChannelHostOnly,
MChannelSummary
} from './video-channel.js'
-import { MVideoFile } from './video-file.js'
+import { MVideoFile, MVideoFileInfoHash } from './video-file.js'
import { MVideoLiveWithSchedules } from './video-live.js'
import { MVideoSource } from './video-source.js'
-import {
- MStreamingPlaylistFiles,
- MStreamingPlaylistRedundancies,
- MStreamingPlaylistRedundanciesAll,
- MStreamingPlaylistRedundanciesOpt
-} from './video-streaming-playlist.js'
+import { MStreamingPlaylistFiles, MStreamingPlaylistFormattable, MStreamingPlaylistRedundanciesAll } from './video-streaming-playlist.js'
type Use<K extends keyof VideoModel, M> = PickWith<VideoModel, K, M>
@@ -153,7 +148,7 @@ export type MVideoWithAllFiles =
export type MVideoAccountLightBlacklistAllFiles =
& MVideo
- & Use<'VideoFiles', MVideoFile[]>
+ & Use<'VideoFiles', MVideoFileInfoHash[]>
& Use<'Thumbnails', MThumbnail[]>
& Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]>
& Use<'VideoChannel', MChannelAccountLight>
@@ -199,9 +194,9 @@ export type MVideoFull =
& Use<'Tags', MTag[]>
& Use<'VideoChannel', MChannelAccountLight>
& Use<'UserVideoHistories', MUserVideoHistoryTime[]>
- & Use<'VideoFiles', MVideoFile[]>
+ & Use<'VideoFiles', MVideoFileInfoHash[]>
& Use<'ScheduleVideoUpdate', MScheduleVideoUpdate>
- & Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]>
+ & Use<'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>
& Use<'VideoLive', MVideoLiveWithSchedules>
// ############################################################################
@@ -212,10 +207,10 @@ export type MVideoAP =
& MVideo
& Use<'Tags', MTag[]>
& Use<'VideoChannel', MChannelAccountLight>
- & Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]>
+ & Use<'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>
& Use<'VideoCaptions', MVideoCaptionLanguageUrl[]>
& Use<'VideoBlacklist', MVideoBlacklistUnfederated>
- & Use<'VideoFiles', MVideoFile[]>
+ & Use<'VideoFiles', MVideoFileInfoHash[]>
& Use<'Thumbnails', MThumbnail[]>
& Use<'VideoLive', MVideoLiveWithSchedules>
& Use<'Storyboard', MStoryboard>
@@ -230,8 +225,8 @@ export type MVideoDetails =
& Use<'ScheduleVideoUpdate', MScheduleVideoUpdate>
& Use<'Thumbnails', MThumbnail[]>
& Use<'UserVideoHistories', MUserVideoHistoryTime[]>
- & Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundancies[]>
- & Use<'VideoFiles', MVideoFile[]>
+ & Use<'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>
+ & Use<'VideoFiles', MVideoFileInfoHash[]>
& Use<'Trackers', MTrackerUrl[]>
export type MVideoForUser =
@@ -253,10 +248,12 @@ export type MVideoFormattable =
& MVideoThumbnails
& PickWithOpt<VideoModel, 'UserVideoHistories', MUserVideoHistoryTime[]>
& Use<'VideoChannel', MChannelAccountSummaryFormattable>
+
+export type MVideoFormattableAdditionalAttributes =
& PickWithOpt<VideoModel, 'ScheduleVideoUpdate', Pick<MScheduleVideoUpdate, 'updateAt' | 'privacy'>>
& PickWithOpt<VideoModel, 'VideoBlacklist', Pick<MVideoBlacklist, 'reason'>>
- & PickWithOpt<VideoModel, 'VideoStreamingPlaylists', MStreamingPlaylistFiles[]>
- & PickWithOpt<VideoModel, 'VideoFiles', MVideoFile[]>
+ & PickWithOpt<VideoModel, 'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>
+ & PickWithOpt<VideoModel, 'VideoFiles', MVideoFileInfoHash[]>
& PickWithOpt<VideoModel, 'VideoLive', MVideoLiveWithSchedules>
& PickWithOpt<VideoModel, 'VideoAutomaticTags', MVideoAutomaticTagWithTag[]>
& PickWithOpt<VideoModel, 'Tags', MTag[]>
@@ -266,6 +263,6 @@ export type MVideoFormattableDetails =
& MVideoFormattable
& Use<'VideoChannel', MChannelFormattable>
& Use<'Tags', MTag[]>
- & Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesOpt[]>
+ & Use<'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>
& Use<'VideoFiles', MVideoFile[]>
& PickWithOpt<VideoModel, 'Trackers', MTrackerUrl[]>
diff --git a/server/scripts/migrations/peertube-5.0.ts b/server/scripts/migrations/peertube-5.0.ts
index 7683848f5a..64505b133e 100644
--- a/server/scripts/migrations/peertube-5.0.ts
+++ b/server/scripts/migrations/peertube-5.0.ts
@@ -1,12 +1,12 @@
-import { ensureDir } from 'fs-extra/esm'
-import { Op } from 'sequelize'
-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
+import { VideoPrivacy } from '@peertube/peertube-models'
import { DIRECTORIES } from '@server/initializers/constants.js'
+import { initDatabaseModels } from '@server/initializers/database.js'
import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy.js'
+import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoFull } from '@server/types/models/index.js'
-import { VideoPrivacy } from '@peertube/peertube-models'
-import { initDatabaseModels } from '@server/initializers/database.js'
+import { ensureDir } from 'fs-extra/esm'
+import { Op } from 'sequelize'
run()
.then(() => process.exit(0))
@@ -57,14 +57,14 @@ async function run () {
async function updateTorrents (video: MVideoFull) {
for (const file of video.VideoFiles) {
- await updateTorrentMetadata(video, file)
+ await updateTorrentForFileAndSave(video, file)
await file.save()
}
const playlist = video.getHLSPlaylist()
for (const file of (playlist?.VideoFiles || [])) {
- await updateTorrentMetadata(playlist, file)
+ await updateTorrentForFileAndSave(playlist, file)
await file.save()
}
diff --git a/server/scripts/update-host.ts b/server/scripts/update-host.ts
index d4a33855d8..c1134cb9a4 100755
--- a/server/scripts/update-host.ts
+++ b/server/scripts/update-host.ts
@@ -1,5 +1,3 @@
-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
-import { getServerActor } from '@server/models/application/application.js'
import { WEBSERVER } from '@server/initializers/constants.js'
import { initDatabaseModels } from '@server/initializers/database.js'
import {
@@ -11,16 +9,18 @@ import {
getLocalVideoPlaylistActivityPubUrl,
getLocalVideoPlaylistElementActivityPubUrl
} from '@server/lib/activitypub/url.js'
+import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'
import { AccountModel } from '@server/models/account/account.js'
import { ActorFollowModel } from '@server/models/actor/actor-follow.js'
import { ActorModel } from '@server/models/actor/actor.js'
+import { getServerActor } from '@server/models/application/application.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoCommentModel } from '@server/models/video/video-comment.js'
+import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
+import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
import { VideoShareModel } from '@server/models/video/video-share.js'
import { VideoModel } from '@server/models/video/video.js'
import { MActorAccount } from '@server/types/models/index.js'
-import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
-import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
run()
.then(() => process.exit(0))
@@ -177,18 +177,14 @@ async function run () {
for (const file of video.VideoFiles) {
console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
- await updateTorrentMetadata(video, file)
-
- await file.save()
+ await updateTorrentForFileAndSave(video, file)
}
const playlist = video.getHLSPlaylist()
for (const file of (playlist?.VideoFiles || [])) {
console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid)
- await updateTorrentMetadata(playlist, file)
-
- await file.save()
+ await updateTorrentForFileAndSave(playlist, file)
}
}
}
diff --git a/server/server.ts b/server/server.ts
index edeecb4974..c0aef4c539 100644
--- a/server/server.ts
+++ b/server/server.ts
@@ -104,7 +104,6 @@ if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) {
}
// ----------- PeerTube modules -----------
-import { omit } from '@peertube/peertube-core-utils'
import { HttpStatusCode } from '@peertube/peertube-models'
import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics.js'
@@ -276,10 +275,10 @@ app.use((err, req, res: express.Response, _next) => {
? (process as any)._getActiveRequests()
: undefined
- // Remove too big metadata
- const sanitizedErr = omit(err, [ 'body' ])
+ // Remove too big metadata and alter original error to keep type
+ err.body = undefined
- logger.error('Error in controller.', { err: sanitizedErr, sql, activeRequests, url: req.originalUrl })
+ logger.error('Error in controller.', { err, sql, activeRequests, url: req.originalUrl })
return res.fail({
status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500,