ActivityPub-federated video streaming platform using P2P directly in your web browser
Move infohash in a dedicated table
Chocobozzza3107fab888c
diff --git a/packages/tests/src/shared/sql-command.ts b/packages/tests/src/shared/sql-command.tsindex de9d848ee3..e7166fe8fc 100644--- a/packages/tests/src/shared/sql-command.ts+++ b/packages/tests/src/shared/sql-command.ts@@ -171,12 +171,13 @@ export class SQLCommand {// ---------------------------------------------------------------------------async getPlaylistInfohash (playlistId: number) {- const query = 'SELECT "p2pMediaLoaderInfohashes" FROM "videoStreamingPlaylist" WHERE id = :playlistId'+ const query = `SELECT CONVERT_FROM("infohash", 'SQL_ASCII') AS "infohash" ` ++ `FROM "videoInfohash" WHERE "videoStreamingPlaylistId" = :playlistId`- const result = await this.selectQuery<{ p2pMediaLoaderInfohashes: string }>(query, { playlistId })+ const result = await this.selectQuery<{ infohash: string }>(query, { playlistId })if (!result || result.length === 0) return []- return result[0].p2pMediaLoaderInfohashes+ return result.map(r => r.infohash)}// ---------------------------------------------------------------------------diff --git a/packages/tests/src/shared/streaming-playlists.ts b/packages/tests/src/shared/streaming-playlists.tsindex 459c687bc6..4cb4dc598f 100644--- a/packages/tests/src/shared/streaming-playlists.ts+++ b/packages/tests/src/shared/streaming-playlists.ts@@ -110,7 +110,7 @@ export async function checkPlaylistInfohash (options: {expect(dbInfohashes).to.include(infohash)- await checkTrackerInfohash(video.account.host, infohash)+ await checkTrackerInfohash(video.account.host, Buffer.from(infohash, 'ascii').toString('hex'))}}diff --git a/packages/tests/src/shared/tracker.ts b/packages/tests/src/shared/tracker.tsindex 38bcc2754a..34f7b22fe9 100644--- a/packages/tests/src/shared/tracker.ts+++ b/packages/tests/src/shared/tracker.ts@@ -1,12 +1,12 @@import { makeGetRequest } from '@peertube/peertube-server-commands'import { expect } from 'chai'-export async function checkTrackerInfohash (serverUrl: string, infohash: string) {+export async function checkTrackerInfohash (serverUrl: string, infohashHex: string) {const path = '/tracker/announce'// From bittorrent-tracker// oxlint-disable-next-line @typescript-eslint/no-deprecated- const infohashBinary = escape(Buffer.from(infohash, 'hex').toString('binary')).replace(/[@*/+]/g, function (char) {+ const infohashBinary = escape(Buffer.from(infohashHex, 'hex').toString('binary')).replace(/[@*/+]/g, function (char) {return '%' + char.charCodeAt(0).toString(16).toUpperCase()})diff --git a/server/core/controllers/api/videos/source.ts b/server/core/controllers/api/videos/source.tsindex 732f91e52a..57784385d5 100644--- a/server/core/controllers/api/videos/source.ts+++ b/server/core/controllers/api/videos/source.ts@@ -14,7 +14,7 @@ import { buildNextVideoState } from '@server/lib/video-state.js'import { openapiOperationDoc } from '@server/middlewares/doc.js'import { VideoChannelActivityModel } from '@server/models/video/video-channel-activity.js'import { VideoModel } from '@server/models/video/video.js'-import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFull } from '@server/types/models/index.js'+import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFileInfoHash, MVideoFull } from '@server/types/models/index.js'import express from 'express'import { move } from 'fs-extra/esm'import { logger, loggerTagsFactory } from '../../../helpers/logger.js'@@ -93,7 +93,12 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.Rconst videoPhysicalFile = res.locals.updateVideoFileResumableconst user = res.locals.oauth.token.User- const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video', ffprobe: res.locals.ffprobe })+ const videoFile = await buildNewFile({+ path: videoPhysicalFile.path,+ mode: 'web-video',+ ffprobe: res.locals.ffprobe+ }) as MVideoFileInfoHash+const originalFilename = videoPhysicalFile.originalnameconst videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoFull.uuid)diff --git a/server/core/controllers/tracker.ts b/server/core/controllers/tracker.tsindex 47cf83bc47..7b7c00bdb2 100644--- a/server/core/controllers/tracker.ts+++ b/server/core/controllers/tracker.ts@@ -7,8 +7,7 @@ import { WebSocketServer } from 'ws'import { logger } from '../helpers/logger.js'import { CONFIG } from '../initializers/config.js'import { LRU_CACHE, TRACKER_RATE_LIMITS } from '../initializers/constants.js'-import { VideoFileModel } from '../models/video/video-file.js'-import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js'+import { VideoInfohashModel } from '../models/video/video-infohash.js'export const trackerRouter = express.Router()@@ -30,32 +29,26 @@ const trackerServer = new TrackerServer({return cb(new Error('Tracker is disabled on this instance.'))}- const infohash = Buffer.from(infohashHex, 'hex').toString('binary')-const ip = params.type === 'ws'? params.ip: params.httpReq.ip- const key = ip + '-' + infohash+ const key = ip + '-' + infohashHexpeersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1if (CONFIG.TRACKER.REJECT_TOO_MANY_ANNOUNCES && peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) {- return cb(new Error(`Too many requests (${peersIpInfoHash[key]} of ip ${ip} for torrent ${infohash}`))+ return cb(new Error(`Too many requests (${peersIpInfoHash[key]} of ip ${ip} for torrent ${infohashHex}`))}try {if (CONFIG.TRACKER.PRIVATE === false) return cb()- const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExistCached(infohash)- if (playlistExists === true) return cb()-- // Classic infohash (not p2p-media-loader custom one), stored as hex, use arg directly- const videoFileExists = await VideoFileModel.doesInfohashExistCached(infohashHex)- if (videoFileExists === true) return cb()+ const infohashExists = await VideoInfohashModel.doesInfohashExistCached(infohashHex)+ if (infohashExists === true) return cb()- cb(new Error(`Unknown infoHash ${infohash} requested by ip ${ip}`))+ cb(new Error(`Unknown infoHash ${infohashHex} requested by ip ${ip}`))// Close socket connection and block IP for a few timeif (params.type === 'ws') {diff --git a/server/core/initializers/database.ts b/server/core/initializers/database.tsindex bc8febbeab..b9ab687ccf 100644--- a/server/core/initializers/database.ts+++ b/server/core/initializers/database.ts@@ -74,6 +74,7 @@ import { VideoLiveModel } from '../models/video/video-live.js'import { VideoPlaylistElementModel } from '../models/video/video-playlist-element.js'import { VideoPlaylistModel } from '../models/video/video-playlist.js'import { VideoShareModel } from '../models/video/video-share.js'+import { VideoInfohashModel } from '../models/video/video-infohash.js'import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js'import { VideoTagModel } from '../models/video/video-tag.js'import { VideoModel } from '../models/video/video.js'@@ -189,6 +190,7 @@ export async function initDatabaseModels (silent: boolean) {UserNotificationModel,UserNotificationSettingModel,VideoStreamingPlaylistModel,+ VideoInfohashModel,VideoPlaylistModel,VideoPlaylistElementModel,LocalVideoViewerModel,diff --git a/server/core/initializers/migrations/1095-video-infohash.ts b/server/core/initializers/migrations/1095-video-infohash.tsnew file mode 100644index 0000000000..4576a406af--- /dev/null+++ b/server/core/initializers/migrations/1095-video-infohash.ts@@ -0,0 +1,72 @@+import * as Sequelize from 'sequelize'++async function up (utils: {+ transaction: Sequelize.Transaction+ queryInterface: Sequelize.QueryInterface+ sequelize: Sequelize.Sequelize+}): Promise<void> {+ const { transaction } = utils++ {+ const query = `+CREATE TABLE IF NOT EXISTS "videoInfohash" (+ "id" SERIAL,+ "infohash" BYTEA NOT NULL,+ "videoStreamingPlaylistId" INTEGER REFERENCES "videoStreamingPlaylist" ("id") ON DELETE CASCADE ON UPDATE CASCADE,+ "videoFileId" INTEGER REFERENCES "videoFile" ("id") ON DELETE CASCADE ON UPDATE CASCADE,+ PRIMARY KEY ("id")+);`++ await utils.sequelize.query(query, { transaction })+ }++ // Don't re-hydrate video streaming playlists infohash, PeerTube will re-generate them on the next startup++ // Classic infohash are stored as 40 char hex on the video file+ {+ await utils.sequelize.query(+ `CREATE OR REPLACE FUNCTION safe_decode_hex(input text)+RETURNS bytea AS $$+BEGIN+ RETURN decode(input, 'hex');+EXCEPTION+ WHEN others THEN+ RETURN NULL;+END;+$$ LANGUAGE plpgsql IMMUTABLE;`,+ { transaction }+ )++ await utils.sequelize.query(+ `INSERT INTO "videoInfohash" ("infohash", "videoFileId") ` ++ `SELECT safe_decode_hex("infoHash"), "id" FROM "videoFile" WHERE "infoHash" IS NOT NULL AND safe_decode_hex("infoHash") IS NOT NULL`,+ { transaction }+ )++ await utils.sequelize.query(`DROP FUNCTION safe_decode_hex(text);`, { transaction })+ }++ // Dropping a column also drops indexes that only depend on it (the old GIN index, and "videoFile"."infoHash")+ {+ await utils.sequelize.query(+ `ALTER TABLE "videoStreamingPlaylist" DROP COLUMN "p2pMediaLoaderInfohashes"`,+ { transaction }+ )+ }++ {+ await utils.sequelize.query(+ `ALTER TABLE "videoFile" DROP COLUMN "infoHash"`,+ { transaction }+ )+ }+}++function down () {+ throw new Error('Not implemented.')+}++export {+ down,+ up+}diff --git a/server/core/lib/activitypub/videos/shared/abstract-builder.ts b/server/core/lib/activitypub/videos/shared/abstract-builder.tsindex 5421942bda..aebcac6f32 100644--- a/server/core/lib/activitypub/videos/shared/abstract-builder.ts+++ b/server/core/lib/activitypub/videos/shared/abstract-builder.ts@@ -13,12 +13,14 @@ import { getServerAccount } from '@server/models/application/application.js'import { StoryboardModel } from '@server/models/video/storyboard.js'import { VideoCaptionModel } from '@server/models/video/video-caption.js'import { VideoFileModel } from '@server/models/video/video-file.js'+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'import { VideoLiveScheduleModel } from '@server/models/video/video-live-schedule.js'import { VideoLiveModel } from '@server/models/video/video-live.js'import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'import {MStreamingPlaylistFiles,MStreamingPlaylistFilesVideo,+ MStreamingPlaylistFormattable,MVideo,MVideoCaption,MVideoFile,@@ -140,14 +142,10 @@ export abstract class APVideoAbstractBuilder {protected async setWebVideoFiles (video: MVideoFull, t: Transaction) {const oldFiles = video.VideoFiles || []- const newVideoFiles = getFileAttributesFromUrl(video, this.videoObject.url, oldFiles).map(a => new VideoFileModel(a))+ const toCreate = getFileAttributesFromUrl(video, this.videoObject.url, oldFiles)+ .map(({ file, infoHash }) => ({ file: new VideoFileModel(file), infoHash }))- // Remove video files that do not exist anymore- await deleteAllModels(filterNonExistingModels(oldFiles, newVideoFiles), t)-- // Update or add other one- const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t))- video.VideoFiles = await Promise.all(upsertTasks)+ video.VideoFiles = await this.saveFiles({ toCreate, oldFiles, t, mode: 'video' })}protected async updateChapters (video: MVideoFull) {@@ -182,20 +180,27 @@ export abstract class APVideoAbstractBuilder {}protected async setStreamingPlaylists (video: MVideoFull, t: Transaction) {- const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject)- const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))+ const toCreate = getStreamingPlaylistAttributesFromObject(video, this.videoObject)// Remove video playlists that do not exist anymore- await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t)+ await deleteAllModels(+ filterNonExistingModels(+ video.VideoStreamingPlaylists || [],+ toCreate.map(({ playlist }) => new VideoStreamingPlaylistModel(playlist))+ ),+ t+ )const oldPlaylists = video.VideoStreamingPlaylistsvideo.VideoStreamingPlaylists = []- for (const playlistAttributes of streamingPlaylistAttributes) {- const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t)+ for (const { playlist, tags, infoHashes } of toCreate) {+ const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlist, t)streamingPlaylistModel.Video = video- await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, playlistAttributes.tagAPObject, t)+ await streamingPlaylistModel.setInfoHashes(infoHashes ?? [], t)++ await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, tags ?? [], t)video.VideoStreamingPlaylists.push(streamingPlaylistModel)}@@ -204,7 +209,7 @@ export abstract class APVideoAbstractBuilder {private async insertOrReplaceStreamingPlaylist (attributes: CreationAttributes<VideoStreamingPlaylistModel>, t: Transaction) {const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t })- return streamingPlaylist as MStreamingPlaylistFilesVideo+ return streamingPlaylist as MStreamingPlaylistFilesVideo & MStreamingPlaylistFormattable}private getStreamingPlaylistFiles (oldPlaylists: MStreamingPlaylistFiles[], type: VideoStreamingPlaylistType_Type) {@@ -222,17 +227,13 @@ export abstract class APVideoAbstractBuilder {) {const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(oldPlaylists || [], playlistModel.type)- const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(+ const toCreate = getFileAttributesFromUrl(playlistModel,tagObjects,oldStreamingPlaylistFiles- ).map(a => new VideoFileModel(a))-- await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t)+ ).map(({ file, infoHash }) => ({ file: new VideoFileModel(file), infoHash }))- // Update or add other one- const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t))- playlistModel.VideoFiles = await Promise.all(upsertTasks)+ playlistModel.VideoFiles = await this.saveFiles({ toCreate, oldFiles: oldStreamingPlaylistFiles, t, mode: 'streaming-playlist' })}protected async setAutomaticTags (options: {@@ -253,4 +254,27 @@ export abstract class APVideoAbstractBuilder {return automaticTagsByAccount}++ protected async saveFiles (options: {+ toCreate: { file: MVideoFile, infoHash: string }[]+ oldFiles: MVideoFile[]+ t: Transaction+ mode: 'video' | 'streaming-playlist'+ }) {+ const { toCreate, oldFiles, t, mode } = options++ // Remove video files that do not exist anymore+ await deleteAllModels(filterNonExistingModels(oldFiles, toCreate.map(({ file }) => file)), t)++ // Update or add other one+ const upsertTasks = toCreate.map(async ({ file, infoHash }) => {+ const newFile = await VideoFileModel.customUpsert(file, mode, t)++ newFile.InfoHash = await VideoInfohashModel.replaceFileInfohash(newFile.id, infoHash, t)++ return newFile+ })++ return Promise.all(upsertTasks)+ }}diff --git a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.tsindex bde81627c4..0ee448ce94 100644--- a/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts+++ b/server/core/lib/activitypub/videos/shared/object-to-model-attributes.ts@@ -61,7 +61,8 @@ export function getFileAttributesFromUrl (const fileUrlObjects = urls.filter(u => isAPVideoUrlObject(u))if (fileUrlObjects.length === 0) return []- const attributes: FilteredModelAttributes<VideoFileModel>[] = []+ const result: { file: FilteredModelAttributes<VideoFileModel>, infoHash: string }[] = []+for (const fileUrlObject of fileUrlObjects) {// Fetch associated metadata url, if anyconst metadata = urls.filter(isAPVideoFileUrlMetadataObject)@@ -100,7 +101,6 @@ export function getFileAttributesFromUrl (filename: basename(fileUrl),fileUrl,- infoHash,torrentFilename,torrentUrl,@@ -112,10 +112,10 @@ export function getFileAttributesFromUrl (videoStreamingPlaylistId}- attributes.push(attribute)+ result.push({ file: attribute, infoHash })}- return attributes+ return result}function buildFileFormatFlags (fileUrl: ActivityVideoUrlObject, isStreamingPlaylist: boolean) {@@ -165,13 +165,16 @@ export function getStreamingPlaylistAttributesFromObject (video: MVideoId & MVidconst playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u))if (playlistUrls.length === 0) return []- const attributes: (FilteredModelAttributes<VideoStreamingPlaylistModel> & { tagAPObject?: ActivityTagObject[] })[] = []+ const result: {+ playlist: FilteredModelAttributes<VideoStreamingPlaylistModel>+ tags?: ActivityTagObject[]+ infoHashes: string[]+ }[] = []+for (const playlistUrlObject of playlistUrls) {const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject)- const files = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u))-- const attribute = {+ const playlist = {type: VideoStreamingPlaylistType.HLS,playlistFilename: basename(playlistUrlObject.href),@@ -182,21 +185,26 @@ export function getStreamingPlaylistAttributesFromObject (video: MVideoId & MVid: null,segmentsSha256Url: segmentsSha256UrlObject?.href ?? null,-- p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(- video.uuid,- files.map(f => ({ resolution: Math.min(f.height ?? Infinity, f.width ?? Infinity) }))- ),p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,- videoId: video.id,-- tagAPObject: playlistUrlObject.tag+ videoId: video.id}- attributes.push(attribute)+ result.push({+ playlist,++ tags: playlistUrlObject.tag,++ // Persisted into the dedicated infohash table once the playlist is upserted (see setStreamingPlaylists)+ infoHashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(+ video.uuid,+ playlistUrlObject.tag+ .filter(u => isAPVideoUrlObject(u))+ .map(f => ({ resolution: Math.min(f.height ?? Infinity, f.width ?? Infinity) }))+ )+ })}- return attributes+ return result}export function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) {diff --git a/server/core/lib/hls.ts b/server/core/lib/hls.tsindex cf2f566c2c..14e38755c8 100644--- a/server/core/lib/hls.ts+++ b/server/core/lib/hls.ts@@ -49,7 +49,7 @@ export async function updateStreamingPlaylistsInfohashesIfNeeded () {const playlist = await VideoStreamingPlaylistModel.loadWithVideo(playlistId, t)const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlistId, t)- playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles)+ await playlist.buildAndSetInfoHashes(playlist.Video, videoFiles, t)playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSIONawait playlist.save({ transaction: t })@@ -67,7 +67,7 @@ export async function updateM3U8AndShaPlaylist (video: MVideo, playlist: MStream// Refresh playlist, operations can take some timeplaylistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id)- playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles)+ await playlistWithFiles.buildAndSetInfoHashes(video, playlistWithFiles.VideoFiles)await playlistWithFiles.save()video.setHLSPlaylist(playlistWithFiles)diff --git a/server/core/lib/job-queue/handlers/manage-video-torrent.ts b/server/core/lib/job-queue/handlers/manage-video-torrent.tsindex e07468c919..465e4f8411 100644--- a/server/core/lib/job-queue/handlers/manage-video-torrent.ts+++ b/server/core/lib/job-queue/handlers/manage-video-torrent.ts@@ -1,11 +1,12 @@-import { Job } from 'bullmq'+import { ManageVideoTorrentPayload } from '@peertube/peertube-models'import { extractVideo } from '@server/helpers/video.js'-import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/lib/webtorrent.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'-import { VideoModel } from '@server/models/video/video.js'+import { createTorrentForFile, updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'import { VideoFileModel } from '@server/models/video/video-file.js'+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'-import { ManageVideoTorrentPayload } from '@peertube/peertube-models'+import { VideoModel } from '@server/models/video/video.js'+import { Job } from 'bullmq'import { logger } from '../../../helpers/logger.js'async function processManageVideoTorrent (job: Job) {@@ -38,17 +39,17 @@ async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'cawait video.reload()await file.reload()- await createTorrentAndSetInfoHash(video, file)+ const { infoHash, torrentFilename } = await createTorrentForFile(video, file)// Refresh videoFile because the createTorrentAndSetInfoHash could be longconst refreshedFile = await VideoFileModel.loadWithVideo(file.id)// File does not exist anymore, remove the generated torrentif (!refreshedFile) return file.removeTorrent()- refreshedFile.infoHash = file.infoHash- refreshedFile.torrentFilename = file.torrentFilename-+ refreshedFile.torrentFilename = torrentFilenameawait refreshedFile.save()++ await VideoInfohashModel.replaceFileInfohash(refreshedFile.id, infoHash)} finally {fileMutexReleaser()}@@ -67,9 +68,7 @@ async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { acconst fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid)try {- await updateTorrentMetadata(video || streamingPlaylist, file)-- await file.save()+ await updateTorrentForFileAndSave(video || streamingPlaylist, file)} finally {fileMutexReleaser()}diff --git a/server/core/lib/job-queue/handlers/video-file-import.ts b/server/core/lib/job-queue/handlers/video-file-import.tsindex 104d5ec2c4..b2506170ae 100644--- a/server/core/lib/job-queue/handlers/video-file-import.ts+++ b/server/core/lib/job-queue/handlers/video-file-import.ts@@ -5,7 +5,8 @@ import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'import { buildNewFile } from '@server/lib/video-file.js'import { buildMoveVideoJob } from '@server/lib/video-jobs.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'-import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'+import { createTorrentForFile } from '@server/lib/webtorrent.js'+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'import { VideoModel } from '@server/models/video/video.js'import { MVideoFull } from '@server/types/models/index.js'import { Job } from 'bullmq'@@ -70,8 +71,12 @@ async function updateVideoFile (video: MVideoFull, inputFilePath: string) {const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)await copy(inputFilePath, outputPath)- video.VideoFiles.push(newVideoFile)- await createTorrentAndSetInfoHash(video, newVideoFile)+ const { infoHash, torrentFilename } = await createTorrentForFile(video, newVideoFile)+ newVideoFile.torrentFilename = torrentFilename++ const infohashModel = await VideoInfohashModel.replaceFileInfohash(newVideoFile.id, infoHash)await newVideoFile.save()++ video.VideoFiles.push(Object.assign(newVideoFile, { InfoHash: infohashModel }))}diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.tsindex 947adeefce..b416f974a5 100644--- a/server/core/lib/job-queue/handlers/video-import.ts+++ b/server/core/lib/job-queue/handlers/video-import.ts@@ -24,9 +24,10 @@ import { buildNewFile } from '@server/lib/video-file.js'import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'import { buildNextVideoState } from '@server/lib/video-state.js'-import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '@server/lib/webtorrent.js'+import { createTorrentForFile, downloadWebTorrentVideo } from '@server/lib/webtorrent.js'import { UserModel } from '@server/models/user/user.js'import { VideoCaptionModel } from '@server/models/video/video-caption.js'+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'import { MUserId, MVideoFile, MVideoFull } from '@server/types/models/index.js'import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'import { Job } from 'bullmq'@@ -215,17 +216,23 @@ async function processFile (options: {const thumbnails = await generateThumbnails({ videoImportWithFiles, videoFile, ffprobe })- // Create torrent- await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)+ const { infoHash, torrentFilename } = await createTorrentForFile(videoImportWithFiles.Video, videoFile)const { videoImportUpdated, video } = await retryTransactionWrapper(() => {return sequelizeTypescript.transaction(async t => {// Refresh videoconst video = await VideoModel.load(videoImportWithFiles.videoId, t)- if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')+ if (!video) {+ await videoFile.removeTorrent()+ throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')+ }++ videoFile.torrentFilename = torrentFilenameawait videoFile.save({ transaction: t })+ await VideoInfohashModel.replaceFileInfohash(videoFile.id, infoHash, t)+// Update video DB objectvideo.duration = durationvideo.state = buildNextVideoState(video.state)diff --git a/server/core/lib/live/shared/muxing-session.ts b/server/core/lib/live/shared/muxing-session.tsindex 0c0a5f7745..feed6ecbb8 100644--- a/server/core/lib/live/shared/muxing-session.ts+++ b/server/core/lib/live/shared/muxing-session.ts@@ -258,7 +258,10 @@ class MuxingSession extends EventEmitter implements MuxingSession {hlsStreams.push(VideoResolution.H_NOVIDEO)}- this.streamingPlaylist.assignP2PMediaLoaderInfoHashes(this.videoLive.Video, Array.from(hlsStreams).map(r => ({ resolution: r })))+ await this.streamingPlaylist.buildAndSetInfoHashes(+ this.videoLive.Video,+ Array.from(hlsStreams).map(r => ({ resolution: r }))+ )await this.streamingPlaylist.save()} catch (err) {@@ -338,7 +341,6 @@ class MuxingSession extends EventEmitter implements MuxingSession {fps,size: -1,extname: '.ts',- infoHash: null,formatFlags: VideoFileFormatFlag.NONE,streams: resolution === VideoResolution.H_NOVIDEO? VideoFileStream.AUDIOdiff --git a/server/core/lib/local-video-creator.ts b/server/core/lib/local-video-creator.tsindex c7cd2901cb..bb5611676c 100644--- a/server/core/lib/local-video-creator.ts+++ b/server/core/lib/local-video-creator.ts@@ -25,7 +25,7 @@ import { VideoLiveScheduleModel } from '@server/models/video/video-live-scheduleimport { VideoLiveModel } from '@server/models/video/video-live.js'import { VideoPasswordModel } from '@server/models/video/video-password.js'import { VideoModel } from '@server/models/video/video.js'-import { MChannel, MChannelAccountLight, MUserAccountId, MVideoFile, MVideoFull } from '@server/types/models/index.js'+import { MChannel, MChannelAccountLight, MUserAccountId, MVideoFileInfoHash, MVideoFull } from '@server/types/models/index.js'import { FilteredModelAttributes } from '@server/types/sequelize.js'import { FfprobeData } from 'fluent-ffmpeg'import { move } from 'fs-extra/esm'@@ -93,7 +93,7 @@ export class LocalVideoCreator {private readonly videoAttributeResultHook: VideoAttributeHookFilterprivate video: MVideoFull- private videoFile: MVideoFile+ private videoFile: MVideoFileInfoHashprivate videoPath: stringconstructor (@@ -142,7 +142,11 @@ export class LocalVideoCreator {this.video.url = getLocalVideoActivityPubUrl(this.video)if (this.videoFilePath) {- this.videoFile = await buildNewFile({ path: this.videoFilePath, mode: 'web-video', ffprobe: this.videoFileProbe })+ this.videoFile = await buildNewFile({+ path: this.videoFilePath,+ mode: 'web-video',+ ffprobe: this.videoFileProbe+ }) as MVideoFileInfoHashthis.videoPath = VideoPathManager.Instance.getFSVideoFileOutputPath(this.video, this.videoFile)await move(this.videoFilePath, this.videoPath)diff --git a/server/core/lib/local-video-updater.ts b/server/core/lib/local-video-updater.tsindex de5689bf72..281de57a00 100644--- a/server/core/lib/local-video-updater.ts+++ b/server/core/lib/local-video-updater.ts@@ -331,7 +331,7 @@ export class LocalVideoUpdater {if (filePathChanged && hls) {logger.debug('Updating HLS playlist file paths after privacy change', this.lTags())- hls.assignP2PMediaLoaderInfoHashes(video, hls.VideoFiles)+ await hls.buildAndSetInfoHashes(video, hls.VideoFiles)await hls.save()}}diff --git a/server/core/lib/move-storage/move-to-file-system.ts b/server/core/lib/move-storage/move-to-file-system.tsindex 2a802c9ff6..fc06e06ac7 100644--- a/server/core/lib/move-storage/move-to-file-system.ts+++ b/server/core/lib/move-storage/move-to-file-system.ts@@ -21,7 +21,7 @@ import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/pimport { updateHLSMasterOnCaptionChange, upsertCaptionPlaylistOnFS } from '@server/lib/video-captions.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'+import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'import { VideoModel } from '@server/models/video/video.js'import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'import { MVideoSource } from '@server/types/models/video/video-source.js'@@ -169,7 +169,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {}if (updatedFile === true) {- playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)+ await playlist.buildAndSetInfoHashes(video, playlist.VideoFiles)playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSIONawait playlist.save()@@ -191,8 +191,7 @@ async function onVideoFileMoved (options: {file.storage = FileStorage.FILE_SYSTEM- await updateTorrentMetadata(videoOrPlaylist, file)- await file.save()+ await updateTorrentForFileAndSave(videoOrPlaylist, file)logger.debug('Removing web video file %s because it\'s now on file system', oldFileUrl, lTagsBase())await objetStorageRemover()diff --git a/server/core/lib/move-storage/move-to-object-storage.ts b/server/core/lib/move-storage/move-to-object-storage.tsindex 78bf0e93c0..62ca4b71bc 100644--- a/server/core/lib/move-storage/move-to-object-storage.ts+++ b/server/core/lib/move-storage/move-to-object-storage.ts@@ -13,7 +13,7 @@ import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/pimport { updateHLSMasterOnCaptionChange } from '@server/lib/video-captions.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'+import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'import { VideoCaptionModel } from '@server/models/video/video-caption.js'import { VideoModel } from '@server/models/video/video.js'import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'@@ -199,7 +199,7 @@ async function moveHLSFiles (video: MVideoWithAllFiles) {}if (updatedFile === true) {- playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)+ await playlist.buildAndSetInfoHashes(video, playlist.VideoFiles)playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSIONawait playlist.save()@@ -222,8 +222,7 @@ async function onVideoFileMoved (options: {file.storage = FileStorage.OBJECT_STORAGE- await updateTorrentMetadata(videoOrPlaylist, file)- await file.save()+ await updateTorrentForFileAndSave(videoOrPlaylist, file)logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())await remove(oldPath)diff --git a/server/core/lib/transcoding/hls-transcoding.ts b/server/core/lib/transcoding/hls-transcoding.tsindex 6f3a50be75..b0738b1977 100644--- a/server/core/lib/transcoding/hls-transcoding.ts+++ b/server/core/lib/transcoding/hls-transcoding.ts@@ -3,8 +3,9 @@ import { canCopyForHLS, getVideoStreamDuration, HLSFromTSTranscodeOptions, HLSTrimport { retryTransactionWrapper } from '@server/helpers/database-utils.js'import { deleteFileAndCatch } from '@server/helpers/fs.js'import { sequelizeTypescript } from '@server/initializers/database.js'-import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'-import { MVideo } from '@server/types/models/index.js'+import { createTorrentForFile } from '@server/lib/webtorrent.js'+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'+import { MVideo, MVideoFile } from '@server/types/models/index.js'import { MutexInterface } from 'async-mutex'import { Job } from 'bullmq'import { ensureDir, move } from 'fs-extra/esm'@@ -116,7 +117,8 @@ export async function onHLSVideoFileTranscoding (options: {await video.save()}- await createTorrentAndSetInfoHash(playlist, newVideoFile)+ const { infoHash, torrentFilename } = await createTorrentForFile(playlist, newVideoFile)+ newVideoFile.torrentFilename = torrentFilenameconst oldFile = await VideoFileModel.loadHLSFile({playlistId: playlist.id,@@ -129,7 +131,13 @@ export async function onHLSVideoFileTranscoding (options: {await oldFile.destroy()}- const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', undefined)+ const savedVideoFile = await sequelizeTypescript.transaction(async t => {+ const savedVideoFile = await VideoFileModel.customUpsert(newVideoFile, 'streaming-playlist', t) as MVideoFile++ await VideoInfohashModel.replaceFileInfohash(savedVideoFile.id, infoHash, t)++ return savedVideoFile+ })if (playlistGenerated) {await createAllCaptionPlaylistsOnFSIfNeeded(video)diff --git a/server/core/lib/transcoding/web-transcoding.ts b/server/core/lib/transcoding/web-transcoding.tsindex 3524742401..485237ca43 100644--- a/server/core/lib/transcoding/web-transcoding.ts+++ b/server/core/lib/transcoding/web-transcoding.ts@@ -8,7 +8,9 @@ import {import { VideoFileStream } from '@peertube/peertube-models'import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js'import { deleteFileAndCatch } from '@server/helpers/fs.js'-import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'+import { sequelizeTypescript } from '@server/initializers/database.js'+import { createTorrentForFile } from '@server/lib/webtorrent.js'+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'import { VideoModel } from '@server/models/video/video.js'import { MVideoFile, MVideoFull } from '@server/types/models/index.js'import { Job } from 'bullmq'@@ -225,7 +227,8 @@ export async function onWebVideoFileTranscoding (options: {await move(videoOutputPath, outputPath, { overwrite: true })- await createTorrentAndSetInfoHash(video, videoFile)+ const { infoHash, torrentFilename } = await createTorrentForFile(video, videoFile)+ videoFile.torrentFilename = torrentFilenameif (deleteWebInputVideoFile) {await saveNewOriginalFileIfNeeded(video, deleteWebInputVideoFile)@@ -237,14 +240,18 @@ export async function onWebVideoFileTranscoding (options: {const existingFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution })if (existingFile) await video.removeWebVideoFile(existingFile)- await VideoFileModel.customUpsert(videoFile, 'video', undefined)+ await sequelizeTypescript.transaction(async t => {+ await VideoFileModel.customUpsert(videoFile, 'video', t)+ await VideoInfohashModel.replaceFileInfohash(videoFile.id, infoHash, t)+ })+video.VideoFiles = await video.$get('VideoFiles')if (wasAudioFile) {await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: false })}- return { video, videoFile }+ return { videoFile }} finally {mutexReleaser()}diff --git a/server/core/lib/video-studio.ts b/server/core/lib/video-studio.tsindex d1dabe60c3..d6c6a7c88f 100644--- a/server/core/lib/video-studio.ts+++ b/server/core/lib/video-studio.ts@@ -3,7 +3,9 @@ import { getVideoStreamDuration } from '@peertube/peertube-ffmpeg'import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@peertube/peertube-models'import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'-import { createTorrentAndSetInfoHashFromPath } from '@server/lib/webtorrent.js'+import { sequelizeTypescript } from '@server/initializers/database.js'+import { createTorrentForFileFromPath } from '@server/lib/webtorrent.js'+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'import { VideoModel } from '@server/models/video/video.js'import { MUser, MVideoFile, MVideoFull, MVideoWithAllFiles, MVideoWithFile } from '@server/types/models/index.js'import { move, remove } from 'fs-extra/esm'@@ -107,10 +109,15 @@ export async function onVideoStudioEnded (options: {await safeCleanupStudioTMPFiles(tasks)- await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath)+ const { infoHash, torrentFilename } = await createTorrentForFileFromPath(video, newFile, outputPath)await removeAllFiles(video, newFile)- await newFile.save()+ await sequelizeTypescript.transaction(async t => {+ newFile.torrentFilename = torrentFilename+ await newFile.save({ transaction: t })++ await VideoInfohashModel.replaceFileInfohash(newFile.id, infoHash, t)+ })video.duration = await getVideoStreamDuration(outputPath)video.aspectRatio = buildAspectRatio({ width: newFile.width, height: newFile.height })diff --git a/server/core/lib/webtorrent.ts b/server/core/lib/webtorrent.tsindex c018ef3869..67dfac584a 100644--- a/server/core/lib/webtorrent.ts+++ b/server/core/lib/webtorrent.ts@@ -3,7 +3,8 @@ import { WEBSERVER } from '@server/initializers/constants.js'import { generateTorrentFileName } from '@server/lib/paths.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'import { createTorrentFromWorker } from '@server/lib/worker/parent-process.js'-import { MVideoFile } from '@server/types/models/video/video-file.js'+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'+import { MVideoFile, MVideoFileInfoHash } from '@server/types/models/video/video-file.js'import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist.js'import { MVideo } from '@server/types/models/video/video.js'import bencode from 'bencode'@@ -111,13 +112,13 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentPat})}-export function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {+export function createTorrentForFile (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), videoPath => {- return createTorrentAndSetInfoHashFromPath(videoOrPlaylist, videoFile, videoPath)+ return createTorrentForFileFromPath(videoOrPlaylist, videoFile, videoPath)})}-export async function createTorrentAndSetInfoHashFromPath (+export async function createTorrentForFileFromPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo,videoFile: MVideoFile,filePath: string@@ -147,11 +148,14 @@ export async function createTorrentAndSetInfoHashFromPath (// FIXME: typings: parseTorrent now returns an async resultconst parsedTorrent = await (parseTorrent(torrentContent) as unknown as Promise<parseTorrent.Instance>)- videoFile.infoHash = parsedTorrent.infoHash- videoFile.torrentFilename = torrentFilename++ return {+ infoHash: parsedTorrent.infoHash,+ torrentFilename: torrentFilename+ }}-export async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {+export async function updateTorrentForFileAndSave (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) {const video = extractVideo(videoOrPlaylist)if (!videoFile.torrentFilename) {@@ -186,12 +190,15 @@ export async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreaminawait remove(oldTorrentPath)videoFile.torrentFilename = newTorrentFilename- videoFile.infoHash = sha1(bencode.encode(decoded.info))++ await VideoInfohashModel.replaceFileInfohash(videoFile.id, sha1(bencode.encode(decoded.info)))++ await videoFile.save()}export function generateMagnetUri (video: MVideo,- videoFile: MVideoFile,+ videoFile: MVideoFileInfoHash,trackerUrls: string[]) {const xs = videoFile.getTorrentUrl()@@ -205,7 +212,7 @@ export function generateMagnetUri (xs,announce,urlList,- infoHash: videoFile.infoHash,+ infoHash: videoFile.InfoHash.toHexInfohash(),name: video.name}diff --git a/server/core/models/video/formatter/video-activity-pub-format.ts b/server/core/models/video/formatter/video-activity-pub-format.tsindex d9d90b89be..7375383381 100644--- a/server/core/models/video/formatter/video-activity-pub-format.ts+++ b/server/core/models/video/formatter/video-activity-pub-format.ts@@ -26,7 +26,7 @@ import {getLocalVideoPlayerSettingsActivityPubUrl,getLocalVideoSharesActivityPubUrl} from '../../../lib/activitypub/url.js'-import { MStreamingPlaylistFiles, MUserId, MVideo, MVideoAP, MVideoFile } from '../../../types/models/index.js'+import { MStreamingPlaylistFormattable, MUserId, MVideo, MVideoAP, MVideoFileInfoHash } from '../../../types/models/index.js'import { sortByResolutionDesc } from './shared/index.js'import { getCategoryLabel, getLanguageLabel, getLicenceLabel } from './video-api-format.js'@@ -203,7 +203,7 @@ function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] {function buildVideoFileUrls (options: {video: MVideo- files: MVideoFile[]+ files: MVideoFileInfoHash[]user?: MUserId}): ActivityUrlObject[] {const { video, files } = options@@ -269,9 +269,9 @@ function buildStreamingPlaylistUrls (video: MVideoAP): ActivityPlaylistUrlObject}))}-function buildStreamingPlaylistTags (video: MVideoAP, playlist: MStreamingPlaylistFiles) {+function buildStreamingPlaylistTags (video: MVideoAP, playlist: MStreamingPlaylistFormattable) {return [- ...playlist.p2pMediaLoaderInfohashes.map(i => ({ type: 'Infohash' as 'Infohash', name: i })),+ ...(playlist.InfoHashes ?? []).map(i => ({ type: 'Infohash' as 'Infohash', name: i.toP2PMediaLoaderInfohash() })),{type: 'Link',diff --git a/server/core/models/video/formatter/video-api-format.ts b/server/core/models/video/formatter/video-api-format.tsindex 980dffb751..dd08126681 100644--- a/server/core/models/video/formatter/video-api-format.ts+++ b/server/core/models/video/formatter/video-api-format.ts@@ -24,8 +24,14 @@ import {VIDEO_PRIVACIES,VIDEO_STATES} from '../../../initializers/constants.js'-import { MServer, MStreamingPlaylistRedundanciesOpt, MVideoFormattable, MVideoFormattableDetails } from '../../../types/models/index.js'-import { MVideoFile } from '../../../types/models/video/video-file.js'+import {+ MServer,+ MStreamingPlaylistFormattable,+ MVideoFormattable,+ MVideoFormattableAdditionalAttributes,+ MVideoFormattableDetails+} from '../../../types/models/index.js'+import { MVideoFileInfoHash } from '../../../types/models/video/video-file.js'import { sortByResolutionDesc } from './shared/index.js'export type VideoFormattingJSONOptions = {@@ -208,7 +214,7 @@ export function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetailexport function streamingPlaylistsModelToFormattedJSON (video: MVideoFormattable,- playlists: MStreamingPlaylistRedundanciesOpt[]+ playlists: MStreamingPlaylistFormattable[]): VideoStreamingPlaylist[] {if (isArray(playlists) === false) return []@@ -232,7 +238,7 @@ export function streamingPlaylistsModelToFormattedJSON (export function videoFilesModelToFormattedJSON (video: MVideoFormattable,- videoFiles: MVideoFile[],+ videoFiles: MVideoFileInfoHash[],options?: {includePlaylistUrl?: trueincludeMagnet?: boolean@@ -241,7 +247,7 @@ export function videoFilesModelToFormattedJSON (export function videoFilesModelToFormattedJSON (video: MVideoFormattable,- videoFiles: MVideoFile[],+ videoFiles: MVideoFileInfoHash[],options: {includePlaylistUrl?: boolean // default falseincludeMagnet?: boolean // default true@@ -332,7 +338,7 @@ export function getStateLabel (id: number) {// Private// ----------------------------------------------------------------------------function buildAdditionalAttributes (video: MVideoFormattable, options: VideoFormattingJSONOptions) {+function buildAdditionalAttributes (video: MVideoFormattable & MVideoFormattableAdditionalAttributes, options: VideoFormattingJSONOptions) {const add = options.additionalAttributesconst result: Partial<VideoAdditionalAttributes> = {}diff --git a/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts b/server/core/models/video/sql/video/shared/abstract-video-query-builder.tsindex 8fe4c46310..10dc1fe160 100644--- a/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts+++ b/server/core/models/video/sql/video/shared/abstract-video-query-builder.ts@@ -4,8 +4,8 @@ import { Sequelize } from 'sequelize'import validator from 'validator'import { AbstractRunQuery } from '../../../../shared/abstract-run-query.js'import { createSafeIn } from '../../../../shared/index.js'-import { VideoTableAttributes } from './video-table-attributes.js'import { TableAttributeOptions } from './table-attributes-options.model.js'+import { VideoTableAttributes } from './video-table-attributes.js'/*** Abstract builder to create SQL query and fetch video models@@ -124,6 +124,8 @@ export class AbstractVideoQueryBuilder extends AbstractRunQuery {...this.buildAttributesObject('VideoFiles', this.tables.getFileAttributes())}++ this.includeFileInfohashJSONJoin('VideoFiles')}protected includeStreamingPlaylistFiles () {@@ -142,6 +144,42 @@ export class AbstractVideoQueryBuilder extends AbstractRunQuery {...this.buildAttributesObject('VideoStreamingPlaylists', this.tables.getStreamingPlaylistAttributes()),...this.buildAttributesObject('VideoStreamingPlaylists->VideoFiles', this.tables.getFileAttributes())}++ this.includeFileInfohashJSONJoin('VideoStreamingPlaylists->VideoFiles')+ this.includePlaylistInfohashesJSON()+ }++ private includePlaylistInfohashesJSON () {+ this.addJoin(+ `LEFT JOIN LATERAL (` ++ `SELECT json_agg(ENCODE("infohash", 'hex')) AS "infohashes" ` ++ `FROM "videoInfohash" WHERE "videoStreamingPlaylistId" = "VideoStreamingPlaylists"."id"` ++ `) AS "VideoStreamingPlaylists->InfohashesJSON" ON TRUE`+ )++ this.attributes = {+ ...this.attributes,++ '"VideoStreamingPlaylists->InfohashesJSON"."infohashes"': '"VideoStreamingPlaylists.InfohashesJSON"'+ }+ }++ private includeFileInfohashJSONJoin (prefix: string) {+ const alias = `${prefix}->InfohashJSON`+ const flatPrefix = prefix.replace(/->/g, '.')++ this.addJoin(+ `LEFT JOIN LATERAL (` ++ `SELECT json_agg(ENCODE("infohash", 'hex')) AS "infoHash" ` ++ `FROM "videoInfohash" WHERE "videoFileId" = "${prefix}"."id"` ++ `) AS "${alias}" ON TRUE`+ )++ this.attributes = {+ ...this.attributes,++ [`"${alias}"."infoHash"`]: `"${flatPrefix}.InfohashJSON"`+ }}protected includeUserHistory (userId: number) {diff --git a/server/core/models/video/sql/video/shared/video-model-builder.ts b/server/core/models/video/sql/video/shared/video-model-builder.tsindex 321d44971a..1042f40baa 100644--- a/server/core/models/video/sql/video/shared/video-model-builder.ts+++ b/server/core/models/video/sql/video/shared/video-model-builder.ts@@ -1,12 +1,12 @@import { VideoInclude, VideoIncludeType } from '@peertube/peertube-models'-import { AccountBlocklistModel } from '@server/models/blocklist/account-blocklist.js'import { AccountModel } from '@server/models/account/account.js'import { ActorImageModel } from '@server/models/actor/actor-image.js'import { ActorModel } from '@server/models/actor/actor.js'import { AutomaticTagModel } from '@server/models/automatic-tag/automatic-tag.js'import { VideoAutomaticTagModel } from '@server/models/automatic-tag/video-automatic-tag.js'-import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy.js'+import { AccountBlocklistModel } from '@server/models/blocklist/account-blocklist.js'import { ServerBlocklistModel } from '@server/models/blocklist/server-blocklist.js'+import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy.js'import { ServerModel } from '@server/models/server/server.js'import { TrackerModel } from '@server/models/server/tracker.js'import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js'@@ -19,11 +19,12 @@ import { ThumbnailModel } from '../../../thumbnail.js'import { VideoBlacklistModel } from '../../../video-blacklist.js'import { VideoChannelModel } from '../../../video-channel.js'import { VideoFileModel } from '../../../video-file.js'+import { VideoInfohashModel } from '../../../video-infohash.js'import { VideoLiveModel } from '../../../video-live.js'import { VideoStreamingPlaylistModel } from '../../../video-streaming-playlist.js'import { VideoModel } from '../../../video.js'-import { VideoTableAttributes } from './video-table-attributes.js'import { TableAttributeOptions } from './table-attributes-options.model.js'+import { VideoTableAttributes } from './video-table-attributes.js'type SQLRow = { [id: string]: string | number }@@ -327,6 +328,7 @@ export class VideoModelBuilder {const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoFiles')const videoFileModel = new VideoFileModel(attributes, this.buildOpts)+ this.addFileInfohash(row, 'VideoFiles', videoFileModel)videoModel.VideoFiles.push(videoFileModel)this.videoFileMemo[id] = videoFileModel@@ -340,6 +342,8 @@ export class VideoModelBuilder {const streamingPlaylist = new VideoStreamingPlaylistModel(attributes, this.buildOpts)streamingPlaylist.VideoFiles = []+ this.addPlaylistInfohashes(row, streamingPlaylist)+videoModel.VideoStreamingPlaylists.push(streamingPlaylist)this.videoStreamingPlaylistMemo[id] = streamingPlaylist@@ -353,11 +357,28 @@ export class VideoModelBuilder {const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoStreamingPlaylists.VideoFiles')const videoFileModel = new VideoFileModel(attributes, this.buildOpts)++ this.addFileInfohash(row, 'VideoStreamingPlaylists.VideoFiles', videoFileModel)+streamingPlaylist.VideoFiles.push(videoFileModel)this.videoFileMemo[id] = videoFileModel}+ private addPlaylistInfohashes (row: SQLRow, streamingPlaylist: VideoStreamingPlaylistModel) {+ const hashes = row['VideoStreamingPlaylists.InfohashesJSON'] as any as string[] || []++ streamingPlaylist.InfoHashes = hashes.map(h => new VideoInfohashModel({ infohash: Buffer.from(h, 'hex') }, this.buildOpts))+ }++ private addFileInfohash (row: SQLRow, prefixKey: string, videoFile: VideoFileModel) {+ const hashes = row[`${prefixKey}.InfohashJSON`] as any as string[] || []++ videoFile.InfoHash = hashes.length !== 0+ ? new VideoInfohashModel({ infohash: Buffer.from(hashes[0], 'hex') }, this.buildOpts)+ : null+ }+private addRedundancy (row: SQLRow, prefix: string, to: VideoStreamingPlaylistModel) {if (!to.RedundancyVideos) to.RedundancyVideos = []diff --git a/server/core/models/video/sql/video/shared/video-table-attributes.ts b/server/core/models/video/sql/video/shared/video-table-attributes.tsindex 93c7449cfb..6fc2d95087 100644--- a/server/core/models/video/sql/video/shared/video-table-attributes.ts+++ b/server/core/models/video/sql/video/shared/video-table-attributes.ts@@ -81,7 +81,6 @@ export class VideoTableAttributes {'fileUrl','torrentFilename','torrentUrl',- 'infoHash','fps','metadataUrl','videoStreamingPlaylistId',@@ -100,7 +99,6 @@ export class VideoTableAttributes {'playlistUrl','playlistFilename','type',- 'p2pMediaLoaderInfohashes','p2pMediaLoaderPeerVersion','segmentsSha256Filename','segmentsSha256Url',diff --git a/server/core/models/video/video-file.ts b/server/core/models/video/video-file.tsindex 5810616a58..bd5a479d77 100644--- a/server/core/models/video/video-file.ts+++ b/server/core/models/video/video-file.ts@@ -24,7 +24,6 @@ import { getVideoFileMimeType } from '@server/lib/video-file.js'import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js'import { MStreamingPlaylistVideo, MVideo, MVideoWithHost, isStreamingPlaylist } from '@server/types/models/index.js'import { remove } from 'fs-extra/esm'-import memoizee from 'memoizee'import { join } from 'path'import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize'import {@@ -36,6 +35,7 @@ import {Default,DefaultScope,ForeignKey,+ HasOne,Is,Scopes,Table,@@ -45,13 +45,13 @@ import validator from 'validator'import {isVideoFPSResolutionValid,isVideoFileExtnameValid,- isVideoFileInfoHashValid,isVideoFileResolutionValid,isVideoFileSizeValid} from '../../helpers/custom-validators/videos.js'-import { DOWNLOAD_PATHS, LAZY_STATIC_PATHS, MEMOIZE_LENGTH, MEMOIZE_TTL, STATIC_PATHS, WEBSERVER } from '../../initializers/constants.js'+import { DOWNLOAD_PATHS, LAZY_STATIC_PATHS, STATIC_PATHS, WEBSERVER } from '../../initializers/constants.js'import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file.js'import { SequelizeModel, doesExist, parseAggregateResult, throwIfNotValid } from '../shared/index.js'+import { VideoInfohashModel } from './video-infohash.js'import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js'import { VideoModel } from './video.js'@@ -106,10 +106,6 @@ export enum ScopeNames {@Table({tableName: 'videoFile',indexes: [- {- fields: [ 'infoHash' ]- },-{fields: [ 'torrentFilename' ],unique: true@@ -170,11 +166,6 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {@Columndeclare extname: string- @AllowNull(true)- @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true))- @Column- declare infoHash: string-@AllowNull(false)@Default(-1)@Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps'))@@ -244,17 +235,13 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {})declare VideoStreamingPlaylist: Awaited<VideoStreamingPlaylistModel>- static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist.bind(VideoFileModel), {- promise: true,- max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,- maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS+ @HasOne(() => VideoInfohashModel, {+ foreignKey: {+ allowNull: true+ },+ onDelete: 'CASCADE'})-- static doesInfohashExist (infoHash: string) {- const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1'-- return doesExist({ sequelize: this.sequelize, query, bind: { infoHash } })- }+ declare InfoHash: Awaited<VideoInfohashModel>static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) {const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID)@@ -467,7 +454,7 @@ export class VideoFileModel extends SequelizeModel<VideoFileModel> {}hasTorrent () {- return this.infoHash && this.torrentFilename+ return !!this.torrentFilename}getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo {diff --git a/server/core/models/video/video-infohash.ts b/server/core/models/video/video-infohash.tsnew file mode 100644index 0000000000..d9d12f0557--- /dev/null+++ b/server/core/models/video/video-infohash.ts@@ -0,0 +1,118 @@+import { logger } from '@server/helpers/logger.js'+import { MInfohash } from '@server/types/models/video/video-infohash.js'+import memoizee from 'memoizee'+import { Op, Transaction } from 'sequelize'+import { AllowNull, Column, DataType, ForeignKey, Table } from 'sequelize-typescript'+import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '../../initializers/constants.js'+import { SequelizeModel, doesExist } from '../shared/index.js'+import { VideoFileModel } from './video-file.js'+import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js'++@Table({+ tableName: 'videoInfohash',+ timestamps: false,+ indexes: [+ {+ fields: [ 'infohash' ]+ },+ {+ fields: [ 'videoStreamingPlaylistId' ],+ where: {+ videoStreamingPlaylistId: {+ [Op.ne]: null+ }+ }+ },+ {+ fields: [ 'videoFileId' ],+ unique: true,+ where: {+ videoFileId: {+ [Op.ne]: null+ }+ }+ }+ ]+})+export class VideoInfohashModel extends SequelizeModel<VideoInfohashModel> {+ @AllowNull(false)+ @Column(DataType.BLOB)+ declare infohash: Buffer++ @ForeignKey(() => VideoStreamingPlaylistModel)+ @AllowNull(true)+ @Column+ declare videoStreamingPlaylistId: number++ @ForeignKey(() => VideoFileModel)+ @AllowNull(true)+ @Column+ declare videoFileId: number++ // The 20 ASCII characters p2p-media-loader announces for this row+ toP2PMediaLoaderInfohash () {+ return this.infohash.toString('binary')+ }++ // The 40 char hex classic BitTorrent clients announce for this row+ toHexInfohash () {+ return this.infohash.toString('hex')+ }++ static doesInfohashExistCached = memoizee(VideoInfohashModel.doesInfohashHexExist.bind(VideoInfohashModel), {+ promise: true,+ max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,+ maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS+ })++ // infohashHex is the hex representation announced on the wire (40 chars for a 20 byte infohash)+ static doesInfohashHexExist (infohashHex: string) {+ const query = 'SELECT 1 FROM "videoInfohash" WHERE "infohash" = $infohash LIMIT 1'++ return doesExist({ sequelize: this.sequelize, query, bind: { infohash: Buffer.from(infohashHex, 'hex') } })+ }++ // ---------------------------------------------------------------------------++ // p2p-media-loader infohashes are 20 ASCII characters (base64), announced as those 20 bytes+ static replacePlaylistInfohashes (playlistId: number, hashes: string[], transaction?: Transaction) {+ logger.debug('Replacing infohash for streaming playlist %d', playlistId, { hashes })++ return this.replaceInfohashes({+ where: { videoStreamingPlaylistId: playlistId },+ rows: hashes.map(h => ({ infohash: Buffer.from(h, 'binary'), videoStreamingPlaylistId: playlistId })),+ transaction+ })+ }++ // Classic BitTorrent infohash stored as its 40 char hex on the video file+ static async replaceFileInfohash (fileId: number, infohashHex: string | null, transaction?: Transaction) {+ logger.debug('Replacing infohash for file %d', fileId, { infohashHex })++ const results = await this.replaceInfohashes({+ where: { videoFileId: fileId },+ rows: infohashHex+ ? [ { infohash: Buffer.from(infohashHex, 'hex'), videoFileId: fileId } ]+ : [],+ transaction+ })++ if (results.length === 0) return undefined++ return results[0]+ }++ private static async replaceInfohashes (options: {+ where: { videoStreamingPlaylistId: number } | { videoFileId: number }+ rows: { infohash: Buffer, videoStreamingPlaylistId?: number, videoFileId?: number }[]+ transaction?: Transaction+ }): Promise<MInfohash[]> {+ const { where, rows, transaction } = options++ await VideoInfohashModel.destroy({ where, transaction })++ if (rows.length === 0) return []++ return VideoInfohashModel.bulkCreate(rows, { transaction })+ }+}diff --git a/server/core/models/video/video-streaming-playlist.ts b/server/core/models/video/video-streaming-playlist.tsindex b385364f0d..4d6e47a1aa 100644--- a/server/core/models/video/video-streaming-playlist.ts+++ b/server/core/models/video/video-streaming-playlist.ts@@ -10,6 +10,7 @@ import {import { generateP2PMediaLoaderHash } from '@peertube/peertube-node-utils'import { logger } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'+import { sequelizeTypescript } from '@server/initializers/database.js'import {buildObjectStorageHLSPrivateFileUrl,buildObjectStoragePublicFileUrl,@@ -27,22 +28,13 @@ import {MVideoPrivacy,MVideoUUID} from '@server/types/models/index.js'-import memoizee from 'memoizee'import { join } from 'path'import { Op, Transaction } from 'sequelize'-import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, Is, Table, UpdatedAt } from 'sequelize-typescript'-import { isArrayOf } from '../../helpers/custom-validators/misc.js'-import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos.js'-import {- CONSTRAINTS_FIELDS,- MEMOIZE_LENGTH,- MEMOIZE_TTL,- P2P_MEDIA_LOADER_PEER_VERSION,- STATIC_PATHS,- WEBSERVER-} from '../../initializers/constants.js'+import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, Table, UpdatedAt } from 'sequelize-typescript'+import { CONSTRAINTS_FIELDS, P2P_MEDIA_LOADER_PEER_VERSION, STATIC_PATHS, WEBSERVER } from '../../initializers/constants.js'import { VideoRedundancyModel } from '../redundancy/video-redundancy.js'-import { SequelizeModel, doesExist, throwIfNotValid } from '../shared/index.js'+import { SequelizeModel, doesExist } from '../shared/index.js'+import { VideoInfohashModel } from './video-infohash.js'import { VideoModel } from './video.js'@Table({@@ -54,10 +46,6 @@ import { VideoModel } from './video.js'{fields: [ 'videoId', 'type' ],unique: true- },- {- fields: [ 'p2pMediaLoaderInfohashes' ],- using: 'gin'}]})@@ -80,11 +68,6 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl@Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max))declare playlistUrl: string- @AllowNull(false)- @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes'))- @Column(DataType.ARRAY(DataType.STRING))- declare p2pMediaLoaderInfohashes: string[]-@AllowNull(false)@Columndeclare p2pMediaLoaderPeerVersion: number@@ -131,18 +114,13 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPl})declare RedundancyVideos: Awaited<VideoRedundancyModel>[]- static doesInfohashExistCached = memoizee(VideoStreamingPlaylistModel.doesInfohashExist.bind(VideoStreamingPlaylistModel), {- promise: true,- max: MEMOIZE_LENGTH.INFO_HASH_EXISTS,- maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS+ @HasMany(() => VideoInfohashModel, {+ foreignKey: {+ allowNull: true+ },+ onDelete: 'CASCADE'})-- static doesInfohashExist (infoHash: string) {- // Don't add a LIMIT 1 here to prevent seq scan by PostgreSQL (not sure why id doesn't use the index when we add a LIMIT)- const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE "p2pMediaLoaderInfohashes" @> $infoHash'-- return doesExist({ sequelize: this.sequelize, query, bind: { infoHash: `{${infoHash}}` } }) // Transform infoHash in a PG array- }+ declare InfoHashes: Awaited<VideoInfohashModel>[]static buildP2PMediaLoaderInfoHashes (videoUUID: string, files: { resolution: number }[]) {const hashes: string[] = []@@ -278,7 +256,6 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPlp2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION,type: VideoStreamingPlaylistType.HLS,storage: FileStorage.FILE_SYSTEM,- p2pMediaLoaderInfohashes: [],playlistFilename: generateHLSMasterPlaylistFilename(video.isLive),segmentsSha256Filename: generateHlsSha256SegmentsFilename(video.isLive),videoId: video.id@@ -299,8 +276,20 @@ export class VideoStreamingPlaylistModel extends SequelizeModel<VideoStreamingPlreturn doesExist({ sequelize: this.sequelize, query, bind: { videoUUID, storage } })}- assignP2PMediaLoaderInfoHashes (video: MVideoUUID, files: { resolution: number }[]) {- this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(video.uuid, files)+ buildAndSetInfoHashes (video: MVideoUUID, files: { resolution: number }[], transaction?: Transaction) {+ const hashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(video.uuid, files)++ return this.setInfoHashes(hashes, transaction)+ }++ async setInfoHashes (hashes: string[], transaction?: Transaction) {+ const replace = (t: Transaction) => VideoInfohashModel.replacePlaylistInfohashes(this.id, hashes, t)++ // Keep the delete + insert atomic even when the caller has no transaction+ if (transaction) await replace(transaction)+ else await sequelizeTypescript.transaction(replace)++ this.InfoHashes = hashes.map(h => new VideoInfohashModel({ infohash: Buffer.from(h, 'binary') }))}// ---------------------------------------------------------------------------diff --git a/server/core/types/models/video/video-file.ts b/server/core/types/models/video/video-file.tsindex 9df420bebb..d22b81dc62 100644--- a/server/core/types/models/video/video-file.ts+++ b/server/core/types/models/video/video-file.ts@@ -1,5 +1,6 @@import { PickWith } from '@peertube/peertube-typescript-utils'import { VideoFileModel } from '../../../models/video/video-file.js'+import { MInfohash } from './video-infohash.js'import { MStreamingPlaylist, MStreamingPlaylistVideo } from './video-streaming-playlist.js'import { MVideo, MVideoUUID } from './video.js'@@ -7,7 +8,11 @@ type Use<K extends keyof VideoFileModel, M> = PickWith<VideoFileModel, K, M>// ############################################################################-export type MVideoFile = Omit<VideoFileModel, 'Video' | 'VideoStreamingPlaylist'>+export type MVideoFile = Omit<VideoFileModel, 'Video' | 'VideoStreamingPlaylist' | 'InfoHash'>++export type MVideoFileInfoHash =+ & MVideoFile+ & Use<'InfoHash', MInfohash>export type MVideoFileVideo =& MVideoFilediff --git a/server/core/types/models/video/video-infohash.ts b/server/core/types/models/video/video-infohash.tsnew file mode 100644index 0000000000..c02387e646--- /dev/null+++ b/server/core/types/models/video/video-infohash.ts@@ -0,0 +1,3 @@+import { VideoInfohashModel } from '@server/models/video/video-infohash.js'++export type MInfohash = VideoInfohashModeldiff --git a/server/core/types/models/video/video-streaming-playlist.ts b/server/core/types/models/video/video-streaming-playlist.tsindex 340858877b..2b9a830fc8 100644--- a/server/core/types/models/video/video-streaming-playlist.ts+++ b/server/core/types/models/video/video-streaming-playlist.ts@@ -1,14 +1,19 @@import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils'import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist.js'-import { MVideo, MVideoUUID } from './video.js'-import { MVideoFile } from './video-file.js'+import { MVideoFile, MVideoFileInfoHash } from './video-file.js'+import { MInfohash } from './video-infohash.js'import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy.js'+import { MVideo, MVideoUUID } from './video.js'type Use<K extends keyof VideoStreamingPlaylistModel, M> = PickWith<VideoStreamingPlaylistModel, K, M>// ############################################################################-export type MStreamingPlaylist = Omit<VideoStreamingPlaylistModel, 'Video' | 'RedundancyVideos' | 'VideoFiles'>+export type MStreamingPlaylist = Omit<VideoStreamingPlaylistModel, 'Video' | 'RedundancyVideos' | 'VideoFiles' | 'InfoHashes'>++export type MStreamingPlaylistInfoHash =+ & MStreamingPlaylist+ & Use<'InfoHashes', MInfohash[]>export type MStreamingPlaylistFiles =& MStreamingPlaylist@@ -37,9 +42,9 @@ export type MStreamingPlaylistRedundancies =& Use<'VideoFiles', MVideoFile[]>& Use<'RedundancyVideos', MVideoRedundancyFileUrl[]>-export type MStreamingPlaylistRedundanciesOpt =- & MStreamingPlaylist- & Use<'VideoFiles', MVideoFile[]>+export type MStreamingPlaylistFormattable =+ & MStreamingPlaylistInfoHash+ & Use<'VideoFiles', MVideoFileInfoHash[]>& PickWithOpt<VideoStreamingPlaylistModel, 'RedundancyVideos', MVideoRedundancyFileUrl[]>export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo {diff --git a/server/core/types/models/video/video.ts b/server/core/types/models/video/video.tsindex 34be96dbc2..45ebf4d106 100644--- a/server/core/types/models/video/video.ts+++ b/server/core/types/models/video/video.ts@@ -19,15 +19,10 @@ import {MChannelHostOnly,MChannelSummary} from './video-channel.js'-import { MVideoFile } from './video-file.js'+import { MVideoFile, MVideoFileInfoHash } from './video-file.js'import { MVideoLiveWithSchedules } from './video-live.js'import { MVideoSource } from './video-source.js'-import {- MStreamingPlaylistFiles,- MStreamingPlaylistRedundancies,- MStreamingPlaylistRedundanciesAll,- MStreamingPlaylistRedundanciesOpt-} from './video-streaming-playlist.js'+import { MStreamingPlaylistFiles, MStreamingPlaylistFormattable, MStreamingPlaylistRedundanciesAll } from './video-streaming-playlist.js'type Use<K extends keyof VideoModel, M> = PickWith<VideoModel, K, M>@@ -153,7 +148,7 @@ export type MVideoWithAllFiles =export type MVideoAccountLightBlacklistAllFiles =& MVideo- & Use<'VideoFiles', MVideoFile[]>+ & Use<'VideoFiles', MVideoFileInfoHash[]>& Use<'Thumbnails', MThumbnail[]>& Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]>& Use<'VideoChannel', MChannelAccountLight>@@ -199,9 +194,9 @@ export type MVideoFull =& Use<'Tags', MTag[]>& Use<'VideoChannel', MChannelAccountLight>& Use<'UserVideoHistories', MUserVideoHistoryTime[]>- & Use<'VideoFiles', MVideoFile[]>+ & Use<'VideoFiles', MVideoFileInfoHash[]>& Use<'ScheduleVideoUpdate', MScheduleVideoUpdate>- & Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]>+ & Use<'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>& Use<'VideoLive', MVideoLiveWithSchedules>// ############################################################################@@ -212,10 +207,10 @@ export type MVideoAP =& MVideo& Use<'Tags', MTag[]>& Use<'VideoChannel', MChannelAccountLight>- & Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]>+ & Use<'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>& Use<'VideoCaptions', MVideoCaptionLanguageUrl[]>& Use<'VideoBlacklist', MVideoBlacklistUnfederated>- & Use<'VideoFiles', MVideoFile[]>+ & Use<'VideoFiles', MVideoFileInfoHash[]>& Use<'Thumbnails', MThumbnail[]>& Use<'VideoLive', MVideoLiveWithSchedules>& Use<'Storyboard', MStoryboard>@@ -230,8 +225,8 @@ export type MVideoDetails =& Use<'ScheduleVideoUpdate', MScheduleVideoUpdate>& Use<'Thumbnails', MThumbnail[]>& Use<'UserVideoHistories', MUserVideoHistoryTime[]>- & Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundancies[]>- & Use<'VideoFiles', MVideoFile[]>+ & Use<'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>+ & Use<'VideoFiles', MVideoFileInfoHash[]>& Use<'Trackers', MTrackerUrl[]>export type MVideoForUser =@@ -253,10 +248,12 @@ export type MVideoFormattable =& MVideoThumbnails& PickWithOpt<VideoModel, 'UserVideoHistories', MUserVideoHistoryTime[]>& Use<'VideoChannel', MChannelAccountSummaryFormattable>++export type MVideoFormattableAdditionalAttributes =& PickWithOpt<VideoModel, 'ScheduleVideoUpdate', Pick<MScheduleVideoUpdate, 'updateAt' | 'privacy'>>& PickWithOpt<VideoModel, 'VideoBlacklist', Pick<MVideoBlacklist, 'reason'>>- & PickWithOpt<VideoModel, 'VideoStreamingPlaylists', MStreamingPlaylistFiles[]>- & PickWithOpt<VideoModel, 'VideoFiles', MVideoFile[]>+ & PickWithOpt<VideoModel, 'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>+ & PickWithOpt<VideoModel, 'VideoFiles', MVideoFileInfoHash[]>& PickWithOpt<VideoModel, 'VideoLive', MVideoLiveWithSchedules>& PickWithOpt<VideoModel, 'VideoAutomaticTags', MVideoAutomaticTagWithTag[]>& PickWithOpt<VideoModel, 'Tags', MTag[]>@@ -266,6 +263,6 @@ export type MVideoFormattableDetails =& MVideoFormattable& Use<'VideoChannel', MChannelFormattable>& Use<'Tags', MTag[]>- & Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesOpt[]>+ & Use<'VideoStreamingPlaylists', MStreamingPlaylistFormattable[]>& Use<'VideoFiles', MVideoFile[]>& PickWithOpt<VideoModel, 'Trackers', MTrackerUrl[]>diff --git a/server/scripts/migrations/peertube-5.0.ts b/server/scripts/migrations/peertube-5.0.tsindex 7683848f5a..64505b133e 100644--- a/server/scripts/migrations/peertube-5.0.ts+++ b/server/scripts/migrations/peertube-5.0.ts@@ -1,12 +1,12 @@-import { ensureDir } from 'fs-extra/esm'-import { Op } from 'sequelize'-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'+import { VideoPrivacy } from '@peertube/peertube-models'import { DIRECTORIES } from '@server/initializers/constants.js'+import { initDatabaseModels } from '@server/initializers/database.js'import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy.js'+import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'import { VideoModel } from '@server/models/video/video.js'import { MVideoFull } from '@server/types/models/index.js'-import { VideoPrivacy } from '@peertube/peertube-models'-import { initDatabaseModels } from '@server/initializers/database.js'+import { ensureDir } from 'fs-extra/esm'+import { Op } from 'sequelize'run().then(() => process.exit(0))@@ -57,14 +57,14 @@ async function run () {async function updateTorrents (video: MVideoFull) {for (const file of video.VideoFiles) {- await updateTorrentMetadata(video, file)+ await updateTorrentForFileAndSave(video, file)await file.save()}const playlist = video.getHLSPlaylist()for (const file of (playlist?.VideoFiles || [])) {- await updateTorrentMetadata(playlist, file)+ await updateTorrentForFileAndSave(playlist, file)await file.save()}diff --git a/server/scripts/update-host.ts b/server/scripts/update-host.tsindex d4a33855d8..c1134cb9a4 100755--- a/server/scripts/update-host.ts+++ b/server/scripts/update-host.ts@@ -1,5 +1,3 @@-import { updateTorrentMetadata } from '@server/lib/webtorrent.js'-import { getServerActor } from '@server/models/application/application.js'import { WEBSERVER } from '@server/initializers/constants.js'import { initDatabaseModels } from '@server/initializers/database.js'import {@@ -11,16 +9,18 @@ import {getLocalVideoPlaylistActivityPubUrl,getLocalVideoPlaylistElementActivityPubUrl} from '@server/lib/activitypub/url.js'+import { updateTorrentForFileAndSave } from '@server/lib/webtorrent.js'import { AccountModel } from '@server/models/account/account.js'import { ActorFollowModel } from '@server/models/actor/actor-follow.js'import { ActorModel } from '@server/models/actor/actor.js'+import { getServerActor } from '@server/models/application/application.js'import { VideoChannelModel } from '@server/models/video/video-channel.js'import { VideoCommentModel } from '@server/models/video/video-comment.js'+import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'+import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'import { VideoShareModel } from '@server/models/video/video-share.js'import { VideoModel } from '@server/models/video/video.js'import { MActorAccount } from '@server/types/models/index.js'-import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'-import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'run().then(() => process.exit(0))@@ -177,18 +177,14 @@ async function run () {for (const file of video.VideoFiles) {console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)- await updateTorrentMetadata(video, file)-- await file.save()+ await updateTorrentForFileAndSave(video, file)}const playlist = video.getHLSPlaylist()for (const file of (playlist?.VideoFiles || [])) {console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid)- await updateTorrentMetadata(playlist, file)-- await file.save()+ await updateTorrentForFileAndSave(playlist, file)}}}diff --git a/server/server.ts b/server/server.tsindex edeecb4974..c0aef4c539 100644--- a/server/server.ts+++ b/server/server.ts@@ -104,7 +104,6 @@ if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) {}// ----------- PeerTube modules ------------import { omit } from '@peertube/peertube-core-utils'import { HttpStatusCode } from '@peertube/peertube-models'import { isTestOrDevInstance } from '@peertube/peertube-node-utils'import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics.js'@@ -276,10 +275,10 @@ app.use((err, req, res: express.Response, _next) => {? (process as any)._getActiveRequests(): undefined- // Remove too big metadata- const sanitizedErr = omit(err, [ 'body' ])+ // Remove too big metadata and alter original error to keep type+ err.body = undefined- logger.error('Error in controller.', { err: sanitizedErr, sql, activeRequests, url: req.originalUrl })+ logger.error('Error in controller.', { err, sql, activeRequests, url: req.originalUrl })return res.fail({status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500,