ActivityPub-federated video streaming platform using P2P directly in your web browser
Fix concurrency issue with same torrent filename
Chocobozzz4e971d175721
diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.tsindex dda828063c..f745397074 100644--- a/packages/models/src/server/job.model.ts+++ b/packages/models/src/server/job.model.ts@@ -113,6 +113,7 @@ export interface VideoImportYoutubeDLPayload extends VideoImportAbstractPayloadexport interface VideoImportTorrentPayload extends VideoImportAbstractPayload {type: VideoImportTorrentPayloadType+ torrentPath: string | null // null if magnet URI}export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayloaddiff --git a/server/core/controllers/api/videos/import.ts b/server/core/controllers/api/videos/import.tsindex 9e37fc87b6..af3b3f2a0a 100644--- a/server/core/controllers/api/videos/import.ts+++ b/server/core/controllers/api/videos/import.ts@@ -6,6 +6,7 @@ import {VideoImportPayload,VideoImportState} from '@peertube/peertube-models'+import { buildUUID } from '@peertube/peertube-node-utils'import { getVideoThumbnailFile } from '@server/helpers/video.js'import { YoutubeDlImportError, YoutubeDlImportErrorCode } from '@server/helpers/youtube-dl/youtube-dl-wrapper.js'import { createLocalVideoThumbnailsFromImage } from '@server/lib/thumbnail.js'@@ -22,7 +23,6 @@ import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../import { isArray } from '../../../helpers/custom-validators/misc.js'import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils.js'import { logger } from '../../../helpers/logger.js'-import { getSecureTorrentName } from '../../../helpers/utils.js'import { CONFIG } from '../../../initializers/config.js'import { MIMETYPES } from '../../../initializers/constants.js'import { JobQueue } from '../../../lib/job-queue/job-queue.js'@@ -161,9 +161,11 @@ async function handleTorrentImport (req: express.Request, res: express.Response,type: torrentfile? 'torrent-file': 'magnet-uri',+videoImportId: videoImport.id,preventException: false,- generateTranscription: body.generateTranscription+ generateTranscription: body.generateTranscription,+ torrentPath: torrentfile?.path ?? null}videoImport.payload = payload@@ -238,7 +240,7 @@ async function processTorrentOrAbortRequest (req: express.Request, res: express.const torrentName = torrentfile.originalname// Rename the torrent to a secured name- const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))+ const newTorrentPath = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, buildUUID() + '.torrent')await move(torrentfile.path, newTorrentPath, { overwrite: true })torrentfile.path = newTorrentPathdiff --git a/server/core/helpers/utils.ts b/server/core/helpers/utils.tsindex 850286b614..cd0cc1b60c 100644--- a/server/core/helpers/utils.ts+++ b/server/core/helpers/utils.ts@@ -19,7 +19,7 @@ async function generateRandomString (size: number) {}interface FormattableToJSON<U, V> {- toFormattedJSON (args?: U): V+ toFormattedJSON(args?: U): V}function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects: T[], objectsTotal: number, formattedArg?: U) {@@ -40,10 +40,6 @@ function generateVideoImportTmpPath (target: string | ParseTorrent, extension =return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)}-function getSecureTorrentName (originalName: string) {- return sha256(originalName) + '.torrent'-}-/*** From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns* only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does@@ -64,7 +60,6 @@ export {deleteFileAndCatch,generateRandomString,getFormattedObjects,- getSecureTorrentName,generateVideoImportTmpPath,getUUIDFromFilename}diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.tsindex 1ee5bb9d0b..c70b620b4f 100644--- a/server/core/lib/job-queue/handlers/video-import.ts+++ b/server/core/lib/job-queue/handlers/video-import.ts@@ -27,6 +27,7 @@ import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@serveimport { VideoPathManager } from '@server/lib/video-path-manager.js'import { buildNextVideoState } from '@server/lib/video-state.js'import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '@server/lib/webtorrent.js'+import { UserModel } from '@server/models/user/user.js'import { VideoCaptionModel } from '@server/models/video/video-caption.js'import { MUserId, MVideoFile, MVideoFull } from '@server/types/models/index.js'import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'@@ -35,7 +36,6 @@ import { FfprobeData } from 'fluent-ffmpeg'import { move, remove } from 'fs-extra/esm'import { stat } from 'fs/promises'import { logger } from '../../../helpers/logger.js'-import { getSecureTorrentName } from '../../../helpers/utils.js'import { CONSTRAINTS_FIELDS, JOB_TTL } from '../../../initializers/constants.js'import { sequelizeTypescript } from '../../../initializers/database.js'import { VideoFileModel } from '../../../models/video/video-file.js'@@ -45,7 +45,6 @@ import { federateVideoIfNeeded } from '../../activitypub/videos/index.js'import { Notifier } from '../../notifier/index.js'import { createLocalVideoThumbnailsFromVideo } from '../../thumbnail.js'import { JobQueue } from '../job-queue.js'-import { UserModel } from '@server/models/user/user.js'async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {const payload = job.data as VideoImportPayload@@ -92,22 +91,16 @@ export {async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportTorrentPayload) {logger.info('Processing torrent video import in job %s.', job.id)- const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: payload.videoImportId }-- const target = {- torrentName: videoImport.torrentName- ? getSecureTorrentName(videoImport.torrentName)- : undefined,- uri: videoImport.magnetUri- }- return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options)+ return processFile(+ () => downloadWebTorrentVideo({ torrentPath: payload.torrentPath, uri: videoImport.magnetUri }, JOB_TTL['video-import']),+ videoImport,+ { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: payload.videoImportId }+ )}async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) {logger.info('Processing youtubeDL video import in job %s.', job.id)- const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: videoImport.id }-const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl,ServerConfigManager.Instance.getEnabledResolutions('vod'),@@ -117,7 +110,7 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaulreturn processFile(() => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']),videoImport,- options+ { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: videoImport.id })}diff --git a/server/core/lib/video-post-import.ts b/server/core/lib/video-post-import.tsindex 2a6bc789ca..cd6c86510a 100644--- a/server/core/lib/video-post-import.ts+++ b/server/core/lib/video-post-import.ts@@ -1,4 +1,4 @@-import { VideoImportPayload, VideoImportState, VideoImportYoutubeDLPayload } from '@peertube/peertube-models'+import { VideoImportPayload, VideoImportState, VideoImportTorrentPayload, VideoImportYoutubeDLPayload } from '@peertube/peertube-models'import { CONFIG } from '@server/initializers/config.js'import { MVideoImport } from '@server/types/models/video/video-import.js'@@ -17,7 +17,8 @@ export async function buildRetryImportJob (videoImport: MVideoImport) {preventException: videoImport.payload?.preventException ?? !!videoImport.videoChannelSyncId,generateTranscription: videoImport.payload?.generateTranscription ?? CONFIG.VIDEO_TRANSCRIPTION.ENABLED,- fileExt: (videoImport.payload as VideoImportYoutubeDLPayload)?.fileExt+ fileExt: (videoImport.payload as VideoImportYoutubeDLPayload)?.fileExt,+ torrentPath: (videoImport.payload as VideoImportTorrentPayload)?.torrentPath}videoImport.state = VideoImportState.PENDINGdiff --git a/server/core/lib/webtorrent.ts b/server/core/lib/webtorrent.tsindex ec5b99507e..c018ef3869 100644--- a/server/core/lib/webtorrent.ts+++ b/server/core/lib/webtorrent.ts@@ -20,12 +20,12 @@ import { generateVideoImportTmpPath } from '../helpers/utils.js'import { extractVideo } from '../helpers/video.js'import { CONFIG } from '../initializers/config.js'-export async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) {- const id = target.uri || target.torrentName+export async function downloadWebTorrentVideo (target: { uri: string, torrentPath: string | null }, timeout: number) {+ const torrentId = target.uri || target.torrentPathlet timer- const path = generateVideoImportTmpPath(id)- logger.info('Importing torrent video %s', id)+ const path = generateVideoImportTmpPath(torrentId)+ logger.info('Importing torrent video %s', torrentId)const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent')await ensureDir(directoryPath)@@ -43,7 +43,6 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNamreturn new Promise<string>(async (res, rej) => {let file: TorrentFile- const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)let torrentInput: string | Buffertry {@@ -61,11 +60,11 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNamdeleteDownloadedFile({ directoryPath, filepath: file.path })}- return safeWebtorrentDestroy(webtorrent, torrentId, undefined, target.torrentName)+ return safeWebtorrentDestroy({ webtorrent, downloadedFile: null, torrentPath: target.torrentPath }).then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it')))}- logger.debug('Got torrent from webtorrent %s.', id, { infoHash: torrent.infoHash })+ logger.debug('Got torrent from webtorrent %s.', torrentId, { infoHash: torrent.infoHash })file = torrent.files[0]@@ -74,9 +73,9 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNamwriteStream.on('finish', () => {if (timer) clearTimeout(timer)- safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName)- .then(() => res(path))+ safeWebtorrentDestroy({ webtorrent, downloadedFile: { directoryPath, filepath: file.path }, torrentPath: target.torrentPath }).catch(err => logger.error('Cannot destroy webtorrent.', { err }))+ .finally(() => res(path))})pipeline(@@ -88,17 +87,26 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNam)})- torrent.on('error', err => rej(err))+ torrent.on('error', err => {+ safeWebtorrentDestroy({+ webtorrent,+ downloadedFile: file ? { directoryPath, filepath: file.path } : null,+ torrentPath: target.torrentPath+ }).catch(err => logger.error('Cannot destroy webtorrent.', { err }))+ .finally(() => rej(err))+ })timer = setTimeout(() => {const err = new Error('Webtorrent download timeout.')- safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName)- .then(() => rej(err))- .catch(destroyErr => {- logger.error('Cannot destroy webtorrent.', { err: destroyErr })- rej(err)- })+ safeWebtorrentDestroy({+ webtorrent,+ downloadedFile: file+ ? { directoryPath, filepath: file.path }+ : null,+ torrentPath: target.torrentPath+ }).catch(destroyErr => logger.error('Cannot destroy webtorrent.', { err: destroyErr }))+ .finally(() => rej(err))}, timeout)})}@@ -208,19 +216,21 @@ export function generateMagnetUri (// Private// ----------------------------------------------------------------------------function safeWebtorrentDestroy (- webtorrent: Instance,- torrentId: string,- downloadedFile?: { directoryPath: string, filepath: string },- torrentName?: string-) {+function safeWebtorrentDestroy (options: {+ webtorrent: Instance+ downloadedFile: { directoryPath: string, filepath: string } | null+ torrentPath: string | null+}) {+ const { webtorrent, downloadedFile, torrentPath } = options+return new Promise<void>(res => {webtorrent.destroy(err => {// Delete torrent file- if (torrentName) {- logger.debug('Removing %s torrent after webtorrent download.', torrentId)- remove(torrentId)- .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))+ if (torrentPath) {+ logger.debug('Removing %s torrent after webtorrent download.', torrentPath)++ remove(torrentPath)+ .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentPath, { err }))}// Delete downloaded file