Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Fix concurrency issue with same torrent filename

Chocobozzz4e971d175721
diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.ts
index dda828063c..f745397074 100644
--- a/packages/models/src/server/job.model.ts
+++ b/packages/models/src/server/job.model.ts
@@ -113,6 +113,7 @@ export interface VideoImportYoutubeDLPayload extends VideoImportAbstractPayload
export interface VideoImportTorrentPayload extends VideoImportAbstractPayload {
type: VideoImportTorrentPayloadType
+ torrentPath: string | null // null if magnet URI
}
export type VideoImportPayload = VideoImportYoutubeDLPayload | VideoImportTorrentPayload
diff --git a/server/core/controllers/api/videos/import.ts b/server/core/controllers/api/videos/import.ts
index 9e37fc87b6..af3b3f2a0a 100644
--- a/server/core/controllers/api/videos/import.ts
+++ b/server/core/controllers/api/videos/import.ts
@@ -6,6 +6,7 @@ import {
VideoImportPayload,
VideoImportState
} from '@peertube/peertube-models'
+import { buildUUID } from '@peertube/peertube-node-utils'
import { getVideoThumbnailFile } from '@server/helpers/video.js'
import { YoutubeDlImportError, YoutubeDlImportErrorCode } from '@server/helpers/youtube-dl/youtube-dl-wrapper.js'
import { createLocalVideoThumbnailsFromImage } from '@server/lib/thumbnail.js'
@@ -22,7 +23,6 @@ import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../
import { isArray } from '../../../helpers/custom-validators/misc.js'
import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils.js'
import { logger } from '../../../helpers/logger.js'
-import { getSecureTorrentName } from '../../../helpers/utils.js'
import { CONFIG } from '../../../initializers/config.js'
import { MIMETYPES } from '../../../initializers/constants.js'
import { JobQueue } from '../../../lib/job-queue/job-queue.js'
@@ -161,9 +161,11 @@ async function handleTorrentImport (req: express.Request, res: express.Response,
type: torrentfile
? 'torrent-file'
: 'magnet-uri',
+
videoImportId: videoImport.id,
preventException: false,
- generateTranscription: body.generateTranscription
+ generateTranscription: body.generateTranscription,
+ torrentPath: torrentfile?.path ?? null
}
videoImport.payload = payload
@@ -238,7 +240,7 @@ async function processTorrentOrAbortRequest (req: express.Request, res: express.
const torrentName = torrentfile.originalname
// Rename the torrent to a secured name
- const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
+ const newTorrentPath = join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, buildUUID() + '.torrent')
await move(torrentfile.path, newTorrentPath, { overwrite: true })
torrentfile.path = newTorrentPath
diff --git a/server/core/helpers/utils.ts b/server/core/helpers/utils.ts
index 850286b614..cd0cc1b60c 100644
--- a/server/core/helpers/utils.ts
+++ b/server/core/helpers/utils.ts
@@ -19,7 +19,7 @@ async function generateRandomString (size: number) {
}
interface FormattableToJSON<U, V> {
- toFormattedJSON (args?: U): V
+ toFormattedJSON(args?: U): V
}
function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects: T[], objectsTotal: number, formattedArg?: U) {
@@ -40,10 +40,6 @@ function generateVideoImportTmpPath (target: string | ParseTorrent, extension =
return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
}
-function getSecureTorrentName (originalName: string) {
- return sha256(originalName) + '.torrent'
-}
-
/**
* From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns
* only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does
@@ -64,7 +60,6 @@ export {
deleteFileAndCatch,
generateRandomString,
getFormattedObjects,
- getSecureTorrentName,
generateVideoImportTmpPath,
getUUIDFromFilename
}
diff --git a/server/core/lib/job-queue/handlers/video-import.ts b/server/core/lib/job-queue/handlers/video-import.ts
index 1ee5bb9d0b..c70b620b4f 100644
--- a/server/core/lib/job-queue/handlers/video-import.ts
+++ b/server/core/lib/job-queue/handlers/video-import.ts
@@ -27,6 +27,7 @@ import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@serve
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '@server/lib/webtorrent.js'
+import { UserModel } from '@server/models/user/user.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { MUserId, MVideoFile, MVideoFull } from '@server/types/models/index.js'
import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
@@ -35,7 +36,6 @@ import { FfprobeData } from 'fluent-ffmpeg'
import { move, remove } from 'fs-extra/esm'
import { stat } from 'fs/promises'
import { logger } from '../../../helpers/logger.js'
-import { getSecureTorrentName } from '../../../helpers/utils.js'
import { CONSTRAINTS_FIELDS, JOB_TTL } from '../../../initializers/constants.js'
import { sequelizeTypescript } from '../../../initializers/database.js'
import { VideoFileModel } from '../../../models/video/video-file.js'
@@ -45,7 +45,6 @@ import { federateVideoIfNeeded } from '../../activitypub/videos/index.js'
import { Notifier } from '../../notifier/index.js'
import { createLocalVideoThumbnailsFromVideo } from '../../thumbnail.js'
import { JobQueue } from '../job-queue.js'
-import { UserModel } from '@server/models/user/user.js'
async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
const payload = job.data as VideoImportPayload
@@ -92,22 +91,16 @@ export {
async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportTorrentPayload) {
logger.info('Processing torrent video import in job %s.', job.id)
- const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: payload.videoImportId }
-
- const target = {
- torrentName: videoImport.torrentName
- ? getSecureTorrentName(videoImport.torrentName)
- : undefined,
- uri: videoImport.magnetUri
- }
- return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options)
+ return processFile(
+ () => downloadWebTorrentVideo({ torrentPath: payload.torrentPath, uri: videoImport.magnetUri }, JOB_TTL['video-import']),
+ videoImport,
+ { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: payload.videoImportId }
+ )
}
async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) {
logger.info('Processing youtubeDL video import in job %s.', job.id)
- const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: videoImport.id }
-
const youtubeDL = new YoutubeDLWrapper(
videoImport.targetUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
@@ -117,7 +110,7 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaul
return processFile(
() => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']),
videoImport,
- options
+ { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: videoImport.id }
)
}
diff --git a/server/core/lib/video-post-import.ts b/server/core/lib/video-post-import.ts
index 2a6bc789ca..cd6c86510a 100644
--- a/server/core/lib/video-post-import.ts
+++ b/server/core/lib/video-post-import.ts
@@ -1,4 +1,4 @@
-import { VideoImportPayload, VideoImportState, VideoImportYoutubeDLPayload } from '@peertube/peertube-models'
+import { VideoImportPayload, VideoImportState, VideoImportTorrentPayload, VideoImportYoutubeDLPayload } from '@peertube/peertube-models'
import { CONFIG } from '@server/initializers/config.js'
import { MVideoImport } from '@server/types/models/video/video-import.js'
@@ -17,7 +17,8 @@ export async function buildRetryImportJob (videoImport: MVideoImport) {
preventException: videoImport.payload?.preventException ?? !!videoImport.videoChannelSyncId,
generateTranscription: videoImport.payload?.generateTranscription ?? CONFIG.VIDEO_TRANSCRIPTION.ENABLED,
- fileExt: (videoImport.payload as VideoImportYoutubeDLPayload)?.fileExt
+ fileExt: (videoImport.payload as VideoImportYoutubeDLPayload)?.fileExt,
+ torrentPath: (videoImport.payload as VideoImportTorrentPayload)?.torrentPath
}
videoImport.state = VideoImportState.PENDING
diff --git a/server/core/lib/webtorrent.ts b/server/core/lib/webtorrent.ts
index ec5b99507e..c018ef3869 100644
--- a/server/core/lib/webtorrent.ts
+++ b/server/core/lib/webtorrent.ts
@@ -20,12 +20,12 @@ import { generateVideoImportTmpPath } from '../helpers/utils.js'
import { extractVideo } from '../helpers/video.js'
import { CONFIG } from '../initializers/config.js'
-export async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) {
- const id = target.uri || target.torrentName
+export async function downloadWebTorrentVideo (target: { uri: string, torrentPath: string | null }, timeout: number) {
+ const torrentId = target.uri || target.torrentPath
let timer
- const path = generateVideoImportTmpPath(id)
- logger.info('Importing torrent video %s', id)
+ const path = generateVideoImportTmpPath(torrentId)
+ logger.info('Importing torrent video %s', torrentId)
const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent')
await ensureDir(directoryPath)
@@ -43,7 +43,6 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNam
return new Promise<string>(async (res, rej) => {
let file: TorrentFile
- const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName)
let torrentInput: string | Buffer
try {
@@ -61,11 +60,11 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNam
deleteDownloadedFile({ directoryPath, filepath: file.path })
}
- return safeWebtorrentDestroy(webtorrent, torrentId, undefined, target.torrentName)
+ return safeWebtorrentDestroy({ webtorrent, downloadedFile: null, torrentPath: target.torrentPath })
.then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it')))
}
- logger.debug('Got torrent from webtorrent %s.', id, { infoHash: torrent.infoHash })
+ logger.debug('Got torrent from webtorrent %s.', torrentId, { infoHash: torrent.infoHash })
file = torrent.files[0]
@@ -74,9 +73,9 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNam
writeStream.on('finish', () => {
if (timer) clearTimeout(timer)
- safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName)
- .then(() => res(path))
+ safeWebtorrentDestroy({ webtorrent, downloadedFile: { directoryPath, filepath: file.path }, torrentPath: target.torrentPath })
.catch(err => logger.error('Cannot destroy webtorrent.', { err }))
+ .finally(() => res(path))
})
pipeline(
@@ -88,17 +87,26 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNam
)
})
- torrent.on('error', err => rej(err))
+ torrent.on('error', err => {
+ safeWebtorrentDestroy({
+ webtorrent,
+ downloadedFile: file ? { directoryPath, filepath: file.path } : null,
+ torrentPath: target.torrentPath
+ }).catch(err => logger.error('Cannot destroy webtorrent.', { err }))
+ .finally(() => rej(err))
+ })
timer = setTimeout(() => {
const err = new Error('Webtorrent download timeout.')
- safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName)
- .then(() => rej(err))
- .catch(destroyErr => {
- logger.error('Cannot destroy webtorrent.', { err: destroyErr })
- rej(err)
- })
+ safeWebtorrentDestroy({
+ webtorrent,
+ downloadedFile: file
+ ? { directoryPath, filepath: file.path }
+ : null,
+ torrentPath: target.torrentPath
+ }).catch(destroyErr => logger.error('Cannot destroy webtorrent.', { err: destroyErr }))
+ .finally(() => rej(err))
}, timeout)
})
}
@@ -208,19 +216,21 @@ export function generateMagnetUri (
// Private
// ---------------------------------------------------------------------------
-function safeWebtorrentDestroy (
- webtorrent: Instance,
- torrentId: string,
- downloadedFile?: { directoryPath: string, filepath: string },
- torrentName?: string
-) {
+function safeWebtorrentDestroy (options: {
+ webtorrent: Instance
+ downloadedFile: { directoryPath: string, filepath: string } | null
+ torrentPath: string | null
+}) {
+ const { webtorrent, downloadedFile, torrentPath } = options
+
return new Promise<void>(res => {
webtorrent.destroy(err => {
// Delete torrent file
- if (torrentName) {
- logger.debug('Removing %s torrent after webtorrent download.', torrentId)
- remove(torrentId)
- .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err }))
+ if (torrentPath) {
+ logger.debug('Removing %s torrent after webtorrent download.', torrentPath)
+
+ remove(torrentPath)
+ .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentPath, { err }))
}
// Delete downloaded file