Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Create torrent in worker

Chocobozzz5ffcbe3185a0
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index 0774831b24..2f111bf3f3 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -1158,6 +1158,10 @@ export const WORKER_THREADS = {
BUILD_DIGEST: {
CONCURRENCY: 1,
MAX_THREADS: 1
+ },
+ CREATE_TORRENT: {
+ CONCURRENCY: 1,
+ MAX_THREADS: 1
}
}
diff --git a/server/core/helpers/webtorrent.ts b/server/core/lib/webtorrent.ts
similarity index 95%
rename from server/core/helpers/webtorrent.ts
rename to server/core/lib/webtorrent.ts
index cbff7208a8..c094f39874 100644
--- a/server/core/helpers/webtorrent.ts
+++ b/server/core/lib/webtorrent.ts
@@ -1,13 +1,12 @@
-import { promisify2 } from '@peertube/peertube-core-utils'
import { sha1 } from '@peertube/peertube-node-utils'
import { WEBSERVER } from '@server/initializers/constants.js'
import { generateTorrentFileName } from '@server/lib/paths.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { createTorrentFromWorker } from '@server/lib/worker/parent-process.js'
import { MVideoFile } from '@server/types/models/video/video-file.js'
import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist.js'
import { MVideo } from '@server/types/models/video/video.js'
import bencode from 'bencode'
-import createTorrent from 'create-torrent'
import { createWriteStream } from 'fs'
import { ensureDir, pathExists, remove } from 'fs-extra/esm'
import { readFile, writeFile } from 'fs/promises'
@@ -15,14 +14,11 @@ import { encode as magnetUriEncode } from 'magnet-uri'
import parseTorrent from 'parse-torrent'
import { dirname, join } from 'path'
import { pipeline } from 'stream'
-import { CONFIG } from '../initializers/config.js'
-import { logger } from './logger.js'
-import { generateVideoImportTmpPath } from './utils.js'
-import { extractVideo } from './video.js'
-
import type { Instance, TorrentFile } from 'webtorrent'
-
-const createTorrentPromise = promisify2<string, any, any>(createTorrent)
+import { logger } from '../helpers/logger.js'
+import { generateVideoImportTmpPath } from '../helpers/utils.js'
+import { extractVideo } from '../helpers/video.js'
+import { CONFIG } from '../initializers/config.js'
export async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) {
const id = target.uri || target.torrentName
@@ -96,7 +92,6 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNam
logger.error('Cannot destroy webtorrent.', { err: destroyErr })
rej(err)
})
-
}, timeout)
})
}
@@ -114,15 +109,15 @@ export async function createTorrentAndSetInfoHashFromPath (
) {
const video = extractVideo(videoOrPlaylist)
- const options = {
+ const torrentContent = await createTorrentFromWorker({
+ path: filePath,
+
// Keep the extname, it's used by the client to stream the file inside a web browser
name: buildInfoName(video, videoFile),
createdBy: 'PeerTube',
announceList: buildAnnounceList(),
urlList: buildUrlList(video, videoFile)
- }
-
- const torrentContent = await createTorrentPromise(filePath, options)
+ })
const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)
const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)
diff --git a/server/core/lib/worker/parent-process.ts b/server/core/lib/worker/parent-process.ts
index 2196e09d66..ffe3acc174 100644
--- a/server/core/lib/worker/parent-process.ts
+++ b/server/core/lib/worker/parent-process.ts
@@ -1,14 +1,15 @@
+import { logger } from '@server/helpers/logger.js'
+import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants.js'
import { join } from 'path'
import { Piscina } from 'piscina'
-import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants.js'
+import type buildDigest from './workers/build-digest.js'
+import type createTorrentPromise from './workers/create-torrent.js'
+import type getImageSize from './workers/get-image-size.js'
import type httpBroadcast from './workers/http-broadcast.js'
+import type httpUnicast from './workers/http-unicast.js'
import type downloadImage from './workers/image-downloader.js'
import type processImage from './workers/image-processor.js'
-import type getImageSize from './workers/get-image-size.js'
import type signJsonLDObject from './workers/sign-json-ld-object.js'
-import type buildDigest from './workers/build-digest.js'
-import type httpUnicast from './workers/http-unicast.js'
-import { logger } from '@server/helpers/logger.js'
let downloadImageWorker: Piscina
@@ -182,6 +183,26 @@ export function buildDigestFromWorker (
// ---------------------------------------------------------------------------
+let createTorrentWorker: Piscina
+
+export function createTorrentFromWorker (options: Parameters<typeof createTorrentPromise>[0]): Promise<Buffer> {
+ if (!createTorrentWorker) {
+ createTorrentWorker = new Piscina({
+ filename: new URL(join('workers', 'create-torrent.js'), import.meta.url).href,
+ concurrentTasksPerWorker: WORKER_THREADS.CREATE_TORRENT.CONCURRENCY,
+ maxThreads: WORKER_THREADS.CREATE_TORRENT.MAX_THREADS,
+ minThreads: 1,
+ idleTimeout: WORKER_THREADS.IDLE_TIMEOUT
+ })
+
+ createTorrentWorker.on('error', err => logger.error('Error in create torrent worker', { err }))
+ }
+
+ return createTorrentWorker.run(options)
+}
+
+// ---------------------------------------------------------------------------
+
export function getWorkersStats () {
return [
{
@@ -223,6 +244,11 @@ export function getWorkersStats () {
label: 'buildDigestWorker',
queueSize: buildDigestWorker?.queueSize || 0,
completed: buildDigestWorker?.completed || 0
+ },
+ {
+ label: 'createTorrentWorker',
+ queueSize: createTorrentWorker?.queueSize || 0,
+ completed: createTorrentWorker?.completed || 0
}
]
}
diff --git a/server/core/lib/worker/workers/create-torrent.ts b/server/core/lib/worker/workers/create-torrent.ts
new file mode 100644
index 0000000000..9c8cc59aeb
--- /dev/null
+++ b/server/core/lib/worker/workers/create-torrent.ts
@@ -0,0 +1,15 @@
+import createTorrent from 'create-torrent'
+
+export default function createTorrentPromise (
+ options: {
+ path: string
+ } & Parameters<typeof createTorrent>[1]
+): Promise<Buffer> {
+ return new Promise<Buffer>((resolve, reject) => {
+ createTorrent(options.path, options, (err, torrent) => {
+ if (err) return reject(err)
+
+ resolve(torrent)
+ })
+ })
+}