ActivityPub-federated video streaming platform using P2P directly in your web browser
Create torrent in worker
Chocobozzz5ffcbe3185a0
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex 0774831b24..2f111bf3f3 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -1158,6 +1158,10 @@ export const WORKER_THREADS = {BUILD_DIGEST: {CONCURRENCY: 1,MAX_THREADS: 1+ },+ CREATE_TORRENT: {+ CONCURRENCY: 1,+ MAX_THREADS: 1}}diff --git a/server/core/helpers/webtorrent.ts b/server/core/lib/webtorrent.tssimilarity index 95%rename from server/core/helpers/webtorrent.tsrename to server/core/lib/webtorrent.tsindex cbff7208a8..c094f39874 100644--- a/server/core/helpers/webtorrent.ts+++ b/server/core/lib/webtorrent.ts@@ -1,13 +1,12 @@-import { promisify2 } from '@peertube/peertube-core-utils'import { sha1 } from '@peertube/peertube-node-utils'import { WEBSERVER } from '@server/initializers/constants.js'import { generateTorrentFileName } from '@server/lib/paths.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'+import { createTorrentFromWorker } from '@server/lib/worker/parent-process.js'import { MVideoFile } from '@server/types/models/video/video-file.js'import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist.js'import { MVideo } from '@server/types/models/video/video.js'import bencode from 'bencode'-import createTorrent from 'create-torrent'import { createWriteStream } from 'fs'import { ensureDir, pathExists, remove } from 'fs-extra/esm'import { readFile, writeFile } from 'fs/promises'@@ -15,14 +14,11 @@ import { encode as magnetUriEncode } from 'magnet-uri'import parseTorrent from 'parse-torrent'import { dirname, join } from 'path'import { pipeline } from 'stream'-import { CONFIG } from '../initializers/config.js'-import { logger } from './logger.js'-import { generateVideoImportTmpPath } from './utils.js'-import { extractVideo } from './video.js'-import type { Instance, TorrentFile } from 'webtorrent'--const createTorrentPromise = promisify2<string, any, any>(createTorrent)+import { logger } from '../helpers/logger.js'+import { generateVideoImportTmpPath } from '../helpers/utils.js'+import { extractVideo } from '../helpers/video.js'+import { CONFIG } from '../initializers/config.js'export async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) {const id = target.uri || target.torrentName@@ -96,7 +92,6 @@ export async function downloadWebTorrentVideo (target: { uri: string, torrentNamlogger.error('Cannot destroy webtorrent.', { err: destroyErr })rej(err)})-}, timeout)})}@@ -114,15 +109,15 @@ export async function createTorrentAndSetInfoHashFromPath () {const video = extractVideo(videoOrPlaylist)- const options = {+ const torrentContent = await createTorrentFromWorker({+ path: filePath,+// Keep the extname, it's used by the client to stream the file inside a web browsername: buildInfoName(video, videoFile),createdBy: 'PeerTube',announceList: buildAnnounceList(),urlList: buildUrlList(video, videoFile)- }-- const torrentContent = await createTorrentPromise(filePath, options)+ })const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution)const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename)diff --git a/server/core/lib/worker/parent-process.ts b/server/core/lib/worker/parent-process.tsindex 2196e09d66..ffe3acc174 100644--- a/server/core/lib/worker/parent-process.ts+++ b/server/core/lib/worker/parent-process.ts@@ -1,14 +1,15 @@+import { logger } from '@server/helpers/logger.js'+import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants.js'import { join } from 'path'import { Piscina } from 'piscina'-import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants.js'+import type buildDigest from './workers/build-digest.js'+import type createTorrentPromise from './workers/create-torrent.js'+import type getImageSize from './workers/get-image-size.js'import type httpBroadcast from './workers/http-broadcast.js'+import type httpUnicast from './workers/http-unicast.js'import type downloadImage from './workers/image-downloader.js'import type processImage from './workers/image-processor.js'-import type getImageSize from './workers/get-image-size.js'import type signJsonLDObject from './workers/sign-json-ld-object.js'-import type buildDigest from './workers/build-digest.js'-import type httpUnicast from './workers/http-unicast.js'-import { logger } from '@server/helpers/logger.js'let downloadImageWorker: Piscina@@ -182,6 +183,26 @@ export function buildDigestFromWorker (// ---------------------------------------------------------------------------+let createTorrentWorker: Piscina++export function createTorrentFromWorker (options: Parameters<typeof createTorrentPromise>[0]): Promise<Buffer> {+ if (!createTorrentWorker) {+ createTorrentWorker = new Piscina({+ filename: new URL(join('workers', 'create-torrent.js'), import.meta.url).href,+ concurrentTasksPerWorker: WORKER_THREADS.CREATE_TORRENT.CONCURRENCY,+ maxThreads: WORKER_THREADS.CREATE_TORRENT.MAX_THREADS,+ minThreads: 1,+ idleTimeout: WORKER_THREADS.IDLE_TIMEOUT+ })++ createTorrentWorker.on('error', err => logger.error('Error in create torrent worker', { err }))+ }++ return createTorrentWorker.run(options)+}++// ---------------------------------------------------------------------------+export function getWorkersStats () {return [{@@ -223,6 +244,11 @@ export function getWorkersStats () {label: 'buildDigestWorker',queueSize: buildDigestWorker?.queueSize || 0,completed: buildDigestWorker?.completed || 0+ },+ {+ label: 'createTorrentWorker',+ queueSize: createTorrentWorker?.queueSize || 0,+ completed: createTorrentWorker?.completed || 0}]}diff --git a/server/core/lib/worker/workers/create-torrent.ts b/server/core/lib/worker/workers/create-torrent.tsnew file mode 100644index 0000000000..9c8cc59aeb--- /dev/null+++ b/server/core/lib/worker/workers/create-torrent.ts@@ -0,0 +1,15 @@+import createTorrent from 'create-torrent'++export default function createTorrentPromise (+ options: {+ path: string+ } & Parameters<typeof createTorrent>[1]+): Promise<Buffer> {+ return new Promise<Buffer>((resolve, reject) => {+ createTorrent(options.path, options, (err, torrent) => {+ if (err) return reject(err)++ resolve(torrent)+ })+ })+}