ActivityPub-federated video streaming platform using P2P directly in your web browser
Do not process images in piscina
Chocobozzz1bcbfdb583a6
diff --git a/packages/tests/src/api/server/lazy-static.ts b/packages/tests/src/api/server/lazy-static.tsindex d47f70c21b..aa8a7e093b 100644--- a/packages/tests/src/api/server/lazy-static.ts+++ b/packages/tests/src/api/server/lazy-static.ts@@ -1,5 +1,6 @@/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await,@typescript-eslint/no-floating-promises */+import { wait } from '@peertube/peertube-core-utils'import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models'import {cleanupTests,@@ -145,6 +146,9 @@ describe('Test lazy static endpoints', function () {await servers[1].run()+ // Wait video info expiration+ await wait(5000)+await testLazyStatic(HttpStatusCode.NOT_FOUND_404)await waitJobs(servers)await testLazyStatic(HttpStatusCode.OK_200)@@ -153,6 +157,8 @@ describe('Test lazy static endpoints', function () {it('Should still have files after a server restart', async function () {this.timeout(60000)+ await fetchRemoteData()+await servers[0].kill()await servers[0].run()diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.tsindex d68edb9924..616701080a 100644--- a/server/core/initializers/constants.ts+++ b/server/core/initializers/constants.ts@@ -1158,18 +1158,6 @@ export const totalCPUs = Math.max(cpus().length, 1)export const WORKER_THREADS = {IDLE_TIMEOUT: 1000 * 10, // 10 seconds- DOWNLOAD_IMAGE: {- CONCURRENCY: 3,- MAX_THREADS: 1- },- PROCESS_IMAGE: {- CONCURRENCY: 1,- MAX_THREADS: Math.min(totalCPUs, 5)- },- GET_IMAGE_SIZE: {- CONCURRENCY: 1,- MAX_THREADS: Math.min(totalCPUs, 5)- },SIGN_JSON_LD_OBJECT: {CONCURRENCY: 1,MAX_THREADS: 1 // FIXME: we would want 2 threads but there is an issue with JSONLD in worker thread where CPU jumps and stays at 100%diff --git a/server/core/lib/files-cache/shared/abstract-file-cache.ts b/server/core/lib/files-cache/shared/abstract-file-cache.tsindex 7770b8247b..435bc1905e 100644--- a/server/core/lib/files-cache/shared/abstract-file-cache.ts+++ b/server/core/lib/files-cache/shared/abstract-file-cache.ts@@ -1,7 +1,7 @@import { HttpStatusCode } from '@peertube/peertube-models'import { logger } from '@server/helpers/logger.js'import { CachePromise } from '@server/helpers/promise-cache.js'-import { doRequestAndSaveToFile } from '@server/helpers/requests.js'+import { doRequestAndSaveToFile, PeerTubeRequestError } from '@server/helpers/requests.js'import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants.js'import express from 'express'import { LRUCache } from 'lru-cache'@@ -67,7 +67,7 @@ export abstract class AbstractFileCache<M extends FileModel> {try {await this.downloadRemoteFile(file)} catch (err) {- if (err.status === HttpStatusCode.NOT_FOUND_404) {+ if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {this.onLazyFetchNotFound(file)} else {logger.warn('Cannot process remote image %s.', file.fileUrl, { err })diff --git a/server/core/lib/files-cache/shared/abstract-image-file-cache.ts b/server/core/lib/files-cache/shared/abstract-image-file-cache.tsindex 8d47800ce2..83a4700c61 100644--- a/server/core/lib/files-cache/shared/abstract-image-file-cache.ts+++ b/server/core/lib/files-cache/shared/abstract-image-file-cache.ts@@ -1,4 +1,4 @@-import { downloadImageFromWorker } from '@server/lib/worker/parent-process.js'+import downloadImage from '@server/lib/image-downloader.js'import { basename, dirname } from 'path'import { AbstractFileCache, FileModel } from './abstract-file-cache.js'@@ -18,7 +18,7 @@ export abstract class AbstractImageFileCache<M extends ImageFileModel> extends Asize: this.getImageSize(image)}- return downloadImageFromWorker(downloaderOptions)+ return downloadImage(downloaderOptions)}private getImageSize (image: M): { width: number, height: number } {diff --git a/server/core/lib/worker/workers/image-downloader.ts b/server/core/lib/image-downloader.tssimilarity index 100%rename from server/core/lib/worker/workers/image-downloader.tsrename to server/core/lib/image-downloader.tsdiff --git a/server/core/lib/job-queue/handlers/generate-storyboard.ts b/server/core/lib/job-queue/handlers/generate-storyboard.tsindex dfe47ab504..c1e52dc293 100644--- a/server/core/lib/job-queue/handlers/generate-storyboard.ts+++ b/server/core/lib/job-queue/handlers/generate-storyboard.ts@@ -1,12 +1,11 @@import { FFmpegImage } from '@peertube/peertube-ffmpeg'import { GenerateStoryboardPayload, VideoFileStream } from '@peertube/peertube-models'import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'-import { generateImageFilename } from '@server/helpers/image-utils.js'+import { generateImageFilename, getImageSize } from '@server/helpers/image-utils.js'import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'import { STORYBOARD } from '@server/initializers/constants.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'-import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'import { VideoModel } from '@server/models/video/video.js'import { Job } from 'bullmq'import { join } from 'path'@@ -85,7 +84,7 @@ export async function processGenerateStoryboard (job: Job): Promise<void> {filename,destination,- imageSize: await getImageSizeFromWorker(destination),+ imageSize: await getImageSize(destination),spriteHeight,spriteWidth,diff --git a/server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts b/server/core/lib/runners/job-handlers/video-storyboard-job-handler.tsindex 1a4c8cbdba..b86656dc66 100644--- a/server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts+++ b/server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts@@ -6,12 +6,11 @@ import {VideoFileStream} from '@peertube/peertube-models'import { buildUUID } from '@peertube/peertube-node-utils'-import { generateImageFilename } from '@server/helpers/image-utils.js'+import { generateImageFilename, getImageSize } from '@server/helpers/image-utils.js'import { logger, loggerTagsFactory } from '@server/helpers/logger.js'import { CONFIG } from '@server/initializers/config.js'import { JOB_PRIORITY, STORYBOARD } from '@server/initializers/constants.js'import { VideoPathManager } from '@server/lib/video-path-manager.js'-import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'import { VideoModel } from '@server/models/video/video.js'import { MRunnerJob } from '@server/types/models/runners/index.js'import { move } from 'fs-extra/esm'@@ -101,7 +100,7 @@ export class VideoStoryboardJobHandler extends AbstractJobHandler<CreateOptions,filename: destinationFilename,destination: destinationPath,- imageSize: await getImageSizeFromWorker(destinationPath),+ imageSize: await getImageSize(destinationPath),spriteHeight: sprites.size.height,spriteWidth: sprites.size.width,diff --git a/server/core/lib/thumbnail.ts b/server/core/lib/thumbnail.tsindex dabd688710..3769bbdfe0 100644--- a/server/core/lib/thumbnail.ts+++ b/server/core/lib/thumbnail.ts@@ -13,8 +13,8 @@ import { ThumbnailModel } from '../models/video/thumbnail.js'import { MVideoFile, MVideoThumbnail, MVideoUUID, MVideoWithAllFiles } from '../types/models/index.js'import { MThumbnail } from '../types/models/video/thumbnail.js'import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist.js'+import downloadImage from './image-downloader.js'import { VideoPathManager } from './video-path-manager.js'-import { downloadImageFromWorker } from './worker/parent-process.js'const lTags = loggerTagsFactory('thumbnail')@@ -240,7 +240,7 @@ function _createLocalVideoThumbnailFromUrl (options: {const { filename, basePath, height, width, aspectRatio } = buildMetadataFromVideo({ video, size, extension })const thumbnailCreator = () => {- return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } })+ return downloadImage({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } })}return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, aspectRatio, cached: false })diff --git a/server/core/lib/video-playlist.ts b/server/core/lib/video-playlist.tsindex 17d2aac8c5..2b40cb413d 100644--- a/server/core/lib/video-playlist.ts+++ b/server/core/lib/video-playlist.ts@@ -12,8 +12,8 @@ import { MAccount, MVideoThumbnail } from '../types/models/index.js'import { MVideoPlaylistOwner, MVideoPlaylistThumbnail } from '../types/models/video/video-playlist.js'import { sendUpdateVideoPlaylist } from './activitypub/send/send-update.js'import { getLocalVideoPlaylistActivityPubUrl } from './activitypub/url.js'+import downloadImage from './image-downloader.js'import { createLocalPlaylistThumbnailFromImage } from './thumbnail.js'-import { downloadImageFromWorker } from './worker/parent-process.js'export async function createWatchLaterPlaylist (account: MAccount, t: Transaction) {const videoPlaylist: MVideoPlaylistOwner = new VideoPlaylistModel({@@ -46,7 +46,7 @@ export async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylisif (video.isLocal() === true) {await copy(videoThumbnail.getFSPath(), join(CONFIG.STORAGE.TMP_DIR, tmpImageName))} else {- await downloadImageFromWorker({+ await downloadImage({url: videoThumbnail.fileUrl,destDir: CONFIG.STORAGE.TMP_DIR,destName: tmpImageName,diff --git a/server/core/lib/worker/parent-process.ts b/server/core/lib/worker/parent-process.tsindex 9341aa2127..24e605ec86 100644--- a/server/core/lib/worker/parent-process.ts+++ b/server/core/lib/worker/parent-process.ts@@ -4,50 +4,10 @@ import { join } from 'path'import { Piscina } from 'piscina'import type buildDigest from './workers/build-digest.js'import type createTorrentPromise from './workers/create-torrent.js'-import type getImageSize from './workers/get-image-size.js'import type httpBroadcast from './workers/http-broadcast.js'import type httpUnicast from './workers/http-unicast.js'-import type downloadImage from './workers/image-downloader.js'import type signJsonLDObject from './workers/sign-json-ld-object.js'-let downloadImageWorker: Piscina--export function downloadImageFromWorker (options: Parameters<typeof downloadImage>[0]): Promise<ReturnType<typeof downloadImage>> {- if (!downloadImageWorker) {- downloadImageWorker = new Piscina({- filename: new URL(join('workers', 'image-downloader.js'), import.meta.url).href,- concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY,- maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS,- minThreads: 1,- idleTimeout: WORKER_THREADS.IDLE_TIMEOUT- })-- downloadImageWorker.on('error', err => logger.error('Error in download image worker', { err }))- }-- return downloadImageWorker.run(options)-}--// -----------------------------------------------------------------------------let getImageSizeWorker: Piscina--export function getImageSizeFromWorker (options: Parameters<typeof getImageSize>[0]): Promise<ReturnType<typeof getImageSize>> {- if (!getImageSizeWorker) {- getImageSizeWorker = new Piscina({- filename: new URL(join('workers', 'get-image-size.js'), import.meta.url).href,- concurrentTasksPerWorker: WORKER_THREADS.GET_IMAGE_SIZE.CONCURRENCY,- maxThreads: WORKER_THREADS.GET_IMAGE_SIZE.MAX_THREADS,- minThreads: 1,- idleTimeout: WORKER_THREADS.IDLE_TIMEOUT- })-- getImageSizeWorker.on('error', err => logger.error('Error in get image size worker', { err }))- }-- return getImageSizeWorker.run(options)-}-// ---------------------------------------------------------------------------let parallelHTTPBroadcastWorker: Piscina@@ -184,16 +144,6 @@ export function createTorrentFromWorker (options: Parameters<typeof createTorrenexport function getWorkersStats () {return [- {- label: 'downloadImage',- queueSize: downloadImageWorker?.queueSize || 0,- completed: downloadImageWorker?.completed || 0- },- {- label: 'getImageSizeWorker',- queueSize: getImageSizeWorker?.queueSize || 0,- completed: getImageSizeWorker?.completed || 0- },{label: 'parallelHTTPBroadcastWorker',queueSize: parallelHTTPBroadcastWorker?.queueSize || 0,diff --git a/server/core/lib/worker/workers/get-image-size.ts b/server/core/lib/worker/workers/get-image-size.tsdeleted file mode 100644index 0d36343082..0000000000--- a/server/core/lib/worker/workers/get-image-size.ts+++ /dev/null@@ -1,3 +0,0 @@-import { getImageSize } from '@server/helpers/image-utils.js'--export default getImageSize