Skip to content

ActivityPub-federated video streaming platform using P2P directly in your web browser

Do not process images in piscina

Chocobozzz1bcbfdb583a6
diff --git a/packages/tests/src/api/server/lazy-static.ts b/packages/tests/src/api/server/lazy-static.ts
index d47f70c21b..aa8a7e093b 100644
--- a/packages/tests/src/api/server/lazy-static.ts
+++ b/packages/tests/src/api/server/lazy-static.ts
@@ -1,5 +1,6 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await,@typescript-eslint/no-floating-promises */
+import { wait } from '@peertube/peertube-core-utils'
import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models'
import {
cleanupTests,
@@ -145,6 +146,9 @@ describe('Test lazy static endpoints', function () {
await servers[1].run()
+ // Wait video info expiration
+ await wait(5000)
+
await testLazyStatic(HttpStatusCode.NOT_FOUND_404)
await waitJobs(servers)
await testLazyStatic(HttpStatusCode.OK_200)
@@ -153,6 +157,8 @@ describe('Test lazy static endpoints', function () {
it('Should still have files after a server restart', async function () {
this.timeout(60000)
+ await fetchRemoteData()
+
await servers[0].kill()
await servers[0].run()
diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts
index d68edb9924..616701080a 100644
--- a/server/core/initializers/constants.ts
+++ b/server/core/initializers/constants.ts
@@ -1158,18 +1158,6 @@ export const totalCPUs = Math.max(cpus().length, 1)
export const WORKER_THREADS = {
IDLE_TIMEOUT: 1000 * 10, // 10 seconds
- DOWNLOAD_IMAGE: {
- CONCURRENCY: 3,
- MAX_THREADS: 1
- },
- PROCESS_IMAGE: {
- CONCURRENCY: 1,
- MAX_THREADS: Math.min(totalCPUs, 5)
- },
- GET_IMAGE_SIZE: {
- CONCURRENCY: 1,
- MAX_THREADS: Math.min(totalCPUs, 5)
- },
SIGN_JSON_LD_OBJECT: {
CONCURRENCY: 1,
MAX_THREADS: 1 // FIXME: we would want 2 threads but there is an issue with JSONLD in worker thread where CPU jumps and stays at 100%
diff --git a/server/core/lib/files-cache/shared/abstract-file-cache.ts b/server/core/lib/files-cache/shared/abstract-file-cache.ts
index 7770b8247b..435bc1905e 100644
--- a/server/core/lib/files-cache/shared/abstract-file-cache.ts
+++ b/server/core/lib/files-cache/shared/abstract-file-cache.ts
@@ -1,7 +1,7 @@
import { HttpStatusCode } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { CachePromise } from '@server/helpers/promise-cache.js'
-import { doRequestAndSaveToFile } from '@server/helpers/requests.js'
+import { doRequestAndSaveToFile, PeerTubeRequestError } from '@server/helpers/requests.js'
import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants.js'
import express from 'express'
import { LRUCache } from 'lru-cache'
@@ -67,7 +67,7 @@ export abstract class AbstractFileCache<M extends FileModel> {
try {
await this.downloadRemoteFile(file)
} catch (err) {
- if (err.status === HttpStatusCode.NOT_FOUND_404) {
+ if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
this.onLazyFetchNotFound(file)
} else {
logger.warn('Cannot process remote image %s.', file.fileUrl, { err })
diff --git a/server/core/lib/files-cache/shared/abstract-image-file-cache.ts b/server/core/lib/files-cache/shared/abstract-image-file-cache.ts
index 8d47800ce2..83a4700c61 100644
--- a/server/core/lib/files-cache/shared/abstract-image-file-cache.ts
+++ b/server/core/lib/files-cache/shared/abstract-image-file-cache.ts
@@ -1,4 +1,4 @@
-import { downloadImageFromWorker } from '@server/lib/worker/parent-process.js'
+import downloadImage from '@server/lib/image-downloader.js'
import { basename, dirname } from 'path'
import { AbstractFileCache, FileModel } from './abstract-file-cache.js'
@@ -18,7 +18,7 @@ export abstract class AbstractImageFileCache<M extends ImageFileModel> extends A
size: this.getImageSize(image)
}
- return downloadImageFromWorker(downloaderOptions)
+ return downloadImage(downloaderOptions)
}
private getImageSize (image: M): { width: number, height: number } {
diff --git a/server/core/lib/worker/workers/image-downloader.ts b/server/core/lib/image-downloader.ts
similarity index 100%
rename from server/core/lib/worker/workers/image-downloader.ts
rename to server/core/lib/image-downloader.ts
diff --git a/server/core/lib/job-queue/handlers/generate-storyboard.ts b/server/core/lib/job-queue/handlers/generate-storyboard.ts
index dfe47ab504..c1e52dc293 100644
--- a/server/core/lib/job-queue/handlers/generate-storyboard.ts
+++ b/server/core/lib/job-queue/handlers/generate-storyboard.ts
@@ -1,12 +1,11 @@
import { FFmpegImage } from '@peertube/peertube-ffmpeg'
import { GenerateStoryboardPayload, VideoFileStream } from '@peertube/peertube-models'
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
-import { generateImageFilename } from '@server/helpers/image-utils.js'
+import { generateImageFilename, getImageSize } from '@server/helpers/image-utils.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { STORYBOARD } from '@server/initializers/constants.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
import { VideoModel } from '@server/models/video/video.js'
import { Job } from 'bullmq'
import { join } from 'path'
@@ -85,7 +84,7 @@ export async function processGenerateStoryboard (job: Job): Promise<void> {
filename,
destination,
- imageSize: await getImageSizeFromWorker(destination),
+ imageSize: await getImageSize(destination),
spriteHeight,
spriteWidth,
diff --git a/server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts b/server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts
index 1a4c8cbdba..b86656dc66 100644
--- a/server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts
+++ b/server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts
@@ -6,12 +6,11 @@ import {
VideoFileStream
} from '@peertube/peertube-models'
import { buildUUID } from '@peertube/peertube-node-utils'
-import { generateImageFilename } from '@server/helpers/image-utils.js'
+import { generateImageFilename, getImageSize } from '@server/helpers/image-utils.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { JOB_PRIORITY, STORYBOARD } from '@server/initializers/constants.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
import { VideoModel } from '@server/models/video/video.js'
import { MRunnerJob } from '@server/types/models/runners/index.js'
import { move } from 'fs-extra/esm'
@@ -101,7 +100,7 @@ export class VideoStoryboardJobHandler extends AbstractJobHandler<CreateOptions,
filename: destinationFilename,
destination: destinationPath,
- imageSize: await getImageSizeFromWorker(destinationPath),
+ imageSize: await getImageSize(destinationPath),
spriteHeight: sprites.size.height,
spriteWidth: sprites.size.width,
diff --git a/server/core/lib/thumbnail.ts b/server/core/lib/thumbnail.ts
index dabd688710..3769bbdfe0 100644
--- a/server/core/lib/thumbnail.ts
+++ b/server/core/lib/thumbnail.ts
@@ -13,8 +13,8 @@ import { ThumbnailModel } from '../models/video/thumbnail.js'
import { MVideoFile, MVideoThumbnail, MVideoUUID, MVideoWithAllFiles } from '../types/models/index.js'
import { MThumbnail } from '../types/models/video/thumbnail.js'
import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist.js'
+import downloadImage from './image-downloader.js'
import { VideoPathManager } from './video-path-manager.js'
-import { downloadImageFromWorker } from './worker/parent-process.js'
const lTags = loggerTagsFactory('thumbnail')
@@ -240,7 +240,7 @@ function _createLocalVideoThumbnailFromUrl (options: {
const { filename, basePath, height, width, aspectRatio } = buildMetadataFromVideo({ video, size, extension })
const thumbnailCreator = () => {
- return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } })
+ return downloadImage({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } })
}
return createThumbnailFromFunction({ thumbnailCreator, filename, height, width, aspectRatio, cached: false })
diff --git a/server/core/lib/video-playlist.ts b/server/core/lib/video-playlist.ts
index 17d2aac8c5..2b40cb413d 100644
--- a/server/core/lib/video-playlist.ts
+++ b/server/core/lib/video-playlist.ts
@@ -12,8 +12,8 @@ import { MAccount, MVideoThumbnail } from '../types/models/index.js'
import { MVideoPlaylistOwner, MVideoPlaylistThumbnail } from '../types/models/video/video-playlist.js'
import { sendUpdateVideoPlaylist } from './activitypub/send/send-update.js'
import { getLocalVideoPlaylistActivityPubUrl } from './activitypub/url.js'
+import downloadImage from './image-downloader.js'
import { createLocalPlaylistThumbnailFromImage } from './thumbnail.js'
-import { downloadImageFromWorker } from './worker/parent-process.js'
export async function createWatchLaterPlaylist (account: MAccount, t: Transaction) {
const videoPlaylist: MVideoPlaylistOwner = new VideoPlaylistModel({
@@ -46,7 +46,7 @@ export async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylis
if (video.isLocal() === true) {
await copy(videoThumbnail.getFSPath(), join(CONFIG.STORAGE.TMP_DIR, tmpImageName))
} else {
- await downloadImageFromWorker({
+ await downloadImage({
url: videoThumbnail.fileUrl,
destDir: CONFIG.STORAGE.TMP_DIR,
destName: tmpImageName,
diff --git a/server/core/lib/worker/parent-process.ts b/server/core/lib/worker/parent-process.ts
index 9341aa2127..24e605ec86 100644
--- a/server/core/lib/worker/parent-process.ts
+++ b/server/core/lib/worker/parent-process.ts
@@ -4,50 +4,10 @@ import { join } from 'path'
import { Piscina } from 'piscina'
import type buildDigest from './workers/build-digest.js'
import type createTorrentPromise from './workers/create-torrent.js'
-import type getImageSize from './workers/get-image-size.js'
import type httpBroadcast from './workers/http-broadcast.js'
import type httpUnicast from './workers/http-unicast.js'
-import type downloadImage from './workers/image-downloader.js'
import type signJsonLDObject from './workers/sign-json-ld-object.js'
-let downloadImageWorker: Piscina
-
-export function downloadImageFromWorker (options: Parameters<typeof downloadImage>[0]): Promise<ReturnType<typeof downloadImage>> {
- if (!downloadImageWorker) {
- downloadImageWorker = new Piscina({
- filename: new URL(join('workers', 'image-downloader.js'), import.meta.url).href,
- concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY,
- maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS,
- minThreads: 1,
- idleTimeout: WORKER_THREADS.IDLE_TIMEOUT
- })
-
- downloadImageWorker.on('error', err => logger.error('Error in download image worker', { err }))
- }
-
- return downloadImageWorker.run(options)
-}
-
-// ---------------------------------------------------------------------------
-
-let getImageSizeWorker: Piscina
-
-export function getImageSizeFromWorker (options: Parameters<typeof getImageSize>[0]): Promise<ReturnType<typeof getImageSize>> {
- if (!getImageSizeWorker) {
- getImageSizeWorker = new Piscina({
- filename: new URL(join('workers', 'get-image-size.js'), import.meta.url).href,
- concurrentTasksPerWorker: WORKER_THREADS.GET_IMAGE_SIZE.CONCURRENCY,
- maxThreads: WORKER_THREADS.GET_IMAGE_SIZE.MAX_THREADS,
- minThreads: 1,
- idleTimeout: WORKER_THREADS.IDLE_TIMEOUT
- })
-
- getImageSizeWorker.on('error', err => logger.error('Error in get image size worker', { err }))
- }
-
- return getImageSizeWorker.run(options)
-}
-
// ---------------------------------------------------------------------------
let parallelHTTPBroadcastWorker: Piscina
@@ -184,16 +144,6 @@ export function createTorrentFromWorker (options: Parameters<typeof createTorren
export function getWorkersStats () {
return [
- {
- label: 'downloadImage',
- queueSize: downloadImageWorker?.queueSize || 0,
- completed: downloadImageWorker?.completed || 0
- },
- {
- label: 'getImageSizeWorker',
- queueSize: getImageSizeWorker?.queueSize || 0,
- completed: getImageSizeWorker?.completed || 0
- },
{
label: 'parallelHTTPBroadcastWorker',
queueSize: parallelHTTPBroadcastWorker?.queueSize || 0,
diff --git a/server/core/lib/worker/workers/get-image-size.ts b/server/core/lib/worker/workers/get-image-size.ts
deleted file mode 100644
index 0d36343082..0000000000
--- a/server/core/lib/worker/workers/get-image-size.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-import { getImageSize } from '@server/helpers/image-utils.js'
-
-export default getImageSize