Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker): turn off useWorkerThreads in sandboxed scrape worker (#1941)

Gergő Móriczbd5adfeae112
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 8375119c4..28fd8a25b 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -348,7 +348,7 @@ const separateWorkerFun = (
path: string,
): Worker => {
// Extract memory size from --max-old-space-size flag if present
- const maxOldSpaceSize = process.execArgv
+ const maxOldSpaceSize = process.env.SCRAPE_WORKER_MAX_OLD_SPACE_SIZE || process.execArgv
.find(arg => arg.startsWith('--max-old-space-size='))
?.split('=')[1];
@@ -362,7 +362,12 @@ const separateWorkerFun = (
stalledInterval: 60 * 1000, // 60 seconds
maxStalledCount: 10, // 10 times
concurrency: 8,
- useWorkerThreads: true,
+ useWorkerThreads: false,
+ workerForkOptions: {
+ execArgv: filteredExecArgv.concat(maxOldSpaceSize ? (
+ ['--max-old-space-size=' + maxOldSpaceSize]
+ ) : []),
+ },
workerThreadsOptions: {
execArgv: filteredExecArgv,
resourceLimits: maxOldSpaceSize ? {