Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker): turn off useWorkerThreads in sandboxed scrape worker (#1941)
Gergő Móriczbd5adfeae112
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 8375119c4..28fd8a25b 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -348,7 +348,7 @@ const separateWorkerFun = (path: string,): Worker => {// Extract memory size from --max-old-space-size flag if present- const maxOldSpaceSize = process.execArgv+ const maxOldSpaceSize = process.env.SCRAPE_WORKER_MAX_OLD_SPACE_SIZE || process.execArgv.find(arg => arg.startsWith('--max-old-space-size='))?.split('=')[1];@@ -362,7 +362,12 @@ const separateWorkerFun = (stalledInterval: 60 * 1000, // 60 secondsmaxStalledCount: 10, // 10 timesconcurrency: 8,- useWorkerThreads: true,+ useWorkerThreads: false,+ workerForkOptions: {+ execArgv: filteredExecArgv.concat(maxOldSpaceSize ? (+ ['--max-old-space-size=' + maxOldSpaceSize]+ ) : []),+ },workerThreadsOptions: {execArgv: filteredExecArgv,resourceLimits: maxOldSpaceSize ? {