Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: handle --max-old-space-size flag for worker threads (#1922)
Josh-M4275e3cd554c46
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 1e9e2fb36..997e54b77 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -332,6 +332,15 @@ const separateWorkerFun = (queue: Queue,path: string,): Worker => {+ // Extract memory size from --max-old-space-size flag if present+ const maxOldSpaceSize = process.execArgv+ .find(arg => arg.startsWith('--max-old-space-size='))+ ?.split('=')[1];++ // Filter out the invalid flag for worker threads+ const filteredExecArgv = process.execArgv+ .filter(arg => !arg.startsWith('--max-old-space-size'));+const worker = new Worker(queue.name, path, {connection: createRedisConnection(),lockDuration: 30 * 1000, // 30 seconds@@ -339,6 +348,12 @@ const separateWorkerFun = (maxStalledCount: 10, // 10 timesconcurrency: 10,useWorkerThreads: true,+ workerThreadsOptions: {+ execArgv: filteredExecArgv,+ resourceLimits: maxOldSpaceSize ? {+ maxOldGenerationSizeMb: parseInt(maxOldSpaceSize)+ } : undefined+ }});return worker;