Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
remove extract logic from queue-worker
Gergő Móricz06e6d6bb716e
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 52472c198..695cbcfe7 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -343,6 +343,17 @@ function startServices(command?: string[]): Services {},);+ const extractWorker = execForward(+ "extract-worker",+ process.argv[2] === "--start-docker"+ ? "node --import ./dist/src/otel.js dist/src/services/extract-worker.js"+ : "pnpm extract-worker:production",+ {+ NUQ_REDUCE_NOISE: "true",+ NUQ_POD_NAME: "extract-worker",+ },+ );+const nuqWorkers = Array.from({ length: 5 }, (_, i) =>execForward(`nuq-worker-${i}`,diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 150187eef..458e18b8f 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -289,7 +289,7 @@ app.get("/liveness", (req, res) => {}});-const workerPort = process.env.WORKER_PORT || process.env.PORT || 3005;+const workerPort = process.env.EXTRACT_WORKER_PORT || process.env.PORT || 3005;app.listen(workerPort, () => {_logger.info(`Liveness endpoint is running on port ${workerPort}`);});diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 37dee72d8..b409e5554 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -467,7 +467,6 @@ app.listen(workerPort, () => {});await Promise.all([- workerFun(getExtractQueue(), processExtractJobInternal),workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),]);