Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

remove extract logic from queue-worker

Gergő Móricz06e6d6bb716e
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 52472c198..695cbcfe7 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -343,6 +343,17 @@ function startServices(command?: string[]): Services {
},
);
+ const extractWorker = execForward(
+ "extract-worker",
+ process.argv[2] === "--start-docker"
+ ? "node --import ./dist/src/otel.js dist/src/services/extract-worker.js"
+ : "pnpm extract-worker:production",
+ {
+ NUQ_REDUCE_NOISE: "true",
+ NUQ_POD_NAME: "extract-worker",
+ },
+ );
+
const nuqWorkers = Array.from({ length: 5 }, (_, i) =>
execForward(
`nuq-worker-${i}`,
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index 150187eef..458e18b8f 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -289,7 +289,7 @@ app.get("/liveness", (req, res) => {
}
});
-const workerPort = process.env.WORKER_PORT || process.env.PORT || 3005;
+const workerPort = process.env.EXTRACT_WORKER_PORT || process.env.PORT || 3005;
app.listen(workerPort, () => {
_logger.info(`Liveness endpoint is running on port ${workerPort}`);
});
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 37dee72d8..b409e5554 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -467,7 +467,6 @@ app.listen(workerPort, () => {
});
await Promise.all([
- workerFun(getExtractQueue(), processExtractJobInternal),
workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),
workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),
]);