Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/prefetch-worker): improvements and heartbeat feature

Gergő Móricz1ade516a50ec
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 13f084dc0..a9f51bb22 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -195,6 +195,8 @@ const configSchema = z.object({
EXTRACT_V3_BETA_URL: z.string().optional(),
AGENT_INTEROP_SECRET: z.string().optional(),
+
+ NUQ_PREFETCH_WORKER_HEARTBEAT_URL: z.string().optional(),
});
export const config = configSchema.parse(process.env);
diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.ts
index 2a590e2f2..1d9126a3c 100644
--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts
+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts
@@ -43,15 +43,29 @@ import { logger } from "../../lib/logger";
process.on("SIGTERM", shutdown);
}
- (async () => {
- while (true) {
- await crawlFinishedQueue.prefetchJobs();
- await new Promise(resolve => setTimeout(resolve, 250));
- }
- })();
-
- while (true) {
- await scrapeQueue.prefetchJobs();
- await new Promise(resolve => setTimeout(resolve, 250));
+ try {
+ await Promise.all([
+ (async () => {
+ while (true) {
+ await crawlFinishedQueue.prefetchJobs();
+ await new Promise(resolve => setTimeout(resolve, 250));
+ }
+ })(),
+ (async () => {
+ while (true) {
+ if (config.NUQ_PREFETCH_WORKER_HEARTBEAT_URL) {
+ fetch(config.NUQ_PREFETCH_WORKER_HEARTBEAT_URL).catch(() => {});
+ }
+ await scrapeQueue.prefetchJobs();
+ await new Promise(resolve => setTimeout(resolve, 250));
+ }
+ })(),
+ ]);
+ } catch (error) {
+ logger.error("Error in prefetch worker", { error });
+ process.exit(1);
}
+
+ logger.info("All prefetch workers exited. Shutting down...");
+ await shutdown();
})();