Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/prefetch-worker): improvements and heartbeat feature
Gergő Móricz1ade516a50ec
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 13f084dc0..a9f51bb22 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -195,6 +195,8 @@ const configSchema = z.object({EXTRACT_V3_BETA_URL: z.string().optional(),AGENT_INTEROP_SECRET: z.string().optional(),++ NUQ_PREFETCH_WORKER_HEARTBEAT_URL: z.string().optional(),});export const config = configSchema.parse(process.env);diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.tsindex 2a590e2f2..1d9126a3c 100644--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts@@ -43,15 +43,29 @@ import { logger } from "../../lib/logger";process.on("SIGTERM", shutdown);}- (async () => {- while (true) {- await crawlFinishedQueue.prefetchJobs();- await new Promise(resolve => setTimeout(resolve, 250));- }- })();-- while (true) {- await scrapeQueue.prefetchJobs();- await new Promise(resolve => setTimeout(resolve, 250));+ try {+ await Promise.all([+ (async () => {+ while (true) {+ await crawlFinishedQueue.prefetchJobs();+ await new Promise(resolve => setTimeout(resolve, 250));+ }+ })(),+ (async () => {+ while (true) {+ if (config.NUQ_PREFETCH_WORKER_HEARTBEAT_URL) {+ fetch(config.NUQ_PREFETCH_WORKER_HEARTBEAT_URL).catch(() => {});+ }+ await scrapeQueue.prefetchJobs();+ await new Promise(resolve => setTimeout(resolve, 250));+ }+ })(),+ ]);+ } catch (error) {+ logger.error("Error in prefetch worker", { error });+ process.exit(1);}++ logger.info("All prefetch workers exited. Shutting down...");+ await shutdown();})();