Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
use separate redis connections for polling
Gergő Móricze698628ca7bf
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex e7e37b1a4..6ebf2a24c 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,4 +1,4 @@-import { getScrapeQueue } from "./queue-service";+import { createRedisConnection, getScrapeQueue } from "./queue-service";import { v4 as uuidv4 } from "uuid";import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";import * as Sentry from "@sentry/node";@@ -390,18 +390,20 @@ export function waitForJob(): Promise<Document> {return new Promise((resolve, reject) => {const start = Date.now();+ const conn = createRedisConnection();+ const queue = getScrapeQueue(conn);const int = setInterval(async () => {logger.debug("WaitforJob ran", { scrapeId: jobId, jobId });if (Date.now() >= start + timeout) {clearInterval(int);reject(new Error("Job wait "));} else {- const state = await getScrapeQueue().getJobState(jobId);+ const state = await queue.getJobState(jobId);logger.debug("Job in state", { state, scrapeId: jobId, jobId });if (state === "completed") {clearInterval(int);let doc: Document;- const job = (await getScrapeQueue().getJob(jobId))!;+ const job = (await queue.getJob(jobId))!;doc = job.returnvalue;if (!doc) {diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex c4c0c3e71..32ae88c49 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -13,9 +13,13 @@ let generateLlmsTxtQueue: Queue;let billingQueue: Queue;let precrawlQueue: Queue;-export const redisConnection = new IORedis(process.env.REDIS_URL!, {- maxRetriesPerRequest: null,-});+export function createRedisConnection(): IORedis {+ return new IORedis(process.env.REDIS_URL!, {+ maxRetriesPerRequest: null,+ });+}++export const redisConnection = createRedisConnection();export const scrapeQueueName = "{scrapeQueue}";export const extractQueueName = "{extractQueue}";@@ -26,21 +30,26 @@ export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export function getScrapeQueue() {- if (!scrapeQueue) {- scrapeQueue = new Queue(scrapeQueueName, {- connection: redisConnection,- defaultJobOptions: {- removeOnComplete: {- age: 3600, // 1 hour- },- removeOnFail: {- age: 3600, // 1 hour- },+export function getScrapeQueue(_redisConnection?: IORedis) {+ const _scrapeQueue = !scrapeQueue || _redisConnection !== undefined ? new Queue(scrapeQueueName, {+ connection: _redisConnection ?? redisConnection,+ defaultJobOptions: {+ removeOnComplete: {+ age: 3600, // 1 hour},- });+ removeOnFail: {+ age: 3600, // 1 hour+ },+ },+ }) : undefined;++ if (_redisConnection !== undefined) {+ return _scrapeQueue!;+ } else if (!scrapeQueue) {+ scrapeQueue = _scrapeQueue!;logger.info("Web scraper queue created");}+return scrapeQueue;}