Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

use separate redis connections for polling

Gergő Móricze698628ca7bf
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index e7e37b1a4..6ebf2a24c 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -1,4 +1,4 @@
-import { getScrapeQueue } from "./queue-service";
+import { createRedisConnection, getScrapeQueue } from "./queue-service";
import { v4 as uuidv4 } from "uuid";
import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";
import * as Sentry from "@sentry/node";
@@ -390,18 +390,20 @@ export function waitForJob(
): Promise<Document> {
return new Promise((resolve, reject) => {
const start = Date.now();
+ const conn = createRedisConnection();
+ const queue = getScrapeQueue(conn);
const int = setInterval(async () => {
logger.debug("WaitforJob ran", { scrapeId: jobId, jobId });
if (Date.now() >= start + timeout) {
clearInterval(int);
reject(new Error("Job wait "));
} else {
- const state = await getScrapeQueue().getJobState(jobId);
+ const state = await queue.getJobState(jobId);
logger.debug("Job in state", { state, scrapeId: jobId, jobId });
if (state === "completed") {
clearInterval(int);
let doc: Document;
- const job = (await getScrapeQueue().getJob(jobId))!;
+ const job = (await queue.getJob(jobId))!;
doc = job.returnvalue;
if (!doc) {
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index c4c0c3e71..32ae88c49 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -13,9 +13,13 @@ let generateLlmsTxtQueue: Queue;
let billingQueue: Queue;
let precrawlQueue: Queue;
-export const redisConnection = new IORedis(process.env.REDIS_URL!, {
- maxRetriesPerRequest: null,
-});
+export function createRedisConnection(): IORedis {
+ return new IORedis(process.env.REDIS_URL!, {
+ maxRetriesPerRequest: null,
+ });
+}
+
+export const redisConnection = createRedisConnection();
export const scrapeQueueName = "{scrapeQueue}";
export const extractQueueName = "{extractQueue}";
@@ -26,21 +30,26 @@ export const deepResearchQueueName = "{deepResearchQueue}";
export const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
-export function getScrapeQueue() {
- if (!scrapeQueue) {
- scrapeQueue = new Queue(scrapeQueueName, {
- connection: redisConnection,
- defaultJobOptions: {
- removeOnComplete: {
- age: 3600, // 1 hour
- },
- removeOnFail: {
- age: 3600, // 1 hour
- },
+export function getScrapeQueue(_redisConnection?: IORedis) {
+ const _scrapeQueue = !scrapeQueue || _redisConnection !== undefined ? new Queue(scrapeQueueName, {
+ connection: _redisConnection ?? redisConnection,
+ defaultJobOptions: {
+ removeOnComplete: {
+ age: 3600, // 1 hour
},
- });
+ removeOnFail: {
+ age: 3600, // 1 hour
+ },
+ },
+ }) : undefined;
+
+ if (_redisConnection !== undefined) {
+ return _scrapeQueue!;
+ } else if (!scrapeQueue) {
+ scrapeQueue = _scrapeQueue!;
logger.info("Web scraper queue created");
}
+
return scrapeQueue;
}