Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): separate redis connections for scrape Queue, QueueEvents, and Worker

Gergő Móricz9f0df422a172
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index ab184efb6..4ba840a91 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -14,12 +14,18 @@ let generateLlmsTxtQueue: Queue;
let billingQueue: Queue;
let precrawlQueue: Queue;
-export const redisConnection = new IORedis(process.env.REDIS_URL!, {
- maxRetriesPerRequest: null,
-});
+export function createRedisConnection() {
+ const connection = new IORedis(process.env.REDIS_URL!, {
+ maxRetriesPerRequest: null,
+ });
-redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));
-redisConnection.on("error", (err) => logger.warn("Redis error", { err }));
+ connection.on("reconnecting", () => logger.warn("Redis reconnecting"));
+ connection.on("error", (err) => logger.warn("Redis error", { err }));
+
+ return connection;
+}
+
+export const redisConnection = createRedisConnection();
export const scrapeQueueName = "{scrapeQueue}";
export const extractQueueName = "{extractQueue}";
@@ -33,7 +39,7 @@ export const precrawlQueueName = "{precrawlQueue}";
export function getScrapeQueue() {
if (!scrapeQueue) {
scrapeQueue = new Queue(scrapeQueueName, {
- connection: redisConnection,
+ connection: createRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
age: 3600, // 1 hour
@@ -50,9 +56,7 @@ export function getScrapeQueue() {
export function getScrapeQueueEvents() {
if (!scrapeQueueEvents) {
scrapeQueueEvents = new QueueEvents(scrapeQueueName, {
- connection: new IORedis(process.env.REDIS_URL!, {
- maxRetriesPerRequest: null,
- }),
+ connection: createRedisConnection(),
});
}
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index ce3abece4..d68659daa 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -8,6 +8,7 @@ import {
redisConnection,
getGenerateLlmsTxtQueue,
scrapeQueueName,
+ createRedisConnection,
} from "./queue-service";
import { Job, Queue, QueueEvents } from "bullmq";
import { logger as _logger } from "../lib/logger";
@@ -332,7 +333,7 @@ const separateWorkerFun = (
path: string,
): Worker => {
const worker = new Worker(queue.name, path, {
- connection: redisConnection,
+ connection: createRedisConnection(),
lockDuration: 30 * 1000, // 30 seconds
stalledInterval: 30 * 1000, // 30 seconds
maxStalledCount: 10, // 10 times