Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): separate redis connections for scrape Queue, QueueEvents, and Worker
Gergő Móricz9f0df422a172
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex ab184efb6..4ba840a91 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -14,12 +14,18 @@ let generateLlmsTxtQueue: Queue;let billingQueue: Queue;let precrawlQueue: Queue;-export const redisConnection = new IORedis(process.env.REDIS_URL!, {- maxRetriesPerRequest: null,-});+export function createRedisConnection() {+ const connection = new IORedis(process.env.REDIS_URL!, {+ maxRetriesPerRequest: null,+ });-redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));-redisConnection.on("error", (err) => logger.warn("Redis error", { err }));+ connection.on("reconnecting", () => logger.warn("Redis reconnecting"));+ connection.on("error", (err) => logger.warn("Redis error", { err }));++ return connection;+}++export const redisConnection = createRedisConnection();export const scrapeQueueName = "{scrapeQueue}";export const extractQueueName = "{extractQueue}";@@ -33,7 +39,7 @@ export const precrawlQueueName = "{precrawlQueue}";export function getScrapeQueue() {if (!scrapeQueue) {scrapeQueue = new Queue(scrapeQueueName, {- connection: redisConnection,+ connection: createRedisConnection(),defaultJobOptions: {removeOnComplete: {age: 3600, // 1 hour@@ -50,9 +56,7 @@ export function getScrapeQueue() {export function getScrapeQueueEvents() {if (!scrapeQueueEvents) {scrapeQueueEvents = new QueueEvents(scrapeQueueName, {- connection: new IORedis(process.env.REDIS_URL!, {- maxRetriesPerRequest: null,- }),+ connection: createRedisConnection(),});}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex ce3abece4..d68659daa 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -8,6 +8,7 @@ import {redisConnection,getGenerateLlmsTxtQueue,scrapeQueueName,+ createRedisConnection,} from "./queue-service";import { Job, Queue, QueueEvents } from "bullmq";import { logger as _logger } from "../lib/logger";@@ -332,7 +333,7 @@ const separateWorkerFun = (path: string,): Worker => {const worker = new Worker(queue.name, path, {- connection: redisConnection,+ connection: createRedisConnection(),lockDuration: 30 * 1000, // 30 secondsstalledInterval: 30 * 1000, // 30 secondsmaxStalledCount: 10, // 10 times