Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-service): reduce redis connections to BullMQ (#1966)
Gergő Móricz308e4f436803
diff --git a/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts b/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.tsindex 2f2ff7359..0ffee0d44 100644--- a/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts+++ b/apps/api/src/__tests__/deep-research/unit/deep-research-redis.test.ts@@ -1,4 +1,4 @@-import { redisConnection } from "../../../services/queue-service";+import { redisEvictConnection } from "../../../services/redis";import {saveDeepResearch,getDeepResearch,@@ -40,11 +40,11 @@ describe("Deep Research Redis Operations", () => {it("should save research data to Redis with TTL", async () => {await saveDeepResearch("test-id", mockResearch);- expect(redisConnection.set).toHaveBeenCalledWith(+ expect(redisEvictConnection.set).toHaveBeenCalledWith("deep-research:test-id",JSON.stringify(mockResearch));- expect(redisConnection.expire).toHaveBeenCalledWith(+ expect(redisEvictConnection.expire).toHaveBeenCalledWith("deep-research:test-id",6 * 60 * 60);@@ -53,17 +53,17 @@ describe("Deep Research Redis Operations", () => {describe("getDeepResearch", () => {it("should retrieve research data from Redis", async () => {- (redisConnection.get as jest.Mock).mockResolvedValue(+ (redisEvictConnection.get as jest.Mock).mockResolvedValue(JSON.stringify(mockResearch));const result = await getDeepResearch("test-id");expect(result).toEqual(mockResearch);- expect(redisConnection.get).toHaveBeenCalledWith("deep-research:test-id");+ expect(redisEvictConnection.get).toHaveBeenCalledWith("deep-research:test-id");});it("should return null when research not found", async () => {- (redisConnection.get as jest.Mock).mockResolvedValue(null);+ (redisEvictConnection.get as jest.Mock).mockResolvedValue(null);const result = await getDeepResearch("non-existent-id");expect(result).toBeNull();@@ -72,7 +72,7 @@ describe("Deep Research Redis Operations", () => {describe("updateDeepResearch", () => {it("should update existing research with new data", async () => {- (redisConnection.get as jest.Mock).mockResolvedValue(+ (redisEvictConnection.get as jest.Mock).mockResolvedValue(JSON.stringify(mockResearch));@@ -98,30 +98,30 @@ describe("Deep Research Redis Operations", () => {activities: [...mockResearch.activities, ...update.activities],};- expect(redisConnection.set).toHaveBeenCalledWith(+ expect(redisEvictConnection.set).toHaveBeenCalledWith("deep-research:test-id",JSON.stringify(expectedUpdate));- expect(redisConnection.expire).toHaveBeenCalledWith(+ expect(redisEvictConnection.expire).toHaveBeenCalledWith("deep-research:test-id",6 * 60 * 60);});it("should do nothing if research not found", async () => {- (redisConnection.get as jest.Mock).mockResolvedValue(null);+ (redisEvictConnection.get as jest.Mock).mockResolvedValue(null);await updateDeepResearch("test-id", { status: "completed" });- expect(redisConnection.set).not.toHaveBeenCalled();- expect(redisConnection.expire).not.toHaveBeenCalled();+ expect(redisEvictConnection.set).not.toHaveBeenCalled();+ expect(redisEvictConnection.expire).not.toHaveBeenCalled();});});describe("getDeepResearchExpiry", () => {it("should return correct expiry date", async () => {const mockTTL = 3600000; // 1 hour in milliseconds- (redisConnection.pttl as jest.Mock).mockResolvedValue(mockTTL);+ (redisEvictConnection.pttl as jest.Mock).mockResolvedValue(mockTTL);const result = await getDeepResearchExpiry("test-id");diff --git a/apps/api/src/__tests__/queue-concurrency-integration.test.ts b/apps/api/src/__tests__/queue-concurrency-integration.test.tsindex c2b75befd..c60fe7276 100644--- a/apps/api/src/__tests__/queue-concurrency-integration.test.ts+++ b/apps/api/src/__tests__/queue-concurrency-integration.test.ts@@ -1,4 +1,4 @@-import { redisConnection } from "../services/queue-service";+import { redisEvictConnection } from "../services/redis";import { addScrapeJob, addScrapeJobs } from "../services/queue-jobs";import {cleanOldConcurrencyLimitEntries,@@ -85,18 +85,18 @@ describe("Queue Concurrency Integration", () => {it("should add job directly to BullMQ when under concurrency limit", async () => {// Mock current active jobs to be under limit- (redisConnection.zrangebyscore as jest.Mock).mockResolvedValue([]);+ (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue([]);await addScrapeJob(mockWebScraperOptions);// Should have checked concurrency- expect(redisConnection.zrangebyscore).toHaveBeenCalled();+ expect(redisEvictConnection.zrangebyscore).toHaveBeenCalled();// Should have added to BullMQexpect(mockAdd).toHaveBeenCalled();// Should have added to active jobs- expect(redisConnection.zadd).toHaveBeenCalledWith(+ expect(redisEvictConnection.zadd).toHaveBeenCalledWith(expect.stringContaining("concurrency-limiter"),expect.any(Number),expect.any(String),@@ -109,20 +109,20 @@ describe("Queue Concurrency Integration", () => {concurrency: 15,} as any);const activeJobs = Array(15).fill("active-job");- (redisConnection.zrangebyscore as jest.Mock).mockResolvedValue(+ (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue(activeJobs,);await addScrapeJob(mockWebScraperOptions);// Should have checked concurrency- expect(redisConnection.zrangebyscore).toHaveBeenCalled();+ expect(redisEvictConnection.zrangebyscore).toHaveBeenCalled();// Should NOT have added to BullMQexpect(mockAdd).not.toHaveBeenCalled();// Should have added to concurrency queue- expect(redisConnection.zadd).toHaveBeenCalledWith(+ expect(redisEvictConnection.zadd).toHaveBeenCalledWith(expect.stringContaining("concurrency-limit-queue"),expect.any(Number),expect.stringContaining("mock-uuid"),@@ -157,7 +157,7 @@ describe("Queue Concurrency Integration", () => {const mockJobs = createMockJobs(totalJobs);// Mock current active jobs to be empty- (redisConnection.zrangebyscore as jest.Mock).mockResolvedValue([]);+ (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValue([]);await addScrapeJobs(mockJobs);@@ -165,7 +165,7 @@ describe("Queue Concurrency Integration", () => {expect(mockAdd).toHaveBeenCalledTimes(maxConcurrency);// Should have added remaining jobs to concurrency queue- expect(redisConnection.zadd).toHaveBeenCalledWith(+ expect(redisEvictConnection.zadd).toHaveBeenCalledWith(expect.stringContaining("concurrency-limit-queue"),expect.any(Number),expect.any(String),@@ -176,7 +176,7 @@ describe("Queue Concurrency Integration", () => {const result = await addScrapeJobs([]);expect(result).toBe(true);expect(mockAdd).not.toHaveBeenCalled();- expect(redisConnection.zadd).not.toHaveBeenCalled();+ expect(redisEvictConnection.zadd).not.toHaveBeenCalled();});});@@ -196,7 +196,7 @@ describe("Queue Concurrency Integration", () => {data: { test: "data" },opts: {},};- (redisConnection.zmpop as jest.Mock).mockResolvedValueOnce([+ (redisEvictConnection.zmpop as jest.Mock).mockResolvedValueOnce(["key",[[JSON.stringify(queuedJob)]],]);@@ -212,7 +212,7 @@ describe("Queue Concurrency Integration", () => {// Should have added new job to active jobsawait pushConcurrencyLimitActiveJob(mockTeamId, nextJob!.id, 2 * 60 * 1000);- expect(redisConnection.zadd).toHaveBeenCalledWith(+ expect(redisEvictConnection.zadd).toHaveBeenCalledWith(expect.stringContaining("concurrency-limiter"),expect.any(Number),nextJob!.id,@@ -235,7 +235,7 @@ describe("Queue Concurrency Integration", () => {await cleanOldConcurrencyLimitEntries(mockTeamId);// Verify job was removed from active jobs- expect(redisConnection.zrem).toHaveBeenCalledWith(+ expect(redisEvictConnection.zrem).toHaveBeenCalledWith(expect.stringContaining("concurrency-limiter"),mockJob.id,);@@ -247,14 +247,14 @@ describe("Queue Concurrency Integration", () => {const stalledTime = mockNow - 3 * 60 * 1000; // 3 minutes ago// Mock stalled jobs in Redis- (redisConnection.zrangebyscore as jest.Mock).mockResolvedValueOnce([+ (redisEvictConnection.zrangebyscore as jest.Mock).mockResolvedValueOnce(["stalled-job",]);await cleanOldConcurrencyLimitEntries(mockTeamId, mockNow);// Should have cleaned up stalled jobs- expect(redisConnection.zremrangebyscore).toHaveBeenCalledWith(+ expect(redisEvictConnection.zremrangebyscore).toHaveBeenCalledWith(expect.stringContaining("concurrency-limiter"),-Infinity,mockNow,@@ -263,7 +263,7 @@ describe("Queue Concurrency Integration", () => {it("should handle race conditions in job queue processing", async () => {// Mock a race condition where job is taken by another worker- (redisConnection.zmpop as jest.Mock).mockResolvedValueOnce(null);+ (redisEvictConnection.zmpop as jest.Mock).mockResolvedValueOnce(null);const nextJob = await takeConcurrencyLimitedJob(mockTeamId);diff --git a/apps/api/src/lib/__tests__/job-priority.test.ts b/apps/api/src/lib/__tests__/job-priority.test.tsindex 118f3fe21..5e1605813 100644--- a/apps/api/src/lib/__tests__/job-priority.test.ts+++ b/apps/api/src/lib/__tests__/job-priority.test.ts@@ -3,7 +3,7 @@ import {addJobPriority,deleteJobPriority,} from "../job-priority";-import { redisConnection } from "../../services/queue-service";+import { redisEvictConnection } from "../../services/redis";import { } from "../../types";jest.mock("../../services/queue-service", () => ({@@ -24,11 +24,11 @@ describe("Job Priority Tests", () => {const team_id = "team1";const job_id = "job1";await addJobPriority(team_id, job_id);- expect(redisConnection.sadd).toHaveBeenCalledWith(+ expect(redisEvictConnection.sadd).toHaveBeenCalledWith(`limit_team_id:${team_id}`,job_id,);- expect(redisConnection.expire).toHaveBeenCalledWith(+ expect(redisEvictConnection.expire).toHaveBeenCalledWith(`limit_team_id:${team_id}`,60,);@@ -38,7 +38,7 @@ describe("Job Priority Tests", () => {const team_id = "team1";const job_id = "job1";await deleteJobPriority(team_id, job_id);- expect(redisConnection.srem).toHaveBeenCalledWith(+ expect(redisEvictConnection.srem).toHaveBeenCalledWith(`limit_team_id:${team_id}`,job_id,);@@ -47,12 +47,12 @@ describe("Job Priority Tests", () => {test("getJobPriority should return correct priority based on plan and set length", async () => {const team_id = "team1";const plan = "standard";- (redisConnection.scard as jest.Mock).mockResolvedValue(150);+ (redisEvictConnection.scard as jest.Mock).mockResolvedValue(150);const priority = await getJobPriority({ team_id });expect(priority).toBe(10);- (redisConnection.scard as jest.Mock).mockResolvedValue(250);+ (redisEvictConnection.scard as jest.Mock).mockResolvedValue(250);const priorityExceeded = await getJobPriority({ team_id });expect(priorityExceeded).toBe(20); // basePriority + Math.ceil((250 - 200) * 0.4)});@@ -60,22 +60,22 @@ describe("Job Priority Tests", () => {test("getJobPriority should handle different plans correctly", async () => {const team_id = "team1";- (redisConnection.scard as jest.Mock).mockResolvedValue(50);+ (redisEvictConnection.scard as jest.Mock).mockResolvedValue(50);let plan = "hobby";let priority = await getJobPriority({ team_id });expect(priority).toBe(10);- (redisConnection.scard as jest.Mock).mockResolvedValue(150);+ (redisEvictConnection.scard as jest.Mock).mockResolvedValue(150);plan = "hobby";priority = await getJobPriority({ team_id });expect(priority).toBe(25); // basePriority + Math.ceil((150 - 50) * 0.3)- (redisConnection.scard as jest.Mock).mockResolvedValue(25);+ (redisEvictConnection.scard as jest.Mock).mockResolvedValue(25);plan = "free";priority = await getJobPriority({ team_id });expect(priority).toBe(10);- (redisConnection.scard as jest.Mock).mockResolvedValue(60);+ (redisEvictConnection.scard as jest.Mock).mockResolvedValue(60);plan = "free";priority = await getJobPriority({ team_id });expect(priority).toBe(28); // basePriority + Math.ceil((60 - 25) * 0.5)@@ -87,17 +87,17 @@ describe("Job Priority Tests", () => {const job_id2 = "job2";await addJobPriority(team_id, job_id1);- expect(redisConnection.expire).toHaveBeenCalledWith(+ expect(redisEvictConnection.expire).toHaveBeenCalledWith(`limit_team_id:${team_id}`,60,);// Clear the mock calls- (redisConnection.expire as jest.Mock).mockClear();+ (redisEvictConnection.expire as jest.Mock).mockClear();// Add another jobawait addJobPriority(team_id, job_id2);- expect(redisConnection.expire).toHaveBeenCalledWith(+ expect(redisEvictConnection.expire).toHaveBeenCalledWith(`limit_team_id:${team_id}`,60,);@@ -110,7 +110,7 @@ describe("Job Priority Tests", () => {jest.useFakeTimers();await addJobPriority(team_id, job_id);- expect(redisConnection.expire).toHaveBeenCalledWith(+ expect(redisEvictConnection.expire).toHaveBeenCalledWith(`limit_team_id:${team_id}`,60,);@@ -119,14 +119,14 @@ describe("Job Priority Tests", () => {jest.advanceTimersByTime(59000);// The set should still exist- expect(redisConnection.scard).not.toHaveBeenCalled();+ expect(redisEvictConnection.scard).not.toHaveBeenCalled();// Fast-forward time by 2 more seconds (total 61 seconds)jest.advanceTimersByTime(2000);// Check if the set has been removed (scard should return 0)- (redisConnection.scard as jest.Mock).mockResolvedValue(0);- const setSize = await redisConnection.scard(`limit_team_id:${team_id}`);+ (redisEvictConnection.scard as jest.Mock).mockResolvedValue(0);+ const setSize = await redisEvictConnection.scard(`limit_team_id:${team_id}`);expect(setSize).toBe(0);jest.useRealTimers();diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex 1a13e8e0a..679bf844e 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -1,5 +1,5 @@import { logger } from "../../lib/logger";-import { redisConnection } from "../queue-service";+import { getRedisConnection } from "../queue-service";import { supabase_service } from "../supabase";import * as Sentry from "@sentry/node";import { Queue } from "bullmq";@@ -33,7 +33,7 @@ interface GroupedBillingOperation {// Function to acquire a lock for batch processingasync function acquireLock(): Promise<boolean> {- const redis = redisConnection;+ const redis = getRedisConnection();// Set lock with NX (only if it doesn't exist) and PX (millisecond expiry)const result = await redis.set(BATCH_LOCK_KEY, "1", "PX", LOCK_TIMEOUT, "NX");const acquired = result === "OK";@@ -45,14 +45,14 @@ async function acquireLock(): Promise<boolean> {// Function to release the lockasync function releaseLock() {- const redis = redisConnection;+ const redis = getRedisConnection();await redis.del(BATCH_LOCK_KEY);logger.info("🔓 Released billing batch processing lock");}// Main function to process the billing batchexport async function processBillingBatch() {- const redis = redisConnection;+ const redis = getRedisConnection();// Try to acquire lockif (!(await acquireLock())) {@@ -156,7 +156,7 @@ export function startBillingBatchProcessing() {logger.info("🔄 Starting periodic billing batch processing");batchInterval = setInterval(async () => {- const queueLength = await redisConnection.llen(BATCH_KEY);+ const queueLength = await getRedisConnection().llen(BATCH_KEY);logger.info(`Checking billing batch queue (${queueLength} items pending)`);await processBillingBatch();}, BATCH_TIMEOUT);@@ -195,9 +195,9 @@ export async function queueBillingOperation(};// Add operation to Redis list- const redis = redisConnection;+ const redis = getRedisConnection();await redis.rpush(BATCH_KEY, JSON.stringify(operation));- const queueLength = await redis.llen(BATCH_KEY);+ const queueLength = await getRedisConnection().llen(BATCH_KEY);logger.info(`📥 Added billing operation to queue (${queueLength} total pending)`, {team_id,creditsdiff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex f39fef2a7..2cf3dd41e 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -4,7 +4,7 @@ import * as Sentry from "@sentry/node";import { Job, Queue, Worker } from "bullmq";import { logger as _logger, logger } from "../../lib/logger";import {- redisConnection,+ getRedisConnection,getBillingQueue,getPrecrawlQueue,precrawlQueueName,@@ -218,7 +218,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =const logger = _logger.child({ module: "index-worker", method: "workerFun" });const worker = new Worker(queue.name, null, {- connection: redisConnection,+ connection: getRedisConnection(),lockDuration: workerLockDuration,stalledInterval: workerStalledCheckInterval,maxStalledCount: queue.name === precrawlQueueName ? 0 : 10,diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 4ba840a91..76c5e4a66 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -13,20 +13,21 @@ let deepResearchQueue: Queue;let generateLlmsTxtQueue: Queue;let billingQueue: Queue;let precrawlQueue: Queue;+let redisConnection: IORedis;-export function createRedisConnection() {- const connection = new IORedis(process.env.REDIS_URL!, {- maxRetriesPerRequest: null,- });-- connection.on("reconnecting", () => logger.warn("Redis reconnecting"));- connection.on("error", (err) => logger.warn("Redis error", { err }));+export function getRedisConnection(): IORedis {+ if (!redisConnection) {+ redisConnection = new IORedis(process.env.REDIS_URL!, {+ maxRetriesPerRequest: null,+ });+ redisConnection.on("connect", () => logger.info("Redis connected"));+ redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));+ redisConnection.on("error", (err) => logger.warn("Redis error", { err }));- return connection;+ }+ return redisConnection;}-export const redisConnection = createRedisConnection();-export const scrapeQueueName = "{scrapeQueue}";export const extractQueueName = "{extractQueue}";export const loggingQueueName = "{loggingQueue}";@@ -39,7 +40,7 @@ export const precrawlQueueName = "{precrawlQueue}";export function getScrapeQueue() {if (!scrapeQueue) {scrapeQueue = new Queue(scrapeQueueName, {- connection: createRedisConnection(),+ connection: getRedisConnection(),defaultJobOptions: {removeOnComplete: {age: 3600, // 1 hour@@ -56,7 +57,7 @@ export function getScrapeQueue() {export function getScrapeQueueEvents() {if (!scrapeQueueEvents) {scrapeQueueEvents = new QueueEvents(scrapeQueueName, {- connection: createRedisConnection(),+ connection: getRedisConnection(),});}@@ -66,7 +67,7 @@ export function getScrapeQueueEvents() {export function getExtractQueue() {if (!extractQueue) {extractQueue = new Queue(extractQueueName, {- connection: redisConnection,+ connection: getRedisConnection(),defaultJobOptions: {removeOnComplete: {age: 90000, // 25 hours@@ -83,7 +84,7 @@ export function getExtractQueue() {export function getGenerateLlmsTxtQueue() {if (!generateLlmsTxtQueue) {generateLlmsTxtQueue = new Queue(generateLlmsTxtQueueName, {- connection: redisConnection,+ connection: getRedisConnection(),defaultJobOptions: {removeOnComplete: {age: 90000, // 25 hours@@ -100,7 +101,7 @@ export function getGenerateLlmsTxtQueue() {export function getDeepResearchQueue() {if (!deepResearchQueue) {deepResearchQueue = new Queue(deepResearchQueueName, {- connection: redisConnection,+ connection: getRedisConnection(),defaultJobOptions: {removeOnComplete: {age: 90000, // 25 hours@@ -117,7 +118,7 @@ export function getDeepResearchQueue() {export function getBillingQueue() {if (!billingQueue) {billingQueue = new Queue(billingQueueName, {- connection: redisConnection,+ connection: getRedisConnection(),defaultJobOptions: {removeOnComplete: {age: 60, // 1 minute@@ -134,7 +135,7 @@ export function getBillingQueue() {export function getPrecrawlQueue() {if (!precrawlQueue) {precrawlQueue = new Queue(precrawlQueueName, {- connection: redisConnection,+ connection: getRedisConnection(),defaultJobOptions: {removeOnComplete: {age: 24 * 60 * 60, // 1 daydiff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 28fd8a25b..fdb4b111a 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -5,10 +5,9 @@ import {getScrapeQueue,getExtractQueue,getDeepResearchQueue,- redisConnection,getGenerateLlmsTxtQueue,scrapeQueueName,- createRedisConnection,+ getRedisConnection,} from "./queue-service";import { Job, Queue, QueueEvents } from "bullmq";import { logger as _logger } from "../lib/logger";@@ -357,7 +356,7 @@ const separateWorkerFun = (.filter(arg => !arg.startsWith('--max-old-space-size'));const worker = new Worker(queue.name, path, {- connection: createRedisConnection(),+ connection: getRedisConnection(),lockDuration: 60 * 1000, // 60 secondsstalledInterval: 60 * 1000, // 60 secondsmaxStalledCount: 10, // 10 times@@ -386,7 +385,7 @@ const workerFun = async (const logger = _logger.child({ module: "queue-worker", method: "workerFun" });const worker = new Worker(queue.name, null, {- connection: redisConnection,+ connection: getRedisConnection(),lockDuration: 60 * 1000, // 60 secondsstalledInterval: 60 * 1000, // 60 secondsmaxStalledCount: 10, // 10 times@@ -534,7 +533,7 @@ app.listen(workerPort, () => {}}- const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: redisConnection });+ const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: getRedisConnection() });scrapeQueueEvents.on("failed", failedListener);const results = await Promise.all([