Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Nick: fixes crawl stuck issue

Nicolasca8f1a328b9d
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.ts
index c54c5c9c2..d7d9c0897 100644
--- a/apps/api/src/controllers/v0/admin/queue.ts
+++ b/apps/api/src/controllers/v0/admin/queue.ts
@@ -2,7 +2,7 @@ import { Request, Response } from "express";
import { Job } from "bullmq";
import { logger } from "../../../lib/logger";
-import { createRedisConnection, getScrapeQueue } from "../../../services/queue-service";
+import { getScrapeQueue } from "../../../services/queue-service";
import { checkAlerts } from "../../../services/alerts";
import { sendSlackWebhook } from "../../../services/alerts/slack";
@@ -11,9 +11,8 @@ export async function cleanBefore24hCompleteJobsController(
res: Response,
) {
logger.info("πŸ‚ Cleaning jobs older than 24h");
- const conn = createRedisConnection();
try {
- const scrapeQueue = getScrapeQueue(conn);
+ const scrapeQueue = getScrapeQueue();
const batchSize = 10;
const numberOfBatches = 9; // Adjust based on your needs
const completedJobsPromises: Promise<Job[]>[] = [];
@@ -55,8 +54,6 @@ export async function cleanBefore24hCompleteJobsController(
} catch (error) {
logger.error(`πŸ‚ Failed to clean last 24h complete jobs: ${error}`);
return res.status(500).send("Failed to clean jobs");
- } finally {
- conn.disconnect();
}
}
@@ -72,9 +69,8 @@ export async function checkQueuesController(req: Request, res: Response) {
// Use this as a "health check" that way we dont destroy the server
export async function queuesController(req: Request, res: Response) {
- const conn = createRedisConnection();
try {
- const scrapeQueue = getScrapeQueue(conn);
+ const scrapeQueue = getScrapeQueue();
const [webScraperActive] = await Promise.all([
scrapeQueue.getActiveCount(),
@@ -89,18 +85,15 @@ export async function queuesController(req: Request, res: Response) {
} catch (error) {
logger.error(error);
return res.status(500).json({ error: error.message });
- } finally {
- conn.disconnect();
}
}
export async function autoscalerController(req: Request, res: Response) {
- const conn = createRedisConnection();
try {
const maxNumberOfMachines = 80;
const minNumberOfMachines = 20;
- const scrapeQueue = getScrapeQueue(conn);
+ const scrapeQueue = getScrapeQueue();
const [webScraperActive, webScraperWaiting, webScraperPriority] =
await Promise.all([
@@ -204,7 +197,5 @@ export async function autoscalerController(req: Request, res: Response) {
} catch (error) {
logger.error(error);
return res.status(500).send("Failed to initialize autoscaler");
- } finally {
- conn.disconnect();
}
}
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 559e14f11..a5ec432f1 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -21,7 +21,7 @@ import {
defaultOrigin,
} from "../../lib/default-values";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { v4 as uuidv4 } from "uuid";
import { logger } from "../../lib/logger";
@@ -139,9 +139,7 @@ export async function scrapeHelper(
return err;
}
- const conn = createRedisConnection();
- await getScrapeQueue(conn).remove(jobId);
- conn.disconnect();
+ await getScrapeQueue().remove(jobId);
if (!doc) {
console.error("!!! PANIC DOC IS", doc);
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index a41b548e7..aaff178c2 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -11,7 +11,7 @@ import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { v4 as uuidv4 } from "uuid";
import { logger } from "../../lib/logger";
-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import * as Sentry from "@sentry/node";
@@ -137,10 +137,8 @@ export async function searchHelper(
return { success: true, error: "No search results found", returnCode: 200 };
}
- const conn = createRedisConnection()
- const sq = getScrapeQueue(conn);
+ const sq = getScrapeQueue();
await Promise.all(jobDatas.map((x) => sq.remove(x.opts.jobId)));
- conn.disconnect();
// make sure doc.content is not empty
const filteredDocs = docs.filter(
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index 263553784..d8d029ee6 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -8,29 +8,23 @@ import {
getCrawl,
getCrawlJobs,
} from "../../lib/crawl-redis";
-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { configDotenv } from "dotenv";
import { Job } from "bullmq";
configDotenv();
export async function getJob(id: string) {
- const conn = createRedisConnection();
- const job = await getScrapeQueue(conn).getJob(id);
- conn.disconnect();
+ const job = await getScrapeQueue().getJob(id);
if (!job) return job;
return job;
}
export async function getJobs(ids: string[]) {
- const conn = createRedisConnection();
- const queue = getScrapeQueue(conn);
- const jobs: (Job & { id: string })[] = (await Promise.all(
- ids.map((x) => queue.getJob(x)),
- )).filter((x) => x) as (Job & { id: string })[];
-
- conn.disconnect();
+ const jobs: (Job & { id: string })[] = (
+ await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))
+ ).filter((x) => x) as (Job & { id: string })[];
return jobs;
}
@@ -48,17 +42,12 @@ export async function crawlErrorsController(
return res.status(403).json({ success: false, error: "Forbidden" });
}
- const conn = createRedisConnection();
- const queue = getScrapeQueue(conn);
-
let jobStatuses = await Promise.all(
(await getCrawlJobs(req.params.jobId)).map(
- async (x) => [x, await queue.getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
),
);
- conn.disconnect();
-
const failedJobIDs: string[] = [];
for (const [id, status] of jobStatuses) {
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 476dc9af3..cc4d6bf2b 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -105,9 +105,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- const conn = createRedisConnection();
- await getScrapeQueue(conn).remove(jobId);
- conn.disconnect();
+ await getScrapeQueue().remove(jobId);
}
if (
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index e1e32f673..82efbcfb3 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -15,7 +15,7 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
import { getJobPriority } from "../../lib/job-priority";
import { Mode } from "../../types";
-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue } from "../../services/queue-service";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
@@ -133,9 +133,7 @@ async function scrapeSearchResult(
teamId: options.teamId,
origin: options.origin,
});
- const conn = createRedisConnection();
- await getScrapeQueue(conn).remove(jobId);
- conn.disconnect();
+ await getScrapeQueue().remove(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 7b6117f57..b6fc1e3fa 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -26,7 +26,6 @@ import { v4 as uuidv4 } from "uuid";
import { RateLimiterMode } from "./types";
import { attachWsProxy } from "./services/agentLivecastWS";
import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
-import { createRedisConnection } from "./services/queue-service";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require('@bull-board/api/bullMQAdapter');
@@ -56,7 +55,7 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [
- new BullMQAdapter(getScrapeQueue(createRedisConnection())),
+ new BullMQAdapter(getScrapeQueue()),
new BullMQAdapter(getExtractQueue()),
new BullMQAdapter(getGenerateLlmsTxtQueue()),
new BullMQAdapter(getDeepResearchQueue()),
@@ -120,8 +119,7 @@ if (require.main === module) {
app.get(`/serverHealthCheck`, async (req, res) => {
try {
- const conn = createRedisConnection();
- const scrapeQueue = getScrapeQueue(conn);
+ const scrapeQueue = getScrapeQueue();
const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);
const noWaitingJobs = waitingJobs === 0;
// 200 if no active jobs, 503 if there are active jobs
@@ -141,8 +139,7 @@ app.get("/serverHealthCheck/notify", async (req, res) => {
const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds
const getWaitingJobsCount = async () => {
- const conn = createRedisConnection();
- const scrapeQueue = getScrapeQueue(conn);
+ const scrapeQueue = getScrapeQueue();
const [waitingJobsCount] = await Promise.all([
scrapeQueue.getWaitingCount(),
]);
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index b68206bb6..022f3523b 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -3,7 +3,7 @@ import { redisEvictConnection } from "../services/redis";
import type { Job, JobsOptions } from "bullmq";
import { getACUCTeam } from "../controllers/auth";
import { getCrawl, StoredCrawl } from "./crawl-redis";
-import { createRedisConnection, getScrapeQueue } from "../services/queue-service";
+import { getScrapeQueue } from "../services/queue-service";
import { logger } from "./logger";
const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;
@@ -258,8 +258,7 @@ export async function concurrentJobDone(job: Job) {
}
}
- const conn = createRedisConnection();
- getScrapeQueue(conn).add(
+ (await getScrapeQueue()).add(
nextJob.job.id,
{
...nextJob.job.data,
@@ -271,7 +270,6 @@ export async function concurrentJobDone(job: Job) {
priority: nextJob.job.priority,
}
);
- conn.disconnect();
}
}
}
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index a00dfeb72..083dd2628 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -1,6 +1,6 @@
import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../controllers/v1/types";
import { logger } from "../logger";
-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue } from "../../services/queue-service";
import { waitForJob } from "../../services/queue-jobs";
import { addScrapeJob } from "../../services/queue-jobs";
import { getJobPriority } from "../job-priority";
@@ -68,9 +68,7 @@ export async function scrapeDocument(
const doc = await waitForJob(jobId, timeout);
- const conn = createRedisConnection();
- await getScrapeQueue(conn).remove(jobId);
- conn.disconnect();
+ await getScrapeQueue().remove(jobId);
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index d5debb776..c6ecca291 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -1,6 +1,6 @@
import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../../controllers/v1/types";
import { logger } from "../../logger";
-import { createRedisConnection, getScrapeQueue } from "../../../services/queue-service";
+import { getScrapeQueue } from "../../../services/queue-service";
import { waitForJob } from "../../../services/queue-jobs";
import { addScrapeJob } from "../../../services/queue-jobs";
import { getJobPriority } from "../../job-priority";
@@ -65,9 +65,7 @@ export async function scrapeDocument_F0(
);
const doc = await waitForJob(jobId, timeout);
- const conn = createRedisConnection();
- await getScrapeQueue(conn).remove(jobId);
- conn.disconnect();
+ await getScrapeQueue().remove(jobId);
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.ts
index 9da5d146f..44f2b8a04 100644
--- a/apps/api/src/services/alerts/index.ts
+++ b/apps/api/src/services/alerts/index.ts
@@ -1,5 +1,5 @@
import { logger } from "../../../src/lib/logger";
-import { createRedisConnection, getScrapeQueue } from "../queue-service";
+import { getScrapeQueue } from "../queue-service";
import { sendSlackWebhook } from "./slack";
export async function checkAlerts() {
@@ -13,11 +13,8 @@ export async function checkAlerts() {
logger.info("Initializing alerts");
const checkActiveJobs = async () => {
try {
- const conn = createRedisConnection();
- const scrapeQueue = getScrapeQueue(conn);
+ const scrapeQueue = getScrapeQueue();
const activeJobs = await scrapeQueue.getActiveCount();
- conn.disconnect();
-
if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
logger.warn(
`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,
@@ -37,10 +34,8 @@ export async function checkAlerts() {
};
const checkWaitingQueue = async () => {
- const conn = createRedisConnection();
- const scrapeQueue = getScrapeQueue(conn);
+ const scrapeQueue = getScrapeQueue();
const waitingJobs = await scrapeQueue.getWaitingCount();
- conn.disconnect();
if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
logger.warn(
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index e0d1bd639..a6302ff5f 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -1,4 +1,4 @@
-import { createRedisConnection, getScrapeQueue, scrapeQueueName } from "./queue-service";
+import { createRedisConnection, getScrapeQueue } from "./queue-service";
import { v4 as uuidv4 } from "uuid";
import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";
import * as Sentry from "@sentry/node";
@@ -165,7 +165,7 @@ export async function addScrapeJob(
op: "queue.publish",
attributes: {
"messaging.message.id": jobId,
- "messaging.destination.name": scrapeQueueName,
+ "messaging.destination.name": getScrapeQueue().name,
"messaging.message.body.size": size,
},
},
@@ -328,7 +328,7 @@ export async function addScrapeJobs(
op: "queue.publish",
attributes: {
"messaging.message.id": job.opts.jobId,
- "messaging.destination.name": scrapeQueueName,
+ "messaging.destination.name": getScrapeQueue().name,
"messaging.message.body.size": size,
},
},
@@ -362,7 +362,7 @@ export async function addScrapeJobs(
op: "queue.publish",
attributes: {
"messaging.message.id": job.opts.jobId,
- "messaging.destination.name": scrapeQueueName,
+ "messaging.destination.name": getScrapeQueue().name,
"messaging.message.body.size": size,
},
},
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 6552df915..32ae88c49 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -30,9 +30,9 @@ export const deepResearchQueueName = "{deepResearchQueue}";
export const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
-export function getScrapeQueue(redisConnection: IORedis) {
- return new Queue(scrapeQueueName, {
- connection: redisConnection,
+export function getScrapeQueue(_redisConnection?: IORedis) {
+ const _scrapeQueue = !scrapeQueue || _redisConnection !== undefined ? new Queue(scrapeQueueName, {
+ connection: _redisConnection ?? redisConnection,
defaultJobOptions: {
removeOnComplete: {
age: 3600, // 1 hour
@@ -41,7 +41,16 @@ export function getScrapeQueue(redisConnection: IORedis) {
age: 3600, // 1 hour
},
},
- });
+ }) : undefined;
+
+ if (_redisConnection !== undefined) {
+ return _scrapeQueue!;
+ } else if (!scrapeQueue) {
+ scrapeQueue = _scrapeQueue!;
+ logger.info("Web scraper queue created");
+ }
+
+ return scrapeQueue;
}
export function getExtractQueue() {