Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixes crawl stuck issue
Nicolasca8f1a328b9d
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.tsindex c54c5c9c2..d7d9c0897 100644--- a/apps/api/src/controllers/v0/admin/queue.ts+++ b/apps/api/src/controllers/v0/admin/queue.ts@@ -2,7 +2,7 @@ import { Request, Response } from "express";import { Job } from "bullmq";import { logger } from "../../../lib/logger";-import { createRedisConnection, getScrapeQueue } from "../../../services/queue-service";+import { getScrapeQueue } from "../../../services/queue-service";import { checkAlerts } from "../../../services/alerts";import { sendSlackWebhook } from "../../../services/alerts/slack";@@ -11,9 +11,8 @@ export async function cleanBefore24hCompleteJobsController(res: Response,) {logger.info("π Cleaning jobs older than 24h");- const conn = createRedisConnection();try {- const scrapeQueue = getScrapeQueue(conn);+ const scrapeQueue = getScrapeQueue();const batchSize = 10;const numberOfBatches = 9; // Adjust based on your needsconst completedJobsPromises: Promise<Job[]>[] = [];@@ -55,8 +54,6 @@ export async function cleanBefore24hCompleteJobsController(} catch (error) {logger.error(`π Failed to clean last 24h complete jobs: ${error}`);return res.status(500).send("Failed to clean jobs");- } finally {- conn.disconnect();}}@@ -72,9 +69,8 @@ export async function checkQueuesController(req: Request, res: Response) {// Use this as a "health check" that way we dont destroy the serverexport async function queuesController(req: Request, res: Response) {- const conn = createRedisConnection();try {- const scrapeQueue = getScrapeQueue(conn);+ const scrapeQueue = getScrapeQueue();const [webScraperActive] = await Promise.all([scrapeQueue.getActiveCount(),@@ -89,18 +85,15 @@ export async function queuesController(req: Request, res: Response) {} catch (error) {logger.error(error);return res.status(500).json({ error: error.message });- } finally {- conn.disconnect();}}export async function autoscalerController(req: Request, res: Response) {- const conn = createRedisConnection();try {const maxNumberOfMachines = 80;const minNumberOfMachines = 20;- const scrapeQueue = getScrapeQueue(conn);+ const scrapeQueue = getScrapeQueue();const [webScraperActive, webScraperWaiting, webScraperPriority] =await Promise.all([@@ -204,7 +197,5 @@ export async function autoscalerController(req: Request, res: Response) {} catch (error) {logger.error(error);return res.status(500).send("Failed to initialize autoscaler");- } finally {- conn.disconnect();}}diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 559e14f11..a5ec432f1 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -21,7 +21,7 @@ import {defaultOrigin,} from "../../lib/default-values";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { v4 as uuidv4 } from "uuid";import { logger } from "../../lib/logger";@@ -139,9 +139,7 @@ export async function scrapeHelper(return err;}- const conn = createRedisConnection();- await getScrapeQueue(conn).remove(jobId);- conn.disconnect();+ await getScrapeQueue().remove(jobId);if (!doc) {console.error("!!! PANIC DOC IS", doc);diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex a41b548e7..aaff178c2 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -11,7 +11,7 @@ import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { v4 as uuidv4 } from "uuid";import { logger } from "../../lib/logger";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import * as Sentry from "@sentry/node";@@ -137,10 +137,8 @@ export async function searchHelper(return { success: true, error: "No search results found", returnCode: 200 };}- const conn = createRedisConnection()- const sq = getScrapeQueue(conn);+ const sq = getScrapeQueue();await Promise.all(jobDatas.map((x) => sq.remove(x.opts.jobId)));- conn.disconnect();// make sure doc.content is not emptyconst filteredDocs = docs.filter(diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex 263553784..d8d029ee6 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -8,29 +8,23 @@ import {getCrawl,getCrawlJobs,} from "../../lib/crawl-redis";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { configDotenv } from "dotenv";import { Job } from "bullmq";configDotenv();export async function getJob(id: string) {- const conn = createRedisConnection();- const job = await getScrapeQueue(conn).getJob(id);- conn.disconnect();+ const job = await getScrapeQueue().getJob(id);if (!job) return job;return job;}export async function getJobs(ids: string[]) {- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);- const jobs: (Job & { id: string })[] = (await Promise.all(- ids.map((x) => queue.getJob(x)),- )).filter((x) => x) as (Job & { id: string })[];-- conn.disconnect();+ const jobs: (Job & { id: string })[] = (+ await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))+ ).filter((x) => x) as (Job & { id: string })[];return jobs;}@@ -48,17 +42,12 @@ export async function crawlErrorsController(return res.status(403).json({ success: false, error: "Forbidden" });}- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);-let jobStatuses = await Promise.all((await getCrawlJobs(req.params.jobId)).map(- async (x) => [x, await queue.getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,),);- conn.disconnect();-const failedJobIDs: string[] = [];for (const [id, status] of jobStatuses) {diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 476dc9af3..cc4d6bf2b 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -105,9 +105,7 @@ export async function scrapeController(});if (zeroDataRetention) {- const conn = createRedisConnection();- await getScrapeQueue(conn).remove(jobId);- conn.disconnect();+ await getScrapeQueue().remove(jobId);}if (diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex e1e32f673..82efbcfb3 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -15,7 +15,7 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";import { getJobPriority } from "../../lib/job-priority";import { Mode } from "../../types";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";@@ -133,9 +133,7 @@ async function scrapeSearchResult(teamId: options.teamId,origin: options.origin,});- const conn = createRedisConnection();- await getScrapeQueue(conn).remove(jobId);- conn.disconnect();+ await getScrapeQueue().remove(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 7b6117f57..b6fc1e3fa 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -26,7 +26,6 @@ import { v4 as uuidv4 } from "uuid";import { RateLimiterMode } from "./types";import { attachWsProxy } from "./services/agentLivecastWS";import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";-import { createRedisConnection } from "./services/queue-service";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require('@bull-board/api/bullMQAdapter');@@ -56,7 +55,7 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({queues: [- new BullMQAdapter(getScrapeQueue(createRedisConnection())),+ new BullMQAdapter(getScrapeQueue()),new BullMQAdapter(getExtractQueue()),new BullMQAdapter(getGenerateLlmsTxtQueue()),new BullMQAdapter(getDeepResearchQueue()),@@ -120,8 +119,7 @@ if (require.main === module) {app.get(`/serverHealthCheck`, async (req, res) => {try {- const conn = createRedisConnection();- const scrapeQueue = getScrapeQueue(conn);+ const scrapeQueue = getScrapeQueue();const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);const noWaitingJobs = waitingJobs === 0;// 200 if no active jobs, 503 if there are active jobs@@ -141,8 +139,7 @@ app.get("/serverHealthCheck/notify", async (req, res) => {const timeout = 60000; // 1 minute // The timeout value for the check in millisecondsconst getWaitingJobsCount = async () => {- const conn = createRedisConnection();- const scrapeQueue = getScrapeQueue(conn);+ const scrapeQueue = getScrapeQueue();const [waitingJobsCount] = await Promise.all([scrapeQueue.getWaitingCount(),]);diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex b68206bb6..022f3523b 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -3,7 +3,7 @@ import { redisEvictConnection } from "../services/redis";import type { Job, JobsOptions } from "bullmq";import { getACUCTeam } from "../controllers/auth";import { getCrawl, StoredCrawl } from "./crawl-redis";-import { createRedisConnection, getScrapeQueue } from "../services/queue-service";+import { getScrapeQueue } from "../services/queue-service";import { logger } from "./logger";const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;@@ -258,8 +258,7 @@ export async function concurrentJobDone(job: Job) {}}- const conn = createRedisConnection();- getScrapeQueue(conn).add(+ (await getScrapeQueue()).add(nextJob.job.id,{...nextJob.job.data,@@ -271,7 +270,6 @@ export async function concurrentJobDone(job: Job) {priority: nextJob.job.priority,});- conn.disconnect();}}}diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex a00dfeb72..083dd2628 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -1,6 +1,6 @@import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../controllers/v1/types";import { logger } from "../logger";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";import { waitForJob } from "../../services/queue-jobs";import { addScrapeJob } from "../../services/queue-jobs";import { getJobPriority } from "../job-priority";@@ -68,9 +68,7 @@ export async function scrapeDocument(const doc = await waitForJob(jobId, timeout);- const conn = createRedisConnection();- await getScrapeQueue(conn).remove(jobId);- conn.disconnect();+ await getScrapeQueue().remove(jobId);if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex d5debb776..c6ecca291 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -1,6 +1,6 @@import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../../controllers/v1/types";import { logger } from "../../logger";-import { createRedisConnection, getScrapeQueue } from "../../../services/queue-service";+import { getScrapeQueue } from "../../../services/queue-service";import { waitForJob } from "../../../services/queue-jobs";import { addScrapeJob } from "../../../services/queue-jobs";import { getJobPriority } from "../../job-priority";@@ -65,9 +65,7 @@ export async function scrapeDocument_F0();const doc = await waitForJob(jobId, timeout);- const conn = createRedisConnection();- await getScrapeQueue(conn).remove(jobId);- conn.disconnect();+ await getScrapeQueue().remove(jobId);if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.tsindex 9da5d146f..44f2b8a04 100644--- a/apps/api/src/services/alerts/index.ts+++ b/apps/api/src/services/alerts/index.ts@@ -1,5 +1,5 @@import { logger } from "../../../src/lib/logger";-import { createRedisConnection, getScrapeQueue } from "../queue-service";+import { getScrapeQueue } from "../queue-service";import { sendSlackWebhook } from "./slack";export async function checkAlerts() {@@ -13,11 +13,8 @@ export async function checkAlerts() {logger.info("Initializing alerts");const checkActiveJobs = async () => {try {- const conn = createRedisConnection();- const scrapeQueue = getScrapeQueue(conn);+ const scrapeQueue = getScrapeQueue();const activeJobs = await scrapeQueue.getActiveCount();- conn.disconnect();-if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {logger.warn(`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,@@ -37,10 +34,8 @@ export async function checkAlerts() {};const checkWaitingQueue = async () => {- const conn = createRedisConnection();- const scrapeQueue = getScrapeQueue(conn);+ const scrapeQueue = getScrapeQueue();const waitingJobs = await scrapeQueue.getWaitingCount();- conn.disconnect();if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {logger.warn(diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex e0d1bd639..a6302ff5f 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,4 +1,4 @@-import { createRedisConnection, getScrapeQueue, scrapeQueueName } from "./queue-service";+import { createRedisConnection, getScrapeQueue } from "./queue-service";import { v4 as uuidv4 } from "uuid";import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";import * as Sentry from "@sentry/node";@@ -165,7 +165,7 @@ export async function addScrapeJob(op: "queue.publish",attributes: {"messaging.message.id": jobId,- "messaging.destination.name": scrapeQueueName,+ "messaging.destination.name": getScrapeQueue().name,"messaging.message.body.size": size,},},@@ -328,7 +328,7 @@ export async function addScrapeJobs(op: "queue.publish",attributes: {"messaging.message.id": job.opts.jobId,- "messaging.destination.name": scrapeQueueName,+ "messaging.destination.name": getScrapeQueue().name,"messaging.message.body.size": size,},},@@ -362,7 +362,7 @@ export async function addScrapeJobs(op: "queue.publish",attributes: {"messaging.message.id": job.opts.jobId,- "messaging.destination.name": scrapeQueueName,+ "messaging.destination.name": getScrapeQueue().name,"messaging.message.body.size": size,},},diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 6552df915..32ae88c49 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -30,9 +30,9 @@ export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export function getScrapeQueue(redisConnection: IORedis) {- return new Queue(scrapeQueueName, {- connection: redisConnection,+export function getScrapeQueue(_redisConnection?: IORedis) {+ const _scrapeQueue = !scrapeQueue || _redisConnection !== undefined ? new Queue(scrapeQueueName, {+ connection: _redisConnection ?? redisConnection,defaultJobOptions: {removeOnComplete: {age: 3600, // 1 hour@@ -41,7 +41,16 @@ export function getScrapeQueue(redisConnection: IORedis) {age: 3600, // 1 hour},},- });+ }) : undefined;++ if (_redisConnection !== undefined) {+ return _scrapeQueue!;+ } else if (!scrapeQueue) {+ scrapeQueue = _scrapeQueue!;+ logger.info("Web scraper queue created");+ }++ return scrapeQueue;}export function getExtractQueue() {