Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Revert "fix: get rid of debug redis reinstantiation""
Gergő Móricz78d008c3d6b7
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex e8ee156a7..9f6e5b7bb 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -1,7 +1,7 @@import { Request, Response } from "express";import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";-import { createRedisConnection, getScrapeQueue } from "../../../src/services/queue-service";+import { getScrapeQueue } from "../../../src/services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { logger } from "../../../src/lib/logger";import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";@@ -12,13 +12,11 @@ import { Job } from "bullmq";import { toLegacyDocument } from "../v1/types";import type { DBJob, PseudoJob } from "../v1/crawl-status";import { getJobFromGCS } from "../../lib/gcs-jobs";-import IORedis from "ioredis";configDotenv();-export async function getJobs(crawlId: string, ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {- const queue = getScrapeQueue(conn);+export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -99,10 +97,8 @@ export async function crawlStatusController(req: Request, res: Response) {return res.status(403).json({ error: "Forbidden" });}let jobIDs = await getCrawlJobs(req.params.jobId);- const conn = createRedisConnection();- let jobs = await getJobs(req.params.jobId, jobIDs, conn);+ let jobs = await getJobs(req.params.jobId, jobIDs);let jobStatuses = await Promise.all(jobs.map((x) => x.getState()));- conn.disconnect();// Combine jobs and jobStatuses into a single array of objectslet jobsWithStatuses = jobs.map((job, index) => ({diff --git a/apps/api/src/controllers/v0/status.ts b/apps/api/src/controllers/v0/status.tsindex 47ef2fd4f..c68579eac 100644--- a/apps/api/src/controllers/v0/status.ts+++ b/apps/api/src/controllers/v0/status.ts@@ -3,7 +3,6 @@ import { logger } from "../../../src/lib/logger";import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";import { getJobs } from "./crawl-status";import * as Sentry from "@sentry/node";-import { createRedisConnection } from "../../services/queue-service";export async function crawlJobStatusPreviewController(req: Request,@@ -26,12 +25,10 @@ export async function crawlJobStatusPreviewController(// }// }- const conn = createRedisConnection();- const jobs = (await getJobs(req.params.jobId, jobIDs, conn)).sort(+ const jobs = (await getJobs(req.params.jobId, jobIDs)).sort((a, b) => a.timestamp - b.timestamp,);const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));- conn.disconnect();const jobStatus = sc.cancelled? "failed": jobStatuses.every((x) => x === "completed")diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex 4d5e3097e..93e1fc9b1 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -20,7 +20,7 @@ import {isCrawlFinished,isCrawlFinishedLocked,} from "../../lib/crawl-redis";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";import { getJob, getJobs } from "./crawl-status";import * as Sentry from "@sentry/node";import { Job, JobState } from "bullmq";@@ -89,8 +89,7 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);+ const queue = getScrapeQueue();const jobStatuses = await Promise.all(notDoneJobIDs.map(async (x) => [@@ -103,7 +102,7 @@ async function crawlStatusWS(.map((x) => x[0]);const newlyDoneJobs: Job[] = (- await Promise.all(newlyDoneJobIDs.map((x) => getJob(x, conn)))+ await Promise.all(newlyDoneJobIDs.map((x) => getJob(x)))).filter((x) => x !== undefined) as Job[];for (const job of newlyDoneJobs) {@@ -118,7 +117,6 @@ async function crawlStatusWS(}doneJobIDs.push(...newlyDoneJobIDs);- conn.disconnect();setTimeout(loop, 1000);};@@ -128,8 +126,7 @@ async function crawlStatusWS(let jobIDs = await getCrawlJobs(req.params.jobId);- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);+ const queue = getScrapeQueue();let jobStatuses = await Promise.all(jobIDs.map(@@ -164,8 +161,7 @@ async function crawlStatusWS(jobIDs = validJobIDs; // Use validJobIDs instead of jobIDs for further processing- const doneJobs = await getJobs(doneJobIDs, conn);- conn.disconnect();+ const doneJobs = await getJobs(doneJobIDs);const data = doneJobs.map((x) => x.returnvalue);await send(ws, {diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 5f115d18e..4da7112cf 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -13,7 +13,7 @@ import {getDoneJobsOrderedLength,isCrawlKickoffFinished,} from "../../lib/crawl-redis";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";import {supabaseGetJobById,supabaseGetJobsById,@@ -24,7 +24,6 @@ import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";import { getJobFromGCS } from "../../lib/gcs-jobs";-import IORedis from "ioredis";configDotenv();export type PseudoJob<T> = {@@ -41,9 +40,9 @@ export type PseudoJob<T> = {export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string }-export async function getJob(id: string, conn: IORedis): Promise<PseudoJob<any> | null> {+export async function getJob(id: string): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue(conn).getJob(id),+ getScrapeQueue().getJob(id),(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);@@ -73,10 +72,9 @@ export async function getJob(id: string, conn: IORedis): Promise<PseudoJob<any>return job;}-export async function getJobs(ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {- const queue = getScrapeQueue(conn);+export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -159,14 +157,11 @@ export async function crawlStatusController(}let jobIDs = await getCrawlJobs(req.params.jobId);- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);let jobStatuses = await Promise.all(jobIDs.map(- async (x) => [x, await queue.getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,),);- conn.disconnect();if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {for (let rangeStart = 0; ; rangeStart += 1000) {@@ -382,9 +377,8 @@ export async function crawlStatusController(i += factor) {// get current chunk and retrieve jobs- const conn = createRedisConnection();const currentIDs = doneJobsOrder.slice(i, i + factor);- const jobs = await getJobs(currentIDs, conn);+ const jobs = await getJobs(currentIDs);// iterate through jobs and add them one them one to the byte counter// both loops will break once we cross the byte counter@@ -407,7 +401,6 @@ export async function crawlStatusController(doneJobs.push(job);bytes += JSON.stringify(job.returnvalue ?? null).length;}- conn.disconnect();}// if we ran over the bytes limit, remove the last document, except if it's the only document@@ -415,15 +408,13 @@ export async function crawlStatusController(doneJobs.splice(doneJobs.length - 1, 1);}} else {- const conn = createRedisConnection();doneJobs = (await Promise.all(- (await getJobs(doneJobsOrder, conn)).map(async (x) =>+ (await getJobs(doneJobsOrder)).map(async (x) =>(await x.getState()) === "failed" ? null : x,),)).filter((x) => x !== null) as PseudoJob<any>[];- conn.disconnect();}const data = doneJobs.map((x) => x.returnvalue);diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.tsindex 802b50a48..78ded113a 100644--- a/apps/api/src/controllers/v1/scrape-status.ts+++ b/apps/api/src/controllers/v1/scrape-status.ts@@ -2,7 +2,6 @@ import { Response } from "express";import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs";import { getJob } from "./crawl-status";import { logger as _logger } from "../../lib/logger";-import { createRedisConnection } from "../../services/queue-service";export async function scrapeStatusController(req: any, res: any) {const logger = _logger.child({@@ -36,9 +35,7 @@ export async function scrapeStatusController(req: any, res: any) {});}- const conn = createRedisConnection();- const jobData = await getJob(req.params.jobId, conn);- conn.disconnect();+ const jobData = await getJob(req.params.jobId);const data = Array.isArray(jobData?.returnvalue)? jobData?.returnvalue[0]: jobData?.returnvalue;diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex cc4d6bf2b..1769edc95 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -10,7 +10,7 @@ import {import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";-import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue } from "../../services/queue-service";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -126,9 +126,7 @@ export async function scrapeController(logger.info("Done with waitForJob");- const conn = createRedisConnection();- await getScrapeQueue(conn).remove(jobId);- conn.disconnect();+ await getScrapeQueue().remove(jobId);logger.info("Removed job from queue");diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex d2c3db669..d6f49cf59 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -5,7 +5,6 @@ import { getJob } from "../../../controllers/v1/crawl-status";import gitDiff from 'git-diff';import parseDiff from 'parse-diff';import { generateCompletions } from "./llmExtract";-import { createRedisConnection } from "../../../services/queue-service";async function extractDataWithSchema(content: string, meta: Meta): Promise<{ extract: any } | null> {try {@@ -66,7 +65,6 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume}const start = Date.now();- const conn = createRedisConnection();const res = await supabase_service.rpc("diff_get_last_scrape_4", {i_team_id: meta.internalOptions.teamId,@@ -85,8 +83,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Documeconst job: {returnvalue: Document,- } | null = data?.o_job_id ? await getJob(data.o_job_id, conn) : null;- conn.disconnect();+ } | null = data?.o_job_id ? await getJob(data.o_job_id) : null;if (data && job && job?.returnvalue) {const previousMarkdown = job.returnvalue.markdown!;const currentMarkdown = document.markdown!;diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 26c0111ab..81dd810d3 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,4 +1,4 @@-import { createRedisConnection, getScrapeQueue } from "./queue-service";+import { getScrapeQueue } from "./queue-service";import { v4 as uuidv4 } from "uuid";import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";import * as Sentry from "@sentry/node";@@ -69,13 +69,11 @@ export async function _addScrapeJobToBullMQ(}}- const conn = createRedisConnection();- await getScrapeQueue(conn).add(jobId, webScraperOptions, {+ await getScrapeQueue().add(jobId, webScraperOptions, {...options,priority: jobPriority,jobId,});- conn.disconnect();}async function addScrapeJobRaw(@@ -397,8 +395,7 @@ export function waitForJob(const interval = setInterval(async () => {logger.debug("WaitforJob ran", { scrapeId: jobId, jobId });- const conn = createRedisConnection();- const queue = getScrapeQueue(conn);+ const queue = getScrapeQueue();if (Date.now() >= start + timeout) {reject(new Error("Job wait "));clearInterval(interval);@@ -434,7 +431,6 @@ export function waitForJob(}}}- conn.disconnect();}, 250);return;});diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 1dca534f5..968805488 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -13,18 +13,12 @@ let generateLlmsTxtQueue: Queue;let billingQueue: Queue;let precrawlQueue: Queue;-export function createRedisConnection(): IORedis {- const redis = new IORedis(process.env.REDIS_URL!, {- maxRetriesPerRequest: null,- });+export const redisConnection = new IORedis(process.env.REDIS_URL!, {+ maxRetriesPerRequest: null,+});- redis.on("reconnecting", () => logger.warn("Redis reconnecting"));- redis.on("error", (err) => logger.warn("Redis error", { err }));-- return redis;-}--export const redisConnection = createRedisConnection();+redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));+redisConnection.on("error", (err) => logger.warn("Redis error", { err }));export const scrapeQueueName = "{scrapeQueue}";export const extractQueueName = "{extractQueue}";@@ -35,26 +29,20 @@ export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export function getScrapeQueue(_redisConnection?: IORedis) {- const _scrapeQueue = !scrapeQueue || _redisConnection !== undefined ? new Queue(scrapeQueueName, {- connection: _redisConnection ?? redisConnection,- defaultJobOptions: {- removeOnComplete: {- age: 3600, // 1 hour- },- removeOnFail: {- age: 3600, // 1 hour+export function getScrapeQueue() {+ if (!scrapeQueue) {+ scrapeQueue = new Queue(scrapeQueueName, {+ connection: redisConnection,+ defaultJobOptions: {+ removeOnComplete: {+ age: 3600, // 1 hour+ },+ removeOnFail: {+ age: 3600, // 1 hour+ },},- },- }) : undefined;-- if (_redisConnection !== undefined) {- return _scrapeQueue!;- } else if (!scrapeQueue) {- scrapeQueue = _scrapeQueue!;- logger.info("Web scraper queue created");+ });}-return scrapeQueue;}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex b9e592ef2..de96f194b 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -10,7 +10,6 @@ import {getGenerateLlmsTxtQueue,getBillingQueue,scrapeQueueName,- createRedisConnection,} from "./queue-service";import { startWebScraperPipeline } from "../main/runWebScraper";import { callWebhook } from "./webhook";@@ -257,11 +256,9 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {if (!job.data.v1) {const jobIDs = await getCrawlJobs(job.data.crawl_id);- const conn = createRedisConnection();- const jobs = (await getJobs(jobIDs, conn)).sort(+ const jobs = (await getJobs(jobIDs)).sort((a, b) => a.timestamp - b.timestamp,);- conn.disconnect();// const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")? "failed"@@ -1640,9 +1637,7 @@ app.listen(workerPort, () => {return;}- const conn = createRedisConnection();- const job = await getScrapeQueue(conn).getJob(args.jobId);- conn.disconnect();+ const job = await getScrapeQueue().getJob(args.jobId);let logger = _logger.child({ jobId: args.jobId, scrapeId: args.jobId, module: "queue-worker", method: "failedListener", zeroDataRetention: job?.data.zeroDataRetention });if (job && job.data.crawl_id) {@@ -1677,14 +1672,12 @@ app.listen(workerPort, () => {const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: redisConnection });scrapeQueueEvents.on("failed", failedListener);- const conn = createRedisConnection();await Promise.all([- workerFun(getScrapeQueue(conn), processJobInternal),+ workerFun(getScrapeQueue(), processJobInternal),workerFun(getExtractQueue(), processExtractJobInternal),workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),]);- conn.disconnect();console.log("All workers exited. Waiting for all jobs to finish...");while (runningJobs.size > 0) {