Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "fix: get rid of debug redis reinstantiation"
Gergő Móricz7a9f859e4424
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 9f6e5b7bb..e8ee156a7 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -1,7 +1,7 @@import { Request, Response } from "express";import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";-import { getScrapeQueue } from "../../../src/services/queue-service";+import { createRedisConnection, getScrapeQueue } from "../../../src/services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { logger } from "../../../src/lib/logger";import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";@@ -12,11 +12,13 @@ import { Job } from "bullmq";import { toLegacyDocument } from "../v1/types";import type { DBJob, PseudoJob } from "../v1/crawl-status";import { getJobFromGCS } from "../../lib/gcs-jobs";+import IORedis from "ioredis";configDotenv();-export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {+export async function getJobs(crawlId: string, ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {+ const queue = getScrapeQueue(conn);const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -97,8 +99,10 @@ export async function crawlStatusController(req: Request, res: Response) {return res.status(403).json({ error: "Forbidden" });}let jobIDs = await getCrawlJobs(req.params.jobId);- let jobs = await getJobs(req.params.jobId, jobIDs);+ const conn = createRedisConnection();+ let jobs = await getJobs(req.params.jobId, jobIDs, conn);let jobStatuses = await Promise.all(jobs.map((x) => x.getState()));+ conn.disconnect();// Combine jobs and jobStatuses into a single array of objectslet jobsWithStatuses = jobs.map((job, index) => ({diff --git a/apps/api/src/controllers/v0/status.ts b/apps/api/src/controllers/v0/status.tsindex c68579eac..47ef2fd4f 100644--- a/apps/api/src/controllers/v0/status.ts+++ b/apps/api/src/controllers/v0/status.ts@@ -3,6 +3,7 @@ import { logger } from "../../../src/lib/logger";import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";import { getJobs } from "./crawl-status";import * as Sentry from "@sentry/node";+import { createRedisConnection } from "../../services/queue-service";export async function crawlJobStatusPreviewController(req: Request,@@ -25,10 +26,12 @@ export async function crawlJobStatusPreviewController(// }// }- const jobs = (await getJobs(req.params.jobId, jobIDs)).sort(+ const conn = createRedisConnection();+ const jobs = (await getJobs(req.params.jobId, jobIDs, conn)).sort((a, b) => a.timestamp - b.timestamp,);const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));+ conn.disconnect();const jobStatus = sc.cancelled? "failed": jobStatuses.every((x) => x === "completed")diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex 93e1fc9b1..4d5e3097e 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -20,7 +20,7 @@ import {isCrawlFinished,isCrawlFinishedLocked,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";+import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";import { getJob, getJobs } from "./crawl-status";import * as Sentry from "@sentry/node";import { Job, JobState } from "bullmq";@@ -89,7 +89,8 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));- const queue = getScrapeQueue();+ const conn = createRedisConnection();+ const queue = getScrapeQueue(conn);const jobStatuses = await Promise.all(notDoneJobIDs.map(async (x) => [@@ -102,7 +103,7 @@ async function crawlStatusWS(.map((x) => x[0]);const newlyDoneJobs: Job[] = (- await Promise.all(newlyDoneJobIDs.map((x) => getJob(x)))+ await Promise.all(newlyDoneJobIDs.map((x) => getJob(x, conn)))).filter((x) => x !== undefined) as Job[];for (const job of newlyDoneJobs) {@@ -117,6 +118,7 @@ async function crawlStatusWS(}doneJobIDs.push(...newlyDoneJobIDs);+ conn.disconnect();setTimeout(loop, 1000);};@@ -126,7 +128,8 @@ async function crawlStatusWS(let jobIDs = await getCrawlJobs(req.params.jobId);- const queue = getScrapeQueue();+ const conn = createRedisConnection();+ const queue = getScrapeQueue(conn);let jobStatuses = await Promise.all(jobIDs.map(@@ -161,7 +164,8 @@ async function crawlStatusWS(jobIDs = validJobIDs; // Use validJobIDs instead of jobIDs for further processing- const doneJobs = await getJobs(doneJobIDs);+ const doneJobs = await getJobs(doneJobIDs, conn);+ conn.disconnect();const data = doneJobs.map((x) => x.returnvalue);await send(ws, {diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 8c6c746ee..de5eb83ba 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -13,7 +13,7 @@ import {getDoneJobsOrderedLength,isCrawlKickoffFinished,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";+import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";import {supabaseGetJobById,supabaseGetJobsById,@@ -24,6 +24,7 @@ import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";import { getJobFromGCS } from "../../lib/gcs-jobs";+import IORedis from "ioredis";configDotenv();export type PseudoJob<T> = {@@ -40,9 +41,9 @@ export type PseudoJob<T> = {export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string }-export async function getJob(id: string): Promise<PseudoJob<any> | null> {+export async function getJob(id: string, conn: IORedis): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue().getJob(id),+ getScrapeQueue(conn).getJob(id),(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);@@ -72,9 +73,10 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {return job;}-export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {+export async function getJobs(ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {+ const queue = getScrapeQueue(conn);const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -157,11 +159,14 @@ export async function crawlStatusController(}let jobIDs = await getCrawlJobs(req.params.jobId);+ const conn = createRedisConnection();+ const queue = getScrapeQueue(conn);let jobStatuses = await Promise.all(jobIDs.map(- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ async (x) => [x, await queue.getJobState(x)] as const,),);+ conn.disconnect();if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {for (let rangeStart = 0; ; rangeStart += 1000) {@@ -380,8 +385,9 @@ export async function crawlStatusController(i += factor) {// get current chunk and retrieve jobs+ const conn = createRedisConnection();const currentIDs = doneJobsOrder.slice(i, i + factor);- const jobs = await getJobs(currentIDs);+ const jobs = await getJobs(currentIDs, conn);// iterate through jobs and add them one them one to the byte counter// both loops will break once we cross the byte counter@@ -404,6 +410,7 @@ export async function crawlStatusController(doneJobs.push(job);bytes += JSON.stringify(job.returnvalue ?? null).length;}+ conn.disconnect();}// if we ran over the bytes limit, remove the last document, except if it's the only document@@ -411,13 +418,15 @@ export async function crawlStatusController(doneJobs.splice(doneJobs.length - 1, 1);}} else {+ const conn = createRedisConnection();doneJobs = (await Promise.all(- (await getJobs(doneJobsOrder)).map(async (x) =>+ (await getJobs(doneJobsOrder, conn)).map(async (x) =>(await x.getState()) === "failed" ? null : x,),)).filter((x) => x !== null) as PseudoJob<any>[];+ conn.disconnect();}const data = doneJobs.map((x) => x.returnvalue);diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.tsindex 78ded113a..802b50a48 100644--- a/apps/api/src/controllers/v1/scrape-status.ts+++ b/apps/api/src/controllers/v1/scrape-status.ts@@ -2,6 +2,7 @@ import { Response } from "express";import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs";import { getJob } from "./crawl-status";import { logger as _logger } from "../../lib/logger";+import { createRedisConnection } from "../../services/queue-service";export async function scrapeStatusController(req: any, res: any) {const logger = _logger.child({@@ -35,7 +36,9 @@ export async function scrapeStatusController(req: any, res: any) {});}- const jobData = await getJob(req.params.jobId);+ const conn = createRedisConnection();+ const jobData = await getJob(req.params.jobId, conn);+ conn.disconnect();const data = Array.isArray(jobData?.returnvalue)? jobData?.returnvalue[0]: jobData?.returnvalue;diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 1769edc95..cc4d6bf2b 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -10,7 +10,7 @@ import {import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";-import { getScrapeQueue } from "../../services/queue-service";+import { createRedisConnection, getScrapeQueue } from "../../services/queue-service";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -126,7 +126,9 @@ export async function scrapeController(logger.info("Done with waitForJob");- await getScrapeQueue().remove(jobId);+ const conn = createRedisConnection();+ await getScrapeQueue(conn).remove(jobId);+ conn.disconnect();logger.info("Removed job from queue");diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex d6f49cf59..d2c3db669 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -5,6 +5,7 @@ import { getJob } from "../../../controllers/v1/crawl-status";import gitDiff from 'git-diff';import parseDiff from 'parse-diff';import { generateCompletions } from "./llmExtract";+import { createRedisConnection } from "../../../services/queue-service";async function extractDataWithSchema(content: string, meta: Meta): Promise<{ extract: any } | null> {try {@@ -65,6 +66,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume}const start = Date.now();+ const conn = createRedisConnection();const res = await supabase_service.rpc("diff_get_last_scrape_4", {i_team_id: meta.internalOptions.teamId,@@ -83,7 +85,8 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Documeconst job: {returnvalue: Document,- } | null = data?.o_job_id ? await getJob(data.o_job_id) : null;+ } | null = data?.o_job_id ? await getJob(data.o_job_id, conn) : null;+ conn.disconnect();if (data && job && job?.returnvalue) {const previousMarkdown = job.returnvalue.markdown!;const currentMarkdown = document.markdown!;diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 4cd7ba80a..a6302ff5f 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,4 +1,4 @@-import { getScrapeQueue } from "./queue-service";+import { createRedisConnection, getScrapeQueue } from "./queue-service";import { v4 as uuidv4 } from "uuid";import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";import * as Sentry from "@sentry/node";@@ -69,11 +69,13 @@ export async function _addScrapeJobToBullMQ(}}- await getScrapeQueue().add(jobId, webScraperOptions, {+ const conn = createRedisConnection();+ await getScrapeQueue(conn).add(jobId, webScraperOptions, {...options,priority: jobPriority,jobId,});+ conn.disconnect();}async function addScrapeJobRaw(@@ -392,7 +394,8 @@ export function waitForJob(): Promise<Document> {return new Promise(async (resolve, reject) => {const start = Date.now();- const queue = getScrapeQueue();+ const conn = createRedisConnection();+ const queue = getScrapeQueue(conn);while (true) {logger.debug("WaitforJob ran", { scrapeId: jobId, jobId });if (Date.now() >= start + timeout) {@@ -434,6 +437,7 @@ export function waitForJob(await new Promise((resolve) => setTimeout(resolve, 750));}+ conn.disconnect();return;});}diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex d5113d889..32ae88c49 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -13,9 +13,13 @@ let generateLlmsTxtQueue: Queue;let billingQueue: Queue;let precrawlQueue: Queue;-export const redisConnection = new IORedis(process.env.REDIS_URL!, {- maxRetriesPerRequest: null,-});+export function createRedisConnection(): IORedis {+ return new IORedis(process.env.REDIS_URL!, {+ maxRetriesPerRequest: null,+ });+}++export const redisConnection = createRedisConnection();export const scrapeQueueName = "{scrapeQueue}";export const extractQueueName = "{extractQueue}";@@ -26,20 +30,26 @@ export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export function getScrapeQueue() {- if (!scrapeQueue) {- scrapeQueue = new Queue(scrapeQueueName, {- connection: redisConnection,- defaultJobOptions: {- removeOnComplete: {- age: 3600, // 1 hour- },- removeOnFail: {- age: 3600, // 1 hour- },+export function getScrapeQueue(_redisConnection?: IORedis) {+ const _scrapeQueue = !scrapeQueue || _redisConnection !== undefined ? new Queue(scrapeQueueName, {+ connection: _redisConnection ?? redisConnection,+ defaultJobOptions: {+ removeOnComplete: {+ age: 3600, // 1 hour},- });+ removeOnFail: {+ age: 3600, // 1 hour+ },+ },+ }) : undefined;++ if (_redisConnection !== undefined) {+ return _scrapeQueue!;+ } else if (!scrapeQueue) {+ scrapeQueue = _scrapeQueue!;+ logger.info("Web scraper queue created");}+return scrapeQueue;}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex de96f194b..b9e592ef2 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -10,6 +10,7 @@ import {getGenerateLlmsTxtQueue,getBillingQueue,scrapeQueueName,+ createRedisConnection,} from "./queue-service";import { startWebScraperPipeline } from "../main/runWebScraper";import { callWebhook } from "./webhook";@@ -256,9 +257,11 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {if (!job.data.v1) {const jobIDs = await getCrawlJobs(job.data.crawl_id);- const jobs = (await getJobs(jobIDs)).sort(+ const conn = createRedisConnection();+ const jobs = (await getJobs(jobIDs, conn)).sort((a, b) => a.timestamp - b.timestamp,);+ conn.disconnect();// const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")? "failed"@@ -1637,7 +1640,9 @@ app.listen(workerPort, () => {return;}- const job = await getScrapeQueue().getJob(args.jobId);+ const conn = createRedisConnection();+ const job = await getScrapeQueue(conn).getJob(args.jobId);+ conn.disconnect();let logger = _logger.child({ jobId: args.jobId, scrapeId: args.jobId, module: "queue-worker", method: "failedListener", zeroDataRetention: job?.data.zeroDataRetention });if (job && job.data.crawl_id) {@@ -1672,12 +1677,14 @@ app.listen(workerPort, () => {const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: redisConnection });scrapeQueueEvents.on("failed", failedListener);+ const conn = createRedisConnection();await Promise.all([- workerFun(getScrapeQueue(), processJobInternal),+ workerFun(getScrapeQueue(conn), processJobInternal),workerFun(getExtractQueue(), processExtractJobInternal),workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),]);+ conn.disconnect();console.log("All workers exited. Waiting for all jobs to finish...");while (runningJobs.size > 0) {