Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix crawl status
Gergő Móricz341eaa64c2f9
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 22b4acf47..e8ee156a7 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -12,17 +12,16 @@ import { Job } from "bullmq";import { toLegacyDocument } from "../v1/types";import type { DBJob, PseudoJob } from "../v1/crawl-status";import { getJobFromGCS } from "../../lib/gcs-jobs";+import IORedis from "ioredis";configDotenv();-export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {- const conn = createRedisConnection();+export async function getJobs(crawlId: string, ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {const queue = getScrapeQueue(conn);const [bullJobs, dbJobs, gcsJobs] = await Promise.all([Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);- conn.disconnect();const bullJobMap = new Map<string, PseudoJob<any>>();const dbJobMap = new Map<string, DBJob>();@@ -100,8 +99,10 @@ export async function crawlStatusController(req: Request, res: Response) {return res.status(403).json({ error: "Forbidden" });}let jobIDs = await getCrawlJobs(req.params.jobId);- let jobs = await getJobs(req.params.jobId, jobIDs);+ const conn = createRedisConnection();+ let jobs = await getJobs(req.params.jobId, jobIDs, conn);let jobStatuses = await Promise.all(jobs.map((x) => x.getState()));+ conn.disconnect();// Combine jobs and jobStatuses into a single array of objectslet jobsWithStatuses = jobs.map((job, index) => ({diff --git a/apps/api/src/controllers/v0/status.ts b/apps/api/src/controllers/v0/status.tsindex c68579eac..47ef2fd4f 100644--- a/apps/api/src/controllers/v0/status.ts+++ b/apps/api/src/controllers/v0/status.ts@@ -3,6 +3,7 @@ import { logger } from "../../../src/lib/logger";import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";import { getJobs } from "./crawl-status";import * as Sentry from "@sentry/node";+import { createRedisConnection } from "../../services/queue-service";export async function crawlJobStatusPreviewController(req: Request,@@ -25,10 +26,12 @@ export async function crawlJobStatusPreviewController(// }// }- const jobs = (await getJobs(req.params.jobId, jobIDs)).sort(+ const conn = createRedisConnection();+ const jobs = (await getJobs(req.params.jobId, jobIDs, conn)).sort((a, b) => a.timestamp - b.timestamp,);const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));+ conn.disconnect();const jobStatus = sc.cancelled? "failed": jobStatuses.every((x) => x === "completed")diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex 190aa73e1..4d5e3097e 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -101,12 +101,11 @@ async function crawlStatusWS(const newlyDoneJobIDs: string[] = jobStatuses.filter((x) => x[1] === "completed" || x[1] === "failed").map((x) => x[0]);+const newlyDoneJobs: Job[] = (- await Promise.all(newlyDoneJobIDs.map((x) => getJob(x)))+ await Promise.all(newlyDoneJobIDs.map((x) => getJob(x, conn)))).filter((x) => x !== undefined) as Job[];- conn.disconnect();-for (const job of newlyDoneJobs) {if (job.returnvalue) {send(ws, {@@ -119,7 +118,7 @@ async function crawlStatusWS(}doneJobIDs.push(...newlyDoneJobIDs);-+ conn.disconnect();setTimeout(loop, 1000);};@@ -138,8 +137,6 @@ async function crawlStatusWS(),);- conn.disconnect();-const throttledJobsSet = await getConcurrencyLimitedJobs(req.auth.team_id);const validJobStatuses: [string, JobState | "unknown"][] = [];@@ -167,7 +164,8 @@ async function crawlStatusWS(jobIDs = validJobIDs; // Use validJobIDs instead of jobIDs for further processing- const doneJobs = await getJobs(doneJobIDs);+ const doneJobs = await getJobs(doneJobIDs, conn);+ conn.disconnect();const data = doneJobs.map((x) => x.returnvalue);await send(ws, {diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 1b49ca3aa..be2d2b21c 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -24,6 +24,7 @@ import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getConcurrencyLimitedJobs, getCrawlConcurrencyLimitActiveJobs } from "../../lib/concurrency-limit";import { getJobFromGCS } from "../../lib/gcs-jobs";+import IORedis from "ioredis";configDotenv();export type PseudoJob<T> = {@@ -40,14 +41,12 @@ export type PseudoJob<T> = {export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null, team_id: string }-export async function getJob(id: string): Promise<PseudoJob<any> | null> {- const conn = createRedisConnection();+export async function getJob(id: string, conn: IORedis): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([getScrapeQueue(conn).getJob(id),(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);- conn.disconnect();if (!bullJob && !dbJob) return null;@@ -74,15 +73,13 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {return job;}-export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {- const conn = createRedisConnection();+export async function getJobs(ids: string[], conn: IORedis): Promise<PseudoJob<any>[]> {const queue = getScrapeQueue(conn);const [bullJobs, dbJobs, gcsJobs] = await Promise.all([Promise.all(ids.map((x) => queue.getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);- conn.disconnect();const bullJobMap = new Map<string, PseudoJob<any>>();const dbJobMap = new Map<string, DBJob>();@@ -116,9 +113,10 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {});}+ const state = await bullJob?.getState();const job: PseudoJob<any> = {id,- getState: bullJob ? (() => bullJob.getState()) : (() => dbJob!.success ? "completed" : "failed"),+ getState: bullJob ? (() => state!) : (() => dbJob!.success ? "completed" : "failed"),returnvalue: Array.isArray(data)? data[0]: data,@@ -388,8 +386,9 @@ export async function crawlStatusController(i += factor) {// get current chunk and retrieve jobs+ const conn = createRedisConnection();const currentIDs = doneJobsOrder.slice(i, i + factor);- const jobs = await getJobs(currentIDs);+ const jobs = await getJobs(currentIDs, conn);// iterate through jobs and add them one them one to the byte counter// both loops will break once we cross the byte counter@@ -412,6 +411,7 @@ export async function crawlStatusController(doneJobs.push(job);bytes += JSON.stringify(job.returnvalue ?? null).length;}+ conn.disconnect();}// if we ran over the bytes limit, remove the last document, except if it's the only document@@ -419,13 +419,15 @@ export async function crawlStatusController(doneJobs.splice(doneJobs.length - 1, 1);}} else {+ const conn = createRedisConnection();doneJobs = (await Promise.all(- (await getJobs(doneJobsOrder)).map(async (x) =>+ (await getJobs(doneJobsOrder, conn)).map(async (x) =>(await x.getState()) === "failed" ? null : x,),)).filter((x) => x !== null) as PseudoJob<any>[];+ conn.disconnect();}const data = doneJobs.map((x) => x.returnvalue);diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.tsindex 78ded113a..802b50a48 100644--- a/apps/api/src/controllers/v1/scrape-status.ts+++ b/apps/api/src/controllers/v1/scrape-status.ts@@ -2,6 +2,7 @@ import { Response } from "express";import { supabaseGetJobByIdOnlyData } from "../../lib/supabase-jobs";import { getJob } from "./crawl-status";import { logger as _logger } from "../../lib/logger";+import { createRedisConnection } from "../../services/queue-service";export async function scrapeStatusController(req: any, res: any) {const logger = _logger.child({@@ -35,7 +36,9 @@ export async function scrapeStatusController(req: any, res: any) {});}- const jobData = await getJob(req.params.jobId);+ const conn = createRedisConnection();+ const jobData = await getJob(req.params.jobId, conn);+ conn.disconnect();const data = Array.isArray(jobData?.returnvalue)? jobData?.returnvalue[0]: jobData?.returnvalue;diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex 79a54b745..d2c3db669 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -5,6 +5,7 @@ import { getJob } from "../../../controllers/v1/crawl-status";import gitDiff from 'git-diff';import parseDiff from 'parse-diff';import { generateCompletions } from "./llmExtract";+import { createRedisConnection } from "../../../services/queue-service";async function extractDataWithSchema(content: string, meta: Meta): Promise<{ extract: any } | null> {try {@@ -65,6 +66,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume}const start = Date.now();+ const conn = createRedisConnection();const res = await supabase_service.rpc("diff_get_last_scrape_4", {i_team_id: meta.internalOptions.teamId,@@ -83,8 +85,8 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Documeconst job: {returnvalue: Document,- } | null = data?.o_job_id ? await getJob(data.o_job_id) : null;-+ } | null = data?.o_job_id ? await getJob(data.o_job_id, conn) : null;+ conn.disconnect();if (data && job && job?.returnvalue) {const previousMarkdown = job.returnvalue.markdown!;const currentMarkdown = document.markdown!;diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex b8e7816a0..b6d4cc60c 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -257,9 +257,11 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {if (!job.data.v1) {const jobIDs = await getCrawlJobs(job.data.crawl_id);- const jobs = (await getJobs(jobIDs)).sort(+ const conn = createRedisConnection();+ const jobs = (await getJobs(jobIDs, conn)).sort((a, b) => a.timestamp - b.timestamp,);+ conn.disconnect();// const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")? "failed"