Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl-status): same for v0
Gergő Móricza7b56ab87ccd
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 60ca0e7f1..7040a857a 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -9,31 +9,53 @@ import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { Job } from "bullmq";import { toLegacyDocument } from "../v1/types";+import type { DBJob, PseudoJob } from "../v1/crawl-status";configDotenv();-export async function getJobs(crawlId: string, ids: string[]) {- const jobs = (- await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))- ).filter((x) => x) as Job[];-- if (process.env.USE_DB_AUTHENTICATION === "true") {- const supabaseData = await supabaseGetJobsByCrawlId(crawlId);-- supabaseData.forEach((x) => {- const job = jobs.find((y) => y.id === x.job_id);- if (job) {- job.returnvalue = x.docs;+export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {+ const [bullJobs, dbJobs] = await Promise.all([+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],+ ]);++ const bullJobMap = new Map<string, PseudoJob<any>>();+ const dbJobMap = new Map<string, DBJob>();++ for (const job of bullJobs) {+ bullJobMap.set(job.id, job);+ }++ for (const job of dbJobs) {+ dbJobMap.set(job.job_id, job);+ }++ const jobs: PseudoJob<any>[] = [];++ for (const id of ids) {+ const bullJob = bullJobMap.get(id);+ const dbJob = dbJobMap.get(id);++ if (!bullJob && !dbJob) continue;++ const data = dbJob?.docs ?? bullJob?.returnvalue;++ const job: PseudoJob<any> = {+ id,+ getState: bullJob ? (() => bullJob.getState()) : (() => dbJob!.success ? "completed" : "failed"),+ returnvalue: Array.isArray(data)+ ? data[0]+ : data,+ data: {+ scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,+ },+ timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),+ failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,}- });- }-- jobs.forEach((job) => {- job.returnvalue = Array.isArray(job.returnvalue)- ? job.returnvalue[0]- : job.returnvalue;- });-- return jobs;++ jobs.push(job);+ }++ return jobs;}export async function crawlStatusController(req: Request, res: Response) {@@ -93,8 +115,9 @@ export async function crawlStatusController(req: Request, res: Response) {if (jobs.length > 0 &&jobs[0].data &&- jobs[0].data.pageOptions &&- !jobs[0].data.pageOptions.includeRawHtml+ jobs[0].data.scrapeOptions &&+ jobs[0].data.scrapeOptions.formats &&+ !jobs[0].data.scrapeOptions.formats.includes("rawHtml")) {data.forEach((item) => {if (item) {diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex a27e791ae..d43562b6e 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -25,7 +25,7 @@ import { logger } from "../../lib/logger";import { supabase_service } from "../../services/supabase";configDotenv();-type PseudoJob<T> = {+export type PseudoJob<T> = {id: string,getState(): Promise<JobState | "unknown"> | JobState | "unknown",returnvalue: T | null,@@ -33,9 +33,10 @@ type PseudoJob<T> = {data: {scrapeOptions: any,},+ failedReason?: string,}-type DBJob = { docs: any, success: boolean, page_options: any, date_added: any }+export type DBJob = { docs: any, success: boolean, page_options: any, date_added: any, message: string | null }export async function getJob(id: string): Promise<PseudoJob<any> | null> {const [bullJob, dbJob] = await Promise.all([@@ -57,6 +58,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,},timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),+ failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,}return job;@@ -99,6 +101,7 @@ export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,},timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),+ failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,}jobs.push(job);