Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolas1a1ac9fd60ba
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 1b1ffdc5f..f809c7be3 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -50,11 +50,15 @@ export async function crawlStatusController(req: Request, res: Response) {return res.status(403).json({ error: "Forbidden" });}- const jobIDs = await getCrawlJobs(req.params.jobId);+ let jobIDs = await getCrawlJobs(req.params.jobId);const jobs = (await getJobs(req.params.jobId, jobIDs)).sort((a, b) => a.timestamp - b.timestamp);- const jobStatuses = await Promise.all(jobs.map(x => x.getState()));- const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : jobs.some((x, i) => jobStatuses[i] === "failed" && x.failedReason !== "Concurrency limit hit") ? "failed" : "active";+ let jobStatuses = await Promise.all(jobs.map(x => x.getState()));+ // filter out failed jobs+ jobIDs = jobIDs.filter(id => !jobStatuses.some(status => status[0] === id && status[1] === "failed"));+ // filter the job statues+ jobStatuses = jobStatuses.filter(x => x[1] !== "failed");+ const jobStatus = sc.cancelled ? "failed" : jobStatuses.every(x => x === "completed") ? "completed" : "active";const data = jobs.filter(x => x.failedReason !== "Concurreny limit hit").map(x => Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue);diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex 9832a948f..b67e559b2 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -94,11 +94,15 @@ async function crawlStatusWS(ws: WebSocket, req: RequestWithAuth<CrawlStatusParadoneJobIDs = await getDoneJobsOrdered(req.params.jobId);- const jobIDs = await getCrawlJobs(req.params.jobId);+ let jobIDs = await getCrawlJobs(req.params.jobId);let jobStatuses = await Promise.all(jobIDs.map(async x => [x, await getScrapeQueue().getJobState(x)] as const));const throttledJobs = new Set(...await getThrottledJobs(req.auth.team_id));jobStatuses = jobStatuses.filter(x => !throttledJobs.has(x[0])); // throttled jobs can have a failed status, but they are not actually failed- const status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] = sc.cancelled ? "cancelled" : jobStatuses.every(x => x[1] === "completed") ? "completed" : jobStatuses.some(x => x[1] === "failed") ? "failed" : "scraping";+ // filter out failed jobs+ jobIDs = jobIDs.filter(id => !jobStatuses.some(status => status[0] === id && status[1] === "failed"));+ // filter the job statues+ jobStatuses = jobStatuses.filter(x => x[1] !== "failed");+ const status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] = sc.cancelled ? "cancelled" : jobStatuses.every(x => x[1] === "completed") ? "completed" : "scraping";const doneJobs = await getJobs(doneJobIDs);const data = doneJobs.map(x => x.returnvalue);diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 9c0026a08..63331c9c3 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -57,11 +57,15 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusParaconst start = typeof req.query.skip === "string" ? parseInt(req.query.skip, 10) : 0;const end = typeof req.query.limit === "string" ? (start + parseInt(req.query.limit, 10) - 1) : undefined;- const jobIDs = await getCrawlJobs(req.params.jobId);+ let jobIDs = await getCrawlJobs(req.params.jobId);let jobStatuses = await Promise.all(jobIDs.map(async x => [x, await getScrapeQueue().getJobState(x)] as const));const throttledJobs = new Set(...await getThrottledJobs(req.auth.team_id));jobStatuses = jobStatuses.filter(x => !throttledJobs.has(x[0])); // throttled jobs can have a failed status, but they are not actually failed- const status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] = sc.cancelled ? "cancelled" : jobStatuses.every(x => x[1] === "completed") ? "completed" : jobStatuses.some(x => x[1] === "failed") ? "failed" : "scraping";+ // filter out failed jobs+ jobIDs = jobIDs.filter(id => !jobStatuses.some(status => status[0] === id && status[1] === "failed"));+ // filter the job statues+ jobStatuses = jobStatuses.filter(x => x[1] !== "failed");+ const status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] = sc.cancelled ? "cancelled" : jobStatuses.every(x => x[1] === "completed") ? "completed" : "scraping";const doneJobsLength = await getDoneJobsOrderedLength(req.params.jobId);const doneJobsOrder = await getDoneJobsOrdered(req.params.jobId, start, end ?? -1);diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 571122f95..6e642c65a 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -59,6 +59,7 @@ export async function startWebScraperPipeline({is_scrape: job.data.is_scrape ?? false,})) as { success: boolean; message: string; docs: Document[] };}+export async function runWebScraper({url,mode,diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 532e8feeb..1ccf486ec 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -395,6 +395,7 @@ async function processJob(job: Job, token: string) {pageOptions: sc.pageOptions,origin: job.data.origin,crawl_id: job.data.crawl_id,+ webhook: job.data.webhook,v1: job.data.v1,},{},@@ -468,9 +469,8 @@ async function processJob(job: Job, token: string) {}} else {const jobIDs = await getCrawlJobs(job.data.crawl_id);- const jobStatuses = await Promise.all(jobIDs.map((x) => getScrapeQueue().getJobState(x)));const jobStatus =- sc.cancelled || jobStatuses.some((x) => x === "failed")+ sc.cancelled? "failed": "completed";@@ -554,16 +554,16 @@ async function processJob(job: Job, token: string) {job.data.v1);}- if (job.data.v1) {- callWebhook(- job.data.team_id,- job.id as string,- [],- job.data.webhook,- job.data.v1,- "crawl.failed"- );- }+ // if (job.data.v1) {+ // callWebhook(+ // job.data.team_id,+ // job.id as string,+ // [],+ // job.data.webhook,+ // job.data.v1,+ // "crawl.failed"+ // );+ // }if (job.data.crawl_id) {await logJob({