Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v1): crawl/batch scrape errors route
Gergő Móricze6531278f654
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsnew file mode 100644index 000000000..b64d02fa3--- /dev/null+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -0,0 +1,81 @@+import { Response } from "express";+import {+ CrawlErrorsResponse,+ CrawlStatusParams,+ CrawlStatusResponse,+ ErrorResponse,+ RequestWithAuth,+} from "./types";+import {+ getCrawl,+ getCrawlExpiry,+ getCrawlJobs,+ getDoneJobsOrdered,+ getDoneJobsOrderedLength,+ getThrottledJobs,+ isCrawlFinished,+} from "../../lib/crawl-redis";+import { getScrapeQueue, redisConnection } from "../../services/queue-service";+import {+ supabaseGetJobById,+ supabaseGetJobsById,+} from "../../lib/supabase-jobs";+import { configDotenv } from "dotenv";+import { Job, JobState } from "bullmq";+import { logger } from "../../lib/logger";+configDotenv();++export async function getJob(id: string) {+ const job = await getScrapeQueue().getJob(id);+ if (!job) return job;++ return job;+}++export async function getJobs(ids: string[]) {+ const jobs: (Job & { id: string })[] = (+ await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))+ ).filter((x) => x) as (Job & { id: string })[];++ return jobs;+}++export async function crawlErrorsController(+ req: RequestWithAuth<CrawlStatusParams, undefined, CrawlErrorsResponse>,+ res: Response<CrawlErrorsResponse>,+) {+ const sc = await getCrawl(req.params.jobId);+ if (!sc) {+ return res.status(404).json({ success: false, error: "Job not found" });+ }++ if (sc.team_id !== req.auth.team_id) {+ return res.status(403).json({ success: false, error: "Forbidden" });+ }++ let jobStatuses = await Promise.all(+ (await getCrawlJobs(req.params.jobId)).map(+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ ),+ );++ const failedJobIDs: string[] = [];++ for (const [id, status] of jobStatuses) {+ if (+ status === "failed"+ ) {+ failedJobIDs.push(id);+ }+ }++ res.status(200).json({+ errors: (await getJobs(failedJobIDs)).map(x => ({+ id: x.id,+ timestamp: x.finishedOn !== undefined ? (new Date(x.finishedOn).toISOString()) : undefined,+ url: x.data.url,+ error: x.failedReason,+ })),+ robotsBlocked: await redisConnection.smembers("crawl:" + req.params.jobId + ":robots_blocked"),+ });+}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex ac3743e86..ee141625b 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -568,6 +568,19 @@ export type CrawlStatusResponse =data: Document[];};++export type CrawlErrorsResponse =+ | ErrorResponse+ | {+ errors: {+ id: string,+ timestamp?: string,+ url: string,+ error: string,+ }[];+ robotsBlocked: string[];+ };+type AuthObject = {team_id: string;plan: PlanType | undefined;diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex a916dd40a..4aacfe185 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -28,6 +28,7 @@ import { extractStatusController } from "../controllers/v1/extract-status";import { creditUsageController } from "../controllers/v1/credit-usage";import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";import { searchController } from "../controllers/v1/search";+import { crawlErrorsController } from "../controllers/v1/crawl-errors";function checkCreditsMiddleware(minimum?: number,@@ -192,6 +193,18 @@ v1Router.get(wrap((req: any, res): any => crawlStatusController(req, res, true)),);+v1Router.get(+ "/crawl/:jobId/errors",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(crawlErrorsController),+);++v1Router.get(+ "/batch/scrape/:jobId/errors",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(crawlErrorsController),+);+v1Router.get("/scrape/:jobId",authMiddleware(RateLimiterMode.CrawlStatus),diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex a66451524..7d4be97b0 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -299,6 +299,16 @@ export class WebCrawler {this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)) {return fullUrl;+ } else if (+ this.isInternalLink(fullUrl) &&+ this.noSections(fullUrl) &&+ !this.matchesExcludes(path) &&+ !this.isRobotsAllowed(fullUrl, this.ignoreRobotsTxt)+ ) {+ (async() => {+ await redisConnection.sadd("crawl:" + this.jobId + ":robots_blocked", fullUrl);+ await redisConnection.expire("crawl:" + this.jobId + ":robots_blocked", 24 * 60 * 60, "NX");+ })();}} else {// EXTERNAL LINKS