Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): surface crawl error message (ENG-4574) (#3057)
tomsideguided558ecb91e2a
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex f39d4103a..27589a35f 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -8,6 +8,7 @@ import {} from "./types";import {getCrawl,+ getCrawlError,getCrawlExpiry,getDoneJobsOrderedLength,isCrawlKickoffFinished,@@ -205,8 +206,11 @@ export async function crawlStatusController(): null;+ // check if the crawl failed during kickoff (e.g. queue full)+ const crawlError = await getCrawlError(req.params.jobId);+let outputBulkA: {- status?: "completed" | "scraping" | "cancelled";+ status?: "completed" | "scraping" | "cancelled" | "failed";completed?: number;total?: number;creditsUsed?: number;@@ -221,6 +225,29 @@ export async function crawlStatusController(creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,};+ // if the crawl has a stored error and no jobs were ever created, mark as failed+ if (+ crawlError &&+ outputBulkA.total === 0 &&+ outputBulkA.status === "completed"+ ) {+ outputBulkA.status = "failed";+ }++ // if the crawl failed during kickoff, return immediately without fetching/processing jobs (there are none)+ if (outputBulkA.status === "failed" && crawlError) {+ return res.status(200).json({+ success: false,+ error: crawlError,+ status: "failed",+ completed: 0,+ total: 0,+ creditsUsed: outputBulkA.creditsUsed ?? 0,+ expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),+ data: [],+ });+ }+let outputBulkB: {data: Document[];next: string | undefined;diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 9eb9b77a2..045c77b53 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1180,6 +1180,16 @@ export type CrawlStatusResponse =expiresAt: string;next?: string;data: Document[];+ }+ | {+ success: false;+ status: "failed";+ error: string;+ completed: number;+ total: number;+ creditsUsed: number;+ expiresAt: string;+ data: Document[];};export type OngoingCrawlsResponse =diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex b139907ad..52211ae96 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -12,6 +12,7 @@ import { v7 as uuidv7 } from "uuid";import { logger } from "../../lib/logger";import {getCrawl,+ getCrawlError,getCrawlExpiry,getCrawlJobs,getDoneJobsOrdered,@@ -139,18 +140,43 @@ async function crawlStatusWS(}}- const status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] =+ // Check if the crawl failed during kickoff (e.g. queue full)+ const crawlError = await getCrawlError(req.params.jobId);++ let status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] =sc.cancelled? "cancelled": validJobStatuses.every(x => x[1] === "completed")? "completed": "scraping";+ if (crawlError && jobIDs.length === 0 && status === "completed") {+ status = "failed";+ }+jobIDs = validJobIDs; // Use validJobIDs instead of jobIDs for further processingconst doneJobs = await getJobs(doneJobIDs, logger);const data = doneJobs.map(x => x.returnvalue);+ if (status === "failed" && crawlError) {+ await send(ws, {+ type: "catchup",+ data: {+ success: false,+ error: crawlError,+ status: "failed",+ total: 0,+ completed: 0,+ creditsUsed: 0,+ expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),+ data: [],+ },+ });+ finished = true;+ return close(ws, 1000, { type: "done" });+ }+await send(ws, {type: "catchup",data: {diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 7b7d44cfa..f15b82f16 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -8,6 +8,7 @@ import {} from "./types";import {getCrawl,+ getCrawlError,getCrawlExpiry,getCrawlQualifiedJobCount,getDoneJobsOrderedLength,@@ -225,8 +226,11 @@ export async function crawlStatusController(): null;+ // check if the crawl failed during kickoff (e.g. queue full)+ const crawlError = await getCrawlError(req.params.jobId);+let outputBulkA: {- status?: "completed" | "scraping" | "cancelled";+ status?: "completed" | "scraping" | "cancelled" | "failed";completed?: number;total?: number;creditsUsed?: number;@@ -241,6 +245,29 @@ export async function crawlStatusController(creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,};+ // if the crawl has a stored error and no jobs were ever created, mark as failed+ if (+ crawlError &&+ outputBulkA.total === 0 &&+ outputBulkA.status === "completed"+ ) {+ outputBulkA.status = "failed";+ }++ // if the crawl failed during kickoff, return immediately without fetching/processing jobs (there are none)+ if (outputBulkA.status === "failed" && crawlError) {+ return res.status(200).json({+ success: false,+ error: crawlError,+ status: "failed",+ completed: 0,+ total: 0,+ creditsUsed: outputBulkA.creditsUsed ?? 0,+ expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),+ data: [],+ });+ }+let outputBulkB: {data: Document[];next: string | undefined;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 5f461da21..2cc7a3617 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1226,6 +1226,16 @@ export type CrawlStatusResponse =next?: string;data: Document[];warning?: string;+ }+ | {+ success: false;+ status: "failed";+ error: string;+ completed: number;+ total: number;+ creditsUsed: number;+ expiresAt: string;+ data: Document[];};export type OngoingCrawlsResponse =diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 83cfe7c9c..77b4f6332 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -259,6 +259,19 @@ export async function finishCrawlKickoff(id: string) {);}+export async function setCrawlError(id: string, error: string) {+ await redisEvictConnection.set(+ "crawl:" + id + ":error",+ error,+ "EX",+ 24 * 60 * 60,+ );+}++export async function getCrawlError(id: string): Promise<string | null> {+ return await redisEvictConnection.get("crawl:" + id + ":error");+}+export async function finishCrawl(id: string, __logger: Logger = _logger) {__logger.debug("Marking crawl as finished.", {module: "crawl-redis",diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 7844f0cc1..0a53dac95 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -26,6 +26,7 @@ import {lockURLsIndividually,normalizeURL,saveCrawl,+ setCrawlError,StoredCrawl,} from "../../lib/crawl-redis";import { redisEvictConnection } from "../redis";@@ -972,7 +973,12 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {} catch (error) {logger.error("An error occurred!", { error });await finishCrawlKickoff(job.data.crawl_id);- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ await setCrawlError(+ job.data.crawl_id,+ error instanceof Error+ ? error.message+ : "An unexpected error occurred during crawl setup",+ );return { success: false, error };}}