Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): surface crawl error message (ENG-4574) (#3057)

tomsideguided558ecb91e2a
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index f39d4103a..27589a35f 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -8,6 +8,7 @@ import {
} from "./types";
import {
getCrawl,
+ getCrawlError,
getCrawlExpiry,
getDoneJobsOrderedLength,
isCrawlKickoffFinished,
@@ -205,8 +206,11 @@ export async function crawlStatusController(
)
: null;
+ // check if the crawl failed during kickoff (e.g. queue full)
+ const crawlError = await getCrawlError(req.params.jobId);
+
let outputBulkA: {
- status?: "completed" | "scraping" | "cancelled";
+ status?: "completed" | "scraping" | "cancelled" | "failed";
completed?: number;
total?: number;
creditsUsed?: number;
@@ -221,6 +225,29 @@ export async function crawlStatusController(
creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,
};
+ // if the crawl has a stored error and no jobs were ever created, mark as failed
+ if (
+ crawlError &&
+ outputBulkA.total === 0 &&
+ outputBulkA.status === "completed"
+ ) {
+ outputBulkA.status = "failed";
+ }
+
+ // if the crawl failed during kickoff, return immediately without fetching/processing jobs (there are none)
+ if (outputBulkA.status === "failed" && crawlError) {
+ return res.status(200).json({
+ success: false,
+ error: crawlError,
+ status: "failed",
+ completed: 0,
+ total: 0,
+ creditsUsed: outputBulkA.creditsUsed ?? 0,
+ expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
+ data: [],
+ });
+ }
+
let outputBulkB: {
data: Document[];
next: string | undefined;
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 9eb9b77a2..045c77b53 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1180,6 +1180,16 @@ export type CrawlStatusResponse =
expiresAt: string;
next?: string;
data: Document[];
+ }
+ | {
+ success: false;
+ status: "failed";
+ error: string;
+ completed: number;
+ total: number;
+ creditsUsed: number;
+ expiresAt: string;
+ data: Document[];
};
export type OngoingCrawlsResponse =
diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts
index b139907ad..52211ae96 100644
--- a/apps/api/src/controllers/v2/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts
@@ -12,6 +12,7 @@ import { v7 as uuidv7 } from "uuid";
import { logger } from "../../lib/logger";
import {
getCrawl,
+ getCrawlError,
getCrawlExpiry,
getCrawlJobs,
getDoneJobsOrdered,
@@ -139,18 +140,43 @@ async function crawlStatusWS(
}
}
- const status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] =
+ // Check if the crawl failed during kickoff (e.g. queue full)
+ const crawlError = await getCrawlError(req.params.jobId);
+
+ let status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] =
sc.cancelled
? "cancelled"
: validJobStatuses.every(x => x[1] === "completed")
? "completed"
: "scraping";
+ if (crawlError && jobIDs.length === 0 && status === "completed") {
+ status = "failed";
+ }
+
jobIDs = validJobIDs; // Use validJobIDs instead of jobIDs for further processing
const doneJobs = await getJobs(doneJobIDs, logger);
const data = doneJobs.map(x => x.returnvalue);
+ if (status === "failed" && crawlError) {
+ await send(ws, {
+ type: "catchup",
+ data: {
+ success: false,
+ error: crawlError,
+ status: "failed",
+ total: 0,
+ completed: 0,
+ creditsUsed: 0,
+ expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
+ data: [],
+ },
+ });
+ finished = true;
+ return close(ws, 1000, { type: "done" });
+ }
+
await send(ws, {
type: "catchup",
data: {
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 7b7d44cfa..f15b82f16 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -8,6 +8,7 @@ import {
} from "./types";
import {
getCrawl,
+ getCrawlError,
getCrawlExpiry,
getCrawlQualifiedJobCount,
getDoneJobsOrderedLength,
@@ -225,8 +226,11 @@ export async function crawlStatusController(
)
: null;
+ // check if the crawl failed during kickoff (e.g. queue full)
+ const crawlError = await getCrawlError(req.params.jobId);
+
let outputBulkA: {
- status?: "completed" | "scraping" | "cancelled";
+ status?: "completed" | "scraping" | "cancelled" | "failed";
completed?: number;
total?: number;
creditsUsed?: number;
@@ -241,6 +245,29 @@ export async function crawlStatusController(
creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,
};
+ // if the crawl has a stored error and no jobs were ever created, mark as failed
+ if (
+ crawlError &&
+ outputBulkA.total === 0 &&
+ outputBulkA.status === "completed"
+ ) {
+ outputBulkA.status = "failed";
+ }
+
+ // if the crawl failed during kickoff, return immediately without fetching/processing jobs (there are none)
+ if (outputBulkA.status === "failed" && crawlError) {
+ return res.status(200).json({
+ success: false,
+ error: crawlError,
+ status: "failed",
+ completed: 0,
+ total: 0,
+ creditsUsed: outputBulkA.creditsUsed ?? 0,
+ expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),
+ data: [],
+ });
+ }
+
let outputBulkB: {
data: Document[];
next: string | undefined;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 5f461da21..2cc7a3617 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1226,6 +1226,16 @@ export type CrawlStatusResponse =
next?: string;
data: Document[];
warning?: string;
+ }
+ | {
+ success: false;
+ status: "failed";
+ error: string;
+ completed: number;
+ total: number;
+ creditsUsed: number;
+ expiresAt: string;
+ data: Document[];
};
export type OngoingCrawlsResponse =
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 83cfe7c9c..77b4f6332 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -259,6 +259,19 @@ export async function finishCrawlKickoff(id: string) {
);
}
+export async function setCrawlError(id: string, error: string) {
+ await redisEvictConnection.set(
+ "crawl:" + id + ":error",
+ error,
+ "EX",
+ 24 * 60 * 60,
+ );
+}
+
+export async function getCrawlError(id: string): Promise<string | null> {
+ return await redisEvictConnection.get("crawl:" + id + ":error");
+}
+
export async function finishCrawl(id: string, __logger: Logger = _logger) {
__logger.debug("Marking crawl as finished.", {
module: "crawl-redis",
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 7844f0cc1..0a53dac95 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -26,6 +26,7 @@ import {
lockURLsIndividually,
normalizeURL,
saveCrawl,
+ setCrawlError,
StoredCrawl,
} from "../../lib/crawl-redis";
import { redisEvictConnection } from "../redis";
@@ -972,7 +973,12 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
} catch (error) {
logger.error("An error occurred!", { error });
await finishCrawlKickoff(job.data.crawl_id);
- const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ await setCrawlError(
+ job.data.crawl_id,
+ error instanceof Error
+ ? error.message
+ : "An unexpected error occurred during crawl setup",
+ );
return { success: false, error };
}
}