Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawl-status): handle failed jobs (oops)

Gergő Móriczd8613899e323
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 8493e855a..1f2f92e93 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -99,8 +99,14 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
// both loops will break once we cross the byte counter
for (let ii = 0; ii < jobs.length && bytes < bytesLimit; ii++) {
const job = jobs[ii];
+ const state = await job.getState();
+
+ if (state === "failed") {
+ continue;
+ }
+
if (job.returnvalue === undefined) {
- logger.warn("Job was considered done, but returnvalue is undefined!", { jobId: job.id });
+ logger.warn("Job was considered done, but returnvalue is undefined!", { jobId: job.id, state });
continue;
}
doneJobs.push(job);
@@ -113,7 +119,7 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara
doneJobs.splice(doneJobs.length - 1, 1);
}
} else {
- doneJobs = await getJobs(doneJobsOrder);
+ doneJobs = (await Promise.all((await getJobs(doneJobsOrder)).map(async x => (await x.getState()) === "failed" ? null : x))).filter(x => x !== null);
}
const data = doneJobs.map(x => x.returnvalue);