Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl-status): handle failed jobs (oops)
Gergő Móriczd8613899e323
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 8493e855a..1f2f92e93 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -99,8 +99,14 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusPara// both loops will break once we cross the byte counterfor (let ii = 0; ii < jobs.length && bytes < bytesLimit; ii++) {const job = jobs[ii];+ const state = await job.getState();++ if (state === "failed") {+ continue;+ }+if (job.returnvalue === undefined) {- logger.warn("Job was considered done, but returnvalue is undefined!", { jobId: job.id });+ logger.warn("Job was considered done, but returnvalue is undefined!", { jobId: job.id, state });continue;}doneJobs.push(job);@@ -113,7 +119,7 @@ export async function crawlStatusController(req: RequestWithAuth<CrawlStatusParadoneJobs.splice(doneJobs.length - 1, 1);}} else {- doneJobs = await getJobs(doneJobsOrder);+ doneJobs = (await Promise.all((await getJobs(doneJobsOrder)).map(async x => (await x.getState()) === "failed" ? null : x))).filter(x => x !== null);}const data = doneJobs.map(x => x.returnvalue);