Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(WebScraper): always report error of last scraper in order
Gergő Móricz43730b5db6b7
diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.tsindex 80491f3c0..66ae744d6 100644--- a/apps/api/src/scraper/WebScraper/single_url.ts+++ b/apps/api/src/scraper/WebScraper/single_url.ts@@ -392,8 +392,12 @@ export async function scrapSingleUrl(if (attempt.pageStatusCode) {pageStatusCode = attempt.pageStatusCode;}- if (attempt.pageError && attempt.pageStatusCode >= 400) {+ if (attempt.pageError && (attempt.pageStatusCode >= 400 || scrapersInOrder.indexOf(scraper) === scrapersInOrder.length - 1)) { // force pageError if it's the last scraper and it failed toopageError = attempt.pageError;++ if (attempt.pageStatusCode < 400 || !attempt.pageStatusCode) {+ pageStatusCode = 500;+ }} else if (attempt && attempt.pageStatusCode && attempt.pageStatusCode < 400) {pageError = undefined;}