Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/metrics): add log on unhappy path
Gergő Móricz450f3a94717e
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex a4099210f..d44f60f9a 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -886,6 +886,15 @@ export async function scrapeURL(// if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {// throw new FEPageLoadFailed();// } else+ meta.logger.debug("scrapeURL metrics", {+ module: "scrapeURL/metrics",+ timeTaken: Date.now() - startTime,+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,+ shouldUseIndex: shouldUseIndex(meta),+ success: false,+ indexHit: false,+ });+if (error instanceof NoEnginesLeftError) {meta.logger.warn("scrapeURL: All scraping engines failed!", { error });} else if (error instanceof LLMRefusalError) {