Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sitemap): reenable fallback to tlsclient
Móricz Gergő3761eb17a7e5
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 7cb9e2742..bb9890778 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -51,45 +51,42 @@ export async function getLinksFromSitemap() {content = fetchResponse.document.rawHtml!;} else {- // logger.debug(- // "Failed to scrape sitemap via fetch, falling back to TLSClient...",- // {- // error: fetchResponse.success- // ? fetchResponse.document- // : fetchResponse.error,- // },- // );-- // const tlsResponse = await scrapeURL(- // "sitemap",- // sitemapUrl,- // scrapeOptions.parse({ formats: ["rawHtml"] }),- // { forceEngine: "fire-engine;tlsclient", v0DisableJsDom: true },- // );-- // if (- // tlsResponse.success &&- // tlsResponse.document.metadata.statusCode >= 200 &&- // tlsResponse.document.metadata.statusCode < 300- // ) {- // content = tlsResponse.document.rawHtml!;- // } else {+ logger.debug(+ "Failed to scrape sitemap via fetch, falling back to TLSClient...",+ {+ error: fetchResponse.success+ ? fetchResponse.document+ : fetchResponse.error,+ },+ );++ const tlsResponse = await scrapeURL(+ "sitemap",+ sitemapUrl,+ scrapeOptions.parse({ formats: ["rawHtml"] }),+ { forceEngine: "fire-engine;tlsclient", v0DisableJsDom: true },+ );++ if (+ tlsResponse.success &&+ tlsResponse.document.metadata.statusCode >= 200 &&+ tlsResponse.document.metadata.statusCode < 300+ ) {+ content = tlsResponse.document.rawHtml!;+ } else {logger.error(`Request failed for ${sitemapUrl}, ran out of engines!`,{method: "getLinksFromSitemap",mode,sitemapUrl,- // error: tlsResponse.success- // ? tlsResponse.document- // : tlsResponse.error,- error: fetchResponse.success- ? fetchResponse.document- : fetchResponse.error,+ error: tlsResponse.success+ ? tlsResponse.document+ : tlsResponse.error,},);return 0;- // }+ }}} else {const fetchResponse = await scrapeURL(