Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(sitemap): reenable fallback to tlsclient

Móricz Gergő3761eb17a7e5
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 7cb9e2742..bb9890778 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -51,45 +51,42 @@ export async function getLinksFromSitemap(
) {
content = fetchResponse.document.rawHtml!;
} else {
- // logger.debug(
- // "Failed to scrape sitemap via fetch, falling back to TLSClient...",
- // {
- // error: fetchResponse.success
- // ? fetchResponse.document
- // : fetchResponse.error,
- // },
- // );
-
- // const tlsResponse = await scrapeURL(
- // "sitemap",
- // sitemapUrl,
- // scrapeOptions.parse({ formats: ["rawHtml"] }),
- // { forceEngine: "fire-engine;tlsclient", v0DisableJsDom: true },
- // );
-
- // if (
- // tlsResponse.success &&
- // tlsResponse.document.metadata.statusCode >= 200 &&
- // tlsResponse.document.metadata.statusCode < 300
- // ) {
- // content = tlsResponse.document.rawHtml!;
- // } else {
+ logger.debug(
+ "Failed to scrape sitemap via fetch, falling back to TLSClient...",
+ {
+ error: fetchResponse.success
+ ? fetchResponse.document
+ : fetchResponse.error,
+ },
+ );
+
+ const tlsResponse = await scrapeURL(
+ "sitemap",
+ sitemapUrl,
+ scrapeOptions.parse({ formats: ["rawHtml"] }),
+ { forceEngine: "fire-engine;tlsclient", v0DisableJsDom: true },
+ );
+
+ if (
+ tlsResponse.success &&
+ tlsResponse.document.metadata.statusCode >= 200 &&
+ tlsResponse.document.metadata.statusCode < 300
+ ) {
+ content = tlsResponse.document.rawHtml!;
+ } else {
logger.error(
`Request failed for ${sitemapUrl}, ran out of engines!`,
{
method: "getLinksFromSitemap",
mode,
sitemapUrl,
- // error: tlsResponse.success
- // ? tlsResponse.document
- // : tlsResponse.error,
- error: fetchResponse.success
- ? fetchResponse.document
- : fetchResponse.error,
+ error: tlsResponse.success
+ ? tlsResponse.document
+ : tlsResponse.error,
},
);
return 0;
- // }
+ }
}
} else {
const fetchResponse = await scrapeURL(