Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(map): use tlsclient to fetch sitemap for better reliability (#2034)
Gergő Móricz879908a810d3
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 4971a6f13..5637b6c3a 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -50,8 +50,8 @@ export async function getLinksFromSitemap({forceEngine: [...(maxAge > 0 ? ["index" as const] : []),+ ...((mode === "fire-engine" && useFireEngine) ? ["fire-engine;tlsclient" as const, "fire-engine;tlsclient;stealth" as const] : []),"fetch",- ...((mode === "fire-engine" && useFireEngine) ? ["fire-engine;tlsclient" as const] : []),],v0DisableJsDom: true,externalAbort: abort ? {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex fdeaad355..4addc40b6 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -348,7 +348,7 @@ export const engineOptions: {atsv: true,location: true,mobile: false,- skipTlsVerification: false,+ skipTlsVerification: true,useFastMode: true,stealthProxy: false,disableAdblock: false,@@ -366,7 +366,7 @@ export const engineOptions: {atsv: true,location: true,mobile: false,- skipTlsVerification: false,+ skipTlsVerification: true,useFastMode: true,stealthProxy: true,disableAdblock: false,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 32781d933..b98448a86 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -630,7 +630,7 @@ export async function scrapeURL(} catch (error) {if (error instanceof AddFeatureError &&- meta.internalOptions.forceEngine === undefined+ (meta.internalOptions.forceEngine === undefined || Array.isArray(meta.internalOptions.forceEngine))) {meta.logger.debug("More feature flags requested by scraper: adding " +@@ -645,7 +645,7 @@ export async function scrapeURL(}} else if (error instanceof RemoveFeatureError &&- meta.internalOptions.forceEngine === undefined+ (meta.internalOptions.forceEngine === undefined || Array.isArray(meta.internalOptions.forceEngine))) {meta.logger.debug("Incorrect feature flags reported by scraper: removing " +