Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(map): use tlsclient to fetch sitemap for better reliability (#2034)

Gergő Móricz879908a810d3
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 4971a6f13..5637b6c3a 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -50,8 +50,8 @@ export async function getLinksFromSitemap(
{
forceEngine: [
...(maxAge > 0 ? ["index" as const] : []),
+ ...((mode === "fire-engine" && useFireEngine) ? ["fire-engine;tlsclient" as const, "fire-engine;tlsclient;stealth" as const] : []),
"fetch",
- ...((mode === "fire-engine" && useFireEngine) ? ["fire-engine;tlsclient" as const] : []),
],
v0DisableJsDom: true,
externalAbort: abort ? {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index fdeaad355..4addc40b6 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -348,7 +348,7 @@ export const engineOptions: {
atsv: true,
location: true,
mobile: false,
- skipTlsVerification: false,
+ skipTlsVerification: true,
useFastMode: true,
stealthProxy: false,
disableAdblock: false,
@@ -366,7 +366,7 @@ export const engineOptions: {
atsv: true,
location: true,
mobile: false,
- skipTlsVerification: false,
+ skipTlsVerification: true,
useFastMode: true,
stealthProxy: true,
disableAdblock: false,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 32781d933..b98448a86 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -630,7 +630,7 @@ export async function scrapeURL(
} catch (error) {
if (
error instanceof AddFeatureError &&
- meta.internalOptions.forceEngine === undefined
+ (meta.internalOptions.forceEngine === undefined || Array.isArray(meta.internalOptions.forceEngine))
) {
meta.logger.debug(
"More feature flags requested by scraper: adding " +
@@ -645,7 +645,7 @@ export async function scrapeURL(
}
} else if (
error instanceof RemoveFeatureError &&
- meta.internalOptions.forceEngine === undefined
+ (meta.internalOptions.forceEngine === undefined || Array.isArray(meta.internalOptions.forceEngine))
) {
meta.logger.debug(
"Incorrect feature flags reported by scraper: removing " +