Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL/index): don't put results by "dumb" engines into the index

Gergő Móricza08d52e45d38
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 8231f78d6..f96b847d4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -10,6 +10,14 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
&& meta.winnerEngine !== "cache"
&& meta.winnerEngine !== "index"
&& meta.winnerEngine !== "index;documents"
+ && (
+ meta.internalOptions.teamId === "sitemap"
+ || (
+ meta.winnerEngine !== "fire-engine;tlsclient"
+ && meta.winnerEngine !== "fire-engine;tlsclient;stealth"
+ && meta.winnerEngine !== "fetch"
+ )
+ )
&& !meta.featureFlags.has("actions")
&& (
meta.options.headers === undefined