Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL/index): don't put results by "dumb" engines into the index
Gergő Móricza08d52e45d38
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 8231f78d6..f96b847d4 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -10,6 +10,14 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {&& meta.winnerEngine !== "cache"&& meta.winnerEngine !== "index"&& meta.winnerEngine !== "index;documents"+ && (+ meta.internalOptions.teamId === "sitemap"+ || (+ meta.winnerEngine !== "fire-engine;tlsclient"+ && meta.winnerEngine !== "fire-engine;tlsclient;stealth"+ && meta.winnerEngine !== "fetch"+ )+ )&& !meta.featureFlags.has("actions")&& (meta.options.headers === undefined