Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): allow index with enhanced mode (#3158)
tomsideguide9cdaae43a8fa
diff --git a/apps/api/src/lib/native-logging.ts b/apps/api/src/lib/native-logging.tsindex a643ab313..5b25bbc34 100644--- a/apps/api/src/lib/native-logging.ts+++ b/apps/api/src/lib/native-logging.ts@@ -3,7 +3,7 @@ import type { Logger } from "winston";const NATIVE_LOGS_SEPARATOR = "\n__native_logs__:";/** Matches the NativeLogEntry struct from Rust (@mendable/firecrawl-rs). */-export interface NativeLogEntry {+interface NativeLogEntry {level: string;target: string;message: string;diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 3524f403d..312c49f96 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -213,7 +213,7 @@ const engineOptions: {location: true,skipTlsVerification: true,useFastMode: true,- stealthProxy: false,+ stealthProxy: true,branding: false,disableAdblock: true,},@@ -270,7 +270,7 @@ const engineOptions: {mobile: true,skipTlsVerification: true,useFastMode: true,- stealthProxy: false,+ stealthProxy: true,branding: false,disableAdblock: false,},@@ -467,8 +467,7 @@ export function shouldUseIndex(meta: Meta) {meta.options.maxAge !== 0 &&(meta.options.headers === undefined ||Object.keys(meta.options.headers).length === 0) &&- (meta.options.actions === undefined || meta.options.actions.length === 0) &&- meta.options.proxy !== "stealth"+ (meta.options.actions === undefined || meta.options.actions.length === 0));}@@ -561,7 +560,11 @@ export async function buildFallbackList(meta: Meta): Promise<}}- if (selectedEngines.some(x => engineOptions[x.engine].quality > 0)) {+ if (+ selectedEngines.some(+ x => engineOptions[x.engine].quality > 0 && !x.engine.startsWith("index"),+ )+ ) {selectedEngines = selectedEngines.filter(x => engineOptions[x.engine].quality > 0,);@@ -628,7 +631,10 @@ export async function scrapeURLWithEngine(});const featureFlags = new Set(meta.featureFlags);- if (engineOptions[engine].features.stealthProxy) {+ if (+ engineOptions[engine].features.stealthProxy &&+ !engine.startsWith("index") // don't force stealth proxy for index+ ) {featureFlags.add("stealthProxy");}diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 2bb91843c..31ba1afd5 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -151,6 +151,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {location_languages: meta.options.location?.languages ?? null,status: document.metadata.statusCode,is_precrawl: meta.internalOptions.isPreCrawl === true,+ is_stealth: meta.featureFlags.has("stealthProxy"),wait_time_ms: meta.options.waitFor > 0 ? meta.options.waitFor : null,...urlSplitsHash.slice(0, 10).reduce((a, x, i) => ({@@ -259,7 +260,7 @@ export async function scrapeURLWithIndex(const checkpoint1 = Date.now();const { data, error } = await index_supabase_service.rpc(- "index_get_recent_3",+ "index_get_recent_4",{p_url_hash: urlHash,p_max_age_ms: maxAge,@@ -275,6 +276,7 @@ export async function scrapeURLWithIndex(? meta.options.location?.languages: null,p_wait_time_ms: meta.options.waitFor,+ p_is_stealth: meta.featureFlags.has("stealthProxy"),p_min_age_ms: meta.options.minAge ?? null,},);