Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): allow index with enhanced mode (#3158)

tomsideguide9cdaae43a8fa
diff --git a/apps/api/src/lib/native-logging.ts b/apps/api/src/lib/native-logging.ts
index a643ab313..5b25bbc34 100644
--- a/apps/api/src/lib/native-logging.ts
+++ b/apps/api/src/lib/native-logging.ts
@@ -3,7 +3,7 @@ import type { Logger } from "winston";
const NATIVE_LOGS_SEPARATOR = "\n__native_logs__:";
/** Matches the NativeLogEntry struct from Rust (@mendable/firecrawl-rs). */
-export interface NativeLogEntry {
+interface NativeLogEntry {
level: string;
target: string;
message: string;
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 3524f403d..312c49f96 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -213,7 +213,7 @@ const engineOptions: {
location: true,
skipTlsVerification: true,
useFastMode: true,
- stealthProxy: false,
+ stealthProxy: true,
branding: false,
disableAdblock: true,
},
@@ -270,7 +270,7 @@ const engineOptions: {
mobile: true,
skipTlsVerification: true,
useFastMode: true,
- stealthProxy: false,
+ stealthProxy: true,
branding: false,
disableAdblock: false,
},
@@ -467,8 +467,7 @@ export function shouldUseIndex(meta: Meta) {
meta.options.maxAge !== 0 &&
(meta.options.headers === undefined ||
Object.keys(meta.options.headers).length === 0) &&
- (meta.options.actions === undefined || meta.options.actions.length === 0) &&
- meta.options.proxy !== "stealth"
+ (meta.options.actions === undefined || meta.options.actions.length === 0)
);
}
@@ -561,7 +560,11 @@ export async function buildFallbackList(meta: Meta): Promise<
}
}
- if (selectedEngines.some(x => engineOptions[x.engine].quality > 0)) {
+ if (
+ selectedEngines.some(
+ x => engineOptions[x.engine].quality > 0 && !x.engine.startsWith("index"),
+ )
+ ) {
selectedEngines = selectedEngines.filter(
x => engineOptions[x.engine].quality > 0,
);
@@ -628,7 +631,10 @@ export async function scrapeURLWithEngine(
});
const featureFlags = new Set(meta.featureFlags);
- if (engineOptions[engine].features.stealthProxy) {
+ if (
+ engineOptions[engine].features.stealthProxy &&
+ !engine.startsWith("index") // don't force stealth proxy for index
+ ) {
featureFlags.add("stealthProxy");
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 2bb91843c..31ba1afd5 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -151,6 +151,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
location_languages: meta.options.location?.languages ?? null,
status: document.metadata.statusCode,
is_precrawl: meta.internalOptions.isPreCrawl === true,
+ is_stealth: meta.featureFlags.has("stealthProxy"),
wait_time_ms: meta.options.waitFor > 0 ? meta.options.waitFor : null,
...urlSplitsHash.slice(0, 10).reduce(
(a, x, i) => ({
@@ -259,7 +260,7 @@ export async function scrapeURLWithIndex(
const checkpoint1 = Date.now();
const { data, error } = await index_supabase_service.rpc(
- "index_get_recent_3",
+ "index_get_recent_4",
{
p_url_hash: urlHash,
p_max_age_ms: maxAge,
@@ -275,6 +276,7 @@ export async function scrapeURLWithIndex(
? meta.options.location?.languages
: null,
p_wait_time_ms: meta.options.waitFor,
+ p_is_stealth: meta.featureFlags.has("stealthProxy"),
p_min_age_ms: meta.options.minAge ?? null,
},
);