Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: allow index use with waitFor (#2346)

toma9e993115d43
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 309a7c2aa..8e1e67166 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -447,7 +447,6 @@ export function shouldUseIndex(meta: Meta) {
return (
useIndex &&
process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true" &&
- meta.options.waitFor === 0 &&
!hasFormatOfType(meta.options.formats, "changeTracking") &&
// Skip index if a non-default PDF maxPages is specified
getPDFMaxPages(meta.options.parsers) === undefined &&
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 765b45b37..e343d223c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -142,6 +142,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
location_languages: meta.options.location?.languages ?? null,
status: document.metadata.statusCode,
is_precrawl: meta.internalOptions.isPreCrawl === true,
+ wait_time_ms: meta.options.waitFor > 0 ? meta.options.waitFor : null,
...urlSplitsHash.slice(0, 10).reduce(
(a, x, i) => ({
...a,
@@ -248,21 +249,25 @@ export async function scrapeURLWithIndex(
const checkpoint1 = Date.now();
- const { data, error } = await index_supabase_service.rpc("index_get_recent", {
- p_url_hash: urlHash,
- p_max_age_ms: maxAge,
- p_is_mobile: meta.options.mobile,
- p_block_ads: meta.options.blockAds,
- p_feature_screenshot: meta.featureFlags.has("screenshot"),
- p_feature_screenshot_fullscreen: meta.featureFlags.has(
- "screenshot@fullScreen",
- ),
- p_location_country: meta.options.location?.country ?? null,
- p_location_languages:
- (meta.options.location?.languages?.length ?? 0) > 0
- ? meta.options.location?.languages
- : null,
- });
+ const { data, error } = await index_supabase_service.rpc(
+ "index_get_recent_2",
+ {
+ p_url_hash: urlHash,
+ p_max_age_ms: maxAge,
+ p_is_mobile: meta.options.mobile,
+ p_block_ads: meta.options.blockAds,
+ p_feature_screenshot: meta.featureFlags.has("screenshot"),
+ p_feature_screenshot_fullscreen: meta.featureFlags.has(
+ "screenshot@fullScreen",
+ ),
+ p_location_country: meta.options.location?.country ?? null,
+ p_location_languages:
+ (meta.options.location?.languages?.length ?? 0) > 0
+ ? meta.options.location?.languages
+ : null,
+ p_wait_time_ms: meta.options.waitFor,
+ },
+ );
if (error || !data) {
throw new EngineError("Failed to retrieve URL from DB index", {