Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: proxy billing for cached scrapes (#3496)

tomsideguidef1f4e88fa6a0
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index bc1e68791..02577f8bd 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -102,6 +102,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
: undefined,
contentType: document.metadata.contentType,
postprocessorsUsed: document.metadata.postprocessorsUsed,
+ proxyUsed: document.metadata.proxyUsed,
});
} catch (error) {
meta.logger.error("Failed to save document to index", {
@@ -404,7 +405,9 @@ export async function scrapeURLWithIndex(
postprocessorsUsed: doc.postprocessorsUsed,
- proxyUsed: doc.proxyUsed ?? "basic",
+ proxyUsed:
+ doc.proxyUsed ??
+ (meta.featureFlags.has("stealthProxy") ? "stealth" : "basic"), // this can be dropped after june 2026, it's here to backfill proxyUsed for older index entries that don't have it
};
}
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index ae71536b7..2711afbc4 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -165,6 +165,7 @@ export async function saveIndexToGCS(
pdfMetadata?: PdfMetadata;
contentType?: string;
postprocessorsUsed?: string[];
+ proxyUsed?: "basic" | "stealth";
},
): Promise<void> {
return await withSpan("firecrawl-index-save-to-gcs", async span => {