Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: proxy billing for cached scrapes (#3496)
tomsideguidef1f4e88fa6a0
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex bc1e68791..02577f8bd 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -102,6 +102,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {: undefined,contentType: document.metadata.contentType,postprocessorsUsed: document.metadata.postprocessorsUsed,+ proxyUsed: document.metadata.proxyUsed,});} catch (error) {meta.logger.error("Failed to save document to index", {@@ -404,7 +405,9 @@ export async function scrapeURLWithIndex(postprocessorsUsed: doc.postprocessorsUsed,- proxyUsed: doc.proxyUsed ?? "basic",+ proxyUsed:+ doc.proxyUsed ??+ (meta.featureFlags.has("stealthProxy") ? "stealth" : "basic"), // this can be dropped after june 2026, it's here to backfill proxyUsed for older index entries that don't have it};}diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex ae71536b7..2711afbc4 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -165,6 +165,7 @@ export async function saveIndexToGCS(pdfMetadata?: PdfMetadata;contentType?: string;postprocessorsUsed?: string[];+ proxyUsed?: "basic" | "stealth";},): Promise<void> {return await withSpan("firecrawl-index-save-to-gcs", async span => {