Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): optional A/B test to staging
Gergő Móricz0c8a54105371
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 5ae390364..266ae3677 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -34,6 +34,7 @@ import { LLMRefusalError } from "./transformers/llmExtract";import { urlSpecificParams } from "./lib/urlSpecificParams";import { loadMock, MockState } from "./lib/mock";import { CostTracking } from "../../lib/extract/extraction-service";+import { robustFetch } from "./lib/fetch";import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";import { checkRobotsTxt } from "../../lib/robots-txt";@@ -590,6 +591,42 @@ export async function scrapeURL(});}+ // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE+ try {+ const abRateEnv = process.env.SCRAPEURL_AB_RATE;+ const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;+ const shouldABTest = !meta.internalOptions.zeroDataRetention && abRate > 0 && Math.random() <= abRate;+ if (shouldABTest) {+ (async () => {+ try {+ const abLogger = meta.logger.child({ method: "scrapeURL/abTestToStaging" });+ abLogger.info("A/B-testing scrapeURL to staging");+ const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));+ await robustFetch({+ url: `http://firecrawl-app-staging:3002/v1/scrape`,+ method: "POST",+ body: {+ url: meta.url,+ ...meta.options,+ origin: (meta.options as any).origin ?? "api",+ timeout: meta.options.timeout ?? 30000,+ },+ logger: abLogger,+ tryCount: 1,+ ignoreResponse: true,+ mock: null,+ abort,+ });+ abLogger.info("A/B-testing scrapeURL (staging) request sent");+ } catch (error) {+ meta.logger.warn("A/B-testing scrapeURL (staging) failed", { error });+ }+ })();+ }+ } catch (error) {+ meta.logger.warn("Failed to initiate A/B test to staging", { error });+ }+try {while (true) {try {