Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): optional A/B test to staging

Gergő Móricz0c8a54105371
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 5ae390364..266ae3677 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -34,6 +34,7 @@ import { LLMRefusalError } from "./transformers/llmExtract";
import { urlSpecificParams } from "./lib/urlSpecificParams";
import { loadMock, MockState } from "./lib/mock";
import { CostTracking } from "../../lib/extract/extraction-service";
+import { robustFetch } from "./lib/fetch";
import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";
import { checkRobotsTxt } from "../../lib/robots-txt";
@@ -590,6 +591,42 @@ export async function scrapeURL(
});
}
+ // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
+ try {
+ const abRateEnv = process.env.SCRAPEURL_AB_RATE;
+ const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
+ const shouldABTest = !meta.internalOptions.zeroDataRetention && abRate > 0 && Math.random() <= abRate;
+ if (shouldABTest) {
+ (async () => {
+ try {
+ const abLogger = meta.logger.child({ method: "scrapeURL/abTestToStaging" });
+ abLogger.info("A/B-testing scrapeURL to staging");
+ const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));
+ await robustFetch({
+ url: `http://firecrawl-app-staging:3002/v1/scrape`,
+ method: "POST",
+ body: {
+ url: meta.url,
+ ...meta.options,
+ origin: (meta.options as any).origin ?? "api",
+ timeout: meta.options.timeout ?? 30000,
+ },
+ logger: abLogger,
+ tryCount: 1,
+ ignoreResponse: true,
+ mock: null,
+ abort,
+ });
+ abLogger.info("A/B-testing scrapeURL (staging) request sent");
+ } catch (error) {
+ meta.logger.warn("A/B-testing scrapeURL (staging) failed", { error });
+ }
+ })();
+ }
+ } catch (error) {
+ meta.logger.warn("Failed to initiate A/B test to staging", { error });
+ }
+
try {
while (true) {
try {