Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: re-introduce direct f-e A/B

Gergő Móricz44a8e06a9ad5
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 93b4e429a..876205831 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -17,6 +17,7 @@ import {
UnsupportedFileError,
} from "../../error";
import { Meta } from "../..";
+import { abTestFireEngine } from "../../../../services/ab-test";
export type FireEngineScrapeRequestCommon = {
url: string;
@@ -181,6 +182,8 @@ export async function fireEngineScrape<
abort?: AbortSignal,
production = true,
): Promise<z.infer<typeof processingSchema> | FireEngineCheckStatusSuccess> {
+ abTestFireEngine(request);
+
let status = await robustFetch({
url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,
method: "POST",
diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.ts
index 6b02c149d..358ae65bd 100644
--- a/apps/api/src/services/ab-test.ts
+++ b/apps/api/src/services/ab-test.ts
@@ -1,6 +1,12 @@
import { WebScraperOptions } from "../types";
import { logger as _logger } from "../lib/logger";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
+import {
+ FireEngineScrapeRequestChromeCDP,
+ FireEngineScrapeRequestCommon,
+ FireEngineScrapeRequestPlaywright,
+ FireEngineScrapeRequestTLSClient,
+} from "../scraper/scrapeURL/engines/fire-engine/scrape";
export function abTestJob(webScraperOptions: WebScraperOptions) {
// Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
@@ -56,3 +62,51 @@ export function abTestJob(webScraperOptions: WebScraperOptions) {
abLogger.warn("Failed to initiate A/B test to staging", { error });
}
}
+
+export function abTestFireEngine(
+ feRequest: FireEngineScrapeRequestCommon &
+ (
+ | FireEngineScrapeRequestChromeCDP
+ | FireEngineScrapeRequestPlaywright
+ | FireEngineScrapeRequestTLSClient
+ ),
+) {
+ // Global A/B test: mirror request to staging fire-engine based on SCRAPEURL_AB_RATE
+ const abLogger = _logger.child({ method: "ABTestToStaging" });
+ try {
+ const abRateEnv = process.env.FIRE_ENGINE_AB_RATE;
+ const abHostEnv = process.env.FIRE_ENGINE_AB_HOST;
+ const abRate =
+ abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
+ const shouldABTest =
+ !feRequest.zeroDataRetention &&
+ abRate > 0 &&
+ Math.random() <= abRate &&
+ abHostEnv;
+ if (shouldABTest) {
+ (async () => {
+ try {
+ abLogger.info("A/B-testing scrapeURL to staging");
+ const abort = AbortSignal.timeout(
+ Math.min(60000, (feRequest.timeout ?? 30000) + 10000),
+ );
+ await robustFetch({
+ url: `http://${abHostEnv}/scrape`,
+ method: "POST",
+ body: feRequest,
+ logger: abLogger,
+ tryCount: 1,
+ ignoreResponse: true,
+ mock: null,
+ abort,
+ });
+ abLogger.info("A/B-testing scrapeURL (staging) request sent");
+ } catch (error) {
+ abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
+ }
+ })();
+ }
+ } catch (error) {
+ abLogger.warn("Failed to initiate A/B test to staging", { error });
+ }
+}