Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: re-introduce direct f-e A/B
Gergő Móricz44a8e06a9ad5
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 93b4e429a..876205831 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -17,6 +17,7 @@ import {UnsupportedFileError,} from "../../error";import { Meta } from "../..";+import { abTestFireEngine } from "../../../../services/ab-test";export type FireEngineScrapeRequestCommon = {url: string;@@ -181,6 +182,8 @@ export async function fireEngineScrape<abort?: AbortSignal,production = true,): Promise<z.infer<typeof processingSchema> | FireEngineCheckStatusSuccess> {+ abTestFireEngine(request);+let status = await robustFetch({url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,method: "POST",diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.tsindex 6b02c149d..358ae65bd 100644--- a/apps/api/src/services/ab-test.ts+++ b/apps/api/src/services/ab-test.ts@@ -1,6 +1,12 @@import { WebScraperOptions } from "../types";import { logger as _logger } from "../lib/logger";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";+import {+ FireEngineScrapeRequestChromeCDP,+ FireEngineScrapeRequestCommon,+ FireEngineScrapeRequestPlaywright,+ FireEngineScrapeRequestTLSClient,+} from "../scraper/scrapeURL/engines/fire-engine/scrape";export function abTestJob(webScraperOptions: WebScraperOptions) {// Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE@@ -56,3 +62,51 @@ export function abTestJob(webScraperOptions: WebScraperOptions) {abLogger.warn("Failed to initiate A/B test to staging", { error });}}++export function abTestFireEngine(+ feRequest: FireEngineScrapeRequestCommon &+ (+ | FireEngineScrapeRequestChromeCDP+ | FireEngineScrapeRequestPlaywright+ | FireEngineScrapeRequestTLSClient+ ),+) {+ // Global A/B test: mirror request to staging fire-engine based on SCRAPEURL_AB_RATE+ const abLogger = _logger.child({ method: "ABTestToStaging" });+ try {+ const abRateEnv = process.env.FIRE_ENGINE_AB_RATE;+ const abHostEnv = process.env.FIRE_ENGINE_AB_HOST;+ const abRate =+ abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;+ const shouldABTest =+ !feRequest.zeroDataRetention &&+ abRate > 0 &&+ Math.random() <= abRate &&+ abHostEnv;+ if (shouldABTest) {+ (async () => {+ try {+ abLogger.info("A/B-testing scrapeURL to staging");+ const abort = AbortSignal.timeout(+ Math.min(60000, (feRequest.timeout ?? 30000) + 10000),+ );+ await robustFetch({+ url: `http://${abHostEnv}/scrape`,+ method: "POST",+ body: feRequest,+ logger: abLogger,+ tryCount: 1,+ ignoreResponse: true,+ mock: null,+ abort,+ });+ abLogger.info("A/B-testing scrapeURL (staging) request sent");+ } catch (error) {+ abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });+ }+ })();+ }+ } catch (error) {+ abLogger.warn("Failed to initiate A/B test to staging", { error });+ }+}