Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): replace f-e a/b with scrapeURL a/b

Gergő Móricza53e2222e325
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 840b350cc..28c519a24 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -223,9 +223,6 @@ export async function scrapeURLWithFireEngineChromeCDP(
const timeout = (timeToRun ?? 300000) + totalWait;
- // const shouldABTest = false;
- const shouldABTest = !meta.internalOptions.zeroDataRetention && Math.random() <= (1/30);
-
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestChromeCDP = {
url: meta.rewrittenUrl ?? meta.url,
@@ -248,31 +245,6 @@ export async function scrapeURLWithFireEngineChromeCDP(
zeroDataRetention: meta.internalOptions.zeroDataRetention,
};
- if (shouldABTest) {
- (async () => {
- try {
- meta.logger.info("AB-testing fire-engine", { request });
-
- await performFireEngineScrape(
- meta,
- meta.logger.child({
- method: "scrapeURLWithFireEngineChromeCDP/callFireEngineAB",
- request,
- }),
- request,
- timeout,
- meta.mock,
- meta.internalOptions.abort ?? AbortSignal.timeout(timeout),
- false,
- );
-
- meta.logger.info("AB-testing fire-engine success", { request });
- } catch (error) {
- meta.logger.error("AB-testing fire-engine failed", { error });
- }
- })();
- }
-
let response = await performFireEngineScrape(
meta,
meta.logger.child({
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 266ae3677..57b4e1062 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -603,7 +603,7 @@ export async function scrapeURL(
abLogger.info("A/B-testing scrapeURL to staging");
const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));
await robustFetch({
- url: `http://firecrawl-app-staging:3002/v1/scrape`,
+ url: `http://firecrawl-staging-app:3002/v1/scrape`,
method: "POST",
body: {
url: meta.url,