Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): replace f-e a/b with scrapeURL a/b
Gergő Móricza53e2222e325
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 840b350cc..28c519a24 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -223,9 +223,6 @@ export async function scrapeURLWithFireEngineChromeCDP(const timeout = (timeToRun ?? 300000) + totalWait;- // const shouldABTest = false;- const shouldABTest = !meta.internalOptions.zeroDataRetention && Math.random() <= (1/30);-const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestChromeCDP = {url: meta.rewrittenUrl ?? meta.url,@@ -248,31 +245,6 @@ export async function scrapeURLWithFireEngineChromeCDP(zeroDataRetention: meta.internalOptions.zeroDataRetention,};- if (shouldABTest) {- (async () => {- try {- meta.logger.info("AB-testing fire-engine", { request });-- await performFireEngineScrape(- meta,- meta.logger.child({- method: "scrapeURLWithFireEngineChromeCDP/callFireEngineAB",- request,- }),- request,- timeout,- meta.mock,- meta.internalOptions.abort ?? AbortSignal.timeout(timeout),- false,- );-- meta.logger.info("AB-testing fire-engine success", { request });- } catch (error) {- meta.logger.error("AB-testing fire-engine failed", { error });- }- })();- }-let response = await performFireEngineScrape(meta,meta.logger.child({diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 266ae3677..57b4e1062 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -603,7 +603,7 @@ export async function scrapeURL(abLogger.info("A/B-testing scrapeURL to staging");const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));await robustFetch({- url: `http://firecrawl-app-staging:3002/v1/scrape`,+ url: `http://firecrawl-staging-app:3002/v1/scrape`,method: "POST",body: {url: meta.url,