Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/ab): make host configurable
Gergő Móricz055f9c36cfb4
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 57b4e1062..fca8e3fc6 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -594,8 +594,9 @@ export async function scrapeURL(// Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATEtry {const abRateEnv = process.env.SCRAPEURL_AB_RATE;+ const abHostEnv = process.env.SCRAPEURL_AB_HOST;const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;- const shouldABTest = !meta.internalOptions.zeroDataRetention && abRate > 0 && Math.random() <= abRate;+ const shouldABTest = !meta.internalOptions.zeroDataRetention && abRate > 0 && Math.random() <= abRate && abHostEnv;if (shouldABTest) {(async () => {try {@@ -603,7 +604,7 @@ export async function scrapeURL(abLogger.info("A/B-testing scrapeURL to staging");const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));await robustFetch({- url: `http://firecrawl-staging-app:3002/v1/scrape`,+ url: `http://${abHostEnv}/v1/scrape`,method: "POST",body: {url: meta.url,