Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(ab-test): SCRAPEURL_AB_EXTEND_MAXAGE support

Gergő Móriczfeecf9e2dc48
diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.ts
index 87c0ce7e5..e2f5d152f 100644
--- a/apps/api/src/services/ab-test.ts
+++ b/apps/api/src/services/ab-test.ts
@@ -8,6 +8,7 @@ export function abTestJob(webScraperOptions: WebScraperOptions) {
try {
const abRateEnv = process.env.SCRAPEURL_AB_RATE;
const abHostEnv = process.env.SCRAPEURL_AB_HOST;
+ const shouldExtendMaxAge = process.env.SCRAPEURL_AB_EXTEND_MAXAGE === "true";
const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
const shouldABTest = webScraperOptions.mode === "single_urls"
&& !webScraperOptions.zeroDataRetention
@@ -29,6 +30,7 @@ export function abTestJob(webScraperOptions: WebScraperOptions) {
url: webScraperOptions.url,
...webScraperOptions.scrapeOptions,
origin: (webScraperOptions.scrapeOptions as any).origin ?? "api",
+ ...(shouldExtendMaxAge ? { maxAge: 900000000 } : {}),
},
logger: abLogger,
tryCount: 1,