Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: a/b test logic (#2708)

tom925adac0ecd1
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 15d134f1e..378c444fd 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -227,7 +227,6 @@ export async function fireEngineScrape<
{
content: successParse.data.content,
pageStatusCode: successParse.data.pageStatusCode,
- timeTaken: successParse.data.timeTaken,
},
productionTimeTaken,
abTest.mirrorPromise,
diff --git a/apps/api/src/services/ab-test-comparison.ts b/apps/api/src/services/ab-test-comparison.ts
index 9cea4c87e..ce7fb8bc0 100644
--- a/apps/api/src/services/ab-test-comparison.ts
+++ b/apps/api/src/services/ab-test-comparison.ts
@@ -1,15 +1,11 @@
import { Logger } from "winston";
-import gitDiff from "git-diff";
-import parseDiff from "parse-diff";
const AB_LOG_PREFIX = "[FE_AB_COMPARE]";
-const DIFF_THRESHOLD_PERCENT = 5;
-const MAX_CONTENT_SIZE = 1000000; // 1 MB
+const MAX_CONTENT_SIZE = 1_000_000; // 1 MB
export interface FireEngineResponse {
content: string;
pageStatusCode: number;
- timeTaken: number;
}
export interface MirrorResult {
@@ -18,131 +14,8 @@ export interface MirrorResult {
timeTaken: number;
}
-interface ContentDiffResult {
- identical: boolean;
- skipped?: boolean;
- skipReason?: string;
- addedLines?: number;
- removedLines?: number;
- diffPercentage?: number;
-}
-
-function computeContentDiff(
- productionContent: string,
- mirrorContent: string,
-): ContentDiffResult {
- // Skip if content too large
- if (
- productionContent.length > MAX_CONTENT_SIZE ||
- mirrorContent.length > MAX_CONTENT_SIZE
- ) {
- return { identical: false, skipped: true, skipReason: "content_too_large" };
- }
-
- // Tier 1: Quick normalized comparison
- const normalize = (s: string) => s.replace(/\s+/g, " ").trim();
- if (normalize(productionContent) === normalize(mirrorContent)) {
- return { identical: true };
- }
-
- // Tier 2: Compute line diff
- const diffText = gitDiff(productionContent, mirrorContent, {
- color: false,
- wordDiff: false,
- });
-
- if (!diffText) {
- return { identical: true };
- }
-
- const parsed = parseDiff(diffText);
- let addedLines = 0;
- let removedLines = 0;
-
- for (const file of parsed) {
- addedLines += file.additions;
- removedLines += file.deletions;
- }
-
- const totalLines = productionContent.split("\n").length;
- const diffPercentage =
- totalLines > 0 ? ((addedLines + removedLines) / totalLines) * 100 : 0;
-
- return {
- identical: false,
- addedLines,
- removedLines,
- diffPercentage: Math.round(diffPercentage * 100) / 100,
- };
-}
-
-async function runComparison(
- url: string,
- productionResponse: FireEngineResponse,
- productionTimeTaken: number,
- mirrorPromise: Promise<MirrorResult>,
- abLogger: Logger,
-): Promise<void> {
- const mirrorResult = await mirrorPromise;
-
- if (mirrorResult.error || !mirrorResult.response) {
- abLogger.warn(
- `${AB_LOG_PREFIX} Mirror request failed - unable to compare`,
- {
- url,
- error: mirrorResult.error?.message ?? "unknown",
- prod_ms: productionTimeTaken,
- },
- );
- return;
- }
-
- const contentDiff = computeContentDiff(
- productionResponse.content,
- mirrorResult.response.content,
- );
-
- const baseLogData = {
- url,
- prod_ms: productionTimeTaken,
- mirror_ms: mirrorResult.timeTaken,
- diff_ms: mirrorResult.timeTaken - productionTimeTaken,
- };
-
- if (contentDiff.skipped) {
- abLogger.info(`${AB_LOG_PREFIX} Comparison skipped - content too large`, {
- ...baseLogData,
- skip_reason: contentDiff.skipReason,
- });
- return;
- }
-
- if (contentDiff.identical) {
- abLogger.info(`${AB_LOG_PREFIX} Content identical`, baseLogData);
- return;
- }
-
- const diffLogData = {
- ...baseLogData,
- diff_pct: contentDiff.diffPercentage,
- added: contentDiff.addedLines,
- removed: contentDiff.removedLines,
- };
- const diffMessage = `+${contentDiff.addedLines}/-${contentDiff.removedLines} lines, ${contentDiff.diffPercentage}%`;
-
- if (contentDiff.diffPercentage! > DIFF_THRESHOLD_PERCENT) {
- abLogger.warn(
- `${AB_LOG_PREFIX} Content mismatch detected (${diffMessage})`,
- diffLogData,
- );
- } else if (contentDiff.diffPercentage! > 0) {
- abLogger.info(
- `${AB_LOG_PREFIX} Minor content differences (${diffMessage})`,
- diffLogData,
- );
- } else {
- abLogger.info(`${AB_LOG_PREFIX} Content identical`, baseLogData);
- }
+function normalizeContent(s: string): string {
+ return s.replace(/\s+/g, " ").trim();
}
export function scheduleABComparison(
@@ -154,16 +27,51 @@ export function scheduleABComparison(
): void {
const abLogger = logger.child({ method: "ABTestComparison" });
- runComparison(
- url,
- productionResponse,
- productionTimeTaken,
- mirrorPromise,
- abLogger,
- ).catch(error => {
- abLogger.error(`${AB_LOG_PREFIX} comparison failed unexpectedly`, {
- error,
- url,
+ mirrorPromise
+ .then(mirrorResult => {
+ const baseLogData = {
+ url,
+ prod_ms: productionTimeTaken,
+ mirror_ms: mirrorResult.timeTaken,
+ diff_ms: mirrorResult.timeTaken - productionTimeTaken,
+ };
+
+ if (mirrorResult.error || !mirrorResult.response) {
+ abLogger.warn(`${AB_LOG_PREFIX} Mirror request failed`, {
+ ...baseLogData,
+ error: mirrorResult.error?.message ?? "unknown",
+ });
+ return;
+ }
+
+ // Skip comparison if content too large
+ if (
+ productionResponse.content.length > MAX_CONTENT_SIZE ||
+ mirrorResult.response.content.length > MAX_CONTENT_SIZE
+ ) {
+ return;
+ }
+
+ const prodNormalized = normalizeContent(productionResponse.content);
+ const mirrorNormalized = normalizeContent(mirrorResult.response.content);
+ const identical = prodNormalized === mirrorNormalized;
+
+ if (identical) {
+ abLogger.info(`${AB_LOG_PREFIX} Content identical`, baseLogData);
+ } else {
+ abLogger.warn(`${AB_LOG_PREFIX} Content mismatch`, {
+ ...baseLogData,
+ prod_len: productionResponse.content.length,
+ mirror_len: mirrorResult.response.content.length,
+ prod_status: productionResponse.pageStatusCode,
+ mirror_status: mirrorResult.response.pageStatusCode,
+ });
+ }
+ })
+ .catch(error => {
+ abLogger.error(`${AB_LOG_PREFIX} Comparison failed unexpectedly`, {
+ error,
+ url,
+ });
});
- });
}
diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.ts
index b349ce44c..f7113b266 100644
--- a/apps/api/src/services/ab-test.ts
+++ b/apps/api/src/services/ab-test.ts
@@ -8,50 +8,51 @@ import {
FireEngineScrapeRequestPlaywright,
FireEngineScrapeRequestTLSClient,
} from "../scraper/scrapeURL/engines/fire-engine/scrape";
+import { getDocFromGCS } from "../lib/gcs-jobs";
import { MirrorResult, FireEngineResponse } from "./ab-test-comparison";
export function abTestJob(webScraperOptions: ScrapeJobData) {
- // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
const abLogger = _logger.child({ method: "ABTestToStaging" });
try {
- const abRateEnv = config.SCRAPEURL_AB_RATE;
- const abHostEnv = config.SCRAPEURL_AB_HOST;
- const shouldExtendMaxAge = config.SCRAPEURL_AB_EXTEND_MAXAGE;
- const abRate =
- abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
+ const abRate = config.SCRAPEURL_AB_RATE
+ ? Math.max(0, Math.min(1, Number(config.SCRAPEURL_AB_RATE)))
+ : 0;
+
const shouldABTest =
webScraperOptions.mode === "single_urls" &&
!webScraperOptions.zeroDataRetention &&
!webScraperOptions.internalOptions?.zeroDataRetention &&
abRate > 0 &&
Math.random() <= abRate &&
- abHostEnv &&
+ config.SCRAPEURL_AB_HOST &&
webScraperOptions.internalOptions?.v1Agent === undefined &&
webScraperOptions.internalOptions?.v1JSONAgent === undefined;
+
if (shouldABTest) {
- let timeout = Math.min(
+ const timeout = Math.min(
60000,
(webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000,
);
(async () => {
const abortController = new AbortController();
- const timeoutHandle = setTimeout(() => {
- if (abortController) {
- abortController.abort();
- }
- }, timeout);
+ const timeoutHandle = setTimeout(
+ () => abortController.abort(),
+ timeout,
+ );
try {
abLogger.info("A/B-testing scrapeURL to staging");
await robustFetch({
- url: `http://${abHostEnv}/v2/scrape`,
+ url: `http://${config.SCRAPEURL_AB_HOST}/v2/scrape`,
method: "POST",
body: {
url: webScraperOptions.url,
...webScraperOptions.scrapeOptions,
origin: (webScraperOptions.scrapeOptions as any).origin ?? "api",
- ...(shouldExtendMaxAge ? { maxAge: 900000000 } : {}),
+ ...(config.SCRAPEURL_AB_EXTEND_MAXAGE
+ ? { maxAge: 900000000 }
+ : {}),
},
logger: abLogger,
tryCount: 1,
@@ -63,7 +64,7 @@ export function abTestJob(webScraperOptions: ScrapeJobData) {
} catch (error) {
abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
} finally {
- if (timeoutHandle) clearTimeout(timeoutHandle);
+ clearTimeout(timeoutHandle);
}
})();
}
@@ -80,19 +81,17 @@ export function abTestFireEngine(
| FireEngineScrapeRequestTLSClient
),
): { shouldCompare: boolean; mirrorPromise: Promise<MirrorResult> | null } {
- const abLogger = _logger.child({ method: "ABTestToStaging" });
+ const abLogger = _logger.child({ method: "ABTestFireEngine" });
- const abRateEnv = config.FIRE_ENGINE_AB_RATE;
- const abUrlEnv = config.FIRE_ENGINE_AB_URL;
- const compareEnabled = config.FIRE_ENGINE_AB_COMPARE_ENABLED;
- const abRate =
- abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
+ const abRate = config.FIRE_ENGINE_AB_RATE
+ ? Math.max(0, Math.min(1, Number(config.FIRE_ENGINE_AB_RATE)))
+ : 0;
const shouldABTest =
!feRequest.zeroDataRetention &&
abRate > 0 &&
Math.random() <= abRate &&
- abUrlEnv;
+ config.FIRE_ENGINE_AB_URL;
if (!shouldABTest) {
return { shouldCompare: false, mirrorPromise: null };
@@ -103,15 +102,16 @@ export function abTestFireEngine(
const mirrorPromise = (async (): Promise<MirrorResult> => {
const abortController = new AbortController();
- const timeoutHandle = setTimeout(() => {
- abortController.abort();
- }, timeout);
+ const timeoutHandle = setTimeout(() => abortController.abort(), timeout);
+ let jobId: string | undefined;
try {
- abLogger.info("A/B-testing scrapeURL to staging");
+ abLogger.info("A/B-testing fire-engine to staging", {
+ url: feRequest.url,
+ });
- const response = await robustFetch({
- url: `${abUrlEnv}/scrape`,
+ let response = await robustFetch({
+ url: `${config.FIRE_ENGINE_AB_URL}/scrape`,
method: "POST",
body: feRequest,
logger: abLogger,
@@ -122,18 +122,27 @@ export function abTestFireEngine(
abort: abortController.signal,
});
- abLogger.info("A/B-testing scrapeURL (staging) request completed");
+ jobId = response?.jobId;
+
+ if (!response.content && response.docUrl) {
+ const doc = await getDocFromGCS(response.docUrl.split("/").pop() ?? "");
+ if (doc) {
+ response = { ...response, ...doc };
+ }
+ }
+
+ abLogger.info("A/B-testing fire-engine (staging) request completed", {
+ url: feRequest.url,
+ hasContent: !!response?.content,
+ });
- // Extract the fields we need for comparison
const feResponse: FireEngineResponse | null =
response &&
typeof response.content === "string" &&
- typeof response.pageStatusCode === "number" &&
- typeof response.timeTaken === "number"
+ typeof response.pageStatusCode === "number"
? {
content: response.content,
pageStatusCode: response.pageStatusCode,
- timeTaken: response.timeTaken,
}
: null;
@@ -143,7 +152,10 @@ export function abTestFireEngine(
timeTaken: Date.now() - startTime,
};
} catch (error) {
- abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
+ abLogger.warn("A/B-testing fire-engine (staging) failed", {
+ error,
+ url: feRequest.url,
+ });
return {
response: null,
error: error as Error,
@@ -151,11 +163,26 @@ export function abTestFireEngine(
};
} finally {
clearTimeout(timeoutHandle);
+
+ if (jobId && config.FIRE_ENGINE_AB_URL) {
+ robustFetch({
+ url: `${config.FIRE_ENGINE_AB_URL}/scrape/${jobId}`,
+ method: "DELETE",
+ headers: {},
+ logger: abLogger.child({ method: "abTestFireEngine/delete", jobId }),
+ mock: null,
+ }).catch(e => {
+ abLogger.warn("Failed to delete AB test job from fire-engine", {
+ error: e,
+ jobId,
+ });
+ });
+ }
}
})();
return {
- shouldCompare: compareEnabled,
+ shouldCompare: config.FIRE_ENGINE_AB_COMPARE_ENABLED,
mirrorPromise,
};
}