Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: add fire engine A/B comparison (#2706)

tom56a5b4dc89e2
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index e0a8ce721..01e261a5f 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -73,6 +73,7 @@ const configSchema = z.object({
FIRE_ENGINE_STAGING_URL: z.string().optional(),
FIRE_ENGINE_AB_URL: z.string().optional(),
FIRE_ENGINE_AB_RATE: z.coerce.number().optional(),
+ FIRE_ENGINE_AB_COMPARE_ENABLED: z.stringbool().default(false),
// ScrapeURL
SCRAPEURL_AB_HOST: z.string().optional(),
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 50d3f46ee..15d134f1e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -18,6 +18,7 @@ import {
} from "../../error";
import { Meta } from "../..";
import { abTestFireEngine } from "../../../../services/ab-test";
+import { scheduleABComparison } from "../../../../services/ab-test-comparison";
import { config } from "../../../../config";
export type FireEngineScrapeRequestCommon = {
@@ -187,7 +188,8 @@ export async function fireEngineScrape<
abort?: AbortSignal,
production = true,
): Promise<z.infer<typeof processingSchema> | FireEngineCheckStatusSuccess> {
- abTestFireEngine(request);
+ const abTest = abTestFireEngine(request);
+ const productionStartTime = Date.now();
let status = await robustFetch({
url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,
@@ -216,6 +218,23 @@ export async function fireEngineScrape<
if (successParse.success) {
logger.debug("Scrape succeeded!");
+
+ // Schedule A/B comparison if enabled (fire-and-forget)
+ if (abTest.shouldCompare && abTest.mirrorPromise) {
+ const productionTimeTaken = Date.now() - productionStartTime;
+ scheduleABComparison(
+ meta.url,
+ {
+ content: successParse.data.content,
+ pageStatusCode: successParse.data.pageStatusCode,
+ timeTaken: successParse.data.timeTaken,
+ },
+ productionTimeTaken,
+ abTest.mirrorPromise,
+ logger,
+ );
+ }
+
return successParse.data;
} else if (processingParse.success) {
return processingParse.data;
diff --git a/apps/api/src/services/ab-test-comparison.ts b/apps/api/src/services/ab-test-comparison.ts
new file mode 100644
index 000000000..5f085bbed
--- /dev/null
+++ b/apps/api/src/services/ab-test-comparison.ts
@@ -0,0 +1,167 @@
+import { Logger } from "winston";
+import gitDiff from "git-diff";
+import parseDiff from "parse-diff";
+
+const AB_LOG_PREFIX = "[FE_AB_COMPARE]";
+const DIFF_THRESHOLD_PERCENT = 5;
+const MAX_CONTENT_SIZE = 1000000; // 1 MB
+
+export interface FireEngineResponse {
+ content: string;
+ pageStatusCode: number;
+ timeTaken: number;
+}
+
+export interface MirrorResult {
+ response: FireEngineResponse | null;
+ error: Error | null;
+ timeTaken: number;
+}
+
+interface ContentDiffResult {
+ identical: boolean;
+ skipped?: boolean;
+ skipReason?: string;
+ addedLines?: number;
+ removedLines?: number;
+ diffPercentage?: number;
+}
+
+function computeContentDiff(
+ productionContent: string,
+ mirrorContent: string,
+): ContentDiffResult {
+ // Skip if content too large
+ if (
+ productionContent.length > MAX_CONTENT_SIZE ||
+ mirrorContent.length > MAX_CONTENT_SIZE
+ ) {
+ return { identical: false, skipped: true, skipReason: "content_too_large" };
+ }
+
+ // Tier 1: Quick normalized comparison
+ const normalize = (s: string) => s.replace(/\s+/g, " ").trim();
+ if (normalize(productionContent) === normalize(mirrorContent)) {
+ return { identical: true };
+ }
+
+ // Tier 2: Compute line diff
+ const diffText = gitDiff(productionContent, mirrorContent, {
+ color: false,
+ wordDiff: false,
+ });
+
+ if (!diffText) {
+ return { identical: true };
+ }
+
+ const parsed = parseDiff(diffText);
+ let addedLines = 0;
+ let removedLines = 0;
+
+ for (const file of parsed) {
+ addedLines += file.additions;
+ removedLines += file.deletions;
+ }
+
+ const totalLines = productionContent.split("\n").length;
+ const diffPercentage =
+ totalLines > 0 ? ((addedLines + removedLines) / totalLines) * 100 : 0;
+
+ return {
+ identical: false,
+ addedLines,
+ removedLines,
+ diffPercentage: Math.round(diffPercentage * 100) / 100,
+ };
+}
+
+async function runComparison(
+ url: string,
+ productionResponse: FireEngineResponse,
+ productionTimeTaken: number,
+ mirrorPromise: Promise<MirrorResult>,
+ abLogger: Logger,
+): Promise<void> {
+ const mirrorResult = await mirrorPromise;
+
+ if (mirrorResult.error || !mirrorResult.response) {
+ abLogger.warn(
+ `${AB_LOG_PREFIX} Mirror request failed - unable to compare`,
+ {
+ url,
+ error: mirrorResult.error?.message ?? "unknown",
+ prod_ms: productionTimeTaken,
+ },
+ );
+ return;
+ }
+
+ const contentDiff = computeContentDiff(
+ productionResponse.content,
+ mirrorResult.response.content,
+ );
+
+ const baseLogData = {
+ url,
+ prod_ms: productionTimeTaken,
+ mirror_ms: mirrorResult.timeTaken,
+ diff_ms: mirrorResult.timeTaken - productionTimeTaken,
+ };
+
+ if (contentDiff.skipped) {
+ abLogger.info(`${AB_LOG_PREFIX} Comparison skipped - content too large`, {
+ ...baseLogData,
+ skip_reason: contentDiff.skipReason,
+ });
+ return;
+ }
+
+ if (contentDiff.identical) {
+ abLogger.info(`${AB_LOG_PREFIX} Content identical`, baseLogData);
+ return;
+ }
+
+ const diffLogData = {
+ ...baseLogData,
+ diff_pct: contentDiff.diffPercentage,
+ added: contentDiff.addedLines,
+ removed: contentDiff.removedLines,
+ };
+ const diffMessage = `+${contentDiff.addedLines}/-${contentDiff.removedLines} lines, ${contentDiff.diffPercentage}%`;
+
+ if (contentDiff.diffPercentage! > DIFF_THRESHOLD_PERCENT) {
+ abLogger.warn(
+ `${AB_LOG_PREFIX} Content mismatch detected (${diffMessage})`,
+ diffLogData,
+ );
+ } else {
+ abLogger.info(
+ `${AB_LOG_PREFIX} Minor content differences (${diffMessage})`,
+ diffLogData,
+ );
+ }
+}
+
+export function scheduleABComparison(
+ url: string,
+ productionResponse: FireEngineResponse,
+ productionTimeTaken: number,
+ mirrorPromise: Promise<MirrorResult>,
+ logger: Logger,
+): void {
+ const abLogger = logger.child({ method: "ABTestComparison" });
+
+ runComparison(
+ url,
+ productionResponse,
+ productionTimeTaken,
+ mirrorPromise,
+ abLogger,
+ ).catch(error => {
+ abLogger.error(`${AB_LOG_PREFIX} comparison failed unexpectedly`, {
+ error,
+ url,
+ });
+ });
+}
diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.ts
index 83d97a887..b349ce44c 100644
--- a/apps/api/src/services/ab-test.ts
+++ b/apps/api/src/services/ab-test.ts
@@ -8,6 +8,7 @@ import {
FireEngineScrapeRequestPlaywright,
FireEngineScrapeRequestTLSClient,
} from "../scraper/scrapeURL/engines/fire-engine/scrape";
+import { MirrorResult, FireEngineResponse } from "./ab-test-comparison";
export function abTestJob(webScraperOptions: ScrapeJobData) {
// Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
@@ -78,51 +79,83 @@ export function abTestFireEngine(
| FireEngineScrapeRequestPlaywright
| FireEngineScrapeRequestTLSClient
),
-) {
- // Global A/B test: mirror request to staging fire-engine based on SCRAPEURL_AB_RATE
+): { shouldCompare: boolean; mirrorPromise: Promise<MirrorResult> | null } {
const abLogger = _logger.child({ method: "ABTestToStaging" });
- try {
- const abRateEnv = config.FIRE_ENGINE_AB_RATE;
- const abUrlEnv = config.FIRE_ENGINE_AB_URL;
- const abRate =
- abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
- const shouldABTest =
- !feRequest.zeroDataRetention &&
- abRate > 0 &&
- Math.random() <= abRate &&
- abUrlEnv;
- if (shouldABTest) {
- let timeout = Math.min(60000, (feRequest.timeout ?? 30000) + 10000);
- (async () => {
- const abortController = new AbortController();
- const timeoutHandle = setTimeout(() => {
- if (abortController) {
- abortController.abort();
- }
- }, timeout);
+ const abRateEnv = config.FIRE_ENGINE_AB_RATE;
+ const abUrlEnv = config.FIRE_ENGINE_AB_URL;
+ const compareEnabled = config.FIRE_ENGINE_AB_COMPARE_ENABLED;
+ const abRate =
+ abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
- try {
- abLogger.info("A/B-testing scrapeURL to staging");
- await robustFetch({
- url: `${abUrlEnv}/scrape`,
- method: "POST",
- body: feRequest,
- logger: abLogger,
- tryCount: 1,
- ignoreResponse: true,
- mock: null,
- abort: abortController.signal,
- });
- abLogger.info("A/B-testing scrapeURL (staging) request sent");
- } catch (error) {
- abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
- } finally {
- if (timeoutHandle) clearTimeout(timeoutHandle);
- }
- })();
- }
- } catch (error) {
- abLogger.warn("Failed to initiate A/B test to staging", { error });
+ const shouldABTest =
+ !feRequest.zeroDataRetention &&
+ abRate > 0 &&
+ Math.random() <= abRate &&
+ abUrlEnv;
+
+ if (!shouldABTest) {
+ return { shouldCompare: false, mirrorPromise: null };
}
+
+ const timeout = Math.min(60000, (feRequest.timeout ?? 30000) + 10000);
+ const startTime = Date.now();
+
+ const mirrorPromise = (async (): Promise<MirrorResult> => {
+ const abortController = new AbortController();
+ const timeoutHandle = setTimeout(() => {
+ abortController.abort();
+ }, timeout);
+
+ try {
+ abLogger.info("A/B-testing scrapeURL to staging");
+
+ const response = await robustFetch({
+ url: `${abUrlEnv}/scrape`,
+ method: "POST",
+ body: feRequest,
+ logger: abLogger,
+ tryCount: 1,
+ ignoreResponse: false,
+ ignoreFailureStatus: true,
+ mock: null,
+ abort: abortController.signal,
+ });
+
+ abLogger.info("A/B-testing scrapeURL (staging) request completed");
+
+ // Extract the fields we need for comparison
+ const feResponse: FireEngineResponse | null =
+ response &&
+ typeof response.content === "string" &&
+ typeof response.pageStatusCode === "number" &&
+ typeof response.timeTaken === "number"
+ ? {
+ content: response.content,
+ pageStatusCode: response.pageStatusCode,
+ timeTaken: response.timeTaken,
+ }
+ : null;
+
+ return {
+ response: feResponse,
+ error: feResponse ? null : new Error("Invalid response format"),
+ timeTaken: Date.now() - startTime,
+ };
+ } catch (error) {
+ abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
+ return {
+ response: null,
+ error: error as Error,
+ timeTaken: Date.now() - startTime,
+ };
+ } finally {
+ clearTimeout(timeoutHandle);
+ }
+ })();
+
+ return {
+ shouldCompare: compareEnabled,
+ mirrorPromise,
+ };
}