Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: add fire engine A/B comparison (#2706)
tom56a5b4dc89e2
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex e0a8ce721..01e261a5f 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -73,6 +73,7 @@ const configSchema = z.object({FIRE_ENGINE_STAGING_URL: z.string().optional(),FIRE_ENGINE_AB_URL: z.string().optional(),FIRE_ENGINE_AB_RATE: z.coerce.number().optional(),+ FIRE_ENGINE_AB_COMPARE_ENABLED: z.stringbool().default(false),// ScrapeURLSCRAPEURL_AB_HOST: z.string().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 50d3f46ee..15d134f1e 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -18,6 +18,7 @@ import {} from "../../error";import { Meta } from "../..";import { abTestFireEngine } from "../../../../services/ab-test";+import { scheduleABComparison } from "../../../../services/ab-test-comparison";import { config } from "../../../../config";export type FireEngineScrapeRequestCommon = {@@ -187,7 +188,8 @@ export async function fireEngineScrape<abort?: AbortSignal,production = true,): Promise<z.infer<typeof processingSchema> | FireEngineCheckStatusSuccess> {- abTestFireEngine(request);+ const abTest = abTestFireEngine(request);+ const productionStartTime = Date.now();let status = await robustFetch({url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,@@ -216,6 +218,23 @@ export async function fireEngineScrape<if (successParse.success) {logger.debug("Scrape succeeded!");++ // Schedule A/B comparison if enabled (fire-and-forget)+ if (abTest.shouldCompare && abTest.mirrorPromise) {+ const productionTimeTaken = Date.now() - productionStartTime;+ scheduleABComparison(+ meta.url,+ {+ content: successParse.data.content,+ pageStatusCode: successParse.data.pageStatusCode,+ timeTaken: successParse.data.timeTaken,+ },+ productionTimeTaken,+ abTest.mirrorPromise,+ logger,+ );+ }+return successParse.data;} else if (processingParse.success) {return processingParse.data;diff --git a/apps/api/src/services/ab-test-comparison.ts b/apps/api/src/services/ab-test-comparison.tsnew file mode 100644index 000000000..5f085bbed--- /dev/null+++ b/apps/api/src/services/ab-test-comparison.ts@@ -0,0 +1,167 @@+import { Logger } from "winston";+import gitDiff from "git-diff";+import parseDiff from "parse-diff";++const AB_LOG_PREFIX = "[FE_AB_COMPARE]";+const DIFF_THRESHOLD_PERCENT = 5;+const MAX_CONTENT_SIZE = 1000000; // 1 MB++export interface FireEngineResponse {+ content: string;+ pageStatusCode: number;+ timeTaken: number;+}++export interface MirrorResult {+ response: FireEngineResponse | null;+ error: Error | null;+ timeTaken: number;+}++interface ContentDiffResult {+ identical: boolean;+ skipped?: boolean;+ skipReason?: string;+ addedLines?: number;+ removedLines?: number;+ diffPercentage?: number;+}++function computeContentDiff(+ productionContent: string,+ mirrorContent: string,+): ContentDiffResult {+ // Skip if content too large+ if (+ productionContent.length > MAX_CONTENT_SIZE ||+ mirrorContent.length > MAX_CONTENT_SIZE+ ) {+ return { identical: false, skipped: true, skipReason: "content_too_large" };+ }++ // Tier 1: Quick normalized comparison+ const normalize = (s: string) => s.replace(/\s+/g, " ").trim();+ if (normalize(productionContent) === normalize(mirrorContent)) {+ return { identical: true };+ }++ // Tier 2: Compute line diff+ const diffText = gitDiff(productionContent, mirrorContent, {+ color: false,+ wordDiff: false,+ });++ if (!diffText) {+ return { identical: true };+ }++ const parsed = parseDiff(diffText);+ let addedLines = 0;+ let removedLines = 0;++ for (const file of parsed) {+ addedLines += file.additions;+ removedLines += file.deletions;+ }++ const totalLines = productionContent.split("\n").length;+ const diffPercentage =+ totalLines > 0 ? ((addedLines + removedLines) / totalLines) * 100 : 0;++ return {+ identical: false,+ addedLines,+ removedLines,+ diffPercentage: Math.round(diffPercentage * 100) / 100,+ };+}++async function runComparison(+ url: string,+ productionResponse: FireEngineResponse,+ productionTimeTaken: number,+ mirrorPromise: Promise<MirrorResult>,+ abLogger: Logger,+): Promise<void> {+ const mirrorResult = await mirrorPromise;++ if (mirrorResult.error || !mirrorResult.response) {+ abLogger.warn(+ `${AB_LOG_PREFIX} Mirror request failed - unable to compare`,+ {+ url,+ error: mirrorResult.error?.message ?? "unknown",+ prod_ms: productionTimeTaken,+ },+ );+ return;+ }++ const contentDiff = computeContentDiff(+ productionResponse.content,+ mirrorResult.response.content,+ );++ const baseLogData = {+ url,+ prod_ms: productionTimeTaken,+ mirror_ms: mirrorResult.timeTaken,+ diff_ms: mirrorResult.timeTaken - productionTimeTaken,+ };++ if (contentDiff.skipped) {+ abLogger.info(`${AB_LOG_PREFIX} Comparison skipped - content too large`, {+ ...baseLogData,+ skip_reason: contentDiff.skipReason,+ });+ return;+ }++ if (contentDiff.identical) {+ abLogger.info(`${AB_LOG_PREFIX} Content identical`, baseLogData);+ return;+ }++ const diffLogData = {+ ...baseLogData,+ diff_pct: contentDiff.diffPercentage,+ added: contentDiff.addedLines,+ removed: contentDiff.removedLines,+ };+ const diffMessage = `+${contentDiff.addedLines}/-${contentDiff.removedLines} lines, ${contentDiff.diffPercentage}%`;++ if (contentDiff.diffPercentage! > DIFF_THRESHOLD_PERCENT) {+ abLogger.warn(+ `${AB_LOG_PREFIX} Content mismatch detected (${diffMessage})`,+ diffLogData,+ );+ } else {+ abLogger.info(+ `${AB_LOG_PREFIX} Minor content differences (${diffMessage})`,+ diffLogData,+ );+ }+}++export function scheduleABComparison(+ url: string,+ productionResponse: FireEngineResponse,+ productionTimeTaken: number,+ mirrorPromise: Promise<MirrorResult>,+ logger: Logger,+): void {+ const abLogger = logger.child({ method: "ABTestComparison" });++ runComparison(+ url,+ productionResponse,+ productionTimeTaken,+ mirrorPromise,+ abLogger,+ ).catch(error => {+ abLogger.error(`${AB_LOG_PREFIX} comparison failed unexpectedly`, {+ error,+ url,+ });+ });+}diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.tsindex 83d97a887..b349ce44c 100644--- a/apps/api/src/services/ab-test.ts+++ b/apps/api/src/services/ab-test.ts@@ -8,6 +8,7 @@ import {FireEngineScrapeRequestPlaywright,FireEngineScrapeRequestTLSClient,} from "../scraper/scrapeURL/engines/fire-engine/scrape";+import { MirrorResult, FireEngineResponse } from "./ab-test-comparison";export function abTestJob(webScraperOptions: ScrapeJobData) {// Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE@@ -78,51 +79,83 @@ export function abTestFireEngine(| FireEngineScrapeRequestPlaywright| FireEngineScrapeRequestTLSClient),-) {- // Global A/B test: mirror request to staging fire-engine based on SCRAPEURL_AB_RATE+): { shouldCompare: boolean; mirrorPromise: Promise<MirrorResult> | null } {const abLogger = _logger.child({ method: "ABTestToStaging" });- try {- const abRateEnv = config.FIRE_ENGINE_AB_RATE;- const abUrlEnv = config.FIRE_ENGINE_AB_URL;- const abRate =- abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;- const shouldABTest =- !feRequest.zeroDataRetention &&- abRate > 0 &&- Math.random() <= abRate &&- abUrlEnv;- if (shouldABTest) {- let timeout = Math.min(60000, (feRequest.timeout ?? 30000) + 10000);- (async () => {- const abortController = new AbortController();- const timeoutHandle = setTimeout(() => {- if (abortController) {- abortController.abort();- }- }, timeout);+ const abRateEnv = config.FIRE_ENGINE_AB_RATE;+ const abUrlEnv = config.FIRE_ENGINE_AB_URL;+ const compareEnabled = config.FIRE_ENGINE_AB_COMPARE_ENABLED;+ const abRate =+ abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;- try {- abLogger.info("A/B-testing scrapeURL to staging");- await robustFetch({- url: `${abUrlEnv}/scrape`,- method: "POST",- body: feRequest,- logger: abLogger,- tryCount: 1,- ignoreResponse: true,- mock: null,- abort: abortController.signal,- });- abLogger.info("A/B-testing scrapeURL (staging) request sent");- } catch (error) {- abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });- } finally {- if (timeoutHandle) clearTimeout(timeoutHandle);- }- })();- }- } catch (error) {- abLogger.warn("Failed to initiate A/B test to staging", { error });+ const shouldABTest =+ !feRequest.zeroDataRetention &&+ abRate > 0 &&+ Math.random() <= abRate &&+ abUrlEnv;++ if (!shouldABTest) {+ return { shouldCompare: false, mirrorPromise: null };}++ const timeout = Math.min(60000, (feRequest.timeout ?? 30000) + 10000);+ const startTime = Date.now();++ const mirrorPromise = (async (): Promise<MirrorResult> => {+ const abortController = new AbortController();+ const timeoutHandle = setTimeout(() => {+ abortController.abort();+ }, timeout);++ try {+ abLogger.info("A/B-testing scrapeURL to staging");++ const response = await robustFetch({+ url: `${abUrlEnv}/scrape`,+ method: "POST",+ body: feRequest,+ logger: abLogger,+ tryCount: 1,+ ignoreResponse: false,+ ignoreFailureStatus: true,+ mock: null,+ abort: abortController.signal,+ });++ abLogger.info("A/B-testing scrapeURL (staging) request completed");++ // Extract the fields we need for comparison+ const feResponse: FireEngineResponse | null =+ response &&+ typeof response.content === "string" &&+ typeof response.pageStatusCode === "number" &&+ typeof response.timeTaken === "number"+ ? {+ content: response.content,+ pageStatusCode: response.pageStatusCode,+ timeTaken: response.timeTaken,+ }+ : null;++ return {+ response: feResponse,+ error: feResponse ? null : new Error("Invalid response format"),+ timeTaken: Date.now() - startTime,+ };+ } catch (error) {+ abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });+ return {+ response: null,+ error: error as Error,+ timeTaken: Date.now() - startTime,+ };+ } finally {+ clearTimeout(timeoutHandle);+ }+ })();++ return {+ shouldCompare: compareEnabled,+ mirrorPromise,+ };}