Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: a/b test logic (#2708)
tom925adac0ecd1
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 15d134f1e..378c444fd 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -227,7 +227,6 @@ export async function fireEngineScrape<{content: successParse.data.content,pageStatusCode: successParse.data.pageStatusCode,- timeTaken: successParse.data.timeTaken,},productionTimeTaken,abTest.mirrorPromise,diff --git a/apps/api/src/services/ab-test-comparison.ts b/apps/api/src/services/ab-test-comparison.tsindex 9cea4c87e..ce7fb8bc0 100644--- a/apps/api/src/services/ab-test-comparison.ts+++ b/apps/api/src/services/ab-test-comparison.ts@@ -1,15 +1,11 @@import { Logger } from "winston";-import gitDiff from "git-diff";-import parseDiff from "parse-diff";const AB_LOG_PREFIX = "[FE_AB_COMPARE]";-const DIFF_THRESHOLD_PERCENT = 5;-const MAX_CONTENT_SIZE = 1000000; // 1 MB+const MAX_CONTENT_SIZE = 1_000_000; // 1 MBexport interface FireEngineResponse {content: string;pageStatusCode: number;- timeTaken: number;}export interface MirrorResult {@@ -18,131 +14,8 @@ export interface MirrorResult {timeTaken: number;}-interface ContentDiffResult {- identical: boolean;- skipped?: boolean;- skipReason?: string;- addedLines?: number;- removedLines?: number;- diffPercentage?: number;-}--function computeContentDiff(- productionContent: string,- mirrorContent: string,-): ContentDiffResult {- // Skip if content too large- if (- productionContent.length > MAX_CONTENT_SIZE ||- mirrorContent.length > MAX_CONTENT_SIZE- ) {- return { identical: false, skipped: true, skipReason: "content_too_large" };- }-- // Tier 1: Quick normalized comparison- const normalize = (s: string) => s.replace(/\s+/g, " ").trim();- if (normalize(productionContent) === normalize(mirrorContent)) {- return { identical: true };- }-- // Tier 2: Compute line diff- const diffText = gitDiff(productionContent, mirrorContent, {- color: false,- wordDiff: false,- });-- if (!diffText) {- return { identical: true };- }-- const parsed = parseDiff(diffText);- let addedLines = 0;- let removedLines = 0;-- for (const file of parsed) {- addedLines += file.additions;- removedLines += file.deletions;- }-- const totalLines = productionContent.split("\n").length;- const diffPercentage =- totalLines > 0 ? ((addedLines + removedLines) / totalLines) * 100 : 0;-- return {- identical: false,- addedLines,- removedLines,- diffPercentage: Math.round(diffPercentage * 100) / 100,- };-}--async function runComparison(- url: string,- productionResponse: FireEngineResponse,- productionTimeTaken: number,- mirrorPromise: Promise<MirrorResult>,- abLogger: Logger,-): Promise<void> {- const mirrorResult = await mirrorPromise;-- if (mirrorResult.error || !mirrorResult.response) {- abLogger.warn(- `${AB_LOG_PREFIX} Mirror request failed - unable to compare`,- {- url,- error: mirrorResult.error?.message ?? "unknown",- prod_ms: productionTimeTaken,- },- );- return;- }-- const contentDiff = computeContentDiff(- productionResponse.content,- mirrorResult.response.content,- );-- const baseLogData = {- url,- prod_ms: productionTimeTaken,- mirror_ms: mirrorResult.timeTaken,- diff_ms: mirrorResult.timeTaken - productionTimeTaken,- };-- if (contentDiff.skipped) {- abLogger.info(`${AB_LOG_PREFIX} Comparison skipped - content too large`, {- ...baseLogData,- skip_reason: contentDiff.skipReason,- });- return;- }-- if (contentDiff.identical) {- abLogger.info(`${AB_LOG_PREFIX} Content identical`, baseLogData);- return;- }-- const diffLogData = {- ...baseLogData,- diff_pct: contentDiff.diffPercentage,- added: contentDiff.addedLines,- removed: contentDiff.removedLines,- };- const diffMessage = `+${contentDiff.addedLines}/-${contentDiff.removedLines} lines, ${contentDiff.diffPercentage}%`;-- if (contentDiff.diffPercentage! > DIFF_THRESHOLD_PERCENT) {- abLogger.warn(- `${AB_LOG_PREFIX} Content mismatch detected (${diffMessage})`,- diffLogData,- );- } else if (contentDiff.diffPercentage! > 0) {- abLogger.info(- `${AB_LOG_PREFIX} Minor content differences (${diffMessage})`,- diffLogData,- );- } else {- abLogger.info(`${AB_LOG_PREFIX} Content identical`, baseLogData);- }+function normalizeContent(s: string): string {+ return s.replace(/\s+/g, " ").trim();}export function scheduleABComparison(@@ -154,16 +27,51 @@ export function scheduleABComparison(): void {const abLogger = logger.child({ method: "ABTestComparison" });- runComparison(- url,- productionResponse,- productionTimeTaken,- mirrorPromise,- abLogger,- ).catch(error => {- abLogger.error(`${AB_LOG_PREFIX} comparison failed unexpectedly`, {- error,- url,+ mirrorPromise+ .then(mirrorResult => {+ const baseLogData = {+ url,+ prod_ms: productionTimeTaken,+ mirror_ms: mirrorResult.timeTaken,+ diff_ms: mirrorResult.timeTaken - productionTimeTaken,+ };++ if (mirrorResult.error || !mirrorResult.response) {+ abLogger.warn(`${AB_LOG_PREFIX} Mirror request failed`, {+ ...baseLogData,+ error: mirrorResult.error?.message ?? "unknown",+ });+ return;+ }++ // Skip comparison if content too large+ if (+ productionResponse.content.length > MAX_CONTENT_SIZE ||+ mirrorResult.response.content.length > MAX_CONTENT_SIZE+ ) {+ return;+ }++ const prodNormalized = normalizeContent(productionResponse.content);+ const mirrorNormalized = normalizeContent(mirrorResult.response.content);+ const identical = prodNormalized === mirrorNormalized;++ if (identical) {+ abLogger.info(`${AB_LOG_PREFIX} Content identical`, baseLogData);+ } else {+ abLogger.warn(`${AB_LOG_PREFIX} Content mismatch`, {+ ...baseLogData,+ prod_len: productionResponse.content.length,+ mirror_len: mirrorResult.response.content.length,+ prod_status: productionResponse.pageStatusCode,+ mirror_status: mirrorResult.response.pageStatusCode,+ });+ }+ })+ .catch(error => {+ abLogger.error(`${AB_LOG_PREFIX} Comparison failed unexpectedly`, {+ error,+ url,+ });});- });}diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.tsindex b349ce44c..f7113b266 100644--- a/apps/api/src/services/ab-test.ts+++ b/apps/api/src/services/ab-test.ts@@ -8,50 +8,51 @@ import {FireEngineScrapeRequestPlaywright,FireEngineScrapeRequestTLSClient,} from "../scraper/scrapeURL/engines/fire-engine/scrape";+import { getDocFromGCS } from "../lib/gcs-jobs";import { MirrorResult, FireEngineResponse } from "./ab-test-comparison";export function abTestJob(webScraperOptions: ScrapeJobData) {- // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATEconst abLogger = _logger.child({ method: "ABTestToStaging" });try {- const abRateEnv = config.SCRAPEURL_AB_RATE;- const abHostEnv = config.SCRAPEURL_AB_HOST;- const shouldExtendMaxAge = config.SCRAPEURL_AB_EXTEND_MAXAGE;- const abRate =- abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;+ const abRate = config.SCRAPEURL_AB_RATE+ ? Math.max(0, Math.min(1, Number(config.SCRAPEURL_AB_RATE)))+ : 0;+const shouldABTest =webScraperOptions.mode === "single_urls" &&!webScraperOptions.zeroDataRetention &&!webScraperOptions.internalOptions?.zeroDataRetention &&abRate > 0 &&Math.random() <= abRate &&- abHostEnv &&+ config.SCRAPEURL_AB_HOST &&webScraperOptions.internalOptions?.v1Agent === undefined &&webScraperOptions.internalOptions?.v1JSONAgent === undefined;+if (shouldABTest) {- let timeout = Math.min(+ const timeout = Math.min(60000,(webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000,);(async () => {const abortController = new AbortController();- const timeoutHandle = setTimeout(() => {- if (abortController) {- abortController.abort();- }- }, timeout);+ const timeoutHandle = setTimeout(+ () => abortController.abort(),+ timeout,+ );try {abLogger.info("A/B-testing scrapeURL to staging");await robustFetch({- url: `http://${abHostEnv}/v2/scrape`,+ url: `http://${config.SCRAPEURL_AB_HOST}/v2/scrape`,method: "POST",body: {url: webScraperOptions.url,...webScraperOptions.scrapeOptions,origin: (webScraperOptions.scrapeOptions as any).origin ?? "api",- ...(shouldExtendMaxAge ? { maxAge: 900000000 } : {}),+ ...(config.SCRAPEURL_AB_EXTEND_MAXAGE+ ? { maxAge: 900000000 }+ : {}),},logger: abLogger,tryCount: 1,@@ -63,7 +64,7 @@ export function abTestJob(webScraperOptions: ScrapeJobData) {} catch (error) {abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });} finally {- if (timeoutHandle) clearTimeout(timeoutHandle);+ clearTimeout(timeoutHandle);}})();}@@ -80,19 +81,17 @@ export function abTestFireEngine(| FireEngineScrapeRequestTLSClient),): { shouldCompare: boolean; mirrorPromise: Promise<MirrorResult> | null } {- const abLogger = _logger.child({ method: "ABTestToStaging" });+ const abLogger = _logger.child({ method: "ABTestFireEngine" });- const abRateEnv = config.FIRE_ENGINE_AB_RATE;- const abUrlEnv = config.FIRE_ENGINE_AB_URL;- const compareEnabled = config.FIRE_ENGINE_AB_COMPARE_ENABLED;- const abRate =- abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;+ const abRate = config.FIRE_ENGINE_AB_RATE+ ? Math.max(0, Math.min(1, Number(config.FIRE_ENGINE_AB_RATE)))+ : 0;const shouldABTest =!feRequest.zeroDataRetention &&abRate > 0 &&Math.random() <= abRate &&- abUrlEnv;+ config.FIRE_ENGINE_AB_URL;if (!shouldABTest) {return { shouldCompare: false, mirrorPromise: null };@@ -103,15 +102,16 @@ export function abTestFireEngine(const mirrorPromise = (async (): Promise<MirrorResult> => {const abortController = new AbortController();- const timeoutHandle = setTimeout(() => {- abortController.abort();- }, timeout);+ const timeoutHandle = setTimeout(() => abortController.abort(), timeout);+ let jobId: string | undefined;try {- abLogger.info("A/B-testing scrapeURL to staging");+ abLogger.info("A/B-testing fire-engine to staging", {+ url: feRequest.url,+ });- const response = await robustFetch({- url: `${abUrlEnv}/scrape`,+ let response = await robustFetch({+ url: `${config.FIRE_ENGINE_AB_URL}/scrape`,method: "POST",body: feRequest,logger: abLogger,@@ -122,18 +122,27 @@ export function abTestFireEngine(abort: abortController.signal,});- abLogger.info("A/B-testing scrapeURL (staging) request completed");+ jobId = response?.jobId;++ if (!response.content && response.docUrl) {+ const doc = await getDocFromGCS(response.docUrl.split("/").pop() ?? "");+ if (doc) {+ response = { ...response, ...doc };+ }+ }++ abLogger.info("A/B-testing fire-engine (staging) request completed", {+ url: feRequest.url,+ hasContent: !!response?.content,+ });- // Extract the fields we need for comparisonconst feResponse: FireEngineResponse | null =response &&typeof response.content === "string" &&- typeof response.pageStatusCode === "number" &&- typeof response.timeTaken === "number"+ typeof response.pageStatusCode === "number"? {content: response.content,pageStatusCode: response.pageStatusCode,- timeTaken: response.timeTaken,}: null;@@ -143,7 +152,10 @@ export function abTestFireEngine(timeTaken: Date.now() - startTime,};} catch (error) {- abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });+ abLogger.warn("A/B-testing fire-engine (staging) failed", {+ error,+ url: feRequest.url,+ });return {response: null,error: error as Error,@@ -151,11 +163,26 @@ export function abTestFireEngine(};} finally {clearTimeout(timeoutHandle);++ if (jobId && config.FIRE_ENGINE_AB_URL) {+ robustFetch({+ url: `${config.FIRE_ENGINE_AB_URL}/scrape/${jobId}`,+ method: "DELETE",+ headers: {},+ logger: abLogger.child({ method: "abTestFireEngine/delete", jobId }),+ mock: null,+ }).catch(e => {+ abLogger.warn("Failed to delete AB test job from fire-engine", {+ error: e,+ jobId,+ });+ });+ }}})();return {- shouldCompare: compareEnabled,+ shouldCompare: config.FIRE_ENGINE_AB_COMPARE_ENABLED,mirrorPromise,};}