Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): improved a/b testing (#2816)
tomsideguide904a9a0204cf
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex ab6c1e81f..8b56294d3 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -88,7 +88,7 @@ const configSchema = z.object({FIRE_ENGINE_STAGING_URL: z.string().optional(),FIRE_ENGINE_AB_URL: z.string().optional(),FIRE_ENGINE_AB_RATE: z.coerce.number().optional(),- FIRE_ENGINE_AB_COMPARE_ENABLED: z.stringbool().default(false),+ FIRE_ENGINE_AB_MODE: z.enum(["mirror", "split"]).default("mirror"),// ScrapeURLSCRAPEURL_AB_HOST: z.string().optional(),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex dce97e034..ca486e842 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -14,7 +14,7 @@ import {ProxySelectionError,} from "../../error";import { MockState } from "../../lib/mock";-import { fireEngineStagingURL, fireEngineURL } from "./scrape";+import { fireEngineURL } from "./scrape";import { getDocFromGCS } from "../../../../lib/gcs-jobs";import { Meta } from "../..";@@ -140,10 +140,10 @@ export async function fireEngineCheckStatus(jobId: string,mock: MockState | null,abort?: AbortSignal,- production = true,+ baseUrl: string = fireEngineURL,): Promise<FireEngineCheckStatusSuccess> {let status = await robustFetch({- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,+ url: `${baseUrl}/scrape/${jobId}`,method: "GET",logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),headers: {},diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.tsindex 22fc538f1..a5a267d6d 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts@@ -2,14 +2,14 @@ import { Logger } from "winston";import { robustFetch } from "../../lib/fetch";import { MockState } from "../../lib/mock";-import { fireEngineStagingURL, fireEngineURL } from "./scrape";+import { fireEngineURL } from "./scrape";export async function fireEngineDelete(logger: Logger,jobId: string | undefined,mock: MockState | null,abort?: AbortSignal,- production = true,+ baseUrl: string = fireEngineURL,) {// jobId only supplied if we need to defer deletionif (!jobId) {@@ -18,7 +18,7 @@ export async function fireEngineDelete(}await robustFetch({- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,+ url: `${baseUrl}/scrape/${jobId}`,method: "DELETE",headers: {},logger: logger.child({ method: "fireEngineDelete/robustFetch", jobId }),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 5c93087b4..721db2ee2 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -2,6 +2,8 @@ import { Logger } from "winston";import { Meta } from "../..";import {fireEngineScrape,+ fireEngineURL,+ fireEngineStagingURL,FireEngineScrapeRequestChromeCDP,FireEngineScrapeRequestCommon,FireEngineScrapeRequestPlaywright,@@ -34,6 +36,8 @@ import { AbortManagerThrownError } from "../../lib/abortManager";import { youtubePostprocessor } from "../../postprocessors/youtube";import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";import { getBrandingScript } from "./brandingScript";+import { abTestFireEngine } from "../../../../services/ab-test";+import { scheduleABComparison } from "../../../../services/ab-test-comparison";/** Default wait (ms) before running the branding script when user did not set waitFor. Lets the page settle so DOM/images are ready and reduces JS errors. */const BRANDING_DEFAULT_WAIT_MS = 2000;@@ -58,6 +62,13 @@ async function performFireEngineScrape<const startTime = Date.now();let pollCount = 0;+ let baseUrl = production ? fireEngineURL : fireEngineStagingURL;++ const abTest = abTestFireEngine(request);+ if (abTest.mode === "split") {+ baseUrl = abTest.baseUrl;+ }+setSpanAttributes(span, {"fire-engine.url": request.url,"fire-engine.priority": request.priority,@@ -68,6 +79,7 @@ async function performFireEngineScrape<"fire-engine.mobile": (request as any).mobile,"fire-engine.skip_tls": (request as any).skipTlsVerification,"fire-engine.production": production,+ "fire-engine.ab_mode": abTest.mode,});const scrape = await fireEngineScrape(meta,@@ -75,7 +87,7 @@ async function performFireEngineScrape<request,mock,abort,- production,+ baseUrl,);let status: FireEngineCheckStatusSuccess | undefined = undefined;@@ -94,7 +106,7 @@ async function performFireEngineScrape<(scrape as any).jobId,mock,undefined,- production,+ baseUrl,).catch(e => {logger.error("Failed to delete job from Fire Engine", { error: e });});@@ -113,7 +125,7 @@ async function performFireEngineScrape<(scrape as any).jobId,mock,abort,- production,+ baseUrl,);} catch (error) {if (error instanceof StillProcessingError) {@@ -136,7 +148,7 @@ async function performFireEngineScrape<(scrape as any).jobId,mock,undefined,- production,+ baseUrl,).catch(e => {logger.error("Failed to delete job from Fire Engine", {error: e,@@ -156,7 +168,7 @@ async function performFireEngineScrape<(scrape as any).jobId,mock,undefined,- production,+ baseUrl,).catch(e => {logger.error("Failed to delete job from Fire Engine", {error: e,@@ -209,11 +221,24 @@ async function performFireEngineScrape<(scrape as any).jobId,mock,undefined,- production,+ baseUrl,).catch(e => {logger.error("Failed to delete job from Fire Engine", { error: e });});+ if (abTest.mode === "mirror") {+ scheduleABComparison(+ meta.url,+ {+ content: status.content,+ pageStatusCode: status.pageStatusCode,+ },+ Date.now() - startTime,+ abTest.mirrorPromise,+ logger,+ );+ }+setSpanAttributes(span, {"fire-engine.poll_count": pollCount,"fire-engine.duration_ms": Date.now() - startTime,diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex beef8c46a..f791e1839 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -1,5 +1,4 @@import { Logger } from "winston";-import * as Sentry from "@sentry/node";import { z } from "zod";import { InternalAction } from "../../../../controllers/v1/types";@@ -17,8 +16,6 @@ import {UnsupportedFileError,} from "../../error";import { Meta } from "../..";-import { abTestFireEngine } from "../../../../services/ab-test";-import { scheduleABComparison } from "../../../../services/ab-test-comparison";import { config } from "../../../../config";export type FireEngineScrapeRequestCommon = {@@ -186,13 +183,10 @@ export async function fireEngineScrape<request: FireEngineScrapeRequestCommon & Engine,mock: MockState | null,abort?: AbortSignal,- production = true,+ baseUrl: string = fireEngineURL,): Promise<z.infer<typeof processingSchema> | FireEngineCheckStatusSuccess> {- const abTest = abTestFireEngine(request);- const productionStartTime = Date.now();-let status = await robustFetch({- url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,+ url: `${baseUrl}/scrape`,method: "POST",headers: {},body: request,@@ -226,22 +220,6 @@ export async function fireEngineScrape<}logger.debug("Scrape succeeded!");-- // Schedule A/B comparison if enabled (fire-and-forget)- if (abTest.shouldCompare && abTest.mirrorPromise) {- const productionTimeTaken = Date.now() - productionStartTime;- scheduleABComparison(- meta.url,- {- content: successParse.data.content,- pageStatusCode: successParse.data.pageStatusCode,- },- productionTimeTaken,- abTest.mirrorPromise,- logger,- );- }-return successParse.data;} else if (processingParse.success) {return processingParse.data;diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.tsindex f7113b266..ae5c302ca 100644--- a/apps/api/src/services/ab-test.ts+++ b/apps/api/src/services/ab-test.ts@@ -73,6 +73,11 @@ export function abTestJob(webScraperOptions: ScrapeJobData) {}}+type ABTestDecision =+ | { mode: "none" }+ | { mode: "mirror"; mirrorPromise: Promise<MirrorResult> }+ | { mode: "split"; baseUrl: string };+export function abTestFireEngine(feRequest: FireEngineScrapeRequestCommon &(@@ -80,7 +85,7 @@ export function abTestFireEngine(| FireEngineScrapeRequestPlaywright| FireEngineScrapeRequestTLSClient),-): { shouldCompare: boolean; mirrorPromise: Promise<MirrorResult> | null } {+): ABTestDecision {const abLogger = _logger.child({ method: "ABTestFireEngine" });const abRate = config.FIRE_ENGINE_AB_RATE@@ -94,7 +99,11 @@ export function abTestFireEngine(config.FIRE_ENGINE_AB_URL;if (!shouldABTest) {- return { shouldCompare: false, mirrorPromise: null };+ return { mode: "none" };+ }++ if (config.FIRE_ENGINE_AB_MODE === "split") {+ return { mode: "split", baseUrl: config.FIRE_ENGINE_AB_URL! };}const timeout = Math.min(60000, (feRequest.timeout ?? 30000) + 10000);@@ -182,7 +191,7 @@ export function abTestFireEngine(})();return {- shouldCompare: config.FIRE_ENGINE_AB_COMPARE_ENABLED,+ mode: "mirror",mirrorPromise,};}