Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL/fire-engine): start AB testing (#1763)
Gergő Móricz5d406643844e
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 8672c00c1..df25b00e1 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -27,6 +27,7 @@ env:TEST_API_KEY_ZDR: ${{ secrets.TEST_API_KEY_ZDR }}TEST_TEAM_ID_ZDR: ${{ secrets.TEST_TEAM_ID_ZDR }}FIRE_ENGINE_BETA_URL: ${{ secrets.FIRE_ENGINE_BETA_URL }}+ FIRE_ENGINE_STAGING_URL: ${{ secrets.FIRE_ENGINE_STAGING_URL }}USE_DB_AUTHENTICATION: trueSERPER_API_KEY: ${{ secrets.SERPER_API_KEY }}ENV: ${{ secrets.ENV }}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 0b43f3e8e..1de240c5d 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -13,7 +13,7 @@ import {FEPageLoadFailed} from "../../error";import { MockState } from "../../lib/mock";-import { fireEngineURL } from "./scrape";+import { fireEngineStagingURL, fireEngineURL } from "./scrape";import { getDocFromGCS } from "../../../../lib/gcs-jobs";import { Meta } from "../..";@@ -133,6 +133,7 @@ export async function fireEngineCheckStatus(jobId: string,mock: MockState | null,abort?: AbortSignal,+ production = true,): Promise<FireEngineCheckStatusSuccess> {let status = await Sentry.startSpan({@@ -143,7 +144,7 @@ export async function fireEngineCheckStatus(},async (span) => {return await robustFetch({- url: `${fireEngineURL}/scrape/${jobId}`,+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,method: "GET",logger: logger.child({ method: "fireEngineCheckStatus/robustFetch" }),headers: {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.tsindex 0ea282926..9af6627ae 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/delete.ts@@ -3,13 +3,14 @@ import * as Sentry from "@sentry/node";import { robustFetch } from "../../lib/fetch";import { MockState } from "../../lib/mock";-import { fireEngineURL } from "./scrape";+import { fireEngineStagingURL, fireEngineURL } from "./scrape";export async function fireEngineDelete(logger: Logger,jobId: string,mock: MockState | null,abort?: AbortSignal,+ production = true,) {await Sentry.startSpan({@@ -20,7 +21,7 @@ export async function fireEngineDelete(},async (span) => {await robustFetch({- url: `${fireEngineURL}/scrape/${jobId}`,+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape/${jobId}`,method: "DELETE",headers: {...(Sentry.isInitialized()diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 91fcda2ea..f2a11a495 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -45,12 +45,14 @@ async function performFireEngineScrape<timeout: number,mock: MockState | null,abort?: AbortSignal,+ production = true,): Promise<FireEngineCheckStatusSuccess> {const scrape = await fireEngineScrape(logger.child({ method: "fireEngineScrape" }),request,mock,abort,+ production,);const startTime = Date.now();@@ -69,6 +71,8 @@ async function performFireEngineScrape<}),scrape.jobId,mock,+ undefined,+ production,);throw new Error("Error limit hit. See e.cause.errors for errors.", {cause: { errors },@@ -93,6 +97,7 @@ async function performFireEngineScrape<scrape.jobId,mock,abort,+ production,);} catch (error) {if (error instanceof StillProcessingError) {@@ -113,6 +118,8 @@ async function performFireEngineScrape<}),scrape.jobId,mock,+ undefined,+ production,);logger.debug("Fire-engine scrape job failed.", {error,@@ -127,6 +134,8 @@ async function performFireEngineScrape<}),scrape.jobId,mock,+ undefined,+ production,);throw error;} else {@@ -170,6 +179,8 @@ async function performFireEngineScrape<}),scrape.jobId,mock,+ undefined,+ production,);return status;@@ -212,6 +223,9 @@ export async function scrapeURLWithFireEngineChromeCDP(const timeout = (timeToRun ?? 300000) + totalWait;+ // const shouldABTest = false;+ const shouldABTest = !meta.internalOptions.zeroDataRetention && Math.random() <= (1/50);+const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestChromeCDP = {url: meta.rewrittenUrl ?? meta.url,@@ -234,6 +248,31 @@ export async function scrapeURLWithFireEngineChromeCDP(zeroDataRetention: meta.internalOptions.zeroDataRetention,};+ if (shouldABTest) {+ (async () => {+ try {+ meta.logger.info("AB-testing fire-engine", { request });++ await performFireEngineScrape(+ meta,+ meta.logger.child({+ method: "scrapeURLWithFireEngineChromeCDP/callFireEngineAB",+ request,+ }),+ request,+ timeout,+ meta.mock,+ meta.internalOptions.abort ?? AbortSignal.timeout(timeout),+ false,+ );++ meta.logger.info("AB-testing fire-engine success", { request });+ } catch (error) {+ meta.logger.error("AB-testing fire-engine failed", { error });+ }+ })();+ }+let response = await performFireEngineScrape(meta,meta.logger.child({@@ -244,6 +283,7 @@ export async function scrapeURLWithFireEngineChromeCDP(timeout,meta.mock,meta.internalOptions.abort ?? AbortSignal.timeout(timeout),+ true,);if (diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex b43989360..f3c566817 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -65,6 +65,7 @@ const schema = z.object({});export const fireEngineURL = process.env.FIRE_ENGINE_BETA_URL ?? "<mock-fire-engine-url>";+export const fireEngineStagingURL = process.env.FIRE_ENGINE_STAGING_URL ?? "<mock-fire-engine-url>";export async function fireEngineScrape<Engine extends@@ -76,6 +77,7 @@ export async function fireEngineScrape<request: FireEngineScrapeRequestCommon & Engine,mock: MockState | null,abort?: AbortSignal,+ production = true,): Promise<z.infer<typeof schema>> {const scrapeRequest = await Sentry.startSpan({@@ -86,7 +88,7 @@ export async function fireEngineScrape<},async (span) => {return await robustFetch({- url: `${fireEngineURL}/scrape`,+ url: `${production ? fireEngineURL : fireEngineStagingURL}/scrape`,method: "POST",headers: {...(Sentry.isInitialized()