Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/abtest): send off AB requests at a higher level
Gergő Móricz71218e7cf3a9
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex fc8ae71b9..560160143 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -632,49 +632,6 @@ export async function scrapeURL(});}- // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE- try {- const abRateEnv = process.env.SCRAPEURL_AB_RATE;- const abHostEnv = process.env.SCRAPEURL_AB_HOST;- const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;- const shouldABTest = !meta.internalOptions.zeroDataRetention- && abRate > 0- && Math.random() <= abRate- && abHostEnv- && meta.internalOptions.v1Agent === undefined- && meta.internalOptions.v1JSONAgent === undefined;- if (shouldABTest) {- (async () => {- try {- const abLogger = meta.logger.child({ method: "scrapeURL/abTestToStaging" });- abLogger.info("A/B-testing scrapeURL to staging");- const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));- await robustFetch({- url: `http://${abHostEnv}/v2/scrape`,- method: "POST",- body: {- url: meta.url,- ...meta.options,- origin: (meta.options as any).origin ?? "api",- timeout: meta.options.timeout ?? 30000,- maxAge: 1000000000,- },- logger: abLogger,- tryCount: 1,- ignoreResponse: true,- mock: null,- abort,- });- abLogger.info("A/B-testing scrapeURL (staging) request sent");- } catch (error) {- meta.logger.warn("A/B-testing scrapeURL (staging) failed", { error });- }- })();- }- } catch (error) {- meta.logger.warn("Failed to initiate A/B test to staging", { error });- }-try {while (true) {try {diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex a4a5ab922..7f80bf705 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -22,6 +22,7 @@ import { Logger } from "winston";import { Job } from "bullmq";import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";import { deserializeTransportableError } from "../lib/error-serde";+import { robustFetch } from '../scraper/scrapeURL/lib/fetch';/*** Checks if a job is a crawl or batch scrape based on its options@@ -58,6 +59,51 @@ export async function _addScrapeJobToBullMQ(jobId: string,jobPriority: number,): Promise<Job> {+ // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE+ const abLogger = _logger.child({ method: "ABTestToStaging" });+ try {+ const abRateEnv = process.env.SCRAPEURL_AB_RATE;+ const abHostEnv = process.env.SCRAPEURL_AB_HOST;+ const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;+ const shouldABTest = webScraperOptions.mode === "single_urls"+ && !webScraperOptions.zeroDataRetention+ && !webScraperOptions.internalOptions?.zeroDataRetention+ && abRate > 0+ && Math.random() <= abRate+ && abHostEnv+ && webScraperOptions.internalOptions?.v1Agent === undefined+ && webScraperOptions.internalOptions?.v1JSONAgent === undefined;+ if (shouldABTest) {+ (async () => {+ try {+ abLogger.info("A/B-testing scrapeURL to staging");+ const abort = AbortSignal.timeout(Math.min(60000, (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000));+ await robustFetch({+ url: `http://${abHostEnv}/v2/scrape`,+ method: "POST",+ body: {+ url: webScraperOptions.url,+ ...webScraperOptions.scrapeOptions,+ origin: (webScraperOptions.scrapeOptions as any).origin ?? "api",+ timeout: webScraperOptions.scrapeOptions.timeout ?? 30000,+ maxAge: 1000000000,+ },+ logger: abLogger,+ tryCount: 1,+ ignoreResponse: true,+ mock: null,+ abort,+ });+ abLogger.info("A/B-testing scrapeURL (staging) request sent");+ } catch (error) {+ abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });+ }+ })();+ }+ } catch (error) {+ abLogger.warn("Failed to initiate A/B test to staging", { error });+ }+if (webScraperOptions &&webScraperOptions.team_id