Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(ab): add consistently at producer level, not consumer level (#1990)
Gergő Móricz03c3bf29e70d
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 52fc2fbdb..3fc996dec 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -5,6 +5,7 @@ import { getACUCTeam } from "../controllers/auth";import { getCrawl, StoredCrawl } from "./crawl-redis";import { getScrapeQueue } from "../services/queue-service";import { logger } from "./logger";+import { abTestJob } from "../services/ab-test";const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;const constructQueueKey = (team_id: string) =>@@ -274,6 +275,8 @@ export async function concurrentJobDone(job: Job) {}}+ abTestJob(nextJob.job.data);+(await getScrapeQueue()).add(nextJob.job.id,{diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.tsnew file mode 100644index 000000000..87c0ce7e5--- /dev/null+++ b/apps/api/src/services/ab-test.ts@@ -0,0 +1,48 @@+import { WebScraperOptions } from "../types";+import { logger as _logger } from "../lib/logger";+import { robustFetch } from "../scraper/scrapeURL/lib/fetch";++export function abTestJob(webScraperOptions: WebScraperOptions) {+ // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE+ const abLogger = _logger.child({ method: "ABTestToStaging" });+ try {+ const abRateEnv = process.env.SCRAPEURL_AB_RATE;+ const abHostEnv = process.env.SCRAPEURL_AB_HOST;+ const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;+ const shouldABTest = webScraperOptions.mode === "single_urls"+ && !webScraperOptions.zeroDataRetention+ && !webScraperOptions.internalOptions?.zeroDataRetention+ && abRate > 0+ && Math.random() <= abRate+ && abHostEnv+ && webScraperOptions.internalOptions?.v1Agent === undefined+ && webScraperOptions.internalOptions?.v1JSONAgent === undefined;+ if (shouldABTest) {+ (async () => {+ try {+ abLogger.info("A/B-testing scrapeURL to staging");+ const abort = AbortSignal.timeout(Math.min(60000, (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000));+ await robustFetch({+ url: `http://${abHostEnv}/v2/scrape`,+ method: "POST",+ body: {+ url: webScraperOptions.url,+ ...webScraperOptions.scrapeOptions,+ origin: (webScraperOptions.scrapeOptions as any).origin ?? "api",+ },+ logger: abLogger,+ tryCount: 1,+ ignoreResponse: true,+ mock: null,+ abort,+ });+ abLogger.info("A/B-testing scrapeURL (staging) request sent");+ } catch (error) {+ abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });+ }+ })();+ }+ } catch (error) {+ abLogger.warn("Failed to initiate A/B test to staging", { error });+ }+}\ No newline at end of filediff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 3e86fab5e..53c4b5d41 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -23,6 +23,7 @@ import { Job } from "bullmq";import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";import { deserializeTransportableError } from "../lib/error-serde";import { robustFetch } from '../scraper/scrapeURL/lib/fetch';+import { abTestJob } from "./ab-test";/*** Checks if a job is a crawl or batch scrape based on its options@@ -59,48 +60,7 @@ export async function _addScrapeJobToBullMQ(jobId: string,jobPriority: number,): Promise<Job> {- // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE- const abLogger = _logger.child({ method: "ABTestToStaging" });- try {- const abRateEnv = process.env.SCRAPEURL_AB_RATE;- const abHostEnv = process.env.SCRAPEURL_AB_HOST;- const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;- const shouldABTest = webScraperOptions.mode === "single_urls"- && !webScraperOptions.zeroDataRetention- && !webScraperOptions.internalOptions?.zeroDataRetention- && abRate > 0- && Math.random() <= abRate- && abHostEnv- && webScraperOptions.internalOptions?.v1Agent === undefined- && webScraperOptions.internalOptions?.v1JSONAgent === undefined;- if (shouldABTest) {- (async () => {- try {- abLogger.info("A/B-testing scrapeURL to staging");- const abort = AbortSignal.timeout(Math.min(60000, (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000));- await robustFetch({- url: `http://${abHostEnv}/v2/scrape`,- method: "POST",- body: {- url: webScraperOptions.url,- ...webScraperOptions.scrapeOptions,- origin: (webScraperOptions.scrapeOptions as any).origin ?? "api",- },- logger: abLogger,- tryCount: 1,- ignoreResponse: true,- mock: null,- abort,- });- abLogger.info("A/B-testing scrapeURL (staging) request sent");- } catch (error) {- abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });- }- })();- }- } catch (error) {- abLogger.warn("Failed to initiate A/B test to staging", { error });- }+ abTestJob(webScraperOptions);if (webScraperOptions &&