Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(ab): add consistently at producer level, not consumer level (#1990)

Gergő Móricz03c3bf29e70d
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 52fc2fbdb..3fc996dec 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -5,6 +5,7 @@ import { getACUCTeam } from "../controllers/auth";
import { getCrawl, StoredCrawl } from "./crawl-redis";
import { getScrapeQueue } from "../services/queue-service";
import { logger } from "./logger";
+import { abTestJob } from "../services/ab-test";
const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;
const constructQueueKey = (team_id: string) =>
@@ -274,6 +275,8 @@ export async function concurrentJobDone(job: Job) {
}
}
+ abTestJob(nextJob.job.data);
+
(await getScrapeQueue()).add(
nextJob.job.id,
{
diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.ts
new file mode 100644
index 000000000..87c0ce7e5
--- /dev/null
+++ b/apps/api/src/services/ab-test.ts
@@ -0,0 +1,48 @@
+import { WebScraperOptions } from "../types";
+import { logger as _logger } from "../lib/logger";
+import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
+
+export function abTestJob(webScraperOptions: WebScraperOptions) {
+ // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
+ const abLogger = _logger.child({ method: "ABTestToStaging" });
+ try {
+ const abRateEnv = process.env.SCRAPEURL_AB_RATE;
+ const abHostEnv = process.env.SCRAPEURL_AB_HOST;
+ const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
+ const shouldABTest = webScraperOptions.mode === "single_urls"
+ && !webScraperOptions.zeroDataRetention
+ && !webScraperOptions.internalOptions?.zeroDataRetention
+ && abRate > 0
+ && Math.random() <= abRate
+ && abHostEnv
+ && webScraperOptions.internalOptions?.v1Agent === undefined
+ && webScraperOptions.internalOptions?.v1JSONAgent === undefined;
+ if (shouldABTest) {
+ (async () => {
+ try {
+ abLogger.info("A/B-testing scrapeURL to staging");
+ const abort = AbortSignal.timeout(Math.min(60000, (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000));
+ await robustFetch({
+ url: `http://${abHostEnv}/v2/scrape`,
+ method: "POST",
+ body: {
+ url: webScraperOptions.url,
+ ...webScraperOptions.scrapeOptions,
+ origin: (webScraperOptions.scrapeOptions as any).origin ?? "api",
+ },
+ logger: abLogger,
+ tryCount: 1,
+ ignoreResponse: true,
+ mock: null,
+ abort,
+ });
+ abLogger.info("A/B-testing scrapeURL (staging) request sent");
+ } catch (error) {
+ abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
+ }
+ })();
+ }
+ } catch (error) {
+ abLogger.warn("Failed to initiate A/B test to staging", { error });
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 3e86fab5e..53c4b5d41 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -23,6 +23,7 @@ import { Job } from "bullmq";
import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";
import { deserializeTransportableError } from "../lib/error-serde";
import { robustFetch } from '../scraper/scrapeURL/lib/fetch';
+import { abTestJob } from "./ab-test";
/**
* Checks if a job is a crawl or batch scrape based on its options
@@ -59,48 +60,7 @@ export async function _addScrapeJobToBullMQ(
jobId: string,
jobPriority: number,
): Promise<Job> {
- // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
- const abLogger = _logger.child({ method: "ABTestToStaging" });
- try {
- const abRateEnv = process.env.SCRAPEURL_AB_RATE;
- const abHostEnv = process.env.SCRAPEURL_AB_HOST;
- const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
- const shouldABTest = webScraperOptions.mode === "single_urls"
- && !webScraperOptions.zeroDataRetention
- && !webScraperOptions.internalOptions?.zeroDataRetention
- && abRate > 0
- && Math.random() <= abRate
- && abHostEnv
- && webScraperOptions.internalOptions?.v1Agent === undefined
- && webScraperOptions.internalOptions?.v1JSONAgent === undefined;
- if (shouldABTest) {
- (async () => {
- try {
- abLogger.info("A/B-testing scrapeURL to staging");
- const abort = AbortSignal.timeout(Math.min(60000, (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000));
- await robustFetch({
- url: `http://${abHostEnv}/v2/scrape`,
- method: "POST",
- body: {
- url: webScraperOptions.url,
- ...webScraperOptions.scrapeOptions,
- origin: (webScraperOptions.scrapeOptions as any).origin ?? "api",
- },
- logger: abLogger,
- tryCount: 1,
- ignoreResponse: true,
- mock: null,
- abort,
- });
- abLogger.info("A/B-testing scrapeURL (staging) request sent");
- } catch (error) {
- abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
- }
- })();
- }
- } catch (error) {
- abLogger.warn("Failed to initiate A/B test to staging", { error });
- }
+ abTestJob(webScraperOptions);
if (
webScraperOptions &&