Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/abtest): send off AB requests at a higher level

Gergő Móricz71218e7cf3a9
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index fc8ae71b9..560160143 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -632,49 +632,6 @@ export async function scrapeURL(
});
}
- // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
- try {
- const abRateEnv = process.env.SCRAPEURL_AB_RATE;
- const abHostEnv = process.env.SCRAPEURL_AB_HOST;
- const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
- const shouldABTest = !meta.internalOptions.zeroDataRetention
- && abRate > 0
- && Math.random() <= abRate
- && abHostEnv
- && meta.internalOptions.v1Agent === undefined
- && meta.internalOptions.v1JSONAgent === undefined;
- if (shouldABTest) {
- (async () => {
- try {
- const abLogger = meta.logger.child({ method: "scrapeURL/abTestToStaging" });
- abLogger.info("A/B-testing scrapeURL to staging");
- const abort = AbortSignal.timeout(Math.min(60000, (meta.options.timeout ?? 30000) + 10000));
- await robustFetch({
- url: `http://${abHostEnv}/v2/scrape`,
- method: "POST",
- body: {
- url: meta.url,
- ...meta.options,
- origin: (meta.options as any).origin ?? "api",
- timeout: meta.options.timeout ?? 30000,
- maxAge: 1000000000,
- },
- logger: abLogger,
- tryCount: 1,
- ignoreResponse: true,
- mock: null,
- abort,
- });
- abLogger.info("A/B-testing scrapeURL (staging) request sent");
- } catch (error) {
- meta.logger.warn("A/B-testing scrapeURL (staging) failed", { error });
- }
- })();
- }
- } catch (error) {
- meta.logger.warn("Failed to initiate A/B test to staging", { error });
- }
-
try {
while (true) {
try {
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index a4a5ab922..7f80bf705 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -22,6 +22,7 @@ import { Logger } from "winston";
import { Job } from "bullmq";
import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";
import { deserializeTransportableError } from "../lib/error-serde";
+import { robustFetch } from '../scraper/scrapeURL/lib/fetch';
/**
* Checks if a job is a crawl or batch scrape based on its options
@@ -58,6 +59,51 @@ export async function _addScrapeJobToBullMQ(
jobId: string,
jobPriority: number,
): Promise<Job> {
+ // Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
+ const abLogger = _logger.child({ method: "ABTestToStaging" });
+ try {
+ const abRateEnv = process.env.SCRAPEURL_AB_RATE;
+ const abHostEnv = process.env.SCRAPEURL_AB_HOST;
+ const abRate = abRateEnv !== undefined ? Math.max(0, Math.min(1, Number(abRateEnv))) : 0;
+ const shouldABTest = webScraperOptions.mode === "single_urls"
+ && !webScraperOptions.zeroDataRetention
+ && !webScraperOptions.internalOptions?.zeroDataRetention
+ && abRate > 0
+ && Math.random() <= abRate
+ && abHostEnv
+ && webScraperOptions.internalOptions?.v1Agent === undefined
+ && webScraperOptions.internalOptions?.v1JSONAgent === undefined;
+ if (shouldABTest) {
+ (async () => {
+ try {
+ abLogger.info("A/B-testing scrapeURL to staging");
+ const abort = AbortSignal.timeout(Math.min(60000, (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000));
+ await robustFetch({
+ url: `http://${abHostEnv}/v2/scrape`,
+ method: "POST",
+ body: {
+ url: webScraperOptions.url,
+ ...webScraperOptions.scrapeOptions,
+ origin: (webScraperOptions.scrapeOptions as any).origin ?? "api",
+ timeout: webScraperOptions.scrapeOptions.timeout ?? 30000,
+ maxAge: 1000000000,
+ },
+ logger: abLogger,
+ tryCount: 1,
+ ignoreResponse: true,
+ mock: null,
+ abort,
+ });
+ abLogger.info("A/B-testing scrapeURL (staging) request sent");
+ } catch (error) {
+ abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
+ }
+ })();
+ }
+ } catch (error) {
+ abLogger.warn("Failed to initiate A/B test to staging", { error });
+ }
+
if (
webScraperOptions &&
webScraperOptions.team_id