Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): ab/b (#2570)
Gergő Móricz47e1379bac5e
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 696d00371..a59599255 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -80,6 +80,25 @@ export async function scrapeController(const zeroDataRetention =req.acuc?.flags?.forceZDR || (req.body.zeroDataRetention ?? false);+ if (+ req.body.__agentInterop &&+ config.AGENT_INTEROP_SECRET &&+ req.body.__agentInterop.auth !== config.AGENT_INTEROP_SECRET+ ) {+ return res.status(403).json({+ success: false,+ error: "Invalid agent interop.",+ });+ } else if (req.body.__agentInterop && !config.AGENT_INTEROP_SECRET) {+ return res.status(403).json({+ success: false,+ error: "Agent interop is not enabled.",+ });+ }++ const shouldBill = req.body.__agentInterop?.shouldBill ?? true;+ const agentRequestId = req.body.__agentInterop?.requestId ?? null;+const logger = _logger.child({method: "scrapeController",jobId,@@ -100,17 +119,19 @@ export async function scrapeController(account: req.account,});- await logRequest({- id: jobId,- kind: "scrape",- api_version: "v2",- team_id: req.auth.team_id,- origin: req.body.origin ?? "api",- integration: req.body.integration,- target_hint: req.body.url,- zeroDataRetention: zeroDataRetention || false,- api_key_id: req.acuc?.api_key_id ?? null,- });+ if (!agentRequestId) {+ await logRequest({+ id: jobId,+ kind: "scrape",+ api_version: "v2",+ team_id: req.auth.team_id,+ origin: req.body.origin ?? "api",+ integration: req.body.integration,+ target_hint: req.body.url,+ zeroDataRetention: zeroDataRetention || false,+ api_key_id: req.acuc?.api_key_id ?? null,+ });+ }setSpanAttributes(span, {"scrape.zero_data_retention": zeroDataRetention,@@ -205,7 +226,7 @@ export async function scrapeController(? true: false,unnormalizedSourceURL: preNormalizedBody.url,- bypassBilling: isDirectToBullMQ,+ bypassBilling: isDirectToBullMQ || !shouldBill,zeroDataRetention,teamFlags: req.acuc?.flags ?? null,},@@ -216,6 +237,7 @@ export async function scrapeController(zeroDataRetention,apiKeyId: req.acuc?.api_key_id ?? null,concurrencyLimited: limited,+ requestId: agentRequestId ?? undefined,},};diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 0997ae671..2e13de248 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -741,6 +741,13 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({origin: z.string().optional().prefault("api"),integration: integrationSchema.optional().transform(val => val || null),zeroDataRetention: z.boolean().optional(),+ __agentInterop: z+ .object({+ auth: z.string(),+ requestId: z.string(),+ shouldBill: z.boolean(),+ })+ .optional(),});export const scrapeRequestSchema = strictWithMessage(scrapeRequestSchemaBase)