Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

conc boost for agent interop (#2770)

Rafael Miller7d125a83e545
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index f7856426f..de76995f0 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -23,6 +23,8 @@ import { logRequest } from "../../services/logging/log_job";
import { getErrorContactMessage } from "../../lib/deployment";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
+const AGENT_INTEROP_CONCURRENCY_BOOST = 3;
+
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
res: Response<ScrapeResponse>,
@@ -100,6 +102,8 @@ export async function scrapeController(
const shouldBill = req.body.__agentInterop?.shouldBill ?? true;
const agentRequestId = req.body.__agentInterop?.requestId ?? null;
+ const boostConcurrency =
+ req.body.__agentInterop?.boostConcurrency ?? false;
const logger = _logger.child({
method: "scrapeController",
@@ -172,10 +176,15 @@ export async function scrapeController(
}
req.on("close", () => aborter.abort());
+ const baseConcurrency = req.acuc?.concurrency || 1;
+ const concurrency = boostConcurrency
+ ? baseConcurrency * AGENT_INTEROP_CONCURRENCY_BOOST
+ : baseConcurrency;
+
doc = await teamConcurrencySemaphore.withSemaphore(
req.auth.team_id,
jobId,
- req.acuc?.concurrency || 1,
+ concurrency,
aborter.signal,
timeout ?? 60_000,
async limited => {
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 218285bc9..0393a080c 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -771,6 +771,7 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({
auth: z.string(),
requestId: z.string(),
shouldBill: z.boolean(),
+ boostConcurrency: z.boolean().optional()
})
.optional(),
});