Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): give an agent-started batch scrape its Bigtable credit row (#4700)
Gergő Móricz22225aadd364
diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex a153bb0be..906c58d90 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -1,4 +1,5 @@import { v7 as uuidv7 } from "uuid";+import { AGENT_REQUEST_CREDITS_SHARDS } from "../../lib/request-credits-store";import { Response } from "express";import {AgentRequest,@@ -205,7 +206,7 @@ export async function agentController(target_hint: req.body.urls?.[0] ?? req.body.prompt ?? "",zeroDataRetention: false, // not supported for agentapi_key_id: req.acuc?.api_key_id ?? null,- creditsShards: 8,+ creditsShards: AGENT_REQUEST_CREDITS_SHARDS,});const passthrough = await fetch(diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex a44e5ab4f..a0ec17177 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -50,7 +50,11 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";import { UnsupportedSiteError } from "../../lib/error";-import { requestCreditsShards } from "../../lib/request-credits-store";+import {+ AGENT_REQUEST_CREDITS_SHARDS,+ initializeRequestCredits,+ requestCreditsShards,+} from "../../lib/request-credits-store";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -351,6 +355,35 @@ export async function batchScrapeController(),creditsShards: requestCreditsShards(urls.length),});+ } else if (!req.body.appendToId) {+ // An agent-started batch is recorded under the agent's request rather+ // than as a request of its own, so logRequest never created a credit+ // row for it. Its status is still polled: v2 reads credits under the+ // saved agent request id, v1 under the batch id, and its children record+ // credits under the agent request id. Make sure both rows exist. The+ // agent row is created once with the agent controller's shard count;+ // for an agent that came through that controller this is a no-op, for+ // one that reaches the API first through interop it is the only writer.+ const agentRequestId = req.body.__agentInterop?.requestId;+ if (agentRequestId) {+ await initializeRequestCredits(+ agentRequestId,+ AGENT_REQUEST_CREDITS_SHARDS,+ ).catch(error => {+ logger.warn("Failed to initialize Bigtable request credits", {+ error,+ requestId: agentRequestId,+ shards: AGENT_REQUEST_CREDITS_SHARDS,+ });+ });+ }+ const creditsShards = requestCreditsShards(urls.length);+ await initializeRequestCredits(id, creditsShards).catch(error => {+ logger.warn("Failed to initialize Bigtable request credits", {+ error,+ shards: creditsShards,+ });+ });}const sc: StoredCrawl = req.body.appendToIddiff --git a/apps/api/src/lib/request-credits-store.ts b/apps/api/src/lib/request-credits-store.tsindex f1e4e87a6..88fcecc06 100644--- a/apps/api/src/lib/request-credits-store.ts+++ b/apps/api/src/lib/request-credits-store.ts@@ -81,6 +81,14 @@ export function requestCreditsShardForJob();}+/**+ * Shard count for an agent request's credit row. An agent's job count is not+ * known up front, and every writer that may create the row for an agent id+ * (the agent controller, and agent-interop calls that reach the API first)+ * must agree on it, because the row is created once and never resized.+ */+export const AGENT_REQUEST_CREDITS_SHARDS = 8;+export function requestCreditsShards(maxJobs: number): number {if (!Number.isFinite(maxJobs) || maxJobs <= 0) return 8;if (maxJobs <= 1_000) return 8;