Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): give an agent-started batch scrape its Bigtable credit row (#4700)

Gergő Móricz22225aadd364
diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.ts
index a153bb0be..906c58d90 100644
--- a/apps/api/src/controllers/v2/agent.ts
+++ b/apps/api/src/controllers/v2/agent.ts
@@ -1,4 +1,5 @@
import { v7 as uuidv7 } from "uuid";
+import { AGENT_REQUEST_CREDITS_SHARDS } from "../../lib/request-credits-store";
import { Response } from "express";
import {
AgentRequest,
@@ -205,7 +206,7 @@ export async function agentController(
target_hint: req.body.urls?.[0] ?? req.body.prompt ?? "",
zeroDataRetention: false, // not supported for agent
api_key_id: req.acuc?.api_key_id ?? null,
- creditsShards: 8,
+ creditsShards: AGENT_REQUEST_CREDITS_SHARDS,
});
const passthrough = await fetch(
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index a44e5ab4f..a0ec17177 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -50,7 +50,11 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { UnsupportedSiteError } from "../../lib/error";
-import { requestCreditsShards } from "../../lib/request-credits-store";
+import {
+ AGENT_REQUEST_CREDITS_SHARDS,
+ initializeRequestCredits,
+ requestCreditsShards,
+} from "../../lib/request-credits-store";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -351,6 +355,35 @@ export async function batchScrapeController(
),
creditsShards: requestCreditsShards(urls.length),
});
+ } else if (!req.body.appendToId) {
+ // An agent-started batch is recorded under the agent's request rather
+ // than as a request of its own, so logRequest never created a credit
+ // row for it. Its status is still polled: v2 reads credits under the
+ // saved agent request id, v1 under the batch id, and its children record
+ // credits under the agent request id. Make sure both rows exist. The
+ // agent row is created once with the agent controller's shard count;
+ // for an agent that came through that controller this is a no-op, for
+ // one that reaches the API first through interop it is the only writer.
+ const agentRequestId = req.body.__agentInterop?.requestId;
+ if (agentRequestId) {
+ await initializeRequestCredits(
+ agentRequestId,
+ AGENT_REQUEST_CREDITS_SHARDS,
+ ).catch(error => {
+ logger.warn("Failed to initialize Bigtable request credits", {
+ error,
+ requestId: agentRequestId,
+ shards: AGENT_REQUEST_CREDITS_SHARDS,
+ });
+ });
+ }
+ const creditsShards = requestCreditsShards(urls.length);
+ await initializeRequestCredits(id, creditsShards).catch(error => {
+ logger.warn("Failed to initialize Bigtable request credits", {
+ error,
+ shards: creditsShards,
+ });
+ });
}
const sc: StoredCrawl = req.body.appendToId
diff --git a/apps/api/src/lib/request-credits-store.ts b/apps/api/src/lib/request-credits-store.ts
index f1e4e87a6..88fcecc06 100644
--- a/apps/api/src/lib/request-credits-store.ts
+++ b/apps/api/src/lib/request-credits-store.ts
@@ -81,6 +81,14 @@ export function requestCreditsShardForJob(
);
}
+/**
+ * Shard count for an agent request's credit row. An agent's job count is not
+ * known up front, and every writer that may create the row for an agent id
+ * (the agent controller, and agent-interop calls that reach the API first)
+ * must agree on it, because the row is created once and never resized.
+ */
+export const AGENT_REQUEST_CREDITS_SHARDS = 8;
+
export function requestCreditsShards(maxJobs: number): number {
if (!Number.isFinite(maxJobs) || maxJobs <= 0) return 8;
if (maxJobs <= 1_000) return 8;