Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): initialize Bigtable credit rows for every crawl starter (#4696)

Gergő Móricz99b3dcc508d0
diff --git a/apps/api/src/controllers/v0/admin/crawl-monitor.ts b/apps/api/src/controllers/v0/admin/crawl-monitor.ts
index 75ba03b18..ccc804ee6 100644
--- a/apps/api/src/controllers/v0/admin/crawl-monitor.ts
+++ b/apps/api/src/controllers/v0/admin/crawl-monitor.ts
@@ -20,6 +20,7 @@ import {
resolveNewGroupBackend,
} from "../../../services/worker/nuq-router";
import { _addScrapeJobToBullMQ } from "../../../services/queue-jobs";
+import { requestCreditsShards } from "../../../lib/request-credits-store";
type ResponseType = {
ok: boolean;
@@ -52,6 +53,7 @@ export async function crawlMonitorController(
target_hint: "https://firecrawl.dev",
zeroDataRetention: false,
api_key_id: req.acuc?.api_key_id ?? null,
+ creditsShards: requestCreditsShards(2),
});
const sc: StoredCrawl = {
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 06acd2184..bb189b483 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -48,6 +48,7 @@ import {
SAFE_MODE_V0_UNSUPPORTED_MESSAGE,
} from "../../lib/safe-mode";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
+import { requestCreditsShards } from "../../lib/request-credits-store";
export async function crawlController(req: Request, res: Response) {
try {
@@ -94,6 +95,9 @@ export async function crawlController(req: Request, res: Response) {
jobAccessExpiresAt: new Date(
Date.now() + (chunk?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
),
+ creditsShards: requestCreditsShards(
+ req.body?.crawlerOptions?.limit ?? defaultCrawlerOptions.limit,
+ ),
});
redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 0ed1de57d..757a35805 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -42,6 +42,7 @@ import { processEngpickerJob } from "../../lib/engpicker";
import { logRequest, shutdownPubSubLogging } from "../logging/log_job";
import { startSiemLoggingConsumer } from "../siem-logging/worker";
import { closeSiemLoggingTransport } from "../../lib/siem-logging/transport";
+import { requestCreditsShards } from "../../lib/request-credits-store";
const workerLockDuration = config.WORKER_LOCK_DURATION;
const workerStalledCheckInterval = config.WORKER_STALLED_CHECK_INTERVAL;
@@ -485,6 +486,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {
target_hint: url,
zeroDataRetention: false,
api_key_id: null,
+ creditsShards: requestCreditsShards(limit),
});
const crawlerOptions = {
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 63aefb0de..6b08c777e 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -80,6 +80,7 @@ import {
reconstructKnownState,
searchStatusToPageStatus,
} from "./search/persist";
+import { requestCreditsShards } from "../../lib/request-credits-store";
const logger = _logger.child({ module: "monitoring-runner" });
export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };
@@ -646,6 +647,9 @@ async function enqueueMonitorCrawlTarget(params: {
target_hint: body.url,
zeroDataRetention: false,
api_key_id: null,
+ creditsShards: requestCreditsShards(
+ body.limit ?? MONITOR_CHECK_PAGE_SCAN_LIMIT,
+ ),
});
const crawlerOptions = {