Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): initialize Bigtable credit rows for every crawl starter (#4696)
Gergő Móricz99b3dcc508d0
diff --git a/apps/api/src/controllers/v0/admin/crawl-monitor.ts b/apps/api/src/controllers/v0/admin/crawl-monitor.tsindex 75ba03b18..ccc804ee6 100644--- a/apps/api/src/controllers/v0/admin/crawl-monitor.ts+++ b/apps/api/src/controllers/v0/admin/crawl-monitor.ts@@ -20,6 +20,7 @@ import {resolveNewGroupBackend,} from "../../../services/worker/nuq-router";import { _addScrapeJobToBullMQ } from "../../../services/queue-jobs";+import { requestCreditsShards } from "../../../lib/request-credits-store";type ResponseType = {ok: boolean;@@ -52,6 +53,7 @@ export async function crawlMonitorController(target_hint: "https://firecrawl.dev",zeroDataRetention: false,api_key_id: req.acuc?.api_key_id ?? null,+ creditsShards: requestCreditsShards(2),});const sc: StoredCrawl = {diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 06acd2184..bb189b483 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -48,6 +48,7 @@ import {SAFE_MODE_V0_UNSUPPORTED_MESSAGE,} from "../../lib/safe-mode";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";+import { requestCreditsShards } from "../../lib/request-credits-store";export async function crawlController(req: Request, res: Response) {try {@@ -94,6 +95,9 @@ export async function crawlController(req: Request, res: Response) {jobAccessExpiresAt: new Date(Date.now() + (chunk?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,),+ creditsShards: requestCreditsShards(+ req.body?.crawlerOptions?.limit ?? defaultCrawlerOptions.limit,+ ),});redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 0ed1de57d..757a35805 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -42,6 +42,7 @@ import { processEngpickerJob } from "../../lib/engpicker";import { logRequest, shutdownPubSubLogging } from "../logging/log_job";import { startSiemLoggingConsumer } from "../siem-logging/worker";import { closeSiemLoggingTransport } from "../../lib/siem-logging/transport";+import { requestCreditsShards } from "../../lib/request-credits-store";const workerLockDuration = config.WORKER_LOCK_DURATION;const workerStalledCheckInterval = config.WORKER_STALLED_CHECK_INTERVAL;@@ -485,6 +486,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {target_hint: url,zeroDataRetention: false,api_key_id: null,+ creditsShards: requestCreditsShards(limit),});const crawlerOptions = {diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 63aefb0de..6b08c777e 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -80,6 +80,7 @@ import {reconstructKnownState,searchStatusToPageStatus,} from "./search/persist";+import { requestCreditsShards } from "../../lib/request-credits-store";const logger = _logger.child({ module: "monitoring-runner" });export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };@@ -646,6 +647,9 @@ async function enqueueMonitorCrawlTarget(params: {target_hint: body.url,zeroDataRetention: false,api_key_id: null,+ creditsShards: requestCreditsShards(+ body.limit ?? MONITOR_CHECK_PAGE_SCAN_LIMIT,+ ),});const crawlerOptions = {