Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: add credits_billed everywhere (FIR-2286) (#1655)
Gergő Móricz3c03d0705121
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 582d8ee2f..6f6ef849f 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -392,6 +392,7 @@ export async function mapController(origin: req.body.origin ?? "api",integration: req.body.integration,num_tokens: 0,+ credits_billed: 1,});const response = {diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex c94480db6..136df1f2f 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -21,7 +21,6 @@ import * as Sentry from "@sentry/node";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";-import { getJobFromGCS } from "../../lib/gcs-jobs";import { CostTracking } from "../../lib/extract/extraction-service";// Used for deep research@@ -268,18 +267,20 @@ export async function searchController(}}+ const credits_billed = responseData.data.reduce((a, x) => {+ if (x.metadata?.numPages !== undefined && x.metadata.numPages > 0) {+ return a + x.metadata.numPages;+ } else {+ return a + 1;+ }+ }, 0)+// Bill team once for all successful resultsif (!isSearchPreview) {billTeam(req.auth.team_id,req.acuc?.sub_id,- responseData.data.reduce((a, x) => {- if (x.metadata?.numPages !== undefined && x.metadata.numPages > 0) {- return a + x.metadata.numPages;- } else {- return a + 1;- }- }, 0),+ credits_billed,).catch((error) => {logger.error(`Failed to bill team ${req.auth.team_id} for ${responseData.data.length} credits: ${error}`,@@ -309,6 +310,7 @@ export async function searchController(origin: req.body.origin,integration: req.body.integration,cost_tracking: costTracking,+ credits_billed,},false,isSearchPreview,diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.tsindex 4c788fe79..8d2c6b6db 100644--- a/apps/api/src/lib/deep-research/deep-research-service.ts+++ b/apps/api/src/lib/deep-research/deep-research-service.ts@@ -347,6 +347,8 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {const progress = state.getProgress();logger.debug("[Deep Research] Research completed successfully");+ const credits_billed = Math.min(urlsAnalyzed, options.maxUrls);+// Log job with token usage and sourcesawait logJob({job_id: researchId,@@ -363,6 +365,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {num_tokens: 0,tokens_billed: 0,cost_tracking: costTracking,+ credits_billed,});await updateDeepResearch(researchId, {status: "completed",@@ -370,7 +373,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {json: finalAnalysisJson,});// Bill team for usage based on URLs analyzed- billTeam(teamId, subId, Math.min(urlsAnalyzed, options.maxUrls), logger).catch(+ billTeam(teamId, subId, credits_billed, logger).catch((error) => {logger.error(`Failed to bill team ${teamId} for ${urlsAnalyzed} URLs analyzed`, { teamId, count: urlsAnalyzed, error },diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.tsindex 1c55f7389..d9f953d6a 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts@@ -242,6 +242,7 @@ export async function performGenerateLlmsTxt(tokens_billed: 0,sources: {},cost_tracking: costTracking,+ credits_billed: urls.length,});// Bill team for usagediff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 584c1ac71..5c46cfde1 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -2,7 +2,7 @@ import { supabase_service } from "../supabase";import { FirecrawlJob } from "../../types";import { posthog } from "../posthog";import "dotenv/config";-import { logger } from "../../lib/logger";+import { logger as _logger } from "../../lib/logger";import { configDotenv } from "dotenv";import { saveJobToGCS } from "../../lib/gcs-jobs";configDotenv();@@ -22,13 +22,26 @@ function cleanOfNull<T>(x: T): T {}export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLogging: boolean = false) {+ const logger = _logger.child({+ module: "log_job",+ method: "logJob",+ ...(job.mode === "scrape" || job.mode === "single_urls" || job.mode === "single_url" ? ({+ scrapeId: job.job_id,+ }) : {}),+ ...(job.mode === "crawl" || job.mode === "batch_scrape" ? ({+ crawlId: job.job_id,+ }) : {}),+ ...(job.mode === "extract" ? ({+ extractId: job.job_id,+ }) : {}),+ });+try {const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";if (!useDbAuthentication) {return;}-// Redact any pages that have an authorization header// actually, Don't. we use the db to retrieve results now. this breaks authed crawls - mogery// if (@@ -88,7 +101,7 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLoif (error) {logger.error("Failed to log job due to Supabase error -- trying again",- { error, scrapeId: job.job_id },+ { error },);await new Promise<void>((resolve) =>setTimeout(() => resolve(), 75),@@ -100,27 +113,26 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLo} catch (error) {logger.error("Failed to log job due to thrown error -- trying again",- { error, scrapeId: job.job_id },+ { error },);await new Promise<void>((resolve) => setTimeout(() => resolve(), 75));}}if (done) {- logger.debug("Job logged successfully!", { scrapeId: job.job_id });+ logger.debug("Job logged successfully!");} else {- logger.error("Failed to log job!", { scrapeId: job.job_id });+ logger.error("Failed to log job!");}} else {const { error } = await supabase_service.from("firecrawl_jobs").insert([jobColumn]);if (error) {- logger.error(`Error logging job: ${error.message}`, {+ logger.error(`Error logging job`, {error,- scrapeId: job.job_id,});} else {- logger.debug("Job logged successfully!", { scrapeId: job.job_id });+ logger.debug("Job logged successfully!");}}@@ -155,6 +167,8 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLo}}} catch (error) {- logger.error(`Error logging job: ${error.message}`);+ logger.error(`Error logging job`, {+ error,+ });}}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 793fa85c8..7ff5283fc 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -310,7 +310,6 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {).filter((x) => x !== null);- logger.info("Logging crawl NOW!");await logJob({job_id: job.data.crawl_id,success: jobStatus === "completed",@@ -326,7 +325,6 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {origin: job.data.origin,integration: job.data.integration,}, false, job.data.internalOptions?.bypassBilling ?? false);- logger.info("Logged crawl!");const data = {success: jobStatus !== "failed",@@ -359,6 +357,23 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);const jobStatus = sc.cancelled ? "failed" : "completed";+ let credits_billed = null;++ if (process.env.USE_DB_AUTHENTICATION === "true") {+ const creditsRpc = await supabase_service+ .rpc("credits_billed_by_crawl_id_1", {+ i_crawl_id: job.data.crawl_id,+ });++ credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;++ if (credits_billed === null) {+ logger.warn("Credits billed is null", {+ error: creditsRpc.error,+ });+ }+ }+await logJob({job_id: job.data.crawl_id,@@ -376,6 +391,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {crawlerOptions: sc.crawlerOptions,origin: job.data.origin,integration: job.data.integration,+ credits_billed,},true,job.data.internalOptions?.bypassBilling ?? false,