Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: add credits_billed everywhere (FIR-2286) (#1655)

Gergő Móricz3c03d0705121
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 582d8ee2f..6f6ef849f 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -392,6 +392,7 @@ export async function mapController(
origin: req.body.origin ?? "api",
integration: req.body.integration,
num_tokens: 0,
+ credits_billed: 1,
});
const response = {
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index c94480db6..136df1f2f 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -21,7 +21,6 @@ import * as Sentry from "@sentry/node";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
-import { getJobFromGCS } from "../../lib/gcs-jobs";
import { CostTracking } from "../../lib/extract/extraction-service";
// Used for deep research
@@ -268,18 +267,20 @@ export async function searchController(
}
}
+ const credits_billed = responseData.data.reduce((a, x) => {
+ if (x.metadata?.numPages !== undefined && x.metadata.numPages > 0) {
+ return a + x.metadata.numPages;
+ } else {
+ return a + 1;
+ }
+ }, 0)
+
// Bill team once for all successful results
if (!isSearchPreview) {
billTeam(
req.auth.team_id,
req.acuc?.sub_id,
- responseData.data.reduce((a, x) => {
- if (x.metadata?.numPages !== undefined && x.metadata.numPages > 0) {
- return a + x.metadata.numPages;
- } else {
- return a + 1;
- }
- }, 0),
+ credits_billed,
).catch((error) => {
logger.error(
`Failed to bill team ${req.auth.team_id} for ${responseData.data.length} credits: ${error}`,
@@ -309,6 +310,7 @@ export async function searchController(
origin: req.body.origin,
integration: req.body.integration,
cost_tracking: costTracking,
+ credits_billed,
},
false,
isSearchPreview,
diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.ts
index 4c788fe79..8d2c6b6db 100644
--- a/apps/api/src/lib/deep-research/deep-research-service.ts
+++ b/apps/api/src/lib/deep-research/deep-research-service.ts
@@ -347,6 +347,8 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
const progress = state.getProgress();
logger.debug("[Deep Research] Research completed successfully");
+ const credits_billed = Math.min(urlsAnalyzed, options.maxUrls);
+
// Log job with token usage and sources
await logJob({
job_id: researchId,
@@ -363,6 +365,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
num_tokens: 0,
tokens_billed: 0,
cost_tracking: costTracking,
+ credits_billed,
});
await updateDeepResearch(researchId, {
status: "completed",
@@ -370,7 +373,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
json: finalAnalysisJson,
});
// Bill team for usage based on URLs analyzed
- billTeam(teamId, subId, Math.min(urlsAnalyzed, options.maxUrls), logger).catch(
+ billTeam(teamId, subId, credits_billed, logger).catch(
(error) => {
logger.error(
`Failed to bill team ${teamId} for ${urlsAnalyzed} URLs analyzed`, { teamId, count: urlsAnalyzed, error },
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
index 1c55f7389..d9f953d6a 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
@@ -242,6 +242,7 @@ export async function performGenerateLlmsTxt(
tokens_billed: 0,
sources: {},
cost_tracking: costTracking,
+ credits_billed: urls.length,
});
// Bill team for usage
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 584c1ac71..5c46cfde1 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -2,7 +2,7 @@ import { supabase_service } from "../supabase";
import { FirecrawlJob } from "../../types";
import { posthog } from "../posthog";
import "dotenv/config";
-import { logger } from "../../lib/logger";
+import { logger as _logger } from "../../lib/logger";
import { configDotenv } from "dotenv";
import { saveJobToGCS } from "../../lib/gcs-jobs";
configDotenv();
@@ -22,13 +22,26 @@ function cleanOfNull<T>(x: T): T {
}
export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLogging: boolean = false) {
+ const logger = _logger.child({
+ module: "log_job",
+ method: "logJob",
+ ...(job.mode === "scrape" || job.mode === "single_urls" || job.mode === "single_url" ? ({
+ scrapeId: job.job_id,
+ }) : {}),
+ ...(job.mode === "crawl" || job.mode === "batch_scrape" ? ({
+ crawlId: job.job_id,
+ }) : {}),
+ ...(job.mode === "extract" ? ({
+ extractId: job.job_id,
+ }) : {}),
+ });
+
try {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
if (!useDbAuthentication) {
return;
}
-
// Redact any pages that have an authorization header
// actually, Don't. we use the db to retrieve results now. this breaks authed crawls - mogery
// if (
@@ -88,7 +101,7 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLo
if (error) {
logger.error(
"Failed to log job due to Supabase error -- trying again",
- { error, scrapeId: job.job_id },
+ { error },
);
await new Promise<void>((resolve) =>
setTimeout(() => resolve(), 75),
@@ -100,27 +113,26 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLo
} catch (error) {
logger.error(
"Failed to log job due to thrown error -- trying again",
- { error, scrapeId: job.job_id },
+ { error },
);
await new Promise<void>((resolve) => setTimeout(() => resolve(), 75));
}
}
if (done) {
- logger.debug("Job logged successfully!", { scrapeId: job.job_id });
+ logger.debug("Job logged successfully!");
} else {
- logger.error("Failed to log job!", { scrapeId: job.job_id });
+ logger.error("Failed to log job!");
}
} else {
const { error } = await supabase_service
.from("firecrawl_jobs")
.insert([jobColumn]);
if (error) {
- logger.error(`Error logging job: ${error.message}`, {
+ logger.error(`Error logging job`, {
error,
- scrapeId: job.job_id,
});
} else {
- logger.debug("Job logged successfully!", { scrapeId: job.job_id });
+ logger.debug("Job logged successfully!");
}
}
@@ -155,6 +167,8 @@ export async function logJob(job: FirecrawlJob, force: boolean = false, bypassLo
}
}
} catch (error) {
- logger.error(`Error logging job: ${error.message}`);
+ logger.error(`Error logging job`, {
+ error,
+ });
}
}
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 793fa85c8..7ff5283fc 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -310,7 +310,6 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
)
.filter((x) => x !== null);
- logger.info("Logging crawl NOW!");
await logJob({
job_id: job.data.crawl_id,
success: jobStatus === "completed",
@@ -326,7 +325,6 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
origin: job.data.origin,
integration: job.data.integration,
}, false, job.data.internalOptions?.bypassBilling ?? false);
- logger.info("Logged crawl!");
const data = {
success: jobStatus !== "failed",
@@ -359,6 +357,23 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);
const jobStatus = sc.cancelled ? "failed" : "completed";
+ let credits_billed = null;
+
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const creditsRpc = await supabase_service
+ .rpc("credits_billed_by_crawl_id_1", {
+ i_crawl_id: job.data.crawl_id,
+ });
+
+ credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;
+
+ if (credits_billed === null) {
+ logger.warn("Credits billed is null", {
+ error: creditsRpc.error,
+ });
+ }
+ }
+
await logJob(
{
job_id: job.data.crawl_id,
@@ -376,6 +391,7 @@ async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {
crawlerOptions: sc.crawlerOptions,
origin: job.data.origin,
integration: job.data.integration,
+ credits_billed,
},
true,
job.data.internalOptions?.bypassBilling ?? false,