Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrape): log FIRE-1 credits billed on failures properly (FIR-2331) (#1682)

Gergő Móriczfbd81b416877
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 5444c3cfa..29d78a420 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -7,13 +7,10 @@ import {
scrapeRequestSchema,
ScrapeResponse,
} from "./types";
-import { billTeam } from "../../services/billing/credit_billing";
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
import { getScrapeQueue } from "../../services/queue-service";
-import { supabaseGetJobById } from "../../lib/supabase-jobs";
-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -86,32 +83,6 @@ export async function scrapeController(
logger.error(`Error in scrapeController`, {
startTime,
});
-
- let creditsToBeBilled = 0;
-
- if (req.body.agent?.model?.toLowerCase() === "fire-1" || req.body.extract?.agent?.model?.toLowerCase() === "fire-1" || req.body.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") {
- if (process.env.USE_DB_AUTHENTICATION === "true") {
- // @Nick this is a hack pushed at 2AM pls help - mogery
- const job = await supabaseGetJobById(jobId);
- if (!job?.cost_tracking) {
- logger.warn("No cost tracking found for job");
- }
- creditsToBeBilled = Math.ceil((job?.cost_tracking?.totalCost ?? 1) * 1800);
- } else {
- creditsToBeBilled = 150;
- }
- }
-
- if (creditsToBeBilled > 0) {
- billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(
- (error) => {
- logger.error(
- `Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`,
- );
- // Optionally, you could notify an admin or add to a retry queue here
- },
- );
- }
if (
e instanceof Error &&
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index aabe42b00..45209aaf3 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -1,40 +1,28 @@
import { Document, ScrapeOptions } from "../controllers/v1/types";
-import { supabaseGetJobById } from "./supabase-jobs";
-import { logger } from "./logger";
import { CostTracking } from "./extract/extraction-service";
const creditsPerPDFPage = 1;
const stealthProxyCostBonus = 4;
-export async function calculateCreditsToBeBilled(options: ScrapeOptions, document: Document, jobId: string, costTracking?: any) {
+export async function calculateCreditsToBeBilled(options: ScrapeOptions, document: Document | null, costTracking: CostTracking) {
+ if (document === null) {
+ // Failure -- check cost tracking if FIRE-1
+ let creditsToBeBilled = 0;
+
+ if (options.agent?.model?.toLowerCase() === "fire-1" || options.extract?.agent?.model?.toLowerCase() === "fire-1" || options.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") {
+ creditsToBeBilled = Math.ceil((costTracking.toJSON().totalCost ?? 1) * 1800);
+ }
+
+ return creditsToBeBilled;
+ }
+
let creditsToBeBilled = 1; // Assuming 1 credit per document
if ((options.extract && options.formats?.includes("extract")) || (options.formats?.includes("changeTracking") && options.changeTrackingOptions?.modes?.includes("json"))) {
creditsToBeBilled = 5;
}
if (options.agent?.model?.toLowerCase() === "fire-1" || options.extract?.agent?.model?.toLowerCase() === "fire-1" || options.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") {
- if (process.env.USE_DB_AUTHENTICATION === "true") {
- // @Nick this is a hack pushed at 2AM pls help - mogery
- if (!costTracking) {
- const job = await supabaseGetJobById(jobId);
- costTracking = job?.cost_tracking;
- }
-
- if (!costTracking) {
- logger.warn("No cost tracking found for job", {
- jobId,
- scrapeId: jobId
- });
- }
-
- if (costTracking instanceof CostTracking) {
- costTracking = costTracking.toJSON();
- }
-
- creditsToBeBilled = Math.ceil((costTracking?.totalCost ?? 1) * 1800);
- } else {
- creditsToBeBilled = 150;
- }
+ creditsToBeBilled = Math.ceil((costTracking.toJSON().totalCost ?? 1) * 1800);
}
if (document.metadata.numPages !== undefined && document.metadata.numPages > 1) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index deff1f0c8..87316fb82 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -1152,11 +1152,11 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {
}
}
-async function billScrapeJob(job: Job & { id: string }, document: Document, logger: Logger, costTracking?: CostTracking) {
+async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking) {
let creditsToBeBilled: number | null = null;
if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {
- creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, document, job.id, costTracking);
+ creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, document, costTracking);
if (
job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&
@@ -1610,6 +1610,8 @@ async function processJob(job: Job & { id: string }, token: string) {
const end = Date.now();
const timeTakenInSeconds = (end - start) / 1000;
+ const credits_billed = await billScrapeJob(job, null, logger, costTracking);
+
logger.debug("Logging job to DB...");
await logJob(
{
@@ -1632,6 +1634,7 @@ async function processJob(job: Job & { id: string }, token: string) {
integration: job.data.integration,
crawl_id: job.data.crawl_id,
cost_tracking: costTracking,
+ credits_billed,
},
true,
job.data.internalOptions?.bypassBilling ?? false,