Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrape): log FIRE-1 credits billed on failures properly (FIR-2331) (#1682)
Gergő Móriczfbd81b416877
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 5444c3cfa..29d78a420 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -7,13 +7,10 @@ import {scrapeRequestSchema,ScrapeResponse,} from "./types";-import { billTeam } from "../../services/billing/credit_billing";import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";import { getScrapeQueue } from "../../services/queue-service";-import { supabaseGetJobById } from "../../lib/supabase-jobs";-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -86,32 +83,6 @@ export async function scrapeController(logger.error(`Error in scrapeController`, {startTime,});-- let creditsToBeBilled = 0;-- if (req.body.agent?.model?.toLowerCase() === "fire-1" || req.body.extract?.agent?.model?.toLowerCase() === "fire-1" || req.body.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") {- if (process.env.USE_DB_AUTHENTICATION === "true") {- // @Nick this is a hack pushed at 2AM pls help - mogery- const job = await supabaseGetJobById(jobId);- if (!job?.cost_tracking) {- logger.warn("No cost tracking found for job");- }- creditsToBeBilled = Math.ceil((job?.cost_tracking?.totalCost ?? 1) * 1800);- } else {- creditsToBeBilled = 150;- }- }-- if (creditsToBeBilled > 0) {- billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(- (error) => {- logger.error(- `Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`,- );- // Optionally, you could notify an admin or add to a retry queue here- },- );- }if (e instanceof Error &&diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex aabe42b00..45209aaf3 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -1,40 +1,28 @@import { Document, ScrapeOptions } from "../controllers/v1/types";-import { supabaseGetJobById } from "./supabase-jobs";-import { logger } from "./logger";import { CostTracking } from "./extract/extraction-service";const creditsPerPDFPage = 1;const stealthProxyCostBonus = 4;-export async function calculateCreditsToBeBilled(options: ScrapeOptions, document: Document, jobId: string, costTracking?: any) {+export async function calculateCreditsToBeBilled(options: ScrapeOptions, document: Document | null, costTracking: CostTracking) {+ if (document === null) {+ // Failure -- check cost tracking if FIRE-1+ let creditsToBeBilled = 0;++ if (options.agent?.model?.toLowerCase() === "fire-1" || options.extract?.agent?.model?.toLowerCase() === "fire-1" || options.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") {+ creditsToBeBilled = Math.ceil((costTracking.toJSON().totalCost ?? 1) * 1800);+ }++ return creditsToBeBilled;+ }+let creditsToBeBilled = 1; // Assuming 1 credit per documentif ((options.extract && options.formats?.includes("extract")) || (options.formats?.includes("changeTracking") && options.changeTrackingOptions?.modes?.includes("json"))) {creditsToBeBilled = 5;}if (options.agent?.model?.toLowerCase() === "fire-1" || options.extract?.agent?.model?.toLowerCase() === "fire-1" || options.jsonOptions?.agent?.model?.toLowerCase() === "fire-1") {- if (process.env.USE_DB_AUTHENTICATION === "true") {- // @Nick this is a hack pushed at 2AM pls help - mogery- if (!costTracking) {- const job = await supabaseGetJobById(jobId);- costTracking = job?.cost_tracking;- }-- if (!costTracking) {- logger.warn("No cost tracking found for job", {- jobId,- scrapeId: jobId- });- }-- if (costTracking instanceof CostTracking) {- costTracking = costTracking.toJSON();- }-- creditsToBeBilled = Math.ceil((costTracking?.totalCost ?? 1) * 1800);- } else {- creditsToBeBilled = 150;- }+ creditsToBeBilled = Math.ceil((costTracking.toJSON().totalCost ?? 1) * 1800);}if (document.metadata.numPages !== undefined && document.metadata.numPages > 1) {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex deff1f0c8..87316fb82 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1152,11 +1152,11 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {}}-async function billScrapeJob(job: Job & { id: string }, document: Document, logger: Logger, costTracking?: CostTracking) {+async function billScrapeJob(job: Job & { id: string }, document: Document | null, logger: Logger, costTracking: CostTracking) {let creditsToBeBilled: number | null = null;if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {- creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, document, job.id, costTracking);+ creditsToBeBilled = await calculateCreditsToBeBilled(job.data.scrapeOptions, document, costTracking);if (job.data.team_id !== process.env.BACKGROUND_INDEX_TEAM_ID! &&@@ -1610,6 +1610,8 @@ async function processJob(job: Job & { id: string }, token: string) {const end = Date.now();const timeTakenInSeconds = (end - start) / 1000;+ const credits_billed = await billScrapeJob(job, null, logger, costTracking);+logger.debug("Logging job to DB...");await logJob({@@ -1632,6 +1634,7 @@ async function processJob(job: Job & { id: string }, token: string) {integration: job.data.integration,crawl_id: job.data.crawl_id,cost_tracking: costTracking,+ credits_billed,},true,job.data.internalOptions?.bypassBilling ?? false,