Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor(v0): credit-check via Autumn, drop dead ACUC credit path (#3871)
firecrawl-spring[bot]9c3fba41b871
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex c75a3ac1c..ccfdb3c19 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -1,5 +1,5 @@import { Request, Response } from "express";-import { checkTeamCredits } from "../../../src/services/billing/credit_billing";+import { autumnService } from "../../services/autumn/autumn.service";import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";import { addScrapeJob } from "../../../src/services/queue-jobs";@@ -128,13 +128,14 @@ export async function crawlController(req: Request, res: Response) {}const limitCheck = req.body?.crawlerOptions?.limit ?? 1;- const {- success: creditsCheckSuccess,- message: creditsCheckMessage,- remainingCredits,- } = await checkTeamCredits(chunk, team_id, limitCheck);+ // Autumn is the source of truth for credits.+ const autumnResult = await autumnService.checkCredits({+ teamId: team_id,+ value: limitCheck,+ properties: { source: "v0/crawl" },+ });- if (!creditsCheckSuccess) {+ if (autumnResult !== null && !autumnResult.allowed) {return res.status(402).json({error: isSelfHosted()? "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. Please check your server configuration."@@ -142,7 +143,13 @@ export async function crawlController(req: Request, res: Response) {});}- // TODO: need to do this to v1+ // When Autumn allows the request (incl. overage) we don't clamp the crawl+ // limit, matching the v1/v2 middleware (which uses Infinity for remaining on+ // allow). null = Autumn unavailable / self-hosted -> also no clamp.+ const remainingCredits =+ autumnResult === null || autumnResult.allowed+ ? Infinity+ : autumnResult.remaining;crawlerOptions.limit = Math.min(remainingCredits, crawlerOptions.limit);let url = urlSchema.parse(req.body.url);diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex cb0b88a09..eee86b19e 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -1,6 +1,6 @@import { ExtractorOptions, PageOptions } from "./../../lib/entities";import { Request, Response } from "express";-import { checkTeamCredits } from "../../services/billing/credit_billing";+import { autumnService } from "../../services/autumn/autumn.service";import { authenticateUser } from "../auth";import { RateLimiterMode, AuthResponse } from "../../types";import { TeamFlags, toLegacyDocument, url as urlSchema } from "../v1/types";@@ -254,11 +254,15 @@ export async function scrapeController(req: Request, res: Response) {timeout = req.body.timeout ?? 90000;}- // checkCredits+ // checkCredits — Autumn is the source of truth for credits.try {- const { success: creditsCheckSuccess, message: creditsCheckMessage } =- await checkTeamCredits(chunk, team_id, 1);- if (!creditsCheckSuccess) {+ const autumnResult = await autumnService.checkCredits({+ teamId: team_id,+ value: 1,+ properties: { source: "v0/scrape" },+ });+ // null = Autumn unavailable / self-hosted -> fail open, matching v1/v2.+ if (autumnResult !== null && !autumnResult.allowed) {earlyReturn = true;return res.status(402).json({error:diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 105dafe77..5a4def2af 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -1,8 +1,6 @@import { Request, Response } from "express";-import {- billTeam,- checkTeamCredits,-} from "../../services/billing/credit_billing";+import { billTeam } from "../../services/billing/credit_billing";+import { autumnService } from "../../services/autumn/autumn.service";import { authenticateUser } from "../auth";import { RateLimiterMode, ScrapeJobSingleUrls } from "../../types";import { logSearch, logRequest } from "../../services/logging/log_job";@@ -237,9 +235,13 @@ export async function searchController(req: Request, res: Response) {const searchOptions = req.body.searchOptions ?? { limit: 5 };try {- const { success: creditsCheckSuccess, message: creditsCheckMessage } =- await checkTeamCredits(chunk, team_id, 1);- if (!creditsCheckSuccess) {+ const autumnResult = await autumnService.checkCredits({+ teamId: team_id,+ value: 1,+ properties: { source: "v0/search" },+ });+ // null = Autumn unavailable / self-hosted -> fail open, matching v1/v2.+ if (autumnResult !== null && !autumnResult.allowed) {return res.status(402).json({ error: "Insufficient credits" });}} catch (error) {diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex 8c5414406..ec97a48bf 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -1,6 +1,4 @@import { withAuth } from "../../lib/withAuth";-import { logger } from "../../lib/logger";-import { AuthCreditUsageChunk } from "../../controllers/v1/types";import { queueBillingOperation } from "./batch_billing";import {autumnService,@@ -66,183 +64,3 @@ export async function billTeam({ success: true, message: "No DB, bypassed." },)(team_id, subscription_id, credits, api_key_id, billing, logger);}--type CheckTeamCreditsResponse = {- success: boolean;- message: string;- remainingCredits: number;- chunk?: AuthCreditUsageChunk;-};--export async function checkTeamCredits(- chunk: AuthCreditUsageChunk | null,- team_id: string,- credits: number,-): Promise<CheckTeamCreditsResponse> {- return withAuth(supaCheckTeamCredits, {- success: true,- message: "No DB, bypassed",- remainingCredits: Infinity,- })(chunk, team_id, credits);-}--function evaluateTeamCredits(- chunk: AuthCreditUsageChunk,- credits: number,- isAutoRechargeEnabled: boolean,-) {- const allowOverages = chunk.price_should_be_graceful && isAutoRechargeEnabled;- const remainingCredits = allowOverages- ? chunk.remaining_credits + chunk.price_credits- : chunk.remaining_credits;- const creditsWillBeUsed = chunk.adjusted_credits_used + credits;- const totalPriceCredits = allowOverages- ? (chunk.total_credits_sum ?? 100000000) + chunk.price_credits- : (chunk.total_credits_sum ?? 100000000);- const creditUsagePercentage =- chunk.adjusted_credits_used / (chunk.total_credits_sum ?? 100000000);-- return {- allowOverages,- remainingCredits,- creditsWillBeUsed,- totalPriceCredits,- creditUsagePercentage,- success: creditsWillBeUsed <= totalPriceCredits,- };-}--// if team has enough credits for the operation, return true, else return false-async function supaCheckTeamCredits(- chunk: AuthCreditUsageChunk | null,- team_id: string,- credits: number,-): Promise<CheckTeamCreditsResponse> {- // WARNING: chunk will be null if team_id is preview -- do not perform operations on it under ANY circumstances - mogery- if (team_id === "preview" || team_id.startsWith("preview_")) {- return {- success: true,- message: "Preview team, no credits used",- remainingCredits: Infinity,- };- } else if (chunk === null) {- throw new Error("NULL ACUC passed to supaCheckTeamCredits");- }-- // If bypassCreditChecks flag is set, return success with infinite credits (infinitely graceful)- if (chunk.flags?.bypassCreditChecks) {- return {- success: true,- message: "Credit checks bypassed",- remainingCredits: Infinity,- chunk,- };- }-- // Auto-recharge is now handled entirely by Autumn. The legacy ACUC-driven- // auto-recharge logic below is disabled to avoid double-charging or firing- // at the wrong threshold.- //- // let isAutoRechargeEnabled = false,- // autoRechargeThreshold = 1000;- // const cacheKey = `team_auto_recharge_${team_id}`;- // let cachedData = await getValue(cacheKey);- // if (cachedData) {- // const parsedData = JSON.parse(cachedData);- // isAutoRechargeEnabled = parsedData.auto_recharge;- // autoRechargeThreshold = parsedData.auto_recharge_threshold;- // } else {- // const { data, error } = await supabase_rr_service- // .from("teams")- // .select("auto_recharge, auto_recharge_threshold")- // .eq("id", team_id)- // .single();- //- // if (data) {- // isAutoRechargeEnabled = data.auto_recharge;- // autoRechargeThreshold = data.auto_recharge_threshold;- // await setValue(cacheKey, JSON.stringify(data), 300);- // }- // }-- const {- success,- remainingCredits,- creditsWillBeUsed,- totalPriceCredits,- creditUsagePercentage,- } = evaluateTeamCredits(chunk, credits, false);-- // if (- // config.AUTO_RECHARGE_ENABLED &&- // isAutoRechargeEnabled &&- // chunk.remaining_credits < autoRechargeThreshold &&- // !chunk.is_extract- // ) {- // logger.info("Auto-recharge triggered", {- // team_id,- // teamId: team_id,- // autoRechargeThreshold,- // remainingCredits: chunk.remaining_credits,- // });- //- // const autoChargeResult = await autoCharge(chunk, autoRechargeThreshold);- //- // if (autoChargeResult && autoChargeResult.success) {- // return {- // success: true,- // message: autoChargeResult.message,- // remainingCredits: allowOverages- // ? autoChargeResult.remainingCredits + chunk.price_credits- // : autoChargeResult.remainingCredits,- // chunk: autoChargeResult.chunk,- // };- // } else if (allowOverages) {- // return {- // success: true,- // message: "Auto-recharge failed, but price should be graceful",- // remainingCredits,- // chunk,- // };- // }- // }-- // Compare the adjusted total credits used with the credits allowed by the plan (and graceful)- if (!success) {- logger.warn("Credit check failed - insufficient credits", {- team_id,- teamId: team_id,- creditsRequested: credits,- is_extract: chunk.is_extract,- bypassCreditChecks: chunk.flags?.bypassCreditChecks,- price_should_be_graceful: chunk.price_should_be_graceful,- price_credits: chunk.price_credits,- coupon_credits: chunk.coupon_credits,- total_credits_sum: chunk.total_credits_sum,- credits_used: chunk.credits_used,- adjusted_credits_used: chunk.adjusted_credits_used,- remaining_credits: chunk.remaining_credits,- sub_current_period_start: chunk.sub_current_period_start,- sub_current_period_end: chunk.sub_current_period_end,- computed_remainingCredits: remainingCredits,- computed_creditsWillBeUsed: creditsWillBeUsed,- computed_totalPriceCredits: totalPriceCredits,- creditUsagePercentage,- sumComponents: chunk.price_credits + chunk.coupon_credits,- });- return {- success: false,- message:- "Insufficient credits to perform this request. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing.",- remainingCredits,- chunk,- };- }-- return {- success: true,- message: "Sufficient credits available",- remainingCredits,- chunk,- };-}