Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): bill search balance/usage against SEARCH_CREDITS (#3735)
firecrawl-spring[bot]869c8de745f8
diff --git a/apps/api/src/controllers/v2/search-feedback.ts b/apps/api/src/controllers/v2/search-feedback.tsindex 29325fe6d..48e8b752d 100644--- a/apps/api/src/controllers/v2/search-feedback.ts+++ b/apps/api/src/controllers/v2/search-feedback.ts@@ -3,7 +3,10 @@ import { v7 as uuidv7 } from "uuid";import { z } from "zod";import { config } from "../../config";import { logger as _logger } from "../../lib/logger";-import { autumnService } from "../../services/autumn/autumn.service";+import {+ autumnService,+ SEARCH_CREDITS_FEATURE_ID,+} from "../../services/autumn/autumn.service";import { and, eq, gte } from "drizzle-orm";import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";@@ -332,6 +335,7 @@ export async function searchFeedbackController(feedbackId,rating: parsedBody.rating,},+ featureId: SEARCH_CREDITS_FEATURE_ID,});creditsRefunded = cappedRefund;} catch (error) {diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex e4215cb82..f63be759e 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -24,10 +24,14 @@ import { validate as isUuid } from "uuid";import { config } from "../config";import { getAgentFreeRequestsLeft } from "../db/rpc";-import { autumnService } from "../services/autumn/autumn.service";+import {+ autumnService,+ CREDITS_FEATURE_ID,+} from "../services/autumn/autumn.service";export function checkCreditsMiddleware(_minimum?: number,+ featureId: string = CREDITS_FEATURE_ID,): (req: RequestWithAuth, res: Response, next: NextFunction) => void {return (req, res, next) => {let minimum = _minimum;@@ -122,6 +126,7 @@ export function checkCreditsMiddleware(source: "checkCreditsMiddleware",path: req.path,},+ featureId,});// Autumn is the source of truth for credits. If it's unavailablediff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex c698eb9b0..5e3b0d930 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -7,6 +7,7 @@ import { crawlStatusController } from "../controllers/v1/crawl-status";import { mapController } from "../controllers/v1/map";import { RequestWithAuth } from "../controllers/v1/types";import { RateLimiterMode } from "../types";+import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";import expressWs from "express-ws";import { crawlStatusWSController } from "../controllers/v1/crawl-status-ws";import { crawlCancelController } from "../controllers/v1/crawl-cancel";@@ -154,7 +155,7 @@ v1Router.post("/search",authMiddleware(RateLimiterMode.Search),countryCheck,- checkCreditsMiddleware(),+ checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),wrap(searchController),);diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 77e8f57e3..71e98d89b 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -2,6 +2,7 @@ import express from "express";import multer from "multer";import { config } from "../config";import { RateLimiterMode } from "../types";+import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";import expressWs from "express-ws";import { searchController } from "../controllers/v2/search";import { searchFeedbackController } from "../controllers/v2/search-feedback";@@ -237,7 +238,7 @@ v2Router.post("/search",authMiddleware(RateLimiterMode.Search),countryCheck,- checkCreditsMiddleware(),+ checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),blocklistMiddleware,wrap(searchController),);diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex 87d52316c..70f66aeec 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -78,6 +78,7 @@ import {AutumnService,BoundedMap,BoundedSet,+ featureIdForBillingEndpoint,isAutumnRequestTrackEnabled,orgBucket,} from "../autumn.service";@@ -390,6 +391,18 @@ describe("checkCredits", () => {const result = await svc.checkCredits({ teamId: "team-1", value: 10 });expect(result).toEqual({ allowed: false, remaining: 0 });});++ it("checks against SEARCH_CREDITS when featureId is provided", async () => {+ const svc = makeService();+ await svc.checkCredits({+ teamId: "team-1",+ value: 5,+ featureId: "SEARCH_CREDITS",+ });+ expect(mockCheck).toHaveBeenCalledWith(+ expect.objectContaining({ featureId: "SEARCH_CREDITS" }),+ );+ });});// ---------------------------------------------------------------------------@@ -440,6 +453,22 @@ describe("trackCredits", () => {expect(await svc.trackCredits({ teamId: "team-1", value: 42 })).toBe(false);});++ it("tracks against SEARCH_CREDITS when featureId is provided", async () => {+ const svc = makeService();++ const result = await svc.trackCredits({+ teamId: "team-1",+ value: 7,+ properties: { source: "test", endpoint: "search" },+ featureId: "SEARCH_CREDITS",+ });++ expect(result).toBe(true);+ const usageCall = mockTrack.mock.calls.find((c: any[]) => c[0].value === 7);+ expect(usageCall).toBeDefined();+ expect((usageCall as any[])[0].featureId).toBe("SEARCH_CREDITS");+ });});// ---------------------------------------------------------------------------@@ -490,6 +519,23 @@ describe("refundCredits", () => {expect(refundCall).toBeDefined();expect((refundCall as any[])[0].properties?.source).toBe("autumn_refund");expect((refundCall as any[])[0].properties?.endpoint).toBe("extract");+ expect((refundCall as any[])[0].featureId).toBe("CREDITS");+ });++ it("refunds against SEARCH_CREDITS when featureId is provided", async () => {+ const svc = makeService();+ await svc.refundCredits({+ teamId: "team-1",+ value: 1,+ properties: { endpoint: "search" },+ featureId: "SEARCH_CREDITS",+ });++ const refundCall = mockTrack.mock.calls.find(+ (c: any[]) => c[0].value === -1,+ );+ expect(refundCall).toBeDefined();+ expect((refundCall as any[])[0].featureId).toBe("SEARCH_CREDITS");});it("is a no-op when autumnClient is null", async () => {@@ -506,6 +552,26 @@ describe("refundCredits", () => {});});+// ---------------------------------------------------------------------------+// featureIdForBillingEndpoint+// ---------------------------------------------------------------------------++describe("featureIdForBillingEndpoint", () => {+ it("maps the search endpoint to SEARCH_CREDITS", () => {+ expect(featureIdForBillingEndpoint("search")).toBe("SEARCH_CREDITS");+ });++ it("maps non-search endpoints to CREDITS", () => {+ for (const endpoint of ["scrape", "crawl", "extract", "agent", "map"]) {+ expect(featureIdForBillingEndpoint(endpoint)).toBe("CREDITS");+ }+ });++ it("maps an undefined endpoint to CREDITS", () => {+ expect(featureIdForBillingEndpoint(undefined)).toBe("CREDITS");+ });+});+// ---------------------------------------------------------------------------// experiment gating// ---------------------------------------------------------------------------diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex 1aa9ce998..15d49728a 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -19,7 +19,20 @@ import type {} from "./types";const TEAM_FEATURE_ID = "TEAM";-const CREDITS_FEATURE_ID = "CREDITS";+export const CREDITS_FEATURE_ID = "CREDITS";+export const SEARCH_CREDITS_FEATURE_ID = "SEARCH_CREDITS";++/**+ * Maps a billing endpoint to the Autumn feature ID it should bill against.+ *+ * Search balance and usage are tracked against a dedicated SEARCH_CREDITS+ * feature; everything else uses the general CREDITS feature. Scrapes performed+ * as part of a search bill themselves under their own (non-search) endpoint, so+ * they correctly remain on CREDITS.+ */+export function featureIdForBillingEndpoint(endpoint?: string): string {+ return endpoint === "search" ? SEARCH_CREDITS_FEATURE_ID : CREDITS_FEATURE_ID;+}/*** Deterministic bucket for an org UUID.@@ -341,6 +354,7 @@ export class AutumnService {teamId,value,properties,+ featureId = CREDITS_FEATURE_ID,}: TrackCreditsParams): Promise<{allowed: boolean;remaining: number;@@ -353,7 +367,7 @@ export class AutumnService {const { allowed, balance } = await autumnClient.check({customerId,entityId: teamId,- featureId: CREDITS_FEATURE_ID,+ featureId,requiredBalance: value,properties,});@@ -363,7 +377,7 @@ export class AutumnService {logger.debug("Autumn checkCredits completed", {customerId,entityId: teamId,- featureId: CREDITS_FEATURE_ID,+ featureId,value,allowed,remaining,@@ -392,6 +406,7 @@ export class AutumnService {lockId,expiresAt,properties,+ featureId = CREDITS_FEATURE_ID,}: LockCreditsParams): Promise<string | null> {if (!autumnClient || this.isPreviewTeam(teamId)) {return null;@@ -403,7 +418,7 @@ export class AutumnService {const { allowed } = await autumnClient.check({customerId,entityId: teamId,- featureId: CREDITS_FEATURE_ID,+ featureId,requiredBalance: value,properties,lock: {@@ -425,7 +440,7 @@ export class AutumnService {logger.info("Autumn lockCredits succeeded", {customerId,entityId: teamId,- featureId: CREDITS_FEATURE_ID,+ featureId,value,lockId: resolvedLockId,properties,@@ -493,6 +508,7 @@ export class AutumnService {value,properties,requestScoped = false,+ featureId = CREDITS_FEATURE_ID,}: TrackCreditsParams): Promise<boolean> {if (requestScoped && !isAutumnRequestTrackEnabled()) return false;if (!autumnClient) return false;@@ -508,7 +524,7 @@ export class AutumnService {return await this.track({customerId,entityId: teamId,- featureId: CREDITS_FEATURE_ID,+ featureId,value,properties,});@@ -533,6 +549,7 @@ export class AutumnService {teamId,value,properties,+ featureId = CREDITS_FEATURE_ID,}: TrackCreditsParams): Promise<void> {if (!autumnClient) return;if (this.isPreviewTeam(teamId)) return;@@ -542,7 +559,7 @@ export class AutumnService {await this.track({customerId,entityId: teamId,- featureId: CREDITS_FEATURE_ID,+ featureId,value: -value,properties: { ...properties, source: "autumn_refund" },});diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.tsindex d863aa5b8..1d75874e7 100644--- a/apps/api/src/services/autumn/types.ts+++ b/apps/api/src/services/autumn/types.ts@@ -43,6 +43,7 @@ export type LockCreditsParams = {lockId?: string;expiresAt?: number;properties?: Record<string, unknown>;+ featureId?: string;};export type FinalizeCreditsLockParams = {@@ -57,6 +58,7 @@ export type TrackCreditsParams = {value: number;properties?: Record<string, unknown>;requestScoped?: boolean;+ featureId?: string;};export type CreateEntityResult =diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex 738ce56f6..920062dda 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -5,7 +5,10 @@ import { billTeam6 } from "../../db/rpc";import * as Sentry from "@sentry/node";import { withAuth } from "../../lib/withAuth";import { setCachedACUC, setCachedACUCTeam } from "../../controllers/auth";-import { autumnService } from "../autumn/autumn.service";+import {+ autumnService,+ featureIdForBillingEndpoint,+} from "../autumn/autumn.service";import {resolveBillingMetadata,toAutumnBillingProperties,@@ -80,6 +83,7 @@ async function refundRequestTrackedCredits(group: GroupedBillingOperation) {apiKeyId: group.api_key_id,subscriptionId: group.subscription_id,},+ featureId: featureIdForBillingEndpoint(group.billing.endpoint),});} catch (error) {logger.warn("Failed to refund Autumn request-tracked credits", {@@ -221,6 +225,7 @@ export async function processBillingBatch() {apiKeyId: group.api_key_id,subscriptionId: group.subscription_id,},+ featureId: featureIdForBillingEndpoint(group.billing.endpoint),});}} catch (error) {diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex cd6a04fe6..56391f67a 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -2,7 +2,10 @@ import { withAuth } from "../../lib/withAuth";import { logger } from "../../lib/logger";import { AuthCreditUsageChunk } from "../../controllers/v1/types";import { queueBillingOperation } from "./batch_billing";-import { autumnService } from "../autumn/autumn.service";+import {+ autumnService,+ featureIdForBillingEndpoint,+} from "../autumn/autumn.service";import { toAutumnBillingProperties, type BillingMetadata } from "./types";import type { Logger } from "winston";@@ -31,11 +34,13 @@ export async function billTeam(...toAutumnBillingProperties(billing),apiKeyId: api_key_id,};+ const featureId = featureIdForBillingEndpoint(billing.endpoint);const trackedInRequest = await autumnService.trackCredits({teamId: team_id,value: credits,properties: autumnProperties,requestScoped: true,+ featureId,});const result = await queueBillingOperation(@@ -53,6 +58,7 @@ export async function billTeam(teamId: team_id,value: credits,properties: autumnProperties,+ featureId,});}