Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): bill search balance/usage against SEARCH_CREDITS (#3735)

firecrawl-spring[bot]869c8de745f8
diff --git a/apps/api/src/controllers/v2/search-feedback.ts b/apps/api/src/controllers/v2/search-feedback.ts
index 29325fe6d..48e8b752d 100644
--- a/apps/api/src/controllers/v2/search-feedback.ts
+++ b/apps/api/src/controllers/v2/search-feedback.ts
@@ -3,7 +3,10 @@ import { v7 as uuidv7 } from "uuid";
import { z } from "zod";
import { config } from "../../config";
import { logger as _logger } from "../../lib/logger";
-import { autumnService } from "../../services/autumn/autumn.service";
+import {
+ autumnService,
+ SEARCH_CREDITS_FEATURE_ID,
+} from "../../services/autumn/autumn.service";
import { and, eq, gte } from "drizzle-orm";
import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
@@ -332,6 +335,7 @@ export async function searchFeedbackController(
feedbackId,
rating: parsedBody.rating,
},
+ featureId: SEARCH_CREDITS_FEATURE_ID,
});
creditsRefunded = cappedRefund;
} catch (error) {
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index e4215cb82..f63be759e 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -24,10 +24,14 @@ import { validate as isUuid } from "uuid";
import { config } from "../config";
import { getAgentFreeRequestsLeft } from "../db/rpc";
-import { autumnService } from "../services/autumn/autumn.service";
+import {
+ autumnService,
+ CREDITS_FEATURE_ID,
+} from "../services/autumn/autumn.service";
export function checkCreditsMiddleware(
_minimum?: number,
+ featureId: string = CREDITS_FEATURE_ID,
): (req: RequestWithAuth, res: Response, next: NextFunction) => void {
return (req, res, next) => {
let minimum = _minimum;
@@ -122,6 +126,7 @@ export function checkCreditsMiddleware(
source: "checkCreditsMiddleware",
path: req.path,
},
+ featureId,
});
// Autumn is the source of truth for credits. If it's unavailable
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index c698eb9b0..5e3b0d930 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -7,6 +7,7 @@ import { crawlStatusController } from "../controllers/v1/crawl-status";
import { mapController } from "../controllers/v1/map";
import { RequestWithAuth } from "../controllers/v1/types";
import { RateLimiterMode } from "../types";
+import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";
import expressWs from "express-ws";
import { crawlStatusWSController } from "../controllers/v1/crawl-status-ws";
import { crawlCancelController } from "../controllers/v1/crawl-cancel";
@@ -154,7 +155,7 @@ v1Router.post(
"/search",
authMiddleware(RateLimiterMode.Search),
countryCheck,
- checkCreditsMiddleware(),
+ checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),
wrap(searchController),
);
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 77e8f57e3..71e98d89b 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -2,6 +2,7 @@ import express from "express";
import multer from "multer";
import { config } from "../config";
import { RateLimiterMode } from "../types";
+import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";
import expressWs from "express-ws";
import { searchController } from "../controllers/v2/search";
import { searchFeedbackController } from "../controllers/v2/search-feedback";
@@ -237,7 +238,7 @@ v2Router.post(
"/search",
authMiddleware(RateLimiterMode.Search),
countryCheck,
- checkCreditsMiddleware(),
+ checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),
blocklistMiddleware,
wrap(searchController),
);
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index 87d52316c..70f66aeec 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -78,6 +78,7 @@ import {
AutumnService,
BoundedMap,
BoundedSet,
+ featureIdForBillingEndpoint,
isAutumnRequestTrackEnabled,
orgBucket,
} from "../autumn.service";
@@ -390,6 +391,18 @@ describe("checkCredits", () => {
const result = await svc.checkCredits({ teamId: "team-1", value: 10 });
expect(result).toEqual({ allowed: false, remaining: 0 });
});
+
+ it("checks against SEARCH_CREDITS when featureId is provided", async () => {
+ const svc = makeService();
+ await svc.checkCredits({
+ teamId: "team-1",
+ value: 5,
+ featureId: "SEARCH_CREDITS",
+ });
+ expect(mockCheck).toHaveBeenCalledWith(
+ expect.objectContaining({ featureId: "SEARCH_CREDITS" }),
+ );
+ });
});
// ---------------------------------------------------------------------------
@@ -440,6 +453,22 @@ describe("trackCredits", () => {
expect(await svc.trackCredits({ teamId: "team-1", value: 42 })).toBe(false);
});
+
+ it("tracks against SEARCH_CREDITS when featureId is provided", async () => {
+ const svc = makeService();
+
+ const result = await svc.trackCredits({
+ teamId: "team-1",
+ value: 7,
+ properties: { source: "test", endpoint: "search" },
+ featureId: "SEARCH_CREDITS",
+ });
+
+ expect(result).toBe(true);
+ const usageCall = mockTrack.mock.calls.find((c: any[]) => c[0].value === 7);
+ expect(usageCall).toBeDefined();
+ expect((usageCall as any[])[0].featureId).toBe("SEARCH_CREDITS");
+ });
});
// ---------------------------------------------------------------------------
@@ -490,6 +519,23 @@ describe("refundCredits", () => {
expect(refundCall).toBeDefined();
expect((refundCall as any[])[0].properties?.source).toBe("autumn_refund");
expect((refundCall as any[])[0].properties?.endpoint).toBe("extract");
+ expect((refundCall as any[])[0].featureId).toBe("CREDITS");
+ });
+
+ it("refunds against SEARCH_CREDITS when featureId is provided", async () => {
+ const svc = makeService();
+ await svc.refundCredits({
+ teamId: "team-1",
+ value: 1,
+ properties: { endpoint: "search" },
+ featureId: "SEARCH_CREDITS",
+ });
+
+ const refundCall = mockTrack.mock.calls.find(
+ (c: any[]) => c[0].value === -1,
+ );
+ expect(refundCall).toBeDefined();
+ expect((refundCall as any[])[0].featureId).toBe("SEARCH_CREDITS");
});
it("is a no-op when autumnClient is null", async () => {
@@ -506,6 +552,26 @@ describe("refundCredits", () => {
});
});
+// ---------------------------------------------------------------------------
+// featureIdForBillingEndpoint
+// ---------------------------------------------------------------------------
+
+describe("featureIdForBillingEndpoint", () => {
+ it("maps the search endpoint to SEARCH_CREDITS", () => {
+ expect(featureIdForBillingEndpoint("search")).toBe("SEARCH_CREDITS");
+ });
+
+ it("maps non-search endpoints to CREDITS", () => {
+ for (const endpoint of ["scrape", "crawl", "extract", "agent", "map"]) {
+ expect(featureIdForBillingEndpoint(endpoint)).toBe("CREDITS");
+ }
+ });
+
+ it("maps an undefined endpoint to CREDITS", () => {
+ expect(featureIdForBillingEndpoint(undefined)).toBe("CREDITS");
+ });
+});
+
// ---------------------------------------------------------------------------
// experiment gating
// ---------------------------------------------------------------------------
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index 1aa9ce998..15d49728a 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -19,7 +19,20 @@ import type {
} from "./types";
const TEAM_FEATURE_ID = "TEAM";
-const CREDITS_FEATURE_ID = "CREDITS";
+export const CREDITS_FEATURE_ID = "CREDITS";
+export const SEARCH_CREDITS_FEATURE_ID = "SEARCH_CREDITS";
+
+/**
+ * Maps a billing endpoint to the Autumn feature ID it should bill against.
+ *
+ * Search balance and usage are tracked against a dedicated SEARCH_CREDITS
+ * feature; everything else uses the general CREDITS feature. Scrapes performed
+ * as part of a search bill themselves under their own (non-search) endpoint, so
+ * they correctly remain on CREDITS.
+ */
+export function featureIdForBillingEndpoint(endpoint?: string): string {
+ return endpoint === "search" ? SEARCH_CREDITS_FEATURE_ID : CREDITS_FEATURE_ID;
+}
/**
* Deterministic bucket for an org UUID.
@@ -341,6 +354,7 @@ export class AutumnService {
teamId,
value,
properties,
+ featureId = CREDITS_FEATURE_ID,
}: TrackCreditsParams): Promise<{
allowed: boolean;
remaining: number;
@@ -353,7 +367,7 @@ export class AutumnService {
const { allowed, balance } = await autumnClient.check({
customerId,
entityId: teamId,
- featureId: CREDITS_FEATURE_ID,
+ featureId,
requiredBalance: value,
properties,
});
@@ -363,7 +377,7 @@ export class AutumnService {
logger.debug("Autumn checkCredits completed", {
customerId,
entityId: teamId,
- featureId: CREDITS_FEATURE_ID,
+ featureId,
value,
allowed,
remaining,
@@ -392,6 +406,7 @@ export class AutumnService {
lockId,
expiresAt,
properties,
+ featureId = CREDITS_FEATURE_ID,
}: LockCreditsParams): Promise<string | null> {
if (!autumnClient || this.isPreviewTeam(teamId)) {
return null;
@@ -403,7 +418,7 @@ export class AutumnService {
const { allowed } = await autumnClient.check({
customerId,
entityId: teamId,
- featureId: CREDITS_FEATURE_ID,
+ featureId,
requiredBalance: value,
properties,
lock: {
@@ -425,7 +440,7 @@ export class AutumnService {
logger.info("Autumn lockCredits succeeded", {
customerId,
entityId: teamId,
- featureId: CREDITS_FEATURE_ID,
+ featureId,
value,
lockId: resolvedLockId,
properties,
@@ -493,6 +508,7 @@ export class AutumnService {
value,
properties,
requestScoped = false,
+ featureId = CREDITS_FEATURE_ID,
}: TrackCreditsParams): Promise<boolean> {
if (requestScoped && !isAutumnRequestTrackEnabled()) return false;
if (!autumnClient) return false;
@@ -508,7 +524,7 @@ export class AutumnService {
return await this.track({
customerId,
entityId: teamId,
- featureId: CREDITS_FEATURE_ID,
+ featureId,
value,
properties,
});
@@ -533,6 +549,7 @@ export class AutumnService {
teamId,
value,
properties,
+ featureId = CREDITS_FEATURE_ID,
}: TrackCreditsParams): Promise<void> {
if (!autumnClient) return;
if (this.isPreviewTeam(teamId)) return;
@@ -542,7 +559,7 @@ export class AutumnService {
await this.track({
customerId,
entityId: teamId,
- featureId: CREDITS_FEATURE_ID,
+ featureId,
value: -value,
properties: { ...properties, source: "autumn_refund" },
});
diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.ts
index d863aa5b8..1d75874e7 100644
--- a/apps/api/src/services/autumn/types.ts
+++ b/apps/api/src/services/autumn/types.ts
@@ -43,6 +43,7 @@ export type LockCreditsParams = {
lockId?: string;
expiresAt?: number;
properties?: Record<string, unknown>;
+ featureId?: string;
};
export type FinalizeCreditsLockParams = {
@@ -57,6 +58,7 @@ export type TrackCreditsParams = {
value: number;
properties?: Record<string, unknown>;
requestScoped?: boolean;
+ featureId?: string;
};
export type CreateEntityResult =
diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.ts
index 738ce56f6..920062dda 100644
--- a/apps/api/src/services/billing/batch_billing.ts
+++ b/apps/api/src/services/billing/batch_billing.ts
@@ -5,7 +5,10 @@ import { billTeam6 } from "../../db/rpc";
import * as Sentry from "@sentry/node";
import { withAuth } from "../../lib/withAuth";
import { setCachedACUC, setCachedACUCTeam } from "../../controllers/auth";
-import { autumnService } from "../autumn/autumn.service";
+import {
+ autumnService,
+ featureIdForBillingEndpoint,
+} from "../autumn/autumn.service";
import {
resolveBillingMetadata,
toAutumnBillingProperties,
@@ -80,6 +83,7 @@ async function refundRequestTrackedCredits(group: GroupedBillingOperation) {
apiKeyId: group.api_key_id,
subscriptionId: group.subscription_id,
},
+ featureId: featureIdForBillingEndpoint(group.billing.endpoint),
});
} catch (error) {
logger.warn("Failed to refund Autumn request-tracked credits", {
@@ -221,6 +225,7 @@ export async function processBillingBatch() {
apiKeyId: group.api_key_id,
subscriptionId: group.subscription_id,
},
+ featureId: featureIdForBillingEndpoint(group.billing.endpoint),
});
}
} catch (error) {
diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts
index cd6a04fe6..56391f67a 100644
--- a/apps/api/src/services/billing/credit_billing.ts
+++ b/apps/api/src/services/billing/credit_billing.ts
@@ -2,7 +2,10 @@ import { withAuth } from "../../lib/withAuth";
import { logger } from "../../lib/logger";
import { AuthCreditUsageChunk } from "../../controllers/v1/types";
import { queueBillingOperation } from "./batch_billing";
-import { autumnService } from "../autumn/autumn.service";
+import {
+ autumnService,
+ featureIdForBillingEndpoint,
+} from "../autumn/autumn.service";
import { toAutumnBillingProperties, type BillingMetadata } from "./types";
import type { Logger } from "winston";
@@ -31,11 +34,13 @@ export async function billTeam(
...toAutumnBillingProperties(billing),
apiKeyId: api_key_id,
};
+ const featureId = featureIdForBillingEndpoint(billing.endpoint);
const trackedInRequest = await autumnService.trackCredits({
teamId: team_id,
value: credits,
properties: autumnProperties,
requestScoped: true,
+ featureId,
});
const result = await queueBillingOperation(
@@ -53,6 +58,7 @@ export async function billTeam(
teamId: team_id,
value: credits,
properties: autumnProperties,
+ featureId,
});
}