Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore: add endpoint properties to autumn track calls (#3128)

Charlie Lambb22ddbd0c9e1
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 865c83cfc..23d00f659 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -238,6 +238,7 @@ export async function crawlController(req: Request, res: Response) {
team_id,
basePriority: 21,
});
+ const billing = { endpoint: "crawl" as const };
const jobs = urls.map(url => {
const uuid = uuidv7();
return {
@@ -251,6 +252,7 @@ export async function crawlController(req: Request, res: Response) {
team_id,
origin: req.body.origin ?? defaultOrigin,
integration: req.body.integration,
+ billing,
crawl_id: id,
sitemapped: true,
zeroDataRetention: false, // not supported on v0
@@ -294,6 +296,7 @@ export async function crawlController(req: Request, res: Response) {
team_id,
origin: req.body.origin ?? defaultOrigin,
integration: req.body.integration,
+ billing: { endpoint: "crawl" },
crawl_id: id,
zeroDataRetention: false, // not supported on v0
apiKeyId: chunk?.api_key_id ?? null,
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 1e7d420d1..8d5294e96 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -91,6 +91,7 @@ async function scrapeHelper(
internalOptions,
origin: req.body.origin ?? defaultOrigin,
integration: req.body.integration,
+ billing: { endpoint: "scrape" },
startTime: Date.now(),
zeroDataRetention: false, // not supported on v0
apiKeyId,
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index e07d8d89f..cfa08d1a7 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -80,14 +80,19 @@ async function searchHelper(
);
if (justSearch) {
- billTeam(team_id, subscription_id, res.length, api_key_id, logger).catch(
- error => {
- logger.error(
- `Failed to bill team ${team_id} for ${res.length} credits: ${error}`,
- );
- // Optionally, you could notify an admin or add to a retry queue here
- },
- );
+ billTeam(
+ team_id,
+ subscription_id,
+ res.length,
+ api_key_id,
+ { endpoint: "search" },
+ logger,
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${team_id} for ${res.length} credits: ${error}`,
+ );
+ // Optionally, you could notify an admin or add to a retry queue here
+ });
return { success: true, data: res, returnCode: 200 };
}
@@ -101,6 +106,7 @@ async function searchHelper(
}
const jobPriority = await getJobPriority({ team_id, basePriority: 20 });
+ const billing = { endpoint: "search" as const };
// filter out social media links
@@ -120,6 +126,7 @@ async function searchHelper(
apiKeyId: api_key_id,
origin: req.body.origin ?? defaultOrigin,
requestId: jobId,
+ billing,
} satisfies ScrapeJobSingleUrls,
};
});
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index fbeb0425b..7a6733978 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -174,6 +174,7 @@ export async function batchScrapeController(
});
}
logger.debug("Using job priority " + jobPriority, { jobPriority });
+ const billing = { endpoint: "batch_scrape" as const };
const jobs = urls.map(x => ({
jobId: uuidv7(),
@@ -185,6 +186,7 @@ export async function batchScrapeController(
scrapeOptions,
origin: "api",
integration: req.body.integration,
+ billing,
crawl_id: id,
sitemapped: true,
v1: true,
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 408239b82..180d86c42 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -171,6 +171,7 @@ export async function crawlController(
internalOptions: sc.internalOptions,
origin: req.body.origin,
integration: req.body.integration,
+ billing: { endpoint: "crawl" },
crawl_id: id,
webhook: req.body.webhook,
v1: true,
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index d24b08ca6..7a1fda94a 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -468,6 +468,7 @@ export async function mapController(
req.acuc?.sub_id,
1,
req.acuc?.api_key_id ?? null,
+ { endpoint: "map" },
).catch(error => {
logger.error(
`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index b5e81eea1..e0eb6a334 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -164,6 +164,7 @@ export async function scrapeController(
skipNuq: true,
origin,
integration: req.body.integration,
+ billing: { endpoint: "scrape" },
startTime: controllerStartTime,
zeroDataRetention: zeroDataRetention ?? false,
apiKeyId: req.acuc?.api_key_id ?? null,
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index f96ffaedd..c8f6ca9cc 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -203,6 +203,7 @@ export async function searchController(
req.acuc?.sub_id ?? undefined,
result.searchCredits,
req.acuc?.api_key_id ?? null,
+ { endpoint: "search" },
).catch(error => {
logger.error(
`Failed to bill team ${req.auth.team_id} for ${result.searchCredits} credits: ${error}`,
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 50da40aa0..7dd236468 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -28,6 +28,7 @@ import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { checkPermissions } from "../../lib/permissions";
import { crawlGroup } from "../../services/worker/nuq";
import { logRequest } from "../../services/logging/log_job";
+import type { BillingMetadata } from "../../services/billing/types";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -68,6 +69,9 @@ export async function batchScrapeController(
}
const id = req.body.appendToId ?? uuidv7();
+ const billing: BillingMetadata = req.body.__agentInterop
+ ? { endpoint: "agent" as const }
+ : { endpoint: "batch_scrape" as const };
const logger = _logger.child({
crawlId: id,
batchScrapeId: id,
@@ -208,6 +212,7 @@ export async function batchScrapeController(
scrapeOptions,
origin: "api",
integration: req.body.integration,
+ billing,
crawl_id: id,
requestId: req.body.__agentInterop?.requestId ?? undefined,
bypassBilling: !(req.body.__agentInterop?.shouldBill ?? true),
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.ts
index fe3fdcd8c..ccd873b1b 100644
--- a/apps/api/src/controllers/v2/browser.ts
+++ b/apps/api/src/controllers/v2/browser.ts
@@ -567,6 +567,7 @@ export async function browserDeleteController(
req.acuc?.sub_id ?? undefined,
creditsBilled,
req.acuc?.api_key_id ?? null,
+ { endpoint: "browser" },
).catch(error => {
logger.error("Failed to bill team for browser session", {
error,
@@ -693,6 +694,7 @@ export async function browserWebhookDestroyedController(
undefined, // subscription_id — billTeam will look it up
creditsBilled,
null, // api_key_id not available in webhook context
+ { endpoint: "browser" },
).catch(error => {
logger.error("Failed to bill team for browser session via webhook", {
error,
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index b36521368..a00c068d3 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -230,6 +230,7 @@ export async function crawlController(
internalOptions: sc.internalOptions,
origin: req.body.origin,
integration: req.body.integration,
+ billing: { endpoint: "crawl" },
crawl_id: id,
webhook: req.body.webhook,
v1: true,
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 4c5adcf13..d3f5ebb4f 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -129,6 +129,7 @@ export async function mapController(
req.acuc?.sub_id ?? undefined,
1,
req.acuc?.api_key_id ?? null,
+ { endpoint: "map" },
).catch(error => {
logger.error(
`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index f7ef64ab4..a99bffc68 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -22,6 +22,7 @@ import { getJobPriority } from "../../lib/job-priority";
import { logRequest } from "../../services/logging/log_job";
import { getErrorContactMessage } from "../../lib/deployment";
import { captureExceptionWithZdrCheck } from "../../services/sentry";
+import type { BillingMetadata } from "../../services/billing/types";
const AGENT_INTEROP_CONCURRENCY_BOOST = 3;
@@ -83,6 +84,9 @@ export async function scrapeController(
const zeroDataRetention =
req.acuc?.flags?.forceZDR || (req.body.zeroDataRetention ?? false);
+ const billing: BillingMetadata = req.body.__agentInterop
+ ? { endpoint: "agent" as const }
+ : { endpoint: "scrape" as const };
if (
req.body.__agentInterop &&
@@ -246,6 +250,7 @@ export async function scrapeController(
skipNuq: true,
origin,
integration: req.body.integration,
+ billing,
startTime: controllerStartTime,
zeroDataRetention,
apiKeyId: req.acuc?.api_key_id ?? null,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 44d738aab..2ca239a7c 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -18,6 +18,7 @@ import {
captureExceptionWithZdrCheck,
} from "../../services/sentry";
import { executeSearch } from "../../search/execute";
+import type { BillingMetadata } from "../../services/billing/types";
export async function searchController(
req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
@@ -72,6 +73,9 @@ export async function searchController(
const shouldBill = req.body.__agentInterop?.shouldBill ?? true;
const agentRequestId = req.body.__agentInterop?.requestId ?? null;
+ const billing: BillingMetadata = req.body.__agentInterop
+ ? { endpoint: "agent" as const }
+ : { endpoint: "search" as const };
logger = logger.child({
version: "v2",
@@ -122,6 +126,7 @@ export async function searchController(
requestId: agentRequestId ?? jobId,
bypassBilling: !shouldBill,
zeroDataRetention: isZDROrAnon,
+ billing,
},
logger,
);
@@ -133,6 +138,7 @@ export async function searchController(
req.acuc?.sub_id ?? undefined,
result.searchCredits,
req.acuc?.api_key_id ?? null,
+ billing,
).catch(error =>
logger.error(
`Failed to bill team ${req.acuc?.sub_id} for ${result.searchCredits} credits: ${error}`,
diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.ts
index 272f83e72..42162478e 100644
--- a/apps/api/src/lib/deep-research/deep-research-service.ts
+++ b/apps/api/src/lib/deep-research/deep-research-service.ts
@@ -425,7 +425,14 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
json: finalAnalysisJson,
});
// Bill team for usage based on URLs analyzed
- billTeam(teamId, subId, credits_billed, apiKeyId, logger).catch(error => {
+ billTeam(
+ teamId,
+ subId,
+ credits_billed,
+ apiKeyId,
+ { endpoint: "deep_research" },
+ logger,
+ ).catch(error => {
logger.error(
`Failed to bill team ${teamId} for ${urlsAnalyzed} URLs analyzed`,
{ teamId, count: urlsAnalyzed, error },
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index 78fa58c21..e14f40cf5 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -148,7 +148,14 @@ async function performExtraction(
cost_tracking: costTracking.toJSON(),
});
- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ await billTeam(
+ teamId,
+ subId,
+ creditsToBill,
+ apiKeyId,
+ { endpoint: "extract" },
+ logger,
+ ).catch(
error => {
logger.error(
`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
@@ -659,7 +666,14 @@ async function performExtraction(
error: "Failed to transform array to object",
cost_tracking: costTracking.toJSON(),
});
- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ await billTeam(
+ teamId,
+ subId,
+ creditsToBill,
+ apiKeyId,
+ { endpoint: "extract" },
+ logger,
+ ).catch(
error => {
logger.error(
`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
@@ -766,7 +780,14 @@ async function performExtraction(
model_kind: "fire-1",
cost_tracking: costTracking.toJSON(),
});
- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ await billTeam(
+ teamId,
+ subId,
+ creditsToBill,
+ apiKeyId,
+ { endpoint: "extract" },
+ logger,
+ ).catch(
error => {
logger.error(
`Failed to bill team ${teamId} for thinking tokens: ${error}`,
@@ -789,7 +810,14 @@ async function performExtraction(
logger.error(errorMessage);
const tokens_billed = 300 + calculateThinkingCost(costTracking);
const creditsToBill = Math.ceil(tokens_billed / 15);
- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ await billTeam(
+ teamId,
+ subId,
+ creditsToBill,
+ apiKeyId,
+ { endpoint: "extract" },
+ logger,
+ ).catch(
error => {
logger.error(
`Failed to bill team ${teamId} for thinking tokens: ${error}`,
@@ -976,7 +1004,14 @@ async function performExtraction(
const creditsToBill = Math.ceil(tokensToBill / 15);
// Bill team for usage
- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ await billTeam(
+ teamId,
+ subId,
+ creditsToBill,
+ apiKeyId,
+ { endpoint: "extract" },
+ logger,
+ ).catch(
error => {
logger.error(
`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
@@ -1039,7 +1074,14 @@ async function performExtraction(
} catch (error) {
const tokens_billed = 300 + calculateThinkingCost(costTracking);
const creditsToBill = Math.ceil(tokens_billed / 15);
- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(
+ await billTeam(
+ teamId,
+ subId,
+ creditsToBill,
+ apiKeyId,
+ { endpoint: "extract" },
+ logger,
+ ).catch(
error => {
logger.error(
`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index 3536f0475..beba75352 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -852,7 +852,7 @@ export async function performExtraction_F0(
const creditsToBill = Math.ceil(tokensToBill / 15);
// Bill team for usage
- billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(error => {
+ billTeam(teamId, subId, creditsToBill, apiKeyId, { endpoint: "extract" }, logger).catch(error => {
logger.error(
`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,
);
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
index c14bda01d..c27493dc8 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
@@ -267,7 +267,7 @@ export async function performGenerateLlmsTxt(
});
// Bill team for usage
- billTeam(teamId, subId, urls.length, apiKeyId, logger).catch(error => {
+ billTeam(teamId, subId, urls.length, apiKeyId, { endpoint: "llms_txt" }, logger).catch(error => {
logger.error(`Failed to bill team ${teamId} for ${urls.length} urls`, {
teamId,
count: urls.length,
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index da9b02090..a0e4b2069 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -246,7 +246,10 @@ export class WebCrawler {
break;
default:
// Use the static enum message for other cases
- fancyDenialReasons.set(key, DenialReason[value]);
+ fancyDenialReasons.set(
+ key,
+ DenialReason[value as keyof typeof DenialReason] ?? value,
+ );
}
});
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 1f790358f..8a847854a 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -13,6 +13,7 @@ import {
mergeScrapedContent,
calculateScrapeCredits,
} from "./scrape";
+import type { BillingMetadata } from "../services/billing/types";
interface SearchOptions {
query: string;
@@ -37,6 +38,7 @@ interface SearchContext {
requestId: string;
bypassBilling?: boolean;
zeroDataRetention?: boolean;
+ billing?: BillingMetadata;
}
interface SearchExecuteResult {
@@ -62,6 +64,7 @@ export async function executeSearch(
requestId,
bypassBilling,
zeroDataRetention,
+ billing,
} = context;
const num_results_buffer = Math.floor(limit * 2);
@@ -149,6 +152,7 @@ export async function executeSearch(
apiKeyId,
zeroDataRetention,
requestId,
+ billing,
};
const allDocsWithCostTracking = await scrapeSearchResults(
diff --git a/apps/api/src/search/scrape.test.ts b/apps/api/src/search/scrape.test.ts
new file mode 100644
index 000000000..554211fbb
--- /dev/null
+++ b/apps/api/src/search/scrape.test.ts
@@ -0,0 +1,55 @@
+jest.mock("uuid", () => ({
+ v7: jest.fn(() => "job-1"),
+}));
+
+import { scrapeSearchResults } from "./scrape";
+import { getJobPriority } from "../lib/job-priority";
+import { processJobInternal } from "../services/worker/scrape-worker";
+
+jest.mock("../lib/job-priority", () => ({
+ getJobPriority: jest.fn().mockResolvedValue(10),
+}));
+
+jest.mock("../services/worker/scrape-worker", () => ({
+ processJobInternal: jest.fn().mockResolvedValue({
+ markdown: "body",
+ metadata: { creditsUsed: 1, statusCode: 200, proxyUsed: "basic" },
+ }),
+}));
+
+describe("scrapeSearchResults", () => {
+ it("preserves billing metadata on spawned scrape jobs", async () => {
+ await scrapeSearchResults(
+ [
+ {
+ url: "https://example.com",
+ title: "Example",
+ description: "Desc",
+ },
+ ],
+ {
+ teamId: "team-1",
+ origin: "api",
+ timeout: 60_000,
+ scrapeOptions: {} as any,
+ apiKeyId: 123,
+ requestId: "req-1",
+ billing: { endpoint: "agent" },
+ },
+ { debug: jest.fn(), info: jest.fn(), error: jest.fn() } as any,
+ null as any,
+ );
+
+ expect(getJobPriority).toHaveBeenCalledWith({
+ team_id: "team-1",
+ basePriority: 10,
+ });
+ expect(processJobInternal).toHaveBeenCalledWith(
+ expect.objectContaining({
+ data: expect.objectContaining({
+ billing: { endpoint: "agent" },
+ }),
+ }),
+ );
+ });
+});
diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.ts
index 731b7eb98..41081098b 100644
--- a/apps/api/src/search/scrape.ts
+++ b/apps/api/src/search/scrape.ts
@@ -8,6 +8,7 @@ import { NuQJob } from "../services/worker/nuq";
import { processJobInternal } from "../services/worker/scrape-worker";
import { ScrapeJobData } from "../types";
import { SearchV2Response } from "../lib/entities";
+import type { BillingMetadata } from "../services/billing/types";
export interface DocumentWithCostTracking {
document: Document;
@@ -35,6 +36,7 @@ interface ScrapeSearchOptions {
apiKeyId: number | null;
zeroDataRetention?: boolean;
requestId?: string;
+ billing?: BillingMetadata;
}
async function scrapeSearchResultDirect(
@@ -83,6 +85,7 @@ async function scrapeSearchResultDirect(
zeroDataRetention,
apiKeyId: options.apiKeyId,
requestId: options.requestId,
+ billing: options.billing,
},
};
diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
index 1e104fd04..cfe251aed 100644
--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts
@@ -201,6 +201,9 @@ describe("ensureTeamProvisioned", () => {
// Only one getEntity call (no confirmation get).
expect(mockEntityGet).toHaveBeenCalledTimes(1);
expect(mockEntityCreate).toHaveBeenCalledTimes(1);
+ expect(mockEntityCreate).toHaveBeenCalledWith(
+ expect.objectContaining({ featureId: "TEAM" }),
+ );
});
it("marks team as ensured on 409 conflict without a second getEntity", async () => {
@@ -286,7 +289,7 @@ describe("reserveCredits", () => {
const result = await svc.reserveCredits({
teamId: "team-1",
value: 42,
- properties: { source: "test" },
+ properties: { source: "test", endpoint: "extract" },
});
expect(result).toBe(true);
@@ -296,6 +299,7 @@ describe("reserveCredits", () => {
(c: any[]) => c[0].featureId === "CREDITS" && c[0].value === 42,
);
expect(usageCall).toBeDefined();
+ expect((usageCall as any[])[0].properties?.endpoint).toBe("extract");
});
});
@@ -306,13 +310,18 @@ describe("reserveCredits", () => {
describe("refundCredits", () => {
it("calls track with the negated value", async () => {
const svc = makeService();
- await svc.refundCredits({ teamId: "team-1", value: 30 });
+ await svc.refundCredits({
+ teamId: "team-1",
+ value: 30,
+ properties: { endpoint: "extract" },
+ });
const refundCall = mockTrack.mock.calls.find(
(c: any[]) => c[0].value === -30,
);
expect(refundCall).toBeDefined();
expect((refundCall as any[])[0].properties?.source).toBe("autumn_refund");
+ expect((refundCall as any[])[0].properties?.endpoint).toBe("extract");
});
it("is a no-op when autumnClient is null", async () => {
diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts
index f35934c43..ca9671447 100644
--- a/apps/api/src/services/autumn/autumn.service.ts
+++ b/apps/api/src/services/autumn/autumn.service.ts
@@ -13,6 +13,7 @@ import type {
TrackParams,
} from "./types";
+const TEAM_FEATURE_ID = "TEAM";
const CREDITS_FEATURE_ID = "CREDITS";
/**
@@ -277,10 +278,10 @@ export class AutumnService {
const result = await this.createEntity({
customerId: resolvedOrgId,
entityId: teamId,
- featureId: CREDITS_FEATURE_ID,
+ featureId: TEAM_FEATURE_ID,
name,
});
- if (result.ok || result.conflict) {
+ if (result.ok || ("conflict" in result && result.conflict)) {
// Entity was just created, or already existed (409 race) — either way
// it's present. No need for a second getEntity confirmation call.
this.ensuredTeams.add(teamId);
diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.ts
index 4b65a870e..04b859208 100644
--- a/apps/api/src/services/billing/batch_billing.ts
+++ b/apps/api/src/services/billing/batch_billing.ts
@@ -6,6 +6,12 @@ import * as Sentry from "@sentry/node";
import { withAuth } from "../../lib/withAuth";
import { setCachedACUC, setCachedACUCTeam } from "../../controllers/auth";
import { autumnService } from "../autumn/autumn.service";
+import {
+ resolveBillingMetadata,
+ toAutumnBillingProperties,
+ type BillingEndpoint,
+ type BillingMetadata,
+} from "./types";
// Configuration constants
const BATCH_KEY = "billing_batch";
@@ -19,6 +25,8 @@ interface BillingOperation {
team_id: string;
subscription_id: string | null;
credits: number;
+ billing?: BillingMetadata;
+ endpoint?: BillingEndpoint;
is_extract: boolean;
timestamp: string;
api_key_id: number | null;
@@ -31,6 +39,7 @@ interface GroupedBillingOperation {
team_id: string;
subscription_id: string | null;
total_credits: number;
+ billing: BillingMetadata;
is_extract: boolean;
api_key_id: number | null;
operations: BillingOperation[];
@@ -93,13 +102,19 @@ export async function processBillingBatch() {
const groupedOperations = new Map<string, GroupedBillingOperation>();
for (const op of operations) {
- const key = `${op.team_id}:${op.subscription_id ?? "null"}:${op.is_extract}:${op.api_key_id}`;
+ const billing = resolveBillingMetadata({
+ billing:
+ op.billing ?? (op.endpoint ? { endpoint: op.endpoint } : undefined),
+ isExtract: op.is_extract,
+ });
+ const key = `${op.team_id}:${op.subscription_id ?? "null"}:${billing.endpoint}:${op.is_extract}:${op.api_key_id}`;
if (!groupedOperations.has(key)) {
groupedOperations.set(key, {
team_id: op.team_id,
subscription_id: op.subscription_id,
total_credits: 0,
+ billing,
is_extract: op.is_extract,
api_key_id: op.api_key_id,
operations: [],
@@ -119,6 +134,7 @@ export async function processBillingBatch() {
team_id: group.team_id,
subscription_id: group.subscription_id,
total_credits: group.total_credits,
+ billing: group.billing,
operation_count: group.operations.length,
is_extract: group.is_extract,
},
@@ -154,7 +170,11 @@ export async function processBillingBatch() {
if (!billingResult.success) {
logger.warn(
`⚠️ Billing returned success: false for team ${group.team_id}, skipping Autumn tracking`,
- { billingResult, team_id: group.team_id, credits: group.total_credits },
+ {
+ billingResult,
+ team_id: group.team_id,
+ credits: group.total_credits,
+ },
);
// Refund only the credits that were actually reserved in Autumn.
if (reservedCredits > 0) {
@@ -163,6 +183,7 @@ export async function processBillingBatch() {
value: reservedCredits,
properties: {
source: "processBillingBatch_failure",
+ ...toAutumnBillingProperties(group.billing),
apiKeyId: group.api_key_id,
subscriptionId: group.subscription_id,
},
@@ -183,6 +204,7 @@ export async function processBillingBatch() {
value: unreservedCredits,
properties: {
source: "processBillingBatch",
+ ...toAutumnBillingProperties(group.billing),
apiKeyId: group.api_key_id,
subscriptionId: group.subscription_id,
},
@@ -207,6 +229,7 @@ export async function processBillingBatch() {
value: reservedCredits,
properties: {
source: "processBillingBatch_exception",
+ ...toAutumnBillingProperties(group.billing),
apiKeyId: group.api_key_id,
subscriptionId: group.subscription_id,
},
@@ -257,6 +280,7 @@ export async function queueBillingOperation(
subscription_id: string | null | undefined,
credits: number,
api_key_id: number | null,
+ billing: BillingMetadata,
is_extract: boolean = false,
autumnReserved: boolean = false,
) {
@@ -270,6 +294,7 @@ export async function queueBillingOperation(
team_id,
subscription_id,
credits,
+ billing,
is_extract,
});
@@ -278,6 +303,7 @@ export async function queueBillingOperation(
team_id,
subscription_id: subscription_id ?? null,
credits,
+ billing,
is_extract,
timestamp: new Date().toISOString(),
api_key_id,
@@ -393,9 +419,11 @@ async function supaBillTeam(
// Fire-and-forget — a Redis failure here must not trigger a false Autumn refund
// after bill_team_6 has already committed.
- getRedisConnection().sadd("billed_teams", team_id).catch(err => {
- _logger.warn("Failed to add team to billed_teams set", { err, team_id });
- });
+ getRedisConnection()
+ .sadd("billed_teams", team_id)
+ .catch(err => {
+ _logger.warn("Failed to add team to billed_teams set", { err, team_id });
+ });
// Update cached ACUC to reflect the new credit usage
(async () => {
diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts
index ed0db30ab..7bcc547e8 100644
--- a/apps/api/src/services/billing/credit_billing.ts
+++ b/apps/api/src/services/billing/credit_billing.ts
@@ -9,6 +9,10 @@ import { autoCharge } from "./auto_charge";
import { getValue, setValue } from "../redis";
import { queueBillingOperation } from "./batch_billing";
import { autumnService } from "../autumn/autumn.service";
+import {
+ toAutumnBillingProperties,
+ type BillingMetadata,
+} from "./types";
import type { Logger } from "winston";
/**
@@ -19,6 +23,7 @@ export async function billTeam(
subscription_id: string | null | undefined,
credits: number,
api_key_id: number | null,
+ billing: BillingMetadata,
logger?: Logger,
) {
return withAuth(
@@ -27,6 +32,7 @@ export async function billTeam(
subscription_id: string | null | undefined,
credits: number,
api_key_id: number | null,
+ billing: BillingMetadata,
logger: Logger | undefined,
) => {
// Reserve in Autumn at request time; await so autumnReserved is accurate.
@@ -34,19 +40,24 @@ export async function billTeam(
const autumnReserved = await autumnService.reserveCredits({
teamId: team_id,
value: credits,
- properties: { source: "billTeam", apiKeyId: api_key_id },
+ properties: {
+ source: "billTeam",
+ ...toAutumnBillingProperties(billing),
+ apiKeyId: api_key_id,
+ },
});
return queueBillingOperation(
team_id,
subscription_id,
credits,
api_key_id,
+ billing,
false,
autumnReserved,
);
},
{ success: true, message: "No DB, bypassed." },
- )(team_id, subscription_id, credits, api_key_id, logger);
+ )(team_id, subscription_id, credits, api_key_id, billing, logger);
}
type CheckTeamCreditsResponse = {
diff --git a/apps/api/src/services/billing/types.ts b/apps/api/src/services/billing/types.ts
new file mode 100644
index 000000000..9a461a5f0
--- /dev/null
+++ b/apps/api/src/services/billing/types.ts
@@ -0,0 +1,43 @@
+export type BillingEndpoint =
+ | "agent"
+ | "batch_scrape"
+ | "browser"
+ | "crawl"
+ | "deep_research"
+ | "extract"
+ | "llms_txt"
+ | "map"
+ | "scrape"
+ | "search";
+
+export type BillingMetadata = {
+ endpoint: BillingEndpoint;
+};
+
+export function resolveBillingMetadata({
+ billing,
+ isExtract = false,
+ crawlId,
+ crawlerOptions,
+}: {
+ billing?: BillingMetadata;
+ isExtract?: boolean;
+ crawlId?: string;
+ crawlerOptions?: unknown;
+}): BillingMetadata {
+ if (billing) return billing;
+ if (crawlId) {
+ return {
+ endpoint: crawlerOptions == null ? "batch_scrape" : "crawl",
+ };
+ }
+ return {
+ endpoint: isExtract ? "extract" : "scrape",
+ };
+}
+
+export function toAutumnBillingProperties(
+ billing: BillingMetadata,
+): Record<string, string> {
+ return { endpoint: billing.endpoint };
+}
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 5b4d3a881..f1fa0dc05 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -16,6 +16,7 @@ import {
queueBillingOperation,
startBillingBatchProcessing,
} from "../billing/batch_billing";
+import { resolveBillingMetadata } from "../billing/types";
import systemMonitor from "../system-monitor";
import { v7 as uuidv7 } from "uuid";
import {
@@ -80,7 +81,7 @@ const processBillingJobInternal = async (token: string, job: Job) => {
await processBillingBatch();
} else if (job.name === "bill_team") {
// This is an individual billing operation that should be queued for batch processing
- const { team_id, subscription_id, credits, is_extract, api_key_id } =
+ const { team_id, subscription_id, credits, billing, endpoint, is_extract, api_key_id } =
job.data;
logger.info(`Adding team ${team_id} billing operation to batch queue`, {
@@ -95,6 +96,10 @@ const processBillingJobInternal = async (token: string, job: Job) => {
subscription_id,
credits,
api_key_id ?? null,
+ resolveBillingMetadata({
+ billing: billing ?? (endpoint ? { endpoint } : undefined),
+ isExtract: is_extract,
+ }),
is_extract,
);
} else {
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 0a53dac95..94cc00dd6 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -30,6 +30,7 @@ import {
StoredCrawl,
} from "../../lib/crawl-redis";
import { redisEvictConnection } from "../redis";
+import { resolveBillingMetadata } from "../billing/types";
import {
_addScrapeJobToBullMQ,
addScrapeJob,
@@ -115,11 +116,17 @@ async function billScrapeJob(
config.USE_DB_AUTHENTICATION
) {
try {
+ const billing = resolveBillingMetadata({
+ billing: job.data.billing,
+ crawlId: job.data.crawl_id,
+ crawlerOptions: job.data.crawlerOptions,
+ });
const billingJobId = uuidv7();
logger.debug(
`Adding billing job to queue for team ${job.data.team_id}`,
{
billingJobId,
+ billing,
credits: creditsToBeBilled,
is_extract: false,
},
@@ -132,6 +139,7 @@ async function billScrapeJob(
team_id: job.data.team_id,
subscription_id: undefined,
credits: creditsToBeBilled,
+ billing,
is_extract: false,
timestamp: new Date().toISOString(),
originating_job_id: job.id,
@@ -408,6 +416,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
integration: job.data.integration,
crawl_id: job.data.crawl_id,
requestId: job.data.requestId,
+ billing: job.data.billing,
webhook: job.data.webhook,
v1: job.data.v1,
zeroDataRetention: job.data.zeroDataRetention,
@@ -797,6 +806,7 @@ async function addKickoffSitemapJob(
integration: sourceJob.data.integration,
crawl_id: sourceJob.data.crawl_id,
requestId: sourceJob.data.requestId,
+ billing: sourceJob.data.billing,
webhook: sourceJob.data.webhook,
v1: sourceJob.data.v1,
apiKeyId: sourceJob.data.apiKeyId,
@@ -849,6 +859,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
integration: job.data.integration,
crawl_id: job.data.crawl_id,
requestId: job.data.requestId,
+ billing: job.data.billing,
webhook: job.data.webhook,
v1: job.data.v1,
isCrawlSourceScrape: true,
@@ -936,6 +947,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
integration: job.data.integration,
crawl_id: job.data.crawl_id,
requestId: job.data.requestId,
+ billing: job.data.billing,
sitemapped: true,
webhook: job.data.webhook,
v1: job.data.v1,
@@ -1048,6 +1060,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
integration: job.data.integration,
crawl_id: job.data.crawl_id,
requestId: job.data.requestId,
+ billing: job.data.billing,
sitemapped: true,
webhook: job.data.webhook,
v1: job.data.v1,
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 9146eb803..b933da50f 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -9,6 +9,7 @@ import { AuthCreditUsageChunk } from "./controllers/v1/types";
import { ExtractorOptions, Document } from "./lib/entities";
import { InternalOptions } from "./scraper/scrapeURL";
import type { CostTracking } from "./lib/cost-tracking";
+import type { BillingMetadata } from "./services/billing/types";
import { webhookSchema } from "./services/webhook/schema";
import { SerializedTraceContext } from "./lib/otel-tracer";
@@ -16,6 +17,7 @@ type ScrapeJobCommon = {
concurrencyLimited?: boolean;
team_id: string;
zeroDataRetention: boolean;
+ billing?: BillingMetadata;
traceContext?: SerializedTraceContext;
skipNuq?: boolean;
requestId?: string;