Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore: add endpoint properties to autumn track calls (#3128)
Charlie Lambb22ddbd0c9e1
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 865c83cfc..23d00f659 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -238,6 +238,7 @@ export async function crawlController(req: Request, res: Response) {team_id,basePriority: 21,});+ const billing = { endpoint: "crawl" as const };const jobs = urls.map(url => {const uuid = uuidv7();return {@@ -251,6 +252,7 @@ export async function crawlController(req: Request, res: Response) {team_id,origin: req.body.origin ?? defaultOrigin,integration: req.body.integration,+ billing,crawl_id: id,sitemapped: true,zeroDataRetention: false, // not supported on v0@@ -294,6 +296,7 @@ export async function crawlController(req: Request, res: Response) {team_id,origin: req.body.origin ?? defaultOrigin,integration: req.body.integration,+ billing: { endpoint: "crawl" },crawl_id: id,zeroDataRetention: false, // not supported on v0apiKeyId: chunk?.api_key_id ?? null,diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 1e7d420d1..8d5294e96 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -91,6 +91,7 @@ async function scrapeHelper(internalOptions,origin: req.body.origin ?? defaultOrigin,integration: req.body.integration,+ billing: { endpoint: "scrape" },startTime: Date.now(),zeroDataRetention: false, // not supported on v0apiKeyId,diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex e07d8d89f..cfa08d1a7 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -80,14 +80,19 @@ async function searchHelper();if (justSearch) {- billTeam(team_id, subscription_id, res.length, api_key_id, logger).catch(- error => {- logger.error(- `Failed to bill team ${team_id} for ${res.length} credits: ${error}`,- );- // Optionally, you could notify an admin or add to a retry queue here- },- );+ billTeam(+ team_id,+ subscription_id,+ res.length,+ api_key_id,+ { endpoint: "search" },+ logger,+ ).catch(error => {+ logger.error(+ `Failed to bill team ${team_id} for ${res.length} credits: ${error}`,+ );+ // Optionally, you could notify an admin or add to a retry queue here+ });return { success: true, data: res, returnCode: 200 };}@@ -101,6 +106,7 @@ async function searchHelper(}const jobPriority = await getJobPriority({ team_id, basePriority: 20 });+ const billing = { endpoint: "search" as const };// filter out social media links@@ -120,6 +126,7 @@ async function searchHelper(apiKeyId: api_key_id,origin: req.body.origin ?? defaultOrigin,requestId: jobId,+ billing,} satisfies ScrapeJobSingleUrls,};});diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex fbeb0425b..7a6733978 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -174,6 +174,7 @@ export async function batchScrapeController(});}logger.debug("Using job priority " + jobPriority, { jobPriority });+ const billing = { endpoint: "batch_scrape" as const };const jobs = urls.map(x => ({jobId: uuidv7(),@@ -185,6 +186,7 @@ export async function batchScrapeController(scrapeOptions,origin: "api",integration: req.body.integration,+ billing,crawl_id: id,sitemapped: true,v1: true,diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 408239b82..180d86c42 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -171,6 +171,7 @@ export async function crawlController(internalOptions: sc.internalOptions,origin: req.body.origin,integration: req.body.integration,+ billing: { endpoint: "crawl" },crawl_id: id,webhook: req.body.webhook,v1: true,diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex d24b08ca6..7a1fda94a 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -468,6 +468,7 @@ export async function mapController(req.acuc?.sub_id,1,req.acuc?.api_key_id ?? null,+ { endpoint: "map" },).catch(error => {logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex b5e81eea1..e0eb6a334 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -164,6 +164,7 @@ export async function scrapeController(skipNuq: true,origin,integration: req.body.integration,+ billing: { endpoint: "scrape" },startTime: controllerStartTime,zeroDataRetention: zeroDataRetention ?? false,apiKeyId: req.acuc?.api_key_id ?? null,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex f96ffaedd..c8f6ca9cc 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -203,6 +203,7 @@ export async function searchController(req.acuc?.sub_id ?? undefined,result.searchCredits,req.acuc?.api_key_id ?? null,+ { endpoint: "search" },).catch(error => {logger.error(`Failed to bill team ${req.auth.team_id} for ${result.searchCredits} credits: ${error}`,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 50da40aa0..7dd236468 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -28,6 +28,7 @@ import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { checkPermissions } from "../../lib/permissions";import { crawlGroup } from "../../services/worker/nuq";import { logRequest } from "../../services/logging/log_job";+import type { BillingMetadata } from "../../services/billing/types";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -68,6 +69,9 @@ export async function batchScrapeController(}const id = req.body.appendToId ?? uuidv7();+ const billing: BillingMetadata = req.body.__agentInterop+ ? { endpoint: "agent" as const }+ : { endpoint: "batch_scrape" as const };const logger = _logger.child({crawlId: id,batchScrapeId: id,@@ -208,6 +212,7 @@ export async function batchScrapeController(scrapeOptions,origin: "api",integration: req.body.integration,+ billing,crawl_id: id,requestId: req.body.__agentInterop?.requestId ?? undefined,bypassBilling: !(req.body.__agentInterop?.shouldBill ?? true),diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex fe3fdcd8c..ccd873b1b 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -567,6 +567,7 @@ export async function browserDeleteController(req.acuc?.sub_id ?? undefined,creditsBilled,req.acuc?.api_key_id ?? null,+ { endpoint: "browser" },).catch(error => {logger.error("Failed to bill team for browser session", {error,@@ -693,6 +694,7 @@ export async function browserWebhookDestroyedController(undefined, // subscription_id — billTeam will look it upcreditsBilled,null, // api_key_id not available in webhook context+ { endpoint: "browser" },).catch(error => {logger.error("Failed to bill team for browser session via webhook", {error,diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex b36521368..a00c068d3 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -230,6 +230,7 @@ export async function crawlController(internalOptions: sc.internalOptions,origin: req.body.origin,integration: req.body.integration,+ billing: { endpoint: "crawl" },crawl_id: id,webhook: req.body.webhook,v1: true,diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 4c5adcf13..d3f5ebb4f 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -129,6 +129,7 @@ export async function mapController(req.acuc?.sub_id ?? undefined,1,req.acuc?.api_key_id ?? null,+ { endpoint: "map" },).catch(error => {logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex f7ef64ab4..a99bffc68 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -22,6 +22,7 @@ import { getJobPriority } from "../../lib/job-priority";import { logRequest } from "../../services/logging/log_job";import { getErrorContactMessage } from "../../lib/deployment";import { captureExceptionWithZdrCheck } from "../../services/sentry";+import type { BillingMetadata } from "../../services/billing/types";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;@@ -83,6 +84,9 @@ export async function scrapeController(const zeroDataRetention =req.acuc?.flags?.forceZDR || (req.body.zeroDataRetention ?? false);+ const billing: BillingMetadata = req.body.__agentInterop+ ? { endpoint: "agent" as const }+ : { endpoint: "scrape" as const };if (req.body.__agentInterop &&@@ -246,6 +250,7 @@ export async function scrapeController(skipNuq: true,origin,integration: req.body.integration,+ billing,startTime: controllerStartTime,zeroDataRetention,apiKeyId: req.acuc?.api_key_id ?? null,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 44d738aab..2ca239a7c 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -18,6 +18,7 @@ import {captureExceptionWithZdrCheck,} from "../../services/sentry";import { executeSearch } from "../../search/execute";+import type { BillingMetadata } from "../../services/billing/types";export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,@@ -72,6 +73,9 @@ export async function searchController(const shouldBill = req.body.__agentInterop?.shouldBill ?? true;const agentRequestId = req.body.__agentInterop?.requestId ?? null;+ const billing: BillingMetadata = req.body.__agentInterop+ ? { endpoint: "agent" as const }+ : { endpoint: "search" as const };logger = logger.child({version: "v2",@@ -122,6 +126,7 @@ export async function searchController(requestId: agentRequestId ?? jobId,bypassBilling: !shouldBill,zeroDataRetention: isZDROrAnon,+ billing,},logger,);@@ -133,6 +138,7 @@ export async function searchController(req.acuc?.sub_id ?? undefined,result.searchCredits,req.acuc?.api_key_id ?? null,+ billing,).catch(error =>logger.error(`Failed to bill team ${req.acuc?.sub_id} for ${result.searchCredits} credits: ${error}`,diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.tsindex 272f83e72..42162478e 100644--- a/apps/api/src/lib/deep-research/deep-research-service.ts+++ b/apps/api/src/lib/deep-research/deep-research-service.ts@@ -425,7 +425,14 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {json: finalAnalysisJson,});// Bill team for usage based on URLs analyzed- billTeam(teamId, subId, credits_billed, apiKeyId, logger).catch(error => {+ billTeam(+ teamId,+ subId,+ credits_billed,+ apiKeyId,+ { endpoint: "deep_research" },+ logger,+ ).catch(error => {logger.error(`Failed to bill team ${teamId} for ${urlsAnalyzed} URLs analyzed`,{ teamId, count: urlsAnalyzed, error },diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 78fa58c21..e14f40cf5 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -148,7 +148,14 @@ async function performExtraction(cost_tracking: costTracking.toJSON(),});- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ await billTeam(+ teamId,+ subId,+ creditsToBill,+ apiKeyId,+ { endpoint: "extract" },+ logger,+ ).catch(error => {logger.error(`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,@@ -659,7 +666,14 @@ async function performExtraction(error: "Failed to transform array to object",cost_tracking: costTracking.toJSON(),});- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ await billTeam(+ teamId,+ subId,+ creditsToBill,+ apiKeyId,+ { endpoint: "extract" },+ logger,+ ).catch(error => {logger.error(`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,@@ -766,7 +780,14 @@ async function performExtraction(model_kind: "fire-1",cost_tracking: costTracking.toJSON(),});- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ await billTeam(+ teamId,+ subId,+ creditsToBill,+ apiKeyId,+ { endpoint: "extract" },+ logger,+ ).catch(error => {logger.error(`Failed to bill team ${teamId} for thinking tokens: ${error}`,@@ -789,7 +810,14 @@ async function performExtraction(logger.error(errorMessage);const tokens_billed = 300 + calculateThinkingCost(costTracking);const creditsToBill = Math.ceil(tokens_billed / 15);- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ await billTeam(+ teamId,+ subId,+ creditsToBill,+ apiKeyId,+ { endpoint: "extract" },+ logger,+ ).catch(error => {logger.error(`Failed to bill team ${teamId} for thinking tokens: ${error}`,@@ -976,7 +1004,14 @@ async function performExtraction(const creditsToBill = Math.ceil(tokensToBill / 15);// Bill team for usage- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ await billTeam(+ teamId,+ subId,+ creditsToBill,+ apiKeyId,+ { endpoint: "extract" },+ logger,+ ).catch(error => {logger.error(`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,@@ -1039,7 +1074,14 @@ async function performExtraction(} catch (error) {const tokens_billed = 300 + calculateThinkingCost(costTracking);const creditsToBill = Math.ceil(tokens_billed / 15);- await billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(+ await billTeam(+ teamId,+ subId,+ creditsToBill,+ apiKeyId,+ { endpoint: "extract" },+ logger,+ ).catch(error => {logger.error(`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 3536f0475..beba75352 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -852,7 +852,7 @@ export async function performExtraction_F0(const creditsToBill = Math.ceil(tokensToBill / 15);// Bill team for usage- billTeam(teamId, subId, creditsToBill, apiKeyId, logger).catch(error => {+ billTeam(teamId, subId, creditsToBill, apiKeyId, { endpoint: "extract" }, logger).catch(error => {logger.error(`Failed to bill team ${teamId} for ${creditsToBill} credits: ${error}`,);diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.tsindex c14bda01d..c27493dc8 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts@@ -267,7 +267,7 @@ export async function performGenerateLlmsTxt(});// Bill team for usage- billTeam(teamId, subId, urls.length, apiKeyId, logger).catch(error => {+ billTeam(teamId, subId, urls.length, apiKeyId, { endpoint: "llms_txt" }, logger).catch(error => {logger.error(`Failed to bill team ${teamId} for ${urls.length} urls`, {teamId,count: urls.length,diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex da9b02090..a0e4b2069 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -246,7 +246,10 @@ export class WebCrawler {break;default:// Use the static enum message for other cases- fancyDenialReasons.set(key, DenialReason[value]);+ fancyDenialReasons.set(+ key,+ DenialReason[value as keyof typeof DenialReason] ?? value,+ );}});diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 1f790358f..8a847854a 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -13,6 +13,7 @@ import {mergeScrapedContent,calculateScrapeCredits,} from "./scrape";+import type { BillingMetadata } from "../services/billing/types";interface SearchOptions {query: string;@@ -37,6 +38,7 @@ interface SearchContext {requestId: string;bypassBilling?: boolean;zeroDataRetention?: boolean;+ billing?: BillingMetadata;}interface SearchExecuteResult {@@ -62,6 +64,7 @@ export async function executeSearch(requestId,bypassBilling,zeroDataRetention,+ billing,} = context;const num_results_buffer = Math.floor(limit * 2);@@ -149,6 +152,7 @@ export async function executeSearch(apiKeyId,zeroDataRetention,requestId,+ billing,};const allDocsWithCostTracking = await scrapeSearchResults(diff --git a/apps/api/src/search/scrape.test.ts b/apps/api/src/search/scrape.test.tsnew file mode 100644index 000000000..554211fbb--- /dev/null+++ b/apps/api/src/search/scrape.test.ts@@ -0,0 +1,55 @@+jest.mock("uuid", () => ({+ v7: jest.fn(() => "job-1"),+}));++import { scrapeSearchResults } from "./scrape";+import { getJobPriority } from "../lib/job-priority";+import { processJobInternal } from "../services/worker/scrape-worker";++jest.mock("../lib/job-priority", () => ({+ getJobPriority: jest.fn().mockResolvedValue(10),+}));++jest.mock("../services/worker/scrape-worker", () => ({+ processJobInternal: jest.fn().mockResolvedValue({+ markdown: "body",+ metadata: { creditsUsed: 1, statusCode: 200, proxyUsed: "basic" },+ }),+}));++describe("scrapeSearchResults", () => {+ it("preserves billing metadata on spawned scrape jobs", async () => {+ await scrapeSearchResults(+ [+ {+ url: "https://example.com",+ title: "Example",+ description: "Desc",+ },+ ],+ {+ teamId: "team-1",+ origin: "api",+ timeout: 60_000,+ scrapeOptions: {} as any,+ apiKeyId: 123,+ requestId: "req-1",+ billing: { endpoint: "agent" },+ },+ { debug: jest.fn(), info: jest.fn(), error: jest.fn() } as any,+ null as any,+ );++ expect(getJobPriority).toHaveBeenCalledWith({+ team_id: "team-1",+ basePriority: 10,+ });+ expect(processJobInternal).toHaveBeenCalledWith(+ expect.objectContaining({+ data: expect.objectContaining({+ billing: { endpoint: "agent" },+ }),+ }),+ );+ });+});diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.tsindex 731b7eb98..41081098b 100644--- a/apps/api/src/search/scrape.ts+++ b/apps/api/src/search/scrape.ts@@ -8,6 +8,7 @@ import { NuQJob } from "../services/worker/nuq";import { processJobInternal } from "../services/worker/scrape-worker";import { ScrapeJobData } from "../types";import { SearchV2Response } from "../lib/entities";+import type { BillingMetadata } from "../services/billing/types";export interface DocumentWithCostTracking {document: Document;@@ -35,6 +36,7 @@ interface ScrapeSearchOptions {apiKeyId: number | null;zeroDataRetention?: boolean;requestId?: string;+ billing?: BillingMetadata;}async function scrapeSearchResultDirect(@@ -83,6 +85,7 @@ async function scrapeSearchResultDirect(zeroDataRetention,apiKeyId: options.apiKeyId,requestId: options.requestId,+ billing: options.billing,},};diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.tsindex 1e104fd04..cfe251aed 100644--- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts+++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts@@ -201,6 +201,9 @@ describe("ensureTeamProvisioned", () => {// Only one getEntity call (no confirmation get).expect(mockEntityGet).toHaveBeenCalledTimes(1);expect(mockEntityCreate).toHaveBeenCalledTimes(1);+ expect(mockEntityCreate).toHaveBeenCalledWith(+ expect.objectContaining({ featureId: "TEAM" }),+ );});it("marks team as ensured on 409 conflict without a second getEntity", async () => {@@ -286,7 +289,7 @@ describe("reserveCredits", () => {const result = await svc.reserveCredits({teamId: "team-1",value: 42,- properties: { source: "test" },+ properties: { source: "test", endpoint: "extract" },});expect(result).toBe(true);@@ -296,6 +299,7 @@ describe("reserveCredits", () => {(c: any[]) => c[0].featureId === "CREDITS" && c[0].value === 42,);expect(usageCall).toBeDefined();+ expect((usageCall as any[])[0].properties?.endpoint).toBe("extract");});});@@ -306,13 +310,18 @@ describe("reserveCredits", () => {describe("refundCredits", () => {it("calls track with the negated value", async () => {const svc = makeService();- await svc.refundCredits({ teamId: "team-1", value: 30 });+ await svc.refundCredits({+ teamId: "team-1",+ value: 30,+ properties: { endpoint: "extract" },+ });const refundCall = mockTrack.mock.calls.find((c: any[]) => c[0].value === -30,);expect(refundCall).toBeDefined();expect((refundCall as any[])[0].properties?.source).toBe("autumn_refund");+ expect((refundCall as any[])[0].properties?.endpoint).toBe("extract");});it("is a no-op when autumnClient is null", async () => {diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex f35934c43..ca9671447 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -13,6 +13,7 @@ import type {TrackParams,} from "./types";+const TEAM_FEATURE_ID = "TEAM";const CREDITS_FEATURE_ID = "CREDITS";/**@@ -277,10 +278,10 @@ export class AutumnService {const result = await this.createEntity({customerId: resolvedOrgId,entityId: teamId,- featureId: CREDITS_FEATURE_ID,+ featureId: TEAM_FEATURE_ID,name,});- if (result.ok || result.conflict) {+ if (result.ok || ("conflict" in result && result.conflict)) {// Entity was just created, or already existed (409 race) — either way// it's present. No need for a second getEntity confirmation call.this.ensuredTeams.add(teamId);diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex 4b65a870e..04b859208 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -6,6 +6,12 @@ import * as Sentry from "@sentry/node";import { withAuth } from "../../lib/withAuth";import { setCachedACUC, setCachedACUCTeam } from "../../controllers/auth";import { autumnService } from "../autumn/autumn.service";+import {+ resolveBillingMetadata,+ toAutumnBillingProperties,+ type BillingEndpoint,+ type BillingMetadata,+} from "./types";// Configuration constantsconst BATCH_KEY = "billing_batch";@@ -19,6 +25,8 @@ interface BillingOperation {team_id: string;subscription_id: string | null;credits: number;+ billing?: BillingMetadata;+ endpoint?: BillingEndpoint;is_extract: boolean;timestamp: string;api_key_id: number | null;@@ -31,6 +39,7 @@ interface GroupedBillingOperation {team_id: string;subscription_id: string | null;total_credits: number;+ billing: BillingMetadata;is_extract: boolean;api_key_id: number | null;operations: BillingOperation[];@@ -93,13 +102,19 @@ export async function processBillingBatch() {const groupedOperations = new Map<string, GroupedBillingOperation>();for (const op of operations) {- const key = `${op.team_id}:${op.subscription_id ?? "null"}:${op.is_extract}:${op.api_key_id}`;+ const billing = resolveBillingMetadata({+ billing:+ op.billing ?? (op.endpoint ? { endpoint: op.endpoint } : undefined),+ isExtract: op.is_extract,+ });+ const key = `${op.team_id}:${op.subscription_id ?? "null"}:${billing.endpoint}:${op.is_extract}:${op.api_key_id}`;if (!groupedOperations.has(key)) {groupedOperations.set(key, {team_id: op.team_id,subscription_id: op.subscription_id,total_credits: 0,+ billing,is_extract: op.is_extract,api_key_id: op.api_key_id,operations: [],@@ -119,6 +134,7 @@ export async function processBillingBatch() {team_id: group.team_id,subscription_id: group.subscription_id,total_credits: group.total_credits,+ billing: group.billing,operation_count: group.operations.length,is_extract: group.is_extract,},@@ -154,7 +170,11 @@ export async function processBillingBatch() {if (!billingResult.success) {logger.warn(`⚠️ Billing returned success: false for team ${group.team_id}, skipping Autumn tracking`,- { billingResult, team_id: group.team_id, credits: group.total_credits },+ {+ billingResult,+ team_id: group.team_id,+ credits: group.total_credits,+ },);// Refund only the credits that were actually reserved in Autumn.if (reservedCredits > 0) {@@ -163,6 +183,7 @@ export async function processBillingBatch() {value: reservedCredits,properties: {source: "processBillingBatch_failure",+ ...toAutumnBillingProperties(group.billing),apiKeyId: group.api_key_id,subscriptionId: group.subscription_id,},@@ -183,6 +204,7 @@ export async function processBillingBatch() {value: unreservedCredits,properties: {source: "processBillingBatch",+ ...toAutumnBillingProperties(group.billing),apiKeyId: group.api_key_id,subscriptionId: group.subscription_id,},@@ -207,6 +229,7 @@ export async function processBillingBatch() {value: reservedCredits,properties: {source: "processBillingBatch_exception",+ ...toAutumnBillingProperties(group.billing),apiKeyId: group.api_key_id,subscriptionId: group.subscription_id,},@@ -257,6 +280,7 @@ export async function queueBillingOperation(subscription_id: string | null | undefined,credits: number,api_key_id: number | null,+ billing: BillingMetadata,is_extract: boolean = false,autumnReserved: boolean = false,) {@@ -270,6 +294,7 @@ export async function queueBillingOperation(team_id,subscription_id,credits,+ billing,is_extract,});@@ -278,6 +303,7 @@ export async function queueBillingOperation(team_id,subscription_id: subscription_id ?? null,credits,+ billing,is_extract,timestamp: new Date().toISOString(),api_key_id,@@ -393,9 +419,11 @@ async function supaBillTeam(// Fire-and-forget — a Redis failure here must not trigger a false Autumn refund// after bill_team_6 has already committed.- getRedisConnection().sadd("billed_teams", team_id).catch(err => {- _logger.warn("Failed to add team to billed_teams set", { err, team_id });- });+ getRedisConnection()+ .sadd("billed_teams", team_id)+ .catch(err => {+ _logger.warn("Failed to add team to billed_teams set", { err, team_id });+ });// Update cached ACUC to reflect the new credit usage(async () => {diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex ed0db30ab..7bcc547e8 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -9,6 +9,10 @@ import { autoCharge } from "./auto_charge";import { getValue, setValue } from "../redis";import { queueBillingOperation } from "./batch_billing";import { autumnService } from "../autumn/autumn.service";+import {+ toAutumnBillingProperties,+ type BillingMetadata,+} from "./types";import type { Logger } from "winston";/**@@ -19,6 +23,7 @@ export async function billTeam(subscription_id: string | null | undefined,credits: number,api_key_id: number | null,+ billing: BillingMetadata,logger?: Logger,) {return withAuth(@@ -27,6 +32,7 @@ export async function billTeam(subscription_id: string | null | undefined,credits: number,api_key_id: number | null,+ billing: BillingMetadata,logger: Logger | undefined,) => {// Reserve in Autumn at request time; await so autumnReserved is accurate.@@ -34,19 +40,24 @@ export async function billTeam(const autumnReserved = await autumnService.reserveCredits({teamId: team_id,value: credits,- properties: { source: "billTeam", apiKeyId: api_key_id },+ properties: {+ source: "billTeam",+ ...toAutumnBillingProperties(billing),+ apiKeyId: api_key_id,+ },});return queueBillingOperation(team_id,subscription_id,credits,api_key_id,+ billing,false,autumnReserved,);},{ success: true, message: "No DB, bypassed." },- )(team_id, subscription_id, credits, api_key_id, logger);+ )(team_id, subscription_id, credits, api_key_id, billing, logger);}type CheckTeamCreditsResponse = {diff --git a/apps/api/src/services/billing/types.ts b/apps/api/src/services/billing/types.tsnew file mode 100644index 000000000..9a461a5f0--- /dev/null+++ b/apps/api/src/services/billing/types.ts@@ -0,0 +1,43 @@+export type BillingEndpoint =+ | "agent"+ | "batch_scrape"+ | "browser"+ | "crawl"+ | "deep_research"+ | "extract"+ | "llms_txt"+ | "map"+ | "scrape"+ | "search";++export type BillingMetadata = {+ endpoint: BillingEndpoint;+};++export function resolveBillingMetadata({+ billing,+ isExtract = false,+ crawlId,+ crawlerOptions,+}: {+ billing?: BillingMetadata;+ isExtract?: boolean;+ crawlId?: string;+ crawlerOptions?: unknown;+}): BillingMetadata {+ if (billing) return billing;+ if (crawlId) {+ return {+ endpoint: crawlerOptions == null ? "batch_scrape" : "crawl",+ };+ }+ return {+ endpoint: isExtract ? "extract" : "scrape",+ };+}++export function toAutumnBillingProperties(+ billing: BillingMetadata,+): Record<string, string> {+ return { endpoint: billing.endpoint };+}diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 5b4d3a881..f1fa0dc05 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -16,6 +16,7 @@ import {queueBillingOperation,startBillingBatchProcessing,} from "../billing/batch_billing";+import { resolveBillingMetadata } from "../billing/types";import systemMonitor from "../system-monitor";import { v7 as uuidv7 } from "uuid";import {@@ -80,7 +81,7 @@ const processBillingJobInternal = async (token: string, job: Job) => {await processBillingBatch();} else if (job.name === "bill_team") {// This is an individual billing operation that should be queued for batch processing- const { team_id, subscription_id, credits, is_extract, api_key_id } =+ const { team_id, subscription_id, credits, billing, endpoint, is_extract, api_key_id } =job.data;logger.info(`Adding team ${team_id} billing operation to batch queue`, {@@ -95,6 +96,10 @@ const processBillingJobInternal = async (token: string, job: Job) => {subscription_id,credits,api_key_id ?? null,+ resolveBillingMetadata({+ billing: billing ?? (endpoint ? { endpoint } : undefined),+ isExtract: is_extract,+ }),is_extract,);} else {diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 0a53dac95..94cc00dd6 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -30,6 +30,7 @@ import {StoredCrawl,} from "../../lib/crawl-redis";import { redisEvictConnection } from "../redis";+import { resolveBillingMetadata } from "../billing/types";import {_addScrapeJobToBullMQ,addScrapeJob,@@ -115,11 +116,17 @@ async function billScrapeJob(config.USE_DB_AUTHENTICATION) {try {+ const billing = resolveBillingMetadata({+ billing: job.data.billing,+ crawlId: job.data.crawl_id,+ crawlerOptions: job.data.crawlerOptions,+ });const billingJobId = uuidv7();logger.debug(`Adding billing job to queue for team ${job.data.team_id}`,{billingJobId,+ billing,credits: creditsToBeBilled,is_extract: false,},@@ -132,6 +139,7 @@ async function billScrapeJob(team_id: job.data.team_id,subscription_id: undefined,credits: creditsToBeBilled,+ billing,is_extract: false,timestamp: new Date().toISOString(),originating_job_id: job.id,@@ -408,6 +416,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {integration: job.data.integration,crawl_id: job.data.crawl_id,requestId: job.data.requestId,+ billing: job.data.billing,webhook: job.data.webhook,v1: job.data.v1,zeroDataRetention: job.data.zeroDataRetention,@@ -797,6 +806,7 @@ async function addKickoffSitemapJob(integration: sourceJob.data.integration,crawl_id: sourceJob.data.crawl_id,requestId: sourceJob.data.requestId,+ billing: sourceJob.data.billing,webhook: sourceJob.data.webhook,v1: sourceJob.data.v1,apiKeyId: sourceJob.data.apiKeyId,@@ -849,6 +859,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {integration: job.data.integration,crawl_id: job.data.crawl_id,requestId: job.data.requestId,+ billing: job.data.billing,webhook: job.data.webhook,v1: job.data.v1,isCrawlSourceScrape: true,@@ -936,6 +947,7 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {integration: job.data.integration,crawl_id: job.data.crawl_id,requestId: job.data.requestId,+ billing: job.data.billing,sitemapped: true,webhook: job.data.webhook,v1: job.data.v1,@@ -1048,6 +1060,7 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {integration: job.data.integration,crawl_id: job.data.crawl_id,requestId: job.data.requestId,+ billing: job.data.billing,sitemapped: true,webhook: job.data.webhook,v1: job.data.v1,diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 9146eb803..b933da50f 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -9,6 +9,7 @@ import { AuthCreditUsageChunk } from "./controllers/v1/types";import { ExtractorOptions, Document } from "./lib/entities";import { InternalOptions } from "./scraper/scrapeURL";import type { CostTracking } from "./lib/cost-tracking";+import type { BillingMetadata } from "./services/billing/types";import { webhookSchema } from "./services/webhook/schema";import { SerializedTraceContext } from "./lib/otel-tracer";@@ -16,6 +17,7 @@ type ScrapeJobCommon = {concurrencyLimited?: boolean;team_id: string;zeroDataRetention: boolean;+ billing?: BillingMetadata;traceContext?: SerializedTraceContext;skipNuq?: boolean;requestId?: string;