Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): propagate api_key_id towards billing function (#2049)
Gergő Móricz4d45f3b890ab
diff --git a/apps/api/src/__tests__/queue-concurrency-integration.test.ts b/apps/api/src/__tests__/queue-concurrency-integration.test.tsindex 2d80c48fd..595079750 100644--- a/apps/api/src/__tests__/queue-concurrency-integration.test.ts+++ b/apps/api/src/__tests__/queue-concurrency-integration.test.ts@@ -67,6 +67,7 @@ describe("Queue Concurrency Integration", () => {scrapeOptions: defaultScrapeOptions,crawlerOptions: null,zeroDataRetention: false,+ apiKeyId: null,} as WebScraperOptions;it("should add job directly to BullMQ when under concurrency limit", async () => {diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 0b5c26594..71cded698 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -74,6 +74,7 @@ export async function setCachedACUC(const mockPreviewACUC: (team_id: string, is_extract: boolean) => AuthCreditUsageChunk = (team_id, is_extract) => ({api_key: "preview",+ api_key_id: 0,team_id,sub_id: null,sub_current_period_start: null,@@ -110,6 +111,7 @@ const mockPreviewACUC: (team_id: string, is_extract: boolean) => AuthCreditUsageconst mockACUC: () => AuthCreditUsageChunk = () => ({api_key: "bypass",+ api_key_id: 0,team_id: "bypass",sub_id: "bypass",sub_current_period_start: new Date().toISOString(),diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex cf0421330..e351caf62 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -213,6 +213,7 @@ export async function crawlController(req: Request, res: Response) {crawl_id: id,sitemapped: true,zeroDataRetention: false, // not supported on v0+ apiKeyId: chunk?.api_key_id ?? null,},opts: {jobId: uuid,@@ -257,6 +258,7 @@ export async function crawlController(req: Request, res: Response) {integration: req.body.integration,crawl_id: id,zeroDataRetention: false, // not supported on v0+ apiKeyId: chunk?.api_key_id ?? null,},{priority: 15, // prioritize request 0 of crawl jobs same as scrape jobsdiff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 8435da31e..f1e0aa735 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -44,6 +44,7 @@ export async function scrapeHelper(extractorOptions: ExtractorOptions,timeout: number,flags: TeamFlags,+ apiKeyId: number | null,): Promise<{success: boolean;error?: string;@@ -87,6 +88,7 @@ export async function scrapeHelper(is_scrape: true,startTime: Date.now(),zeroDataRetention: false, // not supported on v0+ apiKeyId,},{},jobId,@@ -242,6 +244,7 @@ export async function scrapeController(req: Request, res: Response) {extractorOptions,timeout,chunk?.flags ?? null,+ chunk?.api_key_id ?? null,);const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - startTime) / 1000;@@ -270,7 +273,7 @@ export async function scrapeController(req: Request, res: Response) {}if (creditsToBeBilled > 0) {// billing for doc done on queue end, bill only for llm extraction- billTeam(team_id, chunk?.sub_id, creditsToBeBilled, logger).catch(+ billTeam(team_id, chunk?.sub_id, creditsToBeBilled, chunk?.api_key_id ?? null, logger).catch((error) => {logger.error(`Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 6690f1f9b..9b3e82c2b 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -37,6 +37,7 @@ export async function searchHelper(pageOptions: PageOptions,searchOptions: SearchOptions,flags: TeamFlags,+ api_key_id: number | null,): Promise<{success: boolean;error?: string;@@ -82,7 +83,7 @@ export async function searchHelper();if (justSearch) {- billTeam(team_id, subscription_id, res.length).catch((error) => {+ billTeam(team_id, subscription_id, res.length, api_key_id, logger).catch((error) => {logger.error(`Failed to bill team ${team_id} for ${res.length} credits: ${error}`,);@@ -117,6 +118,7 @@ export async function searchHelper(internalOptions,startTime: Date.now(),zeroDataRetention: false, // not supported on v0+ apiKeyId: api_key_id,},opts: {jobId: uuid,@@ -218,6 +220,7 @@ export async function searchController(req: Request, res: Response) {pageOptions,searchOptions,chunk?.flags ?? null,+ chunk?.api_key_id ?? null,);const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - startTime) / 1000;diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex fef7033be..d05165e12 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -148,6 +148,7 @@ export async function batchScrapeController(webhook: req.body.webhook,internalOptions: sc.internalOptions,zeroDataRetention: zeroDataRetention ?? false,+ apiKeyId: req.acuc?.api_key_id ?? null,},opts: {jobId: uuidv4(),diff --git a/apps/api/src/controllers/v1/concurrency-check.ts b/apps/api/src/controllers/v1/concurrency-check.tsindex a6313c72c..517aed515 100644--- a/apps/api/src/controllers/v1/concurrency-check.ts+++ b/apps/api/src/controllers/v1/concurrency-check.ts@@ -22,6 +22,6 @@ export async function concurrencyCheckController(return res.status(200).json({success: true,concurrency: activeJobsOfTeam.length,- maxConcurrency: req.acuc.concurrency,+ maxConcurrency: req.acuc?.concurrency ?? 0,});}diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 62df33703..65ec08faf 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -132,6 +132,7 @@ export async function crawlController(webhook: req.body.webhook,v1: true,zeroDataRetention: zeroDataRetention || false,+ apiKeyId: req.acuc?.api_key_id ?? null,},{},crypto.randomUUID(),diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex 525db6b6c..dd2515e9d 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -56,7 +56,8 @@ export async function deepResearchController(const jobData = {request: req.body,teamId: req.auth.team_id,- subId: req.acuc?.sub_id,+ subId: req.acuc?.sub_id ?? undefined,+ apiKeyId: req.acuc?.api_key_id ?? null,researchId,};diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 0e1fffc95..833191d09 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -31,12 +31,14 @@ export async function oldExtract(request: req.body,teamId: req.auth.team_id,subId: req.acuc?.sub_id ?? undefined,+ apiKeyId: req.acuc?.api_key_id ?? null,});} else {result = await performExtraction_F0(extractId, {request: req.body,teamId: req.auth.team_id,subId: req.acuc?.sub_id ?? undefined,+ apiKeyId: req.acuc?.api_key_id ?? null,});}@@ -103,6 +105,7 @@ export async function extractController(subId: req.acuc?.sub_id,extractId,agent: req.body.agent,+ apiKeyId: req.acuc?.api_key_id ?? null,};if (diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex 6834b819f..97c924620 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -34,7 +34,8 @@ export async function generateLLMsTextController(const jobData = {request: req.body,teamId: req.auth.team_id,- subId: req.acuc?.sub_id,+ subId: req.acuc?.sub_id ?? undefined,+ apiKeyId: req.acuc?.api_key_id ?? null,generationId,};diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 3cb2ef00c..0500c90c9 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -373,7 +373,7 @@ export async function mapController(}// Bill the team- billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {+ billTeam(req.auth.team_id, req.acuc?.sub_id, 1, req.acuc?.api_key_id ?? null).catch((error) => {logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,);diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex b5b82ca43..6210506f9 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -80,6 +80,7 @@ export async function scrapeController(integration: req.body.integration,startTime,zeroDataRetention: zeroDataRetention ?? false,+ apiKeyId: req.acuc?.api_key_id ?? null,},{},jobId,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 6f5a0b7ea..b64a84c74 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -41,6 +41,7 @@ export async function searchAndScrapeSearchResult(origin: string;timeout: number;scrapeOptions: ScrapeOptions;+ apiKeyId: number | null;},logger: Logger,flags: TeamFlags,@@ -80,6 +81,7 @@ async function scrapeSearchResult(origin: string;timeout: number;scrapeOptions: ScrapeOptions;+ apiKeyId: number | null;},logger: Logger,flags: TeamFlags,@@ -124,6 +126,7 @@ async function scrapeSearchResult(is_scrape: true,startTime: Date.now(),zeroDataRetention,+ apiKeyId: options.apiKeyId,},{},jobId,@@ -291,6 +294,7 @@ export async function searchController(origin: req.body.origin,timeout: req.body.timeout,scrapeOptions: req.body.scrapeOptions,+ apiKeyId: req.acuc?.api_key_id ?? null,},logger,req.acuc?.flags ?? null,@@ -357,8 +361,9 @@ export async function searchController(if (!isSearchPreview) {billTeam(req.auth.team_id,- req.acuc?.sub_id,+ req.acuc?.sub_id ?? undefined,credits_billed,+ req.acuc?.api_key_id ?? null,).catch((error) => {logger.error(`Failed to bill team ${req.auth.team_id} for ${responseData.data.length} credits: ${error}`,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 2e0055888..e9beab961 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1073,6 +1073,7 @@ type Account = {export type AuthCreditUsageChunk = {api_key: string;+ api_key_id: number;team_id: string;sub_id: string | null;sub_current_period_start: string | null;@@ -1130,14 +1131,6 @@ export interface RequestWithMaybeACUC<acuc?: AuthCreditUsageChunk;}-export interface RequestWithACUC<- ReqParams = {},- ReqBody = undefined,- ResBody = undefined,-> extends Request<ReqParams, ReqBody, ResBody> {- acuc: AuthCreditUsageChunk;-}-export interface RequestWithAuth<ReqParams = {},ReqBody = undefined,@@ -1160,7 +1153,7 @@ export interface RequestWithAuth<ReqParams = {},ReqBody = undefined,ResBody = undefined,-> extends RequestWithACUC<ReqParams, ReqBody, ResBody> {+> extends RequestWithMaybeACUC<ReqParams, ReqBody, ResBody> {auth: AuthObject;account?: Account;}diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 980ce770c..b82bece72 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -30,46 +30,6 @@ interface DocumentWithCostTracking {costTracking: ReturnType<typeof CostTracking.prototype.toJSON>;}-// Used for deep research-export async function searchAndScrapeX402SearchResult(- query: string,- options: {- teamId: string;- origin: string;- timeout: number;- scrapeOptions: ScrapeOptions;- },- logger: Logger,- flags: TeamFlags,-): Promise<DocumentWithCostTracking[]> {- try {- const searchResults = await search({- query,- logger,- num_results: 5,- });-- const documentsWithCostTracking = await Promise.all(- searchResults.map((result) =>- scrapeX402SearchResult(- {- url: result.url,- title: result.title,- description: result.description,- },- options,- logger,- flags,- ),- ),- );-- return documentsWithCostTracking;- } catch (error) {- return [];- }-}-async function scrapeX402SearchResult(searchResult: { url: string; title: string; description: string },options: {@@ -77,6 +37,7 @@ async function scrapeX402SearchResult(origin: string;timeout: number;scrapeOptions: ScrapeOptions;+ apiKeyId: number | null;},logger: Logger,flags: TeamFlags,@@ -119,6 +80,7 @@ async function scrapeX402SearchResult(is_scrape: true,startTime: Date.now(),zeroDataRetention,+ apiKeyId: options.apiKeyId,},{},jobId,@@ -286,6 +248,7 @@ export async function x402SearchController(origin: req.body.origin,timeout: req.body.timeout,scrapeOptions: req.body.scrapeOptions,+ apiKeyId: req.acuc?.api_key_id ?? null,},logger,req.acuc?.flags ?? null,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 98576dafe..cea446866 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -148,6 +148,7 @@ export async function batchScrapeController(webhook: req.body.webhook,internalOptions: sc.internalOptions,zeroDataRetention,+ apiKeyId: req.acuc?.api_key_id ?? null,},opts: {jobId: uuidv4(),diff --git a/apps/api/src/controllers/v2/concurrency-check.ts b/apps/api/src/controllers/v2/concurrency-check.tsindex 27a71d586..548ced458 100644--- a/apps/api/src/controllers/v2/concurrency-check.ts+++ b/apps/api/src/controllers/v2/concurrency-check.ts@@ -1,9 +1,9 @@import {- AuthCreditUsageChunkFromTeam,ConcurrencyCheckParams,ConcurrencyCheckResponse,RequestWithAuth,} from "./types";+import { AuthCreditUsageChunkFromTeam } from "../v1/types";import { Response } from "express";import { redisEvictConnection } from "../../../src/services/redis";import { getACUCTeam } from "../auth";@@ -14,6 +14,13 @@ export async function concurrencyCheckController(req: RequestWithAuth<ConcurrencyCheckParams, undefined, undefined>,res: Response<ConcurrencyCheckResponse>,) {+ if (!req.acuc) {+ return res.status(401).json({+ success: false,+ error: "Unauthorized",+ });+ }+let otherACUC: AuthCreditUsageChunkFromTeam | null = null;if (!req.acuc.is_extract) {otherACUC = await getACUCTeam(req.auth.team_id, false, true, RateLimiterMode.Extract);diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 9e68a5099..b471040f4 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -139,7 +139,7 @@ export async function crawlController(},team_id: req.auth.team_id,createdAt: Date.now(),- maxConcurrency: req.body.maxConcurrency !== undefined ? Math.min(req.body.maxConcurrency, req.acuc.concurrency) : undefined,+ maxConcurrency: req.body.maxConcurrency !== undefined ? (req.acuc?.concurrency !== undefined ? Math.min(req.body.maxConcurrency, req.acuc.concurrency) : req.body.maxConcurrency) : undefined,zeroDataRetention,};@@ -173,6 +173,7 @@ export async function crawlController(webhook: req.body.webhook,v1: true,zeroDataRetention: zeroDataRetention || false,+ apiKeyId: req.acuc?.api_key_id ?? null,},{},crypto.randomUUID(),diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex d5f0956bc..28fadcd4b 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -362,7 +362,7 @@ export async function mapController(}// Bill the team- billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {+ billTeam(req.auth.team_id, req.acuc?.sub_id ?? undefined, 1, req.acuc?.api_key_id ?? null).catch((error) => {logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`,);diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex b34fe5a31..232d0cb34 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -82,6 +82,7 @@ export async function scrapeController(integration: req.body.integration,startTime,zeroDataRetention,+ apiKeyId: req.acuc?.api_key_id ?? null,},{},jobId,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 84d7a638c..6d2f346e7 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -47,6 +47,7 @@ async function startScrapeJob(timeout: number;scrapeOptions: ScrapeOptions;bypassBilling?: boolean;+ apiKeyId: number | null;},logger: Logger,flags: TeamFlags,@@ -85,6 +86,7 @@ async function startScrapeJob(is_scrape: options.bypassBilling ?? false,startTime: Date.now(),zeroDataRetention,+ apiKeyId: options.apiKeyId,},{},jobId,@@ -103,6 +105,7 @@ async function scrapeSearchResult(timeout: number;scrapeOptions: ScrapeOptions;bypassBilling?: boolean;+ apiKeyId: number | null;},logger: Logger,flags: TeamFlags,@@ -310,6 +313,7 @@ export async function searchController(timeout: req.body.timeout,scrapeOptions: req.body.scrapeOptions,bypassBilling: !isAsyncScraping, // Async mode bills per job, sync mode bills manually+ apiKeyId: req.acuc?.api_key_id ?? null,};const directToBullMQ = (req.acuc?.price_credits ?? 0) <= 3000;@@ -530,9 +534,8 @@ export async function searchController(// - For async scraping: Jobs handle their own billing// - For no scraping: Bill based on search results countif (!isSearchPreview && (!shouldScrape || (shouldScrape && !isAsyncScraping))) {- billTeam(req.auth.team_id, req.acuc?.sub_id, credits_billed).catch((error) => {+ billTeam(req.auth.team_id, req.acuc?.sub_id ?? undefined, credits_billed, req.acuc?.api_key_id ?? null).catch((error) => {logger.error(`Failed to bill team ${req.acuc?.sub_id} for ${credits_billed} credits: ${error}`);-});}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 6d1c002dc..4104edd53 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -9,7 +9,7 @@ import {Document as V0Document,WebSearchResult,} from "../../lib/entities";-import { agentOptionsExtract, ScrapeOptions as V1ScrapeOptions } from "../v1/types";+import { agentOptionsExtract, AuthCreditUsageChunk, ScrapeOptions as V1ScrapeOptions } from "../v1/types";import type { InternalOptions } from "../../scraper/scrapeURL";import { ErrorCodes } from "../../lib/error";import Ajv from "ajv";@@ -899,44 +899,6 @@ type Account = {remainingCredits: number;};-export type AuthCreditUsageChunk = {- api_key: string;- team_id: string;- sub_id: string | null;- sub_current_period_start: string | null;- sub_current_period_end: string | null;- sub_user_id: string | null;- price_id: string | null;- price_credits: number; // credit limit with assoicated price, or free_credits (500) if free plan- price_should_be_graceful: boolean;- credits_used: number;- coupon_credits: number; // do not rely on this number to be up to date after calling a billTeam- adjusted_credits_used: number; // credits this period minus coupons used- remaining_credits: number;- total_credits_sum: number;- plan_priority: {- bucketLimit: number;- planModifier: number;- };- rate_limits: {- crawl: number;- scrape: number;- search: number;- map: number;- extract: number;- preview: number;- crawlStatus: number;- extractStatus: number;- extractAgentPreview?: number;- scrapeAgentPreview?: number;- };- concurrency: number;- flags: TeamFlags;-- // appended on JS-side- is_extract?: boolean;-};-export type TeamFlags = {ignoreRobots?: boolean;unblockedDomains?: string[];@@ -948,8 +910,6 @@ export type TeamFlags = {crawlTtlHours?: number;} | null;-export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;-export interface RequestWithMaybeACUC<ReqParams = {},ReqBody = undefined,@@ -958,23 +918,6 @@ export interface RequestWithMaybeACUC<acuc?: AuthCreditUsageChunk;}-export interface RequestWithACUC<- ReqParams = {},- ReqBody = undefined,- ResBody = undefined,-> extends Request<ReqParams, ReqBody, ResBody> {- acuc: AuthCreditUsageChunk;-}--export interface RequestWithAuth<- ReqParams = {},- ReqBody = undefined,- ResBody = undefined,-> extends Request<ReqParams, ReqBody, ResBody> {- auth: AuthObject;- account?: Account;-}-export interface RequestWithMaybeAuth<ReqParams = {},ReqBody = undefined,@@ -988,7 +931,7 @@ export interface RequestWithAuth<ReqParams = {},ReqBody = undefined,ResBody = undefined,-> extends RequestWithACUC<ReqParams, ReqBody, ResBody> {+> extends RequestWithMaybeACUC<ReqParams, ReqBody, ResBody> {auth: AuthObject;account?: Account;}diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.tsindex e8ebbe6d4..4100cd03c 100644--- a/apps/api/src/lib/deep-research/deep-research-service.ts+++ b/apps/api/src/lib/deep-research/deep-research-service.ts@@ -7,7 +7,7 @@ import { billTeam } from "../../services/billing/credit_billing";import { ExtractOptions } from "../../controllers/v1/types";import { CostTracking } from "../extract/extraction-service";import { getACUCTeam } from "../../controllers/auth";-interface DeepResearchServiceOptions {+export interface DeepResearchServiceOptions {researchId: string;teamId: string;query: string;@@ -19,11 +19,12 @@ interface DeepResearchServiceOptions {formats: string[];jsonOptions: ExtractOptions;subId?: string;+ apiKeyId: number | null;}export async function performDeepResearch(options: DeepResearchServiceOptions) {const costTracking = new CostTracking();- const { researchId, teamId, timeLimit, subId, maxUrls } = options;+ const { researchId, teamId, timeLimit, subId, maxUrls, apiKeyId } = options;const startTime = Date.now();let currentTopic = options.query;let urlsAnalyzed = 0;@@ -140,6 +141,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {storeInCache: true,proxy: "basic",},+ apiKeyId: apiKeyId,}, logger, acuc?.flags ?? null);return response.length > 0 ? response : [];});@@ -395,7 +397,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {json: finalAnalysisJson,});// Bill team for usage based on URLs analyzed- billTeam(teamId, subId, credits_billed, logger).catch(+ billTeam(teamId, subId, credits_billed, apiKeyId, logger).catch((error) => {logger.error(`Failed to bill team ${teamId} for ${urlsAnalyzed} URLs analyzed`, { teamId, count: urlsAnalyzed, error },diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 1b84733b2..db35c56c4 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -13,6 +13,7 @@ interface ScrapeDocumentOptions {timeout: number;isSingleUrl?: boolean;flags: TeamFlags | null;+ apiKeyId: number | null;}export async function scrapeDocument(@@ -58,6 +59,7 @@ export async function scrapeDocument(from_extract: true,startTime: Date.now(),zeroDataRetention: false, // not supported+ apiKeyId: options.apiKeyId,},{},jobId,diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 4d5f2bf20..3b507f466 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -14,7 +14,6 @@ import {} from "../../scraper/scrapeURL/transformers/llmExtract";import { billTeam } from "../../services/billing/credit_billing";import { logJob } from "../../services/logging/log_job";-import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { dereferenceSchema } from "./helpers/dereference-schema";import { spreadSchemas } from "./helpers/spread-schemas";import { transformArrayToObject } from "./helpers/transform-array-to-obj";@@ -47,6 +46,7 @@ interface ExtractServiceOptions {cacheMode?: "load" | "save" | "direct";cacheKey?: string;agent?: boolean;+ apiKeyId: number | null;}export interface ExtractResult {@@ -126,7 +126,7 @@ export async function performExtraction(extractId: string,options: ExtractServiceOptions,): Promise<ExtractResult> {- const { request, teamId, subId } = options;+ const { request, teamId, subId, apiKeyId } = options;const urlTraces: URLTrace[] = [];let docsMap: Map<string, Document> = new Map();let singleAnswerCompletions: completions | null = null;@@ -208,7 +208,7 @@ export async function performExtraction(zeroDataRetention: false, // not supported});- await billTeam(teamId, subId, tokens_billed, logger, true).catch((error) => {+ await billTeam(teamId, subId, tokens_billed, apiKeyId, logger, true).catch((error) => {logger.error(`Failed to bill team ${teamId} for thinking tokens: ${error}`,);@@ -432,6 +432,7 @@ export async function performExtraction(origin: "extract",timeout,flags: acuc?.flags ?? null,+ apiKeyId,},urlTraces,logger.child({@@ -719,7 +720,7 @@ export async function performExtraction(cost_tracking: costTracking,zeroDataRetention: false, // not supported});- await billTeam(teamId, subId, tokens_billed, logger, true).catch((error) => {+ await billTeam(teamId, subId, tokens_billed, apiKeyId, logger, true).catch((error) => {logger.error(`Failed to bill team ${teamId} for thinking tokens: ${error}`,);@@ -773,6 +774,7 @@ export async function performExtraction(origin: "extract",timeout,flags: acuc?.flags ?? null,+ apiKeyId,},urlTraces,logger.child({@@ -829,7 +831,7 @@ export async function performExtraction(cost_tracking: costTracking,zeroDataRetention: false, // not supported});- await billTeam(teamId, subId, tokens_billed, logger, true).catch((error) => {+ await billTeam(teamId, subId, tokens_billed, apiKeyId, logger, true).catch((error) => {logger.error(`Failed to bill team ${teamId} for thinking tokens: ${error}`,);@@ -847,7 +849,7 @@ export async function performExtraction(// All urls are invalidlogger.error("All provided URLs are invalid!");const tokens_billed = 300 + calculateThinkingCost(costTracking);- await billTeam(teamId, subId, tokens_billed, logger, true).catch((error) => {+ await billTeam(teamId, subId, tokens_billed, apiKeyId, logger, true).catch((error) => {logger.error(`Failed to bill team ${teamId} for thinking tokens: ${error}`,);@@ -1036,7 +1038,7 @@ export async function performExtraction(}// Bill team for usage- await billTeam(teamId, subId, tokensToBill, logger, true).catch((error) => {+ await billTeam(teamId, subId, tokensToBill, apiKeyId, logger, true).catch((error) => {logger.error(`Failed to bill team ${teamId} for ${tokensToBill} tokens: ${error}`,);@@ -1109,7 +1111,7 @@ export async function performExtraction(};} catch (error) {const tokens_billed = 300 + calculateThinkingCost(costTracking);- await billTeam(teamId, subId, tokens_billed, logger, true).catch((error) => {+ await billTeam(teamId, subId, tokens_billed, apiKeyId, logger, true).catch((error) => {logger.error(`Failed to bill team ${teamId} for thinking tokens: ${error}`,);diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex a4b524a64..3d3e3fdd3 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -13,6 +13,7 @@ interface ScrapeDocumentOptions {timeout: number;isSingleUrl?: boolean;flags: TeamFlags | null;+ apiKeyId: number | null;}export async function scrapeDocument_F0(@@ -59,6 +60,7 @@ export async function scrapeDocument_F0(from_extract: true,startTime: Date.now(),zeroDataRetention: false, // not supported+ apiKeyId: options.apiKeyId,},{},jobId,diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 212af66c2..7ce1a51c6 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -8,7 +8,6 @@ import {import { scrapeDocument_F0 } from "./document-scraper-f0";import { billTeam } from "../../../services/billing/credit_billing";import { logJob } from "../../../services/logging/log_job";- import { _addScrapeJobToBullMQ } from "../../../services/queue-jobs";import { spreadSchemas_F0 } from "./helpers/spread-schemas-f0";import Ajv from "ajv";const ajv = new Ajv();@@ -42,6 +41,7 @@ import { langfuse } from "../../../services/langfuse";subId?: string;cacheMode?: "load" | "save" | "direct";cacheKey?: string;+ apiKeyId: number | null;}export interface ExtractResult {@@ -70,7 +70,7 @@ import { langfuse } from "../../../services/langfuse";extractId: string,options: ExtractServiceOptions,): Promise<ExtractResult> {- const { request, teamId, subId } = options;+ const { request, teamId, subId, apiKeyId } = options;const urlTraces: URLTrace[] = [];let docsMap: Map<string, Document> = new Map();let singleAnswerCompletions: completions | null = null;@@ -358,6 +358,7 @@ import { langfuse } from "../../../services/langfuse";origin: "extract",timeout,flags: acuc?.flags ?? null,+ apiKeyId,},urlTraces,logger.child({@@ -640,6 +641,7 @@ import { langfuse } from "../../../services/langfuse";origin: "extract",timeout,flags: acuc?.flags ?? null,+ apiKeyId,},urlTraces,logger.child({@@ -860,7 +862,7 @@ import { langfuse } from "../../../services/langfuse";}// Bill team for usage- billTeam(teamId, subId, tokensToBill, logger, true).catch((error) => {+ billTeam(teamId, subId, tokensToBill, apiKeyId, logger, true).catch((error) => {logger.error(`Failed to bill team ${teamId} for ${tokensToBill} tokens: ${error}`,);diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.tsindex 6627abc1e..d37350ad1 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts@@ -16,6 +16,7 @@ import { getACUCTeam } from "../../controllers/auth";interface GenerateLLMsTextServiceOptions {generationId: string;teamId: string;+ apiKeyId: number | null;url: string;maxUrls: number;showFullText: boolean;@@ -64,7 +65,7 @@ function limitLlmsTxtEntries(llmstxt: string, maxEntries: number): string {export async function performGenerateLlmsTxt(options: GenerateLLMsTextServiceOptions,) {- const { generationId, teamId, url, maxUrls = 100, showFullText, cache = true, subId } =+ const { generationId, teamId, url, maxUrls = 100, showFullText, cache = true, subId, apiKeyId } =options;const startTime = Date.now();const logger = _logger.child({@@ -148,6 +149,7 @@ export async function performGenerateLlmsTxt(timeout: 30000,isSingleUrl: true,flags: acuc?.flags ?? null,+ apiKeyId,},[],logger,@@ -252,7 +254,7 @@ export async function performGenerateLlmsTxt(});// Bill team for usage- billTeam(teamId, subId, urls.length, logger).catch((error) => {+ billTeam(teamId, subId, urls.length, apiKeyId, logger).catch((error) => {logger.error(`Failed to bill team ${teamId} for ${urls.length} urls`, {teamId,count: urls.length,diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 118da61f0..c69e58c01 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -2,10 +2,7 @@ import { Job } from "bullmq";import {WebScraperOptions,RunWebScraperParams,- RunWebScraperResult,} from "../types";-import { billTeam } from "../services/billing/credit_billing";-import { Document, TeamFlags } from "../controllers/v1/types";import { supabase_service } from "../services/supabase";import { logger as _logger } from "../lib/logger";import { configDotenv } from "dotenv";@@ -13,7 +10,6 @@ import {scrapeURL,ScrapeUrlResponse,} from "../scraper/scrapeURL";-import { Engine } from "../scraper/scrapeURL/engines";import { CostTracking } from "../lib/extract/extraction-service";configDotenv();diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex cac0dfca5..e2719d8b7 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -2,9 +2,9 @@ import { NextFunction, Request, Response } from "express";// import { crawlStatusController } from "../../src/controllers/v1/crawl-status";import {isAgentExtractModelValid,- RequestWithACUC,RequestWithAuth,RequestWithMaybeAuth,+ RequestWithMaybeACUC,} from "../controllers/v1/types";import { RateLimiterMode } from "../types";import { authenticateUser } from "../controllers/auth";@@ -31,7 +31,7 @@ export function checkCreditsMiddleware(}}const { success, remainingCredits, chunk } = await checkTeamCredits(- req.acuc,+ req.acuc ?? null,req.auth.team_id,minimum ?? 1,);@@ -55,7 +55,7 @@ export function checkCreditsMiddleware(return next();}- const currencyName = req.acuc.is_extract ? "tokens" : "credits";+ const currencyName = req.acuc?.is_extract ? "tokens" : "credits";logger.error(`Insufficient ${currencyName}: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`,{@@ -164,7 +164,7 @@ export function idempotencyMiddleware(})().catch((err) => next(err));}export function blocklistMiddleware(- req: RequestWithACUC<any, any, any>,+ req: RequestWithMaybeACUC<any, any, any>,res: Response,next: NextFunction,) {diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex 679bf844e..a670dd038 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -20,6 +20,7 @@ interface BillingOperation {credits: number;is_extract: boolean;timestamp: string;+ api_key_id: number | null;}// Grouped billing operations for batch processing@@ -28,6 +29,7 @@ interface GroupedBillingOperation {subscription_id: string | null;total_credits: number;is_extract: boolean;+ api_key_id: number | null;operations: BillingOperation[];}@@ -79,7 +81,7 @@ export async function processBillingBatch() {const groupedOperations = new Map<string, GroupedBillingOperation>();for (const op of operations) {- const key = `${op.team_id}:${op.subscription_id ?? 'null'}:${op.is_extract}`;+ const key = `${op.team_id}:${op.subscription_id ?? 'null'}:${op.is_extract}:${op.api_key_id}`;if (!groupedOperations.has(key)) {groupedOperations.set(key, {@@ -87,6 +89,7 @@ export async function processBillingBatch() {subscription_id: op.subscription_id,total_credits: 0,is_extract: op.is_extract,+ api_key_id: op.api_key_id,operations: []});}@@ -118,6 +121,7 @@ export async function processBillingBatch() {group.team_id,group.subscription_id,group.total_credits,+ group.api_key_id,logger,group.is_extract);@@ -170,7 +174,8 @@ export async function queueBillingOperation(team_id: string,subscription_id: string | null | undefined,credits: number,- is_extract: boolean = false+ api_key_id: number | null,+ is_extract: boolean = false,) {// Skip queuing for preview teamsif (team_id === "preview" || team_id.startsWith("preview_")) {@@ -191,7 +196,8 @@ export async function queueBillingOperation(subscription_id: subscription_id ?? null,credits,is_extract,- timestamp: new Date().toISOString()+ timestamp: new Date().toISOString(),+ api_key_id,};// Add operation to Redis list@@ -263,6 +269,7 @@ async function supaBillTeam(team_id: string,subscription_id: string | null | undefined,credits: number,+ api_key_id: number | null,__logger?: any,is_extract: boolean = false,) {@@ -281,11 +288,12 @@ async function supaBillTeam(_logger.info(`Batch billing team ${team_id} for ${credits} credits`);// Perform the actual database operation- const { data, error } = await supabase_service.rpc("bill_team_4_tally", {+ const { data, error } = await supabase_service.rpc("bill_team_5", {_team_id: team_id,sub_id: subscription_id ?? null,fetch_subscription: subscription_id === undefined,credits,+ i_api_key_id: api_key_id ?? null,is_extract_param: is_extract,});diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex 888448488..bbce9836c 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -20,23 +20,25 @@ export async function billTeam(team_id: string,subscription_id: string | null | undefined,credits: number,+ api_key_id: number | null,logger?: Logger,is_extract: boolean = false,) {// Maintain the withAuth wrapper for authenticationreturn withAuth(- async (team_id, subscription_id, credits, logger, is_extract) => {+ async (team_id: string, subscription_id: string | null | undefined, credits: number, api_key_id: number | null, logger: Logger | undefined, is_extract: boolean) => {// Within the authenticated context, queue the billing operation- return queueBillingOperation(team_id, subscription_id, credits, is_extract);+ return queueBillingOperation(team_id, subscription_id, credits, api_key_id, is_extract);},{ success: true, message: "No DB, bypassed." }- )(team_id, subscription_id, credits, logger, is_extract);+ )(team_id, subscription_id, credits, api_key_id, logger, is_extract);}export async function supaBillTeam(team_id: string,subscription_id: string | null | undefined,credits: number,+ api_key_id: number | null,__logger?: Logger,is_extract: boolean = false,) {@@ -51,7 +53,7 @@ export async function supaBillTeam(});_logger.warn("supaBillTeam was called directly. This function is deprecated and should only be called from batch_billing.ts");- queueBillingOperation(team_id, subscription_id, credits, is_extract).catch((err) => {+ queueBillingOperation(team_id, subscription_id, credits, api_key_id, is_extract).catch((err) => {_logger.error("Error queuing billing operation", { err });Sentry.captureException(err);});diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 806349478..aa06b4a66 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -61,7 +61,7 @@ const processBillingJobInternal = async (token: string, job: Job) => {await processBillingBatch();} else if (job.name === "bill_team") {// This is an individual billing operation that should be queued for batch processing- const { team_id, subscription_id, credits, is_extract } = job.data;+ const { team_id, subscription_id, credits, is_extract, api_key_id } = job.data;logger.info(`Adding team ${team_id} billing operation to batch queue`, {credits,@@ -70,7 +70,7 @@ const processBillingJobInternal = async (token: string, job: Job) => {});// Add to the REDIS batch queue- await queueBillingOperation(team_id, subscription_id, credits, is_extract);+ await queueBillingOperation(team_id, subscription_id, credits, api_key_id ?? null, is_extract);} else {logger.warn(`Unknown billing job type: ${job.name}`);}@@ -181,6 +181,7 @@ const processPrecrawlJobInternal = async (token: string, job: Job) => {webhook: undefined,v1: true,zeroDataRetention: false,+ apiKeyId: null,},{},crypto.randomUUID(),diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 6041b9ec8..44ce46ba9 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -2,6 +2,7 @@ import { Queue, QueueEvents } from "bullmq";import { logger } from "../lib/logger";import IORedis from "ioredis";import { BullMQOtel } from "bullmq-otel";+import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";export type QueueFunction = () => Queue<any, any, string, any, any, string>;@@ -104,7 +105,7 @@ export function getGenerateLlmsTxtQueue() {export function getDeepResearchQueue() {if (!deepResearchQueue) {- deepResearchQueue = new Queue(deepResearchQueueName, {+ deepResearchQueue = new Queue<DeepResearchServiceOptions>(deepResearchQueueName, {connection: getRedisConnection(),defaultJobOptions: {removeOnComplete: {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex c2275d4db..8b7ef2103 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -120,12 +120,14 @@ const processExtractJobInternal = async (request: job.data.request,teamId: job.data.teamId,subId: job.data.subId,+ apiKeyId: job.data.apiKeyId,});} else {result = await performExtraction_F0(job.data.extractId, {request: job.data.request,teamId: job.data.teamId,subId: job.data.subId,+ apiKeyId: job.data.apiKeyId,});}// result = await performExtraction_F0(job.data.extractId, {@@ -224,6 +226,7 @@ const processDeepResearchJobInternal = async (systemPrompt: job.data.request.systemPrompt,formats: job.data.request.formats,jsonOptions: job.data.request.jsonOptions,+ apiKeyId: job.data.apiKeyId,});if (result.success) {@@ -294,6 +297,7 @@ const processGenerateLlmsTxtJobInternal = async (showFullText: job.data.request.showFullText,subId: job.data.subId,cache: job.data.request.cache,+ apiKeyId: job.data.apiKeyId,});if (result.success) {diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex aee617f70..eded7fcd8 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -117,6 +117,7 @@ export async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCwebhook: job.data.webhook,v1: job.data.v1,zeroDataRetention: job.data.zeroDataRetention,+ apiKeyId: job.data.apiKeyId,},opts: {jobId: uuid,diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 7b0e3bac6..668f3a9f6 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -99,6 +99,7 @@ async function billScrapeJob(job: Job & { id: string }, document: Document | nulis_extract: false,timestamp: new Date().toISOString(),originating_job_id: job.id,+ api_key_id: job.data.apiKeyId,},{jobId: billingJobId,@@ -331,6 +332,7 @@ async function processJob(job: Job & { id: string }) {webhook: job.data.webhook,v1: job.data.v1,zeroDataRetention: job.data.zeroDataRetention,+ apiKeyId: job.data.apiKeyId,},{},jobId,@@ -623,6 +625,7 @@ async function processKickoffJob(job: Job & { id: string }) {v1: job.data.v1,isCrawlSourceScrape: true,zeroDataRetention: job.data.zeroDataRetention,+ apiKeyId: job.data.apiKeyId,},{priority: 15,@@ -679,6 +682,7 @@ async function processKickoffJob(job: Job & { id: string }) {webhook: job.data.webhook,v1: job.data.v1,zeroDataRetention: job.data.zeroDataRetention,+ apiKeyId: job.data.apiKeyId,},opts: {jobId: uuid,@@ -743,6 +747,7 @@ async function processKickoffJob(job: Job & { id: string }) {webhook: job.data.webhook,v1: job.data.v1,zeroDataRetention: job.data.zeroDataRetention,+ apiKeyId: job.data.apiKeyId,},opts: {jobId: uuid,diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex d5aa33470..4a46da35b 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -1,12 +1,12 @@import { z } from "zod";import {- AuthCreditUsageChunk,BaseScrapeOptions,ScrapeOptions,Document as V2Document,webhookSchema,TeamFlags,} from "./controllers/v2/types";+import { AuthCreditUsageChunk } from "./controllers/v1/types";import { ExtractorOptions, Document } from "./lib/entities";import { InternalOptions } from "./scraper/scrapeURL";import type { CostTracking } from "./lib/extract/extraction-service";@@ -60,6 +60,7 @@ export interface WebScraperOptions {sentry?: any;is_extract?: boolean;concurrencyLimited?: boolean;+ apiKeyId: number | null;}export interface RunWebScraperParams {