Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): carry the caller's external request id on the extract, llms.txt and deep-research charges (#4736)
Gergő Móricz95ff8d6a40a5
diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex ef2aabffd..42b42434c 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -114,6 +114,7 @@ export async function deepResearchController(request: req.body,teamId: req.auth.team_id,apiKeyId: req.acuc?.api_key_id ?? null,+ externalRequestId: externalRequestId(req),researchId,};diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 5250c1940..8bf7bb8d7 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -71,6 +71,7 @@ async function oldExtract(request,teamId: req.auth.team_id,apiKeyId: req.acuc?.api_key_id ?? null,+ externalRequestId: externalRequestId(req),});if (sender) {@@ -282,6 +283,7 @@ export async function extractController(extractId,agent: req.body.agent,apiKeyId: req.acuc?.api_key_id ?? null,+ externalRequestId: externalRequestId(req),createdAt,};diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex 3ad13dda1..b9f82a8a5 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -57,6 +57,7 @@ export async function generateLLMsTextController(request: req.body,teamId: req.auth.team_id,apiKeyId: req.acuc?.api_key_id ?? null,+ externalRequestId: externalRequestId(req),generationId,};diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex af716dd90..747299d22 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -220,6 +220,7 @@ export async function extractController(await addExtractJobToQueue(extractId, {...jobData,apiKeyId: req.acuc?.api_key_id ?? undefined,+ externalRequestId: externalRequestId(req),});return res.status(200).json({diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.tsindex bf0039080..1a10250b7 100644--- a/apps/api/src/lib/deep-research/deep-research-service.ts+++ b/apps/api/src/lib/deep-research/deep-research-service.ts@@ -21,6 +21,8 @@ export interface DeepResearchServiceOptions {formats: string[];jsonOptions: ExtractOptions;apiKeyId: number | null;+ /** The caller's External-Request-Id, carried on the charge for firebill. */+ externalRequestId?: string | null;}export async function performDeepResearch(options: DeepResearchServiceOptions) {@@ -431,7 +433,12 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {orgIdFromAcuc(acuc),credits_billed,apiKeyId,- { endpoint: "deep_research", jobId: researchId, chargeId: researchId },+ {+ endpoint: "deep_research",+ jobId: researchId,+ chargeId: researchId,+ externalRequestId: options.externalRequestId ?? null,+ },logger,).catch(error => {logger.error(diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 15714c7fd..d029fbf6f 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -49,6 +49,8 @@ interface ExtractServiceOptions {cacheKey?: string;apiKeyId: number | null;createdAt?: number;+ /** The caller's External-Request-Id, carried on the charge for firebill. */+ externalRequestId?: string | null;}interface ExtractResult {@@ -875,7 +877,12 @@ export async function performExtraction_F0(orgIdFromAcuc(acuc),creditsToBill,apiKeyId,- { endpoint: "extract", jobId: extractId, chargeId: extractId },+ {+ endpoint: "extract",+ jobId: extractId,+ chargeId: extractId,+ externalRequestId: options.externalRequestId ?? null,+ },logger,).catch(error => {logger.error(diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.tsindex e3d87936e..3d5d1e33f 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts@@ -19,6 +19,8 @@ interface GenerateLLMsTextServiceOptions {generationId: string;teamId: string;apiKeyId: number | null;+ /** The caller's External-Request-Id, carried on the charge for firebill. */+ externalRequestId?: string | null;url: string;maxUrls: number;showFullText: boolean;@@ -284,7 +286,12 @@ export async function performGenerateLlmsTxt(orgIdFromAcuc(acuc),urls.length,apiKeyId,- { endpoint: "llms_txt", jobId: generationId, chargeId: generationId },+ {+ endpoint: "llms_txt",+ jobId: generationId,+ chargeId: generationId,+ externalRequestId: options.externalRequestId ?? null,+ },logger,).catch(error => {logger.error(`Failed to bill team ${teamId} for ${urls.length} urls`, {diff --git a/apps/api/src/services/extract-queue.ts b/apps/api/src/services/extract-queue.tsindex 29ab3a750..f97f1a50c 100644--- a/apps/api/src/services/extract-queue.ts+++ b/apps/api/src/services/extract-queue.ts@@ -13,6 +13,8 @@ export type ExtractJobData = {apiKeyId?: number | null;agent?: any;createdAt: number;+ /** The caller's External-Request-Id, carried on the extract charge. */+ externalRequestId?: string | null;};let connection: amqp.ChannelModel | null = null;diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 043652bae..6789d81c5 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -57,6 +57,7 @@ const processExtractJob = async (request: data.request,teamId: data.teamId,apiKeyId: data.apiKeyId ?? null,+ externalRequestId: data.externalRequestId ?? null,});if (result && result.success) {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 3f98da518..1cd735773 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -81,6 +81,7 @@ const processDeepResearchJobInternal = async (formats: job.data.request.formats,jsonOptions: job.data.request.jsonOptions,apiKeyId: job.data.apiKeyId,+ externalRequestId: job.data.externalRequestId ?? null,});if (result.success) {@@ -142,6 +143,7 @@ const processGenerateLlmsTxtJobInternal = async (showFullText: job.data.request.showFullText,cache: job.data.request.cache,apiKeyId: job.data.apiKeyId,+ externalRequestId: job.data.externalRequestId ?? null,});if (result.success) {