Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/logRequest): associate requests with API keys (#2519)
Gergő Móricz9d0883629fde
diff --git a/apps/api/src/controllers/v0/admin/crawl-monitor.ts b/apps/api/src/controllers/v0/admin/crawl-monitor.tsindex 7a6f8d5bf..c2a6d6e28 100644--- a/apps/api/src/controllers/v0/admin/crawl-monitor.ts+++ b/apps/api/src/controllers/v0/admin/crawl-monitor.ts@@ -48,6 +48,7 @@ export async function crawlMonitorController(integration: null,target_hint: "https://firecrawl.dev",zeroDataRetention: false,+ api_key_id: req.acuc?.api_key_id ?? null,});const sc: StoredCrawl = {diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex a20ea9235..1caa079a0 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -63,6 +63,7 @@ export async function crawlController(req: Request, res: Response) {integration: req.body.integration,target_hint: req.body.url ?? "",zeroDataRetention: false, // not supported on v0+ api_key_id: chunk?.api_key_id ?? null,});redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 337c795e7..ca3957720 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -202,6 +202,7 @@ export async function scrapeController(req: Request, res: Response) {integration: req.body.integration,target_hint: req.body.url ?? "",zeroDataRetention: false, // not supported on v0+ api_key_id: chunk?.api_key_id ?? null,});redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex f9e81fe27..e07d8d89f 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -187,6 +187,7 @@ export async function searchController(req: Request, res: Response) {integration: req.body.integration,target_hint: req.body.query ?? "",zeroDataRetention: false, // not supported on v0+ api_key_id: chunk?.api_key_id ?? null,});redisEvictConnection.sadd("teams_using_v0", team_id).catch(error =>diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 435250be4..57b36d595 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -122,6 +122,7 @@ export async function batchScrapeController(integration: req.body.integration,target_hint: urls[0] ?? "",zeroDataRetention: zeroDataRetention || false,+ api_key_id: req.acuc?.api_key_id ?? null,});}diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex bd57db1bd..408239b82 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -63,6 +63,7 @@ export async function crawlController(integration: req.body.integration,target_hint: req.body.url,zeroDataRetention: zeroDataRetention || false,+ api_key_id: req.acuc?.api_key_id ?? null,});let { remainingCredits } = req.account!;diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex 36be68608..d72a52574 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -101,6 +101,7 @@ export async function deepResearchController(origin: "api",target_hint: req.body.query ?? "",zeroDataRetention: false, // not supported for deep research+ api_key_id: req.acuc?.api_key_id ?? null,});const jobData = {diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex eb2d08e29..f291699aa 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -163,6 +163,7 @@ export async function extractController(integration: req.body.integration,target_hint: req.body.urls?.[0] ?? "",zeroDataRetention: false, // not supported for extract+ api_key_id: req.acuc?.api_key_id ?? null,});const scrapeOptions = req.body.scrapeOptionsdiff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex a7312bbe7..2fe3c83bc 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -48,6 +48,7 @@ export async function generateLLMsTextController(origin: "api", // no origin field for llmstxttarget_hint: req.body.url,zeroDataRetention: false, // not supported for llmstxt+ api_key_id: req.acuc?.api_key_id ?? null,});const jobData = {diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 772ae393d..2a8cbed31 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -397,6 +397,7 @@ export async function mapController(integration: req.body.integration,target_hint: req.body.url,zeroDataRetention: false, // not supported for map+ api_key_id: req.acuc?.api_key_id ?? null,});let result: Awaited<ReturnType<typeof getMapResults>>;diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex d7af937f6..6e9e57c28 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -74,6 +74,7 @@ export async function scrapeController(integration: req.body.integration,target_hint: req.body.url,zeroDataRetention: zeroDataRetention || false,+ api_key_id: req.acuc?.api_key_id ?? null,});const origin = req.body.origin;diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 0fcacdc10..d58b28de4 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -281,6 +281,7 @@ export async function searchController(integration: req.body.integration,target_hint: req.body.query,zeroDataRetention: false, // not supported for search+ api_key_id: req.acuc?.api_key_id ?? null,});let limit = req.body.limit;diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex ff5f9b0e2..a62f0c3af 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -224,6 +224,7 @@ export async function x402SearchController(integration: req.body.integration,target_hint: req.body.query,zeroDataRetention: false, // not supported for x402 search+ api_key_id: req.acuc?.api_key_id ?? null,});let limit = req.body.limit;diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 30c503959..b0d22723d 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -122,6 +122,7 @@ export async function batchScrapeController(integration: req.body.integration,target_hint: urls[0] ?? "",zeroDataRetention: zeroDataRetention || false,+ api_key_id: req.acuc?.api_key_id ?? null,});}diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 4534e01fa..b87bce480 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -70,6 +70,7 @@ export async function crawlController(integration: req.body.integration,target_hint: req.body.url,zeroDataRetention: zeroDataRetention || false,+ api_key_id: req.acuc?.api_key_id ?? null,});let { remainingCredits } = req.account!;diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex b8bbdb525..36a2cc71f 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -70,6 +70,7 @@ export async function extractController(integration: req.body.integration,target_hint: req.body.urls?.[0] ?? "",zeroDataRetention: false, // not supported for extract+ api_key_id: req.acuc?.api_key_id ?? null,});const jobData = {diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 5764929e8..afdba5f15 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -64,6 +64,7 @@ export async function mapController(integration: req.body.integration,target_hint: req.body.url,zeroDataRetention: false, // not supported for map+ api_key_id: req.acuc?.api_key_id ?? null,});let result: MapResult;diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 6fd95c75e..5a5244c42 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -109,6 +109,7 @@ export async function scrapeController(integration: req.body.integration,target_hint: req.body.url,zeroDataRetention: zeroDataRetention || false,+ api_key_id: req.acuc?.api_key_id ?? null,});setSpanAttributes(span, {diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex a85924c51..a61d696d0 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -264,6 +264,7 @@ export async function searchController(integration: req.body.integration,target_hint: req.body.query,zeroDataRetention: isZDROrAnon ?? false, // not supported for search+ api_key_id: req.acuc?.api_key_id ?? null,});let limit = req.body.limit;diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex bc09fb7cb..bc4a92452 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -247,6 +247,7 @@ export async function x402SearchController(integration: req.body.integration,target_hint: req.body.query,zeroDataRetention: false, // not supported for x402 search+ api_key_id: req.acuc?.api_key_id ?? null,});let limit = req.body.limit;diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 4361498a4..7b17f0ef7 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -102,6 +102,7 @@ type LoggedRequest = {integration?: string | null;target_hint: string;zeroDataRetention: boolean;+ api_key_id?: number | null;};export async function logRequest(request: LoggedRequest) {@@ -131,6 +132,7 @@ export async function logRequest(request: LoggedRequest) {dr_clean_by: request.zeroDataRetention? new Date(Date.now() + 24 * 60 * 60 * 1000): null,+ api_key_id: request.api_key_id ?? null,},true,logger,