Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): log to db async (#2845)
tomsideguidecb36f6b75711
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 656acda0a..b5e81eea1 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -67,7 +67,7 @@ export async function scrapeController(account: req.account,});- await logRequest({+ logRequest({id: jobId,kind: "scrape",api_version: "v1",@@ -77,7 +77,9 @@ export async function scrapeController(target_hint: req.body.url,zeroDataRetention: zeroDataRetention || false,api_key_id: req.acuc?.api_key_id ?? null,- });+ }).catch(err =>+ logger.warn("Background request log failed", { error: err, jobId }),+ );const origin = req.body.origin;const timeout = req.body.timeout;diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex de76995f0..4dc37720a 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -126,7 +126,7 @@ export async function scrapeController(});if (!agentRequestId) {- await logRequest({+ logRequest({id: jobId,kind: "scrape",api_version: "v2",@@ -136,7 +136,9 @@ export async function scrapeController(target_hint: req.body.url,zeroDataRetention: zeroDataRetention || false,api_key_id: req.acuc?.api_key_id ?? null,- });+ }).catch(err =>+ logger.warn("Background request log failed", { error: err, jobId }),+ );}setSpanAttributes(span, {diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 965cb7261..f88957362 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -538,7 +538,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {doc.metadata.creditsUsed = credits_billed ?? undefined;- await logScrape(+ logScrape({id: job.id,request_id: job.data.requestId ?? job.data.crawl_id ?? job.id,@@ -555,6 +555,11 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {skipNuq: job.data.skipNuq ?? false,},false,+ ).catch(err =>+ logger.warn("Background scrape log failed", {+ error: err,+ jobId: job.id,+ }),);}