Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Use prom-client methods instead of generating own metrics
Michael Malis3858ced2ff5a
diff --git a/apps/api/src/lib/job-metrics.ts b/apps/api/src/lib/job-metrics.tsindex 9390e7b69..16675ce8f 100644--- a/apps/api/src/lib/job-metrics.ts+++ b/apps/api/src/lib/job-metrics.ts@@ -4,5 +4,5 @@ export const jobDurationSeconds = new Histogram({name: "job_duration_seconds",help: "Duration of background jobs in seconds",labelNames: ["type", "status"],- buckets: [0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300],+ buckets: [0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 900, 1800, 3600],});diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex 1775bec30..3f41a0881 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -4,12 +4,9 @@ import "../sentry";import { setSentryServiceTag } from "../sentry";import { logger as _logger } from "../../lib/logger";import { processJobInternal } from "./scrape-worker";-import {- scrapeQueue,- nuqGetLocalMetrics,- nuqHealthCheck,- recordJobDuration,-} from "./nuq";+import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";+import { jobDurationSeconds } from "../../lib/job-metrics";+import { register } from "prom-client";import Express from "express";import { _ } from "ajv";import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";@@ -32,9 +29,11 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-fconst app = Express();- app.get("/metrics", (_, res) => {- res.contentType("text/plain").send(nuqGetLocalMetrics());- });+ app.get("/metrics", async (_, res) =>+ res+ .contentType("text/plain")+ .send(nuqGetLocalMetrics() + "\n" + (await register.metrics())),+ );app.get("/health", async (_, res) => {if (await nuqHealthCheck()) {res.status(200).send("OK");@@ -94,7 +93,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f| { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }| { ok: false; error: any };- const jobStartTime = Date.now();+ const endJobTimer = jobDurationSeconds.startTimer({ type: job.data.mode });try {processResult = { ok: true, data: await processJobInternal(job) };@@ -104,10 +103,8 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-fclearInterval(lockRenewInterval);- const jobDurationSeconds = (Date.now() - jobStartTime) / 1000;-if (processResult.ok) {- recordJobDuration(job.data.mode, "success", jobDurationSeconds);+ endJobTimer({ status: "success" });if (!(await scrapeQueue.jobFinish(job.id,@@ -119,7 +116,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-flogger.warn("Could not update job status");}} else {- recordJobDuration(job.data.mode, "failed", jobDurationSeconds);+ endJobTimer({ status: "failed" });if (!(await scrapeQueue.jobFail(job.id,diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 4aa6a33d6..934689096 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -1659,65 +1659,10 @@ class NuQJobGroup {}}-const JOB_DURATION_BUCKETS = [0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300];--type HistogramData = {- buckets: Map<number, number>;- sum: number;- count: number;-};--const jobDurationData = new Map<string, HistogramData>();--export function recordJobDuration(- type: string,- status: string,- durationSeconds: number,-) {- const key = `${type}__${status}`;- if (!jobDurationData.has(key)) {- jobDurationData.set(key, {- buckets: new Map(JOB_DURATION_BUCKETS.map(b => [b, 0])),- sum: 0,- count: 0,- });- }- const data = jobDurationData.get(key)!;- for (const bucket of JOB_DURATION_BUCKETS) {- if (durationSeconds <= bucket) {- data.buckets.set(bucket, data.buckets.get(bucket)! + 1);- }- }- data.sum += durationSeconds;- data.count++;-}--function formatJobDurationMetrics(): string {- if (jobDurationData.size === 0) return "";- const lines: string[] = [- "# HELP job_duration_seconds Duration of background jobs in seconds",- "# TYPE job_duration_seconds histogram",- ];- for (const [key, data] of jobDurationData) {- const [type, status] = key.split("__");- const labels = `type="${type}",status="${status}"`;- for (const [le, count] of data.buckets) {- lines.push(`job_duration_seconds_bucket{${labels},le="${le}"} ${count}`);- }- lines.push(- `job_duration_seconds_bucket{${labels},le="+Inf"} ${data.count}`,- );- lines.push(`job_duration_seconds_sum{${labels}} ${data.sum}`);- lines.push(`job_duration_seconds_count{${labels}} ${data.count}`);- }- return lines.join("\n");-}-export function nuqGetLocalMetrics(): string {return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n-# HELP nuq_pool_total_count Number of connections in the pool\n# TYPE nuq_pool_total_count gauge\nnuq_pool_total_count ${nuqPool.totalCount}\n-${formatJobDurationMetrics()}`;+# HELP nuq_pool_total_count Number of connections in the pool\n# TYPE nuq_pool_total_count gauge\nnuq_pool_total_count ${nuqPool.totalCount}\n`;}export async function nuqHealthCheck(): Promise<boolean> {