Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add reporting of job metrics
Michael Malis2dcfbd8ca3d1
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 5053881bd..98304e9c3 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -22,6 +22,7 @@ import {} from "./extract-queue";import { logExtract } from "./logging/log_job";import { jobDurationSeconds } from "../lib/job-metrics";+import { register } from "prom-client";configDotenv();@@ -193,6 +194,9 @@ app.get("/health", (req, res) => {app.get("/liveness", (req, res) => {res.status(200).json({ ok: true });});+app.get("/metrics", async (_, res) => {+ res.contentType("text/plain").send(await register.metrics());+});const workerPort = config.EXTRACT_WORKER_PORT || config.PORT;app.listen(workerPort, () => {diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex 9b0a0487b..1775bec30 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -4,8 +4,12 @@ import "../sentry";import { setSentryServiceTag } from "../sentry";import { logger as _logger } from "../../lib/logger";import { processJobInternal } from "./scrape-worker";-import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";-import { jobDurationSeconds } from "../../lib/job-metrics";+import {+ scrapeQueue,+ nuqGetLocalMetrics,+ nuqHealthCheck,+ recordJobDuration,+} from "./nuq";import Express from "express";import { _ } from "ajv";import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";@@ -90,7 +94,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f| { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }| { ok: false; error: any };- const endJobTimer = jobDurationSeconds.startTimer({ type: job.data.mode });+ const jobStartTime = Date.now();try {processResult = { ok: true, data: await processJobInternal(job) };@@ -100,8 +104,10 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-fclearInterval(lockRenewInterval);+ const jobDurationSeconds = (Date.now() - jobStartTime) / 1000;+if (processResult.ok) {- endJobTimer({ status: "success" });+ recordJobDuration(job.data.mode, "success", jobDurationSeconds);if (!(await scrapeQueue.jobFinish(job.id,@@ -113,7 +119,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-flogger.warn("Could not update job status");}} else {- endJobTimer({ status: "failed" });+ recordJobDuration(job.data.mode, "failed", jobDurationSeconds);if (!(await scrapeQueue.jobFail(job.id,diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 934689096..4aa6a33d6 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -1659,10 +1659,65 @@ class NuQJobGroup {}}+const JOB_DURATION_BUCKETS = [0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300];++type HistogramData = {+ buckets: Map<number, number>;+ sum: number;+ count: number;+};++const jobDurationData = new Map<string, HistogramData>();++export function recordJobDuration(+ type: string,+ status: string,+ durationSeconds: number,+) {+ const key = `${type}__${status}`;+ if (!jobDurationData.has(key)) {+ jobDurationData.set(key, {+ buckets: new Map(JOB_DURATION_BUCKETS.map(b => [b, 0])),+ sum: 0,+ count: 0,+ });+ }+ const data = jobDurationData.get(key)!;+ for (const bucket of JOB_DURATION_BUCKETS) {+ if (durationSeconds <= bucket) {+ data.buckets.set(bucket, data.buckets.get(bucket)! + 1);+ }+ }+ data.sum += durationSeconds;+ data.count++;+}++function formatJobDurationMetrics(): string {+ if (jobDurationData.size === 0) return "";+ const lines: string[] = [+ "# HELP job_duration_seconds Duration of background jobs in seconds",+ "# TYPE job_duration_seconds histogram",+ ];+ for (const [key, data] of jobDurationData) {+ const [type, status] = key.split("__");+ const labels = `type="${type}",status="${status}"`;+ for (const [le, count] of data.buckets) {+ lines.push(`job_duration_seconds_bucket{${labels},le="${le}"} ${count}`);+ }+ lines.push(+ `job_duration_seconds_bucket{${labels},le="+Inf"} ${data.count}`,+ );+ lines.push(`job_duration_seconds_sum{${labels}} ${data.sum}`);+ lines.push(`job_duration_seconds_count{${labels}} ${data.count}`);+ }+ return lines.join("\n");+}+export function nuqGetLocalMetrics(): string {return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n-# HELP nuq_pool_total_count Number of connections in the pool\n# TYPE nuq_pool_total_count gauge\nnuq_pool_total_count ${nuqPool.totalCount}\n`;+# HELP nuq_pool_total_count Number of connections in the pool\n# TYPE nuq_pool_total_count gauge\nnuq_pool_total_count ${nuqPool.totalCount}\n+${formatJobDurationMetrics()}`;}export async function nuqHealthCheck(): Promise<boolean> {