Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add reporting of job metrics

Michael Malis2dcfbd8ca3d1
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index 5053881bd..98304e9c3 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -22,6 +22,7 @@ import {
} from "./extract-queue";
import { logExtract } from "./logging/log_job";
import { jobDurationSeconds } from "../lib/job-metrics";
+import { register } from "prom-client";
configDotenv();
@@ -193,6 +194,9 @@ app.get("/health", (req, res) => {
app.get("/liveness", (req, res) => {
res.status(200).json({ ok: true });
});
+app.get("/metrics", async (_, res) => {
+ res.contentType("text/plain").send(await register.metrics());
+});
const workerPort = config.EXTRACT_WORKER_PORT || config.PORT;
app.listen(workerPort, () => {
diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts
index 9b0a0487b..1775bec30 100644
--- a/apps/api/src/services/worker/nuq-worker.ts
+++ b/apps/api/src/services/worker/nuq-worker.ts
@@ -4,8 +4,12 @@ import "../sentry";
import { setSentryServiceTag } from "../sentry";
import { logger as _logger } from "../../lib/logger";
import { processJobInternal } from "./scrape-worker";
-import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";
-import { jobDurationSeconds } from "../../lib/job-metrics";
+import {
+ scrapeQueue,
+ nuqGetLocalMetrics,
+ nuqHealthCheck,
+ recordJobDuration,
+} from "./nuq";
import Express from "express";
import { _ } from "ajv";
import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";
@@ -90,7 +94,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
| { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }
| { ok: false; error: any };
- const endJobTimer = jobDurationSeconds.startTimer({ type: job.data.mode });
+ const jobStartTime = Date.now();
try {
processResult = { ok: true, data: await processJobInternal(job) };
@@ -100,8 +104,10 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
clearInterval(lockRenewInterval);
+ const jobDurationSeconds = (Date.now() - jobStartTime) / 1000;
+
if (processResult.ok) {
- endJobTimer({ status: "success" });
+ recordJobDuration(job.data.mode, "success", jobDurationSeconds);
if (
!(await scrapeQueue.jobFinish(
job.id,
@@ -113,7 +119,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
logger.warn("Could not update job status");
}
} else {
- endJobTimer({ status: "failed" });
+ recordJobDuration(job.data.mode, "failed", jobDurationSeconds);
if (
!(await scrapeQueue.jobFail(
job.id,
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 934689096..4aa6a33d6 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -1659,10 +1659,65 @@ class NuQJobGroup {
}
}
+const JOB_DURATION_BUCKETS = [0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300];
+
+type HistogramData = {
+ buckets: Map<number, number>;
+ sum: number;
+ count: number;
+};
+
+const jobDurationData = new Map<string, HistogramData>();
+
+export function recordJobDuration(
+ type: string,
+ status: string,
+ durationSeconds: number,
+) {
+ const key = `${type}__${status}`;
+ if (!jobDurationData.has(key)) {
+ jobDurationData.set(key, {
+ buckets: new Map(JOB_DURATION_BUCKETS.map(b => [b, 0])),
+ sum: 0,
+ count: 0,
+ });
+ }
+ const data = jobDurationData.get(key)!;
+ for (const bucket of JOB_DURATION_BUCKETS) {
+ if (durationSeconds <= bucket) {
+ data.buckets.set(bucket, data.buckets.get(bucket)! + 1);
+ }
+ }
+ data.sum += durationSeconds;
+ data.count++;
+}
+
+function formatJobDurationMetrics(): string {
+ if (jobDurationData.size === 0) return "";
+ const lines: string[] = [
+ "# HELP job_duration_seconds Duration of background jobs in seconds",
+ "# TYPE job_duration_seconds histogram",
+ ];
+ for (const [key, data] of jobDurationData) {
+ const [type, status] = key.split("__");
+ const labels = `type="${type}",status="${status}"`;
+ for (const [le, count] of data.buckets) {
+ lines.push(`job_duration_seconds_bucket{${labels},le="${le}"} ${count}`);
+ }
+ lines.push(
+ `job_duration_seconds_bucket{${labels},le="+Inf"} ${data.count}`,
+ );
+ lines.push(`job_duration_seconds_sum{${labels}} ${data.sum}`);
+ lines.push(`job_duration_seconds_count{${labels}} ${data.count}`);
+ }
+ return lines.join("\n");
+}
+
export function nuqGetLocalMetrics(): string {
return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n
# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n
-# HELP nuq_pool_total_count Number of connections in the pool\n# TYPE nuq_pool_total_count gauge\nnuq_pool_total_count ${nuqPool.totalCount}\n`;
+# HELP nuq_pool_total_count Number of connections in the pool\n# TYPE nuq_pool_total_count gauge\nnuq_pool_total_count ${nuqPool.totalCount}\n
+${formatJobDurationMetrics()}`;
}
export async function nuqHealthCheck(): Promise<boolean> {