Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add tracking of job duration

Michael Malised2b874b3b76
diff --git a/apps/api/src/lib/job-metrics.ts b/apps/api/src/lib/job-metrics.ts
new file mode 100644
index 000000000..9390e7b69
--- /dev/null
+++ b/apps/api/src/lib/job-metrics.ts
@@ -0,0 +1,8 @@
+import { Histogram } from "prom-client";
+
+export const jobDurationSeconds = new Histogram({
+ name: "job_duration_seconds",
+ help: "Duration of background jobs in seconds",
+ labelNames: ["type", "status"],
+ buckets: [0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300],
+});
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index d7c53cebd..5053881bd 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -21,6 +21,7 @@ import {
ExtractJobData,
} from "./extract-queue";
import { logExtract } from "./logging/log_job";
+import { jobDurationSeconds } from "../lib/job-metrics";
configDotenv();
@@ -43,6 +44,8 @@ const processExtractJob = async (
v0: false,
});
+ const endJobTimer = jobDurationSeconds.startTimer({ type: "extract" });
+
try {
if (sender) {
sender.send(WebhookEvent.EXTRACT_STARTED, {
@@ -75,6 +78,7 @@ const processExtractJob = async (
});
}
+ endJobTimer({ status: "success" });
ack();
return;
} else {
@@ -90,10 +94,12 @@ const processExtractJob = async (
});
}
+ endJobTimer({ status: "failed" });
ack();
return;
}
} catch (error) {
+ endJobTimer({ status: "failed" });
logger.error(`🚫 Extract job errored ${data.extractId} - ${error}`, {
error,
});
diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts
index 69f68eb19..9b0a0487b 100644
--- a/apps/api/src/services/worker/nuq-worker.ts
+++ b/apps/api/src/services/worker/nuq-worker.ts
@@ -5,6 +5,7 @@ import { setSentryServiceTag } from "../sentry";
import { logger as _logger } from "../../lib/logger";
import { processJobInternal } from "./scrape-worker";
import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";
+import { jobDurationSeconds } from "../../lib/job-metrics";
import Express from "express";
import { _ } from "ajv";
import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";
@@ -89,6 +90,8 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
| { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }
| { ok: false; error: any };
+ const endJobTimer = jobDurationSeconds.startTimer({ type: job.data.mode });
+
try {
processResult = { ok: true, data: await processJobInternal(job) };
} catch (error) {
@@ -98,6 +101,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
clearInterval(lockRenewInterval);
if (processResult.ok) {
+ endJobTimer({ status: "success" });
if (
!(await scrapeQueue.jobFinish(
job.id,
@@ -109,6 +113,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f
logger.warn("Could not update job status");
}
} else {
+ endJobTimer({ status: "failed" });
if (
!(await scrapeQueue.jobFail(
job.id,