Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add tracking of job duration
Michael Malised2b874b3b76
diff --git a/apps/api/src/lib/job-metrics.ts b/apps/api/src/lib/job-metrics.tsnew file mode 100644index 000000000..9390e7b69--- /dev/null+++ b/apps/api/src/lib/job-metrics.ts@@ -0,0 +1,8 @@+import { Histogram } from "prom-client";++export const jobDurationSeconds = new Histogram({+ name: "job_duration_seconds",+ help: "Duration of background jobs in seconds",+ labelNames: ["type", "status"],+ buckets: [0.1, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300],+});diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex d7c53cebd..5053881bd 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -21,6 +21,7 @@ import {ExtractJobData,} from "./extract-queue";import { logExtract } from "./logging/log_job";+import { jobDurationSeconds } from "../lib/job-metrics";configDotenv();@@ -43,6 +44,8 @@ const processExtractJob = async (v0: false,});+ const endJobTimer = jobDurationSeconds.startTimer({ type: "extract" });+try {if (sender) {sender.send(WebhookEvent.EXTRACT_STARTED, {@@ -75,6 +78,7 @@ const processExtractJob = async (});}+ endJobTimer({ status: "success" });ack();return;} else {@@ -90,10 +94,12 @@ const processExtractJob = async (});}+ endJobTimer({ status: "failed" });ack();return;}} catch (error) {+ endJobTimer({ status: "failed" });logger.error(`🚫 Extract job errored ${data.extractId} - ${error}`, {error,});diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex 69f68eb19..9b0a0487b 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -5,6 +5,7 @@ import { setSentryServiceTag } from "../sentry";import { logger as _logger } from "../../lib/logger";import { processJobInternal } from "./scrape-worker";import { scrapeQueue, nuqGetLocalMetrics, nuqHealthCheck } from "./nuq";+import { jobDurationSeconds } from "../../lib/job-metrics";import Express from "express";import { _ } from "ajv";import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";@@ -89,6 +90,8 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-f| { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }| { ok: false; error: any };+ const endJobTimer = jobDurationSeconds.startTimer({ type: job.data.mode });+try {processResult = { ok: true, data: await processJobInternal(job) };} catch (error) {@@ -98,6 +101,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-fclearInterval(lockRenewInterval);if (processResult.ok) {+ endJobTimer({ status: "success" });if (!(await scrapeQueue.jobFinish(job.id,@@ -109,6 +113,7 @@ import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-flogger.warn("Could not update job status");}} else {+ endJobTimer({ status: "failed" });if (!(await scrapeQueue.jobFail(job.id,