Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): export Node runtime metrics (event-loop lag, heap, GC) from app and workers (#4923)
Gergő Móriczd0786a8c28bf
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 8e1b29bf1..31969064e 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -11,6 +11,7 @@ import {getPrecrawlQueue,} from "./services/queue-service";import { v0Router } from "./routes/v0";+import { startNodeRuntimeMetrics } from "./lib/node-runtime-metrics";import os from "os";import { logger } from "./lib/logger";import { adminRouter } from "./routes/admin";@@ -141,6 +142,7 @@ const DEFAULT_PORT = config.PORT;const HOST = config.HOST;async function startServer(port = DEFAULT_PORT) {+ startNodeRuntimeMetrics();try {await initializeBlocklist();initializeEngineForcing();diff --git a/apps/api/src/lib/node-runtime-metrics.test.ts b/apps/api/src/lib/node-runtime-metrics.test.tsnew file mode 100644index 000000000..0d64342d3--- /dev/null+++ b/apps/api/src/lib/node-runtime-metrics.test.ts@@ -0,0 +1,36 @@+import { register } from "prom-client";+import { startNodeRuntimeMetrics } from "./node-runtime-metrics";++describe("startNodeRuntimeMetrics", () => {+ it("registers the event-loop, heap and GC series once, with live values", async () => {+ startNodeRuntimeMetrics();+ // A second call must not throw on the already-registered names.+ startNodeRuntimeMetrics();++ const metrics = await register.getMetricsAsJSON();+ const names = metrics.map(m => m.name);+ for (const expected of [+ "nodejs_eventloop_lag_seconds",+ "nodejs_eventloop_lag_p99_seconds",+ "nodejs_eventloop_lag_max_seconds",+ "nodejs_heap_size_used_bytes",+ "nodejs_gc_duration_seconds",+ "nodejs_active_handles_total",+ "process_cpu_seconds_total",+ ]) {+ expect(names).toContain(expected);+ }++ // Registered but empty would be invisible in production too: the gauges+ // that sample on collection must carry a value right away.+ for (const sampled of [+ "nodejs_heap_size_used_bytes",+ "process_cpu_seconds_total",+ "process_resident_memory_bytes",+ ]) {+ const metric = metrics.find(m => m.name === sampled);+ expect(metric?.values.length).toBeGreaterThan(0);+ expect(metric?.values[0].value).toBeGreaterThan(0);+ }+ });+});diff --git a/apps/api/src/lib/node-runtime-metrics.ts b/apps/api/src/lib/node-runtime-metrics.tsnew file mode 100644index 000000000..4b57ec111--- /dev/null+++ b/apps/api/src/lib/node-runtime-metrics.ts@@ -0,0 +1,22 @@+import { collectDefaultMetrics, register } from "prom-client";++let started = false;++/**+ * Export the Node runtime's own health through the default registry, next to+ * the request histograms the app and the workers already serve.+ *+ * The interesting series are `nodejs_eventloop_lag_*_seconds` (sampled via+ * `perf_hooks.monitorEventLoopDelay`), the heap and GC series and the active+ * handle/request counts: a pod that stops answering its liveness probe while+ * its CPU, memory and descriptor counts look normal is otherwise invisible.+ *+ * Idempotent: `collectDefaultMetrics` throws when a metric name is registered+ * twice, and several entry points share this module. The flag is set only+ * after a successful registration so a failed attempt can be retried.+ */+export function startNodeRuntimeMetrics(): void {+ if (started) return;+ collectDefaultMetrics({ register });+ started = true;+}diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 6789d81c5..d7b36487b 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -8,6 +8,7 @@ import { updateExtract } from "../lib/extract/extract-redis";import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f0";import { createWebhookSender, WebhookEvent } from "./webhook";import Express from "express";+import { startNodeRuntimeMetrics } from "../lib/node-runtime-metrics";import { getErrorContactMessage } from "../lib/deployment";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";@@ -175,6 +176,7 @@ const processDLQJob = async (data: ExtractJobData) => {// Start the workerconst app = Express();+startNodeRuntimeMetrics();app.get("/health", (req, res) => {res.status(200).json({ ok: true });diff --git a/apps/api/src/services/worker/nuq-reconciler-worker.ts b/apps/api/src/services/worker/nuq-reconciler-worker.tsindex 16e6f9322..fadd39cf8 100644--- a/apps/api/src/services/worker/nuq-reconciler-worker.ts+++ b/apps/api/src/services/worker/nuq-reconciler-worker.ts@@ -6,6 +6,7 @@ import { reconcileConcurrencyQueue } from "../../lib/concurrency-queue-reconcileimport { repairCrawlJobDoneMarkers } from "../../lib/crawl-redis";import { Counter, register } from "prom-client";import Express from "express";+import { startNodeRuntimeMetrics } from "../../lib/node-runtime-metrics";const RECONCILER_INTERVAL_MS = 60 * 1000;@@ -29,6 +30,7 @@ const reconcilerJobsRecoveredTotal = new Counter({let reconcilerInFlight = false;const app = Express();+ startNodeRuntimeMetrics();app.get("/metrics", async (_, res) => {try {diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.tsindex 259a332e1..481cb2fb7 100644--- a/apps/api/src/services/worker/nuq-worker-runner.ts+++ b/apps/api/src/services/worker/nuq-worker-runner.ts@@ -7,6 +7,7 @@ import { processJobInternal } from "./scrape-worker";import { NuQJob } from "./nuq";import { register } from "prom-client";import Express from "express";+import { startNodeRuntimeMetrics } from "../../lib/node-runtime-metrics";import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";import { shutdownPubSubLogging } from "../logging/log_job";@@ -74,6 +75,7 @@ export async function runNuqWorker(options: {let shutdownStartedAt: number | undefined;const app = Express();+ startNodeRuntimeMetrics();app.get("/metrics", async (_, res) => {const localMetrics = options.metrics ? await options.metrics() : "";