Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): export Node runtime metrics (event-loop lag, heap, GC) from app and workers (#4923)

Gergő Móriczd0786a8c28bf
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 8e1b29bf1..31969064e 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -11,6 +11,7 @@ import {
getPrecrawlQueue,
} from "./services/queue-service";
import { v0Router } from "./routes/v0";
+import { startNodeRuntimeMetrics } from "./lib/node-runtime-metrics";
import os from "os";
import { logger } from "./lib/logger";
import { adminRouter } from "./routes/admin";
@@ -141,6 +142,7 @@ const DEFAULT_PORT = config.PORT;
const HOST = config.HOST;
async function startServer(port = DEFAULT_PORT) {
+ startNodeRuntimeMetrics();
try {
await initializeBlocklist();
initializeEngineForcing();
diff --git a/apps/api/src/lib/node-runtime-metrics.test.ts b/apps/api/src/lib/node-runtime-metrics.test.ts
new file mode 100644
index 000000000..0d64342d3
--- /dev/null
+++ b/apps/api/src/lib/node-runtime-metrics.test.ts
@@ -0,0 +1,36 @@
+import { register } from "prom-client";
+import { startNodeRuntimeMetrics } from "./node-runtime-metrics";
+
+describe("startNodeRuntimeMetrics", () => {
+ it("registers the event-loop, heap and GC series once, with live values", async () => {
+ startNodeRuntimeMetrics();
+ // A second call must not throw on the already-registered names.
+ startNodeRuntimeMetrics();
+
+ const metrics = await register.getMetricsAsJSON();
+ const names = metrics.map(m => m.name);
+ for (const expected of [
+ "nodejs_eventloop_lag_seconds",
+ "nodejs_eventloop_lag_p99_seconds",
+ "nodejs_eventloop_lag_max_seconds",
+ "nodejs_heap_size_used_bytes",
+ "nodejs_gc_duration_seconds",
+ "nodejs_active_handles_total",
+ "process_cpu_seconds_total",
+ ]) {
+ expect(names).toContain(expected);
+ }
+
+ // Registered but empty would be invisible in production too: the gauges
+ // that sample on collection must carry a value right away.
+ for (const sampled of [
+ "nodejs_heap_size_used_bytes",
+ "process_cpu_seconds_total",
+ "process_resident_memory_bytes",
+ ]) {
+ const metric = metrics.find(m => m.name === sampled);
+ expect(metric?.values.length).toBeGreaterThan(0);
+ expect(metric?.values[0].value).toBeGreaterThan(0);
+ }
+ });
+});
diff --git a/apps/api/src/lib/node-runtime-metrics.ts b/apps/api/src/lib/node-runtime-metrics.ts
new file mode 100644
index 000000000..4b57ec111
--- /dev/null
+++ b/apps/api/src/lib/node-runtime-metrics.ts
@@ -0,0 +1,22 @@
+import { collectDefaultMetrics, register } from "prom-client";
+
+let started = false;
+
+/**
+ * Export the Node runtime's own health through the default registry, next to
+ * the request histograms the app and the workers already serve.
+ *
+ * The interesting series are `nodejs_eventloop_lag_*_seconds` (sampled via
+ * `perf_hooks.monitorEventLoopDelay`), the heap and GC series and the active
+ * handle/request counts: a pod that stops answering its liveness probe while
+ * its CPU, memory and descriptor counts look normal is otherwise invisible.
+ *
+ * Idempotent: `collectDefaultMetrics` throws when a metric name is registered
+ * twice, and several entry points share this module. The flag is set only
+ * after a successful registration so a failed attempt can be retried.
+ */
+export function startNodeRuntimeMetrics(): void {
+ if (started) return;
+ collectDefaultMetrics({ register });
+ started = true;
+}
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index 6789d81c5..d7b36487b 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -8,6 +8,7 @@ import { updateExtract } from "../lib/extract/extract-redis";
import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f0";
import { createWebhookSender, WebhookEvent } from "./webhook";
import Express from "express";
+import { startNodeRuntimeMetrics } from "../lib/node-runtime-metrics";
import { getErrorContactMessage } from "../lib/deployment";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";
@@ -175,6 +176,7 @@ const processDLQJob = async (data: ExtractJobData) => {
// Start the worker
const app = Express();
+startNodeRuntimeMetrics();
app.get("/health", (req, res) => {
res.status(200).json({ ok: true });
diff --git a/apps/api/src/services/worker/nuq-reconciler-worker.ts b/apps/api/src/services/worker/nuq-reconciler-worker.ts
index 16e6f9322..fadd39cf8 100644
--- a/apps/api/src/services/worker/nuq-reconciler-worker.ts
+++ b/apps/api/src/services/worker/nuq-reconciler-worker.ts
@@ -6,6 +6,7 @@ import { reconcileConcurrencyQueue } from "../../lib/concurrency-queue-reconcile
import { repairCrawlJobDoneMarkers } from "../../lib/crawl-redis";
import { Counter, register } from "prom-client";
import Express from "express";
+import { startNodeRuntimeMetrics } from "../../lib/node-runtime-metrics";
const RECONCILER_INTERVAL_MS = 60 * 1000;
@@ -29,6 +30,7 @@ const reconcilerJobsRecoveredTotal = new Counter({
let reconcilerInFlight = false;
const app = Express();
+ startNodeRuntimeMetrics();
app.get("/metrics", async (_, res) => {
try {
diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.ts
index 259a332e1..481cb2fb7 100644
--- a/apps/api/src/services/worker/nuq-worker-runner.ts
+++ b/apps/api/src/services/worker/nuq-worker-runner.ts
@@ -7,6 +7,7 @@ import { processJobInternal } from "./scrape-worker";
import { NuQJob } from "./nuq";
import { register } from "prom-client";
import Express from "express";
+import { startNodeRuntimeMetrics } from "../../lib/node-runtime-metrics";
import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";
import { shutdownPubSubLogging } from "../logging/log_job";
@@ -74,6 +75,7 @@ export async function runNuqWorker(options: {
let shutdownStartedAt: number | undefined;
const app = Express();
+ startNodeRuntimeMetrics();
app.get("/metrics", async (_, res) => {
const localMetrics = options.metrics ? await options.metrics() : "";