Infisical is the open-source platform for securing secrets, certificates, access, and AI Agents.
TypeScript acmeagentsai-agentscertificate-managementdevsecopsenvironment-variableskubernetesopen-sourcepampkipostgresprivate-caprivileged-access-managementsecret-managementsecret-managersecretssecrets-managementsecuritysecurity-toolsvault
feat(telemetry): enhance HTTP instrumentation and metrics
Victor Hugo dos Santosd3a96a716d4a
diff --git a/backend/src/lib/telemetry/instrumentation.ts b/backend/src/lib/telemetry/instrumentation.tsindex be861a2857..f2e7a0bac4 100644--- a/backend/src/lib/telemetry/instrumentation.ts+++ b/backend/src/lib/telemetry/instrumentation.ts@@ -18,7 +18,12 @@ import tracer from "dd-trace";import dotenv from "dotenv";import { getTelemetryConfig } from "../config/env";-import { HIGH_CARDINALITY_METER_NAMES, INFISICAL_CORE_METER_ATTRIBUTES } from "./telemetry-attributes";+import {+ HIGH_CARDINALITY_METER_NAMES,+ HTTP_INSTRUMENTATION_METER_ATTRIBUTES,+ HTTP_INSTRUMENTATION_METER_NAME,+ INFISICAL_CORE_METER_ATTRIBUTES+} from "./telemetry-attributes";dotenv.config();@@ -80,6 +85,10 @@ const initTelemetryInstrumentation = ({{meterName: "InfisicalCore",attributesProcessors: [createAllowListAttributesProcessor(INFISICAL_CORE_METER_ATTRIBUTES)]+ },+ {+ meterName: HTTP_INSTRUMENTATION_METER_NAME,+ attributesProcessors: [createAllowListAttributesProcessor(HTTP_INSTRUMENTATION_METER_ATTRIBUTES)]}];@@ -107,6 +116,14 @@ const initTelemetryInstrumentation = ({opentelemetry.metrics.setGlobalMeterProvider(meterProvider);+ // HttpInstrumentation reads this in its constructor and exposes no config equivalent, so it has to be set+ // here rather than left to the deployment. Unset means the old HTTP semconv, whose http.server.duration+ // carries net.host.name, the raw client-controlled Host header, and so grows a new series for every value+ // a scanner sends. Stable drops it, normalizes the method to a fixed set, and adds error.type.+ // Assigning it (rather than defaulting it) keeps one metric name across cloud and self-hosted, and shuts+ // out "http/dup", which would emit both attribute sets and double the cardinality this is here to bound.+ process.env.OTEL_SEMCONV_STABILITY_OPT_IN = "http";+registerInstrumentations({instrumentations: [new HttpInstrumentation(), new RuntimeNodeInstrumentation()]});diff --git a/backend/src/lib/telemetry/metrics.ts b/backend/src/lib/telemetry/metrics.tsindex 941c67f210..eeb2f339f3 100644--- a/backend/src/lib/telemetry/metrics.ts+++ b/backend/src/lib/telemetry/metrics.ts@@ -440,6 +440,32 @@ export const coreHttpErrorCounter = infisicalCoreMeter.createCounter("infisical.unit: "{error}"});+// Denominator for coreHttpErrorCounter so error rate per route survives OTEL_DROP_HIGH_CARDINALITY_METERS.+// normalizeHttpMethod mirrors @opentelemetry/instrumentation-http KNOWN_METHODS (_OTHER for the rest) for joinable labels.+const KNOWN_HTTP_METHODS = new Set([+ "GET",+ "HEAD",+ "POST",+ "PUT",+ "DELETE",+ "CONNECT",+ "OPTIONS",+ "TRACE",+ "PATCH",+ "QUERY"+]);++export const normalizeHttpMethod = (method?: string): string => {+ if (!method) return "GET";+ const upper = method.toUpperCase();+ return KNOWN_HTTP_METHODS.has(upper) ? upper : "_OTHER";+};++export const coreHttpRequestCounter = infisicalCoreMeter.createCounter("infisical.core.http.request.count", {+ description: "API requests with bounded labels. Labels limited to InfisicalCore View allowlist.",+ unit: "{request}"+});+// -- Signup abuse (InfisicalCore meter) -------------------------------------------------------------export enum EmailDispatchPurpose {diff --git a/backend/src/lib/telemetry/telemetry-attributes.ts b/backend/src/lib/telemetry/telemetry-attributes.tsindex 97435eddef..7ef42c1e21 100644--- a/backend/src/lib/telemetry/telemetry-attributes.ts+++ b/backend/src/lib/telemetry/telemetry-attributes.ts@@ -54,3 +54,14 @@ export const INFISICAL_CORE_METER_ATTRIBUTES = [// labels (user.email, client.address, syncId, ...) flow through unchanged unless dropped wholesale via// OTEL_DROP_HIGH_CARDINALITY_METERS. Kept on by default for self-hosted; dropped in multi-tenant/cloud.export const HIGH_CARDINALITY_METER_NAMES = ["Infisical", "API", "SecretSyncs", "PkiSyncs", "Integrations"];++// Meter used by @opentelemetry/instrumentation-http for http.server.duration / http.client.duration.+export const HTTP_INSTRUMENTATION_METER_NAME = "@opentelemetry/instrumentation-http";++// This View keeps http.server.duration bounded if an upgrade reintroduces unbounded host related attributes like net.host.name and constant labels like url.scheme+export const HTTP_INSTRUMENTATION_METER_ATTRIBUTES = [+ "http.request.method",+ "http.response.status_code",+ "http.route",+ "error.type"+];diff --git a/backend/src/server/plugins/api-metrics.ts b/backend/src/server/plugins/api-metrics.tsindex ebe1e2d146..b8ef0d534c 100644--- a/backend/src/server/plugins/api-metrics.ts+++ b/backend/src/server/plugins/api-metrics.ts@@ -2,7 +2,13 @@ import { requestContext } from "@fastify/request-context";import fp from "fastify-plugin";import { RequestContextKey } from "@app/lib/request-context/request-context-keys";-import { highCardinalityMeter, shouldRecordHighCardinalityMetrics } from "@app/lib/telemetry/metrics";+import {+ coreHttpRequestCounter,+ highCardinalityMeter,+ isTelemetryEnabled,+ normalizeHttpMethod,+ shouldRecordHighCardinalityMetrics+} from "@app/lib/telemetry/metrics";const apiMeter = highCardinalityMeter("API");@@ -24,14 +30,23 @@ const requestDurationHistogram = infisicalMeter.createHistogram("infisical.http.export const apiMetrics = fp(async (fastify) => {fastify.addHook("onResponse", async (request, reply) => {- // Checked before assembling the attribute object below: it is built per response and every label on- // it is per-actor, so it is the largest source of the cardinality these meters would retain.- if (!shouldRecordHighCardinalityMetrics()) return;+ if (!isTelemetryEnabled()) return;- const { method } = request;+ const method = normalizeHttpMethod(request.method);const route = request.routeOptions.url;const { statusCode } = reply;+ // Recorded ahead of the high-cardinality gate: these three labels are bounded (the route is Fastify's+ // template, never the raw path), and they are the denominator for coreHttpErrorCounter, so a deployment+ // that drops the per-actor meters must not lose them too.+ coreHttpRequestCounter.add(1, {+ "http.request.method": method,+ "http.route": route ?? "unknown",+ "http.response.status_code": statusCode+ });++ if (!shouldRecordHighCardinalityMetrics()) return;+latencyHistogram.record(reply.elapsedTime, {route,method,diff --git a/backend/src/server/plugins/error-handler.ts b/backend/src/server/plugins/error-handler.tsindex c2d6dad22c..755ee23c40 100644--- a/backend/src/server/plugins/error-handler.ts+++ b/backend/src/server/plugins/error-handler.ts@@ -28,6 +28,7 @@ import { RequestContextKey } from "@app/lib/request-context/request-context-keysimport {coreHttpErrorCounter,highCardinalityMeter,+ normalizeHttpMethod,rateLimitExceededCounter,shouldRecordHighCardinalityMetrics} from "@app/lib/telemetry/metrics";@@ -108,7 +109,7 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider}if (appCfg.OTEL_TELEMETRY_COLLECTION_ENABLED) {- const { method } = req;+ const method = normalizeHttpMethod(req.method);const route = req.routeOptions.url;if (shouldRecordHighCardinalityMetrics()) {@@ -295,7 +296,7 @@ export const fastifyErrHandler = fastifyPlugin(async (server: FastifyZodProvider} else if (error instanceof RateLimitError) {rateLimitExceededCounter.add(1, {"http.route": req.routeOptions.url ?? "unknown",- "http.request.method": req.method+ "http.request.method": normalizeHttpMethod(req.method)});void res.status(HttpStatusCodes.TooManyRequests).send({reqId: req.id,diff --git a/docs/self-hosting/guides/monitoring-telemetry.mdx b/docs/self-hosting/guides/monitoring-telemetry.mdxindex e2fab44b76..fa718a2015 100644--- a/docs/self-hosting/guides/monitoring-telemetry.mdx+++ b/docs/self-hosting/guides/monitoring-telemetry.mdx@@ -461,6 +461,21 @@ These metrics track all HTTP API requests to Infisical, including request counts- `user_agent.original` (string, optional): User agent information</Accordion>+ <Accordion title="Total API Requests (InfisicalCore)">+ **Metric Name**: `infisical.core.http.request.count`++ **Type**: Counter++ **Unit**: `{request}`++ **Description**: Total API requests with bounded labels. The low-cardinality counterpart to `infisical.http.server.request.count`; same request volume, but labels are restricted to the InfisicalCore allowlist so the series count stays independent of org/user/identity count. Divide `infisical.core.http.error.count` by this to get an error rate per route. Unlike its high-cardinality twin, it is emitted even when `OTEL_DROP_HIGH_CARDINALITY_METERS=true`.++ **Attributes**:+ - `http.request.method` (string): HTTP method+ - `http.route` (string): Parameterized route template (e.g. `/api/v1/secrets/:secretName`), never the raw path. `unknown` for a request that matched no route.+ - `http.response.status_code` (int): HTTP status code+ </Accordion>+<Accordion title="API Errors (InfisicalCore)">**Metric Name**: `infisical.core.http.error.count`@@ -1073,12 +1088,36 @@ Heap usage, GC pause, event loop lag, and other Node runtime metrics are auto-emThese low-level HTTP metrics are automatically collected by OpenTelemetry's instrumentation layer, providing baseline performance data for all HTTP traffic.<AccordionGroup>- <Accordion title="http_server_duration">- HTTP server request duration metrics (histogram buckets, count, sum)+ <Accordion title="http_server_request_duration">+ **Metric Name**: `http.server.request.duration`++ **Type**: Histogram++ **Unit**: `s` (seconds)++ **Description**: Inbound HTTP request duration, measured at the socket rather than at the route handler, so it also covers requests that never matched a route.++ **Attributes**:+ - `http.request.method` (string): HTTP method, normalized to a fixed set (`_OTHER` for anything unrecognized)+ - `http.response.status_code` (int): HTTP status code+ - `error.type` (string, optional): Present only when the response failed++ Infisical opts in to the stable OpenTelemetry HTTP semantic conventions. The older conventions labelled this metric with `net.host.name`, the raw `Host` request header, which is client-controlled and grows a new time series per value; the stable conventions omit it. `http.route` is not populated, so use `infisical.core.http.request.count` for a per-route breakdown.</Accordion>-- <Accordion title="http_client_duration">- HTTP client request duration metrics (histogram buckets, count, sum)++ <Accordion title="http_client_request_duration">+ **Metric Name**: `http.client.request.duration`++ **Type**: Histogram++ **Unit**: `s` (seconds)++ **Description**: Outbound HTTP request duration.++ **Attributes**:+ - `http.request.method` (string): HTTP method, normalized to a fixed set+ - `http.response.status_code` (int): HTTP status code+ - `error.type` (string, optional): Present only when the request failed</Accordion></AccordionGroup>