Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

Expose Prometheus metrics for inbound HTTP requests (#1053)

Raman Tehlan8cd3f3334708
diff --git a/.changeset/prometheus-http-metrics.md b/.changeset/prometheus-http-metrics.md
new file mode 100644
index 000000000..38c33b7dc
--- /dev/null
+++ b/.changeset/prometheus-http-metrics.md
@@ -0,0 +1,5 @@
+---
+'@truefoundry/trueforge': patch
+---
+
+Expose Prometheus request metrics at `GET /metrics` so a scraper can see rate, status, and latency for `/api` requests.
diff --git a/charts/trueforge/README.md b/charts/trueforge/README.md
index 07ba62e00..85ed7cab8 100644
--- a/charts/trueforge/README.md
+++ b/charts/trueforge/README.md
@@ -394,6 +394,7 @@ extraObjects:
| `redis.nameOverride` | `trueforge-redis` | Bitnami name prefix for bundled Redis objects. |
| `service.type` | `ClusterIP` | Service type. |
| `service.port` | `8790` | Service port. |
+| `serviceMonitor.enabled` | `false` | Prometheus Operator ServiceMonitor for `GET /metrics`. Set `labels` so the operator's Prometheus selects the monitor. `mtls.enabled` scrapes `https` and skips certificate verification. |
| `server.port` | `8790` | Container port (`PORT`). |
| `autoscaling.enabled` | `false` | Enable a HorizontalPodAutoscaler. |
| `podDisruptionBudget.enabled` | `true` | Server PodDisruptionBudget (`minAvailable` defaults to `1`); rendered only when the server runs more than one replica. |
diff --git a/charts/trueforge/templates/_helpers.tpl b/charts/trueforge/templates/_helpers.tpl
index b36bcebc8..fc638797e 100644
--- a/charts/trueforge/templates/_helpers.tpl
+++ b/charts/trueforge/templates/_helpers.tpl
@@ -86,6 +86,25 @@ Pod labels / annotations, global merged under the chart's own.
{{- end }}
{{- end }}
+{{/*
+ServiceMonitor labels. Chart labels first, then serviceMonitor.labels, so a
+parent can add the label its Prometheus selects on.
+*/}}
+{{- define "trueforge.serviceMonitor.labels" -}}
+{{- $merged := mergeOverwrite (include "trueforge.labels" . | fromYaml) (deepCopy (.Values.serviceMonitor.labels | default dict)) -}}
+{{- toYaml $merged -}}
+{{- end }}
+
+{{/*
+ServiceMonitor annotations. Common annotations first, then serviceMonitor.annotations.
+*/}}
+{{- define "trueforge.serviceMonitor.annotations" -}}
+{{- $merged := mergeOverwrite (include "trueforge.annotations" . | fromYaml) (deepCopy (.Values.serviceMonitor.annotations | default dict)) -}}
+{{- with $merged }}
+{{- toYaml . }}
+{{- end }}
+{{- end }}
+
{{/*
Selector labels.
*/}}
diff --git a/charts/trueforge/templates/servicemonitor.yaml b/charts/trueforge/templates/servicemonitor.yaml
new file mode 100644
index 000000000..559a26cda
--- /dev/null
+++ b/charts/trueforge/templates/servicemonitor.yaml
@@ -0,0 +1,29 @@
+{{- if and .Values.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ name: {{ include "trueforge.server.fullname" . }}
+ labels:
+ {{- include "trueforge.serviceMonitor.labels" . | nindent 4 }}
+ {{- with (include "trueforge.serviceMonitor.annotations" .) }}
+ annotations:
+ {{- . | nindent 4 }}
+ {{- end }}
+spec:
+ namespaceSelector:
+ matchNames:
+ - {{ .Release.Namespace }}
+ selector:
+ matchLabels:
+ {{- include "trueforge.selectorLabels" . | nindent 6 }}
+ endpoints:
+ - port: http
+ path: {{ .Values.serviceMonitor.path | quote }}
+ honorLabels: true
+ interval: {{ .Values.serviceMonitor.interval }}
+ {{- if .Values.mtls.enabled }}
+ scheme: https
+ tlsConfig:
+ insecureSkipVerify: true
+ {{- end }}
+{{- end }}
diff --git a/charts/trueforge/values.yaml b/charts/trueforge/values.yaml
index 248a00220..386e2cd7c 100644
--- a/charts/trueforge/values.yaml
+++ b/charts/trueforge/values.yaml
@@ -86,6 +86,16 @@ service:
port: 8790
annotations: {}
labels: {}
+# Prometheus Operator ServiceMonitor for the server Service (GET /metrics).
+# Off by default. A parent chart sets labels so its Prometheus selects this
+# monitor. When mtls.enabled, the endpoint is https and skips certificate
+# verification; /metrics does not require a client certificate.
+serviceMonitor:
+ enabled: false
+ interval: 30s
+ path: /metrics
+ labels: {}
+ annotations: {}
# Server runtime configuration. Keys map to environment variables read in
# packages/trueforge/src/config.ts.
server:
diff --git a/packages/trueforge/package.json b/packages/trueforge/package.json
index 04f295c10..776bc0152 100644
--- a/packages/trueforge/package.json
+++ b/packages/trueforge/package.json
@@ -63,6 +63,7 @@
"@anthropic-ai/sandbox-runtime": "0.0.77",
"@daytona/sdk": "^0.218.0",
"@hono/node-server": "^2.1.1",
+ "@hono/prometheus": "^1.0.3",
"@hono/swagger-ui": "^0.6.1",
"@hono/zod-openapi": "^1.6.3",
"@modelcontextprotocol/sdk": "^1.31.0",
@@ -78,6 +79,7 @@
"lru-cache": "^11.5.3",
"openid-client": "^6.8.8",
"pg": "^8.23.0",
+ "prom-client": "^15.1.3",
"redis": "^5.12.1",
"ulid": "^3.0.2",
"undici": "7.30.0",
diff --git a/packages/trueforge/src/app.ts b/packages/trueforge/src/app.ts
index 39f984e11..6bcbe44b0 100644
--- a/packages/trueforge/src/app.ts
+++ b/packages/trueforge/src/app.ts
@@ -50,6 +50,7 @@ import type { ISessionMetricsStore } from './db/sessionMetricsStore';
import type { ISkillStore } from './db/skillStore';
import type { WithTransaction } from './db/transaction';
import type { IWebSearchProviderStore } from './db/webSearchProviderStore';
+import { createHttpMetrics } from './http/prometheus';
import { createClientCertificateMiddleware } from './http/tls';
import type { IOAuthTokenStore } from './mcp/auth/types';
import { PACKAGE_VERSION } from './packageVersion';
@@ -271,6 +272,9 @@ export function createServerApp<TTransaction>(deps: ServerDeps<TTransaction>) {
turnSkillsResolverStore: deps.turnSkillsResolverStore,
};
+ const httpMetrics = createHttpMetrics();
+ // Health, the scrape endpoint, and static files sit outside /api, so they never enter this middleware.
+ app.use('/api/*', httpMetrics.middleware);
if (configuration.ACCESS_LOGS) {
app.use('*', createAccessLogMiddleware(deps.logger));
}
@@ -280,6 +284,7 @@ export function createServerApp<TTransaction>(deps: ServerDeps<TTransaction>) {
app.use('*', createRequestBodyLimitMiddleware(configuration.MAX_REQUEST_BODY_BYTES));
app.get('/healthz', c => c.json({ status: 'ok', version: PACKAGE_VERSION }));
+ app.get('/metrics', httpMetrics.handler);
app.route(
'/api/v1/auth',
diff --git a/packages/trueforge/src/frontend.ts b/packages/trueforge/src/frontend.ts
index 9444d7626..0f49de2ec 100644
--- a/packages/trueforge/src/frontend.ts
+++ b/packages/trueforge/src/frontend.ts
@@ -8,7 +8,7 @@ import { existsSync, readFileSync } from 'node:fs';
import path from 'node:path';
/** Routes the server answers itself; never served from the build. /api covers every version below it. */
-const SERVER_PATH_PREFIXES = ['/api', '/healthz'];
+const SERVER_PATH_PREFIXES = ['/api', '/healthz', '/metrics'];
/** Only Vite's hashed asset names can be cached forever. */
const HASHED_ASSET_PREFIX = '/assets/';
diff --git a/packages/trueforge/src/http/prometheus.ts b/packages/trueforge/src/http/prometheus.ts
new file mode 100644
index 000000000..a124dab22
--- /dev/null
+++ b/packages/trueforge/src/http/prometheus.ts
@@ -0,0 +1,23 @@
+import { prometheus } from '@hono/prometheus';
+
+/** Buckets cover multi-minute turn handlers. Longer requests land in +Inf. */
+const DURATION_BUCKETS_SECONDS = [0.01, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300];
+
+/**
+ * `help` is shared by both standard metrics. The package's option union drops
+ * histogram-only fields, so `buckets` has to ride along with a shared field.
+ */
+const requestDuration: { help: string; buckets: number[] } = {
+ help: 'Duration of HTTP requests in seconds',
+ buckets: DURATION_BUCKETS_SECONDS,
+};
+
+export function createHttpMetrics() {
+ const { printMetrics, registerMetrics } = prometheus({
+ collectDefaultMetrics: true,
+ metricOptions: {
+ requestDuration,
+ },
+ });
+ return { middleware: registerMetrics, handler: printMetrics };
+}
diff --git a/packages/trueforge/src/http/tls.ts b/packages/trueforge/src/http/tls.ts
index 4f1e48104..8c22e109d 100644
--- a/packages/trueforge/src/http/tls.ts
+++ b/packages/trueforge/src/http/tls.ts
@@ -116,16 +116,18 @@ export function serverTlsServeOptions(
cert: Buffer.from(read(TLS_CERT_FILE)),
ca: Buffer.from(read(TLS_CA_FILE)),
requestCert: true,
- // Do not reject at handshake so /healthz probes without a cert still connect.
+ // Do not reject at handshake so /healthz probes and /metrics scrapes without a cert still connect.
rejectUnauthorized: false,
},
};
}
-/** Rejects certless requests except `/healthz`. Mount only when mTLS is enabled. */
+const CLIENT_CERT_EXEMPT_PATHS = new Set(['/healthz', '/metrics']);
+
+/** Rejects certless requests except `/healthz` and `/metrics`. Mount only when mTLS is enabled. */
export function createClientCertificateMiddleware(logger: Logger): MiddlewareHandler {
return async (c, next) => {
- if (c.req.path === '/healthz') {
+ if (CLIENT_CERT_EXEMPT_PATHS.has(c.req.path)) {
await next();
return;
}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index d67d652d9..032dab2f9 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -83,7 +83,7 @@ importers:
version: 19.3.0
tsup:
specifier: ^8.5.0
typescript:
specifier: ^5.9.3
version: 5.9.3
@@ -172,6 +172,9 @@ importers:
'@hono/node-server':
specifier: ^2.1.1
version: 2.1.1([email protected])
+ '@hono/prometheus':
+ specifier: ^1.0.3
'@hono/swagger-ui':
specifier: ^0.6.1
version: 0.6.1([email protected])
@@ -217,6 +220,9 @@ importers:
pg:
specifier: ^8.23.0
version: 8.23.0
+ prom-client:
+ specifier: ^15.1.3
+ version: 15.1.3
redis:
specifier: ^5.12.1
version: 5.12.1(@opentelemetry/[email protected])
@@ -1800,6 +1806,12 @@ packages:
peerDependencies:
hono: ^4 || ^5.0.0-0
+ resolution: {integrity: sha512-oOBMk2yJL62l6unLkFrAct5IJejSac0yYjomxZuPqDTGKWPcmPD94zQGkLQNdzPRCOW9jEUN3QfxL0Rh5kiNOA==}
+ peerDependencies:
+ hono: '>=3.0.0'
+ prom-client: ^15.0.0
+
resolution: {integrity: sha512-sJTvldu1GPeEPfyeLG7gRj+W4vEuD+JDi+JjJ3TJs/DvMUtBLs0KJO5yokGegWWdy5qrbdnQGekbhgNRmPmYKQ==}
peerDependencies:
@@ -4311,6 +4323,9 @@ packages:
resolution: {integrity: sha512-fX1Onk0tdVPC7obPWB5EbJ1z7NVhLq4m2xZLq2YXBkxzMXIGRpNMU88n0EPgWseKl12J7zXs7qrDxPK4sRs2fg==}
+ resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
+
resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==}
engines: {node: '>=18'}
@@ -6637,6 +6652,11 @@ packages:
resolution: {integrity: sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==}
engines: {node: '>=6'}
+ resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
+ engines: {node: ^16 || ^18 || >=20}
+ deprecated: prom-client has been replaced by @prometheus-io/client
+
resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==}
engines: {node: '>= 6'}
@@ -7221,6 +7241,9 @@ packages:
resolution: {integrity: sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==}
+
resolution: {integrity: sha512-myiQ6aFnxDOjdiXdTlC8ngVccQD88uHXTx5RUQOSEnarDYgJMjDagwAQszcOusjvmf4YqWsxoD1+MY+oAJRmpw==}
engines: {node: '>=10'}
@@ -9139,6 +9162,11 @@ snapshots:
dependencies:
hono: 4.13.13
+ dependencies:
+ hono: 4.13.9
+ prom-client: 15.1.3
+
dependencies:
hono: 4.13.9
@@ -11858,6 +11886,8 @@ snapshots:
dependencies:
require-from-string: 2.0.2
+
dependencies:
bytes: 3.1.2
@@ -14624,15 +14654,6 @@ snapshots:
mlly: 1.8.2
pathe: 2.0.3
- dependencies:
- lilconfig: 3.1.3
- optionalDependencies:
- jiti: 2.7.0
- postcss: 8.5.26
- tsx: 4.23.12
- yaml: 2.9.0
-
dependencies:
lilconfig: 3.1.3
@@ -14701,6 +14722,11 @@ snapshots:
+ dependencies:
+ '@opentelemetry/api': 1.9.1
+ tdigest: 0.1.3
+
dependencies:
kleur: 3.0.3
@@ -15447,6 +15473,10 @@ snapshots:
minizlib: 3.1.0
yallist: 5.0.0
+ dependencies:
+ bintrees: 1.0.2
+
dependencies:
'@jridgewell/source-map': 0.3.11
@@ -15550,35 +15580,6 @@ snapshots:
- dependencies:
- bundle-require: 5.1.0([email protected])
- cac: 6.7.14
- chokidar: 4.0.3
- consola: 3.4.2
- debug: 4.4.3([email protected])
- esbuild: 0.27.7
- fix-dts-default-cjs-exports: 1.0.1
- joycon: 3.1.1
- picocolors: 1.1.1
- resolve-from: 5.0.0
- rollup: 4.62.2
- source-map: 0.7.6
- sucrase: 3.35.1
- tinyexec: 0.3.2
- tinyglobby: 0.2.17
- tree-kill: 1.2.2
- optionalDependencies:
- '@swc/core': 1.16.2
- postcss: 8.5.26
- typescript: 5.9.3
- transitivePeerDependencies:
- - jiti
- - supports-color
- - tsx
- - yaml
-
dependencies:
bundle-require: 5.1.0([email protected])