The open-source agent harness - the runtime layer that turns an LLM into a working agent.
Expose Prometheus metrics for inbound HTTP requests (#1053)
Raman Tehlan8cd3f3334708
diff --git a/.changeset/prometheus-http-metrics.md b/.changeset/prometheus-http-metrics.mdnew file mode 100644index 000000000..38c33b7dc--- /dev/null+++ b/.changeset/prometheus-http-metrics.md@@ -0,0 +1,5 @@+---+'@truefoundry/trueforge': patch+---++Expose Prometheus request metrics at `GET /metrics` so a scraper can see rate, status, and latency for `/api` requests.diff --git a/charts/trueforge/README.md b/charts/trueforge/README.mdindex 07ba62e00..85ed7cab8 100644--- a/charts/trueforge/README.md+++ b/charts/trueforge/README.md@@ -394,6 +394,7 @@ extraObjects:| `redis.nameOverride` | `trueforge-redis` | Bitnami name prefix for bundled Redis objects. || `service.type` | `ClusterIP` | Service type. || `service.port` | `8790` | Service port. |+| `serviceMonitor.enabled` | `false` | Prometheus Operator ServiceMonitor for `GET /metrics`. Set `labels` so the operator's Prometheus selects the monitor. `mtls.enabled` scrapes `https` and skips certificate verification. || `server.port` | `8790` | Container port (`PORT`). || `autoscaling.enabled` | `false` | Enable a HorizontalPodAutoscaler. || `podDisruptionBudget.enabled` | `true` | Server PodDisruptionBudget (`minAvailable` defaults to `1`); rendered only when the server runs more than one replica. |diff --git a/charts/trueforge/templates/_helpers.tpl b/charts/trueforge/templates/_helpers.tplindex b36bcebc8..fc638797e 100644--- a/charts/trueforge/templates/_helpers.tpl+++ b/charts/trueforge/templates/_helpers.tpl@@ -86,6 +86,25 @@ Pod labels / annotations, global merged under the chart's own.{{- end }}{{- end }}+{{/*+ServiceMonitor labels. Chart labels first, then serviceMonitor.labels, so a+parent can add the label its Prometheus selects on.+*/}}+{{- define "trueforge.serviceMonitor.labels" -}}+{{- $merged := mergeOverwrite (include "trueforge.labels" . | fromYaml) (deepCopy (.Values.serviceMonitor.labels | default dict)) -}}+{{- toYaml $merged -}}+{{- end }}++{{/*+ServiceMonitor annotations. Common annotations first, then serviceMonitor.annotations.+*/}}+{{- define "trueforge.serviceMonitor.annotations" -}}+{{- $merged := mergeOverwrite (include "trueforge.annotations" . | fromYaml) (deepCopy (.Values.serviceMonitor.annotations | default dict)) -}}+{{- with $merged }}+{{- toYaml . }}+{{- end }}+{{- end }}+{{/*Selector labels.*/}}diff --git a/charts/trueforge/templates/servicemonitor.yaml b/charts/trueforge/templates/servicemonitor.yamlnew file mode 100644index 000000000..559a26cda--- /dev/null+++ b/charts/trueforge/templates/servicemonitor.yaml@@ -0,0 +1,29 @@+{{- if and .Values.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") }}+apiVersion: monitoring.coreos.com/v1+kind: ServiceMonitor+metadata:+ name: {{ include "trueforge.server.fullname" . }}+ labels:+ {{- include "trueforge.serviceMonitor.labels" . | nindent 4 }}+ {{- with (include "trueforge.serviceMonitor.annotations" .) }}+ annotations:+ {{- . | nindent 4 }}+ {{- end }}+spec:+ namespaceSelector:+ matchNames:+ - {{ .Release.Namespace }}+ selector:+ matchLabels:+ {{- include "trueforge.selectorLabels" . | nindent 6 }}+ endpoints:+ - port: http+ path: {{ .Values.serviceMonitor.path | quote }}+ honorLabels: true+ interval: {{ .Values.serviceMonitor.interval }}+ {{- if .Values.mtls.enabled }}+ scheme: https+ tlsConfig:+ insecureSkipVerify: true+ {{- end }}+{{- end }}diff --git a/charts/trueforge/values.yaml b/charts/trueforge/values.yamlindex 248a00220..386e2cd7c 100644--- a/charts/trueforge/values.yaml+++ b/charts/trueforge/values.yaml@@ -86,6 +86,16 @@ service:port: 8790annotations: {}labels: {}+# Prometheus Operator ServiceMonitor for the server Service (GET /metrics).+# Off by default. A parent chart sets labels so its Prometheus selects this+# monitor. When mtls.enabled, the endpoint is https and skips certificate+# verification; /metrics does not require a client certificate.+serviceMonitor:+ enabled: false+ interval: 30s+ path: /metrics+ labels: {}+ annotations: {}# Server runtime configuration. Keys map to environment variables read in# packages/trueforge/src/config.ts.server:diff --git a/packages/trueforge/package.json b/packages/trueforge/package.jsonindex 04f295c10..776bc0152 100644--- a/packages/trueforge/package.json+++ b/packages/trueforge/package.json@@ -63,6 +63,7 @@"@anthropic-ai/sandbox-runtime": "0.0.77","@daytona/sdk": "^0.218.0","@hono/node-server": "^2.1.1",+ "@hono/prometheus": "^1.0.3","@hono/swagger-ui": "^0.6.1","@hono/zod-openapi": "^1.6.3","@modelcontextprotocol/sdk": "^1.31.0",@@ -78,6 +79,7 @@"lru-cache": "^11.5.3","openid-client": "^6.8.8","pg": "^8.23.0",+ "prom-client": "^15.1.3","redis": "^5.12.1","ulid": "^3.0.2","undici": "7.30.0",diff --git a/packages/trueforge/src/app.ts b/packages/trueforge/src/app.tsindex 39f984e11..6bcbe44b0 100644--- a/packages/trueforge/src/app.ts+++ b/packages/trueforge/src/app.ts@@ -50,6 +50,7 @@ import type { ISessionMetricsStore } from './db/sessionMetricsStore';import type { ISkillStore } from './db/skillStore';import type { WithTransaction } from './db/transaction';import type { IWebSearchProviderStore } from './db/webSearchProviderStore';+import { createHttpMetrics } from './http/prometheus';import { createClientCertificateMiddleware } from './http/tls';import type { IOAuthTokenStore } from './mcp/auth/types';import { PACKAGE_VERSION } from './packageVersion';@@ -271,6 +272,9 @@ export function createServerApp<TTransaction>(deps: ServerDeps<TTransaction>) {turnSkillsResolverStore: deps.turnSkillsResolverStore,};+ const httpMetrics = createHttpMetrics();+ // Health, the scrape endpoint, and static files sit outside /api, so they never enter this middleware.+ app.use('/api/*', httpMetrics.middleware);if (configuration.ACCESS_LOGS) {app.use('*', createAccessLogMiddleware(deps.logger));}@@ -280,6 +284,7 @@ export function createServerApp<TTransaction>(deps: ServerDeps<TTransaction>) {app.use('*', createRequestBodyLimitMiddleware(configuration.MAX_REQUEST_BODY_BYTES));app.get('/healthz', c => c.json({ status: 'ok', version: PACKAGE_VERSION }));+ app.get('/metrics', httpMetrics.handler);app.route('/api/v1/auth',diff --git a/packages/trueforge/src/frontend.ts b/packages/trueforge/src/frontend.tsindex 9444d7626..0f49de2ec 100644--- a/packages/trueforge/src/frontend.ts+++ b/packages/trueforge/src/frontend.ts@@ -8,7 +8,7 @@ import { existsSync, readFileSync } from 'node:fs';import path from 'node:path';/** Routes the server answers itself; never served from the build. /api covers every version below it. */-const SERVER_PATH_PREFIXES = ['/api', '/healthz'];+const SERVER_PATH_PREFIXES = ['/api', '/healthz', '/metrics'];/** Only Vite's hashed asset names can be cached forever. */const HASHED_ASSET_PREFIX = '/assets/';diff --git a/packages/trueforge/src/http/prometheus.ts b/packages/trueforge/src/http/prometheus.tsnew file mode 100644index 000000000..a124dab22--- /dev/null+++ b/packages/trueforge/src/http/prometheus.ts@@ -0,0 +1,23 @@+import { prometheus } from '@hono/prometheus';++/** Buckets cover multi-minute turn handlers. Longer requests land in +Inf. */+const DURATION_BUCKETS_SECONDS = [0.01, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300];++/**+ * `help` is shared by both standard metrics. The package's option union drops+ * histogram-only fields, so `buckets` has to ride along with a shared field.+ */+const requestDuration: { help: string; buckets: number[] } = {+ help: 'Duration of HTTP requests in seconds',+ buckets: DURATION_BUCKETS_SECONDS,+};++export function createHttpMetrics() {+ const { printMetrics, registerMetrics } = prometheus({+ collectDefaultMetrics: true,+ metricOptions: {+ requestDuration,+ },+ });+ return { middleware: registerMetrics, handler: printMetrics };+}diff --git a/packages/trueforge/src/http/tls.ts b/packages/trueforge/src/http/tls.tsindex 4f1e48104..8c22e109d 100644--- a/packages/trueforge/src/http/tls.ts+++ b/packages/trueforge/src/http/tls.ts@@ -116,16 +116,18 @@ export function serverTlsServeOptions(cert: Buffer.from(read(TLS_CERT_FILE)),ca: Buffer.from(read(TLS_CA_FILE)),requestCert: true,- // Do not reject at handshake so /healthz probes without a cert still connect.+ // Do not reject at handshake so /healthz probes and /metrics scrapes without a cert still connect.rejectUnauthorized: false,},};}-/** Rejects certless requests except `/healthz`. Mount only when mTLS is enabled. */+const CLIENT_CERT_EXEMPT_PATHS = new Set(['/healthz', '/metrics']);++/** Rejects certless requests except `/healthz` and `/metrics`. Mount only when mTLS is enabled. */export function createClientCertificateMiddleware(logger: Logger): MiddlewareHandler {return async (c, next) => {- if (c.req.path === '/healthz') {+ if (CLIENT_CERT_EXEMPT_PATHS.has(c.req.path)) {await next();return;}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex d67d652d9..032dab2f9 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -83,7 +83,7 @@ importers:version: 19.3.0tsup:specifier: ^8.5.0- version: 8.5.1(@swc/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])+ version: 8.5.1(@swc/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected])typescript:specifier: ^5.9.3version: 5.9.3@@ -172,6 +172,9 @@ importers:'@hono/node-server':specifier: ^2.1.1version: 2.1.1([email protected])+ '@hono/prometheus':+ specifier: ^1.0.3+ version: 1.0.3([email protected])([email protected])'@hono/swagger-ui':specifier: ^0.6.1version: 0.6.1([email protected])@@ -217,6 +220,9 @@ importers:pg:specifier: ^8.23.0version: 8.23.0+ prom-client:+ specifier: ^15.1.3+ version: 15.1.3redis:specifier: ^5.12.1version: 5.12.1(@opentelemetry/[email protected])@@ -1800,6 +1806,12 @@ packages:peerDependencies:hono: ^4 || ^5.0.0-0+ '@hono/[email protected]':+ resolution: {integrity: sha512-oOBMk2yJL62l6unLkFrAct5IJejSac0yYjomxZuPqDTGKWPcmPD94zQGkLQNdzPRCOW9jEUN3QfxL0Rh5kiNOA==}+ peerDependencies:+ hono: '>=3.0.0'+ prom-client: ^15.0.0+'@hono/[email protected]':resolution: {integrity: sha512-sJTvldu1GPeEPfyeLG7gRj+W4vEuD+JDi+JjJ3TJs/DvMUtBLs0KJO5yokGegWWdy5qrbdnQGekbhgNRmPmYKQ==}peerDependencies:@@ -4311,6 +4323,9 @@ packages:resolution: {integrity: sha512-fX1Onk0tdVPC7obPWB5EbJ1z7NVhLq4m2xZLq2YXBkxzMXIGRpNMU88n0EPgWseKl12J7zXs7qrDxPK4sRs2fg==}+ resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}+resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==}engines: {node: '>=18'}@@ -6637,6 +6652,11 @@ packages:resolution: {integrity: sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==}engines: {node: '>=6'}+ resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}+ engines: {node: ^16 || ^18 || >=20}+ deprecated: prom-client has been replaced by @prometheus-io/client+resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==}engines: {node: '>= 6'}@@ -7221,6 +7241,9 @@ packages:resolution: {integrity: sha512-MFO/QzvtAOmJbkhOaCTvbGcFN9L9b+JunIsDwaKljSOdcLMea3NJ1k9Usz/rjdfSXTq4dfzfeS7W4p4YOAAHeA==}engines: {node: '>=18'}+ resolution: {integrity: sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==}+resolution: {integrity: sha512-myiQ6aFnxDOjdiXdTlC8ngVccQD88uHXTx5RUQOSEnarDYgJMjDagwAQszcOusjvmf4YqWsxoD1+MY+oAJRmpw==}engines: {node: '>=10'}@@ -9139,6 +9162,11 @@ snapshots:dependencies:hono: 4.13.13+ dependencies:+ hono: 4.13.9+ prom-client: 15.1.3+'@hono/[email protected]([email protected])':dependencies:hono: 4.13.9@@ -11858,6 +11886,8 @@ snapshots:dependencies:require-from-string: 2.0.2+ [email protected]: {}+dependencies:bytes: 3.1.2@@ -14624,15 +14654,6 @@ snapshots:mlly: 1.8.2pathe: 2.0.3- dependencies:- lilconfig: 3.1.3- optionalDependencies:- jiti: 2.7.0- postcss: 8.5.26- tsx: 4.23.12- yaml: 2.9.0-dependencies:lilconfig: 3.1.3@@ -14701,6 +14722,11 @@ snapshots:+ dependencies:+ '@opentelemetry/api': 1.9.1+ tdigest: 0.1.3+dependencies:kleur: 3.0.3@@ -15447,6 +15473,10 @@ snapshots:minizlib: 3.1.0yallist: 5.0.0+ dependencies:+ bintrees: 1.0.2+dependencies:'@jridgewell/source-map': 0.3.11@@ -15550,35 +15580,6 @@ snapshots:- [email protected](@swc/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]):- dependencies:- bundle-require: 5.1.0([email protected])- cac: 6.7.14- chokidar: 4.0.3- consola: 3.4.2- debug: 4.4.3([email protected])- esbuild: 0.27.7- fix-dts-default-cjs-exports: 1.0.1- joycon: 3.1.1- picocolors: 1.1.1- postcss-load-config: 6.0.1([email protected])([email protected])([email protected])([email protected])- resolve-from: 5.0.0- rollup: 4.62.2- source-map: 0.7.6- sucrase: 3.35.1- tinyexec: 0.3.2- tinyglobby: 0.2.17- tree-kill: 1.2.2- optionalDependencies:- '@swc/core': 1.16.2- postcss: 8.5.26- typescript: 5.9.3- transitivePeerDependencies:- - jiti- - supports-color- - tsx- - yaml-[email protected](@swc/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]):dependencies:bundle-require: 5.1.0([email protected])