Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): add operational logs, traces, and metrics (#17490)

Hassieb Pakzadd0092797ffee
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lock
index 3a4d768c93..b20a6bdff7 100644
--- a/ai-gateway/Cargo.lock
+++ b/ai-gateway/Cargo.lock
@@ -2,6 +2,15 @@
# It is not intended for manual editing.
version = 4
+[[package]]
+name = "aho-corasick"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "ai-gateway"
version = "0.1.0"
@@ -10,18 +19,46 @@ dependencies = [
"chrono",
"futures-util",
"hmac",
+ "http-body",
"http-body-util",
+ "opentelemetry",
+ "opentelemetry-http",
+ "opentelemetry-otlp",
+ "opentelemetry-proto",
+ "opentelemetry_sdk",
+ "prost",
"reqwest",
+ "reqwest-middleware",
+ "reqwest-tracing",
"serde",
"serde_json",
"sha2",
"tokio",
"tower",
+ "tower-http 0.7.1",
"tracing",
+ "tracing-opentelemetry",
"tracing-subscriber",
"uuid",
]
+[[package]]
+name = "anyhow"
+version = "1.0.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
+
+[[package]]
+name = "async-trait"
+version = "0.1.92"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.5",
+]
+
[[package]]
name = "atomic-waker"
version = "1.1.2"
@@ -34,6 +71,29 @@ version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+[[package]]
+name = "aws-lc-rs"
+version = "1.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
+dependencies = [
+ "aws-lc-sys",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-lc-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
+dependencies = [
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+ "pkg-config",
+]
+
[[package]]
name = "axum"
version = "0.8.9"
@@ -88,6 +148,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
[[package]]
name = "bitflags"
version = "2.13.2"
@@ -122,6 +188,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80"
dependencies = [
"find-msvc-tools",
+ "jobserver",
+ "libc",
"shlex",
]
@@ -145,7 +213,7 @@ checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
dependencies = [
"cfg-if",
"cpufeatures",
- "rand_core",
+ "rand_core 0.10.1",
]
[[package]]
@@ -157,12 +225,31 @@ dependencies = [
"num-traits",
]
+[[package]]
+name = "cmake"
+version = "0.1.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "cmov"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
+[[package]]
+name = "combine"
+version = "4.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e"
+dependencies = [
+ "bytes",
+ "memchr",
+]
+
[[package]]
name = "const-oid"
version = "0.10.2"
@@ -235,6 +322,18 @@ dependencies = [
"syn 3.0.5",
]
+[[package]]
+name = "dunce"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
+
+[[package]]
+name = "either"
+version = "1.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
+
[[package]]
name = "errno"
version = "0.3.14"
@@ -260,6 +359,12 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
[[package]]
name = "futures-channel"
version = "0.3.34"
@@ -267,6 +372,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"
dependencies = [
"futures-core",
+ "futures-sink",
]
[[package]]
@@ -275,6 +381,17 @@ version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
+[[package]]
+name = "futures-executor"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
[[package]]
name = "futures-io"
version = "0.3.34"
@@ -333,6 +450,18 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 5.3.0",
+ "wasip2",
+]
+
[[package]]
name = "getrandom"
version = "0.4.3"
@@ -342,8 +471,8 @@ dependencies = [
"cfg-if",
"js-sys",
"libc",
- "r-efi",
- "rand_core",
+ "r-efi 6.0.0",
+ "rand_core 0.10.1",
"wasm-bindgen",
]
@@ -441,7 +570,6 @@ dependencies = [
"hyper",
"hyper-util",
"rustls",
- "rustls-native-certs",
"tokio",
"tokio-rustls",
"tower-service",
@@ -453,7 +581,7 @@ version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
- "base64",
+ "base64 0.22.1",
"bytes",
"futures-channel",
"futures-util",
@@ -580,12 +708,80 @@ version = "2.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"
+[[package]]
+name = "itertools"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+[[package]]
+name = "jni"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
+dependencies = [
+ "cfg-if",
+ "combine",
+ "jni-macros",
+ "jni-sys",
+ "log",
+ "simd_cesu8",
+ "thiserror",
+ "walkdir",
+ "windows-link",
+]
+
+[[package]]
+name = "jni-macros"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "simd_cesu8",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
+dependencies = [
+ "jni-sys-macros",
+]
+
+[[package]]
+name = "jni-sys-macros"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
+dependencies = [
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "jobserver"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
+dependencies = [
+ "getrandom 0.4.3",
+ "libc",
+]
+
[[package]]
name = "js-sys"
version = "0.3.105"
@@ -627,6 +823,15 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
+[[package]]
+name = "matchers"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
+dependencies = [
+ "regex-automata",
+]
+
[[package]]
name = "matchit"
version = "0.8.4"
@@ -677,6 +882,77 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+[[package]]
+name = "opentelemetry"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "js-sys",
+ "pin-project-lite",
+ "thiserror",
+ "tracing",
+]
+
+[[package]]
+name = "opentelemetry-http"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "http",
+ "opentelemetry",
+ "reqwest",
+]
+
+[[package]]
+name = "opentelemetry-otlp"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35"
+dependencies = [
+ "http",
+ "opentelemetry",
+ "opentelemetry-http",
+ "opentelemetry-proto",
+ "opentelemetry_sdk",
+ "prost",
+ "reqwest",
+ "thiserror",
+]
+
+[[package]]
+name = "opentelemetry-proto"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"
+dependencies = [
+ "opentelemetry",
+ "opentelemetry_sdk",
+ "prost",
+]
+
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.32.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9"
+dependencies = [
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "opentelemetry",
+ "percent-encoding",
+ "portable-atomic",
+ "rand 0.9.5",
+ "thiserror",
+ "tokio",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -689,6 +965,18 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+[[package]]
+name = "pkg-config"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
+
+[[package]]
+name = "portable-atomic"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
+
[[package]]
name = "potential_utf"
version = "0.1.6"
@@ -698,6 +986,15 @@ dependencies = [
"zerovec",
]
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
[[package]]
name = "proc-macro2"
version = "1.0.107"
@@ -707,6 +1004,29 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "prost"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
[[package]]
name = "quinn"
version = "0.11.11"
@@ -733,10 +1053,11 @@ version = "0.11.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83"
dependencies = [
+ "aws-lc-rs",
"bytes",
"getrandom 0.4.3",
"lru-slab",
- "rand",
+ "rand 0.10.2",
"rand_pcg",
"ring",
"rustc-hash",
@@ -772,12 +1093,28 @@ dependencies = [
"proc-macro2",
]
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
[[package]]
name = "r-efi"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+[[package]]
+name = "rand"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
+dependencies = [
+ "rand_chacha",
+ "rand_core 0.9.5",
+]
+
[[package]]
name = "rand"
version = "0.10.2"
@@ -786,7 +1123,26 @@ checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"
dependencies = [
"chacha20",
"getrandom 0.4.3",
- "rand_core",
+ "rand_core 0.10.1",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
+dependencies = [
+ "getrandom 0.3.4",
]
[[package]]
@@ -801,17 +1157,35 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
dependencies = [
- "rand_core",
+ "rand_core 0.10.1",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
]
+[[package]]
+name = "regex-syntax"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+
[[package]]
name = "reqwest"
-version = "0.12.28"
+version = "0.13.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
+checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"
dependencies = [
- "base64",
+ "base64 0.23.1",
"bytes",
+ "futures-channel",
"futures-core",
"futures-util",
"http",
@@ -826,17 +1200,14 @@ dependencies = [
"pin-project-lite",
"quinn",
"rustls",
- "rustls-native-certs",
"rustls-pki-types",
- "serde",
- "serde_json",
- "serde_urlencoded",
+ "rustls-platform-verifier",
"sync_wrapper",
"tokio",
"tokio-rustls",
"tokio-util",
"tower",
- "tower-http",
+ "tower-http 0.6.11",
"tower-service",
"url",
"wasm-bindgen",
@@ -845,6 +1216,38 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "reqwest-middleware"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "http",
+ "reqwest",
+ "thiserror",
+ "tower-service",
+]
+
+[[package]]
+name = "reqwest-tracing"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5e5af0cd6fc3d3c8f703d597af70b6e4e62432c63157b49419fa1ffaf481702"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "getrandom 0.2.17",
+ "http",
+ "matchit",
+ "opentelemetry",
+ "reqwest",
+ "reqwest-middleware",
+ "tracing",
+ "tracing-opentelemetry",
+]
+
[[package]]
name = "ring"
version = "0.17.14"
@@ -865,14 +1268,23 @@ version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
[[package]]
name = "rustls"
-version = "0.23.44"
+version = "0.23.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba"
+checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
dependencies = [
+ "aws-lc-rs",
"once_cell",
- "ring",
"rustls-pki-types",
"rustls-webpki",
"subtle",
@@ -901,12 +1313,40 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "rustls-platform-verifier"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
+dependencies = [
+ "core-foundation",
+ "core-foundation-sys",
+ "jni",
+ "log",
+ "once_cell",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-platform-verifier-android",
+ "rustls-webpki",
+ "security-framework",
+ "security-framework-sys",
+ "webpki-root-certs",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "rustls-platform-verifier-android"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
+
[[package]]
name = "rustls-webpki"
version = "0.103.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
dependencies = [
+ "aws-lc-rs",
"ring",
"rustls-pki-types",
"untrusted",
@@ -919,10 +1359,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
[[package]]
-name = "ryu"
-version = "1.0.23"
+name = "same-file"
+version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
[[package]]
name = "schannel"
@@ -956,6 +1399,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "semver"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
+
[[package]]
name = "serde"
version = "1.0.229"
@@ -1010,18 +1459,6 @@ dependencies = [
"serde_core",
]
-[[package]]
-name = "serde_urlencoded"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
-dependencies = [
- "form_urlencoded",
- "itoa",
- "ryu",
- "serde",
-]
-
[[package]]
name = "sha2"
version = "0.11.0"
@@ -1058,6 +1495,22 @@ dependencies = [
"libc",
]
+[[package]]
+name = "simd_cesu8"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520"
+dependencies = [
+ "rustc_version",
+ "simdutf8",
+]
+
+[[package]]
+name = "simdutf8"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+
[[package]]
name = "slab"
version = "0.4.12"
@@ -1271,6 +1724,23 @@ dependencies = [
"url",
]
+[[package]]
+name = "tower-http"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
+dependencies = [
+ "bitflags",
+ "bytes",
+ "http",
+ "http-body",
+ "percent-encoding",
+ "pin-project-lite",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
[[package]]
name = "tower-layer"
version = "0.3.3"
@@ -1315,6 +1785,33 @@ dependencies = [
"valuable",
]
+[[package]]
+name = "tracing-log"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
+dependencies = [
+ "log",
+ "once_cell",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-opentelemetry"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"
+dependencies = [
+ "js-sys",
+ "opentelemetry",
+ "smallvec",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-subscriber",
+ "web-time",
+]
+
[[package]]
name = "tracing-serde"
version = "0.2.0"
@@ -1331,10 +1828,14 @@ version = "0.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
dependencies = [
+ "matchers",
+ "once_cell",
+ "regex-automata",
"serde",
"serde_json",
"sharded-slab",
"thread_local",
+ "tracing",
"tracing-core",
"tracing-serde",
]
@@ -1398,6 +1899,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
[[package]]
name = "want"
version = "0.3.1"
@@ -1413,6 +1924,15 @@ version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+[[package]]
+name = "wasip2"
+version = "1.0.4+wasi-0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
+dependencies = [
+ "wit-bindgen",
+]
+
[[package]]
name = "wasm-bindgen"
version = "0.2.128"
@@ -1470,9 +1990,9 @@ dependencies = [
[[package]]
name = "wasm-streams"
-version = "0.4.2"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
+checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
dependencies = [
"futures-util",
"js-sys",
@@ -1501,6 +2021,24 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "webpki-root-certs"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "windows-link"
version = "0.2.1"
@@ -1589,6 +2127,12 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+
[[package]]
name = "writeable"
version = "0.6.4"
@@ -1618,6 +2162,26 @@ dependencies = [
"synstructure",
]
+[[package]]
+name = "zerocopy"
+version = "0.8.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.57"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
[[package]]
name = "zerofrom"
version = "0.1.8"
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.toml
index c56efb94eb..975d5aaedc 100644
--- a/ai-gateway/Cargo.toml
+++ b/ai-gateway/Cargo.toml
@@ -6,21 +6,33 @@ publish = false
license = "MIT"
[dependencies]
-axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] }
+axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "matched-path", "tokio"] }
chrono = { version = "0.4", default-features = false, features = ["std"] }
hmac = "0.13"
http-body-util = "0.1"
futures-util = "0.3"
-reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-native-roots", "stream"] }
+opentelemetry-http = "0.32"
+opentelemetry = { version = "0.32", default-features = false, features = ["trace", "metrics", "internal-logs"] }
+opentelemetry_sdk = { version = "0.32", default-features = false, features = ["trace", "metrics", "internal-logs"] }
+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "metrics", "http-proto", "reqwest-blocking-client", "internal-logs"] }
+reqwest = { version = "0.13.5", default-features = false, features = ["rustls", "stream"] }
+reqwest-middleware = "0.5"
+reqwest-tracing = { version = "0.7", default-features = false, features = ["opentelemetry_0_32"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "0.11"
tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }
+tower-http = { version = "0.7", features = ["trace"] }
tracing = "0.1.44"
-tracing-subscriber = { version = "0.3.23", default-features = false, features = ["fmt", "json"] }
+tracing-opentelemetry = { version = "0.33", default-features = false }
+tracing-subscriber = { version = "0.3.23", default-features = false, features = ["env-filter", "fmt", "json"] }
uuid = { version = "1", features = ["v4"] }
[dev-dependencies]
+http-body = "1"
+opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic-messages", "trace", "metrics"] }
+opentelemetry_sdk = { version = "0.32", default-features = false, features = ["testing"] }
+prost = "0.14"
tokio = { version = "1.53.1", features = ["io-util"] }
tower = { version = "0.5", features = ["util"] }
diff --git a/ai-gateway/Dockerfile b/ai-gateway/Dockerfile
index 49d1447925..25b7491895 100644
--- a/ai-gateway/Dockerfile
+++ b/ai-gateway/Dockerfile
@@ -8,7 +8,7 @@ COPY --chown=gateway:gateway src ./src
RUN cargo build --release --locked
FROM debian:bookworm-slim AS runtime
-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /app/target/release/ai-gateway /usr/local/bin/ai-gateway
USER 10001:10001
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
index 263bfa86e7..cce2022dac 100644
--- a/ai-gateway/README.md
+++ b/ai-gateway/README.md
@@ -3,7 +3,7 @@
A standalone Rust gateway for `POST /openai/v1/responses`. Web resolves the gateway
key to a trusted provider connection; Rust relays native JSON or SSE without
rewriting provider bytes. A bounded capture layer captures request/response facts and
-logs the finalized record at debug level. Each finalized execution is immediately
+logs capture completeness at debug level. Each finalized execution is immediately
uploaded as a Langfuse generation through OTLP. Building and testing need no real
Web, database or provider credentials.
@@ -67,7 +67,7 @@ do not load dotenv files:
| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |
| `LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT` | `false` | `true`, `false`; the pnpm dev task sets `true` |
| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |
-| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |
+| `LANGFUSE_LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error`, `fatal` |
| `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 |
| `LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS` | `128` | Positive integer up to Tokio's semaphore capacity; authenticated requests per instance |
| `LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS` | `128` | Positive integer up to Tokio's semaphore capacity; concurrent Web resolutions per instance |
@@ -84,11 +84,64 @@ Both formats use the same log level and preserve event fields. For example:
```
Invalid values fail startup without echoing their contents. At the default `info`
-level, logs contain service lifecycle events. Debug logging also includes the
-captured request and completed output when Web resolves `ingestion_mode: full`;
-see capture details below. For direct
+level, logs contain service lifecycle events and HTTP response summaries.
+Debug logging adds content-free phase and capture details. For direct
host-only development, set `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS=127.0.0.1:8080`.
+## Operational observability
+
+Logs and operational OpenTelemetry exports describe gateway health independently
+of the inference generations sent to Langfuse. They do not include request/response
+content, credentials, key metadata, or URL query strings. Health probes are excluded
+from request logs and spans.
+
+Use `LANGFUSE_LOG_FORMAT=text` locally and `json` for log collectors. Built-in `tracing-subscriber` formatters handle both modes. JSON event fields
+are top-level attributes; span fields are nested. HTTP response and execution
+summaries include top-level `trace_id` and `span_id` for Datadog correlation.
+`info` emits lifecycle, HTTP response, and execution summaries; `debug` adds
+sanitized capture details. Other log events do not automatically include trace IDs. Dependency logs stay at `warn`. Log severity does not control
+trace sampling. `trace` is not an accepted gateway log level.
+
+| Variable | Default | Purpose |
+| --- | --- | --- |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | unset | Enables operational traces and metrics; HTTP/protobuf base URL, e.g. `http://localhost:4318` |
+| `OTEL_SERVICE_NAME` | `ai-gateway` | Operational service name |
+| `DD_ENV` | `development` | Deployment environment resource attribute |
+| `BUILD_ID` | Cargo package version | Service version; set to the deployed image's commit SHA |
+| `OTEL_TRACES_SAMPLER_ARG` | `1` | Sampling ratio from 0 to 1 for root traces; incoming W3C sampling decisions are respected |
+
+Tower's `TraceLayer` keeps the server span alive through the response body.
+The HTTP response log and `http.server.request.duration` measure time to response
+headers, not full SSE duration. Execution summaries and `gateway.phase.duration`
+with `phase=execution` measure the provider relay through completion, cancellation,
+timeout, or transport error. HTTP status and stream outcome are separate: a 200
+response can still fail while streaming.
+
+`reqwest-tracing` instruments resolver, provider-header, and ingestion requests.
+Only trusted Web calls receive W3C trace context; baggage and operational trace
+headers are not sent to the external provider. Inference generations retain their
+separate identity. Client spans measure the HTTP send through response headers;
+streaming execution and inference-telemetry delivery are tracked separately.
+
+A small Axum middleware records `http.server.request.duration` through the
+OpenTelemetry SDK, with method, matched route, and HTTP status as its only
+dimensions. The histogram's count supplies request volume and error rates,
+including responses rejected before provider execution. Metrics are independent
+of trace sampling. Health probes are excluded.
+
+Gateway-specific metrics include `gateway.active` (resolution/execution),
+`gateway.phase.duration` (execution/provider.first_byte), `gateway.executions`,
+`gateway.admission.rejected`, and `gateway.telemetry.records`. Durations use seconds;
+first-byte timing measures provider bytes arriving at the gateway, not first-token
+delivery to the caller. Metric attributes contain bounded categories, not tenant
+or request identifiers.
+
+Trace export uses a 512-span queue, batches of up to 64, and a one-second interval.
+Metrics export every 30 seconds. Export calls use a three-second timeout; failures
+do not fail inference. Shutdown flushes within the remaining shared drain deadline.
+Telemetry delivery warnings report the first failure/drop and every hundredth;
+metrics count each occurrence. This is best-effort operational telemetry.
+
## Call the gateway
Set these in the repository root `.env` for `pnpm dev` (or export them for Cargo):
@@ -206,17 +259,15 @@ compatible grant. Capture and provider byte forwarding do not need to change.
## Response capture
Set `LANGFUSE_LOG_LEVEL=debug` to print one `gateway response captured` event at
-execution end. Its `capture` field is JSON with `input`, `output`, model,
-parameters, usage, attribution, timestamps, capture completeness and relay outcome.
-Text logs pretty-print the captured JSON across multiple lines.
-`LANGFUSE_LOG_FORMAT=json` keeps each surrounding log event on one line, with
-newlines escaped inside the `capture` string.
+execution end with HTTP status, timing, capture completeness and relay outcome.
+Operational logs exclude prompts, outputs, model parameters and key metadata,
+including at debug level. `LANGFUSE_LOG_FORMAT=json` emits one JSON object per line.
Web's resolved ingestion mode controls content capture:
- `full`: input is the native request JSON object, including `input`, `instructions`,
tools, parameters, context references and unknown fields. Output is an ordered
- array of native completed items. Debug logs intentionally contain this content.
+ array of native completed items. Content is sent only through Langfuse ingestion.
- `usage`: input and output are null. Model, scalar parameters, native usage,
timing and trusted attribution are retained; request schemas/content are omitted.
@@ -250,7 +301,7 @@ fully captured response, and downstream cancellation does not erase completed ca
The implementation separates shared `ExecutionCapture` lifecycle/timing, the
`OpenAiResponsesCapture` adapter and bounded `SseDecoder`. A private `ProtocolCapture`
enum dispatches to the adapter. Finalization hands an owned `InferenceFacts` record
-to telemetry for debug logging and immediate upload. Another provider can supply
+to telemetry for a safe debug summary and immediate upload. Another provider can supply
native facts through the same interface without changing the relay. The current
usage mapping targets OpenAI Responses; no Anthropic adapter is implemented yet.
Capture and upload run independently of log level; debug emission alone is gated.
@@ -265,8 +316,7 @@ these limits never cap the actual provider response. Media is not fetched/upload
Provider credentials, the gateway credential and ingestion tokens are not part of
the capture record; raw headers are not logged. Full captured content and resolved
-key metadata may themselves be sensitive. Keep debug logging enabled only where
-that content is intended to be recorded. No data is posted to Langfuse yet.
+key metadata remain in Langfuse ingestion and are excluded from operational logs.
## Process lifecycle
diff --git a/ai-gateway/scripts/smoke-image.sh b/ai-gateway/scripts/smoke-image.sh
index 70c740b016..15baa74f05 100644
--- a/ai-gateway/scripts/smoke-image.sh
+++ b/ai-gateway/scripts/smoke-image.sh
@@ -20,6 +20,7 @@ for route in health ready; do
*) echo "Unexpected probe response: $route" >&2; exit 1 ;;
esac
done
+docker exec "$container" sh -c 'wget -qO /dev/null http://localhost:8080/health || exit 1'
[[ "$(docker exec "$container" id -u)" != 0 ]]
[[ "$(curl --silent --output /dev/null --write-out '%{http_code}' \
--max-time 2 -X POST "http://$port/openai/v1/responses")" == 401 ]]
@@ -27,4 +28,4 @@ docker stop --time 5 "$container" >/dev/null
[[ "$(docker inspect --format '{{.State.ExitCode}}' "$container")" == 0 ]]
docker logs "$container" 2>&1 | grep -q 'gateway draining'
docker logs "$container" 2>&1 | grep -q 'gateway stopped'
-printf '%s\n' 'Image smoke: non-root, probes, unauthenticated inference, SIGTERM passed'
+printf '%s\n' 'Image smoke: non-root, probes, ECS health command, unauthenticated inference, SIGTERM passed'
diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rs
index 03683f4151..61c9c62c59 100644
--- a/ai-gateway/src/capture/mod.rs
+++ b/ai-gateway/src/capture/mod.rs
@@ -55,6 +55,7 @@ impl ProtocolCapture {
/// Owned before dispatch and moved into the response body. Drop also covers a
/// cancelled provider future before response headers have arrived.
pub(crate) struct ExecutionCapture {
+ span: tracing::Span,
protocol: Option<ProtocolCapture>,
started: Instant,
start_time_unix_ms: u128,
@@ -92,6 +93,7 @@ impl ExecutionCapture {
.collect();
let full = context.ingestion_mode() == IngestionMode::Full;
Self {
+ span: tracing::Span::current(),
protocol: Some(ProtocolCapture::OpenAiResponses(
OpenAiResponsesCapture::new(headers, body, context.ingestion_mode()),
)),
@@ -146,6 +148,8 @@ impl ExecutionCapture {
}
pub fn finish(&mut self, outcome: RelayOutcome) {
+ let span = self.span.clone();
+ let _entered = span.enter();
let Some(protocol) = self.protocol.take() else {
return;
};
@@ -161,6 +165,7 @@ impl ExecutionCapture {
inference,
};
telemetry::debug_record(&facts);
+ crate::observability::execution_finished(&facts);
if let Some((telemetry, context)) = self.delivery.take() {
telemetry.record(context, facts);
}
diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rs
index 89c78611d4..d03fdd9a65 100644
--- a/ai-gateway/src/capture/openai_responses/tests.rs
+++ b/ai-gateway/src/capture/openai_responses/tests.rs
@@ -95,6 +95,10 @@ async fn captures_completed_items_only_in_order_without_duplication() {
let stored = json!({"record":capture.facts,"items":capture.items}).to_string();
assert!(!stored.contains("unfinished-canary"));
assert!(!stored.contains("terminal-output-must-not-be-copied"));
+ let (_, facts) = observer.protocol.take().unwrap().into_facts();
+ let output = facts.output.unwrap();
+ assert_eq!(output[0]["content"][0]["text"], "héllo 🌍");
+ assert_eq!(output[1]["content"][0]["text"], "second");
}
#[tokio::test]
@@ -250,9 +254,15 @@ async fn debug_record_is_emitted_once_and_only_at_debug_level() {
for level in [tracing::Level::DEBUG, tracing::Level::INFO] {
let mut observer = observer("full").await;
response(&mut observer, "text/event-stream");
- observer.bytes(completed_item(0, "captured-output").as_bytes());
+ observer.bytes(completed_item(0, "output-canary").as_bytes());
observer.bytes(terminal("completed", 1).as_bytes());
observer.end_body();
+ observer.metadata["key_metadata"] = json!({"secret": "metadata-canary"});
+ let ProtocolCapture::OpenAiResponses(capture) = observer.protocol.as_mut().unwrap();
+ capture
+ .facts
+ .model_parameters
+ .insert("user".into(), json!("parameter-canary"));
let writer = LogWriter::default();
let subscriber = tracing_subscriber::fmt()
.json()
@@ -266,25 +276,28 @@ async fn debug_record_is_emitted_once_and_only_at_debug_level() {
});
let text = String::from_utf8(writer.0.lock().unwrap().clone()).unwrap();
if level == tracing::Level::DEBUG {
- assert_eq!(text.lines().count(), 1);
- let event: Value = serde_json::from_str(&text).unwrap();
- let capture: Value =
- serde_json::from_str(event["fields"]["capture"].as_str().unwrap()).unwrap();
- assert_eq!(
- capture["output"][0]["content"][0]["text"],
- "captured-output"
- );
+ let events = records(&writer);
+ assert_eq!(events.len(), 1);
+ let capture = &events[0];
assert_eq!(capture["outcome"], "eof");
- assert_eq!(capture["input"]["instructions"], "prompt-canary");
+ assert_eq!(capture["capture_complete"], true);
+ for excluded in ["input", "output", "model_parameters", "metadata"] {
+ assert!(capture.get(excluded).is_none());
+ }
for secret in [
+ "prompt-canary",
+ "schema-canary",
+ "output-canary",
+ "metadata-canary",
+ "parameter-canary",
"provider-secret",
"private-ingestion-token",
"gateway-secret",
] {
- assert!(!text.contains(secret));
+ assert!(!text.contains(secret), "operational logs leaked {secret}");
}
} else {
- assert!(text.is_empty());
+ assert!(records(&writer).is_empty());
}
}
}
@@ -295,9 +308,8 @@ fn records(writer: &LogWriter) -> Vec<Value> {
.lines()
.filter_map(|line| {
let event: Value = serde_json::from_str(line).unwrap();
- event["fields"]["capture"]
- .as_str()
- .map(|capture| serde_json::from_str(capture).unwrap())
+ (event["fields"]["message"] == "gateway response captured")
+ .then(|| event["fields"].clone())
})
.collect()
}
@@ -411,10 +423,6 @@ async fn native_http_relay_logs_once_on_eof_drop_and_unpolled_deadline() {
assert_eq!(records.len(), 1);
assert_eq!(records[0]["outcome"], json!(outcome));
if outcome == RelayOutcome::Eof {
- assert_eq!(
- records[0]["output"][0]["content"][0]["text"],
- "native-result"
- );
assert_eq!(records[0]["output_complete"], true);
assert!(records[0]["first_byte_ms"].is_number());
}
@@ -481,6 +489,17 @@ async fn provider_future_finalizes_on_timeout_and_cancellation_before_headers()
}
}
+fn uploaded_attribute(upload: &Value, key: &str) -> Value {
+ let attributes = upload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]
+ .as_array()
+ .unwrap();
+ let attribute = attributes
+ .iter()
+ .find(|attribute| attribute["key"] == key)
+ .unwrap();
+ serde_json::from_str(attribute["value"]["stringValue"].as_str().unwrap()).unwrap()
+}
+
#[tokio::test]
async fn client_compression_preferences_do_not_disable_capture() {
for streaming in [false, true] {
@@ -512,24 +531,34 @@ async fn client_compression_preferences_do_not_disable_capture() {
}
})
.await;
+ let uploaded = Arc::new(Mutex::new(Value::Null));
+ let received = uploaded.clone();
+ let collector = FakeServer::start(move |request| {
+ let received = received.clone();
+ async move {
+ let bytes = to_bytes(request.into_body(), 64 * 1024).await.unwrap();
+ *received.lock().unwrap() = serde_json::from_slice(&bytes).unwrap();
+ Response::new(Body::from("{}"))
+ }
+ })
+ .await;
+ let telemetry = crate::telemetry::Telemetry::new(
+ &crate::resolution::ControlPlaneConfig::new(&collector.url, "test-service-key")
+ .unwrap(),
+ )
+ .unwrap();
let context = resolved_request_context_with_mode("provider-secret", "full").await;
let provider = OpenAiProvider::for_test(
format!("{}/v1/responses", upstream.url),
ProviderLimits::default(),
- );
- let writer = LogWriter::default();
- let subscriber = tracing_subscriber::fmt()
- .json()
- .with_max_level(tracing::Level::DEBUG)
- .with_writer(writer.clone())
- .finish();
- let _guard = tracing::subscriber::set_default(subscriber);
+ )
+ .with_telemetry(telemetry.clone());
let mut headers = HeaderMap::new();
headers.insert(
header::ACCEPT_ENCODING,
HeaderValue::from_static("gzip, deflate, br"),
);
- let response = provider
+ let forwarded = provider
.forward(
provider.try_admit().unwrap(),
context,
@@ -539,18 +568,22 @@ async fn client_compression_preferences_do_not_disable_capture() {
.await
.unwrap();
assert_eq!(
- to_bytes(response.into_body(), 4096).await.unwrap(),
+ to_bytes(forwarded.into_body(), 4096).await.unwrap(),
expected
);
- let records = records(&writer);
- assert_eq!(records.len(), 1);
- assert_eq!(records[0]["capture_complete"], true);
- assert_eq!(records[0]["output_complete"], true);
- assert_eq!(records[0]["provider_response_id"], "resp-1");
- assert_eq!(records[0]["provider_status"], "completed");
- assert!(records[0]["usage_details"]["input_tokens"].is_number());
+ telemetry
+ .shutdown(tokio::time::Instant::now() + Duration::from_secs(2))
+ .await;
+ assert_eq!(collector.calls(), 1);
+ let upload = uploaded.lock().unwrap();
+ let metadata = uploaded_attribute(&upload, "langfuse.observation.metadata");
+ assert_eq!(metadata["capture_complete"], true);
+ assert_eq!(metadata["output_complete"], true);
+ assert_eq!(metadata["provider_response_id"], "resp-1");
+ assert_eq!(metadata["provider_status"], "completed");
+ assert!(metadata["native_usage"]["input_tokens"].is_number());
assert_eq!(
- records[0]["output"][0]["content"][0]["text"],
+ uploaded_attribute(&upload, "langfuse.observation.output")[0]["content"][0]["text"],
"captured output"
);
}
@@ -660,7 +693,6 @@ async fn downstream_cancellation_does_not_erase_completed_capture() {
let facts = records(&writer);
assert_eq!(facts.len(), 1);
assert_eq!(facts[0]["outcome"], "cancelled");
- assert_eq!(facts[0]["provider_status"], "completed");
assert_eq!(facts[0]["output_complete"], true);
assert_eq!(facts[0]["capture_complete"], true);
}
diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rs
index 9deb1dd077..aa8a2fd709 100644
--- a/ai-gateway/src/config.rs
+++ b/ai-gateway/src/config.rs
@@ -105,7 +105,6 @@ impl GatewayConfig {
));
}
let log_level = match log_level.unwrap_or("info") {
- "trace" => LevelFilter::TRACE,
"debug" => LevelFilter::DEBUG,
"info" => LevelFilter::INFO,
"warn" => LevelFilter::WARN,
@@ -113,7 +112,7 @@ impl GatewayConfig {
"error" | "fatal" => LevelFilter::ERROR,
_ => {
return Err(GatewayConfigError(
- "LANGFUSE_LOG_LEVEL must be trace, debug, info, warn, error or fatal",
+ "LANGFUSE_LOG_LEVEL must be debug, info, warn, error or fatal",
));
}
};
@@ -232,7 +231,6 @@ mod tests {
#[test]
fn accepts_shared_langfuse_log_levels() {
for (value, expected) in [
- ("trace", LevelFilter::TRACE),
("debug", LevelFilter::DEBUG),
("info", LevelFilter::INFO),
("warn", LevelFilter::WARN),
@@ -254,6 +252,7 @@ mod tests {
(None, Some(sensitive_input), None),
(None, None, Some(sensitive_input)),
(None, None, Some("off")),
+ (None, None, Some("trace")),
(None, None, Some("INFO")),
(None, None, Some("3")),
(None, None, Some("")),
diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rs
index f56f4138cd..aad0966047 100644
--- a/ai-gateway/src/http.rs
+++ b/ai-gateway/src/http.rs
@@ -121,9 +121,46 @@ struct OpenAiErrorDetail {
code: &'static str,
}
+impl InferenceHttpError {
+ fn category(&self) -> (&'static str, &'static str) {
+ use ResolutionError as R;
+ match self {
+ Self::Unavailable => ("admission", "unavailable"),
+ Self::Credential => ("authentication", "invalid_credential"),
+ Self::TooLarge => ("request", "body_too_large"),
+ Self::RequestTimeout => ("request", "body_timeout"),
+ Self::InvalidBody => ("request", "invalid_body"),
+ Self::Resolution(error) => (
+ "resolution",
+ match error {
+ R::InvalidCredential | R::Authentication => "authentication",
+ R::Forbidden => "forbidden",
+ R::NoRoute => "no_route",
+ R::Timeout => "timeout",
+ R::Unavailable => "unavailable",
+ R::Configuration => "configuration",
+ R::Transport => "transport",
+ R::InvalidResponse => "invalid_response",
+ R::ResponseTooLarge => "response_too_large",
+ },
+ ),
+ Self::Provider(error) => (
+ "provider",
+ match error {
+ ProviderError::Busy => "capacity",
+ ProviderError::Timeout => "timeout",
+ ProviderError::Transport => "transport",
+ ProviderError::Configuration => "configuration",
+ },
+ ),
+ }
+ }
+}
+
impl IntoResponse for InferenceHttpError {
fn into_response(self) -> Response<Body> {
use ResolutionError as R;
+ let (phase, reason) = self.category();
let (status, message, kind, code) = match self {
Self::Credential | Self::Resolution(R::InvalidCredential | R::Authentication) => (
StatusCode::UNAUTHORIZED,
@@ -183,6 +220,21 @@ impl IntoResponse for InferenceHttpError {
"upstream_error",
),
};
+ if status.is_server_error() {
+ tracing::warn!(
+ phase,
+ reason,
+ status = status.as_u16(),
+ "gateway request rejected"
+ );
+ } else {
+ tracing::debug!(
+ phase,
+ reason,
+ status = status.as_u16(),
+ "gateway request rejected"
+ );
+ }
(
status,
Json(OpenAiErrorResponse {
diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rs
index 37fba9035c..b7bc47246e 100644
--- a/ai-gateway/src/inference.rs
+++ b/ai-gateway/src/inference.rs
@@ -58,10 +58,11 @@ impl InferenceService {
gateway_key: &str,
) -> Result<(RequestPermit, ResolvedRequestContext), RequestPreparationError> {
let context = {
- let _permit = self
- .resolution_capacity
- .try_acquire()
- .map_err(|_| RequestPreparationError::Resolution(ResolutionError::Unavailable))?;
+ let _permit = self.resolution_capacity.try_acquire().map_err(|_| {
+ crate::observability::rejected("resolution");
+ RequestPreparationError::Resolution(ResolutionError::Unavailable)
+ })?;
+ let _active = crate::observability::Active::new("resolution");
self.control_plane
.resolve(gateway_key, ApiFormat::OpenAiResponses)
.await
diff --git a/ai-gateway/src/lib.rs b/ai-gateway/src/lib.rs
index 1d06512e43..0b9caa6ed0 100644
--- a/ai-gateway/src/lib.rs
+++ b/ai-gateway/src/lib.rs
@@ -2,6 +2,7 @@ mod capture;
pub mod config;
pub mod http;
pub mod inference;
+pub mod observability;
pub mod providers;
pub mod resolution;
pub mod server;
diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rs
index 94c87f689f..71c7193383 100644
--- a/ai-gateway/src/main.rs
+++ b/ai-gateway/src/main.rs
@@ -1,9 +1,10 @@
use std::{error::Error, io, net::SocketAddr, process::ExitCode};
use ai_gateway::{
- config::{GatewayConfig, LogFormat},
+ config::GatewayConfig,
http,
inference::InferenceService,
+ observability,
server::{self, GatewayLifecycleState},
};
use tokio::net::TcpListener;
@@ -21,13 +22,7 @@ async fn main() -> ExitCode {
async fn run() -> Result<(), Box<dyn Error>> {
let config = GatewayConfig::from_env()?;
- let logging = tracing_subscriber::fmt()
- .with_max_level(config.log_level)
- .with_target(false);
- match config.log_format {
- LogFormat::Text => logging.compact().init(),
- LogFormat::Json => logging.json().init(),
- }
+ let observability = observability::init(&config)?;
let inference = config
.control_plane
.map(|control_plane| {
@@ -45,10 +40,13 @@ async fn run() -> Result<(), Box<dyn Error>> {
} else {
GatewayLifecycleState::unconfigured()
};
- let app = server::router(state.clone()).merge(http::router(inference, state.clone()));
+ let app = server::router(state.clone()).merge(observability::instrument(http::router(
+ inference,
+ state.clone(),
+ )));
let shutdown = shutdown_signal()?;
let listener = bind_listener(config.listen_address, config.auto_increment_listen_port).await?;
- tracing::info!(address = %listener.local_addr()?, inference_enabled, "gateway listening");
+ tracing::info!(address = %listener.local_addr()?, inference_enabled, max_active_requests = config.max_active_requests, max_concurrent_resolutions = config.max_concurrent_resolutions, shutdown_timeout_seconds = config.shutdown_timeout.as_secs(), "gateway listening");
let result = server::serve(
listener,
app,
@@ -66,9 +64,19 @@ async fn run() -> Result<(), Box<dyn Error>> {
)
.await;
}
- result?;
- tracing::info!("gateway stopped");
- Ok(())
+ if result.is_err() {
+ tracing::error!("gateway shutdown failed");
+ } else {
+ tracing::info!("gateway stopped");
+ }
+ observability
+ .shutdown(
+ state
+ .drain_deadline()
+ .unwrap_or_else(tokio::time::Instant::now),
+ )
+ .await;
+ result.map_err(Into::into)
}
async fn bind_listener(
diff --git a/ai-gateway/src/observability/client.rs b/ai-gateway/src/observability/client.rs
new file mode 100644
index 0000000000..45d5e3c4eb
--- /dev/null
+++ b/ai-gateway/src/observability/client.rs
@@ -0,0 +1,161 @@
+use axum::http::Extensions;
+use reqwest::{Client, Request, Response};
+use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, Extension, Result};
+use reqwest_tracing::{
+ DefaultSpanBackend, OtelName, ReqwestOtelSpanBackend, TracingMiddleware,
+ default_on_request_success,
+};
+use tracing::Span;
+
+pub(crate) fn instrument_client(client: Client, name: &'static str) -> ClientWithMiddleware {
+ ClientBuilder::new(client)
+ .with_init(Extension(OtelName(name.into())))
+ .with(TracingMiddleware::<SafeSpanBackend>::new())
+ .build()
+}
+
+struct SafeSpanBackend;
+
+impl ReqwestOtelSpanBackend for SafeSpanBackend {
+ fn on_request_start(request: &Request, extensions: &mut Extensions) -> Span {
+ DefaultSpanBackend::on_request_start(request, extensions)
+ }
+
+ fn on_request_end(span: &Span, outcome: &Result<Response>, _: &mut Extensions) {
+ match outcome {
+ Ok(response) => {
+ default_on_request_success(span, response);
+ span.record(
+ "http.response.status_code",
+ i64::from(response.status().as_u16()),
+ );
+ }
+ // Default failure fields include URLs and raw cause chains.
+ Err(error) => {
+ span.record("otel.status_code", "ERROR");
+ span.record(
+ "error.message",
+ if error.is_timeout() {
+ "timeout"
+ } else {
+ "transport"
+ },
+ );
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use axum::{Router, http::HeaderMap, routing::post};
+ use opentelemetry::{
+ KeyValue,
+ baggage::BaggageExt,
+ trace::{FutureExt, TraceContextExt, TracerProvider},
+ };
+ use opentelemetry_sdk::{
+ propagation::TraceContextPropagator,
+ trace::{InMemorySpanExporter, SdkTracerProvider},
+ };
+ use reqwest_tracing::DisableOtelPropagation;
+ use tracing::instrument::WithSubscriber;
+ use tracing_opentelemetry::OpenTelemetrySpanExt;
+ use tracing_subscriber::prelude::*;
+
+ #[tokio::test]
+ async fn middleware_preserves_safe_context_and_sanitizes_transport_errors() {
+ opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let subscriber = tracing::Dispatch::new(
+ tracing_subscriber::registry()
+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test"))),
+ );
+ let parent = tracing::dispatcher::with_default(&subscriber, || {
+ tracing::info_span!("request").context()
+ })
+ .with_baggage([KeyValue::new("customer", "secret-baggage")]);
+ // A detached upload can inherit trace IDs without extending the server span.
+ assert_eq!(exporter.get_finished_spans().unwrap().len(), 1);
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let endpoint = format!("http://{}/", listener.local_addr().unwrap());
+ let (headers_tx, mut headers_rx) = tokio::sync::mpsc::unbounded_channel();
+ let server = tokio::spawn(async move {
+ axum::serve(
+ listener,
+ Router::new().route(
+ "/",
+ post(move |headers: HeaderMap| async move {
+ headers_tx.send(headers).unwrap();
+ axum::http::StatusCode::NO_CONTENT
+ }),
+ ),
+ )
+ .await
+ .unwrap();
+ });
+ let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let failed_url = format!(
+ "http://{}:unused@{}/private-content?token=secret-query",
+ "secret-user",
+ closed.local_addr().unwrap()
+ );
+ drop(closed);
+ async {
+ let client = Client::builder().no_proxy().build().unwrap();
+ instrument_client(client.clone(), "trusted")
+ .post(&endpoint)
+ .send()
+ .await
+ .unwrap();
+ let trusted = headers_rx.recv().await.unwrap();
+ assert!(trusted.contains_key("traceparent"));
+ assert!(!trusted.contains_key("baggage"));
+ instrument_client(client.clone(), "external")
+ .post(&endpoint)
+ .with_extension(DisableOtelPropagation)
+ .send()
+ .await
+ .unwrap();
+ let external = headers_rx.recv().await.unwrap();
+ assert!(!external.contains_key("traceparent"));
+ assert!(!external.contains_key("baggage"));
+ assert!(
+ instrument_client(client, "failed")
+ .post(failed_url)
+ .send()
+ .await
+ .is_err()
+ );
+ }
+ .with_context(parent.clone())
+ .with_subscriber(subscriber)
+ .await;
+ server.abort();
+ let spans = exporter.get_finished_spans().unwrap();
+ assert_eq!(spans.len(), 4);
+ for span in &spans[1..] {
+ assert_eq!(span.parent_span_id, parent.span().span_context().span_id());
+ assert_eq!(
+ span.span_context.trace_id(),
+ parent.span().span_context().trace_id()
+ );
+ }
+ let failed = spans.iter().find(|span| span.name == "failed").unwrap();
+ assert!(matches!(
+ failed.status,
+ opentelemetry::trace::Status::Error { .. }
+ ));
+ let attributes = format!("{:?}", failed.attributes);
+ assert!(attributes.contains("transport"));
+ for secret in ["secret-user", "unused", "private-content", "secret-query"] {
+ assert!(!attributes.contains(secret), "leaked {secret}");
+ }
+ assert!(!attributes.contains("error.cause_chain"));
+ provider.shutdown().unwrap();
+ }
+}
diff --git a/ai-gateway/src/observability/http.rs b/ai-gateway/src/observability/http.rs
new file mode 100644
index 0000000000..bfa2800049
--- /dev/null
+++ b/ai-gateway/src/observability/http.rs
@@ -0,0 +1,73 @@
+use std::time::{Duration, Instant};
+
+use axum::{
+ Router,
+ body::Body,
+ extract::MatchedPath,
+ http::Request,
+ middleware::{self, Next},
+ response::Response,
+};
+use opentelemetry::{propagation::TextMapPropagator, trace::TraceContextExt};
+use opentelemetry_http::HeaderExtractor;
+use opentelemetry_sdk::propagation::TraceContextPropagator;
+use tower_http::trace::TraceLayer;
+use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+/// Instrument application routes; mount health probes outside this router.
+pub fn instrument(router: Router) -> Router {
+ router
+ .layer(middleware::from_fn(record_http_duration))
+ .layer(
+ TraceLayer::new_for_http()
+ .make_span_with(|request: &Request<Body>| {
+ let (method, route) = request_labels(request);
+ let span = tracing::info_span!("http.server", otel.name = %format_args!("{method} {route}"), otel.kind = "server", http.request.method = method, http.route = route, http.response.status_code = tracing::field::Empty, otel.status_code = tracing::field::Empty, provider_request_id = tracing::field::Empty);
+ let parent = TraceContextPropagator::new().extract(&HeaderExtractor(request.headers()));
+ let _ = span.set_parent(parent);
+ span
+ })
+ .on_request(())
+ .on_response(|response: &Response, latency: Duration, span: &Span| {
+ let status = response.status();
+ span.record("http.response.status_code", i64::from(status.as_u16()));
+ if status.is_server_error() {
+ span.record("otel.status_code", "ERROR");
+ }
+ let context = span.context();
+ let context_span = context.span();
+ let ids = context_span.span_context();
+ tracing::info!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), status = status.as_u16(), duration_ms = latency.as_secs_f64() * 1000.0, "gateway response started");
+ })
+ .on_body_chunk(())
+ .on_eos(())
+ .on_failure(()),
+ )
+}
+
+async fn record_http_duration(request: Request<Body>, next: Next) -> Response {
+ let (method, route) = request_labels(&request);
+ let (method, route) = (method.to_owned(), route.to_owned());
+ let start = Instant::now();
+ let response = next.run(request).await;
+ super::metrics::http_response(start.elapsed(), method, route, response.status().as_u16());
+ response
+}
+
+fn request_labels(request: &Request<Body>) -> (&str, &str) {
+ let route = request
+ .extensions()
+ .get::<MatchedPath>()
+ .map_or("unmatched", MatchedPath::as_str);
+ let method = match request.method().as_str() {
+ "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" | "CONNECT" | "TRACE" => {
+ request.method().as_str()
+ }
+ _ => "_OTHER",
+ };
+ (method, route)
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/ai-gateway/src/observability/http/tests.rs b/ai-gateway/src/observability/http/tests.rs
new file mode 100644
index 0000000000..1c92a28973
--- /dev/null
+++ b/ai-gateway/src/observability/http/tests.rs
@@ -0,0 +1,211 @@
+use super::*;
+use axum::body::Bytes;
+use axum::{
+ Router,
+ http::{HeaderMap, Request as HttpRequest},
+};
+use http_body::Frame;
+use http_body_util::{BodyExt, StreamBody};
+use opentelemetry::trace::{SpanKind, TracerProvider};
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};
+use serde_json::Value;
+use std::{
+ convert::Infallible,
+ io,
+ sync::{Arc, Mutex},
+};
+use tower::ServiceExt;
+use tracing_subscriber::prelude::*;
+
+#[derive(Clone, Default)]
+struct Buffer(Arc<Mutex<Vec<u8>>>);
+impl io::Write for Buffer {
+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+ self.0.lock().unwrap().extend_from_slice(bytes);
+ Ok(bytes.len())
+ }
+ fn flush(&mut self) -> io::Result<()> {
+ Ok(())
+ }
+}
+
+struct Recording {
+ _guard: tracing::subscriber::DefaultGuard,
+ _provider: SdkTracerProvider,
+ exporter: InMemorySpanExporter,
+ output: Buffer,
+}
+impl Recording {
+ fn start() -> Self {
+ let exporter = InMemorySpanExporter::default();
+ let provider = SdkTracerProvider::builder()
+ .with_simple_exporter(exporter.clone())
+ .build();
+ let output = Buffer::default();
+ let buffer = output.clone();
+ let subscriber = tracing_subscriber::registry()
+ .with(
+ tracing_subscriber::fmt::layer()
+ .json()
+ .flatten_event(true)
+ .with_writer(move || buffer.clone()),
+ )
+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));
+ Self {
+ _guard: tracing::subscriber::set_default(subscriber),
+ _provider: provider,
+ exporter,
+ output,
+ }
+ }
+ fn summaries(&self) -> Vec<Value> {
+ String::from_utf8(self.output.0.lock().unwrap().clone())
+ .unwrap()
+ .lines()
+ .map(|line| serde_json::from_str::<Value>(line).unwrap())
+ .filter(|event| event["message"] == "gateway response started")
+ .collect()
+ }
+}
+
+fn incoming() -> HttpRequest<Body> {
+ HttpRequest::builder()
+ .method("POST")
+ .uri("/openai/v1/responses")
+ .header(
+ "traceparent",
+ "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
+ )
+ .header("baggage", "private=content-canary")
+ .body(Body::empty())
+ .unwrap()
+}
+
+#[tokio::test]
+async fn inherits_trace_and_preserves_stream_lifetime_bytes_and_trailers() {
+ let recording = Recording::start();
+ let app = instrument(Router::new().fallback(|| async {
+ let mut trailers = HeaderMap::new();
+ trailers.insert("x-checksum", "verified".parse().unwrap());
+ let frames = [
+ Ok::<_, Infallible>(Frame::data(Bytes::from_static(b"native bytes"))),
+ Ok(Frame::trailers(trailers)),
+ ];
+ Body::new(StreamBody::new(futures_util::stream::iter(frames)))
+ }));
+ let response = app.oneshot(incoming()).await.unwrap();
+ assert!(recording.exporter.get_finished_spans().unwrap().is_empty());
+ let summaries = recording.summaries();
+ assert_eq!(summaries.len(), 1);
+ assert_eq!(summaries[0]["status"], 200);
+ let mut body = response.into_body();
+ assert_eq!(
+ body.frame().await.unwrap().unwrap().into_data().unwrap(),
+ "native bytes"
+ );
+ let trailers = body
+ .frame()
+ .await
+ .unwrap()
+ .unwrap()
+ .into_trailers()
+ .unwrap();
+ assert_eq!(trailers["x-checksum"], "verified");
+ assert!(body.frame().await.is_none());
+ drop(body);
+ let spans = recording.exporter.get_finished_spans().unwrap();
+ assert_eq!(spans.len(), 1);
+ let server = &spans[0];
+ assert_eq!(server.span_kind, SpanKind::Server);
+ assert_eq!(
+ server.span_context.trace_id().to_string(),
+ "4bf92f3577b34da6a3ce929d0e0e4736"
+ );
+ assert_eq!(server.parent_span_id.to_string(), "00f067aa0ba902b7");
+ assert_eq!(
+ summaries[0]["trace_id"],
+ server.span_context.trace_id().to_string()
+ );
+ assert_eq!(
+ summaries[0]["span_id"],
+ server.span_context.span_id().to_string()
+ );
+}
+
+#[tokio::test]
+async fn cancellation_releases_spans_before_and_after_headers() {
+ for before_headers in [true, false] {
+ let recording = Recording::start();
+ let app = instrument(Router::new().fallback(move || async move {
+ if before_headers {
+ std::future::pending::<()>().await;
+ }
+ Body::from_stream(futures_util::stream::pending::<Result<Bytes, Infallible>>())
+ }));
+ if before_headers {
+ let mut call = Box::pin(app.oneshot(incoming()));
+ assert!(futures_util::poll!(call.as_mut()).is_pending());
+ drop(call);
+ } else {
+ let response = app.oneshot(incoming()).await.unwrap();
+ assert!(recording.exporter.get_finished_spans().unwrap().is_empty());
+ drop(response);
+ }
+ assert_eq!(recording.exporter.get_finished_spans().unwrap().len(), 1);
+ }
+}
+
+#[tokio::test]
+async fn provider_http_errors_are_traced_without_changing_the_response() {
+ use crate::{
+ providers::openai::{OpenAiProvider, ProviderLimits},
+ test_support::{FakeServer, resolved_request_context},
+ };
+ for status in [429, 500] {
+ let context = resolved_request_context("provider-secret").await;
+ let upstream = FakeServer::start(move |_| async move {
+ Response::builder()
+ .status(status)
+ .body(Body::from("native error"))
+ .unwrap()
+ })
+ .await;
+ let provider = OpenAiProvider::for_test(upstream.url.clone(), ProviderLimits::default());
+ let recording = Recording::start();
+ let response = provider
+ .forward(
+ provider.try_admit().unwrap(),
+ context,
+ &HeaderMap::new(),
+ Bytes::from_static(b"{}"),
+ )
+ .await
+ .unwrap();
+ assert_eq!(response.status().as_u16(), status);
+ assert_eq!(
+ axum::body::to_bytes(response.into_body(), 100)
+ .await
+ .unwrap(),
+ "native error"
+ );
+ let spans = recording.exporter.get_finished_spans().unwrap();
+ let upstream_span = spans
+ .iter()
+ .find(|span| span.name == "provider.headers")
+ .unwrap();
+ assert!(matches!(
+ upstream_span.status,
+ opentelemetry::trace::Status::Error { .. }
+ ));
+ assert!(
+ upstream_span
+ .attributes
+ .iter()
+ .any(
+ |attribute| attribute.key.as_str() == "http.response.status_code"
+ && attribute.value == opentelemetry::Value::I64(i64::from(status))
+ )
+ );
+ assert_eq!(upstream.calls(), 1);
+ }
+}
diff --git a/ai-gateway/src/observability/metrics.rs b/ai-gateway/src/observability/metrics.rs
new file mode 100644
index 0000000000..d780259b15
--- /dev/null
+++ b/ai-gateway/src/observability/metrics.rs
@@ -0,0 +1,130 @@
+use std::sync::LazyLock;
+
+use crate::capture::RelayOutcome;
+use opentelemetry::trace::TraceContextExt;
+use opentelemetry::{
+ KeyValue, global,
+ metrics::{Counter, Histogram, UpDownCounter},
+};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+struct Metrics {
+ http_duration: Histogram<f64>,
+ active: UpDownCounter<i64>,
+ phases: Histogram<f64>,
+ rejections: Counter<u64>,
+ delivery: Counter<u64>,
+ executions: Counter<u64>,
+}
+
+static METRICS: LazyLock<Metrics> = LazyLock::new(|| {
+ let meter = global::meter("ai-gateway");
+ let seconds = vec![
+ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 600.0,
+ ];
+ Metrics {
+ http_duration: meter
+ .f64_histogram("http.server.request.duration")
+ .with_unit("s")
+ .with_boundaries(vec![
+ 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0,
+ ])
+ .build(),
+ active: meter.i64_up_down_counter("gateway.active").build(),
+ phases: meter
+ .f64_histogram("gateway.phase.duration")
+ .with_unit("s")
+ .with_boundaries(seconds)
+ .build(),
+ rejections: meter.u64_counter("gateway.admission.rejected").build(),
+ delivery: meter.u64_counter("gateway.telemetry.records").build(),
+ executions: meter.u64_counter("gateway.executions").build(),
+ }
+});
+
+pub(super) fn http_response(
+ duration: std::time::Duration,
+ method: String,
+ route: String,
+ status: u16,
+) {
+ METRICS.http_duration.record(
+ duration.as_secs_f64(),
+ &[
+ KeyValue::new("http.request.method", method),
+ KeyValue::new("http.route", route),
+ KeyValue::new("http.response.status_code", i64::from(status)),
+ ],
+ );
+}
+
+pub(crate) struct Active(&'static str);
+
+impl Active {
+ pub(crate) fn new(phase: &'static str) -> Self {
+ METRICS.active.add(1, &[KeyValue::new("phase", phase)]);
+ Self(phase)
+ }
+}
+
+impl Drop for Active {
+ fn drop(&mut self) {
+ METRICS.active.add(-1, &[KeyValue::new("phase", self.0)]);
+ }
+}
+
+pub(crate) fn phase_finished(phase: &'static str, duration: std::time::Duration) {
+ METRICS
+ .phases
+ .record(duration.as_secs_f64(), &[KeyValue::new("phase", phase)]);
+}
+
+pub(crate) fn rejected(phase: &'static str) {
+ METRICS.rejections.add(1, &[KeyValue::new("phase", phase)]);
+}
+
+pub(crate) fn delivery(outcome: &'static str, reason: &'static str) {
+ METRICS.delivery.add(
+ 1,
+ &[
+ KeyValue::new("outcome", outcome),
+ KeyValue::new("reason", reason),
+ ],
+ );
+}
+
+pub(crate) fn execution_finished(facts: &crate::capture::InferenceFacts) {
+ if let Some(request_id) = &facts.inference.provider_request_id {
+ tracing::Span::current().record("provider_request_id", request_id.as_str());
+ }
+ let outcome = match facts.outcome {
+ RelayOutcome::Eof => "complete",
+ RelayOutcome::Cancelled => "cancelled",
+ RelayOutcome::Timeout => "timeout",
+ RelayOutcome::TransportError => "transport_error",
+ };
+ METRICS
+ .executions
+ .add(1, &[KeyValue::new("outcome", outcome)]);
+ if let Some(first_byte_ms) = facts.first_byte_ms {
+ phase_finished(
+ "provider.first_byte",
+ std::time::Duration::from_millis(u64::try_from(first_byte_ms).unwrap_or(u64::MAX)),
+ );
+ }
+ phase_finished(
+ "execution",
+ std::time::Duration::from_millis(u64::try_from(facts.duration_ms).unwrap_or(u64::MAX)),
+ );
+ let context = tracing::Span::current().context();
+ let context_span = context.span();
+ let ids = context_span.span_context();
+ tracing::info!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), outcome, duration_ms = facts.duration_ms, "gateway execution finished");
+ if matches!(
+ facts.outcome,
+ RelayOutcome::Timeout | RelayOutcome::TransportError
+ ) {
+ tracing::Span::current().record("otel.status_code", "ERROR");
+ tracing::warn!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), phase = "provider", outcome, "gateway stream failed");
+ }
+}
diff --git a/ai-gateway/src/observability/mod.rs b/ai-gateway/src/observability/mod.rs
new file mode 100644
index 0000000000..02641e3371
--- /dev/null
+++ b/ai-gateway/src/observability/mod.rs
@@ -0,0 +1,191 @@
+mod client;
+mod http;
+mod metrics;
+
+pub(crate) use client::instrument_client;
+pub use http::instrument;
+pub(crate) use metrics::{Active, delivery, execution_finished, rejected};
+
+use crate::config::{GatewayConfig, LogFormat};
+use opentelemetry::{KeyValue, global, trace::TracerProvider};
+use opentelemetry_otlp::{Protocol, WithExportConfig};
+use opentelemetry_sdk::{
+ Resource,
+ metrics::{PeriodicReader, SdkMeterProvider},
+ propagation::TraceContextPropagator,
+ trace::{BatchConfigBuilder, BatchSpanProcessor, Sampler, SdkTracerProvider},
+};
+use std::{env, error::Error, time::Duration};
+use tokio::time::Instant;
+use tracing_subscriber::{EnvFilter, Layer, filter::filter_fn, prelude::*};
+
+pub struct Observability {
+ traces: SdkTracerProvider,
+ metrics: Option<SdkMeterProvider>,
+}
+
+/// Initialize process logging and optional operational OTLP exports.
+///
+/// # Errors
+/// Returns a sanitized error for invalid settings, exporter construction, or an
+/// already installed tracing subscriber.
+pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {
+ let service = setting("OTEL_SERVICE_NAME")?.unwrap_or_else(|| "ai-gateway".into());
+ let resource = Resource::builder()
+ .with_service_name(service)
+ .with_attributes([
+ KeyValue::new(
+ "deployment.environment.name",
+ setting("DD_ENV")?.unwrap_or_else(|| "development".into()),
+ ),
+ KeyValue::new(
+ "service.version",
+ setting("BUILD_ID")?.unwrap_or_else(|| env!("CARGO_PKG_VERSION").into()),
+ ),
+ ])
+ .build();
+ let ratio = sampling_ratio(setting("OTEL_TRACES_SAMPLER_ARG")?.as_deref())?;
+ let mut traces = SdkTracerProvider::builder()
+ .with_resource(resource.clone())
+ .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(
+ ratio,
+ ))));
+ let metrics = if let Some(endpoint) = setting("OTEL_EXPORTER_OTLP_ENDPOINT")? {
+ let endpoint =
+ reqwest::Url::parse(&endpoint).map_err(|_| "invalid OTEL_EXPORTER_OTLP_ENDPOINT")?;
+ if !matches!(endpoint.scheme(), "http" | "https") || endpoint.host_str().is_none() {
+ return Err("invalid OTEL_EXPORTER_OTLP_ENDPOINT".into());
+ }
+ let exporter = opentelemetry_otlp::SpanExporter::builder()
+ .with_http()
+ .with_protocol(Protocol::HttpBinary)
+ .with_timeout(Duration::from_secs(3))
+ .build()
+ .map_err(|_| "failed to initialize operational trace exporter")?;
+ traces = traces.with_span_processor(
+ BatchSpanProcessor::builder(exporter)
+ .with_batch_config(
+ BatchConfigBuilder::default()
+ .with_max_queue_size(512)
+ .with_max_export_batch_size(64)
+ .with_scheduled_delay(Duration::from_secs(1))
+ .build(),
+ )
+ .build(),
+ );
+ let exporter = opentelemetry_otlp::MetricExporter::builder()
+ .with_http()
+ .with_protocol(Protocol::HttpBinary)
+ .with_timeout(Duration::from_secs(3))
+ .build()
+ .map_err(|_| "failed to initialize operational metric exporter")?;
+ Some(
+ SdkMeterProvider::builder()
+ .with_resource(resource)
+ .with_reader(
+ PeriodicReader::builder(exporter)
+ .with_interval(Duration::from_secs(30))
+ .build(),
+ )
+ .build(),
+ )
+ } else {
+ None
+ };
+ let traces = traces.build();
+ global::set_text_map_propagator(TraceContextPropagator::new());
+ if let Some(metrics) = &metrics {
+ global::set_meter_provider(metrics.clone());
+ }
+ let log_filter = EnvFilter::try_new(format!("warn,ai_gateway={}", config.log_level))?;
+ let logger = match config.log_format {
+ LogFormat::Json => tracing_subscriber::fmt::layer()
+ .json()
+ .flatten_event(true)
+ .with_filter(log_filter)
+ .boxed(),
+ LogFormat::Text => tracing_subscriber::fmt::layer()
+ .compact()
+ .with_ansi(false)
+ .with_filter(log_filter)
+ .boxed(),
+ };
+ let spans = tracing_opentelemetry::layer()
+ .with_tracer(traces.tracer("ai-gateway"))
+ .with_tracked_inactivity(false)
+ .with_filter(filter_fn(|metadata| {
+ metadata.is_span()
+ && (metadata.target().starts_with("ai_gateway")
+ || metadata.target().starts_with("reqwest_tracing"))
+ && *metadata.level() <= tracing::Level::INFO
+ }));
+ tracing_subscriber::registry()
+ .with(logger)
+ .with(spans)
+ .try_init()
+ .map_err(|_| "failed to initialize gateway logging")?;
+ Ok(Observability { traces, metrics })
+}
+
+impl Observability {
+ /// Flush exporters within the shared gateway shutdown deadline.
+ pub async fn shutdown(self, deadline: Instant) {
+ let (trace_sender, traces) = tokio::sync::oneshot::channel();
+ let (metric_sender, metrics) = tokio::sync::oneshot::channel();
+ std::thread::spawn(move || {
+ let _ = trace_sender.send(
+ self.traces
+ .shutdown_with_timeout(deadline.saturating_duration_since(Instant::now())),
+ );
+ });
+ std::thread::spawn(move || {
+ let _ = metric_sender.send(self.metrics.map(|provider| {
+ provider.shutdown_with_timeout(deadline.saturating_duration_since(Instant::now()))
+ }));
+ });
+ match tokio::time::timeout_at(deadline, async { tokio::join!(traces, metrics) }).await {
+ Ok((Ok(Ok(())), Ok(None | Some(Ok(()))))) => {}
+ Ok(_) => tracing::warn!("operational telemetry shutdown failed"),
+ Err(_) => tracing::warn!("operational telemetry shutdown deadline reached"),
+ }
+ }
+}
+
+fn setting(name: &str) -> Result<Option<String>, Box<dyn Error>> {
+ match env::var(name) {
+ Ok(value) if value.trim().is_empty() => Ok(None),
+ Ok(value) => Ok(Some(value)),
+ Err(env::VarError::NotPresent) => Ok(None),
+ Err(env::VarError::NotUnicode(_)) => {
+ Err("observability configuration must be Unicode".into())
+ }
+ }
+}
+
+fn sampling_ratio(value: Option<&str>) -> Result<f64, Box<dyn Error>> {
+ value
+ .unwrap_or("1")
+ .parse::<f64>()
+ .ok()
+ .filter(|value| (0.0..=1.0).contains(value))
+ .ok_or_else(|| "OTEL_TRACES_SAMPLER_ARG must be a number from 0 to 1".into())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::sampling_ratio;
+
+ #[test]
+ fn sampling_rejects_nonfinite_out_of_range_and_sensitive_values() {
+ for value in ["NaN", "inf", "-0.1", "1.1", "secret-token"] {
+ let error = sampling_ratio(Some(value)).unwrap_err();
+ assert_eq!(
+ error.to_string(),
+ "OTEL_TRACES_SAMPLER_ARG must be a number from 0 to 1"
+ );
+ }
+ for (value, expected) in [(None, 1.0), (Some("0"), 0.0), (Some("0.25"), 0.25)] {
+ assert!((sampling_ratio(value).unwrap() - expected).abs() < f64::EPSILON);
+ }
+ }
+}
diff --git a/ai-gateway/src/providers/openai.rs b/ai-gateway/src/providers/openai.rs
index 7b05775e35..c036702b43 100644
--- a/ai-gateway/src/providers/openai.rs
+++ b/ai-gateway/src/providers/openai.rs
@@ -6,6 +6,8 @@ use axum::{
http::{HeaderMap, HeaderValue, Response, header},
};
use reqwest::Client;
+use reqwest_middleware::ClientWithMiddleware;
+use reqwest_tracing::DisableOtelPropagation;
use tokio::{
sync::{OwnedSemaphorePermit, Semaphore},
time::Instant,
@@ -40,12 +42,13 @@ impl Default for ProviderLimits {
/// An admitted execution. Dropping it releases capacity; there is no waiting queue.
pub struct RequestPermit {
_permit: OwnedSemaphorePermit,
+ _active: crate::observability::Active,
deadline: Instant,
}
/// A pooled client for the official `OpenAI` Responses endpoint.
pub struct OpenAiProvider {
- client: Client,
+ client: ClientWithMiddleware,
capacity: Arc<Semaphore>,
limits: ProviderLimits,
telemetry: Option<crate::telemetry::Telemetry>,
@@ -83,7 +86,7 @@ impl OpenAiProvider {
.build()
.map_err(|_| ProviderError::Configuration)?;
Ok(Self {
- client,
+ client: crate::observability::instrument_client(client, "provider.headers"),
capacity: Arc::new(Semaphore::new(limits.active)),
limits,
telemetry: None,
@@ -102,13 +105,13 @@ impl OpenAiProvider {
/// # Errors
/// Returns [`ProviderError::Busy`] immediately when all execution slots are occupied.
pub fn try_admit(&self) -> Result<RequestPermit, ProviderError> {
- let permit = self
- .capacity
- .clone()
- .try_acquire_owned()
- .map_err(|_| ProviderError::Busy)?;
+ let permit = self.capacity.clone().try_acquire_owned().map_err(|_| {
+ crate::observability::rejected("execution");
+ ProviderError::Busy
+ })?;
Ok(RequestPermit {
_permit: permit,
+ _active: crate::observability::Active::new("execution"),
deadline: Instant::now() + self.limits.execution_timeout,
})
}
@@ -145,6 +148,7 @@ impl OpenAiProvider {
.min(Instant::now() + self.limits.headers_timeout),
self.client
.post(endpoint)
+ .with_extension(DisableOtelPropagation)
.headers(transport::request_headers(headers))
// Observe plain JSON/SSE while relaying the provider bytes unchanged.
.header(header::ACCEPT_ENCODING, "identity")
diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rs
index fe5adfc95a..6897ce4d4b 100644
--- a/ai-gateway/src/resolution/mod.rs
+++ b/ai-gateway/src/resolution/mod.rs
@@ -10,6 +10,7 @@ use reqwest::{
Client, Url,
header::{AUTHORIZATION, CONTENT_TYPE, HeaderValue},
};
+use reqwest_middleware::ClientWithMiddleware;
use std::{
fmt,
net::IpAddr,
@@ -79,7 +80,7 @@ impl ControlPlaneConfig {
/// Reuses the HTTP connection pool; credentials belong exclusively to each request.
pub struct ControlPlaneClient {
- client: Client,
+ client: ClientWithMiddleware,
config: ControlPlaneConfig,
}
@@ -99,7 +100,10 @@ impl ControlPlaneClient {
.no_deflate()
.build()
.map_err(|_| ResolutionError::Configuration)?;
- Ok(Self { client, config })
+ Ok(Self {
+ client: crate::observability::instrument_client(client, "resolver"),
+ config,
+ })
}
/// Resolve a gateway credential and API format into a validated execution contract.
diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rs
index 7a9b2383a3..6e624c47ec 100644
--- a/ai-gateway/src/telemetry/mod.rs
+++ b/ai-gateway/src/telemetry/mod.rs
@@ -10,7 +10,9 @@ use std::{
},
};
+use opentelemetry::trace::{FutureExt, TraceContextExt};
use tokio::{sync::Semaphore, task::JoinSet, time::Instant};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
use uuid::Uuid;
use crate::{
@@ -115,27 +117,47 @@ impl Telemetry {
while tasks.try_join_next().is_some() {}
let uploader = self.0.uploader.clone();
let stats = self.0.stats.clone();
- tasks.spawn(async move {
- let _capacity = capacity;
- let _bytes = bytes;
- let span = mapping::span(facts, &context.trace_id, &context.observation_id);
- match uploader.export(&context, &[span]).await {
- Ok(()) => {
- stats.accepted.fetch_add(1, Ordering::Relaxed);
- tracing::debug!("gateway telemetry accepted");
- }
- Err(error) => {
- stats.failed.fetch_add(1, Ordering::Relaxed);
- // Error categories contain no URLs, credentials, response bodies or content.
- tracing::warn!(reason = error.reason(), "gateway telemetry upload failed");
+ let parent = tracing::Span::current().context();
+ tasks.spawn(
+ async move {
+ let _capacity = capacity;
+ let _bytes = bytes;
+ let span = mapping::span(facts, &context.trace_id, &context.observation_id);
+ match uploader.export(&context, &[span]).await {
+ Ok(()) => {
+ stats.accepted.fetch_add(1, Ordering::Relaxed);
+ crate::observability::delivery("accepted", "success");
+ tracing::debug!("gateway telemetry accepted");
+ }
+ Err(error) => {
+ let failed = stats.failed.fetch_add(1, Ordering::Relaxed) + 1;
+ crate::observability::delivery("failed", error.reason());
+ // Error categories contain no URLs, credentials, response bodies or content.
+ if failed == 1 || failed.is_multiple_of(100) {
+ let context = opentelemetry::Context::current();
+ let span = context.span();
+ let context = span.span_context();
+ tracing::warn!(
+ trace_id = %context.trace_id(),
+ span_id = %context.span_id(),
+ reason = error.reason(),
+ failed,
+ "gateway telemetry upload failed"
+ );
+ }
+ }
}
}
- });
+ .with_context(parent),
+ );
}
fn drop_record(&self, reason: &'static str) {
- self.0.stats.dropped.fetch_add(1, Ordering::Relaxed);
- tracing::warn!(reason, "gateway telemetry dropped");
+ let dropped = self.0.stats.dropped.fetch_add(1, Ordering::Relaxed) + 1;
+ crate::observability::delivery("dropped", reason);
+ if dropped == 1 || dropped.is_multiple_of(100) {
+ tracing::warn!(reason, dropped, "gateway telemetry dropped");
+ }
}
/// Stop accepting work and finish uploads within the existing process drain deadline.
@@ -158,6 +180,7 @@ impl Telemetry {
Ok(Some(Ok(()))) => {}
Ok(Some(Err(_))) => {
self.0.stats.failed.fetch_add(1, Ordering::Relaxed);
+ crate::observability::delivery("failed", "task");
}
Ok(None) => break,
Err(_) => {
@@ -165,6 +188,7 @@ impl Telemetry {
while let Some(result) = tasks.join_next().await {
if result.is_err() {
self.0.stats.dropped.fetch_add(1, Ordering::Relaxed);
+ crate::observability::delivery("dropped", "shutdown");
}
}
break;
@@ -200,8 +224,24 @@ impl Write for SizeCounter {
}
pub(crate) fn debug_record(facts: &InferenceFacts) {
- // Full mode intentionally includes native request and completed output content.
- tracing::debug!(capture = %format_args!("{:#}", serde_json::to_value(facts).expect("captured facts must serialize")), "gateway response captured");
+ tracing::debug!(
+ api_format = facts.api_format,
+ outcome = match facts.outcome {
+ crate::capture::RelayOutcome::Eof => "eof",
+ crate::capture::RelayOutcome::Cancelled => "cancelled",
+ crate::capture::RelayOutcome::Timeout => "timeout",
+ crate::capture::RelayOutcome::TransportError => "transport_error",
+ },
+ http_status = facts.http_status,
+ duration_ms = u64::try_from(facts.duration_ms).unwrap_or(u64::MAX),
+ first_byte_ms = facts
+ .first_byte_ms
+ .map(|ms| u64::try_from(ms).unwrap_or(u64::MAX)),
+ input_complete = facts.inference.input_complete,
+ output_complete = facts.inference.output_complete,
+ capture_complete = facts.inference.capture_complete,
+ "gateway response captured"
+ );
}
#[cfg(test)]
diff --git a/ai-gateway/src/telemetry/otlp.rs b/ai-gateway/src/telemetry/otlp.rs
index c35a5d1528..66a2cced08 100644
--- a/ai-gateway/src/telemetry/otlp.rs
+++ b/ai-gateway/src/telemetry/otlp.rs
@@ -5,6 +5,7 @@ use std::{
use axum::http::{HeaderValue, header};
use reqwest::{Client, Url};
+use reqwest_middleware::ClientWithMiddleware;
use serde_json::{Value, json};
use super::DeliveryContext;
@@ -16,7 +17,7 @@ const MAX_RESPONSE_BYTES: usize = 64 * 1024;
const UPLOAD_TIMEOUT: Duration = Duration::from_secs(5);
pub(super) struct Uploader {
- client: Client,
+ client: ClientWithMiddleware,
endpoint: Url,
service_key: String,
}
@@ -60,7 +61,7 @@ impl Uploader {
.build()
.map_err(|_| ResolutionError::Configuration)?;
Ok(Self {
- client,
+ client: crate::observability::instrument_client(client, "ingestion"),
endpoint: config.endpoint(INGESTION_PATH),
service_key: config.service_key().to_owned(),
})
diff --git a/ai-gateway/tests/observability.rs b/ai-gateway/tests/observability.rs
new file mode 100644
index 0000000000..22d1acc8e9
--- /dev/null
+++ b/ai-gateway/tests/observability.rs
@@ -0,0 +1,282 @@
+#![cfg(unix)]
+
+use axum::{Router, body::Bytes, extract::Request, routing::post};
+use opentelemetry_proto::tonic::collector::{
+ metrics::v1::ExportMetricsServiceRequest, trace::v1::ExportTraceServiceRequest,
+};
+use opentelemetry_proto::tonic::{
+ common::v1::any_value::Value as AttributeValue,
+ metrics::v1::{ResourceMetrics, metric::Data},
+};
+use prost::Message;
+use serde_json::Value;
+use std::{
+ io::{BufRead, BufReader},
+ process::{Child, Command, Stdio},
+ sync::{Arc, Mutex, mpsc},
+ thread::JoinHandle,
+ time::{Duration, Instant},
+};
+
+struct Gateway {
+ child: Child,
+ url: String,
+ logs: Arc<Mutex<Vec<Value>>>,
+ reader: Option<JoinHandle<()>>,
+}
+
+impl Gateway {
+ fn start(endpoint: &str, shutdown_seconds: &str, level: &str) -> Self {
+ let mut child = Command::new(env!("CARGO_BIN_EXE_ai-gateway"))
+ .env_clear()
+ .env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS", "127.0.0.1:0")
+ .env(
+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS",
+ shutdown_seconds,
+ )
+ .env("LANGFUSE_LOG_FORMAT", "json")
+ .env("LANGFUSE_LOG_LEVEL", level)
+ .env("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint)
+ .env("OTEL_SERVICE_NAME", "gateway-observability-test")
+ .env("DD_ENV", "staging-test")
+ .env("BUILD_ID", "test-version")
+ .stdout(Stdio::piped())
+ .stderr(Stdio::inherit())
+ .spawn()
+ .unwrap();
+ let stdout = child.stdout.take().unwrap();
+ let logs = Arc::new(Mutex::new(Vec::new()));
+ let output = logs.clone();
+ let (sender, receiver) = mpsc::channel();
+ let reader = std::thread::spawn(move || {
+ for line in BufReader::new(stdout).lines().map_while(Result::ok) {
+ let value: Value = serde_json::from_str(&line).unwrap();
+ if value["message"] == "gateway listening" {
+ let _ = sender.send(value["address"].as_str().unwrap().to_owned());
+ }
+ output.lock().unwrap().push(value);
+ }
+ });
+ let address = receiver
+ .recv_timeout(Duration::from_secs(10))
+ .expect("gateway startup");
+ Self {
+ child,
+ url: format!("http://{address}"),
+ logs,
+ reader: Some(reader),
+ }
+ }
+
+ async fn request(&self, sampled: bool) {
+ let response = reqwest::Client::new()
+ .post(format!(
+ "{}/openai/v1/responses?secret=query-canary",
+ self.url
+ ))
+ .header("host", "host-canary.invalid")
+ .header("x-forwarded-proto", "scheme-canary")
+ .header(
+ "traceparent",
+ format!(
+ "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-{:02x}",
+ u8::from(sampled)
+ ),
+ )
+ .header("authorization", "Bearer secret-auth-token")
+ .body(r#"{"input":"secret-prompt-content"}"#)
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.status(), 503);
+ let _ = response.bytes().await.unwrap();
+ }
+
+ async fn stop(&mut self) {
+ assert!(
+ Command::new("/bin/kill")
+ .args(["-TERM", &self.child.id().to_string()])
+ .status()
+ .unwrap()
+ .success()
+ );
+ tokio::time::timeout(Duration::from_secs(6), async {
+ loop {
+ if let Some(status) = self.child.try_wait().unwrap() {
+ assert!(status.success());
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("gateway shutdown deadline");
+ self.reader.take().unwrap().join().unwrap();
+ }
+}
+
+impl Drop for Gateway {
+ fn drop(&mut self) {
+ let _ = self.child.kill();
+ let _ = self.child.wait();
+ }
+}
+
+#[tokio::test]
+async fn exports_otlp_traces_metrics_and_correlated_content_free_logs() {
+ let captured = Arc::new(Mutex::new(Vec::new()));
+ let receiver = captured.clone();
+ let app = Router::new().route(
+ "/{*path}",
+ post(move |request: Request| {
+ let receiver = receiver.clone();
+ async move {
+ let path = request.uri().path().to_owned();
+ let body = axum::body::to_bytes(request.into_body(), 1_000_000)
+ .await
+ .unwrap();
+ receiver.lock().unwrap().push((path, body));
+ Bytes::new()
+ }
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let endpoint = format!("http://{}", listener.local_addr().unwrap());
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ let mut gateway = Gateway::start(&endpoint, "5", "debug");
+ gateway.request(true).await;
+ gateway.request(false).await;
+ assert_eq!(
+ reqwest::get(format!("{}/health", gateway.url))
+ .await
+ .unwrap()
+ .status(),
+ 200
+ );
+ gateway.stop().await;
+ server.abort();
+ let captured = captured.lock().unwrap();
+ let traces: Vec<_> = captured
+ .iter()
+ .filter(|(path, _)| path == "/v1/traces")
+ .flat_map(|(_, body)| {
+ ExportTraceServiceRequest::decode(body.clone())
+ .unwrap()
+ .resource_spans
+ })
+ .collect();
+ let metrics: Vec<_> = captured
+ .iter()
+ .filter(|(path, _)| path == "/v1/metrics")
+ .flat_map(|(_, body)| {
+ ExportMetricsServiceRequest::decode(body.clone())
+ .unwrap()
+ .resource_metrics
+ })
+ .collect();
+ assert!(!traces.is_empty(), "traces must flush on shutdown");
+ assert!(!metrics.is_empty(), "metrics must flush on shutdown");
+ let spans: Vec<_> = traces
+ .iter()
+ .flat_map(|resource| &resource.scope_spans)
+ .flat_map(|scope| &scope.spans)
+ .collect();
+ assert_eq!(spans.len(), 1);
+ assert_eq!(spans[0].name, "POST /openai/v1/responses");
+ assert_eq!(
+ spans[0].parent_span_id,
+ [0x00, 0xf0, 0x67, 0xaa, 0x0b, 0xa9, 0x02, 0xb7]
+ );
+ assert_eq!(spans[0].kind, 2);
+ assert!(
+ spans[0].events.is_empty(),
+ "log events must not be exported as span events"
+ );
+ let logs = gateway.logs.lock().unwrap();
+ let summaries: Vec<_> = logs
+ .iter()
+ .filter(|value| value["message"] == "gateway response started")
+ .collect();
+ assert_eq!(summaries.len(), 2);
+ assert_eq!(summaries[0]["trace_id"], "4bf92f3577b34da6a3ce929d0e0e4736");
+ assert_eq!(
+ summaries[0]["span_id"].as_str().unwrap(),
+ opentelemetry::trace::SpanId::from_bytes(spans[0].span_id.as_slice().try_into().unwrap())
+ .to_string()
+ );
+ assert_eq!(summaries[0]["status"], 503);
+ let serialized = serde_json::to_string(&*logs).unwrap();
+ assert!(!serialized.contains("secret-auth-token"));
+ assert!(!serialized.contains("secret-prompt-content"));
+ let exported = format!("{traces:?}{metrics:?}");
+ for secret in [
+ "secret-auth-token",
+ "secret-prompt-content",
+ "query-canary",
+ "host-canary",
+ "scheme-canary",
+ ] {
+ assert!(!exported.contains(secret));
+ assert!(!serialized.contains(secret));
+ }
+ assert_http_metrics(&metrics);
+}
+
+fn assert_http_metrics(metrics: &[ResourceMetrics]) {
+ let http_duration = metrics
+ .iter()
+ .flat_map(|resource| &resource.scope_metrics)
+ .flat_map(|scope| &scope.metrics)
+ .find(|metric| metric.name == "http.server.request.duration")
+ .expect("HTTP duration must export even for requests rejected before execution");
+ let Some(Data::Histogram(histogram)) = &http_duration.data else {
+ panic!("HTTP duration must be a histogram");
+ };
+ assert_eq!(histogram.data_points.len(), 1);
+ let point = &histogram.data_points[0];
+ assert_eq!(
+ point.count, 2,
+ "count includes unsampled requests and excludes health probes"
+ );
+ assert!(point.sum.unwrap() > 0.0);
+ let attributes: std::collections::BTreeMap<_, _> = point
+ .attributes
+ .iter()
+ .map(|attribute| {
+ (
+ attribute.key.as_str(),
+ attribute.value.as_ref().unwrap().value.as_ref().unwrap(),
+ )
+ })
+ .collect();
+ assert_eq!(attributes.len(), 3);
+ assert_eq!(
+ attributes["http.request.method"],
+ &AttributeValue::StringValue("POST".into())
+ );
+ assert_eq!(
+ attributes["http.route"],
+ &AttributeValue::StringValue("/openai/v1/responses".into())
+ );
+ assert_eq!(
+ attributes["http.response.status_code"],
+ &AttributeValue::IntValue(503)
+ );
+}
+
+#[tokio::test]
+async fn an_unresponsive_collector_does_not_extend_the_shutdown_deadline() {
+ let app = Router::new().route(
+ "/{*path}",
+ post(|| async { std::future::pending::<()>().await }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let endpoint = format!("http://{}", listener.local_addr().unwrap());
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ let mut gateway = Gateway::start(&endpoint, "1", "info");
+ gateway.request(true).await;
+ let start = Instant::now();
+ gateway.stop().await;
+ assert!(start.elapsed() < Duration::from_secs(2));
+ server.abort();
+}