🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): add operational logs, traces, and metrics (#17490)
Hassieb Pakzadd0092797ffee
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lockindex 3a4d768c93..b20a6bdff7 100644--- a/ai-gateway/Cargo.lock+++ b/ai-gateway/Cargo.lock@@ -2,6 +2,15 @@# It is not intended for manual editing.version = 4+[[package]]+name = "aho-corasick"+version = "1.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"+dependencies = [+ "memchr",+]+[[package]]name = "ai-gateway"version = "0.1.0"@@ -10,18 +19,46 @@ dependencies = ["chrono","futures-util","hmac",+ "http-body","http-body-util",+ "opentelemetry",+ "opentelemetry-http",+ "opentelemetry-otlp",+ "opentelemetry-proto",+ "opentelemetry_sdk",+ "prost","reqwest",+ "reqwest-middleware",+ "reqwest-tracing","serde","serde_json","sha2","tokio","tower",+ "tower-http 0.7.1","tracing",+ "tracing-opentelemetry","tracing-subscriber","uuid",]+[[package]]+name = "anyhow"+version = "1.0.104"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"++[[package]]+name = "async-trait"+version = "0.1.92"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.5",+]+[[package]]name = "atomic-waker"version = "1.1.2"@@ -34,6 +71,29 @@ version = "1.5.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"+[[package]]+name = "aws-lc-rs"+version = "1.18.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"+dependencies = [+ "aws-lc-sys",+ "zeroize",+]++[[package]]+name = "aws-lc-sys"+version = "0.45.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"+dependencies = [+ "cc",+ "cmake",+ "dunce",+ "fs_extra",+ "pkg-config",+]+[[package]]name = "axum"version = "0.8.9"@@ -88,6 +148,12 @@ version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"+[[package]]+name = "base64"+version = "0.23.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"+[[package]]name = "bitflags"version = "2.13.2"@@ -122,6 +188,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80"dependencies = ["find-msvc-tools",+ "jobserver",+ "libc","shlex",]@@ -145,7 +213,7 @@ checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"dependencies = ["cfg-if","cpufeatures",- "rand_core",+ "rand_core 0.10.1",][[package]]@@ -157,12 +225,31 @@ dependencies = ["num-traits",]+[[package]]+name = "cmake"+version = "0.1.58"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"+dependencies = [+ "cc",+]+[[package]]name = "cmov"version = "0.5.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"+[[package]]+name = "combine"+version = "4.6.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e"+dependencies = [+ "bytes",+ "memchr",+]+[[package]]name = "const-oid"version = "0.10.2"@@ -235,6 +322,18 @@ dependencies = ["syn 3.0.5",]+[[package]]+name = "dunce"+version = "1.0.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"++[[package]]+name = "either"+version = "1.18.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"+[[package]]name = "errno"version = "0.3.14"@@ -260,6 +359,12 @@ dependencies = ["percent-encoding",]+[[package]]+name = "fs_extra"+version = "1.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"+[[package]]name = "futures-channel"version = "0.3.34"@@ -267,6 +372,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"dependencies = ["futures-core",+ "futures-sink",][[package]]@@ -275,6 +381,17 @@ version = "0.3.34"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"+[[package]]+name = "futures-executor"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432"+dependencies = [+ "futures-core",+ "futures-task",+ "futures-util",+]+[[package]]name = "futures-io"version = "0.3.34"@@ -333,6 +450,18 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "getrandom"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"+dependencies = [+ "cfg-if",+ "libc",+ "r-efi 5.3.0",+ "wasip2",+]+[[package]]name = "getrandom"version = "0.4.3"@@ -342,8 +471,8 @@ dependencies = ["cfg-if","js-sys","libc",- "r-efi",- "rand_core",+ "r-efi 6.0.0",+ "rand_core 0.10.1","wasm-bindgen",]@@ -441,7 +570,6 @@ dependencies = ["hyper","hyper-util","rustls",- "rustls-native-certs","tokio","tokio-rustls","tower-service",@@ -453,7 +581,7 @@ version = "0.1.20"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"dependencies = [- "base64",+ "base64 0.22.1","bytes","futures-channel","futures-util",@@ -580,12 +708,80 @@ version = "2.12.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"+[[package]]+name = "itertools"+version = "0.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"+dependencies = [+ "either",+]+[[package]]name = "itoa"version = "1.0.18"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"+[[package]]+name = "jni"+version = "0.22.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"+dependencies = [+ "cfg-if",+ "combine",+ "jni-macros",+ "jni-sys",+ "log",+ "simd_cesu8",+ "thiserror",+ "walkdir",+ "windows-link",+]++[[package]]+name = "jni-macros"+version = "0.22.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"+dependencies = [+ "proc-macro2",+ "quote",+ "rustc_version",+ "simd_cesu8",+ "syn 2.0.119",+]++[[package]]+name = "jni-sys"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"+dependencies = [+ "jni-sys-macros",+]++[[package]]+name = "jni-sys-macros"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"+dependencies = [+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "jobserver"+version = "0.1.35"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"+dependencies = [+ "getrandom 0.4.3",+ "libc",+]+[[package]]name = "js-sys"version = "0.3.105"@@ -627,6 +823,15 @@ version = "0.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"+[[package]]+name = "matchers"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"+dependencies = [+ "regex-automata",+]+[[package]]name = "matchit"version = "0.8.4"@@ -677,6 +882,77 @@ version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"+[[package]]+name = "opentelemetry"+version = "0.32.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682"+dependencies = [+ "futures-core",+ "futures-sink",+ "js-sys",+ "pin-project-lite",+ "thiserror",+ "tracing",+]++[[package]]+name = "opentelemetry-http"+version = "0.32.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331"+dependencies = [+ "async-trait",+ "bytes",+ "http",+ "opentelemetry",+ "reqwest",+]++[[package]]+name = "opentelemetry-otlp"+version = "0.32.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35"+dependencies = [+ "http",+ "opentelemetry",+ "opentelemetry-http",+ "opentelemetry-proto",+ "opentelemetry_sdk",+ "prost",+ "reqwest",+ "thiserror",+]++[[package]]+name = "opentelemetry-proto"+version = "0.32.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"+dependencies = [+ "opentelemetry",+ "opentelemetry_sdk",+ "prost",+]++[[package]]+name = "opentelemetry_sdk"+version = "0.32.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9"+dependencies = [+ "futures-channel",+ "futures-executor",+ "futures-util",+ "opentelemetry",+ "percent-encoding",+ "portable-atomic",+ "rand 0.9.5",+ "thiserror",+ "tokio",+]+[[package]]name = "percent-encoding"version = "2.3.2"@@ -689,6 +965,18 @@ version = "0.2.17"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"+[[package]]+name = "pkg-config"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"++[[package]]+name = "portable-atomic"+version = "1.15.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"+[[package]]name = "potential_utf"version = "0.1.6"@@ -698,6 +986,15 @@ dependencies = ["zerovec",]+[[package]]+name = "ppv-lite86"+version = "0.2.21"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"+dependencies = [+ "zerocopy",+]+[[package]]name = "proc-macro2"version = "1.0.107"@@ -707,6 +1004,29 @@ dependencies = ["unicode-ident",]+[[package]]+name = "prost"+version = "0.14.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"+dependencies = [+ "bytes",+ "prost-derive",+]++[[package]]+name = "prost-derive"+version = "0.14.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"+dependencies = [+ "anyhow",+ "itertools",+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]+[[package]]name = "quinn"version = "0.11.11"@@ -733,10 +1053,11 @@ version = "0.11.17"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83"dependencies = [+ "aws-lc-rs","bytes","getrandom 0.4.3","lru-slab",- "rand",+ "rand 0.10.2","rand_pcg","ring","rustc-hash",@@ -772,12 +1093,28 @@ dependencies = ["proc-macro2",]+[[package]]+name = "r-efi"+version = "5.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"+[[package]]name = "r-efi"version = "6.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"+[[package]]+name = "rand"+version = "0.9.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"+dependencies = [+ "rand_chacha",+ "rand_core 0.9.5",+]+[[package]]name = "rand"version = "0.10.2"@@ -786,7 +1123,26 @@ checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80"dependencies = ["chacha20","getrandom 0.4.3",- "rand_core",+ "rand_core 0.10.1",+]++[[package]]+name = "rand_chacha"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"+dependencies = [+ "ppv-lite86",+ "rand_core 0.9.5",+]++[[package]]+name = "rand_core"+version = "0.9.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"+dependencies = [+ "getrandom 0.3.4",][[package]]@@ -801,17 +1157,35 @@ version = "0.10.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"dependencies = [- "rand_core",+ "rand_core 0.10.1",+]++[[package]]+name = "regex-automata"+version = "0.4.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-syntax",]+[[package]]+name = "regex-syntax"+version = "0.8.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"+[[package]]name = "reqwest"-version = "0.12.28"+version = "0.13.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"+checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"dependencies = [- "base64",+ "base64 0.23.1","bytes",+ "futures-channel","futures-core","futures-util","http",@@ -826,17 +1200,14 @@ dependencies = ["pin-project-lite","quinn","rustls",- "rustls-native-certs","rustls-pki-types",- "serde",- "serde_json",- "serde_urlencoded",+ "rustls-platform-verifier","sync_wrapper","tokio","tokio-rustls","tokio-util","tower",- "tower-http",+ "tower-http 0.6.11","tower-service","url","wasm-bindgen",@@ -845,6 +1216,38 @@ dependencies = ["web-sys",]+[[package]]+name = "reqwest-middleware"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58"+dependencies = [+ "anyhow",+ "async-trait",+ "http",+ "reqwest",+ "thiserror",+ "tower-service",+]++[[package]]+name = "reqwest-tracing"+version = "0.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b5e5af0cd6fc3d3c8f703d597af70b6e4e62432c63157b49419fa1ffaf481702"+dependencies = [+ "anyhow",+ "async-trait",+ "getrandom 0.2.17",+ "http",+ "matchit",+ "opentelemetry",+ "reqwest",+ "reqwest-middleware",+ "tracing",+ "tracing-opentelemetry",+]+[[package]]name = "ring"version = "0.17.14"@@ -865,14 +1268,23 @@ version = "2.1.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"+[[package]]+name = "rustc_version"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"+dependencies = [+ "semver",+]+[[package]]name = "rustls"-version = "0.23.44"+version = "0.23.45"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6725596c3f2c3a0aef021139e145d4eafe314a6623e4680ca83852b2c67ab2ba"+checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"dependencies = [+ "aws-lc-rs","once_cell",- "ring","rustls-pki-types","rustls-webpki","subtle",@@ -901,12 +1313,40 @@ dependencies = ["zeroize",]+[[package]]+name = "rustls-platform-verifier"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"+dependencies = [+ "core-foundation",+ "core-foundation-sys",+ "jni",+ "log",+ "once_cell",+ "rustls",+ "rustls-native-certs",+ "rustls-platform-verifier-android",+ "rustls-webpki",+ "security-framework",+ "security-framework-sys",+ "webpki-root-certs",+ "windows-sys 0.61.2",+]++[[package]]+name = "rustls-platform-verifier-android"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"+[[package]]name = "rustls-webpki"version = "0.103.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"dependencies = [+ "aws-lc-rs","ring","rustls-pki-types","untrusted",@@ -919,10 +1359,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"[[package]]-name = "ryu"-version = "1.0.23"+name = "same-file"+version = "1.0.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"+dependencies = [+ "winapi-util",+][[package]]name = "schannel"@@ -956,6 +1399,12 @@ dependencies = ["libc",]+[[package]]+name = "semver"+version = "1.0.28"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"+[[package]]name = "serde"version = "1.0.229"@@ -1010,18 +1459,6 @@ dependencies = ["serde_core",]-[[package]]-name = "serde_urlencoded"-version = "0.7.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"-dependencies = [- "form_urlencoded",- "itoa",- "ryu",- "serde",-]-[[package]]name = "sha2"version = "0.11.0"@@ -1058,6 +1495,22 @@ dependencies = ["libc",]+[[package]]+name = "simd_cesu8"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520"+dependencies = [+ "rustc_version",+ "simdutf8",+]++[[package]]+name = "simdutf8"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"+[[package]]name = "slab"version = "0.4.12"@@ -1271,6 +1724,23 @@ dependencies = ["url",]+[[package]]+name = "tower-http"+version = "0.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"+dependencies = [+ "bitflags",+ "bytes",+ "http",+ "http-body",+ "percent-encoding",+ "pin-project-lite",+ "tower-layer",+ "tower-service",+ "tracing",+]+[[package]]name = "tower-layer"version = "0.3.3"@@ -1315,6 +1785,33 @@ dependencies = ["valuable",]+[[package]]+name = "tracing-log"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"+dependencies = [+ "log",+ "once_cell",+ "tracing-core",+]++[[package]]+name = "tracing-opentelemetry"+version = "0.33.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"+dependencies = [+ "js-sys",+ "opentelemetry",+ "smallvec",+ "tracing",+ "tracing-core",+ "tracing-log",+ "tracing-subscriber",+ "web-time",+]+[[package]]name = "tracing-serde"version = "0.2.0"@@ -1331,10 +1828,14 @@ version = "0.3.23"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"dependencies = [+ "matchers",+ "once_cell",+ "regex-automata","serde","serde_json","sharded-slab","thread_local",+ "tracing","tracing-core","tracing-serde",]@@ -1398,6 +1899,16 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"+[[package]]+name = "walkdir"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"+dependencies = [+ "same-file",+ "winapi-util",+]+[[package]]name = "want"version = "0.3.1"@@ -1413,6 +1924,15 @@ version = "0.11.1+wasi-snapshot-preview1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"+[[package]]+name = "wasip2"+version = "1.0.4+wasi-0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"+dependencies = [+ "wit-bindgen",+]+[[package]]name = "wasm-bindgen"version = "0.2.128"@@ -1470,9 +1990,9 @@ dependencies = [[[package]]name = "wasm-streams"-version = "0.4.2"+version = "0.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"+checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"dependencies = ["futures-util","js-sys",@@ -1501,6 +2021,24 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "webpki-root-certs"+version = "1.0.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"+dependencies = [+ "rustls-pki-types",+]++[[package]]+name = "winapi-util"+version = "0.1.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"+dependencies = [+ "windows-sys 0.61.2",+]+[[package]]name = "windows-link"version = "0.2.1"@@ -1589,6 +2127,12 @@ version = "0.52.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"+[[package]]+name = "wit-bindgen"+version = "0.57.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"+[[package]]name = "writeable"version = "0.6.4"@@ -1618,6 +2162,26 @@ dependencies = ["synstructure",]+[[package]]+name = "zerocopy"+version = "0.8.57"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873"+dependencies = [+ "zerocopy-derive",+]++[[package]]+name = "zerocopy-derive"+version = "0.8.57"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]+[[package]]name = "zerofrom"version = "0.1.8"diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.tomlindex c56efb94eb..975d5aaedc 100644--- a/ai-gateway/Cargo.toml+++ b/ai-gateway/Cargo.toml@@ -6,21 +6,33 @@ publish = falselicense = "MIT"[dependencies]-axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] }+axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "matched-path", "tokio"] }chrono = { version = "0.4", default-features = false, features = ["std"] }hmac = "0.13"http-body-util = "0.1"futures-util = "0.3"-reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-native-roots", "stream"] }+opentelemetry-http = "0.32"+opentelemetry = { version = "0.32", default-features = false, features = ["trace", "metrics", "internal-logs"] }+opentelemetry_sdk = { version = "0.32", default-features = false, features = ["trace", "metrics", "internal-logs"] }+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["trace", "metrics", "http-proto", "reqwest-blocking-client", "internal-logs"] }+reqwest = { version = "0.13.5", default-features = false, features = ["rustls", "stream"] }+reqwest-middleware = "0.5"+reqwest-tracing = { version = "0.7", default-features = false, features = ["opentelemetry_0_32"] }serde = { version = "1", features = ["derive"] }serde_json = "1"sha2 = "0.11"tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }+tower-http = { version = "0.7", features = ["trace"] }tracing = "0.1.44"-tracing-subscriber = { version = "0.3.23", default-features = false, features = ["fmt", "json"] }+tracing-opentelemetry = { version = "0.33", default-features = false }+tracing-subscriber = { version = "0.3.23", default-features = false, features = ["env-filter", "fmt", "json"] }uuid = { version = "1", features = ["v4"] }[dev-dependencies]+http-body = "1"+opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic-messages", "trace", "metrics"] }+opentelemetry_sdk = { version = "0.32", default-features = false, features = ["testing"] }+prost = "0.14"tokio = { version = "1.53.1", features = ["io-util"] }tower = { version = "0.5", features = ["util"] }diff --git a/ai-gateway/Dockerfile b/ai-gateway/Dockerfileindex 49d1447925..25b7491895 100644--- a/ai-gateway/Dockerfile+++ b/ai-gateway/Dockerfile@@ -8,7 +8,7 @@ COPY --chown=gateway:gateway src ./srcRUN cargo build --release --lockedFROM debian:bookworm-slim AS runtime-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget \&& rm -rf /var/lib/apt/lists/*COPY --from=builder /app/target/release/ai-gateway /usr/local/bin/ai-gatewayUSER 10001:10001diff --git a/ai-gateway/README.md b/ai-gateway/README.mdindex 263bfa86e7..cce2022dac 100644--- a/ai-gateway/README.md+++ b/ai-gateway/README.md@@ -3,7 +3,7 @@A standalone Rust gateway for `POST /openai/v1/responses`. Web resolves the gatewaykey to a trusted provider connection; Rust relays native JSON or SSE withoutrewriting provider bytes. A bounded capture layer captures request/response facts and-logs the finalized record at debug level. Each finalized execution is immediately+logs capture completeness at debug level. Each finalized execution is immediatelyuploaded as a Langfuse generation through OTLP. Building and testing need no realWeb, database or provider credentials.@@ -67,7 +67,7 @@ do not load dotenv files:| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` || `LANGFUSE_AI_GATEWAY_AUTO_INCREMENT_LISTEN_PORT` | `false` | `true`, `false`; the pnpm dev task sets `true` || `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |-| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |+| `LANGFUSE_LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error`, `fatal` || `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 || `LANGFUSE_AI_GATEWAY_MAX_ACTIVE_REQUESTS` | `128` | Positive integer up to Tokio's semaphore capacity; authenticated requests per instance || `LANGFUSE_AI_GATEWAY_MAX_CONCURRENT_RESOLUTIONS` | `128` | Positive integer up to Tokio's semaphore capacity; concurrent Web resolutions per instance |@@ -84,11 +84,64 @@ Both formats use the same log level and preserve event fields. For example:```Invalid values fail startup without echoing their contents. At the default `info`-level, logs contain service lifecycle events. Debug logging also includes the-captured request and completed output when Web resolves `ingestion_mode: full`;-see capture details below. For direct+level, logs contain service lifecycle events and HTTP response summaries.+Debug logging adds content-free phase and capture details. For directhost-only development, set `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS=127.0.0.1:8080`.+## Operational observability++Logs and operational OpenTelemetry exports describe gateway health independently+of the inference generations sent to Langfuse. They do not include request/response+content, credentials, key metadata, or URL query strings. Health probes are excluded+from request logs and spans.++Use `LANGFUSE_LOG_FORMAT=text` locally and `json` for log collectors. Built-in `tracing-subscriber` formatters handle both modes. JSON event fields+are top-level attributes; span fields are nested. HTTP response and execution+summaries include top-level `trace_id` and `span_id` for Datadog correlation.+`info` emits lifecycle, HTTP response, and execution summaries; `debug` adds+sanitized capture details. Other log events do not automatically include trace IDs. Dependency logs stay at `warn`. Log severity does not control+trace sampling. `trace` is not an accepted gateway log level.++| Variable | Default | Purpose |+| --- | --- | --- |+| `OTEL_EXPORTER_OTLP_ENDPOINT` | unset | Enables operational traces and metrics; HTTP/protobuf base URL, e.g. `http://localhost:4318` |+| `OTEL_SERVICE_NAME` | `ai-gateway` | Operational service name |+| `DD_ENV` | `development` | Deployment environment resource attribute |+| `BUILD_ID` | Cargo package version | Service version; set to the deployed image's commit SHA |+| `OTEL_TRACES_SAMPLER_ARG` | `1` | Sampling ratio from 0 to 1 for root traces; incoming W3C sampling decisions are respected |++Tower's `TraceLayer` keeps the server span alive through the response body.+The HTTP response log and `http.server.request.duration` measure time to response+headers, not full SSE duration. Execution summaries and `gateway.phase.duration`+with `phase=execution` measure the provider relay through completion, cancellation,+timeout, or transport error. HTTP status and stream outcome are separate: a 200+response can still fail while streaming.++`reqwest-tracing` instruments resolver, provider-header, and ingestion requests.+Only trusted Web calls receive W3C trace context; baggage and operational trace+headers are not sent to the external provider. Inference generations retain their+separate identity. Client spans measure the HTTP send through response headers;+streaming execution and inference-telemetry delivery are tracked separately.++A small Axum middleware records `http.server.request.duration` through the+OpenTelemetry SDK, with method, matched route, and HTTP status as its only+dimensions. The histogram's count supplies request volume and error rates,+including responses rejected before provider execution. Metrics are independent+of trace sampling. Health probes are excluded.++Gateway-specific metrics include `gateway.active` (resolution/execution),+`gateway.phase.duration` (execution/provider.first_byte), `gateway.executions`,+`gateway.admission.rejected`, and `gateway.telemetry.records`. Durations use seconds;+first-byte timing measures provider bytes arriving at the gateway, not first-token+delivery to the caller. Metric attributes contain bounded categories, not tenant+or request identifiers.++Trace export uses a 512-span queue, batches of up to 64, and a one-second interval.+Metrics export every 30 seconds. Export calls use a three-second timeout; failures+do not fail inference. Shutdown flushes within the remaining shared drain deadline.+Telemetry delivery warnings report the first failure/drop and every hundredth;+metrics count each occurrence. This is best-effort operational telemetry.+## Call the gatewaySet these in the repository root `.env` for `pnpm dev` (or export them for Cargo):@@ -206,17 +259,15 @@ compatible grant. Capture and provider byte forwarding do not need to change.## Response captureSet `LANGFUSE_LOG_LEVEL=debug` to print one `gateway response captured` event at-execution end. Its `capture` field is JSON with `input`, `output`, model,-parameters, usage, attribution, timestamps, capture completeness and relay outcome.-Text logs pretty-print the captured JSON across multiple lines.-`LANGFUSE_LOG_FORMAT=json` keeps each surrounding log event on one line, with-newlines escaped inside the `capture` string.+execution end with HTTP status, timing, capture completeness and relay outcome.+Operational logs exclude prompts, outputs, model parameters and key metadata,+including at debug level. `LANGFUSE_LOG_FORMAT=json` emits one JSON object per line.Web's resolved ingestion mode controls content capture:- `full`: input is the native request JSON object, including `input`, `instructions`,tools, parameters, context references and unknown fields. Output is an ordered- array of native completed items. Debug logs intentionally contain this content.+ array of native completed items. Content is sent only through Langfuse ingestion.- `usage`: input and output are null. Model, scalar parameters, native usage,timing and trusted attribution are retained; request schemas/content are omitted.@@ -250,7 +301,7 @@ fully captured response, and downstream cancellation does not erase completed caThe implementation separates shared `ExecutionCapture` lifecycle/timing, the`OpenAiResponsesCapture` adapter and bounded `SseDecoder`. A private `ProtocolCapture`enum dispatches to the adapter. Finalization hands an owned `InferenceFacts` record-to telemetry for debug logging and immediate upload. Another provider can supply+to telemetry for a safe debug summary and immediate upload. Another provider can supplynative facts through the same interface without changing the relay. The currentusage mapping targets OpenAI Responses; no Anthropic adapter is implemented yet.Capture and upload run independently of log level; debug emission alone is gated.@@ -265,8 +316,7 @@ these limits never cap the actual provider response. Media is not fetched/uploadProvider credentials, the gateway credential and ingestion tokens are not part ofthe capture record; raw headers are not logged. Full captured content and resolved-key metadata may themselves be sensitive. Keep debug logging enabled only where-that content is intended to be recorded. No data is posted to Langfuse yet.+key metadata remain in Langfuse ingestion and are excluded from operational logs.## Process lifecyclediff --git a/ai-gateway/scripts/smoke-image.sh b/ai-gateway/scripts/smoke-image.shindex 70c740b016..15baa74f05 100644--- a/ai-gateway/scripts/smoke-image.sh+++ b/ai-gateway/scripts/smoke-image.sh@@ -20,6 +20,7 @@ for route in health ready; do*) echo "Unexpected probe response: $route" >&2; exit 1 ;;esacdone+docker exec "$container" sh -c 'wget -qO /dev/null http://localhost:8080/health || exit 1'[[ "$(docker exec "$container" id -u)" != 0 ]][[ "$(curl --silent --output /dev/null --write-out '%{http_code}' \--max-time 2 -X POST "http://$port/openai/v1/responses")" == 401 ]]@@ -27,4 +28,4 @@ docker stop --time 5 "$container" >/dev/null[[ "$(docker inspect --format '{{.State.ExitCode}}' "$container")" == 0 ]]docker logs "$container" 2>&1 | grep -q 'gateway draining'docker logs "$container" 2>&1 | grep -q 'gateway stopped'-printf '%s\n' 'Image smoke: non-root, probes, unauthenticated inference, SIGTERM passed'+printf '%s\n' 'Image smoke: non-root, probes, ECS health command, unauthenticated inference, SIGTERM passed'diff --git a/ai-gateway/src/capture/mod.rs b/ai-gateway/src/capture/mod.rsindex 03683f4151..61c9c62c59 100644--- a/ai-gateway/src/capture/mod.rs+++ b/ai-gateway/src/capture/mod.rs@@ -55,6 +55,7 @@ impl ProtocolCapture {/// Owned before dispatch and moved into the response body. Drop also covers a/// cancelled provider future before response headers have arrived.pub(crate) struct ExecutionCapture {+ span: tracing::Span,protocol: Option<ProtocolCapture>,started: Instant,start_time_unix_ms: u128,@@ -92,6 +93,7 @@ impl ExecutionCapture {.collect();let full = context.ingestion_mode() == IngestionMode::Full;Self {+ span: tracing::Span::current(),protocol: Some(ProtocolCapture::OpenAiResponses(OpenAiResponsesCapture::new(headers, body, context.ingestion_mode()),)),@@ -146,6 +148,8 @@ impl ExecutionCapture {}pub fn finish(&mut self, outcome: RelayOutcome) {+ let span = self.span.clone();+ let _entered = span.enter();let Some(protocol) = self.protocol.take() else {return;};@@ -161,6 +165,7 @@ impl ExecutionCapture {inference,};telemetry::debug_record(&facts);+ crate::observability::execution_finished(&facts);if let Some((telemetry, context)) = self.delivery.take() {telemetry.record(context, facts);}diff --git a/ai-gateway/src/capture/openai_responses/tests.rs b/ai-gateway/src/capture/openai_responses/tests.rsindex 89c78611d4..d03fdd9a65 100644--- a/ai-gateway/src/capture/openai_responses/tests.rs+++ b/ai-gateway/src/capture/openai_responses/tests.rs@@ -95,6 +95,10 @@ async fn captures_completed_items_only_in_order_without_duplication() {let stored = json!({"record":capture.facts,"items":capture.items}).to_string();assert!(!stored.contains("unfinished-canary"));assert!(!stored.contains("terminal-output-must-not-be-copied"));+ let (_, facts) = observer.protocol.take().unwrap().into_facts();+ let output = facts.output.unwrap();+ assert_eq!(output[0]["content"][0]["text"], "héllo 🌍");+ assert_eq!(output[1]["content"][0]["text"], "second");}#[tokio::test]@@ -250,9 +254,15 @@ async fn debug_record_is_emitted_once_and_only_at_debug_level() {for level in [tracing::Level::DEBUG, tracing::Level::INFO] {let mut observer = observer("full").await;response(&mut observer, "text/event-stream");- observer.bytes(completed_item(0, "captured-output").as_bytes());+ observer.bytes(completed_item(0, "output-canary").as_bytes());observer.bytes(terminal("completed", 1).as_bytes());observer.end_body();+ observer.metadata["key_metadata"] = json!({"secret": "metadata-canary"});+ let ProtocolCapture::OpenAiResponses(capture) = observer.protocol.as_mut().unwrap();+ capture+ .facts+ .model_parameters+ .insert("user".into(), json!("parameter-canary"));let writer = LogWriter::default();let subscriber = tracing_subscriber::fmt().json()@@ -266,25 +276,28 @@ async fn debug_record_is_emitted_once_and_only_at_debug_level() {});let text = String::from_utf8(writer.0.lock().unwrap().clone()).unwrap();if level == tracing::Level::DEBUG {- assert_eq!(text.lines().count(), 1);- let event: Value = serde_json::from_str(&text).unwrap();- let capture: Value =- serde_json::from_str(event["fields"]["capture"].as_str().unwrap()).unwrap();- assert_eq!(- capture["output"][0]["content"][0]["text"],- "captured-output"- );+ let events = records(&writer);+ assert_eq!(events.len(), 1);+ let capture = &events[0];assert_eq!(capture["outcome"], "eof");- assert_eq!(capture["input"]["instructions"], "prompt-canary");+ assert_eq!(capture["capture_complete"], true);+ for excluded in ["input", "output", "model_parameters", "metadata"] {+ assert!(capture.get(excluded).is_none());+ }for secret in [+ "prompt-canary",+ "schema-canary",+ "output-canary",+ "metadata-canary",+ "parameter-canary","provider-secret","private-ingestion-token","gateway-secret",] {- assert!(!text.contains(secret));+ assert!(!text.contains(secret), "operational logs leaked {secret}");}} else {- assert!(text.is_empty());+ assert!(records(&writer).is_empty());}}}@@ -295,9 +308,8 @@ fn records(writer: &LogWriter) -> Vec<Value> {.lines().filter_map(|line| {let event: Value = serde_json::from_str(line).unwrap();- event["fields"]["capture"]- .as_str()- .map(|capture| serde_json::from_str(capture).unwrap())+ (event["fields"]["message"] == "gateway response captured")+ .then(|| event["fields"].clone())}).collect()}@@ -411,10 +423,6 @@ async fn native_http_relay_logs_once_on_eof_drop_and_unpolled_deadline() {assert_eq!(records.len(), 1);assert_eq!(records[0]["outcome"], json!(outcome));if outcome == RelayOutcome::Eof {- assert_eq!(- records[0]["output"][0]["content"][0]["text"],- "native-result"- );assert_eq!(records[0]["output_complete"], true);assert!(records[0]["first_byte_ms"].is_number());}@@ -481,6 +489,17 @@ async fn provider_future_finalizes_on_timeout_and_cancellation_before_headers()}}+fn uploaded_attribute(upload: &Value, key: &str) -> Value {+ let attributes = upload["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]+ .as_array()+ .unwrap();+ let attribute = attributes+ .iter()+ .find(|attribute| attribute["key"] == key)+ .unwrap();+ serde_json::from_str(attribute["value"]["stringValue"].as_str().unwrap()).unwrap()+}+#[tokio::test]async fn client_compression_preferences_do_not_disable_capture() {for streaming in [false, true] {@@ -512,24 +531,34 @@ async fn client_compression_preferences_do_not_disable_capture() {}}).await;+ let uploaded = Arc::new(Mutex::new(Value::Null));+ let received = uploaded.clone();+ let collector = FakeServer::start(move |request| {+ let received = received.clone();+ async move {+ let bytes = to_bytes(request.into_body(), 64 * 1024).await.unwrap();+ *received.lock().unwrap() = serde_json::from_slice(&bytes).unwrap();+ Response::new(Body::from("{}"))+ }+ })+ .await;+ let telemetry = crate::telemetry::Telemetry::new(+ &crate::resolution::ControlPlaneConfig::new(&collector.url, "test-service-key")+ .unwrap(),+ )+ .unwrap();let context = resolved_request_context_with_mode("provider-secret", "full").await;let provider = OpenAiProvider::for_test(format!("{}/v1/responses", upstream.url),ProviderLimits::default(),- );- let writer = LogWriter::default();- let subscriber = tracing_subscriber::fmt()- .json()- .with_max_level(tracing::Level::DEBUG)- .with_writer(writer.clone())- .finish();- let _guard = tracing::subscriber::set_default(subscriber);+ )+ .with_telemetry(telemetry.clone());let mut headers = HeaderMap::new();headers.insert(header::ACCEPT_ENCODING,HeaderValue::from_static("gzip, deflate, br"),);- let response = provider+ let forwarded = provider.forward(provider.try_admit().unwrap(),context,@@ -539,18 +568,22 @@ async fn client_compression_preferences_do_not_disable_capture() {.await.unwrap();assert_eq!(- to_bytes(response.into_body(), 4096).await.unwrap(),+ to_bytes(forwarded.into_body(), 4096).await.unwrap(),expected);- let records = records(&writer);- assert_eq!(records.len(), 1);- assert_eq!(records[0]["capture_complete"], true);- assert_eq!(records[0]["output_complete"], true);- assert_eq!(records[0]["provider_response_id"], "resp-1");- assert_eq!(records[0]["provider_status"], "completed");- assert!(records[0]["usage_details"]["input_tokens"].is_number());+ telemetry+ .shutdown(tokio::time::Instant::now() + Duration::from_secs(2))+ .await;+ assert_eq!(collector.calls(), 1);+ let upload = uploaded.lock().unwrap();+ let metadata = uploaded_attribute(&upload, "langfuse.observation.metadata");+ assert_eq!(metadata["capture_complete"], true);+ assert_eq!(metadata["output_complete"], true);+ assert_eq!(metadata["provider_response_id"], "resp-1");+ assert_eq!(metadata["provider_status"], "completed");+ assert!(metadata["native_usage"]["input_tokens"].is_number());assert_eq!(- records[0]["output"][0]["content"][0]["text"],+ uploaded_attribute(&upload, "langfuse.observation.output")[0]["content"][0]["text"],"captured output");}@@ -660,7 +693,6 @@ async fn downstream_cancellation_does_not_erase_completed_capture() {let facts = records(&writer);assert_eq!(facts.len(), 1);assert_eq!(facts[0]["outcome"], "cancelled");- assert_eq!(facts[0]["provider_status"], "completed");assert_eq!(facts[0]["output_complete"], true);assert_eq!(facts[0]["capture_complete"], true);}diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rsindex 9deb1dd077..aa8a2fd709 100644--- a/ai-gateway/src/config.rs+++ b/ai-gateway/src/config.rs@@ -105,7 +105,6 @@ impl GatewayConfig {));}let log_level = match log_level.unwrap_or("info") {- "trace" => LevelFilter::TRACE,"debug" => LevelFilter::DEBUG,"info" => LevelFilter::INFO,"warn" => LevelFilter::WARN,@@ -113,7 +112,7 @@ impl GatewayConfig {"error" | "fatal" => LevelFilter::ERROR,_ => {return Err(GatewayConfigError(- "LANGFUSE_LOG_LEVEL must be trace, debug, info, warn, error or fatal",+ "LANGFUSE_LOG_LEVEL must be debug, info, warn, error or fatal",));}};@@ -232,7 +231,6 @@ mod tests {#[test]fn accepts_shared_langfuse_log_levels() {for (value, expected) in [- ("trace", LevelFilter::TRACE),("debug", LevelFilter::DEBUG),("info", LevelFilter::INFO),("warn", LevelFilter::WARN),@@ -254,6 +252,7 @@ mod tests {(None, Some(sensitive_input), None),(None, None, Some(sensitive_input)),(None, None, Some("off")),+ (None, None, Some("trace")),(None, None, Some("INFO")),(None, None, Some("3")),(None, None, Some("")),diff --git a/ai-gateway/src/http.rs b/ai-gateway/src/http.rsindex f56f4138cd..aad0966047 100644--- a/ai-gateway/src/http.rs+++ b/ai-gateway/src/http.rs@@ -121,9 +121,46 @@ struct OpenAiErrorDetail {code: &'static str,}+impl InferenceHttpError {+ fn category(&self) -> (&'static str, &'static str) {+ use ResolutionError as R;+ match self {+ Self::Unavailable => ("admission", "unavailable"),+ Self::Credential => ("authentication", "invalid_credential"),+ Self::TooLarge => ("request", "body_too_large"),+ Self::RequestTimeout => ("request", "body_timeout"),+ Self::InvalidBody => ("request", "invalid_body"),+ Self::Resolution(error) => (+ "resolution",+ match error {+ R::InvalidCredential | R::Authentication => "authentication",+ R::Forbidden => "forbidden",+ R::NoRoute => "no_route",+ R::Timeout => "timeout",+ R::Unavailable => "unavailable",+ R::Configuration => "configuration",+ R::Transport => "transport",+ R::InvalidResponse => "invalid_response",+ R::ResponseTooLarge => "response_too_large",+ },+ ),+ Self::Provider(error) => (+ "provider",+ match error {+ ProviderError::Busy => "capacity",+ ProviderError::Timeout => "timeout",+ ProviderError::Transport => "transport",+ ProviderError::Configuration => "configuration",+ },+ ),+ }+ }+}+impl IntoResponse for InferenceHttpError {fn into_response(self) -> Response<Body> {use ResolutionError as R;+ let (phase, reason) = self.category();let (status, message, kind, code) = match self {Self::Credential | Self::Resolution(R::InvalidCredential | R::Authentication) => (StatusCode::UNAUTHORIZED,@@ -183,6 +220,21 @@ impl IntoResponse for InferenceHttpError {"upstream_error",),};+ if status.is_server_error() {+ tracing::warn!(+ phase,+ reason,+ status = status.as_u16(),+ "gateway request rejected"+ );+ } else {+ tracing::debug!(+ phase,+ reason,+ status = status.as_u16(),+ "gateway request rejected"+ );+ }(status,Json(OpenAiErrorResponse {diff --git a/ai-gateway/src/inference.rs b/ai-gateway/src/inference.rsindex 37fba9035c..b7bc47246e 100644--- a/ai-gateway/src/inference.rs+++ b/ai-gateway/src/inference.rs@@ -58,10 +58,11 @@ impl InferenceService {gateway_key: &str,) -> Result<(RequestPermit, ResolvedRequestContext), RequestPreparationError> {let context = {- let _permit = self- .resolution_capacity- .try_acquire()- .map_err(|_| RequestPreparationError::Resolution(ResolutionError::Unavailable))?;+ let _permit = self.resolution_capacity.try_acquire().map_err(|_| {+ crate::observability::rejected("resolution");+ RequestPreparationError::Resolution(ResolutionError::Unavailable)+ })?;+ let _active = crate::observability::Active::new("resolution");self.control_plane.resolve(gateway_key, ApiFormat::OpenAiResponses).awaitdiff --git a/ai-gateway/src/lib.rs b/ai-gateway/src/lib.rsindex 1d06512e43..0b9caa6ed0 100644--- a/ai-gateway/src/lib.rs+++ b/ai-gateway/src/lib.rs@@ -2,6 +2,7 @@ mod capture;pub mod config;pub mod http;pub mod inference;+pub mod observability;pub mod providers;pub mod resolution;pub mod server;diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rsindex 94c87f689f..71c7193383 100644--- a/ai-gateway/src/main.rs+++ b/ai-gateway/src/main.rs@@ -1,9 +1,10 @@use std::{error::Error, io, net::SocketAddr, process::ExitCode};use ai_gateway::{- config::{GatewayConfig, LogFormat},+ config::GatewayConfig,http,inference::InferenceService,+ observability,server::{self, GatewayLifecycleState},};use tokio::net::TcpListener;@@ -21,13 +22,7 @@ async fn main() -> ExitCode {async fn run() -> Result<(), Box<dyn Error>> {let config = GatewayConfig::from_env()?;- let logging = tracing_subscriber::fmt()- .with_max_level(config.log_level)- .with_target(false);- match config.log_format {- LogFormat::Text => logging.compact().init(),- LogFormat::Json => logging.json().init(),- }+ let observability = observability::init(&config)?;let inference = config.control_plane.map(|control_plane| {@@ -45,10 +40,13 @@ async fn run() -> Result<(), Box<dyn Error>> {} else {GatewayLifecycleState::unconfigured()};- let app = server::router(state.clone()).merge(http::router(inference, state.clone()));+ let app = server::router(state.clone()).merge(observability::instrument(http::router(+ inference,+ state.clone(),+ )));let shutdown = shutdown_signal()?;let listener = bind_listener(config.listen_address, config.auto_increment_listen_port).await?;- tracing::info!(address = %listener.local_addr()?, inference_enabled, "gateway listening");+ tracing::info!(address = %listener.local_addr()?, inference_enabled, max_active_requests = config.max_active_requests, max_concurrent_resolutions = config.max_concurrent_resolutions, shutdown_timeout_seconds = config.shutdown_timeout.as_secs(), "gateway listening");let result = server::serve(listener,app,@@ -66,9 +64,19 @@ async fn run() -> Result<(), Box<dyn Error>> {).await;}- result?;- tracing::info!("gateway stopped");- Ok(())+ if result.is_err() {+ tracing::error!("gateway shutdown failed");+ } else {+ tracing::info!("gateway stopped");+ }+ observability+ .shutdown(+ state+ .drain_deadline()+ .unwrap_or_else(tokio::time::Instant::now),+ )+ .await;+ result.map_err(Into::into)}async fn bind_listener(diff --git a/ai-gateway/src/observability/client.rs b/ai-gateway/src/observability/client.rsnew file mode 100644index 0000000000..45d5e3c4eb--- /dev/null+++ b/ai-gateway/src/observability/client.rs@@ -0,0 +1,161 @@+use axum::http::Extensions;+use reqwest::{Client, Request, Response};+use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, Extension, Result};+use reqwest_tracing::{+ DefaultSpanBackend, OtelName, ReqwestOtelSpanBackend, TracingMiddleware,+ default_on_request_success,+};+use tracing::Span;++pub(crate) fn instrument_client(client: Client, name: &'static str) -> ClientWithMiddleware {+ ClientBuilder::new(client)+ .with_init(Extension(OtelName(name.into())))+ .with(TracingMiddleware::<SafeSpanBackend>::new())+ .build()+}++struct SafeSpanBackend;++impl ReqwestOtelSpanBackend for SafeSpanBackend {+ fn on_request_start(request: &Request, extensions: &mut Extensions) -> Span {+ DefaultSpanBackend::on_request_start(request, extensions)+ }++ fn on_request_end(span: &Span, outcome: &Result<Response>, _: &mut Extensions) {+ match outcome {+ Ok(response) => {+ default_on_request_success(span, response);+ span.record(+ "http.response.status_code",+ i64::from(response.status().as_u16()),+ );+ }+ // Default failure fields include URLs and raw cause chains.+ Err(error) => {+ span.record("otel.status_code", "ERROR");+ span.record(+ "error.message",+ if error.is_timeout() {+ "timeout"+ } else {+ "transport"+ },+ );+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use axum::{Router, http::HeaderMap, routing::post};+ use opentelemetry::{+ KeyValue,+ baggage::BaggageExt,+ trace::{FutureExt, TraceContextExt, TracerProvider},+ };+ use opentelemetry_sdk::{+ propagation::TraceContextPropagator,+ trace::{InMemorySpanExporter, SdkTracerProvider},+ };+ use reqwest_tracing::DisableOtelPropagation;+ use tracing::instrument::WithSubscriber;+ use tracing_opentelemetry::OpenTelemetrySpanExt;+ use tracing_subscriber::prelude::*;++ #[tokio::test]+ async fn middleware_preserves_safe_context_and_sanitizes_transport_errors() {+ opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let subscriber = tracing::Dispatch::new(+ tracing_subscriber::registry()+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test"))),+ );+ let parent = tracing::dispatcher::with_default(&subscriber, || {+ tracing::info_span!("request").context()+ })+ .with_baggage([KeyValue::new("customer", "secret-baggage")]);+ // A detached upload can inherit trace IDs without extending the server span.+ assert_eq!(exporter.get_finished_spans().unwrap().len(), 1);+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let endpoint = format!("http://{}/", listener.local_addr().unwrap());+ let (headers_tx, mut headers_rx) = tokio::sync::mpsc::unbounded_channel();+ let server = tokio::spawn(async move {+ axum::serve(+ listener,+ Router::new().route(+ "/",+ post(move |headers: HeaderMap| async move {+ headers_tx.send(headers).unwrap();+ axum::http::StatusCode::NO_CONTENT+ }),+ ),+ )+ .await+ .unwrap();+ });+ let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let failed_url = format!(+ "http://{}:unused@{}/private-content?token=secret-query",+ "secret-user",+ closed.local_addr().unwrap()+ );+ drop(closed);+ async {+ let client = Client::builder().no_proxy().build().unwrap();+ instrument_client(client.clone(), "trusted")+ .post(&endpoint)+ .send()+ .await+ .unwrap();+ let trusted = headers_rx.recv().await.unwrap();+ assert!(trusted.contains_key("traceparent"));+ assert!(!trusted.contains_key("baggage"));+ instrument_client(client.clone(), "external")+ .post(&endpoint)+ .with_extension(DisableOtelPropagation)+ .send()+ .await+ .unwrap();+ let external = headers_rx.recv().await.unwrap();+ assert!(!external.contains_key("traceparent"));+ assert!(!external.contains_key("baggage"));+ assert!(+ instrument_client(client, "failed")+ .post(failed_url)+ .send()+ .await+ .is_err()+ );+ }+ .with_context(parent.clone())+ .with_subscriber(subscriber)+ .await;+ server.abort();+ let spans = exporter.get_finished_spans().unwrap();+ assert_eq!(spans.len(), 4);+ for span in &spans[1..] {+ assert_eq!(span.parent_span_id, parent.span().span_context().span_id());+ assert_eq!(+ span.span_context.trace_id(),+ parent.span().span_context().trace_id()+ );+ }+ let failed = spans.iter().find(|span| span.name == "failed").unwrap();+ assert!(matches!(+ failed.status,+ opentelemetry::trace::Status::Error { .. }+ ));+ let attributes = format!("{:?}", failed.attributes);+ assert!(attributes.contains("transport"));+ for secret in ["secret-user", "unused", "private-content", "secret-query"] {+ assert!(!attributes.contains(secret), "leaked {secret}");+ }+ assert!(!attributes.contains("error.cause_chain"));+ provider.shutdown().unwrap();+ }+}diff --git a/ai-gateway/src/observability/http.rs b/ai-gateway/src/observability/http.rsnew file mode 100644index 0000000000..bfa2800049--- /dev/null+++ b/ai-gateway/src/observability/http.rs@@ -0,0 +1,73 @@+use std::time::{Duration, Instant};++use axum::{+ Router,+ body::Body,+ extract::MatchedPath,+ http::Request,+ middleware::{self, Next},+ response::Response,+};+use opentelemetry::{propagation::TextMapPropagator, trace::TraceContextExt};+use opentelemetry_http::HeaderExtractor;+use opentelemetry_sdk::propagation::TraceContextPropagator;+use tower_http::trace::TraceLayer;+use tracing::Span;+use tracing_opentelemetry::OpenTelemetrySpanExt;++/// Instrument application routes; mount health probes outside this router.+pub fn instrument(router: Router) -> Router {+ router+ .layer(middleware::from_fn(record_http_duration))+ .layer(+ TraceLayer::new_for_http()+ .make_span_with(|request: &Request<Body>| {+ let (method, route) = request_labels(request);+ let span = tracing::info_span!("http.server", otel.name = %format_args!("{method} {route}"), otel.kind = "server", http.request.method = method, http.route = route, http.response.status_code = tracing::field::Empty, otel.status_code = tracing::field::Empty, provider_request_id = tracing::field::Empty);+ let parent = TraceContextPropagator::new().extract(&HeaderExtractor(request.headers()));+ let _ = span.set_parent(parent);+ span+ })+ .on_request(())+ .on_response(|response: &Response, latency: Duration, span: &Span| {+ let status = response.status();+ span.record("http.response.status_code", i64::from(status.as_u16()));+ if status.is_server_error() {+ span.record("otel.status_code", "ERROR");+ }+ let context = span.context();+ let context_span = context.span();+ let ids = context_span.span_context();+ tracing::info!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), status = status.as_u16(), duration_ms = latency.as_secs_f64() * 1000.0, "gateway response started");+ })+ .on_body_chunk(())+ .on_eos(())+ .on_failure(()),+ )+}++async fn record_http_duration(request: Request<Body>, next: Next) -> Response {+ let (method, route) = request_labels(&request);+ let (method, route) = (method.to_owned(), route.to_owned());+ let start = Instant::now();+ let response = next.run(request).await;+ super::metrics::http_response(start.elapsed(), method, route, response.status().as_u16());+ response+}++fn request_labels(request: &Request<Body>) -> (&str, &str) {+ let route = request+ .extensions()+ .get::<MatchedPath>()+ .map_or("unmatched", MatchedPath::as_str);+ let method = match request.method().as_str() {+ "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" | "CONNECT" | "TRACE" => {+ request.method().as_str()+ }+ _ => "_OTHER",+ };+ (method, route)+}++#[cfg(test)]+mod tests;diff --git a/ai-gateway/src/observability/http/tests.rs b/ai-gateway/src/observability/http/tests.rsnew file mode 100644index 0000000000..1c92a28973--- /dev/null+++ b/ai-gateway/src/observability/http/tests.rs@@ -0,0 +1,211 @@+use super::*;+use axum::body::Bytes;+use axum::{+ Router,+ http::{HeaderMap, Request as HttpRequest},+};+use http_body::Frame;+use http_body_util::{BodyExt, StreamBody};+use opentelemetry::trace::{SpanKind, TracerProvider};+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider};+use serde_json::Value;+use std::{+ convert::Infallible,+ io,+ sync::{Arc, Mutex},+};+use tower::ServiceExt;+use tracing_subscriber::prelude::*;++#[derive(Clone, Default)]+struct Buffer(Arc<Mutex<Vec<u8>>>);+impl io::Write for Buffer {+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {+ self.0.lock().unwrap().extend_from_slice(bytes);+ Ok(bytes.len())+ }+ fn flush(&mut self) -> io::Result<()> {+ Ok(())+ }+}++struct Recording {+ _guard: tracing::subscriber::DefaultGuard,+ _provider: SdkTracerProvider,+ exporter: InMemorySpanExporter,+ output: Buffer,+}+impl Recording {+ fn start() -> Self {+ let exporter = InMemorySpanExporter::default();+ let provider = SdkTracerProvider::builder()+ .with_simple_exporter(exporter.clone())+ .build();+ let output = Buffer::default();+ let buffer = output.clone();+ let subscriber = tracing_subscriber::registry()+ .with(+ tracing_subscriber::fmt::layer()+ .json()+ .flatten_event(true)+ .with_writer(move || buffer.clone()),+ )+ .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("test")));+ Self {+ _guard: tracing::subscriber::set_default(subscriber),+ _provider: provider,+ exporter,+ output,+ }+ }+ fn summaries(&self) -> Vec<Value> {+ String::from_utf8(self.output.0.lock().unwrap().clone())+ .unwrap()+ .lines()+ .map(|line| serde_json::from_str::<Value>(line).unwrap())+ .filter(|event| event["message"] == "gateway response started")+ .collect()+ }+}++fn incoming() -> HttpRequest<Body> {+ HttpRequest::builder()+ .method("POST")+ .uri("/openai/v1/responses")+ .header(+ "traceparent",+ "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",+ )+ .header("baggage", "private=content-canary")+ .body(Body::empty())+ .unwrap()+}++#[tokio::test]+async fn inherits_trace_and_preserves_stream_lifetime_bytes_and_trailers() {+ let recording = Recording::start();+ let app = instrument(Router::new().fallback(|| async {+ let mut trailers = HeaderMap::new();+ trailers.insert("x-checksum", "verified".parse().unwrap());+ let frames = [+ Ok::<_, Infallible>(Frame::data(Bytes::from_static(b"native bytes"))),+ Ok(Frame::trailers(trailers)),+ ];+ Body::new(StreamBody::new(futures_util::stream::iter(frames)))+ }));+ let response = app.oneshot(incoming()).await.unwrap();+ assert!(recording.exporter.get_finished_spans().unwrap().is_empty());+ let summaries = recording.summaries();+ assert_eq!(summaries.len(), 1);+ assert_eq!(summaries[0]["status"], 200);+ let mut body = response.into_body();+ assert_eq!(+ body.frame().await.unwrap().unwrap().into_data().unwrap(),+ "native bytes"+ );+ let trailers = body+ .frame()+ .await+ .unwrap()+ .unwrap()+ .into_trailers()+ .unwrap();+ assert_eq!(trailers["x-checksum"], "verified");+ assert!(body.frame().await.is_none());+ drop(body);+ let spans = recording.exporter.get_finished_spans().unwrap();+ assert_eq!(spans.len(), 1);+ let server = &spans[0];+ assert_eq!(server.span_kind, SpanKind::Server);+ assert_eq!(+ server.span_context.trace_id().to_string(),+ "4bf92f3577b34da6a3ce929d0e0e4736"+ );+ assert_eq!(server.parent_span_id.to_string(), "00f067aa0ba902b7");+ assert_eq!(+ summaries[0]["trace_id"],+ server.span_context.trace_id().to_string()+ );+ assert_eq!(+ summaries[0]["span_id"],+ server.span_context.span_id().to_string()+ );+}++#[tokio::test]+async fn cancellation_releases_spans_before_and_after_headers() {+ for before_headers in [true, false] {+ let recording = Recording::start();+ let app = instrument(Router::new().fallback(move || async move {+ if before_headers {+ std::future::pending::<()>().await;+ }+ Body::from_stream(futures_util::stream::pending::<Result<Bytes, Infallible>>())+ }));+ if before_headers {+ let mut call = Box::pin(app.oneshot(incoming()));+ assert!(futures_util::poll!(call.as_mut()).is_pending());+ drop(call);+ } else {+ let response = app.oneshot(incoming()).await.unwrap();+ assert!(recording.exporter.get_finished_spans().unwrap().is_empty());+ drop(response);+ }+ assert_eq!(recording.exporter.get_finished_spans().unwrap().len(), 1);+ }+}++#[tokio::test]+async fn provider_http_errors_are_traced_without_changing_the_response() {+ use crate::{+ providers::openai::{OpenAiProvider, ProviderLimits},+ test_support::{FakeServer, resolved_request_context},+ };+ for status in [429, 500] {+ let context = resolved_request_context("provider-secret").await;+ let upstream = FakeServer::start(move |_| async move {+ Response::builder()+ .status(status)+ .body(Body::from("native error"))+ .unwrap()+ })+ .await;+ let provider = OpenAiProvider::for_test(upstream.url.clone(), ProviderLimits::default());+ let recording = Recording::start();+ let response = provider+ .forward(+ provider.try_admit().unwrap(),+ context,+ &HeaderMap::new(),+ Bytes::from_static(b"{}"),+ )+ .await+ .unwrap();+ assert_eq!(response.status().as_u16(), status);+ assert_eq!(+ axum::body::to_bytes(response.into_body(), 100)+ .await+ .unwrap(),+ "native error"+ );+ let spans = recording.exporter.get_finished_spans().unwrap();+ let upstream_span = spans+ .iter()+ .find(|span| span.name == "provider.headers")+ .unwrap();+ assert!(matches!(+ upstream_span.status,+ opentelemetry::trace::Status::Error { .. }+ ));+ assert!(+ upstream_span+ .attributes+ .iter()+ .any(+ |attribute| attribute.key.as_str() == "http.response.status_code"+ && attribute.value == opentelemetry::Value::I64(i64::from(status))+ )+ );+ assert_eq!(upstream.calls(), 1);+ }+}diff --git a/ai-gateway/src/observability/metrics.rs b/ai-gateway/src/observability/metrics.rsnew file mode 100644index 0000000000..d780259b15--- /dev/null+++ b/ai-gateway/src/observability/metrics.rs@@ -0,0 +1,130 @@+use std::sync::LazyLock;++use crate::capture::RelayOutcome;+use opentelemetry::trace::TraceContextExt;+use opentelemetry::{+ KeyValue, global,+ metrics::{Counter, Histogram, UpDownCounter},+};+use tracing_opentelemetry::OpenTelemetrySpanExt;++struct Metrics {+ http_duration: Histogram<f64>,+ active: UpDownCounter<i64>,+ phases: Histogram<f64>,+ rejections: Counter<u64>,+ delivery: Counter<u64>,+ executions: Counter<u64>,+}++static METRICS: LazyLock<Metrics> = LazyLock::new(|| {+ let meter = global::meter("ai-gateway");+ let seconds = vec![+ 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 600.0,+ ];+ Metrics {+ http_duration: meter+ .f64_histogram("http.server.request.duration")+ .with_unit("s")+ .with_boundaries(vec![+ 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0,+ ])+ .build(),+ active: meter.i64_up_down_counter("gateway.active").build(),+ phases: meter+ .f64_histogram("gateway.phase.duration")+ .with_unit("s")+ .with_boundaries(seconds)+ .build(),+ rejections: meter.u64_counter("gateway.admission.rejected").build(),+ delivery: meter.u64_counter("gateway.telemetry.records").build(),+ executions: meter.u64_counter("gateway.executions").build(),+ }+});++pub(super) fn http_response(+ duration: std::time::Duration,+ method: String,+ route: String,+ status: u16,+) {+ METRICS.http_duration.record(+ duration.as_secs_f64(),+ &[+ KeyValue::new("http.request.method", method),+ KeyValue::new("http.route", route),+ KeyValue::new("http.response.status_code", i64::from(status)),+ ],+ );+}++pub(crate) struct Active(&'static str);++impl Active {+ pub(crate) fn new(phase: &'static str) -> Self {+ METRICS.active.add(1, &[KeyValue::new("phase", phase)]);+ Self(phase)+ }+}++impl Drop for Active {+ fn drop(&mut self) {+ METRICS.active.add(-1, &[KeyValue::new("phase", self.0)]);+ }+}++pub(crate) fn phase_finished(phase: &'static str, duration: std::time::Duration) {+ METRICS+ .phases+ .record(duration.as_secs_f64(), &[KeyValue::new("phase", phase)]);+}++pub(crate) fn rejected(phase: &'static str) {+ METRICS.rejections.add(1, &[KeyValue::new("phase", phase)]);+}++pub(crate) fn delivery(outcome: &'static str, reason: &'static str) {+ METRICS.delivery.add(+ 1,+ &[+ KeyValue::new("outcome", outcome),+ KeyValue::new("reason", reason),+ ],+ );+}++pub(crate) fn execution_finished(facts: &crate::capture::InferenceFacts) {+ if let Some(request_id) = &facts.inference.provider_request_id {+ tracing::Span::current().record("provider_request_id", request_id.as_str());+ }+ let outcome = match facts.outcome {+ RelayOutcome::Eof => "complete",+ RelayOutcome::Cancelled => "cancelled",+ RelayOutcome::Timeout => "timeout",+ RelayOutcome::TransportError => "transport_error",+ };+ METRICS+ .executions+ .add(1, &[KeyValue::new("outcome", outcome)]);+ if let Some(first_byte_ms) = facts.first_byte_ms {+ phase_finished(+ "provider.first_byte",+ std::time::Duration::from_millis(u64::try_from(first_byte_ms).unwrap_or(u64::MAX)),+ );+ }+ phase_finished(+ "execution",+ std::time::Duration::from_millis(u64::try_from(facts.duration_ms).unwrap_or(u64::MAX)),+ );+ let context = tracing::Span::current().context();+ let context_span = context.span();+ let ids = context_span.span_context();+ tracing::info!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), outcome, duration_ms = facts.duration_ms, "gateway execution finished");+ if matches!(+ facts.outcome,+ RelayOutcome::Timeout | RelayOutcome::TransportError+ ) {+ tracing::Span::current().record("otel.status_code", "ERROR");+ tracing::warn!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), phase = "provider", outcome, "gateway stream failed");+ }+}diff --git a/ai-gateway/src/observability/mod.rs b/ai-gateway/src/observability/mod.rsnew file mode 100644index 0000000000..02641e3371--- /dev/null+++ b/ai-gateway/src/observability/mod.rs@@ -0,0 +1,191 @@+mod client;+mod http;+mod metrics;++pub(crate) use client::instrument_client;+pub use http::instrument;+pub(crate) use metrics::{Active, delivery, execution_finished, rejected};++use crate::config::{GatewayConfig, LogFormat};+use opentelemetry::{KeyValue, global, trace::TracerProvider};+use opentelemetry_otlp::{Protocol, WithExportConfig};+use opentelemetry_sdk::{+ Resource,+ metrics::{PeriodicReader, SdkMeterProvider},+ propagation::TraceContextPropagator,+ trace::{BatchConfigBuilder, BatchSpanProcessor, Sampler, SdkTracerProvider},+};+use std::{env, error::Error, time::Duration};+use tokio::time::Instant;+use tracing_subscriber::{EnvFilter, Layer, filter::filter_fn, prelude::*};++pub struct Observability {+ traces: SdkTracerProvider,+ metrics: Option<SdkMeterProvider>,+}++/// Initialize process logging and optional operational OTLP exports.+///+/// # Errors+/// Returns a sanitized error for invalid settings, exporter construction, or an+/// already installed tracing subscriber.+pub fn init(config: &GatewayConfig) -> Result<Observability, Box<dyn Error>> {+ let service = setting("OTEL_SERVICE_NAME")?.unwrap_or_else(|| "ai-gateway".into());+ let resource = Resource::builder()+ .with_service_name(service)+ .with_attributes([+ KeyValue::new(+ "deployment.environment.name",+ setting("DD_ENV")?.unwrap_or_else(|| "development".into()),+ ),+ KeyValue::new(+ "service.version",+ setting("BUILD_ID")?.unwrap_or_else(|| env!("CARGO_PKG_VERSION").into()),+ ),+ ])+ .build();+ let ratio = sampling_ratio(setting("OTEL_TRACES_SAMPLER_ARG")?.as_deref())?;+ let mut traces = SdkTracerProvider::builder()+ .with_resource(resource.clone())+ .with_sampler(Sampler::ParentBased(Box::new(Sampler::TraceIdRatioBased(+ ratio,+ ))));+ let metrics = if let Some(endpoint) = setting("OTEL_EXPORTER_OTLP_ENDPOINT")? {+ let endpoint =+ reqwest::Url::parse(&endpoint).map_err(|_| "invalid OTEL_EXPORTER_OTLP_ENDPOINT")?;+ if !matches!(endpoint.scheme(), "http" | "https") || endpoint.host_str().is_none() {+ return Err("invalid OTEL_EXPORTER_OTLP_ENDPOINT".into());+ }+ let exporter = opentelemetry_otlp::SpanExporter::builder()+ .with_http()+ .with_protocol(Protocol::HttpBinary)+ .with_timeout(Duration::from_secs(3))+ .build()+ .map_err(|_| "failed to initialize operational trace exporter")?;+ traces = traces.with_span_processor(+ BatchSpanProcessor::builder(exporter)+ .with_batch_config(+ BatchConfigBuilder::default()+ .with_max_queue_size(512)+ .with_max_export_batch_size(64)+ .with_scheduled_delay(Duration::from_secs(1))+ .build(),+ )+ .build(),+ );+ let exporter = opentelemetry_otlp::MetricExporter::builder()+ .with_http()+ .with_protocol(Protocol::HttpBinary)+ .with_timeout(Duration::from_secs(3))+ .build()+ .map_err(|_| "failed to initialize operational metric exporter")?;+ Some(+ SdkMeterProvider::builder()+ .with_resource(resource)+ .with_reader(+ PeriodicReader::builder(exporter)+ .with_interval(Duration::from_secs(30))+ .build(),+ )+ .build(),+ )+ } else {+ None+ };+ let traces = traces.build();+ global::set_text_map_propagator(TraceContextPropagator::new());+ if let Some(metrics) = &metrics {+ global::set_meter_provider(metrics.clone());+ }+ let log_filter = EnvFilter::try_new(format!("warn,ai_gateway={}", config.log_level))?;+ let logger = match config.log_format {+ LogFormat::Json => tracing_subscriber::fmt::layer()+ .json()+ .flatten_event(true)+ .with_filter(log_filter)+ .boxed(),+ LogFormat::Text => tracing_subscriber::fmt::layer()+ .compact()+ .with_ansi(false)+ .with_filter(log_filter)+ .boxed(),+ };+ let spans = tracing_opentelemetry::layer()+ .with_tracer(traces.tracer("ai-gateway"))+ .with_tracked_inactivity(false)+ .with_filter(filter_fn(|metadata| {+ metadata.is_span()+ && (metadata.target().starts_with("ai_gateway")+ || metadata.target().starts_with("reqwest_tracing"))+ && *metadata.level() <= tracing::Level::INFO+ }));+ tracing_subscriber::registry()+ .with(logger)+ .with(spans)+ .try_init()+ .map_err(|_| "failed to initialize gateway logging")?;+ Ok(Observability { traces, metrics })+}++impl Observability {+ /// Flush exporters within the shared gateway shutdown deadline.+ pub async fn shutdown(self, deadline: Instant) {+ let (trace_sender, traces) = tokio::sync::oneshot::channel();+ let (metric_sender, metrics) = tokio::sync::oneshot::channel();+ std::thread::spawn(move || {+ let _ = trace_sender.send(+ self.traces+ .shutdown_with_timeout(deadline.saturating_duration_since(Instant::now())),+ );+ });+ std::thread::spawn(move || {+ let _ = metric_sender.send(self.metrics.map(|provider| {+ provider.shutdown_with_timeout(deadline.saturating_duration_since(Instant::now()))+ }));+ });+ match tokio::time::timeout_at(deadline, async { tokio::join!(traces, metrics) }).await {+ Ok((Ok(Ok(())), Ok(None | Some(Ok(()))))) => {}+ Ok(_) => tracing::warn!("operational telemetry shutdown failed"),+ Err(_) => tracing::warn!("operational telemetry shutdown deadline reached"),+ }+ }+}++fn setting(name: &str) -> Result<Option<String>, Box<dyn Error>> {+ match env::var(name) {+ Ok(value) if value.trim().is_empty() => Ok(None),+ Ok(value) => Ok(Some(value)),+ Err(env::VarError::NotPresent) => Ok(None),+ Err(env::VarError::NotUnicode(_)) => {+ Err("observability configuration must be Unicode".into())+ }+ }+}++fn sampling_ratio(value: Option<&str>) -> Result<f64, Box<dyn Error>> {+ value+ .unwrap_or("1")+ .parse::<f64>()+ .ok()+ .filter(|value| (0.0..=1.0).contains(value))+ .ok_or_else(|| "OTEL_TRACES_SAMPLER_ARG must be a number from 0 to 1".into())+}++#[cfg(test)]+mod tests {+ use super::sampling_ratio;++ #[test]+ fn sampling_rejects_nonfinite_out_of_range_and_sensitive_values() {+ for value in ["NaN", "inf", "-0.1", "1.1", "secret-token"] {+ let error = sampling_ratio(Some(value)).unwrap_err();+ assert_eq!(+ error.to_string(),+ "OTEL_TRACES_SAMPLER_ARG must be a number from 0 to 1"+ );+ }+ for (value, expected) in [(None, 1.0), (Some("0"), 0.0), (Some("0.25"), 0.25)] {+ assert!((sampling_ratio(value).unwrap() - expected).abs() < f64::EPSILON);+ }+ }+}diff --git a/ai-gateway/src/providers/openai.rs b/ai-gateway/src/providers/openai.rsindex 7b05775e35..c036702b43 100644--- a/ai-gateway/src/providers/openai.rs+++ b/ai-gateway/src/providers/openai.rs@@ -6,6 +6,8 @@ use axum::{http::{HeaderMap, HeaderValue, Response, header},};use reqwest::Client;+use reqwest_middleware::ClientWithMiddleware;+use reqwest_tracing::DisableOtelPropagation;use tokio::{sync::{OwnedSemaphorePermit, Semaphore},time::Instant,@@ -40,12 +42,13 @@ impl Default for ProviderLimits {/// An admitted execution. Dropping it releases capacity; there is no waiting queue.pub struct RequestPermit {_permit: OwnedSemaphorePermit,+ _active: crate::observability::Active,deadline: Instant,}/// A pooled client for the official `OpenAI` Responses endpoint.pub struct OpenAiProvider {- client: Client,+ client: ClientWithMiddleware,capacity: Arc<Semaphore>,limits: ProviderLimits,telemetry: Option<crate::telemetry::Telemetry>,@@ -83,7 +86,7 @@ impl OpenAiProvider {.build().map_err(|_| ProviderError::Configuration)?;Ok(Self {- client,+ client: crate::observability::instrument_client(client, "provider.headers"),capacity: Arc::new(Semaphore::new(limits.active)),limits,telemetry: None,@@ -102,13 +105,13 @@ impl OpenAiProvider {/// # Errors/// Returns [`ProviderError::Busy`] immediately when all execution slots are occupied.pub fn try_admit(&self) -> Result<RequestPermit, ProviderError> {- let permit = self- .capacity- .clone()- .try_acquire_owned()- .map_err(|_| ProviderError::Busy)?;+ let permit = self.capacity.clone().try_acquire_owned().map_err(|_| {+ crate::observability::rejected("execution");+ ProviderError::Busy+ })?;Ok(RequestPermit {_permit: permit,+ _active: crate::observability::Active::new("execution"),deadline: Instant::now() + self.limits.execution_timeout,})}@@ -145,6 +148,7 @@ impl OpenAiProvider {.min(Instant::now() + self.limits.headers_timeout),self.client.post(endpoint)+ .with_extension(DisableOtelPropagation).headers(transport::request_headers(headers))// Observe plain JSON/SSE while relaying the provider bytes unchanged..header(header::ACCEPT_ENCODING, "identity")diff --git a/ai-gateway/src/resolution/mod.rs b/ai-gateway/src/resolution/mod.rsindex fe5adfc95a..6897ce4d4b 100644--- a/ai-gateway/src/resolution/mod.rs+++ b/ai-gateway/src/resolution/mod.rs@@ -10,6 +10,7 @@ use reqwest::{Client, Url,header::{AUTHORIZATION, CONTENT_TYPE, HeaderValue},};+use reqwest_middleware::ClientWithMiddleware;use std::{fmt,net::IpAddr,@@ -79,7 +80,7 @@ impl ControlPlaneConfig {/// Reuses the HTTP connection pool; credentials belong exclusively to each request.pub struct ControlPlaneClient {- client: Client,+ client: ClientWithMiddleware,config: ControlPlaneConfig,}@@ -99,7 +100,10 @@ impl ControlPlaneClient {.no_deflate().build().map_err(|_| ResolutionError::Configuration)?;- Ok(Self { client, config })+ Ok(Self {+ client: crate::observability::instrument_client(client, "resolver"),+ config,+ })}/// Resolve a gateway credential and API format into a validated execution contract.diff --git a/ai-gateway/src/telemetry/mod.rs b/ai-gateway/src/telemetry/mod.rsindex 7a9b2383a3..6e624c47ec 100644--- a/ai-gateway/src/telemetry/mod.rs+++ b/ai-gateway/src/telemetry/mod.rs@@ -10,7 +10,9 @@ use std::{},};+use opentelemetry::trace::{FutureExt, TraceContextExt};use tokio::{sync::Semaphore, task::JoinSet, time::Instant};+use tracing_opentelemetry::OpenTelemetrySpanExt;use uuid::Uuid;use crate::{@@ -115,27 +117,47 @@ impl Telemetry {while tasks.try_join_next().is_some() {}let uploader = self.0.uploader.clone();let stats = self.0.stats.clone();- tasks.spawn(async move {- let _capacity = capacity;- let _bytes = bytes;- let span = mapping::span(facts, &context.trace_id, &context.observation_id);- match uploader.export(&context, &[span]).await {- Ok(()) => {- stats.accepted.fetch_add(1, Ordering::Relaxed);- tracing::debug!("gateway telemetry accepted");- }- Err(error) => {- stats.failed.fetch_add(1, Ordering::Relaxed);- // Error categories contain no URLs, credentials, response bodies or content.- tracing::warn!(reason = error.reason(), "gateway telemetry upload failed");+ let parent = tracing::Span::current().context();+ tasks.spawn(+ async move {+ let _capacity = capacity;+ let _bytes = bytes;+ let span = mapping::span(facts, &context.trace_id, &context.observation_id);+ match uploader.export(&context, &[span]).await {+ Ok(()) => {+ stats.accepted.fetch_add(1, Ordering::Relaxed);+ crate::observability::delivery("accepted", "success");+ tracing::debug!("gateway telemetry accepted");+ }+ Err(error) => {+ let failed = stats.failed.fetch_add(1, Ordering::Relaxed) + 1;+ crate::observability::delivery("failed", error.reason());+ // Error categories contain no URLs, credentials, response bodies or content.+ if failed == 1 || failed.is_multiple_of(100) {+ let context = opentelemetry::Context::current();+ let span = context.span();+ let context = span.span_context();+ tracing::warn!(+ trace_id = %context.trace_id(),+ span_id = %context.span_id(),+ reason = error.reason(),+ failed,+ "gateway telemetry upload failed"+ );+ }+ }}}- });+ .with_context(parent),+ );}fn drop_record(&self, reason: &'static str) {- self.0.stats.dropped.fetch_add(1, Ordering::Relaxed);- tracing::warn!(reason, "gateway telemetry dropped");+ let dropped = self.0.stats.dropped.fetch_add(1, Ordering::Relaxed) + 1;+ crate::observability::delivery("dropped", reason);+ if dropped == 1 || dropped.is_multiple_of(100) {+ tracing::warn!(reason, dropped, "gateway telemetry dropped");+ }}/// Stop accepting work and finish uploads within the existing process drain deadline.@@ -158,6 +180,7 @@ impl Telemetry {Ok(Some(Ok(()))) => {}Ok(Some(Err(_))) => {self.0.stats.failed.fetch_add(1, Ordering::Relaxed);+ crate::observability::delivery("failed", "task");}Ok(None) => break,Err(_) => {@@ -165,6 +188,7 @@ impl Telemetry {while let Some(result) = tasks.join_next().await {if result.is_err() {self.0.stats.dropped.fetch_add(1, Ordering::Relaxed);+ crate::observability::delivery("dropped", "shutdown");}}break;@@ -200,8 +224,24 @@ impl Write for SizeCounter {}pub(crate) fn debug_record(facts: &InferenceFacts) {- // Full mode intentionally includes native request and completed output content.- tracing::debug!(capture = %format_args!("{:#}", serde_json::to_value(facts).expect("captured facts must serialize")), "gateway response captured");+ tracing::debug!(+ api_format = facts.api_format,+ outcome = match facts.outcome {+ crate::capture::RelayOutcome::Eof => "eof",+ crate::capture::RelayOutcome::Cancelled => "cancelled",+ crate::capture::RelayOutcome::Timeout => "timeout",+ crate::capture::RelayOutcome::TransportError => "transport_error",+ },+ http_status = facts.http_status,+ duration_ms = u64::try_from(facts.duration_ms).unwrap_or(u64::MAX),+ first_byte_ms = facts+ .first_byte_ms+ .map(|ms| u64::try_from(ms).unwrap_or(u64::MAX)),+ input_complete = facts.inference.input_complete,+ output_complete = facts.inference.output_complete,+ capture_complete = facts.inference.capture_complete,+ "gateway response captured"+ );}#[cfg(test)]diff --git a/ai-gateway/src/telemetry/otlp.rs b/ai-gateway/src/telemetry/otlp.rsindex c35a5d1528..66a2cced08 100644--- a/ai-gateway/src/telemetry/otlp.rs+++ b/ai-gateway/src/telemetry/otlp.rs@@ -5,6 +5,7 @@ use std::{use axum::http::{HeaderValue, header};use reqwest::{Client, Url};+use reqwest_middleware::ClientWithMiddleware;use serde_json::{Value, json};use super::DeliveryContext;@@ -16,7 +17,7 @@ const MAX_RESPONSE_BYTES: usize = 64 * 1024;const UPLOAD_TIMEOUT: Duration = Duration::from_secs(5);pub(super) struct Uploader {- client: Client,+ client: ClientWithMiddleware,endpoint: Url,service_key: String,}@@ -60,7 +61,7 @@ impl Uploader {.build().map_err(|_| ResolutionError::Configuration)?;Ok(Self {- client,+ client: crate::observability::instrument_client(client, "ingestion"),endpoint: config.endpoint(INGESTION_PATH),service_key: config.service_key().to_owned(),})diff --git a/ai-gateway/tests/observability.rs b/ai-gateway/tests/observability.rsnew file mode 100644index 0000000000..22d1acc8e9--- /dev/null+++ b/ai-gateway/tests/observability.rs@@ -0,0 +1,282 @@+#![cfg(unix)]++use axum::{Router, body::Bytes, extract::Request, routing::post};+use opentelemetry_proto::tonic::collector::{+ metrics::v1::ExportMetricsServiceRequest, trace::v1::ExportTraceServiceRequest,+};+use opentelemetry_proto::tonic::{+ common::v1::any_value::Value as AttributeValue,+ metrics::v1::{ResourceMetrics, metric::Data},+};+use prost::Message;+use serde_json::Value;+use std::{+ io::{BufRead, BufReader},+ process::{Child, Command, Stdio},+ sync::{Arc, Mutex, mpsc},+ thread::JoinHandle,+ time::{Duration, Instant},+};++struct Gateway {+ child: Child,+ url: String,+ logs: Arc<Mutex<Vec<Value>>>,+ reader: Option<JoinHandle<()>>,+}++impl Gateway {+ fn start(endpoint: &str, shutdown_seconds: &str, level: &str) -> Self {+ let mut child = Command::new(env!("CARGO_BIN_EXE_ai-gateway"))+ .env_clear()+ .env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS", "127.0.0.1:0")+ .env(+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS",+ shutdown_seconds,+ )+ .env("LANGFUSE_LOG_FORMAT", "json")+ .env("LANGFUSE_LOG_LEVEL", level)+ .env("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint)+ .env("OTEL_SERVICE_NAME", "gateway-observability-test")+ .env("DD_ENV", "staging-test")+ .env("BUILD_ID", "test-version")+ .stdout(Stdio::piped())+ .stderr(Stdio::inherit())+ .spawn()+ .unwrap();+ let stdout = child.stdout.take().unwrap();+ let logs = Arc::new(Mutex::new(Vec::new()));+ let output = logs.clone();+ let (sender, receiver) = mpsc::channel();+ let reader = std::thread::spawn(move || {+ for line in BufReader::new(stdout).lines().map_while(Result::ok) {+ let value: Value = serde_json::from_str(&line).unwrap();+ if value["message"] == "gateway listening" {+ let _ = sender.send(value["address"].as_str().unwrap().to_owned());+ }+ output.lock().unwrap().push(value);+ }+ });+ let address = receiver+ .recv_timeout(Duration::from_secs(10))+ .expect("gateway startup");+ Self {+ child,+ url: format!("http://{address}"),+ logs,+ reader: Some(reader),+ }+ }++ async fn request(&self, sampled: bool) {+ let response = reqwest::Client::new()+ .post(format!(+ "{}/openai/v1/responses?secret=query-canary",+ self.url+ ))+ .header("host", "host-canary.invalid")+ .header("x-forwarded-proto", "scheme-canary")+ .header(+ "traceparent",+ format!(+ "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-{:02x}",+ u8::from(sampled)+ ),+ )+ .header("authorization", "Bearer secret-auth-token")+ .body(r#"{"input":"secret-prompt-content"}"#)+ .send()+ .await+ .unwrap();+ assert_eq!(response.status(), 503);+ let _ = response.bytes().await.unwrap();+ }++ async fn stop(&mut self) {+ assert!(+ Command::new("/bin/kill")+ .args(["-TERM", &self.child.id().to_string()])+ .status()+ .unwrap()+ .success()+ );+ tokio::time::timeout(Duration::from_secs(6), async {+ loop {+ if let Some(status) = self.child.try_wait().unwrap() {+ assert!(status.success());+ break;+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .expect("gateway shutdown deadline");+ self.reader.take().unwrap().join().unwrap();+ }+}++impl Drop for Gateway {+ fn drop(&mut self) {+ let _ = self.child.kill();+ let _ = self.child.wait();+ }+}++#[tokio::test]+async fn exports_otlp_traces_metrics_and_correlated_content_free_logs() {+ let captured = Arc::new(Mutex::new(Vec::new()));+ let receiver = captured.clone();+ let app = Router::new().route(+ "/{*path}",+ post(move |request: Request| {+ let receiver = receiver.clone();+ async move {+ let path = request.uri().path().to_owned();+ let body = axum::body::to_bytes(request.into_body(), 1_000_000)+ .await+ .unwrap();+ receiver.lock().unwrap().push((path, body));+ Bytes::new()+ }+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let endpoint = format!("http://{}", listener.local_addr().unwrap());+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ let mut gateway = Gateway::start(&endpoint, "5", "debug");+ gateway.request(true).await;+ gateway.request(false).await;+ assert_eq!(+ reqwest::get(format!("{}/health", gateway.url))+ .await+ .unwrap()+ .status(),+ 200+ );+ gateway.stop().await;+ server.abort();+ let captured = captured.lock().unwrap();+ let traces: Vec<_> = captured+ .iter()+ .filter(|(path, _)| path == "/v1/traces")+ .flat_map(|(_, body)| {+ ExportTraceServiceRequest::decode(body.clone())+ .unwrap()+ .resource_spans+ })+ .collect();+ let metrics: Vec<_> = captured+ .iter()+ .filter(|(path, _)| path == "/v1/metrics")+ .flat_map(|(_, body)| {+ ExportMetricsServiceRequest::decode(body.clone())+ .unwrap()+ .resource_metrics+ })+ .collect();+ assert!(!traces.is_empty(), "traces must flush on shutdown");+ assert!(!metrics.is_empty(), "metrics must flush on shutdown");+ let spans: Vec<_> = traces+ .iter()+ .flat_map(|resource| &resource.scope_spans)+ .flat_map(|scope| &scope.spans)+ .collect();+ assert_eq!(spans.len(), 1);+ assert_eq!(spans[0].name, "POST /openai/v1/responses");+ assert_eq!(+ spans[0].parent_span_id,+ [0x00, 0xf0, 0x67, 0xaa, 0x0b, 0xa9, 0x02, 0xb7]+ );+ assert_eq!(spans[0].kind, 2);+ assert!(+ spans[0].events.is_empty(),+ "log events must not be exported as span events"+ );+ let logs = gateway.logs.lock().unwrap();+ let summaries: Vec<_> = logs+ .iter()+ .filter(|value| value["message"] == "gateway response started")+ .collect();+ assert_eq!(summaries.len(), 2);+ assert_eq!(summaries[0]["trace_id"], "4bf92f3577b34da6a3ce929d0e0e4736");+ assert_eq!(+ summaries[0]["span_id"].as_str().unwrap(),+ opentelemetry::trace::SpanId::from_bytes(spans[0].span_id.as_slice().try_into().unwrap())+ .to_string()+ );+ assert_eq!(summaries[0]["status"], 503);+ let serialized = serde_json::to_string(&*logs).unwrap();+ assert!(!serialized.contains("secret-auth-token"));+ assert!(!serialized.contains("secret-prompt-content"));+ let exported = format!("{traces:?}{metrics:?}");+ for secret in [+ "secret-auth-token",+ "secret-prompt-content",+ "query-canary",+ "host-canary",+ "scheme-canary",+ ] {+ assert!(!exported.contains(secret));+ assert!(!serialized.contains(secret));+ }+ assert_http_metrics(&metrics);+}++fn assert_http_metrics(metrics: &[ResourceMetrics]) {+ let http_duration = metrics+ .iter()+ .flat_map(|resource| &resource.scope_metrics)+ .flat_map(|scope| &scope.metrics)+ .find(|metric| metric.name == "http.server.request.duration")+ .expect("HTTP duration must export even for requests rejected before execution");+ let Some(Data::Histogram(histogram)) = &http_duration.data else {+ panic!("HTTP duration must be a histogram");+ };+ assert_eq!(histogram.data_points.len(), 1);+ let point = &histogram.data_points[0];+ assert_eq!(+ point.count, 2,+ "count includes unsampled requests and excludes health probes"+ );+ assert!(point.sum.unwrap() > 0.0);+ let attributes: std::collections::BTreeMap<_, _> = point+ .attributes+ .iter()+ .map(|attribute| {+ (+ attribute.key.as_str(),+ attribute.value.as_ref().unwrap().value.as_ref().unwrap(),+ )+ })+ .collect();+ assert_eq!(attributes.len(), 3);+ assert_eq!(+ attributes["http.request.method"],+ &AttributeValue::StringValue("POST".into())+ );+ assert_eq!(+ attributes["http.route"],+ &AttributeValue::StringValue("/openai/v1/responses".into())+ );+ assert_eq!(+ attributes["http.response.status_code"],+ &AttributeValue::IntValue(503)+ );+}++#[tokio::test]+async fn an_unresponsive_collector_does_not_extend_the_shutdown_deadline() {+ let app = Router::new().route(+ "/{*path}",+ post(|| async { std::future::pending::<()>().await }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let endpoint = format!("http://{}", listener.local_addr().unwrap());+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ let mut gateway = Gateway::start(&endpoint, "1", "info");+ gateway.request(true).await;+ let start = Instant::now();+ gateway.stop().await;+ assert!(start.elapsed() < Duration::from_secs(2));+ server.abort();+}