🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): bootstrap Rust service and development workflow (#17188)
Hassieb Pakzaddabb3bb8ba4e
diff --git a/.dockerignore b/.dockerignoreindex d9a86cca91..f65b5e3152 100644--- a/.dockerignore+++ b/.dockerignore@@ -13,3 +13,4 @@ README.md**/*.tsbuildinfo.git**/node_modules+ai-gateway/targetdiff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.ymlindex deb537ffe2..a636d826c2 100644--- a/.github/workflows/pipeline.yml+++ b/.github/workflows/pipeline.yml@@ -150,6 +150,32 @@ jobs:- 'packages/shared/src/server/llm/**'- 'worker/src/__tests__/llmConnections.test.ts'+ tests-ai-gateway:+ timeout-minutes: 20+ runs-on: blacksmith-4vcpu-ubuntu-2404+ needs:+ - pre-job+ if: ${{ !cancelled() && (github.event_name != 'push' || needs.pre-job.outputs.should_skip != 'true') }}+ defaults:+ run:+ working-directory: ai-gateway+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ - name: Install pinned Rust toolchain+ run: rustup show+ - name: Check Rust formatting+ run: cargo fmt --all -- --check+ - name: Lint Rust targets+ run: cargo clippy --locked --all-targets -- -D warnings+ - name: Run Rust tests+ run: cargo test --locked+ - name: Build runtime image+ run: docker build --target runtime -t langfuse-ai-gateway:test .+ - name: Smoke test runtime image+ run: bash scripts/smoke-image.sh langfuse-ai-gateway:test+lint:timeout-minutes: 12runs-on: blacksmith-16vcpu-ubuntu-2404@@ -214,9 +240,9 @@ jobs:cp .env.dev.example .env- name: lint web# Three packages with four ESLint workers leave headroom on 16 vCPUs.- run: pnpm turbo run lint --concurrency=3 -- --concurrency=4+ run: pnpm turbo run lint --filter=!ai-gateway --concurrency=3 -- --concurrency=4- name: typecheck- run: pnpm run typecheck+ run: pnpm run typecheck --filter=!ai-gatewayknip:runs-on: blacksmith-4vcpu-ubuntu-2404@@ -1124,7 +1150,7 @@ jobs:run: |(set +e; pnpm --filter=web exec playwright install --with-deps --only-shell chromium > /tmp/playwright-install.log 2>&1; echo $? > /tmp/playwright-install.exit) &- name: Build- run: pnpm run build+ run: pnpm run build --filter=!ai-gatewayenv:NODE_OPTIONS: --max_old_space_size=8192# TypeScript is checked explicitly in the lint job; skip duplicate@@ -1234,7 +1260,7 @@ jobs:run: |(set +e; docker compose -f docker-compose.dev.yml up -d --wait --wait-timeout 180 > /tmp/compose-up.log 2>&1; echo $? > /tmp/compose-up.exit) &- name: Build- run: pnpm run build+ run: pnpm run build --filter=!ai-gatewayenv:NODE_OPTIONS: --max_old_space_size=8192# TypeScript is checked explicitly in the lint job; skip duplicate@@ -1273,6 +1299,7 @@ jobs:lint,knip,prettier-check,+ tests-ai-gateway,tests-eslint-plugin,tests-in-app-agent-sandbox-runtime,tests-storybook,diff --git a/.github/workflows/warm-caches.yml b/.github/workflows/warm-caches.ymlindex 18936a008c..09c253d4d9 100644--- a/.github/workflows/warm-caches.yml+++ b/.github/workflows/warm-caches.yml@@ -98,7 +98,7 @@ jobs:if [ "${{ matrix.env_profile }}" = "tests-web" ]; thenpnpm turbo run build:testelse- pnpm run build+ pnpm run build --filter=!ai-gatewayfienv:NODE_OPTIONS: --max_old_space_size=8192diff --git a/.husky/pre-commit b/.husky/pre-commitindex 49a544abc5..592340d1ec 100755--- a/.husky/pre-commit+++ b/.husky/pre-commit@@ -7,26 +7,26 @@ current_branch=$(git rev-parse --abbrev-ref HEAD)protected_branch="main"run_lint() {- if [ -n "${LANGFUSE_PRE_COMMIT_SKIP_LINT-}" ]; then- echo "Skipping lint because LANGFUSE_PRE_COMMIT_SKIP_LINT is set."- return 0- fi+ if [ -n "${LANGFUSE_PRE_COMMIT_SKIP_LINT-}" ]; then+ echo "Skipping lint because LANGFUSE_PRE_COMMIT_SKIP_LINT is set."+ return 0+ fi- pnpm run lint+ pnpm run lint --filter=!ai-gateway}# Check if the current branch is the protected branchif [ "$current_branch" = "$protected_branch" ]; then- echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"- read -r answer < /dev/tty- if [ "$answer" != "${answer#[Yy]}" ]; then- # Commit approved, run checks- pnpm run format:check- run_lint- else- echo "Commit to $protected_branch branch has been canceled."- exit 1 # Commit will be blocked- fi+ echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"+ read -r answer </dev/tty+ if [ "$answer" != "${answer#[Yy]}" ]; then+ # Commit approved, run checks+ pnpm run format:check+ run_lint+ else+ echo "Commit to $protected_branch branch has been canceled."+ exit 1 # Commit will be blocked+ fifi# If not the protected branch, run checksdiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 49f183ac87..90c53d9836 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -130,6 +130,7 @@ We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https:/Requirements- Node.js 24 as specified in the [.nvmrc](.nvmrc)+- [Rust via rustup](https://rust-lang.org/tools/install/) and a native compiler/linker for the AI gateway, which starts with `pnpm dev`. See [gateway setup](ai-gateway/README.md#run-locally).- pnpm 12.3.1 as specified in `package.json`- Docker to run the database locally- Clickhouse clientdiff --git a/ai-gateway/.dockerignore b/ai-gateway/.dockerignorenew file mode 100644index 0000000000..27387a05fe--- /dev/null+++ b/ai-gateway/.dockerignore@@ -0,0 +1,6 @@+**+!Cargo.toml+!Cargo.lock+!rust-toolchain.toml+!src+!src/**diff --git a/ai-gateway/.gitignore b/ai-gateway/.gitignorenew file mode 100644index 0000000000..b83d22266a--- /dev/null+++ b/ai-gateway/.gitignore@@ -0,0 +1 @@+/target/diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.locknew file mode 100644index 0000000000..77c3e5df65--- /dev/null+++ b/ai-gateway/Cargo.lock@@ -0,0 +1,568 @@+# This file is automatically @generated by Cargo.+# It is not intended for manual editing.+version = 4++[[package]]+name = "ai-gateway"+version = "0.1.0"+dependencies = [+ "axum",+ "serde",+ "tokio",+ "tower",+ "tracing",+ "tracing-subscriber",+]++[[package]]+name = "atomic-waker"+version = "1.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"++[[package]]+name = "axum"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"+dependencies = [+ "axum-core",+ "bytes",+ "futures-util",+ "http",+ "http-body",+ "http-body-util",+ "hyper",+ "hyper-util",+ "itoa",+ "matchit",+ "memchr",+ "mime",+ "percent-encoding",+ "pin-project-lite",+ "serde_core",+ "serde_json",+ "serde_path_to_error",+ "sync_wrapper",+ "tokio",+ "tower",+ "tower-layer",+ "tower-service",+]++[[package]]+name = "axum-core"+version = "0.5.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"+dependencies = [+ "bytes",+ "futures-core",+ "http",+ "http-body",+ "http-body-util",+ "mime",+ "pin-project-lite",+ "sync_wrapper",+ "tower-layer",+ "tower-service",+]++[[package]]+name = "bytes"+version = "1.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"++[[package]]+name = "cfg-if"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"++[[package]]+name = "errno"+version = "0.3.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"+dependencies = [+ "libc",+ "windows-sys",+]++[[package]]+name = "futures-channel"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"+dependencies = [+ "futures-core",+]++[[package]]+name = "futures-core"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"++[[package]]+name = "futures-task"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"++[[package]]+name = "futures-util"+version = "0.3.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"+dependencies = [+ "futures-core",+ "futures-task",+ "pin-project-lite",+ "slab",+]++[[package]]+name = "http"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"+dependencies = [+ "bytes",+ "itoa",+]++[[package]]+name = "http-body"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"+dependencies = [+ "bytes",+ "http",+]++[[package]]+name = "http-body-util"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c"+dependencies = [+ "bytes",+ "futures-core",+ "http",+ "http-body",+ "pin-project-lite",+]++[[package]]+name = "httparse"+version = "1.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"++[[package]]+name = "httpdate"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"++[[package]]+name = "hyper"+version = "1.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43"+dependencies = [+ "atomic-waker",+ "bytes",+ "futures-channel",+ "futures-core",+ "http",+ "http-body",+ "httparse",+ "httpdate",+ "itoa",+ "pin-project-lite",+ "smallvec",+ "tokio",+]++[[package]]+name = "hyper-util"+version = "0.1.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"+dependencies = [+ "bytes",+ "http",+ "http-body",+ "hyper",+ "pin-project-lite",+ "tokio",+ "tower-service",+]++[[package]]+name = "itoa"+version = "1.0.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"++[[package]]+name = "lazy_static"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"++[[package]]+name = "libc"+version = "0.2.189"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"++[[package]]+name = "matchit"+version = "0.8.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"++[[package]]+name = "memchr"+version = "2.8.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"++[[package]]+name = "mime"+version = "0.3.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"++[[package]]+name = "mio"+version = "1.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8"+dependencies = [+ "libc",+ "wasi",+ "windows-sys",+]++[[package]]+name = "once_cell"+version = "1.21.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"++[[package]]+name = "percent-encoding"+version = "2.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"++[[package]]+name = "pin-project-lite"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"++[[package]]+name = "proc-macro2"+version = "1.0.107"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"+dependencies = [+ "unicode-ident",+]++[[package]]+name = "quote"+version = "1.0.47"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"+dependencies = [+ "proc-macro2",+]++[[package]]+name = "serde"+version = "1.0.229"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"+dependencies = [+ "serde_core",+ "serde_derive",+]++[[package]]+name = "serde_core"+version = "1.0.229"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"+dependencies = [+ "serde_derive",+]++[[package]]+name = "serde_derive"+version = "1.0.229"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.5",+]++[[package]]+name = "serde_json"+version = "1.0.151"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"+dependencies = [+ "itoa",+ "memchr",+ "serde",+ "serde_core",+ "zmij",+]++[[package]]+name = "serde_path_to_error"+version = "0.1.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"+dependencies = [+ "itoa",+ "serde",+ "serde_core",+]++[[package]]+name = "sharded-slab"+version = "0.1.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"+dependencies = [+ "lazy_static",+]++[[package]]+name = "signal-hook-registry"+version = "1.4.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"+dependencies = [+ "errno",+ "libc",+]++[[package]]+name = "slab"+version = "0.4.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"++[[package]]+name = "smallvec"+version = "1.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"++[[package]]+name = "socket2"+version = "0.6.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"+dependencies = [+ "libc",+ "windows-sys",+]++[[package]]+name = "syn"+version = "2.0.119"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "syn"+version = "3.0.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "sync_wrapper"+version = "1.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"++[[package]]+name = "thread_local"+version = "1.1.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070"+dependencies = [+ "cfg-if",+]++[[package]]+name = "tokio"+version = "1.53.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"+dependencies = [+ "bytes",+ "libc",+ "mio",+ "pin-project-lite",+ "signal-hook-registry",+ "socket2",+ "tokio-macros",+ "windows-sys",+]++[[package]]+name = "tokio-macros"+version = "2.7.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 3.0.5",+]++[[package]]+name = "tower"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"+dependencies = [+ "futures-core",+ "futures-util",+ "pin-project-lite",+ "sync_wrapper",+ "tokio",+ "tower-layer",+ "tower-service",+]++[[package]]+name = "tower-layer"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"++[[package]]+name = "tower-service"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"++[[package]]+name = "tracing"+version = "0.1.44"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"+dependencies = [+ "pin-project-lite",+ "tracing-attributes",+ "tracing-core",+]++[[package]]+name = "tracing-attributes"+version = "0.1.31"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.119",+]++[[package]]+name = "tracing-core"+version = "0.1.36"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"+dependencies = [+ "once_cell",+ "valuable",+]++[[package]]+name = "tracing-serde"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1"+dependencies = [+ "serde",+ "tracing-core",+]++[[package]]+name = "tracing-subscriber"+version = "0.3.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"+dependencies = [+ "serde",+ "serde_json",+ "sharded-slab",+ "thread_local",+ "tracing-core",+ "tracing-serde",+]++[[package]]+name = "unicode-ident"+version = "1.0.24"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"++[[package]]+name = "valuable"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"++[[package]]+name = "wasi"+version = "0.11.1+wasi-snapshot-preview1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"++[[package]]+name = "windows-link"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"++[[package]]+name = "windows-sys"+version = "0.61.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"+dependencies = [+ "windows-link",+]++[[package]]+name = "zmij"+version = "1.0.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.tomlnew file mode 100644index 0000000000..2aa7f0ec8f--- /dev/null+++ b/ai-gateway/Cargo.toml@@ -0,0 +1,21 @@+[package]+name = "ai-gateway"+version = "0.1.0"+edition = "2024"+publish = false+license = "MIT"++[dependencies]+axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] }+serde = { version = "1", features = ["derive"] }+tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }+tracing = "0.1.44"+tracing-subscriber = { version = "0.3.23", default-features = false, features = ["fmt", "json"] }++[dev-dependencies]+tokio = { version = "1.53.1", features = ["io-util"] }+tower = { version = "0.5", features = ["util"] }++[profile.release]+lto = "thin"+strip = truediff --git a/ai-gateway/Dockerfile b/ai-gateway/Dockerfilenew file mode 100644index 0000000000..49d1447925--- /dev/null+++ b/ai-gateway/Dockerfile@@ -0,0 +1,17 @@+FROM rust:1.98.0-bookworm AS builder+RUN useradd --create-home --uid 10001 gateway && mkdir /app && chown gateway:gateway /app+WORKDIR /app+ENV CARGO_HOME=/home/gateway/.cargo+USER gateway+COPY --chown=gateway:gateway Cargo.toml Cargo.lock rust-toolchain.toml ./+COPY --chown=gateway:gateway src ./src+RUN cargo build --release --locked++FROM debian:bookworm-slim AS runtime+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \+ && rm -rf /var/lib/apt/lists/*+COPY --from=builder /app/target/release/ai-gateway /usr/local/bin/ai-gateway+USER 10001:10001+EXPOSE 8080+STOPSIGNAL SIGTERM+ENTRYPOINT ["/usr/local/bin/ai-gateway"]diff --git a/ai-gateway/README.md b/ai-gateway/README.mdnew file mode 100644index 0000000000..b88f953785--- /dev/null+++ b/ai-gateway/README.md@@ -0,0 +1,151 @@+# AI Gateway service foundation++A standalone Rust HTTP process with health probes, bounded shutdown, structured+logs and a container. Inference, Web resolution and customer telemetry are not+implemented yet; inference paths return 404. No Web, database or provider+credentials are needed to build, start or test this package.++## Run locally++Alongside the repository's Node.js/pnpm setup, install [Rust via rustup](https://rust-lang.org/tools/install/)+and a native compiler/linker: Xcode Command Line Tools on macOS (`xcode-select --install`),+GCC/Clang on Linux (Ubuntu: `sudo apt install build-essential`), or MSVC C++ Build Tools on Windows.+Ensure `~/.cargo/bin` is on your PATH (reopen the terminal after installing rustup).+`rust-toolchain.toml` selects Rust 1.98.0 with Clippy and rustfmt automatically;+the first run needs network access to download the toolchain and Cargo dependencies.+No separate Rust watcher is needed.++After the usual repository `pnpm install`, run from the repository root:++```sh+pnpm dev+```++This starts the gateway alongside the other development services. Turbo Watch+recompiles and restarts the gateway after Rust source changes; Web and worker+keep their built-in watchers. The gateway loads the root `.env` and listens on+port 8080 by default. Add overrides from the table below to your root `.env`;+exported shell variables take precedence.+No gateway-specific credentials are needed for this foundation slice.++To work on only the gateway, use `pnpm dev --filter=ai-gateway`.+Both commands watch Rust source changes; do not run both on the same port.++To start once without watching, run `pnpm dev` from `ai-gateway/`. Cargo also+works independently of Node/pnpm; it reads exported environment variables only,+without automatically loading `.env`. Run it from this directory so rustup+selects the pinned toolchain:++```sh+cd ai-gateway+cargo run --locked+```++In another terminal:++```sh+curl --fail http://localhost:8080/health+# {"status":"ok"}+curl --fail http://localhost:8080/ready+# {"status":"ready"}+```++The binary reads configuration from the process environment. The pnpm development+script loads the root `.env` before starting it; production and direct Cargo runs+do not load dotenv files. All settings have defaults:++| Variable | Default | Validation |+| ---------------------------------------------- | -------------- | -------------------------------------------------- |+| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |+| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |+| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |+| `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 |++The gateway shares `LANGFUSE_LOG_LEVEL` with Web and worker. Values are lowercase;+`fatal` maps to Rust's `error` level and therefore includes ordinary error logs.++Set `LANGFUSE_LOG_FORMAT=text` in the root `.env` for compact, readable logs+(the default). Use `LANGFUSE_LOG_FORMAT=json` for structured production logs.+Both formats use the same log level and preserve event fields. For example:++```text+2026-09-10T13:20:00Z INFO gateway listening address=0.0.0.0:8080+```++Invalid values fail startup without echoing their contents. Logs contain service+lifecycle events, not request bodies or headers. For direct+host-only development, set `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS=127.0.0.1:8080`.++## Process lifecycle++- `/health` returns 200 while HTTP is serving. It never probes dependencies.+- `/ready` returns 200 after initialization. On shutdown, readiness changes to+ 503 with `{"status":"draining"}` and the listener stops accepting connections.+ An external probe may see a closed connection instead of the brief 503 state.+- SIGTERM or Ctrl-C starts graceful shutdown. In-flight requests may finish+ within the configured timeout. Deadline expiry exits nonzero and terminates+ remaining runtime tasks; ordinary shutdown exits zero.+- The deadline starts on the shutdown signal, not at process startup. Future+ stream/export slices must integrate their own work into this lifecycle.++## Container++From the repository root:++```sh+docker build --target runtime -t langfuse-ai-gateway:dev ./ai-gateway+docker run --rm --name langfuse-ai-gateway-dev \+ -e LANGFUSE_LOG_FORMAT=json -p 127.0.0.1:8080:8080 langfuse-ai-gateway:dev+# In another terminal:+docker stop --time 15 langfuse-ai-gateway-dev+bash ai-gateway/scripts/smoke-image.sh langfuse-ai-gateway:dev+```++The runtime image runs as UID/GID 10001, includes CA certificates, and executes+the binary directly so it receives signals. Set the container/orchestrator stop+grace period longer than the gateway shutdown timeout. The smoke test uses an+isolated container and an ephemeral host port, checks read-only/non-root startup,+probes, missing inference routes and a clean SIGTERM exit, then removes it.++## Tests and module boundaries++From the repository root:++```sh+pnpm --filter ai-gateway build+pnpm --filter ai-gateway typecheck+pnpm --filter ai-gateway lint+pnpm --filter ai-gateway test+pnpm --filter ai-gateway format+```++These scripts delegate to Cargo. Root `pnpm build`, `pnpm typecheck`, `pnpm lint`+and `pnpm test` include the gateway; its Turbo tasks have no Prisma/JavaScript+dependencies and disable Turbo caching, leaving incremental builds to Cargo.+The JavaScript CI jobs and commit-hook lint exclude it; the dedicated Rust CI job+runs its checks without Node. Root Prettier formatting remains JavaScript/CSS-only;+use the gateway `format` script for Rust. Docker builds also use Cargo directly.++Equivalent Cargo checks, without pnpm:++```sh+cd ai-gateway+cargo fmt --all -- --check+cargo clippy --locked --all-targets -- -D warnings+cargo test --locked+```++- `config.rs`: pure configuration parsing plus the environment adapter.+- `server.rs`: router, probe state and bounded graceful shutdown. A listener and+ shutdown future are injected, so tests do not depend on fixed ports or signals.+- `main.rs`: configuration, logging, signal registration and process exit.+- `tests/support`: local ephemeral-port HTTP servers with injected Axum routers.+ `fake_dependency_records_requests_and_returns_scripted_response` demonstrates+ configurable status/body/headers and request recording. Specialized provider+ streaming scripts belong with the slices that consume them.++Tests exercise real local HTTP requests, config rejection/redaction, readiness+transition, in-flight draining and a stuck-handler deadline. CI runs these Cargo+checks and the container smoke test independently of JavaScript builds; its result+is a dependency of the required `all-ci-passed` gate. It currently runs for all+eligible PRs, so source, lockfile, toolchain and Docker changes are all covered.diff --git a/ai-gateway/package.json b/ai-gateway/package.jsonnew file mode 100644index 0000000000..8e48d235b5--- /dev/null+++ b/ai-gateway/package.json@@ -0,0 +1,15 @@+{+ "name": "ai-gateway",+ "private": true,+ "scripts": {+ "dev": "dotenv -e ../.env -- cargo run --locked",+ "build": "cargo build --release --locked",+ "typecheck": "cargo check --locked --all-targets",+ "lint": "cargo fmt --all -- --check && cargo clippy --locked --all-targets -- -D warnings",+ "test": "cargo test --locked",+ "format": "cargo fmt --all"+ },+ "devDependencies": {+ "dotenv-cli": "^7.4.4"+ }+}diff --git a/ai-gateway/rust-toolchain.toml b/ai-gateway/rust-toolchain.tomlnew file mode 100644index 0000000000..d1ad785747--- /dev/null+++ b/ai-gateway/rust-toolchain.toml@@ -0,0 +1,4 @@+[toolchain]+channel = "1.98.0"+profile = "minimal"+components = ["clippy", "rustfmt"]diff --git a/ai-gateway/scripts/smoke-image.sh b/ai-gateway/scripts/smoke-image.shnew file mode 100644index 0000000000..57c10d31ba--- /dev/null+++ b/ai-gateway/scripts/smoke-image.sh@@ -0,0 +1,28 @@+#!/usr/bin/env bash+set -euo pipefail++image="${1:?Usage: bash scripts/smoke-image.sh IMAGE}"+container="$(docker run -d --read-only --cap-drop ALL --security-opt no-new-privileges \+ -e LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS=2 -p 127.0.0.1::8080 "$image")"+cleanup() {+ docker logs "$container" || true+ docker rm -f "$container" >/dev/null || true+}+trap cleanup EXIT+port="$(docker port "$container" 8080/tcp)"+for route in health ready; do+ body="$(curl --fail --silent --show-error --retry 20 --retry-connrefused \+ --retry-delay 1 --max-time 2 "http://$port/$route")"+ case "$route:$body" in+ 'health:{"status":"ok"}'|'ready:{"status":"ready"}') ;;+ *) echo "Unexpected probe response: $route" >&2; exit 1 ;;+ esac+done+[[ "$(docker exec "$container" id -u)" != 0 ]]+[[ "$(curl --silent --output /dev/null --write-out '%{http_code}' \+ --max-time 2 -X POST "http://$port/openai/v1/responses")" == 404 ]]+docker stop --time 5 "$container" >/dev/null+[[ "$(docker inspect --format '{{.State.ExitCode}}' "$container")" == 0 ]]+docker logs "$container" 2>&1 | grep -q 'gateway draining'+docker logs "$container" 2>&1 | grep -q 'gateway stopped'+printf '%s\n' 'Image smoke: non-root, probes, unimplemented inference, SIGTERM passed'diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rsnew file mode 100644index 0000000000..1571d23cd0--- /dev/null+++ b/ai-gateway/src/config.rs@@ -0,0 +1,176 @@+use std::{env, error::Error, fmt, net::SocketAddr, time::Duration};+use tracing::level_filters::LevelFilter;++pub struct Config {+ pub listen_address: SocketAddr,+ pub shutdown_timeout: Duration,+ pub log_level: LevelFilter,+ pub log_format: LogFormat,+}++#[derive(Debug, PartialEq, Eq)]+pub enum LogFormat {+ Text,+ Json,+}++#[derive(Debug, PartialEq, Eq)]+pub struct ConfigError(&'static str);++impl fmt::Display for ConfigError {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.write_str(self.0)+ }+}++impl Error for ConfigError {}++impl Config {+ pub fn from_env() -> Result<Self, ConfigError> {+ Self::from_values(+ read_env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS")?.as_deref(),+ read_env("LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS")?.as_deref(),+ read_env("LANGFUSE_LOG_LEVEL")?.as_deref(),+ read_env("LANGFUSE_LOG_FORMAT")?.as_deref(),+ )+ }++ pub fn from_values(+ listen_address: Option<&str>,+ shutdown_timeout: Option<&str>,+ log_level: Option<&str>,+ log_format: Option<&str>,+ ) -> Result<Self, ConfigError> {+ let listen_address = listen_address+ .unwrap_or("0.0.0.0:8080")+ .parse()+ .map_err(|_| {+ ConfigError("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS must be an IP address and port")+ })?;+ let seconds: u64 = shutdown_timeout.unwrap_or("10").parse().map_err(|_| {+ ConfigError(+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",+ )+ })?;+ if !(1..=300).contains(&seconds) {+ return Err(ConfigError(+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",+ ));+ }+ let log_level = match log_level.unwrap_or("info") {+ "trace" => LevelFilter::TRACE,+ "debug" => LevelFilter::DEBUG,+ "info" => LevelFilter::INFO,+ "warn" => LevelFilter::WARN,+ // Tracing has no separate fatal severity.+ "error" | "fatal" => LevelFilter::ERROR,+ _ => {+ return Err(ConfigError(+ "LANGFUSE_LOG_LEVEL must be trace, debug, info, warn, error or fatal",+ ));+ }+ };+ let log_format = match log_format.unwrap_or("text") {+ "text" => LogFormat::Text,+ "json" => LogFormat::Json,+ _ => return Err(ConfigError("LANGFUSE_LOG_FORMAT must be text or json")),+ };+ Ok(Self {+ listen_address,+ shutdown_timeout: Duration::from_secs(seconds),+ log_level,+ log_format,+ })+ }+}++fn read_env(name: &'static str) -> Result<Option<String>, ConfigError> {+ match env::var(name) {+ Ok(value) => Ok(Some(value)),+ Err(env::VarError::NotPresent) => Ok(None),+ Err(env::VarError::NotUnicode(_)) => Err(ConfigError(+ "gateway configuration contains a non-Unicode environment value",+ )),+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn configuration_parses_defaults_and_overrides() {+ let default = Config::from_values(None, None, None, None).unwrap();+ assert_eq!(default.listen_address, "0.0.0.0:8080".parse().unwrap());+ assert_eq!(default.shutdown_timeout, Duration::from_secs(10));+ assert_eq!(default.log_level, LevelFilter::INFO);+ assert_eq!(default.log_format, LogFormat::Text);+ let custom =+ Config::from_values(Some("[::1]:9000"), Some("30"), Some("debug"), Some("json"))+ .unwrap();+ assert_eq!(custom.listen_address, "[::1]:9000".parse().unwrap());+ assert_eq!(custom.shutdown_timeout, Duration::from_secs(30));+ assert_eq!(custom.log_level, LevelFilter::DEBUG);+ assert_eq!(custom.log_format, LogFormat::Json);+ }++ #[test]+ fn validates_shared_log_format() {+ for (value, expected) in [("text", LogFormat::Text), ("json", LogFormat::Json)] {+ assert_eq!(+ Config::from_values(None, None, None, Some(value))+ .unwrap()+ .log_format,+ expected+ );+ }+ for value in ["", "TEXT", "pretty", "0", "secret-that-must-not-appear"] {+ let error = Config::from_values(None, None, None, Some(value))+ .err()+ .unwrap();+ assert_eq!(+ error.to_string(),+ "LANGFUSE_LOG_FORMAT must be text or json"+ );+ }+ }++ #[test]+ fn accepts_shared_langfuse_log_levels() {+ for (value, expected) in [+ ("trace", LevelFilter::TRACE),+ ("debug", LevelFilter::DEBUG),+ ("info", LevelFilter::INFO),+ ("warn", LevelFilter::WARN),+ ("error", LevelFilter::ERROR),+ ("fatal", LevelFilter::ERROR),+ ] {+ let config = Config::from_values(None, None, Some(value), None).unwrap();+ assert_eq!(config.log_level, expected, "{value}");+ }+ }++ #[test]+ fn rejects_invalid_configuration_without_echoing_input() {+ let sensitive_input = "secret-that-must-not-appear";+ for (address, timeout, level) in [+ (Some(sensitive_input), None, None),+ (None, Some(sensitive_input), None),+ (None, None, Some(sensitive_input)),+ (None, None, Some("off")),+ (None, None, Some("INFO")),+ (None, None, Some("3")),+ (None, None, Some("")),+ (None, Some("0"), None),+ (None, Some("301"), None),+ (None, Some("-1"), None),+ (Some("localhost:8080"), None, None),+ ] {+ let error = Config::from_values(address, timeout, level, None)+ .err()+ .unwrap();+ assert!(!error.to_string().contains(sensitive_input));+ assert!(!format!("{error:?}").contains(sensitive_input));+ }+ }+}diff --git a/ai-gateway/src/lib.rs b/ai-gateway/src/lib.rsnew file mode 100644index 0000000000..6d51ea9018--- /dev/null+++ b/ai-gateway/src/lib.rs@@ -0,0 +1,2 @@+pub mod config;+pub mod server;diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rsnew file mode 100644index 0000000000..44e0d6819e--- /dev/null+++ b/ai-gateway/src/main.rs@@ -0,0 +1,65 @@+use std::{error::Error, process::ExitCode};++use ai_gateway::{+ config::{Config, LogFormat},+ server::{self, AppState},+};+use tokio::net::TcpListener;++#[tokio::main]+async fn main() -> ExitCode {+ match run().await {+ Ok(()) => ExitCode::SUCCESS,+ Err(error) => {+ eprintln!("gateway startup or shutdown failed: {error}");+ ExitCode::FAILURE+ }+ }+}++async fn run() -> Result<(), Box<dyn Error>> {+ let config = Config::from_env()?;+ let logging = tracing_subscriber::fmt()+ .with_max_level(config.log_level)+ .with_target(false);+ match config.log_format {+ LogFormat::Text => logging.compact().init(),+ LogFormat::Json => logging.json().init(),+ }+ let shutdown = shutdown_signal()?;+ let listener = TcpListener::bind(config.listen_address).await?;+ tracing::info!(address = %listener.local_addr()?, "gateway listening");+ let state = AppState::default();+ server::serve(+ listener,+ server::router(state.clone()),+ state,+ shutdown,+ config.shutdown_timeout,+ )+ .await?;+ tracing::info!("gateway stopped");+ Ok(())+}++#[cfg(unix)]+fn shutdown_signal() -> std::io::Result<impl Future<Output = ()> + Send> {+ use tokio::signal::unix::{SignalKind, signal};+ let mut terminate = signal(SignalKind::terminate())?;+ let mut interrupt = signal(SignalKind::interrupt())?;+ Ok(async move {+ tokio::select! {+ _ = terminate.recv() => {},+ _ = interrupt.recv() => {},+ }+ })+}++#[cfg(not(unix))]+fn shutdown_signal() -> std::io::Result<impl Future<Output = ()> + Send> {+ Ok(async {+ if let Err(error) = tokio::signal::ctrl_c().await {+ tracing::error!(%error, "failed to receive shutdown signal");+ }+ })+}diff --git a/ai-gateway/src/server.rs b/ai-gateway/src/server.rsnew file mode 100644index 0000000000..48d4d15f84--- /dev/null+++ b/ai-gateway/src/server.rs@@ -0,0 +1,79 @@+use std::{+ future::{Future, IntoFuture},+ io,+ sync::{+ Arc,+ atomic::{AtomicBool, Ordering},+ },+ time::Duration,+};++use axum::{Json, Router, extract::State, http::StatusCode, routing::get};+use serde::Serialize;+use tokio::{net::TcpListener, sync::oneshot};++#[derive(Clone, Default)]+pub struct AppState {+ ready: Arc<AtomicBool>,+}++impl AppState {+ pub fn is_ready(&self) -> bool {+ self.ready.load(Ordering::Acquire)+ }+}++#[derive(Serialize)]+struct Probe {+ status: &'static str,+}++pub fn router(state: AppState) -> Router {+ Router::new()+ .route("/health", get(|| async { Json(Probe { status: "ok" }) }))+ .route("/ready", get(readiness))+ .with_state(state)+}++async fn readiness(State(state): State<AppState>) -> (StatusCode, Json<Probe>) {+ if state.is_ready() {+ (StatusCode::OK, Json(Probe { status: "ready" }))+ } else {+ (+ StatusCode::SERVICE_UNAVAILABLE,+ Json(Probe { status: "draining" }),+ )+ }+}++/// Serve an initialized router until shutdown, then bound connection draining.+/// A timeout must terminate the owning process/runtime to stop remaining tasks.+pub async fn serve(+ listener: TcpListener,+ app: Router,+ state: AppState,+ shutdown: impl Future<Output = ()> + Send + 'static,+ shutdown_timeout: Duration,+) -> io::Result<()> {+ let (draining_tx, draining_rx) = oneshot::channel();+ let shutdown_state = state.clone();+ let server = axum::serve(listener, app)+ .with_graceful_shutdown(async move {+ shutdown.await;+ shutdown_state.ready.store(false, Ordering::Release);+ tracing::info!("gateway draining");+ let _ = draining_tx.send(());+ })+ .into_future();+ tokio::pin!(server);+ state.ready.store(true, Ordering::Release);+ let result = tokio::select! {+ result = &mut server => result,+ _ = async {+ let _ = draining_rx.await;+ tokio::time::sleep(shutdown_timeout).await;+ } => Err(io::Error::new(io::ErrorKind::TimedOut, "gateway shutdown deadline exceeded")),+ };+ state.ready.store(false, Ordering::Release);+ result+}diff --git a/ai-gateway/tests/lifecycle.rs b/ai-gateway/tests/lifecycle.rsnew file mode 100644index 0000000000..613d92eac0--- /dev/null+++ b/ai-gateway/tests/lifecycle.rs@@ -0,0 +1,153 @@+mod support;++use ai_gateway::server::{self, AppState};+use axum::{+ Router,+ body::{Body, to_bytes},+ http::{Request, StatusCode},+ routing::get,+};+use std::{+ sync::{Arc, Mutex},+ time::Duration,+};+use support::TestServer;+use tokio::sync::Notify;+use tower::ServiceExt;++#[tokio::test]+async fn probes_and_unimplemented_routes_over_http() {+ let state = AppState::default();+ let mut server =+ TestServer::start(server::router(state.clone()), state, Duration::from_secs(1)).await;+ let health = support::get(server.address, "/health").await;+ assert!(health.starts_with("HTTP/1.1 200"));+ assert!(health.ends_with(r#"{"status":"ok"}"#));+ let ready = support::get(server.address, "/ready").await;+ assert!(ready.starts_with("HTTP/1.1 200"));+ assert!(ready.ends_with(r#"{"status":"ready"}"#));+ let unimplemented = server::router(server.state.clone())+ .oneshot(+ Request::post("/openai/v1/responses")+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();+ assert_eq!(unimplemented.status(), StatusCode::NOT_FOUND);+ server.begin_shutdown();+ server.finish().await.unwrap();+}++#[tokio::test]+async fn shutdown_marks_unready_and_waits_for_in_flight_request() {+ let entered = Arc::new(Notify::new());+ let release = Arc::new(Notify::new());+ let state = AppState::default();+ let app = server::router(state.clone()).route(+ "/slow",+ get({+ let entered = entered.clone();+ let release = release.clone();+ move || {+ let entered = entered.clone();+ let release = release.clone();+ async move {+ entered.notify_one();+ release.notified().await;+ "finished"+ }+ }+ }),+ );+ let mut server = TestServer::start(app, state.clone(), Duration::from_secs(2)).await;+ let request = tokio::spawn(support::get(server.address, "/slow"));+ tokio::time::timeout(Duration::from_secs(2), entered.notified())+ .await+ .unwrap();+ server.begin_shutdown();+ tokio::time::timeout(Duration::from_secs(2), async {+ while state.is_ready() {+ tokio::task::yield_now().await;+ }+ })+ .await+ .unwrap();+ let response = server::router(state.clone())+ .oneshot(Request::get("/ready").body(Body::empty()).unwrap())+ .await+ .unwrap();+ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);+ assert_eq!(+ to_bytes(response.into_body(), 1024).await.unwrap(),+ r#"{"status":"draining"}"#+ );+ assert!(!request.is_finished());+ release.notify_one();+ assert!(request.await.unwrap().ends_with("finished"));+ server.finish().await.unwrap();+ assert!(!state.is_ready());+}++#[tokio::test]+async fn shutdown_deadline_bounds_a_stuck_handler() {+ let entered = Arc::new(Notify::new());+ let app = Router::new().route(+ "/stuck",+ get({+ let entered = entered.clone();+ move || {+ let entered = entered.clone();+ async move {+ entered.notify_one();+ std::future::pending::<()>().await;+ "unreachable"+ }+ }+ }),+ );+ let mut server = TestServer::start(app, AppState::default(), Duration::from_millis(50)).await;+ let request = tokio::spawn(support::get(server.address, "/stuck"));+ tokio::time::timeout(Duration::from_secs(2), entered.notified())+ .await+ .unwrap();+ server.begin_shutdown();+ assert_eq!(+ server.finish().await.unwrap_err().kind(),+ std::io::ErrorKind::TimedOut+ );+ request.abort();+}++#[tokio::test]+async fn fake_dependency_records_requests_and_returns_scripted_response() {+ let requests = Arc::new(Mutex::new(Vec::new()));+ let app = Router::new().fallback({+ let requests = requests.clone();+ move |request: Request<Body>| {+ let requests = requests.clone();+ async move {+ requests+ .lock()+ .unwrap()+ .push((request.method().clone(), request.uri().to_string()));+ (+ StatusCode::TOO_MANY_REQUESTS,+ [("retry-after", "1")],+ "try later",+ )+ }+ }+ });+ let mut fake = TestServer::start(app, AppState::default(), Duration::from_secs(1)).await;+ let response = support::get(fake.address, "/example").await;+ assert!(response.starts_with("HTTP/1.1 429"));+ assert!(response.contains("retry-after: 1\r\n"));+ assert!(response.ends_with("try later"));+ assert_eq!(+ requests.lock().unwrap().as_slice(),+ &[(axum::http::Method::GET, "/example".to_string())]+ );+ fake.begin_shutdown();+ fake.finish().await.unwrap();+}diff --git a/ai-gateway/tests/startup.rs b/ai-gateway/tests/startup.rsnew file mode 100644index 0000000000..999e70e276--- /dev/null+++ b/ai-gateway/tests/startup.rs@@ -0,0 +1,100 @@+use std::{net::TcpListener, process::Command};++#[test]+fn invalid_configuration_exits_without_disclosing_the_value() {+ // Occupy the listener so an ignored setting cannot leave a child server running.+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ for name in [+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS",+ "LANGFUSE_LOG_LEVEL",+ "LANGFUSE_LOG_FORMAT",+ ] {+ let output = Command::new(env!("CARGO_BIN_EXE_ai-gateway"))+ .env_clear()+ .env(+ "LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS",+ listener.local_addr().unwrap().to_string(),+ )+ .env(name, "example-sensitive-value")+ .output()+ .unwrap();+ assert!(!output.status.success());+ let error = String::from_utf8(output.stderr).unwrap();+ assert!(error.contains(name), "{error}");+ assert!(!error.contains("example-sensitive-value"));+ assert!(output.stdout.is_empty());+ }+}++#[test]+fn occupied_listener_exits_unsuccessfully() {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ let output = Command::new(env!("CARGO_BIN_EXE_ai-gateway"))+ .env_clear()+ .env(+ "LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS",+ listener.local_addr().unwrap().to_string(),+ )+ .output()+ .unwrap();+ assert!(!output.status.success());+ assert!(+ String::from_utf8(output.stderr)+ .unwrap()+ .contains("gateway startup or shutdown failed")+ );+}++#[test]+fn lifecycle_logs_follow_the_shared_format() {+ use std::{+ io::{BufRead, BufReader},+ process::Stdio,+ sync::mpsc,+ time::Duration,+ };++ for format in [None, Some("text"), Some("json")] {+ let mut command = Command::new(env!("CARGO_BIN_EXE_ai-gateway"));+ command+ .env_clear()+ .env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS", "127.0.0.1:0")+ .stdout(Stdio::piped())+ .stderr(Stdio::null());+ if let Some(format) = format {+ command.env("LANGFUSE_LOG_FORMAT", format);+ }+ let mut child = command.spawn().unwrap();+ let stdout = child.stdout.take().unwrap();+ let (sender, receiver) = mpsc::channel();+ let reader = std::thread::spawn(move || {+ let mut line = String::new();+ let result = BufReader::new(stdout).read_line(&mut line).map(|_| line);+ let _ = sender.send(result);+ });+ let result = receiver.recv_timeout(Duration::from_secs(10));+ let _ = child.kill();+ child.wait().unwrap();+ reader.join().unwrap();+ let line = result.expect("gateway must emit a startup log").unwrap();+ assert!(line.contains("gateway listening"), "{format:?}: {line}");+ assert!(+ !line.contains('\u{1b}'),+ "logs must not contain ANSI escapes: {line}"+ );+ if format == Some("json") {+ assert!(+ line.starts_with('{') && line.trim_end().ends_with('}'),+ "{line}"+ );+ assert!(line.contains("\"level\":\"INFO\""), "{line}");+ assert!(line.contains("\"address\":\"127.0.0.1:"), "{line}");+ } else {+ assert!(!line.starts_with('{'), "{line}");+ assert!(+ line.contains("INFO") && line.contains("address=127.0.0.1:"),+ "{line}"+ );+ }+ }+}diff --git a/ai-gateway/tests/support/mod.rs b/ai-gateway/tests/support/mod.rsnew file mode 100644index 0000000000..00ebf2d74f--- /dev/null+++ b/ai-gateway/tests/support/mod.rs@@ -0,0 +1,76 @@+use std::{io, net::SocketAddr, time::Duration};++use ai_gateway::server::{self, AppState};+use axum::Router;+use tokio::{+ io::{AsyncReadExt, AsyncWriteExt},+ net::{TcpListener, TcpStream},+ sync::oneshot,+ task::JoinHandle,+};++pub struct TestServer {+ pub address: SocketAddr,+ pub state: AppState,+ shutdown: Option<oneshot::Sender<()>>,+ task: Option<JoinHandle<io::Result<()>>>,+}++impl TestServer {+ pub async fn start(app: Router, state: AppState, budget: Duration) -> Self {+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let (shutdown, receiver) = oneshot::channel();+ let task = tokio::spawn(server::serve(+ listener,+ app,+ state.clone(),+ async {+ let _ = receiver.await;+ },+ budget,+ ));+ Self {+ address,+ state,+ shutdown: Some(shutdown),+ task: Some(task),+ }+ }++ pub fn begin_shutdown(&mut self) {+ self.shutdown.take().unwrap().send(()).unwrap();+ }++ pub async fn finish(mut self) -> io::Result<()> {+ tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap())+ .await+ .expect("server did not stop")+ .unwrap()+ }+}++impl Drop for TestServer {+ fn drop(&mut self) {+ // Dropping the sender also requests shutdown, including after a failed assertion.+ self.shutdown.take();+ }+}++pub async fn get(address: SocketAddr, path: &str) -> String {+ tokio::time::timeout(Duration::from_secs(5), async {+ let mut socket = TcpStream::connect(address).await.unwrap();+ socket+ .write_all(+ format!("GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")+ .as_bytes(),+ )+ .await+ .unwrap();+ let mut response = String::new();+ socket.read_to_string(&mut response).await.unwrap();+ response+ })+ .await+ .expect("HTTP request timed out")+}diff --git a/ai-gateway/turbo.json b/ai-gateway/turbo.jsonnew file mode 100644index 0000000000..d7757ad0e4--- /dev/null+++ b/ai-gateway/turbo.json@@ -0,0 +1,15 @@+{+ "extends": ["//"],+ "tasks": {+ "dev": {+ "dependsOn": [],+ "cache": false,+ "persistent": true,+ "interruptible": true+ },+ "build": { "dependsOn": [], "cache": false, "outputs": [] },+ "typecheck": { "dependsOn": [], "cache": false, "outputs": [] },+ "lint": { "dependsOn": [], "cache": false, "outputs": [] },+ "test": { "dependsOn": [], "cache": false, "outputs": [] }+ }+}diff --git a/package.json b/package.jsonindex 1efba881e9..74c9905d77 100644--- a/package.json+++ b/package.json@@ -33,7 +33,7 @@"typecheck": "turbo run typecheck","tc": "turbo run typecheck","start": "turbo run start",- "dev": "turbo run dev",+ "dev": "turbo watch dev","dev:worker": "turbo run dev --filter=worker","dev:web": "turbo run dev --filter=web","dev:web-webpack": "turbo run dev --filter=web -- --webpack",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex acc9839bbd..dcfa532cb6 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -115,6 +115,12 @@ importers:specifier: 2.10.5version: 2.10.5+ ai-gateway:+ devDependencies:+ dotenv-cli:+ specifier: ^7.4.4+ version: 7.4.4+ee:dependencies:'@langfuse/shared':diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex dc9fb04727..5c2a17ed18 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -2,6 +2,7 @@ packages:# include packages in subfolders (e.g. apps/ and packages/)- "web"- "worker"+ - "ai-gateway"- "packages/**"- "ee"catalog: