Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): bootstrap Rust service and development workflow (#17188)

Hassieb Pakzaddabb3bb8ba4e
diff --git a/.dockerignore b/.dockerignore
index d9a86cca91..f65b5e3152 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -13,3 +13,4 @@ README.md
**/*.tsbuildinfo
.git
**/node_modules
+ai-gateway/target
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index deb537ffe2..a636d826c2 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -150,6 +150,32 @@ jobs:
- 'packages/shared/src/server/llm/**'
- 'worker/src/__tests__/llmConnections.test.ts'
+ tests-ai-gateway:
+ timeout-minutes: 20
+ runs-on: blacksmith-4vcpu-ubuntu-2404
+ needs:
+ - pre-job
+ if: ${{ !cancelled() && (github.event_name != 'push' || needs.pre-job.outputs.should_skip != 'true') }}
+ defaults:
+ run:
+ working-directory: ai-gateway
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - name: Install pinned Rust toolchain
+ run: rustup show
+ - name: Check Rust formatting
+ run: cargo fmt --all -- --check
+ - name: Lint Rust targets
+ run: cargo clippy --locked --all-targets -- -D warnings
+ - name: Run Rust tests
+ run: cargo test --locked
+ - name: Build runtime image
+ run: docker build --target runtime -t langfuse-ai-gateway:test .
+ - name: Smoke test runtime image
+ run: bash scripts/smoke-image.sh langfuse-ai-gateway:test
+
lint:
timeout-minutes: 12
runs-on: blacksmith-16vcpu-ubuntu-2404
@@ -214,9 +240,9 @@ jobs:
cp .env.dev.example .env
- name: lint web
# Three packages with four ESLint workers leave headroom on 16 vCPUs.
- run: pnpm turbo run lint --concurrency=3 -- --concurrency=4
+ run: pnpm turbo run lint --filter=!ai-gateway --concurrency=3 -- --concurrency=4
- name: typecheck
- run: pnpm run typecheck
+ run: pnpm run typecheck --filter=!ai-gateway
knip:
runs-on: blacksmith-4vcpu-ubuntu-2404
@@ -1124,7 +1150,7 @@ jobs:
run: |
(set +e; pnpm --filter=web exec playwright install --with-deps --only-shell chromium > /tmp/playwright-install.log 2>&1; echo $? > /tmp/playwright-install.exit) &
- name: Build
- run: pnpm run build
+ run: pnpm run build --filter=!ai-gateway
env:
NODE_OPTIONS: --max_old_space_size=8192
# TypeScript is checked explicitly in the lint job; skip duplicate
@@ -1234,7 +1260,7 @@ jobs:
run: |
(set +e; docker compose -f docker-compose.dev.yml up -d --wait --wait-timeout 180 > /tmp/compose-up.log 2>&1; echo $? > /tmp/compose-up.exit) &
- name: Build
- run: pnpm run build
+ run: pnpm run build --filter=!ai-gateway
env:
NODE_OPTIONS: --max_old_space_size=8192
# TypeScript is checked explicitly in the lint job; skip duplicate
@@ -1273,6 +1299,7 @@ jobs:
lint,
knip,
prettier-check,
+ tests-ai-gateway,
tests-eslint-plugin,
tests-in-app-agent-sandbox-runtime,
tests-storybook,
diff --git a/.github/workflows/warm-caches.yml b/.github/workflows/warm-caches.yml
index 18936a008c..09c253d4d9 100644
--- a/.github/workflows/warm-caches.yml
+++ b/.github/workflows/warm-caches.yml
@@ -98,7 +98,7 @@ jobs:
if [ "${{ matrix.env_profile }}" = "tests-web" ]; then
pnpm turbo run build:test
else
- pnpm run build
+ pnpm run build --filter=!ai-gateway
fi
env:
NODE_OPTIONS: --max_old_space_size=8192
diff --git a/.husky/pre-commit b/.husky/pre-commit
index 49a544abc5..592340d1ec 100755
--- a/.husky/pre-commit
+++ b/.husky/pre-commit
@@ -7,26 +7,26 @@ current_branch=$(git rev-parse --abbrev-ref HEAD)
protected_branch="main"
run_lint() {
- if [ -n "${LANGFUSE_PRE_COMMIT_SKIP_LINT-}" ]; then
- echo "Skipping lint because LANGFUSE_PRE_COMMIT_SKIP_LINT is set."
- return 0
- fi
+ if [ -n "${LANGFUSE_PRE_COMMIT_SKIP_LINT-}" ]; then
+ echo "Skipping lint because LANGFUSE_PRE_COMMIT_SKIP_LINT is set."
+ return 0
+ fi
- pnpm run lint
+ pnpm run lint --filter=!ai-gateway
}
# Check if the current branch is the protected branch
if [ "$current_branch" = "$protected_branch" ]; then
- echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"
- read -r answer < /dev/tty
- if [ "$answer" != "${answer#[Yy]}" ]; then
- # Commit approved, run checks
- pnpm run format:check
- run_lint
- else
- echo "Commit to $protected_branch branch has been canceled."
- exit 1 # Commit will be blocked
- fi
+ echo "🚨 You are about to commit to the $protected_branch branch. Are you sure? (y/n)"
+ read -r answer </dev/tty
+ if [ "$answer" != "${answer#[Yy]}" ]; then
+ # Commit approved, run checks
+ pnpm run format:check
+ run_lint
+ else
+ echo "Commit to $protected_branch branch has been canceled."
+ exit 1 # Commit will be blocked
+ fi
fi
# If not the protected branch, run checks
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 49f183ac87..90c53d9836 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -130,6 +130,7 @@ We built a monorepo using [pnpm](https://pnpm.io/motivation) and [turbo](https:/
Requirements
- Node.js 24 as specified in the [.nvmrc](.nvmrc)
+- [Rust via rustup](https://rust-lang.org/tools/install/) and a native compiler/linker for the AI gateway, which starts with `pnpm dev`. See [gateway setup](ai-gateway/README.md#run-locally).
- pnpm 12.3.1 as specified in `package.json`
- Docker to run the database locally
- Clickhouse client
diff --git a/ai-gateway/.dockerignore b/ai-gateway/.dockerignore
new file mode 100644
index 0000000000..27387a05fe
--- /dev/null
+++ b/ai-gateway/.dockerignore
@@ -0,0 +1,6 @@
+**
+!Cargo.toml
+!Cargo.lock
+!rust-toolchain.toml
+!src
+!src/**
diff --git a/ai-gateway/.gitignore b/ai-gateway/.gitignore
new file mode 100644
index 0000000000..b83d22266a
--- /dev/null
+++ b/ai-gateway/.gitignore
@@ -0,0 +1 @@
+/target/
diff --git a/ai-gateway/Cargo.lock b/ai-gateway/Cargo.lock
new file mode 100644
index 0000000000..77c3e5df65
--- /dev/null
+++ b/ai-gateway/Cargo.lock
@@ -0,0 +1,568 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "ai-gateway"
+version = "0.1.0"
+dependencies = [
+ "axum",
+ "serde",
+ "tokio",
+ "tower",
+ "tracing",
+ "tracing-subscriber",
+]
+
+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
+[[package]]
+name = "axum"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
+dependencies = [
+ "axum-core",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-util",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde_core",
+ "serde_json",
+ "serde_path_to_error",
+ "sync_wrapper",
+ "tokio",
+ "tower",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "axum-core"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http",
+ "http-body",
+ "http-body-util",
+ "mime",
+ "pin-project-lite",
+ "sync_wrapper",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "bytes"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4"
+dependencies = [
+ "futures-core",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
+
+[[package]]
+name = "futures-task"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
+
+[[package]]
+name = "futures-util"
+version = "0.3.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "pin-project-lite",
+ "slab",
+]
+
+[[package]]
+name = "http"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
+dependencies = [
+ "bytes",
+ "itoa",
+]
+
+[[package]]
+name = "http-body"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
+dependencies = [
+ "bytes",
+ "http",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http",
+ "http-body",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
+[[package]]
+name = "httpdate"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+
+[[package]]
+name = "hyper"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "http",
+ "http-body",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
+dependencies = [
+ "bytes",
+ "http",
+ "http-body",
+ "hyper",
+ "pin-project-lite",
+ "tokio",
+ "tower-service",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
+[[package]]
+name = "libc"
+version = "0.2.189"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+
+[[package]]
+name = "matchit"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "mime"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+
+[[package]]
+name = "mio"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8"
+dependencies = [
+ "libc",
+ "wasi",
+ "windows-sys",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.5",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.151"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "serde_path_to_error"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
+dependencies = [
+ "itoa",
+ "serde",
+ "serde_core",
+]
+
+[[package]]
+name = "sharded-slab"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
+dependencies = [
+ "lazy_static",
+]
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
+dependencies = [
+ "errno",
+ "libc",
+]
+
+[[package]]
+name = "slab"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+
+[[package]]
+name = "smallvec"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"
+
+[[package]]
+name = "socket2"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.119"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "sync_wrapper"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
+
+[[package]]
+name = "thread_local"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "tokio"
+version = "1.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
+dependencies = [
+ "bytes",
+ "libc",
+ "mio",
+ "pin-project-lite",
+ "signal-hook-registry",
+ "socket2",
+ "tokio-macros",
+ "windows-sys",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 3.0.5",
+]
+
+[[package]]
+name = "tower"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
+
+[[package]]
+name = "tower-service"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
+
+[[package]]
+name = "tracing"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
+dependencies = [
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
+dependencies = [
+ "once_cell",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-serde"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
+dependencies = [
+ "serde",
+ "serde_json",
+ "sharded-slab",
+ "thread_local",
+ "tracing-core",
+ "tracing-serde",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "valuable"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
+
+[[package]]
+name = "wasi"
+version = "0.11.1+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
diff --git a/ai-gateway/Cargo.toml b/ai-gateway/Cargo.toml
new file mode 100644
index 0000000000..2aa7f0ec8f
--- /dev/null
+++ b/ai-gateway/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "ai-gateway"
+version = "0.1.0"
+edition = "2024"
+publish = false
+license = "MIT"
+
+[dependencies]
+axum = { version = "0.8.9", default-features = false, features = ["http1", "json", "tokio"] }
+serde = { version = "1", features = ["derive"] }
+tokio = { version = "1.53.1", features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] }
+tracing = "0.1.44"
+tracing-subscriber = { version = "0.3.23", default-features = false, features = ["fmt", "json"] }
+
+[dev-dependencies]
+tokio = { version = "1.53.1", features = ["io-util"] }
+tower = { version = "0.5", features = ["util"] }
+
+[profile.release]
+lto = "thin"
+strip = true
diff --git a/ai-gateway/Dockerfile b/ai-gateway/Dockerfile
new file mode 100644
index 0000000000..49d1447925
--- /dev/null
+++ b/ai-gateway/Dockerfile
@@ -0,0 +1,17 @@
+FROM rust:1.98.0-bookworm AS builder
+RUN useradd --create-home --uid 10001 gateway && mkdir /app && chown gateway:gateway /app
+WORKDIR /app
+ENV CARGO_HOME=/home/gateway/.cargo
+USER gateway
+COPY --chown=gateway:gateway Cargo.toml Cargo.lock rust-toolchain.toml ./
+COPY --chown=gateway:gateway src ./src
+RUN cargo build --release --locked
+
+FROM debian:bookworm-slim AS runtime
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /app/target/release/ai-gateway /usr/local/bin/ai-gateway
+USER 10001:10001
+EXPOSE 8080
+STOPSIGNAL SIGTERM
+ENTRYPOINT ["/usr/local/bin/ai-gateway"]
diff --git a/ai-gateway/README.md b/ai-gateway/README.md
new file mode 100644
index 0000000000..b88f953785
--- /dev/null
+++ b/ai-gateway/README.md
@@ -0,0 +1,151 @@
+# AI Gateway service foundation
+
+A standalone Rust HTTP process with health probes, bounded shutdown, structured
+logs and a container. Inference, Web resolution and customer telemetry are not
+implemented yet; inference paths return 404. No Web, database or provider
+credentials are needed to build, start or test this package.
+
+## Run locally
+
+Alongside the repository's Node.js/pnpm setup, install [Rust via rustup](https://rust-lang.org/tools/install/)
+and a native compiler/linker: Xcode Command Line Tools on macOS (`xcode-select --install`),
+GCC/Clang on Linux (Ubuntu: `sudo apt install build-essential`), or MSVC C++ Build Tools on Windows.
+Ensure `~/.cargo/bin` is on your PATH (reopen the terminal after installing rustup).
+`rust-toolchain.toml` selects Rust 1.98.0 with Clippy and rustfmt automatically;
+the first run needs network access to download the toolchain and Cargo dependencies.
+No separate Rust watcher is needed.
+
+After the usual repository `pnpm install`, run from the repository root:
+
+```sh
+pnpm dev
+```
+
+This starts the gateway alongside the other development services. Turbo Watch
+recompiles and restarts the gateway after Rust source changes; Web and worker
+keep their built-in watchers. The gateway loads the root `.env` and listens on
+port 8080 by default. Add overrides from the table below to your root `.env`;
+exported shell variables take precedence.
+No gateway-specific credentials are needed for this foundation slice.
+
+To work on only the gateway, use `pnpm dev --filter=ai-gateway`.
+Both commands watch Rust source changes; do not run both on the same port.
+
+To start once without watching, run `pnpm dev` from `ai-gateway/`. Cargo also
+works independently of Node/pnpm; it reads exported environment variables only,
+without automatically loading `.env`. Run it from this directory so rustup
+selects the pinned toolchain:
+
+```sh
+cd ai-gateway
+cargo run --locked
+```
+
+In another terminal:
+
+```sh
+curl --fail http://localhost:8080/health
+# {"status":"ok"}
+curl --fail http://localhost:8080/ready
+# {"status":"ready"}
+```
+
+The binary reads configuration from the process environment. The pnpm development
+script loads the root `.env` before starting it; production and direct Cargo runs
+do not load dotenv files. All settings have defaults:
+
+| Variable | Default | Validation |
+| ---------------------------------------------- | -------------- | -------------------------------------------------- |
+| `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS` | `0.0.0.0:8080` | IP address and port; IPv6 uses `[::]:8080` |
+| `LANGFUSE_LOG_FORMAT` | `text` | `text`, `json` |
+| `LANGFUSE_LOG_LEVEL` | `info` | `trace`, `debug`, `info`, `warn`, `error`, `fatal` |
+| `LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS` | `10` | Integer from 1 to 300 |
+
+The gateway shares `LANGFUSE_LOG_LEVEL` with Web and worker. Values are lowercase;
+`fatal` maps to Rust's `error` level and therefore includes ordinary error logs.
+
+Set `LANGFUSE_LOG_FORMAT=text` in the root `.env` for compact, readable logs
+(the default). Use `LANGFUSE_LOG_FORMAT=json` for structured production logs.
+Both formats use the same log level and preserve event fields. For example:
+
+```text
+2026-09-10T13:20:00Z INFO gateway listening address=0.0.0.0:8080
+```
+
+Invalid values fail startup without echoing their contents. Logs contain service
+lifecycle events, not request bodies or headers. For direct
+host-only development, set `LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS=127.0.0.1:8080`.
+
+## Process lifecycle
+
+- `/health` returns 200 while HTTP is serving. It never probes dependencies.
+- `/ready` returns 200 after initialization. On shutdown, readiness changes to
+ 503 with `{"status":"draining"}` and the listener stops accepting connections.
+ An external probe may see a closed connection instead of the brief 503 state.
+- SIGTERM or Ctrl-C starts graceful shutdown. In-flight requests may finish
+ within the configured timeout. Deadline expiry exits nonzero and terminates
+ remaining runtime tasks; ordinary shutdown exits zero.
+- The deadline starts on the shutdown signal, not at process startup. Future
+ stream/export slices must integrate their own work into this lifecycle.
+
+## Container
+
+From the repository root:
+
+```sh
+docker build --target runtime -t langfuse-ai-gateway:dev ./ai-gateway
+docker run --rm --name langfuse-ai-gateway-dev \
+ -e LANGFUSE_LOG_FORMAT=json -p 127.0.0.1:8080:8080 langfuse-ai-gateway:dev
+# In another terminal:
+docker stop --time 15 langfuse-ai-gateway-dev
+bash ai-gateway/scripts/smoke-image.sh langfuse-ai-gateway:dev
+```
+
+The runtime image runs as UID/GID 10001, includes CA certificates, and executes
+the binary directly so it receives signals. Set the container/orchestrator stop
+grace period longer than the gateway shutdown timeout. The smoke test uses an
+isolated container and an ephemeral host port, checks read-only/non-root startup,
+probes, missing inference routes and a clean SIGTERM exit, then removes it.
+
+## Tests and module boundaries
+
+From the repository root:
+
+```sh
+pnpm --filter ai-gateway build
+pnpm --filter ai-gateway typecheck
+pnpm --filter ai-gateway lint
+pnpm --filter ai-gateway test
+pnpm --filter ai-gateway format
+```
+
+These scripts delegate to Cargo. Root `pnpm build`, `pnpm typecheck`, `pnpm lint`
+and `pnpm test` include the gateway; its Turbo tasks have no Prisma/JavaScript
+dependencies and disable Turbo caching, leaving incremental builds to Cargo.
+The JavaScript CI jobs and commit-hook lint exclude it; the dedicated Rust CI job
+runs its checks without Node. Root Prettier formatting remains JavaScript/CSS-only;
+use the gateway `format` script for Rust. Docker builds also use Cargo directly.
+
+Equivalent Cargo checks, without pnpm:
+
+```sh
+cd ai-gateway
+cargo fmt --all -- --check
+cargo clippy --locked --all-targets -- -D warnings
+cargo test --locked
+```
+
+- `config.rs`: pure configuration parsing plus the environment adapter.
+- `server.rs`: router, probe state and bounded graceful shutdown. A listener and
+ shutdown future are injected, so tests do not depend on fixed ports or signals.
+- `main.rs`: configuration, logging, signal registration and process exit.
+- `tests/support`: local ephemeral-port HTTP servers with injected Axum routers.
+ `fake_dependency_records_requests_and_returns_scripted_response` demonstrates
+ configurable status/body/headers and request recording. Specialized provider
+ streaming scripts belong with the slices that consume them.
+
+Tests exercise real local HTTP requests, config rejection/redaction, readiness
+transition, in-flight draining and a stuck-handler deadline. CI runs these Cargo
+checks and the container smoke test independently of JavaScript builds; its result
+is a dependency of the required `all-ci-passed` gate. It currently runs for all
+eligible PRs, so source, lockfile, toolchain and Docker changes are all covered.
diff --git a/ai-gateway/package.json b/ai-gateway/package.json
new file mode 100644
index 0000000000..8e48d235b5
--- /dev/null
+++ b/ai-gateway/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "ai-gateway",
+ "private": true,
+ "scripts": {
+ "dev": "dotenv -e ../.env -- cargo run --locked",
+ "build": "cargo build --release --locked",
+ "typecheck": "cargo check --locked --all-targets",
+ "lint": "cargo fmt --all -- --check && cargo clippy --locked --all-targets -- -D warnings",
+ "test": "cargo test --locked",
+ "format": "cargo fmt --all"
+ },
+ "devDependencies": {
+ "dotenv-cli": "^7.4.4"
+ }
+}
diff --git a/ai-gateway/rust-toolchain.toml b/ai-gateway/rust-toolchain.toml
new file mode 100644
index 0000000000..d1ad785747
--- /dev/null
+++ b/ai-gateway/rust-toolchain.toml
@@ -0,0 +1,4 @@
+[toolchain]
+channel = "1.98.0"
+profile = "minimal"
+components = ["clippy", "rustfmt"]
diff --git a/ai-gateway/scripts/smoke-image.sh b/ai-gateway/scripts/smoke-image.sh
new file mode 100644
index 0000000000..57c10d31ba
--- /dev/null
+++ b/ai-gateway/scripts/smoke-image.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+image="${1:?Usage: bash scripts/smoke-image.sh IMAGE}"
+container="$(docker run -d --read-only --cap-drop ALL --security-opt no-new-privileges \
+ -e LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS=2 -p 127.0.0.1::8080 "$image")"
+cleanup() {
+ docker logs "$container" || true
+ docker rm -f "$container" >/dev/null || true
+}
+trap cleanup EXIT
+port="$(docker port "$container" 8080/tcp)"
+for route in health ready; do
+ body="$(curl --fail --silent --show-error --retry 20 --retry-connrefused \
+ --retry-delay 1 --max-time 2 "http://$port/$route")"
+ case "$route:$body" in
+ 'health:{"status":"ok"}'|'ready:{"status":"ready"}') ;;
+ *) echo "Unexpected probe response: $route" >&2; exit 1 ;;
+ esac
+done
+[[ "$(docker exec "$container" id -u)" != 0 ]]
+[[ "$(curl --silent --output /dev/null --write-out '%{http_code}' \
+ --max-time 2 -X POST "http://$port/openai/v1/responses")" == 404 ]]
+docker stop --time 5 "$container" >/dev/null
+[[ "$(docker inspect --format '{{.State.ExitCode}}' "$container")" == 0 ]]
+docker logs "$container" 2>&1 | grep -q 'gateway draining'
+docker logs "$container" 2>&1 | grep -q 'gateway stopped'
+printf '%s\n' 'Image smoke: non-root, probes, unimplemented inference, SIGTERM passed'
diff --git a/ai-gateway/src/config.rs b/ai-gateway/src/config.rs
new file mode 100644
index 0000000000..1571d23cd0
--- /dev/null
+++ b/ai-gateway/src/config.rs
@@ -0,0 +1,176 @@
+use std::{env, error::Error, fmt, net::SocketAddr, time::Duration};
+use tracing::level_filters::LevelFilter;
+
+pub struct Config {
+ pub listen_address: SocketAddr,
+ pub shutdown_timeout: Duration,
+ pub log_level: LevelFilter,
+ pub log_format: LogFormat,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum LogFormat {
+ Text,
+ Json,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct ConfigError(&'static str);
+
+impl fmt::Display for ConfigError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str(self.0)
+ }
+}
+
+impl Error for ConfigError {}
+
+impl Config {
+ pub fn from_env() -> Result<Self, ConfigError> {
+ Self::from_values(
+ read_env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS")?.as_deref(),
+ read_env("LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS")?.as_deref(),
+ read_env("LANGFUSE_LOG_LEVEL")?.as_deref(),
+ read_env("LANGFUSE_LOG_FORMAT")?.as_deref(),
+ )
+ }
+
+ pub fn from_values(
+ listen_address: Option<&str>,
+ shutdown_timeout: Option<&str>,
+ log_level: Option<&str>,
+ log_format: Option<&str>,
+ ) -> Result<Self, ConfigError> {
+ let listen_address = listen_address
+ .unwrap_or("0.0.0.0:8080")
+ .parse()
+ .map_err(|_| {
+ ConfigError("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS must be an IP address and port")
+ })?;
+ let seconds: u64 = shutdown_timeout.unwrap_or("10").parse().map_err(|_| {
+ ConfigError(
+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",
+ )
+ })?;
+ if !(1..=300).contains(&seconds) {
+ return Err(ConfigError(
+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS must be an integer from 1 to 300",
+ ));
+ }
+ let log_level = match log_level.unwrap_or("info") {
+ "trace" => LevelFilter::TRACE,
+ "debug" => LevelFilter::DEBUG,
+ "info" => LevelFilter::INFO,
+ "warn" => LevelFilter::WARN,
+ // Tracing has no separate fatal severity.
+ "error" | "fatal" => LevelFilter::ERROR,
+ _ => {
+ return Err(ConfigError(
+ "LANGFUSE_LOG_LEVEL must be trace, debug, info, warn, error or fatal",
+ ));
+ }
+ };
+ let log_format = match log_format.unwrap_or("text") {
+ "text" => LogFormat::Text,
+ "json" => LogFormat::Json,
+ _ => return Err(ConfigError("LANGFUSE_LOG_FORMAT must be text or json")),
+ };
+ Ok(Self {
+ listen_address,
+ shutdown_timeout: Duration::from_secs(seconds),
+ log_level,
+ log_format,
+ })
+ }
+}
+
+fn read_env(name: &'static str) -> Result<Option<String>, ConfigError> {
+ match env::var(name) {
+ Ok(value) => Ok(Some(value)),
+ Err(env::VarError::NotPresent) => Ok(None),
+ Err(env::VarError::NotUnicode(_)) => Err(ConfigError(
+ "gateway configuration contains a non-Unicode environment value",
+ )),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn configuration_parses_defaults_and_overrides() {
+ let default = Config::from_values(None, None, None, None).unwrap();
+ assert_eq!(default.listen_address, "0.0.0.0:8080".parse().unwrap());
+ assert_eq!(default.shutdown_timeout, Duration::from_secs(10));
+ assert_eq!(default.log_level, LevelFilter::INFO);
+ assert_eq!(default.log_format, LogFormat::Text);
+ let custom =
+ Config::from_values(Some("[::1]:9000"), Some("30"), Some("debug"), Some("json"))
+ .unwrap();
+ assert_eq!(custom.listen_address, "[::1]:9000".parse().unwrap());
+ assert_eq!(custom.shutdown_timeout, Duration::from_secs(30));
+ assert_eq!(custom.log_level, LevelFilter::DEBUG);
+ assert_eq!(custom.log_format, LogFormat::Json);
+ }
+
+ #[test]
+ fn validates_shared_log_format() {
+ for (value, expected) in [("text", LogFormat::Text), ("json", LogFormat::Json)] {
+ assert_eq!(
+ Config::from_values(None, None, None, Some(value))
+ .unwrap()
+ .log_format,
+ expected
+ );
+ }
+ for value in ["", "TEXT", "pretty", "0", "secret-that-must-not-appear"] {
+ let error = Config::from_values(None, None, None, Some(value))
+ .err()
+ .unwrap();
+ assert_eq!(
+ error.to_string(),
+ "LANGFUSE_LOG_FORMAT must be text or json"
+ );
+ }
+ }
+
+ #[test]
+ fn accepts_shared_langfuse_log_levels() {
+ for (value, expected) in [
+ ("trace", LevelFilter::TRACE),
+ ("debug", LevelFilter::DEBUG),
+ ("info", LevelFilter::INFO),
+ ("warn", LevelFilter::WARN),
+ ("error", LevelFilter::ERROR),
+ ("fatal", LevelFilter::ERROR),
+ ] {
+ let config = Config::from_values(None, None, Some(value), None).unwrap();
+ assert_eq!(config.log_level, expected, "{value}");
+ }
+ }
+
+ #[test]
+ fn rejects_invalid_configuration_without_echoing_input() {
+ let sensitive_input = "secret-that-must-not-appear";
+ for (address, timeout, level) in [
+ (Some(sensitive_input), None, None),
+ (None, Some(sensitive_input), None),
+ (None, None, Some(sensitive_input)),
+ (None, None, Some("off")),
+ (None, None, Some("INFO")),
+ (None, None, Some("3")),
+ (None, None, Some("")),
+ (None, Some("0"), None),
+ (None, Some("301"), None),
+ (None, Some("-1"), None),
+ (Some("localhost:8080"), None, None),
+ ] {
+ let error = Config::from_values(address, timeout, level, None)
+ .err()
+ .unwrap();
+ assert!(!error.to_string().contains(sensitive_input));
+ assert!(!format!("{error:?}").contains(sensitive_input));
+ }
+ }
+}
diff --git a/ai-gateway/src/lib.rs b/ai-gateway/src/lib.rs
new file mode 100644
index 0000000000..6d51ea9018
--- /dev/null
+++ b/ai-gateway/src/lib.rs
@@ -0,0 +1,2 @@
+pub mod config;
+pub mod server;
diff --git a/ai-gateway/src/main.rs b/ai-gateway/src/main.rs
new file mode 100644
index 0000000000..44e0d6819e
--- /dev/null
+++ b/ai-gateway/src/main.rs
@@ -0,0 +1,65 @@
+use std::{error::Error, process::ExitCode};
+
+use ai_gateway::{
+ config::{Config, LogFormat},
+ server::{self, AppState},
+};
+use tokio::net::TcpListener;
+
+#[tokio::main]
+async fn main() -> ExitCode {
+ match run().await {
+ Ok(()) => ExitCode::SUCCESS,
+ Err(error) => {
+ eprintln!("gateway startup or shutdown failed: {error}");
+ ExitCode::FAILURE
+ }
+ }
+}
+
+async fn run() -> Result<(), Box<dyn Error>> {
+ let config = Config::from_env()?;
+ let logging = tracing_subscriber::fmt()
+ .with_max_level(config.log_level)
+ .with_target(false);
+ match config.log_format {
+ LogFormat::Text => logging.compact().init(),
+ LogFormat::Json => logging.json().init(),
+ }
+ let shutdown = shutdown_signal()?;
+ let listener = TcpListener::bind(config.listen_address).await?;
+ tracing::info!(address = %listener.local_addr()?, "gateway listening");
+ let state = AppState::default();
+ server::serve(
+ listener,
+ server::router(state.clone()),
+ state,
+ shutdown,
+ config.shutdown_timeout,
+ )
+ .await?;
+ tracing::info!("gateway stopped");
+ Ok(())
+}
+
+#[cfg(unix)]
+fn shutdown_signal() -> std::io::Result<impl Future<Output = ()> + Send> {
+ use tokio::signal::unix::{SignalKind, signal};
+ let mut terminate = signal(SignalKind::terminate())?;
+ let mut interrupt = signal(SignalKind::interrupt())?;
+ Ok(async move {
+ tokio::select! {
+ _ = terminate.recv() => {},
+ _ = interrupt.recv() => {},
+ }
+ })
+}
+
+#[cfg(not(unix))]
+fn shutdown_signal() -> std::io::Result<impl Future<Output = ()> + Send> {
+ Ok(async {
+ if let Err(error) = tokio::signal::ctrl_c().await {
+ tracing::error!(%error, "failed to receive shutdown signal");
+ }
+ })
+}
diff --git a/ai-gateway/src/server.rs b/ai-gateway/src/server.rs
new file mode 100644
index 0000000000..48d4d15f84
--- /dev/null
+++ b/ai-gateway/src/server.rs
@@ -0,0 +1,79 @@
+use std::{
+ future::{Future, IntoFuture},
+ io,
+ sync::{
+ Arc,
+ atomic::{AtomicBool, Ordering},
+ },
+ time::Duration,
+};
+
+use axum::{Json, Router, extract::State, http::StatusCode, routing::get};
+use serde::Serialize;
+use tokio::{net::TcpListener, sync::oneshot};
+
+#[derive(Clone, Default)]
+pub struct AppState {
+ ready: Arc<AtomicBool>,
+}
+
+impl AppState {
+ pub fn is_ready(&self) -> bool {
+ self.ready.load(Ordering::Acquire)
+ }
+}
+
+#[derive(Serialize)]
+struct Probe {
+ status: &'static str,
+}
+
+pub fn router(state: AppState) -> Router {
+ Router::new()
+ .route("/health", get(|| async { Json(Probe { status: "ok" }) }))
+ .route("/ready", get(readiness))
+ .with_state(state)
+}
+
+async fn readiness(State(state): State<AppState>) -> (StatusCode, Json<Probe>) {
+ if state.is_ready() {
+ (StatusCode::OK, Json(Probe { status: "ready" }))
+ } else {
+ (
+ StatusCode::SERVICE_UNAVAILABLE,
+ Json(Probe { status: "draining" }),
+ )
+ }
+}
+
+/// Serve an initialized router until shutdown, then bound connection draining.
+/// A timeout must terminate the owning process/runtime to stop remaining tasks.
+pub async fn serve(
+ listener: TcpListener,
+ app: Router,
+ state: AppState,
+ shutdown: impl Future<Output = ()> + Send + 'static,
+ shutdown_timeout: Duration,
+) -> io::Result<()> {
+ let (draining_tx, draining_rx) = oneshot::channel();
+ let shutdown_state = state.clone();
+ let server = axum::serve(listener, app)
+ .with_graceful_shutdown(async move {
+ shutdown.await;
+ shutdown_state.ready.store(false, Ordering::Release);
+ tracing::info!("gateway draining");
+ let _ = draining_tx.send(());
+ })
+ .into_future();
+ tokio::pin!(server);
+ state.ready.store(true, Ordering::Release);
+ let result = tokio::select! {
+ result = &mut server => result,
+ _ = async {
+ let _ = draining_rx.await;
+ tokio::time::sleep(shutdown_timeout).await;
+ } => Err(io::Error::new(io::ErrorKind::TimedOut, "gateway shutdown deadline exceeded")),
+ };
+ state.ready.store(false, Ordering::Release);
+ result
+}
diff --git a/ai-gateway/tests/lifecycle.rs b/ai-gateway/tests/lifecycle.rs
new file mode 100644
index 0000000000..613d92eac0
--- /dev/null
+++ b/ai-gateway/tests/lifecycle.rs
@@ -0,0 +1,153 @@
+mod support;
+
+use ai_gateway::server::{self, AppState};
+use axum::{
+ Router,
+ body::{Body, to_bytes},
+ http::{Request, StatusCode},
+ routing::get,
+};
+use std::{
+ sync::{Arc, Mutex},
+ time::Duration,
+};
+use support::TestServer;
+use tokio::sync::Notify;
+use tower::ServiceExt;
+
+#[tokio::test]
+async fn probes_and_unimplemented_routes_over_http() {
+ let state = AppState::default();
+ let mut server =
+ TestServer::start(server::router(state.clone()), state, Duration::from_secs(1)).await;
+ let health = support::get(server.address, "/health").await;
+ assert!(health.starts_with("HTTP/1.1 200"));
+ assert!(health.ends_with(r#"{"status":"ok"}"#));
+ let ready = support::get(server.address, "/ready").await;
+ assert!(ready.starts_with("HTTP/1.1 200"));
+ assert!(ready.ends_with(r#"{"status":"ready"}"#));
+ let unimplemented = server::router(server.state.clone())
+ .oneshot(
+ Request::post("/openai/v1/responses")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(unimplemented.status(), StatusCode::NOT_FOUND);
+ server.begin_shutdown();
+ server.finish().await.unwrap();
+}
+
+#[tokio::test]
+async fn shutdown_marks_unready_and_waits_for_in_flight_request() {
+ let entered = Arc::new(Notify::new());
+ let release = Arc::new(Notify::new());
+ let state = AppState::default();
+ let app = server::router(state.clone()).route(
+ "/slow",
+ get({
+ let entered = entered.clone();
+ let release = release.clone();
+ move || {
+ let entered = entered.clone();
+ let release = release.clone();
+ async move {
+ entered.notify_one();
+ release.notified().await;
+ "finished"
+ }
+ }
+ }),
+ );
+ let mut server = TestServer::start(app, state.clone(), Duration::from_secs(2)).await;
+ let request = tokio::spawn(support::get(server.address, "/slow"));
+ tokio::time::timeout(Duration::from_secs(2), entered.notified())
+ .await
+ .unwrap();
+ server.begin_shutdown();
+ tokio::time::timeout(Duration::from_secs(2), async {
+ while state.is_ready() {
+ tokio::task::yield_now().await;
+ }
+ })
+ .await
+ .unwrap();
+ let response = server::router(state.clone())
+ .oneshot(Request::get("/ready").body(Body::empty()).unwrap())
+ .await
+ .unwrap();
+ assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
+ assert_eq!(
+ to_bytes(response.into_body(), 1024).await.unwrap(),
+ r#"{"status":"draining"}"#
+ );
+ assert!(!request.is_finished());
+ release.notify_one();
+ assert!(request.await.unwrap().ends_with("finished"));
+ server.finish().await.unwrap();
+ assert!(!state.is_ready());
+}
+
+#[tokio::test]
+async fn shutdown_deadline_bounds_a_stuck_handler() {
+ let entered = Arc::new(Notify::new());
+ let app = Router::new().route(
+ "/stuck",
+ get({
+ let entered = entered.clone();
+ move || {
+ let entered = entered.clone();
+ async move {
+ entered.notify_one();
+ std::future::pending::<()>().await;
+ "unreachable"
+ }
+ }
+ }),
+ );
+ let mut server = TestServer::start(app, AppState::default(), Duration::from_millis(50)).await;
+ let request = tokio::spawn(support::get(server.address, "/stuck"));
+ tokio::time::timeout(Duration::from_secs(2), entered.notified())
+ .await
+ .unwrap();
+ server.begin_shutdown();
+ assert_eq!(
+ server.finish().await.unwrap_err().kind(),
+ std::io::ErrorKind::TimedOut
+ );
+ request.abort();
+}
+
+#[tokio::test]
+async fn fake_dependency_records_requests_and_returns_scripted_response() {
+ let requests = Arc::new(Mutex::new(Vec::new()));
+ let app = Router::new().fallback({
+ let requests = requests.clone();
+ move |request: Request<Body>| {
+ let requests = requests.clone();
+ async move {
+ requests
+ .lock()
+ .unwrap()
+ .push((request.method().clone(), request.uri().to_string()));
+ (
+ StatusCode::TOO_MANY_REQUESTS,
+ [("retry-after", "1")],
+ "try later",
+ )
+ }
+ }
+ });
+ let mut fake = TestServer::start(app, AppState::default(), Duration::from_secs(1)).await;
+ let response = support::get(fake.address, "/example").await;
+ assert!(response.starts_with("HTTP/1.1 429"));
+ assert!(response.contains("retry-after: 1\r\n"));
+ assert!(response.ends_with("try later"));
+ assert_eq!(
+ requests.lock().unwrap().as_slice(),
+ &[(axum::http::Method::GET, "/example".to_string())]
+ );
+ fake.begin_shutdown();
+ fake.finish().await.unwrap();
+}
diff --git a/ai-gateway/tests/startup.rs b/ai-gateway/tests/startup.rs
new file mode 100644
index 0000000000..999e70e276
--- /dev/null
+++ b/ai-gateway/tests/startup.rs
@@ -0,0 +1,100 @@
+use std::{net::TcpListener, process::Command};
+
+#[test]
+fn invalid_configuration_exits_without_disclosing_the_value() {
+ // Occupy the listener so an ignored setting cannot leave a child server running.
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ for name in [
+ "LANGFUSE_AI_GATEWAY_SHUTDOWN_TIMEOUT_SECONDS",
+ "LANGFUSE_LOG_LEVEL",
+ "LANGFUSE_LOG_FORMAT",
+ ] {
+ let output = Command::new(env!("CARGO_BIN_EXE_ai-gateway"))
+ .env_clear()
+ .env(
+ "LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS",
+ listener.local_addr().unwrap().to_string(),
+ )
+ .env(name, "example-sensitive-value")
+ .output()
+ .unwrap();
+ assert!(!output.status.success());
+ let error = String::from_utf8(output.stderr).unwrap();
+ assert!(error.contains(name), "{error}");
+ assert!(!error.contains("example-sensitive-value"));
+ assert!(output.stdout.is_empty());
+ }
+}
+
+#[test]
+fn occupied_listener_exits_unsuccessfully() {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let output = Command::new(env!("CARGO_BIN_EXE_ai-gateway"))
+ .env_clear()
+ .env(
+ "LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS",
+ listener.local_addr().unwrap().to_string(),
+ )
+ .output()
+ .unwrap();
+ assert!(!output.status.success());
+ assert!(
+ String::from_utf8(output.stderr)
+ .unwrap()
+ .contains("gateway startup or shutdown failed")
+ );
+}
+
+#[test]
+fn lifecycle_logs_follow_the_shared_format() {
+ use std::{
+ io::{BufRead, BufReader},
+ process::Stdio,
+ sync::mpsc,
+ time::Duration,
+ };
+
+ for format in [None, Some("text"), Some("json")] {
+ let mut command = Command::new(env!("CARGO_BIN_EXE_ai-gateway"));
+ command
+ .env_clear()
+ .env("LANGFUSE_AI_GATEWAY_LISTEN_ADDRESS", "127.0.0.1:0")
+ .stdout(Stdio::piped())
+ .stderr(Stdio::null());
+ if let Some(format) = format {
+ command.env("LANGFUSE_LOG_FORMAT", format);
+ }
+ let mut child = command.spawn().unwrap();
+ let stdout = child.stdout.take().unwrap();
+ let (sender, receiver) = mpsc::channel();
+ let reader = std::thread::spawn(move || {
+ let mut line = String::new();
+ let result = BufReader::new(stdout).read_line(&mut line).map(|_| line);
+ let _ = sender.send(result);
+ });
+ let result = receiver.recv_timeout(Duration::from_secs(10));
+ let _ = child.kill();
+ child.wait().unwrap();
+ reader.join().unwrap();
+ let line = result.expect("gateway must emit a startup log").unwrap();
+ assert!(line.contains("gateway listening"), "{format:?}: {line}");
+ assert!(
+ !line.contains('\u{1b}'),
+ "logs must not contain ANSI escapes: {line}"
+ );
+ if format == Some("json") {
+ assert!(
+ line.starts_with('{') && line.trim_end().ends_with('}'),
+ "{line}"
+ );
+ assert!(line.contains("\"level\":\"INFO\""), "{line}");
+ assert!(line.contains("\"address\":\"127.0.0.1:"), "{line}");
+ } else {
+ assert!(!line.starts_with('{'), "{line}");
+ assert!(
+ line.contains("INFO") && line.contains("address=127.0.0.1:"),
+ "{line}"
+ );
+ }
+ }
+}
diff --git a/ai-gateway/tests/support/mod.rs b/ai-gateway/tests/support/mod.rs
new file mode 100644
index 0000000000..00ebf2d74f
--- /dev/null
+++ b/ai-gateway/tests/support/mod.rs
@@ -0,0 +1,76 @@
+use std::{io, net::SocketAddr, time::Duration};
+
+use ai_gateway::server::{self, AppState};
+use axum::Router;
+use tokio::{
+ io::{AsyncReadExt, AsyncWriteExt},
+ net::{TcpListener, TcpStream},
+ sync::oneshot,
+ task::JoinHandle,
+};
+
+pub struct TestServer {
+ pub address: SocketAddr,
+ pub state: AppState,
+ shutdown: Option<oneshot::Sender<()>>,
+ task: Option<JoinHandle<io::Result<()>>>,
+}
+
+impl TestServer {
+ pub async fn start(app: Router, state: AppState, budget: Duration) -> Self {
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let (shutdown, receiver) = oneshot::channel();
+ let task = tokio::spawn(server::serve(
+ listener,
+ app,
+ state.clone(),
+ async {
+ let _ = receiver.await;
+ },
+ budget,
+ ));
+ Self {
+ address,
+ state,
+ shutdown: Some(shutdown),
+ task: Some(task),
+ }
+ }
+
+ pub fn begin_shutdown(&mut self) {
+ self.shutdown.take().unwrap().send(()).unwrap();
+ }
+
+ pub async fn finish(mut self) -> io::Result<()> {
+ tokio::time::timeout(Duration::from_secs(5), self.task.take().unwrap())
+ .await
+ .expect("server did not stop")
+ .unwrap()
+ }
+}
+
+impl Drop for TestServer {
+ fn drop(&mut self) {
+ // Dropping the sender also requests shutdown, including after a failed assertion.
+ self.shutdown.take();
+ }
+}
+
+pub async fn get(address: SocketAddr, path: &str) -> String {
+ tokio::time::timeout(Duration::from_secs(5), async {
+ let mut socket = TcpStream::connect(address).await.unwrap();
+ socket
+ .write_all(
+ format!("GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")
+ .as_bytes(),
+ )
+ .await
+ .unwrap();
+ let mut response = String::new();
+ socket.read_to_string(&mut response).await.unwrap();
+ response
+ })
+ .await
+ .expect("HTTP request timed out")
+}
diff --git a/ai-gateway/turbo.json b/ai-gateway/turbo.json
new file mode 100644
index 0000000000..d7757ad0e4
--- /dev/null
+++ b/ai-gateway/turbo.json
@@ -0,0 +1,15 @@
+{
+ "extends": ["//"],
+ "tasks": {
+ "dev": {
+ "dependsOn": [],
+ "cache": false,
+ "persistent": true,
+ "interruptible": true
+ },
+ "build": { "dependsOn": [], "cache": false, "outputs": [] },
+ "typecheck": { "dependsOn": [], "cache": false, "outputs": [] },
+ "lint": { "dependsOn": [], "cache": false, "outputs": [] },
+ "test": { "dependsOn": [], "cache": false, "outputs": [] }
+ }
+}
diff --git a/package.json b/package.json
index 1efba881e9..74c9905d77 100644
--- a/package.json
+++ b/package.json
@@ -33,7 +33,7 @@
"typecheck": "turbo run typecheck",
"tc": "turbo run typecheck",
"start": "turbo run start",
- "dev": "turbo run dev",
+ "dev": "turbo watch dev",
"dev:worker": "turbo run dev --filter=worker",
"dev:web": "turbo run dev --filter=web",
"dev:web-webpack": "turbo run dev --filter=web -- --webpack",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index acc9839bbd..dcfa532cb6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -115,6 +115,12 @@ importers:
specifier: 2.10.5
version: 2.10.5
+ ai-gateway:
+ devDependencies:
+ dotenv-cli:
+ specifier: ^7.4.4
+ version: 7.4.4
+
ee:
dependencies:
'@langfuse/shared':
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index dc9fb04727..5c2a17ed18 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -2,6 +2,7 @@ packages:
# include packages in subfolders (e.g. apps/ and packages/)
- "web"
- "worker"
+ - "ai-gateway"
- "packages/**"
- "ee"
catalog: