🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(docker): create zero vuln base images and add fips mode (#18009)
Steffen Reutherbb1b6645182e
diff --git a/.github/workflows/_deploy_ecs_service.yml b/.github/workflows/_deploy_ecs_service.ymlindex bb07fe7cd7..55ad829026 100644--- a/.github/workflows/_deploy_ecs_service.yml+++ b/.github/workflows/_deploy_ecs_service.yml@@ -19,6 +19,10 @@ on:required: trueSENTRY_AUTH_TOKEN:required: false+ IRONBANK_USER:+ required: true+ IRONBANK_PASS:+ required: truejobs:ecs-deploy:if: ${{ inputs.service != 'ai-gateway' || inputs.environment == 'staging' }}@@ -26,6 +30,10 @@ jobs:environment: ${{ inputs.environment }}permissions:contents: read+ env:+ # Iron Bank UBI 9 bases, the same ones pipeline.yml uses for release images.+ BASE_IMAGE: registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8+ MICRO_BASE_IMAGE: registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8steps:- name: Get app nameuses: winterjung/split@a211a1c46e35fcdc4097d59dd6282d4a9859651b # v2@@ -48,6 +56,12 @@ jobs:- name: Login to AWS ECRid: login-ecruses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6+ - name: Log in to Iron Bank+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0+ with:+ registry: registry1.dso.mil+ username: ${{ secrets.IRONBANK_USER }}+ password: ${{ secrets.IRONBANK_PASS }}- name: Build, tag, and push AI Gateway imageif: ${{ inputs.service == 'ai-gateway' }}env:@@ -59,6 +73,8 @@ jobs:--platform linux/amd64 \--target runtime \--build-arg BUILD_ID="$IMAGE_TAG" \+ --build-arg BASE_IMAGE="$BASE_IMAGE" \+ --build-arg MICRO_BASE_IMAGE="$MICRO_BASE_IMAGE" \-t "$REGISTRY/ai-gateway:$IMAGE_TAG" \./ai-gatewaydocker push "$REGISTRY/ai-gateway:$IMAGE_TAG"@@ -87,6 +103,8 @@ jobs:docker build \-t $REGISTRY/$REPOSITORY:$IMAGE_TAG \-f ./${STEPS_SPLIT_OUTPUTS__0}/Dockerfile \+ --build-arg BASE_IMAGE="$BASE_IMAGE" \+ --build-arg MICRO_BASE_IMAGE="$MICRO_BASE_IMAGE" \--build-arg NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${VARS_NEXT_PUBLIC_LANGFUSE_CLOUD_REGION} \--build-arg NEXT_LANGFUSE_TRACING_SAMPLE_RATE=${VARS_NEXT_LANGFUSE_TRACING_SAMPLE_RATE} \--build-arg NEXT_PUBLIC_SENTRY_ENVIRONMENT=${VARS_NEXT_PUBLIC_SENTRY_ENVIRONMENT} \diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.ymlindex b58d9a9271..0409aa82a4 100644--- a/.github/workflows/deploy.yml+++ b/.github/workflows/deploy.yml@@ -117,6 +117,8 @@ jobs:AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}+ IRONBANK_USER: ${{ secrets.IRONBANK_USER }}+ IRONBANK_PASS: ${{ secrets.IRONBANK_PASS }}strategy:matrix:service: ${{ fromJson(needs.affected-services.outputs.services) }}diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.ymlindex 29e68e6951..64cf16d61d 100644--- a/.github/workflows/pipeline.yml+++ b/.github/workflows/pipeline.yml@@ -643,6 +643,42 @@ jobs:with:username: ${{ secrets.DOCKERHUB_USERNAME_READ }}password: ${{ secrets.DOCKERHUB_TOKEN_READ }}+ # Iron Bank needs a registry1.dso.mil account, and those credentials are+ # not available to forks or to pull_request runs from a fork. Fall back+ # to the Dockerfile default, Red Hat's public UBI9, which has the same+ # content and the same FIPS-validated OpenSSL.+ - name: Resolve FIPS base images+ id: bases+ env:+ IRONBANK_USER: ${{ secrets.IRONBANK_USER }}+ # True on runs that execute in this repo's own context, which is+ # where the Registry1 secrets exist: push, merge_group, tags, and+ # PRs raised from a branch here rather than from a fork.+ SECRETS_EXPECTED: ${{ github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}+ run: |+ if [ -n "$IRONBANK_USER" ]; then+ echo "ironbank=true" >> "$GITHUB_OUTPUT"+ echo "base=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8" >> "$GITHUB_OUTPUT"+ echo "micro=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8" >> "$GITHUB_OUTPUT"+ elif [ "$SECRETS_EXPECTED" = "true" ]; then+ # Never degrade quietly where the credential is supposed to be+ # present: a green build on the wrong base misreports which image+ # was validated.+ echo "::error::IRONBANK_USER is empty on a run that should have it. Check the repository secrets." >&2+ exit 1+ else+ echo "Fork run without Registry1 access, building on Red Hat's public UBI9." >&2+ echo "ironbank=false" >> "$GITHUB_OUTPUT"+ echo "base=registry.access.redhat.com/ubi9/ubi:9.8" >> "$GITHUB_OUTPUT"+ echo "micro=registry.access.redhat.com/ubi9/ubi-micro:9.8" >> "$GITHUB_OUTPUT"+ fi+ - name: Login to Iron Bank+ if: steps.bases.outputs.ironbank == 'true'+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0+ with:+ registry: registry1.dso.mil+ username: ${{ secrets.IRONBANK_USER }}+ password: ${{ secrets.IRONBANK_PASS }}- name: Setup Blacksmith Builderuses: useblacksmith/setup-docker-builder@1e75355f81cbb3f93cba1eb5dd5e5b05110ca8f9 # v2.0.1with:@@ -656,14 +692,22 @@ jobs:(set +e; docker compose -f "$DOCKER_COMPOSE_FILE" up -d --wait --wait-timeout 180 postgres redis minio clickhouse > /tmp/deps-up.log 2>&1; echo $? > /tmp/deps-up.exit) &- name: Build images# Type-checking runs in the lint job; release builds retain it.+ env:+ BASE_IMAGE: ${{ steps.bases.outputs.base }}+ MICRO_BASE_IMAGE: ${{ steps.bases.outputs.micro }}run: |docker compose -f "$DOCKER_COMPOSE_FILE" build --print > /tmp/bake.jsonif [ "${{ matrix.component }}" = "web" ]; thendocker buildx bake -f /tmp/bake.json \+ --set "*.args.BASE_IMAGE=$BASE_IMAGE" \+ --set "*.args.MICRO_BASE_IMAGE=$MICRO_BASE_IMAGE" \--set "langfuse-web.args.NEXT_IGNORE_BUILD_ERRORS=true" \langfuse-webelse- docker buildx bake -f /tmp/bake.json langfuse-worker+ docker buildx bake -f /tmp/bake.json \+ --set "*.args.BASE_IMAGE=$BASE_IMAGE" \+ --set "*.args.MICRO_BASE_IMAGE=$MICRO_BASE_IMAGE" \+ langfuse-workerfi- name: Start compose services and wait for healthchecksrun: |@@ -767,25 +811,10 @@ jobs:pnpm install- name: Load default envrun: |- write_test_env() {- {- grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' "$1"- echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1"- echo "LANGFUSE_CACHE_PROMPT_ENABLED=false"- echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1"- echo "LANGFUSE_TRACE_DELETE_DELAY_MS=1"- echo "LANGFUSE_TRACE_DELETE_CONCURRENCY=100"- echo "ADMIN_API_KEY=admin-api-key"- echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test"- echo "LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION=true"- echo "LANGFUSE_ENABLE_SCORES_V3_API=true"- } > "$2"- }-# Compile once under the default test environment. Next's generate# phase below reapplies the mode-specific config and prerendering.- write_test_env .env.dev.example .env- write_test_env .env.dev${{ matrix.deploy-mode }}.example .env.runtime+ bash scripts/ci/write-web-test-env.sh .env.dev.example .env+ bash scripts/ci/write-web-test-env.sh .env.dev${{ matrix.deploy-mode }}.example .env.runtimeecho "LANGFUSE_OTEL_INGESTION_WORKER_SHADOW_ENABLED=${{ matrix.deploy-mode != '-azure' }}" >> .env.runtime- name: Setup Turbo remote cacheif: env.CI_CACHE_ALLOWED == 'true'@@ -1026,6 +1055,130 @@ jobs:run: pnpm --filter=worker run test:exclude-llm-connectionsenv:LANGFUSE_CODE_EVAL_AWS_LAMBDA_ENDPOINT: ${{ matrix.deploy-mode == '' && 'http://localhost:4566' || '' }}++ tests-fips:+ # Runs the web and worker suites the way a FIPS host would: on the UBI+ # toolchain from worker/Dockerfile with OPENSSL_FORCE_FIPS_MODE=1, which+ # puts Red Hat's OpenSSL in FIPS mode without a FIPS kernel. Code paths+ # that need an algorithm FIPS refuses (md5, ...) fail with+ # ERR_OSSL_EVP_UNSUPPORTED. Databases and migrations run on the runner as+ # in tests-web/tests-worker; install, build, the web and worker processes+ # and the tests run in the container (scripts/ci/fips-tests.sh).+ # Not part of all-ci-passed yet.+ timeout-minutes: 45+ runs-on: ${{ matrix.runner }}+ needs:+ - pre-job+ if: ${{ !cancelled() && (github.event_name != 'push' || needs.pre-job.outputs.should_skip != 'true') }}+ name: tests-fips (${{ matrix.suite }})+ env:+ VITEST_MAX_WORKERS: ${{ matrix.max-workers }}+ strategy:+ fail-fast: false+ matrix:+ include:+ - suite: web+ runner: blacksmith-16vcpu-ubuntu-2404+ max-workers: "12"+ - suite: worker+ runner: blacksmith-8vcpu-ubuntu-2404+ max-workers: "4"+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ - uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0+ with:+ install: false+ cache: false+ - name: Login to Docker Hub+ if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0+ with:+ username: ${{ secrets.DOCKERHUB_USERNAME_READ }}+ password: ${{ secrets.DOCKERHUB_TOKEN_READ }}+ - name: Use Node.js 24 without cache+ if: env.CI_CACHE_ALLOWED != 'true'+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+ with:+ node-version: 24+ package-manager-cache: false+ - name: Use Node.js 24 with pnpm cache+ if: env.CI_CACHE_ALLOWED == 'true'+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # zizmor: ignore[cache-poisoning] test job dependency cache only; no released artifacts are built or published from this cached state+ with:+ node-version: 24+ cache: "pnpm"+ cache-dependency-path: "pnpm-lock.yaml"+ - name: install dependencies+ # For the migrations and seeding on the runner; the container installs+ # its own copy for UBI.+ run: |+ pnpm install+ - name: Install golang-migrate for Clickhouse migrations+ run: |+ curl --fail --location --retry 5 --retry-delay 2 --retry-all-errors \+ --output migrate.linux-amd64.tar.gz \+ https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz+ tar xzf migrate.linux-amd64.tar.gz+ sudo mv migrate /usr/bin/migrate+ which migrate+ - name: Load default env+ run: |+ if [ "${{ matrix.suite }}" = "web" ]; then+ bash scripts/ci/write-web-test-env.sh .env.dev.example .env+ cp .env .env.runtime+ echo "LANGFUSE_OTEL_INGESTION_WORKER_SHADOW_ENABLED=true" >> .env.runtime+ else+ cp .env.dev.example .env+ cp .env.dev.example web/.env+ cp .env.dev.example worker/.env+ fi+ - name: Setup Blacksmith Builder+ uses: useblacksmith/setup-docker-builder@1e75355f81cbb3f93cba1eb5dd5e5b05110ca8f9 # v2.0.1+ with:+ cache-key: langfuse/test-fips-toolchain+ - name: Build UBI toolchain image+ run: docker buildx build --load --target toolchain -f worker/Dockerfile -t langfuse-fips-toolchain .+ - name: Run + migrate+ run: |+ docker compose -f docker-compose.dev.yml up -d --wait --wait-timeout 180+ docker compose -f docker-compose.dev.yml ps+ pnpm run db:migrate+ if [ "${{ matrix.suite }}" = "worker" ]; then+ pnpm --filter=shared run db:seed+ fi+ pnpm --filter=shared ch:up+ env:+ POSTGRES_VERSION: 15+ # The worker's code-eval tests need the extra floci container.+ COMPOSE_PROFILES: ${{ matrix.suite == 'worker' && 'worker-tests' || '' }}+ - name: Provide ClickHouse client via dev container for dev-tables setup+ # The clickhouse-server container already ships the client binary;+ # a shim avoids re-downloading the ~300MB client package every run.+ run: |+ sudo tee /usr/local/bin/clickhouse > /dev/null <<'EOF'+ #!/bin/bash+ exec docker exec -i langfuse-clickhouse clickhouse "$@"+ EOF+ sudo chmod +x /usr/local/bin/clickhouse+ - name: Setup Dev Tables+ run: |+ pnpm --filter=shared ch:dev-tables+ - name: Run ${{ matrix.suite }} tests under forced FIPS+ env:+ LANGFUSE_CODE_EVAL_AWS_LAMBDA_ENDPOINT: ${{ matrix.suite == 'worker' && 'http://localhost:4566' || '' }}+ SUITE: ${{ matrix.suite }}+ run: |+ docker run --rm --network host \+ -e OPENSSL_FORCE_FIPS_MODE=1 \+ -e CI -e VITEST_MAX_WORKERS -e LANGFUSE_CODE_EVAL_AWS_LAMBDA_ENDPOINT \+ -e PNPM_STORE_DIR=/pnpm-store \+ -v "$(pnpm store path --silent):/pnpm-store" \+ -v "$PWD:/src:ro" \+ langfuse-fips-toolchain \+ bash /src/scripts/ci/fips-tests.sh "$SUITE"+test-worker-llm-connections:timeout-minutes: 20runs-on: blacksmith-4vcpu-ubuntu-2404@@ -1549,6 +1702,30 @@ jobs:with:username: ${{ secrets.DOCKERHUB_USERNAME }}password: ${{ secrets.DOCKERHUB_TOKEN }}+ # Released images always build on the Iron Bank bases. Both ubi9 and+ # ubi9-micro publish an OCI index covering linux/amd64 and linux/arm64,+ # so every leg of the platform matrix resolves there.+ - name: Resolve FIPS base images+ id: bases+ env:+ IRONBANK_USER: ${{ secrets.IRONBANK_USER }}+ run: |+ if [ -z "$IRONBANK_USER" ]; then+ # Releases only ever build in this repo, so a missing credential is+ # a misconfiguration. Publishing images that claim Iron Bank+ # provenance but were built elsewhere is worse than a failed+ # release, so stop here.+ echo "::error::IRONBANK_USER is empty on a release build. Check the repository secrets." >&2+ exit 1+ fi+ echo "base=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8" >> "$GITHUB_OUTPUT"+ echo "micro=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8" >> "$GITHUB_OUTPUT"+ - name: Log in to Iron Bank+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0+ with:+ registry: registry1.dso.mil+ username: ${{ secrets.IRONBANK_USER }}+ password: ${{ secrets.IRONBANK_PASS }}# build-push-action v2 requires this step; it no longer sets up buildkit# itself. Both inputs are load-bearing, do not drop them:# - cache-key is required as of setup-docker-builder v2. Without it the@@ -1588,6 +1765,9 @@ jobs:outputs: type=image,name=ghcr.io/langfuse/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=truelabels: ${{ steps.meta.outputs.labels }}platforms: ${{ matrix.platform }}+ build-args: |+ BASE_IMAGE=${{ steps.bases.outputs.base }}+ MICRO_BASE_IMAGE=${{ steps.bases.outputs.micro }}provenance: falsesbom: false- name: Build and push image by digest to Docker Hub (${{ matrix.component }}, ${{ matrix.platform_tag }})@@ -1599,6 +1779,9 @@ jobs:outputs: type=image,name=langfuse/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=truelabels: ${{ steps.meta.outputs.labels }}platforms: ${{ matrix.platform }}+ build-args: |+ BASE_IMAGE=${{ steps.bases.outputs.base }}+ MICRO_BASE_IMAGE=${{ steps.bases.outputs.micro }}provenance: falsesbom: false- name: Record pushed digestsdiff --git a/ai-gateway/Dockerfile b/ai-gateway/Dockerfileindex 8e656645e0..4e72ff3a99 100644--- a/ai-gateway/Dockerfile+++ b/ai-gateway/Dockerfile@@ -1,18 +1,101 @@-FROM rust:1.98.0-trixie AS builder-RUN useradd --create-home --uid 10001 gateway && mkdir /app && chown gateway:gateway /app+# FIPS 140-3 base: UBI 9 builder and ubi9-micro runtime+# The gateway performs no TLS today (plain HTTP; the dependency graph contains+# no openssl or rustls crates), so FIPS readiness here means the whole image+# lifecycle stays on the RHEL platform: the binary is compiled and linked+# against UBI9 glibc, and the runtime carries the CMVP-validated OpenSSL FIPS+# provider, so outbound TLS added later links the system libcrypto.so.3 and+# inherits the validated module, the same pattern the RPM Node uses in+# web/worker. The crypto policy stays at the default until the gateway does TLS.+# The defaults are Red Hat's public UBI9 images, so the build needs no+# credentials. Iron Bank serves the same content from an authenticated registry+# with DoD provenance and scan attestations. Release builds opt in by overriding+# both args after `docker login registry1.dso.mil` (free P1 account plus a CLI+# secret):+# --build-arg BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8+# --build-arg MICRO_BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8+ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.8+ARG MICRO_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi-micro:9.8++FROM --platform=${TARGETPLATFORM:-linux/amd64} ${BASE_IMAGE} AS builder++ARG TARGETARCH++# gcc provides the linker; tar and gzip unpack the pinned toolchain (curl is+# already present as curl-minimal in the base). Red Hat's UBI9 rust-toolset+# (1.84.1) cannot compile this crate's edition 2024 (needs rustc >= 1.85), and+# the official rust:1.98.0-bookworm image links against Debian's glibc 2.36,+# so binaries built there refuse to run on UBI9 (glibc 2.34). Building on UBI9+# keeps the toolchain and the binary on the FIPS-validated platform.+RUN dnf -y upgrade && \+ dnf -y install --setopt=install_weak_deps=0 gcc tar gzip && \+ dnf clean all++# Pinned standalone Rust 1.98.0 installer, the exact channel from+# rust-toolchain.toml (keep the version in sync with it), checksum-verified+# like the dumb-init binary in web/Dockerfile. rustup-init would fetch the+# rustup binary from an unversioned URL, and only rustc, cargo and the+# target's std are needed to build; docs, clippy and rustfmt stay out of the+# builder image (CI runs fmt and clippy outside Docker).+RUN set -eux; \+ case "${TARGETARCH:-amd64}" in \+ amd64) triple=x86_64-unknown-linux-gnu; toolchain_sha256=aa30409afa67bd1ada244cefd82c7980e6a65bc113bb978e934b2413c75e3900 ;; \+ arm64) triple=aarch64-unknown-linux-gnu; toolchain_sha256=5fbb4282403046d52a4672765c6761a809bf9f33e699b17e6eb7a93ab7770cc3 ;; \+ *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \+ esac; \+ curl -fsSL -o /tmp/rust.tar.gz "https://static.rust-lang.org/dist/rust-1.98.0-${triple}.tar.gz"; \+ echo "${toolchain_sha256} /tmp/rust.tar.gz" | sha256sum -c -; \+ tar -xzf /tmp/rust.tar.gz -C /tmp; \+ /tmp/rust-1.98.0-${triple}/install.sh --prefix=/usr/local \+ --components=rustc,cargo,rust-std-${triple}; \+ rm -rf /tmp/rust.tar.gz /tmp/rust-1.98.0-${triple}+WORKDIR /app-ENV CARGO_HOME=/home/gateway/.cargo-USER gateway-COPY --chown=gateway:gateway Cargo.toml Cargo.lock rust-toolchain.toml ./-COPY --chown=gateway:gateway src ./src++COPY Cargo.toml Cargo.lock rust-toolchain.toml ./+COPY src ./src++# rust-toolchain.toml is honored by rustup only; the standalone installer's+# cargo ignores it, so --locked plus the pinned installer above guarantee the+# 1.98.0 toolchain and the checked-in lockfile.RUN cargo build --release --locked-FROM debian:trixie-slim AS runtime-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget \- && rm -rf /var/lib/apt/lists/*+# Runtime rootfs assembled with dnf --installroot and layered onto ubi9-micro:+# the final image carries no package manager, python, or curl. It holds only+# the packages listed below plus their deps, with an intact rpmdb for scanners.+# --disableexcludes=all: the Iron Bank base excludes filesystem-* in+# /etc/dnf/dnf.conf (it protects its own rootfs during in-place upgrades),+# which would also block installing the dir-skeleton package into a fresh+# installroot.+FROM --platform=${TARGETPLATFORM:-linux/amd64} ${BASE_IMAGE} AS rpm-installer++# glibc and libgcc are the binary's dynamic dependencies (libgcc_s for panics).+# openssl-libs ships the FIPS provider module (/usr/lib64/ossl-modules/fips.so),+# unlinked by today's binary but required the moment an OpenSSL-dependent+# binary lands, so the validated module is already present. crypto-policies+# provides the system policy OpenSSL reads; ca-certificates is the trust store+# for future outbound TLS. wget serves the ECS container health command+# (`sh -c 'wget -qO /dev/null http://localhost:8080/health'`, exercised by+# scripts/smoke-image.sh); ubi9-micro already provides the shell.+RUN dnf -y --installroot /mnt/rootfs --releasever 9 --setopt=install_weak_deps=0 \+ --setopt=tsflags=nodocs --disableexcludes=all \+ install glibc libgcc ca-certificates crypto-policies openssl-libs wget && \+ dnf --installroot /mnt/rootfs clean all && \+ rm -rf /mnt/rootfs/var/cache /mnt/rootfs/var/log/dnf*++# Bake the non-root user into the rootfs (micro has no shadow-utils). uid/gid+# 10001 preserves the upstream gateway image's identity.+RUN echo "gateway:x:10001:" >> /mnt/rootfs/etc/group && \+ echo "gateway:x:10001:10001::/home/gateway:/sbin/nologin" >> /mnt/rootfs/etc/passwd++FROM --platform=${TARGETPLATFORM:-linux/amd64} ${MICRO_BASE_IMAGE} AS runtime++COPY --from=rpm-installer /mnt/rootfs /COPY --from=builder /app/target/release/ai-gateway /usr/local/bin/ai-gatewayARG BUILD_IDENV BUILD_ID=$BUILD_ID++# The gateway handles SIGTERM itself with a bounded drain (see+# scripts/smoke-image.sh), so no init process is needed.USER 10001:10001EXPOSE 8080STOPSIGNAL SIGTERMdiff --git a/packages/native/AGENTS.md b/packages/native/AGENTS.mdindex 5a3b9046db..036219302f 100644--- a/packages/native/AGENTS.md+++ b/packages/native/AGENTS.md@@ -12,10 +12,12 @@- `index.js` and `index.d.ts` are generated by `napi build`. Never hand-editthem; rebuild and commit the result whenever `src/lib.rs` exports change.- Building the worker needs the pinned Rust toolchain: CI runners get it from- rustup auto-install on first `cargo` use; `worker/Dockerfile` copies the- same version from the `rust:<version>-alpine` image. Bump- `rust-toolchain.toml` and that image tag together. The pin starts equal to- `ai-gateway/rust-toolchain.toml` but the files are deliberately separate.+ rustup auto-install on first `cargo` use; `worker/Dockerfile` installs the+ same version from the checksum-pinned standalone installer on UBI9. Bump+ `rust-toolchain.toml` together with that version and both per-arch sha256+ values (`https://static.rust-lang.org/dist/rust-<version>-<triple>.tar.gz.sha256`).+ The pin starts equal to `ai-gateway/rust-toolchain.toml` but the files are+ deliberately separate.- Native code reports through the `metrics` and `tracing` facades set up in`src/telemetry.rs`; never return values for Node to record. Configurationmirrors dd-trace and the winston logger (`DD_*`, `LANGFUSE_LOG_*`).diff --git a/packages/native/README.md b/packages/native/README.mdindex 7f320dbcb1..2b0e28c1ef 100644--- a/packages/native/README.md+++ b/packages/native/README.md@@ -15,17 +15,17 @@ CPU-bound fits can be killed without blocking the worker.## Layout-| Path | Purpose |-| --------------------- | -------------------------------------------------------------------------- |-| `src/lib.rs` | Rust source. Every `#[napi]` item is exported to Node.js. |-| `src/telemetry.rs` | Metrics and log setup shared by all native code; see Observability. |-| `src/topics.rs` | Seeded UMAP reductions and HDBSCAN for Topics. |-| `Cargo.toml` | Crate manifest (`cdylib`). `Cargo.lock` is committed. |-| `build.rs` | napi-rs build hook. |-| `rust-toolchain.toml` | Pinned compiler version, matched by the Rust image in `worker/Dockerfile`. |-| `package.json` | npm package; `napi.binaryName` names the compiled `.node` file. |-| `index.js` | Generated loader that picks the `.node` file for the current platform. |-| `index.d.ts` | Generated TypeScript declarations, derived from the `#[napi]` signatures. |+| Path | Purpose |+| --------------------- | ---------------------------------------------------------------------------- |+| `src/lib.rs` | Rust source. Every `#[napi]` item is exported to Node.js. |+| `src/telemetry.rs` | Metrics and log setup shared by all native code; see Observability. |+| `src/topics.rs` | Seeded UMAP reductions and HDBSCAN for Topics. |+| `Cargo.toml` | Crate manifest (`cdylib`). `Cargo.lock` is committed. |+| `build.rs` | napi-rs build hook. |+| `rust-toolchain.toml` | Pinned compiler version, kept equal to the one `worker/Dockerfile` installs. |+| `package.json` | npm package; `napi.binaryName` names the compiled `.node` file. |+| `index.js` | Generated loader that picks the `.node` file for the current platform. |+| `index.d.ts` | Generated TypeScript declarations, derived from the `#[napi]` signatures. |`index.js` and `index.d.ts` are regenerated by every build and committed sotype checks and tooling work without a Rust toolchain. `*.node` binaries and@@ -79,9 +79,9 @@ is always present when the worker starts or its tests run. A direct## How it ships-`worker/Dockerfile` copies the toolchain pinned in `rust-toolchain.toml` from-the official `rust:<version>-alpine` image into the builder stage;-`turbo run build --filter=worker...` compiles the addon for musl, and+`worker/Dockerfile` installs the toolchain pinned in `rust-toolchain.toml` from+the checksum-verified standalone Rust installer into its UBI9 builder stage;+`turbo run build --filter=worker...` compiles the addon for glibc, and`pnpm deploy` copies the `.node` file into the runtime image next to theloader. Each architecture builds on a native runner, so no crosscompilation is involved. The runtime image gains only the compiled library.diff --git a/packages/native/rust-toolchain.toml b/packages/native/rust-toolchain.tomlindex 612a791279..7a82122ecb 100644--- a/packages/native/rust-toolchain.toml+++ b/packages/native/rust-toolchain.toml@@ -1,7 +1,7 @@# Same Rust version as ai-gateway/rust-toolchain.toml so the repository starts# out on one compiler; the two files stay separate so either crate can move on-# its own. worker/Dockerfile copies this exact version from the official-# `rust:<version>-alpine` image, so bump the two together.+# its own. worker/Dockerfile installs this exact version from the pinned+# standalone Rust installer, so bump the two together.[toolchain]channel = "1.98.0"components = ["clippy", "rustfmt"]diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.mdindex 1abd74207a..e0d93045c8 100644--- a/packages/shared/AGENTS.md+++ b/packages/shared/AGENTS.md@@ -152,6 +152,9 @@`@langfuse/shared/src/server/auth/apiKeys`,`@langfuse/shared/src/server/clickhouse/clickhouseIdentifiers`,`@langfuse/shared/src/server/ee/ingestionMasking`,+ `@langfuse/shared/src/server/ee/fips` (startup FIPS-mode enforcement; it+ loads only env and the license check, so it can run before anything+ connects),`@langfuse/shared/src/server/llm/llmText`, and`@langfuse/shared/src/utils/chatml`. The`@langfuse/shared/src/utils/normalized-io` parser is client-safe and is thediff --git a/packages/shared/package.json b/packages/shared/package.jsonindex ae95ab53af..1391c43a3d 100644--- a/packages/shared/package.json+++ b/packages/shared/package.json@@ -83,6 +83,10 @@"import": "./dist/src/server/ee/ingestionMasking/index.js","require": "./dist/src/server/ee/ingestionMasking/index.js"},+ "./src/server/ee/fips": {+ "import": "./dist/src/server/ee/fips/index.js",+ "require": "./dist/src/server/ee/fips/index.js"+ },"./src/server/ee/licenseCheck": {"import": "./dist/src/server/ee/licenseCheck/index.js","require": "./dist/src/server/ee/licenseCheck/index.js"diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prismaindex b29f7da1e2..61fe8d0cc7 100644--- a/packages/shared/prisma/schema.prisma+++ b/packages/shared/prisma/schema.prisma@@ -4,6 +4,10 @@generator client {provider = "prisma-client-js"previewFeatures = ["views", "relationJoins", "metrics"]+ // rhel-openssl-3.0.x: UBI9 runtime images (glibc); these engines link the+ // system libcrypto.so.3, so Postgres TLS runs inside the platform's+ // FIPS-validated OpenSSL module.+ binaryTargets = ["native", "rhel-openssl-3.0.x"]}datasource db {diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.tsindex 7b8ba5331a..e4bb3bd8de 100644--- a/packages/shared/src/env.ts+++ b/packages/shared/src/env.ts@@ -695,6 +695,10 @@ const EnvSchema = z.object({// EE LicenseLANGFUSE_EE_LICENSE_KEY: z.string().optional(),+ // FIPS mode (EE feature): refuse to start without an enterprise license or+ // an active OpenSSL FIPS provider. See server/ee/fips.+ LANGFUSE_REQUIRE_FIPS: z.enum(["true", "false"]).default("false"),+// Ingestion Masking (EE feature)LANGFUSE_INGESTION_MASKING_CALLBACK_URL: z.url().optional(),LANGFUSE_INGESTION_MASKING_CALLBACK_TIMEOUT_MS: z.coercediff --git a/packages/shared/src/server/ee/fips/assertFipsMode.test.ts b/packages/shared/src/server/ee/fips/assertFipsMode.test.tsnew file mode 100644index 0000000000..0316321b2b--- /dev/null+++ b/packages/shared/src/server/ee/fips/assertFipsMode.test.ts@@ -0,0 +1,70 @@+import { beforeEach, describe, expect, it, vi } from "vitest";+import type { SharedEnv } from "../../../env";+import { assertFipsMode } from "./assertFipsMode";++const { getFips } = vi.hoisted(() => ({ getFips: vi.fn() }));++vi.mock("crypto", async (importOriginal) => ({+ ...(await importOriginal<typeof import("crypto")>()),+ getFips,+}));++const createTestEnv = (overrides: Partial<SharedEnv> = {}): SharedEnv =>+ ({+ LANGFUSE_REQUIRE_FIPS: "true",+ LANGFUSE_EE_LICENSE_KEY: "langfuse_ee_test",+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,+ ...overrides,+ }) as SharedEnv;++describe("assertFipsMode", () => {+ beforeEach(() => {+ getFips.mockReset();+ getFips.mockReturnValue(1);+ });++ it("does nothing when FIPS mode is not requested", () => {+ getFips.mockReturnValue(0);++ expect(() =>+ assertFipsMode(+ createTestEnv({+ LANGFUSE_REQUIRE_FIPS: "false",+ LANGFUSE_EE_LICENSE_KEY: undefined,+ }),+ ),+ ).not.toThrow();+ });++ it("passes with an enterprise license and an active FIPS provider", () => {+ expect(() => assertFipsMode(createTestEnv())).not.toThrow();+ });++ it("passes on Langfuse Cloud with an active FIPS provider", () => {+ expect(() =>+ assertFipsMode(+ createTestEnv({+ LANGFUSE_EE_LICENSE_KEY: undefined,+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",+ }),+ ),+ ).not.toThrow();+ });++ it.each([+ ["no license key", undefined],+ ["a pro license key", "langfuse_pro_test"],+ ])("refuses to start with %s, even on a FIPS host", (_label, licenseKey) => {+ expect(() =>+ assertFipsMode(createTestEnv({ LANGFUSE_EE_LICENSE_KEY: licenseKey })),+ ).toThrow(/enterprise license/);+ });++ it("refuses to start with a license when the FIPS provider is inactive", () => {+ getFips.mockReturnValue(0);++ expect(() => assertFipsMode(createTestEnv())).toThrow(+ /FIPS provider is not active/,+ );+ });+});diff --git a/packages/shared/src/server/ee/fips/assertFipsMode.ts b/packages/shared/src/server/ee/fips/assertFipsMode.tsnew file mode 100644index 0000000000..d84057647e--- /dev/null+++ b/packages/shared/src/server/ee/fips/assertFipsMode.ts@@ -0,0 +1,34 @@+import { getFips } from "crypto";+import { env, type SharedEnv } from "../../../env";+import { isEnterpriseLicenseAvailable } from "../licenseCheck";++/**+ * Enforce FIPS mode (LANGFUSE_REQUIRE_FIPS=true), an enterprise feature.+ *+ * Fails closed: when FIPS mode is requested, startup must stop unless an+ * enterprise license is available and Node's OpenSSL FIPS provider is active.+ * The provider is only active when the host kernel runs in FIPS mode, so a+ * container cannot switch it on by itself.+ *+ * Imports nothing that opens connections, so callers can run it before any+ * other startup code.+ */+export function assertFipsMode(envOverride?: SharedEnv): void {+ const e = envOverride ?? env;++ if (e.LANGFUSE_REQUIRE_FIPS !== "true") {+ return;+ }++ if (!isEnterpriseLicenseAvailable(e)) {+ throw new Error(+ "LANGFUSE_REQUIRE_FIPS=true requires a Langfuse enterprise license (LANGFUSE_EE_LICENSE_KEY).",+ );+ }++ if (getFips() !== 1) {+ throw new Error(+ "LANGFUSE_REQUIRE_FIPS=true but Node's OpenSSL FIPS provider is not active (crypto.getFips() != 1). Run Langfuse on a host with FIPS mode enabled.",+ );+ }+}diff --git a/packages/shared/src/server/ee/fips/index.ts b/packages/shared/src/server/ee/fips/index.tsnew file mode 100644index 0000000000..7f7d01771f--- /dev/null+++ b/packages/shared/src/server/ee/fips/index.ts@@ -0,0 +1 @@+export * from "./assertFipsMode";diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.tsindex 783dfaf095..d97f99227b 100644--- a/packages/shared/src/server/redis/redis.ts+++ b/packages/shared/src/server/redis/redis.ts@@ -1,5 +1,5 @@import Redis, { RedisOptions, Cluster, ClusterOptions } from "ioredis";-import type { QueueBaseOptions } from "bullmq";+import type { QueueBaseOptions, QueueOptions } from "bullmq";import fs from "fs";import { env } from "../../env";import { logger } from "../logger";@@ -76,10 +76,13 @@ export const redisQueueRetryOptions: Partial<RedisOptions> = {},};+// "settings" comes from QueueOptions (not QueueBaseOptions); the same+// AdvancedRepeatOptions shape is accepted by WorkerOptions["settings"].type BullMQOptionsWithRedis = Pick<QueueBaseOptions,"connection" | "prefix" | "skipVersionCheck"->;+> &+ Pick<QueueOptions, "settings">;/*** Parse Redis node definitions from environment variable@@ -369,6 +372,15 @@ const getBullMQOptionsForRedisConnection = (): BullMQOptionsWithRedis => ({connection,prefix: getQueuePrefix(queueName),+ // bullmq hashes with md5 by default, and an OpenSSL FIPS provider refuses+ // md5, so any call into the legacy repeat API throws+ // ERR_OSSL_EVP_UNSUPPORTED on a FIPS image. Job scheduler registration does+ // not hash and is unaffected, but the legacy-schedule cleanup in+ // scheduleRecurringJob calls removeRepeatableByKey, which hashes before it+ // reaches Redis. sha256 is FIPS-approved and keeps that call working; the+ // digest itself is only used to build an identifier the removal script+ // never reads.+ settings: { repeatKeyHashAlgorithm: "sha256" },...(env.LANGFUSE_BULLMQ_SKIP_REDIS_VERSION_CHECK === "true"? { skipVersionCheck: true }: {}),diff --git a/packages/shared/src/server/redis/scheduleRecurringJob.test.ts b/packages/shared/src/server/redis/scheduleRecurringJob.test.tsindex 17c4cb4816..6fd8ad9d45 100644--- a/packages/shared/src/server/redis/scheduleRecurringJob.test.ts+++ b/packages/shared/src/server/redis/scheduleRecurringJob.test.ts@@ -11,15 +11,46 @@ vi.mock("../logger", () => ({},}));-const createQueueMock = () => {- const removeRepeatable = vi.fn().mockResolvedValue(true);+/** Legacy entries are keyed by a hash of the repeat options, never by name. */+const legacyEntry = (name: string, pattern: string, key: string) => ({+ key,+ name,+ pattern,+ endDate: null,+ tz: null,+ every: null,+ next: 0,+});++/** The job scheduler stores its entry under the scheduler id, i.e. the name. */+const schedulerEntry = (name: string, pattern: string) => ({+ key: name,+ name,+ pattern,+ endDate: null,+ tz: null,+ every: null,+ next: 0,+});++const createQueueMock = (+ repeatables: ReturnType<typeof legacyEntry>[] = [],+) => {+ const getRepeatableJobs = vi.fn().mockResolvedValue(repeatables);+ const removeRepeatableByKey = vi.fn().mockResolvedValue(true);const upsertJobScheduler = vi.fn().mockResolvedValue({});const queue = {name: "test-queue",- removeRepeatable,+ getRepeatableJobs,+ removeRepeatableByKey,upsertJobScheduler,} as unknown as Queue;- return { queue, removeRepeatable, upsertJobScheduler };+ return {+ queue,+ getRepeatableJobs,+ removeRepeatableByKey,+ upsertJobScheduler,+ };};describe("scheduleRecurringJob", () => {@@ -28,7 +59,11 @@ describe("scheduleRecurringJob", () => {});it("removes legacy schedules for the current and previous patterns before upserting", async () => {- const { queue, removeRepeatable, upsertJobScheduler } = createQueueMock();+ const { queue, removeRepeatableByKey, upsertJobScheduler } =+ createQueueMock([+ legacyEntry("my-job", "*/15 * * * *", "8f14e45fceea167a"),+ legacyEntry("my-job", "25 * * * *", "45c48cce2e2d7fbd"),+ ]);await scheduleRecurringJob(queue, {jobName: "my-job",@@ -36,13 +71,9 @@ describe("scheduleRecurringJob", () => {previousPatterns: ["25 * * * *"],});- expect(removeRepeatable).toHaveBeenCalledTimes(2);- expect(removeRepeatable).toHaveBeenCalledWith("my-job", {- pattern: "*/15 * * * *",- });- expect(removeRepeatable).toHaveBeenCalledWith("my-job", {- pattern: "25 * * * *",- });+ expect(removeRepeatableByKey).toHaveBeenCalledTimes(2);+ expect(removeRepeatableByKey).toHaveBeenCalledWith("8f14e45fceea167a");+ expect(removeRepeatableByKey).toHaveBeenCalledWith("45c48cce2e2d7fbd");expect(upsertJobScheduler).toHaveBeenCalledWith("my-job",{ pattern: "*/15 * * * *" },@@ -50,13 +81,67 @@ describe("scheduleRecurringJob", () => {);// The legacy cleanup must complete before the scheduler is upserted, so// a concurrently created legacy entry cannot outlive the migration.- for (const removeCall of removeRepeatable.mock.invocationCallOrder) {+ for (const removeCall of removeRepeatableByKey.mock.invocationCallOrder) {expect(removeCall).toBeLessThan(upsertJobScheduler.mock.invocationCallOrder[0]!,);}});+ it("never removes the job scheduler's own entry", async () => {+ // The scheduler shares the `repeat` sorted set with legacy entries and is+ // keyed by the scheduler id. Matching on name alone would delete the live+ // schedule this call is about to (re)create.+ const { queue, removeRepeatableByKey } = createQueueMock([+ schedulerEntry("my-job", "*/15 * * * *"),+ ]);++ await scheduleRecurringJob(queue, {+ jobName: "my-job",+ pattern: "*/15 * * * *",+ });++ expect(removeRepeatableByKey).not.toHaveBeenCalled();+ });++ it("leaves other jobs and unrelated patterns alone", async () => {+ const { queue, removeRepeatableByKey } = createQueueMock([+ legacyEntry("other-job", "*/15 * * * *", "aaa111"),+ legacyEntry("my-job", "0 3 * * *", "bbb222"),+ legacyEntry("my-job", "*/15 * * * *", "ccc333"),+ ]);++ await scheduleRecurringJob(queue, {+ jobName: "my-job",+ pattern: "*/15 * * * *",+ });++ expect(removeRepeatableByKey).toHaveBeenCalledTimes(1);+ expect(removeRepeatableByKey).toHaveBeenCalledWith("ccc333");+ });++ it("does not hash to locate legacy entries, so it works under FIPS", async () => {+ // Regression guard: Queue.removeRepeatable() recomputes an md5 digest to+ // find the entry, which throws ERR_OSSL_EVP_UNSUPPORTED when OpenSSL runs+ // in FIPS mode. The cleanup must go through the enumerate-then-remove-by-+ // key path instead, which never derives a key.+ const { queue, getRepeatableJobs, removeRepeatableByKey } = createQueueMock(+ [legacyEntry("my-job", "*/15 * * * *", "ddd444")],+ );+ (queue as unknown as { removeRepeatable: unknown }).removeRepeatable =+ vi.fn(() => {+ throw new Error("ERR_OSSL_EVP_UNSUPPORTED");+ });++ await scheduleRecurringJob(queue, {+ jobName: "my-job",+ pattern: "*/15 * * * *",+ });++ expect(getRepeatableJobs).toHaveBeenCalled();+ expect(removeRepeatableByKey).toHaveBeenCalledWith("ddd444");+ });+it("passes template data through to the scheduler", async () => {const { queue, upsertJobScheduler } = createQueueMock();@@ -74,8 +159,8 @@ describe("scheduleRecurringJob", () => {});it("still upserts the scheduler when legacy cleanup fails", async () => {- const { queue, removeRepeatable, upsertJobScheduler } = createQueueMock();- removeRepeatable.mockRejectedValue(new Error("redis down"));+ const { queue, getRepeatableJobs, upsertJobScheduler } = createQueueMock();+ getRepeatableJobs.mockRejectedValue(new Error("redis down"));await scheduleRecurringJob(queue, {jobName: "my-job",@@ -85,6 +170,25 @@ describe("scheduleRecurringJob", () => {expect(upsertJobScheduler).toHaveBeenCalledTimes(1);});+ it("keeps removing the other legacy schedules when one removal fails", async () => {+ const { queue, removeRepeatableByKey, upsertJobScheduler } =+ createQueueMock([+ legacyEntry("my-job", "*/15 * * * *", "eee555"),+ legacyEntry("my-job", "25 * * * *", "fff666"),+ ]);+ removeRepeatableByKey.mockRejectedValueOnce(new Error("redis timeout"));++ await scheduleRecurringJob(queue, {+ jobName: "my-job",+ pattern: "*/15 * * * *",+ previousPatterns: ["25 * * * *"],+ });++ expect(removeRepeatableByKey).toHaveBeenCalledWith("eee555");+ expect(removeRepeatableByKey).toHaveBeenCalledWith("fff666");+ expect(upsertJobScheduler).toHaveBeenCalledTimes(1);+ });+it("resolves instead of rejecting when the upsert fails", async () => {const { queue, upsertJobScheduler } = createQueueMock();upsertJobScheduler.mockRejectedValue(new Error("redis down"));diff --git a/packages/shared/src/server/redis/scheduleRecurringJob.ts b/packages/shared/src/server/redis/scheduleRecurringJob.tsindex ef31ecff67..941c14deb1 100644--- a/packages/shared/src/server/redis/scheduleRecurringJob.ts+++ b/packages/shared/src/server/redis/scheduleRecurringJob.ts@@ -7,7 +7,8 @@ import { logger } from "../logger";// TRIGGERS FOREVER (bullmq v6 workers keep iterating stranded md5-keyed// chains; only manual Redis cleanup stops them).// HOLD THE BULLMQ v6 UPGRADE UNTIL LANGFUSE V5: v6 deletes-// Queue.removeRepeatable(), which this cleanup requires.+// Queue.getRepeatableJobs() and Queue.removeRepeatableByKey(), which this+// cleanup requires./*** Registers a recurring cron job on a queue via a BullMQ job scheduler.@@ -23,11 +24,13 @@ import { logger } from "../logger";* entirely.** The scheduler id is the job name. Legacy schedules are keyed by- * md5(name + pattern) instead, so this helper also removes the legacy- * entries for the current pattern and any historical ones - without that- * cleanup the old chain keeps firing in parallel with the scheduler. The- * cleanup is idempotent and must stay in place for as long as a direct- * upgrade from a Langfuse version that scheduled via the legacy API is+ * md5(name:jobId:endDate:tz:pattern) instead, so this helper also removes the+ * legacy entries for the current pattern and any historical ones - without+ * that cleanup the old chain keeps firing in parallel with the scheduler. The+ * legacy keys are read back from Redis rather than recomputed, because+ * recomputing them requires md5 and that is unavailable when OpenSSL runs in+ * FIPS mode. The cleanup is idempotent and must stay in place for as long as a+ * direct upgrade from a Langfuse version that scheduled via the legacy API is* supported.** Never rejects: scheduling runs fire-and-forget from queue singleton@@ -50,17 +53,50 @@ export const scheduleRecurringJob = async (},): Promise<void> => {const { jobName, pattern, previousPatterns = [], data } = opts;+ const legacyPatterns = new Set([pattern, ...previousPatterns]);- for (const legacyPattern of [pattern, ...previousPatterns]) {- try {- // eslint-disable-next-line @typescript-eslint/no-deprecated -- Removes schedules created by the deprecated repeat API in earlier releases; keep while direct upgrades from those releases are supported.- await queue.removeRepeatable(jobName, { pattern: legacyPattern });- } catch (err) {- logger.error(- `Error removing legacy ${jobName} schedule (${legacyPattern}) on ${queue.name}`,- err,- );+ try {+ // Read the stored keys back rather than recomputing them.+ // Queue.removeRepeatable() derives md5(name:jobId:endDate:tz:pattern) to+ // locate the entry, and an OpenSSL FIPS provider refuses md5 outright, so+ // on a FIPS image that call throws ERR_OSSL_EVP_UNSUPPORTED and the+ // cleanup silently never happens - producing exactly the duplicate cron+ // triggers this file exists to prevent. Enumerating hashes nothing, and+ // removal by stored key is hash-free in the Lua script.+ // eslint-disable-next-line @typescript-eslint/no-deprecated -- legacy repeat entries are invisible to getJobSchedulers; keep while direct upgrades from legacy-scheduling releases are supported.+ const stored = await queue.getRepeatableJobs();++ const staleKeys = stored+ .filter(+ (job) =>+ job.name === jobName &&+ // The scheduler keeps its own entry in this same sorted set, keyed+ // by the scheduler id, which is the job name. Legacy entries are+ // keyed by a hash, so an entry keyed by the job name is the live+ // scheduler and must not be removed here.+ job.key !== jobName &&+ job.pattern !== null &&+ legacyPatterns.has(job.pattern),+ )+ .map((job) => job.key);++ // One failing key must not leave the remaining legacy chains firing.+ for (const key of staleKeys) {+ try {+ // eslint-disable-next-line @typescript-eslint/no-deprecated -- pairs with getRepeatableJobs above; removeJobScheduler cannot address legacy entries.+ await queue.removeRepeatableByKey(key);+ } catch (err) {+ logger.error(+ `Error removing legacy ${jobName} schedule ${key} on ${queue.name}`,+ err,+ );+ }}+ } catch (err) {+ logger.error(+ `Error listing legacy ${jobName} schedules on ${queue.name}`,+ err,+ );}try {diff --git a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.tsindex b4e1130054..fc0fecb16b 100644--- a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts+++ b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts@@ -13,13 +13,15 @@ describe("V4LegacyApiUsageQueue schedule", () => {});it("removes the legacy repeatable schedules and upserts a job scheduler", async () => {- const removeRepeatable = vi.fn().mockResolvedValue(true);+ const getRepeatableJobs = vi.fn();+ const removeRepeatableByKey = vi.fn().mockResolvedValue(true);const upsertJobScheduler = vi.fn().mockResolvedValue({});const on = vi.fn();vi.doMock("bullmq", () => ({Queue: class {- removeRepeatable = removeRepeatable;+ getRepeatableJobs = getRepeatableJobs;+ removeRepeatableByKey = removeRepeatableByKey;upsertJobScheduler = upsertJobScheduler;on = on;},@@ -43,6 +45,21 @@ describe("V4LegacyApiUsageQueue schedule", () => {await import("./v4LegacyApiUsageQueue.js");const { QueueJobs } = await import("../queues.js");+ // Legacy repeatable entries are stored under a hash of their repeat+ // options, one per pattern the job was ever scheduled with.+ getRepeatableJobs.mockResolvedValue([+ {+ key: "legacy-current-pattern",+ name: QueueJobs.V4LegacyApiUsageJob,+ pattern: V4_LEGACY_API_USAGE_CRON_PATTERN,+ },+ {+ key: "legacy-hourly-pattern",+ name: QueueJobs.V4LegacyApiUsageJob,+ pattern: "25 * * * *",+ },+ ]);+V4LegacyApiUsageQueue.getInstance();// Scheduling is fire-and-forget from getInstance, so wait for the chain.@@ -50,23 +67,20 @@ describe("V4LegacyApiUsageQueue schedule", () => {expect(upsertJobScheduler).toHaveBeenCalledTimes(1);});- // Legacy repeatable entries are keyed by md5(name + pattern), so both the- // current pattern and the pre-migration hourly pattern must be cleaned up.- expect(removeRepeatable).toHaveBeenCalledWith(- QueueJobs.V4LegacyApiUsageJob,- { pattern: V4_LEGACY_API_USAGE_CRON_PATTERN },- );- expect(removeRepeatable).toHaveBeenCalledWith(- QueueJobs.V4LegacyApiUsageJob,- { pattern: "25 * * * *" },+ // Both the current pattern and the pre-migration hourly pattern must be+ // cleaned up, by their stored keys.+ expect(removeRepeatableByKey).toHaveBeenCalledTimes(2);+ expect(removeRepeatableByKey).toHaveBeenCalledWith(+ "legacy-current-pattern",);+ expect(removeRepeatableByKey).toHaveBeenCalledWith("legacy-hourly-pattern");expect(upsertJobScheduler).toHaveBeenCalledWith(QueueJobs.V4LegacyApiUsageJob,{ pattern: V4_LEGACY_API_USAGE_CRON_PATTERN },{ name: QueueJobs.V4LegacyApiUsageJob, data: {} },);expect(V4_LEGACY_API_USAGE_CRON_PATTERN).toBe("*/15 * * * *");- for (const removeCall of removeRepeatable.mock.invocationCallOrder) {+ for (const removeCall of removeRepeatableByKey.mock.invocationCallOrder) {expect(removeCall).toBeLessThan(upsertJobScheduler.mock.invocationCallOrder[0]!,);diff --git a/packages/shared/src/server/services/StorageService.test.ts b/packages/shared/src/server/services/StorageService.test.tsindex d473c6dd96..f0652e3d6a 100644--- a/packages/shared/src/server/services/StorageService.test.ts+++ b/packages/shared/src/server/services/StorageService.test.ts@@ -1,4 +1,4 @@-import { createHash } from "crypto";+import { createHash, getFips } from "crypto";import { PassThrough, Readable } from "stream";import { afterEach, describe, expect, it, vi } from "vitest";@@ -525,27 +525,31 @@ describe("S3StorageService DeleteObjects checksum", () => {expect(findHeader(request, "content-md5")).toBeUndefined();});- it("sends Content-MD5 on DeleteObjects when the algorithm is set to MD5", async () => {- setChecksumAlgorithm("MD5");- const { service, captured } = makeServiceWithCapture(- EMPTY_DELETE_RESULT_XML,- );-- await service.deleteFiles([- "events/project-1/file-1.json",- "media/project-1/file-2.png",- ]);-- expect(captured).toHaveLength(1);- const request = captured[0];- expect(typeof request.body).toBe("string");-- const expectedMd5 = createHash("md5")- .update(request.body as string)- .digest("base64");- expect(findHeader(request, "content-md5")).toBe(expectedMd5);- expect(findHeader(request, "x-amz-checksum-crc32")).toBeUndefined();- });+ // MD5 checksums are unavailable when OpenSSL runs in FIPS mode.+ it.skipIf(getFips() === 1)(+ "sends Content-MD5 on DeleteObjects when the algorithm is set to MD5",+ async () => {+ setChecksumAlgorithm("MD5");+ const { service, captured } = makeServiceWithCapture(+ EMPTY_DELETE_RESULT_XML,+ );++ await service.deleteFiles([+ "events/project-1/file-1.json",+ "media/project-1/file-2.png",+ ]);++ expect(captured).toHaveLength(1);+ const request = captured[0];+ expect(typeof request.body).toBe("string");++ const expectedMd5 = createHash("md5")+ .update(request.body as string)+ .digest("base64");+ expect(findHeader(request, "content-md5")).toBe(expectedMd5);+ expect(findHeader(request, "x-amz-checksum-crc32")).toBeUndefined();+ },+ );});describe("S3StorageService non-buffered upload part size", () => {diff --git a/scripts/ci/fips-tests.sh b/scripts/ci/fips-tests.shnew file mode 100755index 0000000000..5f151f8cef--- /dev/null+++ b/scripts/ci/fips-tests.sh@@ -0,0 +1,121 @@+#!/usr/bin/env bash+# Runs the web or worker test suites with OpenSSL's FIPS provider active, so a+# code path that needs an algorithm a FIPS host refuses (md5, ...) fails here+# with ERR_OSSL_EVP_UNSUPPORTED instead of in a customer deployment.+#+# Runs inside the UBI toolchain stage of worker/Dockerfile: Red Hat's Node links+# the system OpenSSL, and OPENSSL_FORCE_FIPS_MODE=1 switches it into FIPS mode+# without a FIPS host kernel. The databases, migrations and env files must+# already be set up on the host, as the tests-fips CI job does.+#+# Local run, from the repo root with the dev containers up and migrated:+# docker buildx build --load --target toolchain -f worker/Dockerfile -t langfuse-fips-toolchain .+# docker run --rm --network host -e OPENSSL_FORCE_FIPS_MODE=1 -v "$PWD:/src:ro" \+# langfuse-fips-toolchain bash /src/scripts/ci/fips-tests.sh worker [vitest filters...]++set -euo pipefail++readonly SUITE="${1:?usage: fips-tests.sh web|worker [vitest filters...]}"+shift+readonly SRC_DIR="${FIPS_SOURCE_DIR:-/src}"+readonly WORK_DIR="/work"++export HUSKY=0 TURBO_TELEMETRY_DISABLED=1++group() { echo "::group::$*"; }+endgroup() { echo "::endgroup::"; }++if ! node -e 'process.exit(require("node:crypto").getFips() === 1 ? 0 : 1)'; then+ echo "The OpenSSL FIPS provider is not active; run with OPENSSL_FORCE_FIPS_MODE=1." >&2+ exit 1+fi++# Build on a private copy: the host's node_modules and build outputs were made+# for the host, not for UBI. Env files written by the host come along.+group "Copy sources"+mkdir -p "$WORK_DIR"+tar -C "$SRC_DIR" --exclude=.git --exclude=node_modules --exclude=.next \+ --exclude=.turbo --exclude=dist --exclude=target -cf - . | tar -C "$WORK_DIR" -xf -+cd "$WORK_DIR"+endgroup++group "Install dependencies"+pnpm install --frozen-lockfile ${PNPM_STORE_DIR:+--store-dir "$PNPM_STORE_DIR"}+pnpm --filter=shared run db:generate+endgroup++wait_for_health() {+ local url="$1" pid="$2"+ for _ in $(seq 1 150); do+ if curl --silent --fail --output /dev/null "$url"; then+ return 0+ fi+ if ! kill -0 "$pid" 2>/dev/null; then+ echo "Langfuse exited before $url became healthy:" >&2+ tail -n 100 /tmp/langfuse-start.log >&2+ return 1+ fi+ sleep 2+ done+ echo "Timed out waiting for $url:" >&2+ tail -n 100 /tmp/langfuse-start.log >&2+ return 1+}++case "$SUITE" in+ worker)+ group "Build worker"+ pnpm --filter=worker... run build+ endgroup++ NODE_ENV="test" pnpm --filter @langfuse/shared run test --passWithNoTests "$@"+ pnpm --filter=worker run test:exclude-llm-connections --passWithNoTests "$@"+ ;;++ web)+ # Mirrors tests-web: compile once, finalize for the runtime env, then run+ # web and worker so the server tests can call them.+ export NODE_ENV=test+ group "Build web and worker"+ NEXT_DISABLE_BUILD_CACHE=true NODE_OPTIONS=--max_old_space_size=8192 \+ NEXT_IGNORE_BUILD_ERRORS=true pnpm turbo run build:test+ cp .env.runtime .env+ NEXT_DISABLE_BUILD_CACHE=true NODE_OPTIONS=--max_old_space_size=8192 \+ NEXT_IGNORE_BUILD_ERRORS=true \+ pnpm --filter web exec dotenv -e ../.env -- next build --experimental-build-mode generate+ pnpm --filter web run build:otel-worker+ endgroup++ group "Start Langfuse"+ # Docker sets HOSTNAME to the container's host name, and the worker binds+ # to HOSTNAME; bind all interfaces so localhost reaches it.+ HOSTNAME="0.0.0.0" \+ LANGFUSE_INIT_ORG_ID="seed-org-id" \+ LANGFUSE_INIT_ORG_NAME="Seed Org" \+ LANGFUSE_INIT_ORG_CLOUD_PLAN="Team" \+ LANGFUSE_INIT_PROJECT_ID="7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" \+ LANGFUSE_INIT_PROJECT_NAME="Seed Project" \+ LANGFUSE_INIT_PROJECT_PUBLIC_KEY="pk-lf-1234567890" \+ LANGFUSE_INIT_PROJECT_SECRET_KEY="sk-lf-1234567890" \+ LANGFUSE_INIT_USER_EMAIL="[email protected]" \+ LANGFUSE_INIT_USER_NAME="Demo User" \+ LANGFUSE_INIT_USER_PASSWORD="password" \+ pnpm run start > /tmp/langfuse-start.log 2>&1 &+ start_pid=$!+ wait_for_health http://localhost:3000/api/public/health "$start_pid"+ wait_for_health http://localhost:3030/api/health "$start_pid"+ endgroup++ cd web+ NODE_COMPILE_CACHE=/tmp/node-compile-cache npx dotenv -e ../.env.test -e ../.env -- \+ vitest run --project server --project server-isolated \+ --project server-shared-source --project server-unit \+ --project server-shared-source-unit \+ --maxWorkers="${VITEST_MAX_WORKERS:-8}" --passWithNoTests "$@"+ ;;++ *)+ echo "Unknown suite: $SUITE (expected web or worker)" >&2+ exit 1+ ;;+esacdiff --git a/scripts/ci/write-web-test-env.sh b/scripts/ci/write-web-test-env.shnew file mode 100755index 0000000000..cd9735927b--- /dev/null+++ b/scripts/ci/write-web-test-env.sh@@ -0,0 +1,21 @@+#!/usr/bin/env bash+# Writes the env file the web server tests run with: a dev env example plus+# the test overrides. Used by the tests-web and tests-fips CI jobs.++set -euo pipefail++readonly SOURCE="${1:?usage: write-web-test-env.sh <env example> <output>}"+readonly OUTPUT="${2:?usage: write-web-test-env.sh <env example> <output>}"++{+ grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' "$SOURCE"+ echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1"+ echo "LANGFUSE_CACHE_PROMPT_ENABLED=false"+ echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1"+ echo "LANGFUSE_TRACE_DELETE_DELAY_MS=1"+ echo "LANGFUSE_TRACE_DELETE_CONCURRENCY=100"+ echo "ADMIN_API_KEY=admin-api-key"+ echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test"+ echo "LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION=true"+ echo "LANGFUSE_ENABLE_SCORES_V3_API=true"+} > "$OUTPUT"diff --git a/web/Dockerfile b/web/Dockerfileindex 19e246d35c..5eaa232d18 100644--- a/web/Dockerfile+++ b/web/Dockerfile@@ -1,27 +1,111 @@-# The node alpine image is available here: https://github.com/nodejs/docker-node-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:24-alpine AS alpine--# It's important to update the index before installing packages to ensure you're getting the latest versions.-# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.-RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client zlib--FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS build-base+# FIPS 140-3 base: UBI 9 with the nodejs:24 module-stream RPM+# instead of node:24-alpine. The RPM Node dynamically links the system+# libcrypto.so.3, so node:crypto routes through RHEL's CMVP-validated OpenSSL+# FIPS provider (/usr/lib64/ossl-modules/fips.so). Upstream node images+# statically bundle their own non-validated OpenSSL and bypass the platform+# module entirely.+# The defaults are Red Hat's public UBI9 images, so the build needs no+# credentials. Iron Bank serves the same content from an authenticated registry+# with DoD provenance and scan attestations. Release builds opt in by overriding+# both args after `docker login registry1.dso.mil` (free P1 account plus a CLI+# secret):+# --build-arg BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8+# --build-arg MICRO_BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8+ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.8+ARG MICRO_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi-micro:9.8++FROM --platform=${TARGETPLATFORM:-linux/amd64} ${BASE_IMAGE} AS base++# nodejs:24 and openssl (incl. the FIPS provider) come from the freely+# redistributable UBI repos, so no RHEL entitlement is required.+# nodejs-full-i18n is required: the RPM Node is a small-ICU build and V8+# segfaults in Intl.Segmenter (used e.g. by the prisma CLI's output+# formatting) without the full ICU data.+RUN dnf -y upgrade && \+ dnf -y module enable nodejs:24 && \+ dnf -y install --setopt=install_weak_deps=0 nodejs npm nodejs-full-i18n && \+ dnf clean all++FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS build-base# Pin turbo to avoid nondeterministic prune output from future patch releases.-RUN npm install [email protected] --global-ENV PNPM_HOME="/pnpm"-ENV PATH="$PNPM_HOME:$PATH"+# pnpm is installed pinned via npm instead of corepack: the corepack shim would+# fetch pnpm from an unpinned CDN on first use.+RUN npm install -g [email protected] [email protected]# Pass --build-arg NODE_USE_ENV_PROXY=1 when building behind a proxy.ARG NODE_USE_ENV_PROXY-RUN npm install --global [email protected] \- && corepack enable \- && corepack prepare [email protected] --activate-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS runtime-base-# Remove build-only package managers. npm stays here because the runner stage-# uses it to install prisma and optional dd-trace before removing it.-RUN rm -rf /usr/local/lib/node_modules/corepack \- /root/.cache/node/corepack && \- rm -f /usr/local/bin/corepack /usr/local/bin/yarn /usr/local/bin/yarnpkg+# Runtime rootfs assembled with dnf --installroot and layered onto ubi9-micro:+# the final image carries no package manager, python, or curl. It holds only+# the packages listed below plus their deps, with an intact rpmdb for scanners.+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS rpm-installer++ARG TARGETARCH+ARG UID=1001+ARG GID=1001++# nodejs-full-i18n is required: the RPM Node is a small-ICU build and V8+# segfaults in Intl.Segmenter (used e.g. by the prisma CLI's output+# formatting) without the full ICU data. sed is used by entrypoint.sh.+# wget keeps exec-style container health checks working (e.g. `wget --spider+# http://127.0.0.1:<port>/...` in ECS task definitions); it links GnuTLS,+# which follows the host's FIPS mode like OpenSSL does.+RUN dnf -y --installroot /mnt/rootfs --releasever 9 --setopt=install_weak_deps=0 \+ --disableexcludes=all \+ module enable nodejs:24 && \+ dnf -y --installroot /mnt/rootfs --releasever 9 \+ --setopt=install_weak_deps=0 --setopt=tsflags=nodocs \+ --disableexcludes=all \+ install nodejs nodejs-full-i18n bash coreutils-single sed \+ tzdata ca-certificates crypto-policies wget && \+ dnf --installroot /mnt/rootfs clean all && \+ rm -rf /mnt/rootfs/var/cache /mnt/rootfs/var/log/dnf*++# The nodejs RPM drags npm in as a dependency; the runtime image must carry+# no package manager, so remove it from the rootfs (rpmdb stays consistent).+RUN if rpm --root /mnt/rootfs -q npm >/dev/null 2>&1; then \+ rpm --root /mnt/rootfs -e --nodeps npm; \+ fi++# The FIPS crypto policy for OpenSSL ships as an opt-in config instead of+# being activated system-wide, so installs without FIPS mode keep the default+# TLS policy. entrypoint.sh points OPENSSL_CONF at it when+# LANGFUSE_REQUIRE_FIPS=true; deployments that override the entrypoint set+# OPENSSL_CONF=/etc/pki/tls/openssl-fips.cnf themselves. The FIPS provider+# itself loads only when the host kernel runs in FIPS mode; with+# LANGFUSE_REQUIRE_FIPS=true, startup fails unless it is active and an+# enterprise license is configured.+RUN sed 's#/etc/crypto-policies/back-ends/opensslcnf.config#/usr/share/crypto-policies/FIPS/opensslcnf.txt#' \+ /mnt/rootfs/etc/pki/tls/openssl.cnf > /mnt/rootfs/etc/pki/tls/openssl-fips.cnf && \+ grep -q '/usr/share/crypto-policies/FIPS/opensslcnf.txt' /mnt/rootfs/etc/pki/tls/openssl-fips.cnf++# dumb-init is not packaged in UBI; install the pinned, checksum-verified+# static release binary (preserves the semver-covered ENTRYPOINT contract).+RUN set -eux; \+ case "${TARGETARCH:-amd64}" in \+ amd64) di_arch=x86_64; di_sha256=e874b55f3279ca41415d290c512a7ba9d08f98041b28ae7c2acb19a545f1c4df ;; \+ arm64) di_arch=aarch64; di_sha256=b7d648f97154a99c539b63c55979cd29f005f88430fb383007fe3458340b795e ;; \+ *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \+ esac; \+ curl -fsSL -o /mnt/rootfs/usr/bin/dumb-init "https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_${di_arch}"; \+ echo "${di_sha256} /mnt/rootfs/usr/bin/dumb-init" | sha256sum -c -; \+ chmod 0755 /mnt/rootfs/usr/bin/dumb-init++# Bake the non-root user into the rootfs (micro has no shadow-utils).+RUN echo "nodejs:x:${GID}:" >> /mnt/rootfs/etc/group && \+ echo "nextjs:x:${UID}:${GID}::/home/nextjs:/sbin/nologin" >> /mnt/rootfs/etc/passwd++FROM --platform=${TARGETPLATFORM:-linux/amd64} ${MICRO_BASE_IMAGE} AS runtime-base+COPY --from=rpm-installer /mnt/rootfs /++# prisma CLI + optional dd-trace are npm-installed here and copied into the+# package-manager-less runtime image.+FROM --platform=${TARGETPLATFORM:-linux/amd64} build-base AS web-tools+RUN npm install -g --prefix /opt/prisma-cli --no-package-lock --no-save [email protected]+ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION+RUN mkdir -p /opt/dd/node_modules && \+ if [ -n "$NEXT_PUBLIC_LANGFUSE_CLOUD_REGION" ]; then \+ cd /opt/dd && npm install --no-package-lock --no-save [email protected]; \+ fiFROM --platform=${BUILDPLATFORM} golang:1.26 AS migrate-builderARG TARGETOS@@ -30,6 +114,10 @@ ENV CGO_ENABLED=0 \GOBIN=/out \GOOS=${TARGETOS} \GOARCH=${TARGETARCH}+# Build against the validated Go Cryptographic Module snapshot (Go >= 1.24) so+# the migrate CLI's ClickHouse TLS uses FIPS-approved crypto; GOFIPS140 also+# defaults GODEBUG=fips140=on at runtime.+ENV GOFIPS140=v1.0.0# Build only the ClickHouse migrate CLI Langfuse uses at runtime.# compile this ourselves instead of downloading the upstream release# because prebuilt bins bundle many unused drivers and thus inherit CVEs@@ -164,25 +252,17 @@ ENV DOCKER_BUILD 0# Set NEXT_MANUAL_SIG_HANDLE for runtimeENV NEXT_MANUAL_SIG_HANDLE true-RUN apk add --no-cache dumb-init tzdata+# The non-root user (nextjs, uid 1001) is baked into the runtime rootfs by+# the rpm-installer stage, since micro has no shadow-utils.-# Don't run production as root-ARG UID=1001-ARG GID=1001-RUN addgroup --system --gid ${GID} nodejs-RUN adduser --system --uid ${UID} nextjs+# prisma CLI from the tools stage (bin/prisma symlink + lib/node_modules+# structure lands under /usr/local); the runtime image has no npm.+COPY --from=web-tools /opt/prisma-cli /usr/local-RUN npm install -g --no-package-lock --no-save [email protected]--# Install dd-trace only if NEXT_PUBLIC_LANGFUSE_CLOUD_REGION is configured-ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION-RUN if [ -n "$NEXT_PUBLIC_LANGFUSE_CLOUD_REGION" ]; then \- npm install --no-package-lock --no-save [email protected]; \- fi--# npm is only used for the installs above; remove it from the final runtime image.-RUN rm -rf /usr/local/lib/node_modules/npm && \- rm -f /usr/local/bin/npm /usr/local/bin/npx+# dd-trace, npm-installed in the tools stage (node_modules is empty unless+# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION was set); the standalone output copied+# below merges on top of it.+COPY --from=web-tools --chown=nextjs:nodejs /opt/dd/node_modules ./node_modulesCOPY --from=migrate-builder /out/migrate /usr/bin/migrate@@ -202,7 +282,10 @@ COPY --chown=nextjs:nodejs ./web/entrypoint.sh ./web/entrypoint.shCOPY --chown=nextjs:nodejs ./packages/shared/scripts/cleanup.sql ./packages/shared/scripts/cleanup.sqlRUN chmod +x ./web/entrypoint.sh-USER nextjs+# Numeric USER (uid of nextjs) so runAsNonRoot checks and Iron Bank hardening+# validation can verify the user without resolving names.+ARG UID=1001+USER ${UID}# Default port to 3000ENV PORT 3000diff --git a/web/entrypoint.sh b/web/entrypoint.shindex aaf947916d..e2a3204b74 100644--- a/web/entrypoint.sh+++ b/web/entrypoint.sh@@ -1,5 +1,20 @@#!/bin/sh+# Fail closed when FIPS mode is required but the OpenSSL FIPS provider is not+# active for Node (e.g. a host without FIPS mode, or an image rebuilt on a+# non-FIPS base). This runs before the migrations below open database+# connections. The app additionally requires an enterprise license for FIPS+# mode at startup (packages/shared/src/server/ee/fips).+if [ "$LANGFUSE_REQUIRE_FIPS" = "true" ]; then+ # Opt into the FIPS crypto policy for OpenSSL (see the Dockerfile). An+ # OPENSSL_CONF set by the deployment takes precedence.+ export OPENSSL_CONF="${OPENSSL_CONF:-/etc/pki/tls/openssl-fips.cnf}"+ if ! node -e 'process.exit(require("node:crypto").getFips() === 1 ? 0 : 1)'; then+ echo "Error: LANGFUSE_REQUIRE_FIPS=true but Node's OpenSSL FIPS provider is not active (crypto.getFips() != 1). Exiting..."+ exit 1+ fi+fi+# Check whether a database URL's credentials contain characters that typically# need percent-encoding for Prisma (@ : / % # ?). Best-effort heuristic —# strips the scheme, extracts the authority (user:pass@host) before the firstdiff --git a/web/src/instrumentation.ts b/web/src/instrumentation.tsindex 5d7a17fefd..5ef2f2d0a3 100644--- a/web/src/instrumentation.ts+++ b/web/src/instrumentation.ts@@ -9,6 +9,19 @@ export async function register() {const isNodeRuntime = process.env.NEXT_RUNTIME === "nodejs";+ // Enforce LANGFUSE_REQUIRE_FIPS before any init script connects, also on+ // secondary replicas that skip init and when entrypoint.sh is overridden.+ if (isNodeRuntime) {+ const { assertFipsMode } =+ await import("@langfuse/shared/src/server/ee/fips");+ try {+ assertFipsMode();+ } catch (error) {+ console.error(error instanceof Error ? error.message : String(error));+ process.exit(1);+ }+ }+if (isNodeRuntime && isInitLoadingEnabled) {console.log("Running init scripts...");await import("./observability.config");diff --git a/worker/Dockerfile b/worker/Dockerfileindex 56c523d1ab..c0a4d011b4 100644--- a/worker/Dockerfile+++ b/worker/Dockerfile@@ -1,26 +1,101 @@-# The node alpine image is available here: https://github.com/nodejs/docker-node-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:24-alpine AS alpine--# It's important to update the index before installing packages to ensure you're getting the latest versions.-# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.-RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client zlib--FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS build-base+# FIPS 140-3 base: UBI 9 with the nodejs:24 module-stream RPM+# instead of node:24-alpine. The RPM Node dynamically links the system+# libcrypto.so.3, so node:crypto routes through RHEL's CMVP-validated OpenSSL+# FIPS provider (/usr/lib64/ossl-modules/fips.so). Upstream node images+# statically bundle their own non-validated OpenSSL and bypass the platform+# module entirely.+# The defaults are Red Hat's public UBI9 images, so the build needs no+# credentials. Iron Bank serves the same content from an authenticated registry+# with DoD provenance and scan attestations. Release builds opt in by overriding+# both args after `docker login registry1.dso.mil` (free P1 account plus a CLI+# secret):+# --build-arg BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8+# --build-arg MICRO_BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8+ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.8+ARG MICRO_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi-micro:9.8++FROM --platform=${TARGETPLATFORM:-linux/amd64} ${BASE_IMAGE} AS base++# nodejs:24 and openssl (incl. the FIPS provider) come from the freely+# redistributable UBI repos, so no RHEL entitlement is required.+# nodejs-full-i18n is required: the RPM Node is a small-ICU build and V8+# segfaults in Intl.Segmenter (used e.g. by the prisma CLI's output+# formatting) without the full ICU data.+RUN dnf -y upgrade && \+ dnf -y module enable nodejs:24 && \+ dnf -y install --setopt=install_weak_deps=0 nodejs npm nodejs-full-i18n && \+ dnf clean all++FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS build-base# Pin turbo to avoid nondeterministic prune output from future patch releases.-RUN npm install [email protected] --global-ENV PNPM_HOME="/pnpm"-ENV PATH="$PNPM_HOME:$PATH"+# pnpm is installed pinned via npm instead of corepack: the corepack shim would+# fetch pnpm from an unpinned CDN on first use.+RUN npm install -g [email protected] [email protected]# Pass --build-arg NODE_USE_ENV_PROXY=1 when building behind a proxy.ARG NODE_USE_ENV_PROXY-RUN npm install --global [email protected] \- && corepack enable \- && corepack prepare [email protected] --activate-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS runtime-base-# package managers and build-only CLIs only increase exposure to CVEs -> remove them-RUN rm -rf /usr/local/lib/node_modules/corepack /usr/local/lib/node_modules/npm \- /root/.cache/node/corepack && \- rm -f /usr/local/bin/corepack /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/yarn /usr/local/bin/yarnpkg+# Runtime rootfs assembled with dnf --installroot and layered onto ubi9-micro:+# the final image carries no package manager, python, or curl. It holds only+# the packages listed below plus their deps, with an intact rpmdb for scanners.+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS rpm-installer++ARG TARGETARCH+ARG UID=1001+ARG GID=1001++# nodejs-full-i18n is required: the RPM Node is a small-ICU build and V8+# segfaults in Intl.Segmenter (used e.g. by the prisma CLI's output+# formatting) without the full ICU data.+# wget keeps exec-style container health checks working (e.g. `wget --spider+# http://127.0.0.1:<port>/...` in ECS task definitions); it links GnuTLS,+# which follows the host's FIPS mode like OpenSSL does.+RUN dnf -y --installroot /mnt/rootfs --releasever 9 --setopt=install_weak_deps=0 \+ --disableexcludes=all \+ module enable nodejs:24 && \+ dnf -y --installroot /mnt/rootfs --releasever 9 \+ --setopt=install_weak_deps=0 --setopt=tsflags=nodocs \+ --disableexcludes=all \+ install nodejs nodejs-full-i18n bash coreutils-single sed \+ tzdata ca-certificates crypto-policies wget && \+ dnf --installroot /mnt/rootfs clean all && \+ rm -rf /mnt/rootfs/var/cache /mnt/rootfs/var/log/dnf*++# The nodejs RPM drags npm in as a dependency; the runtime image must carry+# no package manager, so remove it from the rootfs (rpmdb stays consistent).+RUN if rpm --root /mnt/rootfs -q npm >/dev/null 2>&1; then \+ rpm --root /mnt/rootfs -e --nodeps npm; \+ fi++# The FIPS crypto policy for OpenSSL ships as an opt-in config instead of+# being activated system-wide, so installs without FIPS mode keep the default+# TLS policy. entrypoint.sh points OPENSSL_CONF at it when+# LANGFUSE_REQUIRE_FIPS=true; deployments that override the entrypoint set+# OPENSSL_CONF=/etc/pki/tls/openssl-fips.cnf themselves. The FIPS provider+# itself loads only when the host kernel runs in FIPS mode; with+# LANGFUSE_REQUIRE_FIPS=true, startup fails unless it is active and an+# enterprise license is configured.+RUN sed 's#/etc/crypto-policies/back-ends/opensslcnf.config#/usr/share/crypto-policies/FIPS/opensslcnf.txt#' \+ /mnt/rootfs/etc/pki/tls/openssl.cnf > /mnt/rootfs/etc/pki/tls/openssl-fips.cnf && \+ grep -q '/usr/share/crypto-policies/FIPS/opensslcnf.txt' /mnt/rootfs/etc/pki/tls/openssl-fips.cnf++# dumb-init is not packaged in UBI; install the pinned, checksum-verified+# static release binary (preserves the semver-covered ENTRYPOINT contract).+RUN set -eux; \+ case "${TARGETARCH:-amd64}" in \+ amd64) di_arch=x86_64; di_sha256=e874b55f3279ca41415d290c512a7ba9d08f98041b28ae7c2acb19a545f1c4df ;; \+ arm64) di_arch=aarch64; di_sha256=b7d648f97154a99c539b63c55979cd29f005f88430fb383007fe3458340b795e ;; \+ *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \+ esac; \+ curl -fsSL -o /mnt/rootfs/usr/bin/dumb-init "https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_${di_arch}"; \+ echo "${di_sha256} /mnt/rootfs/usr/bin/dumb-init" | sha256sum -c -; \+ chmod 0755 /mnt/rootfs/usr/bin/dumb-init++# Bake the non-root user into the rootfs (micro has no shadow-utils).+RUN echo "expressjs:x:${GID}:" >> /mnt/rootfs/etc/group && \+ echo "expressjs:x:${UID}:${GID}::/home/expressjs:/sbin/nologin" >> /mnt/rootfs/etc/passwd++FROM --platform=${TARGETPLATFORM:-linux/amd64} ${MICRO_BASE_IMAGE} AS runtime-base+COPY --from=rpm-installer /mnt/rootfs /FROM --platform=${TARGETPLATFORM:-linux/amd64} build-base AS pruner@@ -29,24 +104,42 @@ WORKDIR /appCOPY . .RUN turbo prune worker --docker-FROM --platform=${TARGETPLATFORM:-linux/amd64} build-base AS builder+# UBI build toolchain: RPM Node, pnpm, turbo, gcc and Rust. CI's tests-fips job+# also runs the web and worker test suites on this stage with the OpenSSL FIPS+# provider forced on (scripts/ci/fips-tests.sh).+FROM --platform=${TARGETPLATFORM:-linux/amd64} build-base AS toolchain++ARG TARGETARCH++# @langfuse/native is a Rust addon (napi-rs) compiled by `turbo run build` for+# the glibc (`*-unknown-linux-gnu`) target, linked against UBI9's glibc so it+# loads in the runtime image. gcc provides the linker; tar and gzip unpack the+# pinned toolchain; patch (with the base image's curl-minimal) prepares the+# small clickhouse source overlay used by the native encoder.+RUN dnf -y install --setopt=install_weak_deps=0 gcc tar gzip patch && \+ dnf clean all++# Pinned standalone Rust installer, the exact channel from+# packages/native/rust-toolchain.toml (bump the two together), checksum-verified+# like the dumb-init binary above. Only rustc, cargo and the target's std are+# needed to build; clippy and rustfmt run outside Docker.+RUN set -eux; \+ case "${TARGETARCH:-amd64}" in \+ amd64) triple=x86_64-unknown-linux-gnu; toolchain_sha256=aa30409afa67bd1ada244cefd82c7980e6a65bc113bb978e934b2413c75e3900 ;; \+ arm64) triple=aarch64-unknown-linux-gnu; toolchain_sha256=5fbb4282403046d52a4672765c6761a809bf9f33e699b17e6eb7a93ab7770cc3 ;; \+ *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \+ esac; \+ curl -fsSL -o /tmp/rust.tar.gz "https://static.rust-lang.org/dist/rust-1.98.0-${triple}.tar.gz"; \+ echo "${toolchain_sha256} /tmp/rust.tar.gz" | sha256sum -c -; \+ tar -xzf /tmp/rust.tar.gz -C /tmp; \+ /tmp/rust-1.98.0-${triple}/install.sh --prefix=/usr/local \+ --components=rustc,cargo,rust-std-${triple}; \+ rm -rf /tmp/rust.tar.gz /tmp/rust-1.98.0-${triple}++FROM --platform=${TARGETPLATFORM:-linux/amd64} toolchain AS builderWORKDIR /app-# @langfuse/native is a Rust addon (napi-rs) compiled by `turbo run build`.-# The toolchain comes from the official Rust image for the same Alpine release,-# so the addon compiles with the exact version pinned in-# packages/native/rust-toolchain.toml; bump the two together. gcc and musl-dev-# provide the linker and C runtime objects the build needs; curl and patch-# prepare the small clickhouse source overlay used by the native encoder.-COPY --from=rust:1.98.0-alpine3.24 /usr/local/rustup /usr/local/rustup-COPY --from=rust:1.98.0-alpine3.24 /usr/local/cargo /usr/local/cargo-ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo PATH=/usr/local/cargo/bin:$PATH-# rustc's musl targets default to a `<arch>-linux-musl-gcc` linker name that-# Alpine does not ship; its gcc is reachable as `cc`.-ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=cc CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=cc-RUN apk add --no-cache gcc musl-dev curl patch-# First install the dependencies (as they change less often)COPY --from=pruner /app/out/pnpm-lock.yaml ./pnpm-lock.yamlCOPY --from=pruner /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml@@ -94,8 +187,6 @@ FROM --platform=${TARGETPLATFORM:-linux/amd64} runtime-base AS runnerARG TARGETPLATFORMARG BUILDPLATFORM-RUN apk add --no-cache dumb-init-WORKDIR /appARG NEXT_PUBLIC_BUILD_ID@@ -104,16 +195,16 @@ ENV BUILD_ID=$NEXT_PUBLIC_BUILD_IDENV NODE_ENV productionENV DOCKER_BUILD 0-# Don't run production as root-ARG UID=1001-ARG GID=1001-RUN addgroup --system --gid ${GID} expressjs-RUN adduser --system --uid ${UID} expressjs+# The non-root user (expressjs, uid 1001) is baked into the runtime rootfs by+# the rpm-installer stage, since micro has no shadow-utils.# Copy only production worker payload instead of full builder workspace (just /prod/worker not entire /app)COPY --from=prod-deps --chown=expressjs:expressjs /prod/worker ./workerRUN chmod +x ./worker/entrypoint.sh-USER expressjs+# Numeric USER (uid of expressjs) so runAsNonRoot checks and Iron Bank+# hardening validation can verify the user without resolving names.+ARG UID=1001+USER ${UID}EXPOSE 3030ENV PORT=3030diff --git a/worker/entrypoint.sh b/worker/entrypoint.shindex 6c6cbce9ff..081e8bcf41 100644--- a/worker/entrypoint.sh+++ b/worker/entrypoint.sh@@ -1,5 +1,20 @@#!/bin/sh+# Fail closed when FIPS mode is required but the OpenSSL FIPS provider is not+# active for Node (e.g. a host without FIPS mode, or an image rebuilt on a+# non-FIPS base). This runs before the migrations below open database+# connections. The app additionally requires an enterprise license for FIPS+# mode at startup (packages/shared/src/server/ee/fips).+if [ "$LANGFUSE_REQUIRE_FIPS" = "true" ]; then+ # Opt into the FIPS crypto policy for OpenSSL (see the Dockerfile). An+ # OPENSSL_CONF set by the deployment takes precedence.+ export OPENSSL_CONF="${OPENSSL_CONF:-/etc/pki/tls/openssl-fips.cnf}"+ if ! node -e 'process.exit(require("node:crypto").getFips() === 1 ? 0 : 1)'; then+ echo "Error: LANGFUSE_REQUIRE_FIPS=true but Node's OpenSSL FIPS provider is not active (crypto.getFips() != 1). Exiting..."+ exit 1+ fi+fi+# Run cleanup script before running migrations# Check if DATABASE_URL is not setif [ -z "$DATABASE_URL" ]; thendiff --git a/worker/src/__tests__/bullmqVersionCheckOptions.test.ts b/worker/src/__tests__/bullmqVersionCheckOptions.test.tsindex 3840c0d1ae..4f1f2fefe6 100644--- a/worker/src/__tests__/bullmqVersionCheckOptions.test.ts+++ b/worker/src/__tests__/bullmqVersionCheckOptions.test.ts@@ -124,6 +124,25 @@ describe("BullMQ Redis version check options", () => {});});+ test("hashes repeatable-job keys with a FIPS-approved algorithm", async () => {+ const { redisModule } = await importRedisModule();++ // bullmq defaults repeatKeyHashAlgorithm to md5, which an OpenSSL FIPS+ // provider refuses. Without this setting the legacy-schedule cleanup in+ // scheduleRecurringJob throws ERR_OSSL_EVP_UNSUPPORTED on a FIPS image,+ // leaving stale md5-keyed schedules firing alongside the job scheduler.+ expect(+ redisModule.createBullMQQueueOptionsWithRedis("test-queue"),+ ).toMatchObject({+ settings: { repeatKeyHashAlgorithm: "sha256" },+ });+ expect(+ redisModule.createBullMQWorkerOptionsWithRedis("test-queue"),+ ).toMatchObject({+ settings: { repeatKeyHashAlgorithm: "sha256" },+ });+ });+test("uses worker Redis retry options without disabling offline queue", async () => {const { redisInstances, redisModule } = await importRedisModule();const options =diff --git a/worker/src/ee/fipsMode.ts b/worker/src/ee/fipsMode.tsnew file mode 100644index 0000000000..ebd164a080--- /dev/null+++ b/worker/src/ee/fipsMode.ts@@ -0,0 +1,11 @@+import { assertFipsMode } from "@langfuse/shared/src/server/ee/fips";++// Side-effect module: index.ts imports it before anything that connects to+// Redis, Postgres or ClickHouse, so FIPS mode also holds when a deployment+// overrides the image's entrypoint.sh.+try {+ assertFipsMode();+} catch (error) {+ console.error(error instanceof Error ? error.message : String(error));+ process.exit(1);+}diff --git a/worker/src/features/batch-data-retention-cleaner/index.ts b/worker/src/features/batch-data-retention-cleaner/index.tsindex 0b86a96254..0d39b89ab4 100644--- a/worker/src/features/batch-data-retention-cleaner/index.ts+++ b/worker/src/features/batch-data-retention-cleaner/index.ts@@ -89,9 +89,11 @@ function summarizeBacklog(workloads: ProjectWorkload[]) {/*** Hash projectId to a short key for ClickHouse parameter names.+ * sha256 rather than md5: an OpenSSL FIPS provider refuses md5. The key only+ * names query parameters within one query, so it is never persisted.*/function toParamKey(projectId: string): string {- return createHash("md5").update(projectId).digest("hex").slice(0, 8);+ return createHash("sha256").update(projectId).digest("hex").slice(0, 8);}/**diff --git a/worker/src/index.ts b/worker/src/index.tsindex 463ef272f5..5dd5112e1d 100644--- a/worker/src/index.ts+++ b/worker/src/index.ts@@ -1,4 +1,5 @@import "./instrumentation"; // instrumenting the application+import "./ee/fipsMode"; // enforce LANGFUSE_REQUIRE_FIPS before anything connectsimport type { Server } from "http";import { initializeWorker } from "./initialize";import { env } from "./env";