Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(docker): create zero vuln base images and add fips mode (#18009)

Steffen Reutherbb1b6645182e
diff --git a/.github/workflows/_deploy_ecs_service.yml b/.github/workflows/_deploy_ecs_service.yml
index bb07fe7cd7..55ad829026 100644
--- a/.github/workflows/_deploy_ecs_service.yml
+++ b/.github/workflows/_deploy_ecs_service.yml
@@ -19,6 +19,10 @@ on:
required: true
SENTRY_AUTH_TOKEN:
required: false
+ IRONBANK_USER:
+ required: true
+ IRONBANK_PASS:
+ required: true
jobs:
ecs-deploy:
if: ${{ inputs.service != 'ai-gateway' || inputs.environment == 'staging' }}
@@ -26,6 +30,10 @@ jobs:
environment: ${{ inputs.environment }}
permissions:
contents: read
+ env:
+ # Iron Bank UBI 9 bases, the same ones pipeline.yml uses for release images.
+ BASE_IMAGE: registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8
+ MICRO_BASE_IMAGE: registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8
steps:
- name: Get app name
uses: winterjung/split@a211a1c46e35fcdc4097d59dd6282d4a9859651b # v2
@@ -48,6 +56,12 @@ jobs:
- name: Login to AWS ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2.1.6
+ - name: Log in to Iron Bank
+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+ with:
+ registry: registry1.dso.mil
+ username: ${{ secrets.IRONBANK_USER }}
+ password: ${{ secrets.IRONBANK_PASS }}
- name: Build, tag, and push AI Gateway image
if: ${{ inputs.service == 'ai-gateway' }}
env:
@@ -59,6 +73,8 @@ jobs:
--platform linux/amd64 \
--target runtime \
--build-arg BUILD_ID="$IMAGE_TAG" \
+ --build-arg BASE_IMAGE="$BASE_IMAGE" \
+ --build-arg MICRO_BASE_IMAGE="$MICRO_BASE_IMAGE" \
-t "$REGISTRY/ai-gateway:$IMAGE_TAG" \
./ai-gateway
docker push "$REGISTRY/ai-gateway:$IMAGE_TAG"
@@ -87,6 +103,8 @@ jobs:
docker build \
-t $REGISTRY/$REPOSITORY:$IMAGE_TAG \
-f ./${STEPS_SPLIT_OUTPUTS__0}/Dockerfile \
+ --build-arg BASE_IMAGE="$BASE_IMAGE" \
+ --build-arg MICRO_BASE_IMAGE="$MICRO_BASE_IMAGE" \
--build-arg NEXT_PUBLIC_LANGFUSE_CLOUD_REGION=${VARS_NEXT_PUBLIC_LANGFUSE_CLOUD_REGION} \
--build-arg NEXT_LANGFUSE_TRACING_SAMPLE_RATE=${VARS_NEXT_LANGFUSE_TRACING_SAMPLE_RATE} \
--build-arg NEXT_PUBLIC_SENTRY_ENVIRONMENT=${VARS_NEXT_PUBLIC_SENTRY_ENVIRONMENT} \
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index b58d9a9271..0409aa82a4 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -117,6 +117,8 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+ IRONBANK_USER: ${{ secrets.IRONBANK_USER }}
+ IRONBANK_PASS: ${{ secrets.IRONBANK_PASS }}
strategy:
matrix:
service: ${{ fromJson(needs.affected-services.outputs.services) }}
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
index 29e68e6951..64cf16d61d 100644
--- a/.github/workflows/pipeline.yml
+++ b/.github/workflows/pipeline.yml
@@ -643,6 +643,42 @@ jobs:
with:
username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
+ # Iron Bank needs a registry1.dso.mil account, and those credentials are
+ # not available to forks or to pull_request runs from a fork. Fall back
+ # to the Dockerfile default, Red Hat's public UBI9, which has the same
+ # content and the same FIPS-validated OpenSSL.
+ - name: Resolve FIPS base images
+ id: bases
+ env:
+ IRONBANK_USER: ${{ secrets.IRONBANK_USER }}
+ # True on runs that execute in this repo's own context, which is
+ # where the Registry1 secrets exist: push, merge_group, tags, and
+ # PRs raised from a branch here rather than from a fork.
+ SECRETS_EXPECTED: ${{ github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }}
+ run: |
+ if [ -n "$IRONBANK_USER" ]; then
+ echo "ironbank=true" >> "$GITHUB_OUTPUT"
+ echo "base=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8" >> "$GITHUB_OUTPUT"
+ echo "micro=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8" >> "$GITHUB_OUTPUT"
+ elif [ "$SECRETS_EXPECTED" = "true" ]; then
+ # Never degrade quietly where the credential is supposed to be
+ # present: a green build on the wrong base misreports which image
+ # was validated.
+ echo "::error::IRONBANK_USER is empty on a run that should have it. Check the repository secrets." >&2
+ exit 1
+ else
+ echo "Fork run without Registry1 access, building on Red Hat's public UBI9." >&2
+ echo "ironbank=false" >> "$GITHUB_OUTPUT"
+ echo "base=registry.access.redhat.com/ubi9/ubi:9.8" >> "$GITHUB_OUTPUT"
+ echo "micro=registry.access.redhat.com/ubi9/ubi-micro:9.8" >> "$GITHUB_OUTPUT"
+ fi
+ - name: Login to Iron Bank
+ if: steps.bases.outputs.ironbank == 'true'
+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+ with:
+ registry: registry1.dso.mil
+ username: ${{ secrets.IRONBANK_USER }}
+ password: ${{ secrets.IRONBANK_PASS }}
- name: Setup Blacksmith Builder
uses: useblacksmith/setup-docker-builder@1e75355f81cbb3f93cba1eb5dd5e5b05110ca8f9 # v2.0.1
with:
@@ -656,14 +692,22 @@ jobs:
(set +e; docker compose -f "$DOCKER_COMPOSE_FILE" up -d --wait --wait-timeout 180 postgres redis minio clickhouse > /tmp/deps-up.log 2>&1; echo $? > /tmp/deps-up.exit) &
- name: Build images
# Type-checking runs in the lint job; release builds retain it.
+ env:
+ BASE_IMAGE: ${{ steps.bases.outputs.base }}
+ MICRO_BASE_IMAGE: ${{ steps.bases.outputs.micro }}
run: |
docker compose -f "$DOCKER_COMPOSE_FILE" build --print > /tmp/bake.json
if [ "${{ matrix.component }}" = "web" ]; then
docker buildx bake -f /tmp/bake.json \
+ --set "*.args.BASE_IMAGE=$BASE_IMAGE" \
+ --set "*.args.MICRO_BASE_IMAGE=$MICRO_BASE_IMAGE" \
--set "langfuse-web.args.NEXT_IGNORE_BUILD_ERRORS=true" \
langfuse-web
else
- docker buildx bake -f /tmp/bake.json langfuse-worker
+ docker buildx bake -f /tmp/bake.json \
+ --set "*.args.BASE_IMAGE=$BASE_IMAGE" \
+ --set "*.args.MICRO_BASE_IMAGE=$MICRO_BASE_IMAGE" \
+ langfuse-worker
fi
- name: Start compose services and wait for healthchecks
run: |
@@ -767,25 +811,10 @@ jobs:
pnpm install
- name: Load default env
run: |
- write_test_env() {
- {
- grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' "$1"
- echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1"
- echo "LANGFUSE_CACHE_PROMPT_ENABLED=false"
- echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1"
- echo "LANGFUSE_TRACE_DELETE_DELAY_MS=1"
- echo "LANGFUSE_TRACE_DELETE_CONCURRENCY=100"
- echo "ADMIN_API_KEY=admin-api-key"
- echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test"
- echo "LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION=true"
- echo "LANGFUSE_ENABLE_SCORES_V3_API=true"
- } > "$2"
- }
-
# Compile once under the default test environment. Next's generate
# phase below reapplies the mode-specific config and prerendering.
- write_test_env .env.dev.example .env
- write_test_env .env.dev${{ matrix.deploy-mode }}.example .env.runtime
+ bash scripts/ci/write-web-test-env.sh .env.dev.example .env
+ bash scripts/ci/write-web-test-env.sh .env.dev${{ matrix.deploy-mode }}.example .env.runtime
echo "LANGFUSE_OTEL_INGESTION_WORKER_SHADOW_ENABLED=${{ matrix.deploy-mode != '-azure' }}" >> .env.runtime
- name: Setup Turbo remote cache
if: env.CI_CACHE_ALLOWED == 'true'
@@ -1026,6 +1055,130 @@ jobs:
run: pnpm --filter=worker run test:exclude-llm-connections
env:
LANGFUSE_CODE_EVAL_AWS_LAMBDA_ENDPOINT: ${{ matrix.deploy-mode == '' && 'http://localhost:4566' || '' }}
+
+ tests-fips:
+ # Runs the web and worker suites the way a FIPS host would: on the UBI
+ # toolchain from worker/Dockerfile with OPENSSL_FORCE_FIPS_MODE=1, which
+ # puts Red Hat's OpenSSL in FIPS mode without a FIPS kernel. Code paths
+ # that need an algorithm FIPS refuses (md5, ...) fail with
+ # ERR_OSSL_EVP_UNSUPPORTED. Databases and migrations run on the runner as
+ # in tests-web/tests-worker; install, build, the web and worker processes
+ # and the tests run in the container (scripts/ci/fips-tests.sh).
+ # Not part of all-ci-passed yet.
+ timeout-minutes: 45
+ runs-on: ${{ matrix.runner }}
+ needs:
+ - pre-job
+ if: ${{ !cancelled() && (github.event_name != 'push' || needs.pre-job.outputs.should_skip != 'true') }}
+ name: tests-fips (${{ matrix.suite }})
+ env:
+ VITEST_MAX_WORKERS: ${{ matrix.max-workers }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - suite: web
+ runner: blacksmith-16vcpu-ubuntu-2404
+ max-workers: "12"
+ - suite: worker
+ runner: blacksmith-8vcpu-ubuntu-2404
+ max-workers: "4"
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: pnpm/setup@703c52620218391530e48b9e8870d5c0082e1b9b # v2.1.0
+ with:
+ install: false
+ cache: false
+ - name: Login to Docker Hub
+ if: github.repository == 'langfuse/langfuse' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME_READ }}
+ password: ${{ secrets.DOCKERHUB_TOKEN_READ }}
+ - name: Use Node.js 24 without cache
+ if: env.CI_CACHE_ALLOWED != 'true'
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+ with:
+ node-version: 24
+ package-manager-cache: false
+ - name: Use Node.js 24 with pnpm cache
+ if: env.CI_CACHE_ALLOWED == 'true'
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # zizmor: ignore[cache-poisoning] test job dependency cache only; no released artifacts are built or published from this cached state
+ with:
+ node-version: 24
+ cache: "pnpm"
+ cache-dependency-path: "pnpm-lock.yaml"
+ - name: install dependencies
+ # For the migrations and seeding on the runner; the container installs
+ # its own copy for UBI.
+ run: |
+ pnpm install
+ - name: Install golang-migrate for Clickhouse migrations
+ run: |
+ curl --fail --location --retry 5 --retry-delay 2 --retry-all-errors \
+ --output migrate.linux-amd64.tar.gz \
+ https://github.com/golang-migrate/migrate/releases/download/v4.19.1/migrate.linux-amd64.tar.gz
+ tar xzf migrate.linux-amd64.tar.gz
+ sudo mv migrate /usr/bin/migrate
+ which migrate
+ - name: Load default env
+ run: |
+ if [ "${{ matrix.suite }}" = "web" ]; then
+ bash scripts/ci/write-web-test-env.sh .env.dev.example .env
+ cp .env .env.runtime
+ echo "LANGFUSE_OTEL_INGESTION_WORKER_SHADOW_ENABLED=true" >> .env.runtime
+ else
+ cp .env.dev.example .env
+ cp .env.dev.example web/.env
+ cp .env.dev.example worker/.env
+ fi
+ - name: Setup Blacksmith Builder
+ uses: useblacksmith/setup-docker-builder@1e75355f81cbb3f93cba1eb5dd5e5b05110ca8f9 # v2.0.1
+ with:
+ cache-key: langfuse/test-fips-toolchain
+ - name: Build UBI toolchain image
+ run: docker buildx build --load --target toolchain -f worker/Dockerfile -t langfuse-fips-toolchain .
+ - name: Run + migrate
+ run: |
+ docker compose -f docker-compose.dev.yml up -d --wait --wait-timeout 180
+ docker compose -f docker-compose.dev.yml ps
+ pnpm run db:migrate
+ if [ "${{ matrix.suite }}" = "worker" ]; then
+ pnpm --filter=shared run db:seed
+ fi
+ pnpm --filter=shared ch:up
+ env:
+ POSTGRES_VERSION: 15
+ # The worker's code-eval tests need the extra floci container.
+ COMPOSE_PROFILES: ${{ matrix.suite == 'worker' && 'worker-tests' || '' }}
+ - name: Provide ClickHouse client via dev container for dev-tables setup
+ # The clickhouse-server container already ships the client binary;
+ # a shim avoids re-downloading the ~300MB client package every run.
+ run: |
+ sudo tee /usr/local/bin/clickhouse > /dev/null <<'EOF'
+ #!/bin/bash
+ exec docker exec -i langfuse-clickhouse clickhouse "$@"
+ EOF
+ sudo chmod +x /usr/local/bin/clickhouse
+ - name: Setup Dev Tables
+ run: |
+ pnpm --filter=shared ch:dev-tables
+ - name: Run ${{ matrix.suite }} tests under forced FIPS
+ env:
+ LANGFUSE_CODE_EVAL_AWS_LAMBDA_ENDPOINT: ${{ matrix.suite == 'worker' && 'http://localhost:4566' || '' }}
+ SUITE: ${{ matrix.suite }}
+ run: |
+ docker run --rm --network host \
+ -e OPENSSL_FORCE_FIPS_MODE=1 \
+ -e CI -e VITEST_MAX_WORKERS -e LANGFUSE_CODE_EVAL_AWS_LAMBDA_ENDPOINT \
+ -e PNPM_STORE_DIR=/pnpm-store \
+ -v "$(pnpm store path --silent):/pnpm-store" \
+ -v "$PWD:/src:ro" \
+ langfuse-fips-toolchain \
+ bash /src/scripts/ci/fips-tests.sh "$SUITE"
+
test-worker-llm-connections:
timeout-minutes: 20
runs-on: blacksmith-4vcpu-ubuntu-2404
@@ -1549,6 +1702,30 @@ jobs:
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
+ # Released images always build on the Iron Bank bases. Both ubi9 and
+ # ubi9-micro publish an OCI index covering linux/amd64 and linux/arm64,
+ # so every leg of the platform matrix resolves there.
+ - name: Resolve FIPS base images
+ id: bases
+ env:
+ IRONBANK_USER: ${{ secrets.IRONBANK_USER }}
+ run: |
+ if [ -z "$IRONBANK_USER" ]; then
+ # Releases only ever build in this repo, so a missing credential is
+ # a misconfiguration. Publishing images that claim Iron Bank
+ # provenance but were built elsewhere is worse than a failed
+ # release, so stop here.
+ echo "::error::IRONBANK_USER is empty on a release build. Check the repository secrets." >&2
+ exit 1
+ fi
+ echo "base=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8" >> "$GITHUB_OUTPUT"
+ echo "micro=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8" >> "$GITHUB_OUTPUT"
+ - name: Log in to Iron Bank
+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
+ with:
+ registry: registry1.dso.mil
+ username: ${{ secrets.IRONBANK_USER }}
+ password: ${{ secrets.IRONBANK_PASS }}
# build-push-action v2 requires this step; it no longer sets up buildkit
# itself. Both inputs are load-bearing, do not drop them:
# - cache-key is required as of setup-docker-builder v2. Without it the
@@ -1588,6 +1765,9 @@ jobs:
outputs: type=image,name=ghcr.io/langfuse/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true
labels: ${{ steps.meta.outputs.labels }}
platforms: ${{ matrix.platform }}
+ build-args: |
+ BASE_IMAGE=${{ steps.bases.outputs.base }}
+ MICRO_BASE_IMAGE=${{ steps.bases.outputs.micro }}
provenance: false
sbom: false
- name: Build and push image by digest to Docker Hub (${{ matrix.component }}, ${{ matrix.platform_tag }})
@@ -1599,6 +1779,9 @@ jobs:
outputs: type=image,name=langfuse/${{ matrix.image_name }},push-by-digest=true,name-canonical=true,push=true
labels: ${{ steps.meta.outputs.labels }}
platforms: ${{ matrix.platform }}
+ build-args: |
+ BASE_IMAGE=${{ steps.bases.outputs.base }}
+ MICRO_BASE_IMAGE=${{ steps.bases.outputs.micro }}
provenance: false
sbom: false
- name: Record pushed digests
diff --git a/ai-gateway/Dockerfile b/ai-gateway/Dockerfile
index 8e656645e0..4e72ff3a99 100644
--- a/ai-gateway/Dockerfile
+++ b/ai-gateway/Dockerfile
@@ -1,18 +1,101 @@
-FROM rust:1.98.0-trixie AS builder
-RUN useradd --create-home --uid 10001 gateway && mkdir /app && chown gateway:gateway /app
+# FIPS 140-3 base: UBI 9 builder and ubi9-micro runtime
+# The gateway performs no TLS today (plain HTTP; the dependency graph contains
+# no openssl or rustls crates), so FIPS readiness here means the whole image
+# lifecycle stays on the RHEL platform: the binary is compiled and linked
+# against UBI9 glibc, and the runtime carries the CMVP-validated OpenSSL FIPS
+# provider, so outbound TLS added later links the system libcrypto.so.3 and
+# inherits the validated module, the same pattern the RPM Node uses in
+# web/worker. The crypto policy stays at the default until the gateway does TLS.
+# The defaults are Red Hat's public UBI9 images, so the build needs no
+# credentials. Iron Bank serves the same content from an authenticated registry
+# with DoD provenance and scan attestations. Release builds opt in by overriding
+# both args after `docker login registry1.dso.mil` (free P1 account plus a CLI
+# secret):
+# --build-arg BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8
+# --build-arg MICRO_BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8
+ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.8
+ARG MICRO_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi-micro:9.8
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} ${BASE_IMAGE} AS builder
+
+ARG TARGETARCH
+
+# gcc provides the linker; tar and gzip unpack the pinned toolchain (curl is
+# already present as curl-minimal in the base). Red Hat's UBI9 rust-toolset
+# (1.84.1) cannot compile this crate's edition 2024 (needs rustc >= 1.85), and
+# the official rust:1.98.0-bookworm image links against Debian's glibc 2.36,
+# so binaries built there refuse to run on UBI9 (glibc 2.34). Building on UBI9
+# keeps the toolchain and the binary on the FIPS-validated platform.
+RUN dnf -y upgrade && \
+ dnf -y install --setopt=install_weak_deps=0 gcc tar gzip && \
+ dnf clean all
+
+# Pinned standalone Rust 1.98.0 installer, the exact channel from
+# rust-toolchain.toml (keep the version in sync with it), checksum-verified
+# like the dumb-init binary in web/Dockerfile. rustup-init would fetch the
+# rustup binary from an unversioned URL, and only rustc, cargo and the
+# target's std are needed to build; docs, clippy and rustfmt stay out of the
+# builder image (CI runs fmt and clippy outside Docker).
+RUN set -eux; \
+ case "${TARGETARCH:-amd64}" in \
+ amd64) triple=x86_64-unknown-linux-gnu; toolchain_sha256=aa30409afa67bd1ada244cefd82c7980e6a65bc113bb978e934b2413c75e3900 ;; \
+ arm64) triple=aarch64-unknown-linux-gnu; toolchain_sha256=5fbb4282403046d52a4672765c6761a809bf9f33e699b17e6eb7a93ab7770cc3 ;; \
+ *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
+ esac; \
+ curl -fsSL -o /tmp/rust.tar.gz "https://static.rust-lang.org/dist/rust-1.98.0-${triple}.tar.gz"; \
+ echo "${toolchain_sha256} /tmp/rust.tar.gz" | sha256sum -c -; \
+ tar -xzf /tmp/rust.tar.gz -C /tmp; \
+ /tmp/rust-1.98.0-${triple}/install.sh --prefix=/usr/local \
+ --components=rustc,cargo,rust-std-${triple}; \
+ rm -rf /tmp/rust.tar.gz /tmp/rust-1.98.0-${triple}
+
WORKDIR /app
-ENV CARGO_HOME=/home/gateway/.cargo
-USER gateway
-COPY --chown=gateway:gateway Cargo.toml Cargo.lock rust-toolchain.toml ./
-COPY --chown=gateway:gateway src ./src
+
+COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
+COPY src ./src
+
+# rust-toolchain.toml is honored by rustup only; the standalone installer's
+# cargo ignores it, so --locked plus the pinned installer above guarantee the
+# 1.98.0 toolchain and the checked-in lockfile.
RUN cargo build --release --locked
-FROM debian:trixie-slim AS runtime
-RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget \
- && rm -rf /var/lib/apt/lists/*
+# Runtime rootfs assembled with dnf --installroot and layered onto ubi9-micro:
+# the final image carries no package manager, python, or curl. It holds only
+# the packages listed below plus their deps, with an intact rpmdb for scanners.
+# --disableexcludes=all: the Iron Bank base excludes filesystem-* in
+# /etc/dnf/dnf.conf (it protects its own rootfs during in-place upgrades),
+# which would also block installing the dir-skeleton package into a fresh
+# installroot.
+FROM --platform=${TARGETPLATFORM:-linux/amd64} ${BASE_IMAGE} AS rpm-installer
+
+# glibc and libgcc are the binary's dynamic dependencies (libgcc_s for panics).
+# openssl-libs ships the FIPS provider module (/usr/lib64/ossl-modules/fips.so),
+# unlinked by today's binary but required the moment an OpenSSL-dependent
+# binary lands, so the validated module is already present. crypto-policies
+# provides the system policy OpenSSL reads; ca-certificates is the trust store
+# for future outbound TLS. wget serves the ECS container health command
+# (`sh -c 'wget -qO /dev/null http://localhost:8080/health'`, exercised by
+# scripts/smoke-image.sh); ubi9-micro already provides the shell.
+RUN dnf -y --installroot /mnt/rootfs --releasever 9 --setopt=install_weak_deps=0 \
+ --setopt=tsflags=nodocs --disableexcludes=all \
+ install glibc libgcc ca-certificates crypto-policies openssl-libs wget && \
+ dnf --installroot /mnt/rootfs clean all && \
+ rm -rf /mnt/rootfs/var/cache /mnt/rootfs/var/log/dnf*
+
+# Bake the non-root user into the rootfs (micro has no shadow-utils). uid/gid
+# 10001 preserves the upstream gateway image's identity.
+RUN echo "gateway:x:10001:" >> /mnt/rootfs/etc/group && \
+ echo "gateway:x:10001:10001::/home/gateway:/sbin/nologin" >> /mnt/rootfs/etc/passwd
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} ${MICRO_BASE_IMAGE} AS runtime
+
+COPY --from=rpm-installer /mnt/rootfs /
COPY --from=builder /app/target/release/ai-gateway /usr/local/bin/ai-gateway
ARG BUILD_ID
ENV BUILD_ID=$BUILD_ID
+
+# The gateway handles SIGTERM itself with a bounded drain (see
+# scripts/smoke-image.sh), so no init process is needed.
USER 10001:10001
EXPOSE 8080
STOPSIGNAL SIGTERM
diff --git a/packages/native/AGENTS.md b/packages/native/AGENTS.md
index 5a3b9046db..036219302f 100644
--- a/packages/native/AGENTS.md
+++ b/packages/native/AGENTS.md
@@ -12,10 +12,12 @@
- `index.js` and `index.d.ts` are generated by `napi build`. Never hand-edit
them; rebuild and commit the result whenever `src/lib.rs` exports change.
- Building the worker needs the pinned Rust toolchain: CI runners get it from
- rustup auto-install on first `cargo` use; `worker/Dockerfile` copies the
- same version from the `rust:<version>-alpine` image. Bump
- `rust-toolchain.toml` and that image tag together. The pin starts equal to
- `ai-gateway/rust-toolchain.toml` but the files are deliberately separate.
+ rustup auto-install on first `cargo` use; `worker/Dockerfile` installs the
+ same version from the checksum-pinned standalone installer on UBI9. Bump
+ `rust-toolchain.toml` together with that version and both per-arch sha256
+ values (`https://static.rust-lang.org/dist/rust-<version>-<triple>.tar.gz.sha256`).
+ The pin starts equal to `ai-gateway/rust-toolchain.toml` but the files are
+ deliberately separate.
- Native code reports through the `metrics` and `tracing` facades set up in
`src/telemetry.rs`; never return values for Node to record. Configuration
mirrors dd-trace and the winston logger (`DD_*`, `LANGFUSE_LOG_*`).
diff --git a/packages/native/README.md b/packages/native/README.md
index 7f320dbcb1..2b0e28c1ef 100644
--- a/packages/native/README.md
+++ b/packages/native/README.md
@@ -15,17 +15,17 @@ CPU-bound fits can be killed without blocking the worker.
## Layout
-| Path | Purpose |
-| --------------------- | -------------------------------------------------------------------------- |
-| `src/lib.rs` | Rust source. Every `#[napi]` item is exported to Node.js. |
-| `src/telemetry.rs` | Metrics and log setup shared by all native code; see Observability. |
-| `src/topics.rs` | Seeded UMAP reductions and HDBSCAN for Topics. |
-| `Cargo.toml` | Crate manifest (`cdylib`). `Cargo.lock` is committed. |
-| `build.rs` | napi-rs build hook. |
-| `rust-toolchain.toml` | Pinned compiler version, matched by the Rust image in `worker/Dockerfile`. |
-| `package.json` | npm package; `napi.binaryName` names the compiled `.node` file. |
-| `index.js` | Generated loader that picks the `.node` file for the current platform. |
-| `index.d.ts` | Generated TypeScript declarations, derived from the `#[napi]` signatures. |
+| Path | Purpose |
+| --------------------- | ---------------------------------------------------------------------------- |
+| `src/lib.rs` | Rust source. Every `#[napi]` item is exported to Node.js. |
+| `src/telemetry.rs` | Metrics and log setup shared by all native code; see Observability. |
+| `src/topics.rs` | Seeded UMAP reductions and HDBSCAN for Topics. |
+| `Cargo.toml` | Crate manifest (`cdylib`). `Cargo.lock` is committed. |
+| `build.rs` | napi-rs build hook. |
+| `rust-toolchain.toml` | Pinned compiler version, kept equal to the one `worker/Dockerfile` installs. |
+| `package.json` | npm package; `napi.binaryName` names the compiled `.node` file. |
+| `index.js` | Generated loader that picks the `.node` file for the current platform. |
+| `index.d.ts` | Generated TypeScript declarations, derived from the `#[napi]` signatures. |
`index.js` and `index.d.ts` are regenerated by every build and committed so
type checks and tooling work without a Rust toolchain. `*.node` binaries and
@@ -79,9 +79,9 @@ is always present when the worker starts or its tests run. A direct
## How it ships
-`worker/Dockerfile` copies the toolchain pinned in `rust-toolchain.toml` from
-the official `rust:<version>-alpine` image into the builder stage;
-`turbo run build --filter=worker...` compiles the addon for musl, and
+`worker/Dockerfile` installs the toolchain pinned in `rust-toolchain.toml` from
+the checksum-verified standalone Rust installer into its UBI9 builder stage;
+`turbo run build --filter=worker...` compiles the addon for glibc, and
`pnpm deploy` copies the `.node` file into the runtime image next to the
loader. Each architecture builds on a native runner, so no cross
compilation is involved. The runtime image gains only the compiled library.
diff --git a/packages/native/rust-toolchain.toml b/packages/native/rust-toolchain.toml
index 612a791279..7a82122ecb 100644
--- a/packages/native/rust-toolchain.toml
+++ b/packages/native/rust-toolchain.toml
@@ -1,7 +1,7 @@
# Same Rust version as ai-gateway/rust-toolchain.toml so the repository starts
# out on one compiler; the two files stay separate so either crate can move on
-# its own. worker/Dockerfile copies this exact version from the official
-# `rust:<version>-alpine` image, so bump the two together.
+# its own. worker/Dockerfile installs this exact version from the pinned
+# standalone Rust installer, so bump the two together.
[toolchain]
channel = "1.98.0"
components = ["clippy", "rustfmt"]
diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md
index 1abd74207a..e0d93045c8 100644
--- a/packages/shared/AGENTS.md
+++ b/packages/shared/AGENTS.md
@@ -152,6 +152,9 @@
`@langfuse/shared/src/server/auth/apiKeys`,
`@langfuse/shared/src/server/clickhouse/clickhouseIdentifiers`,
`@langfuse/shared/src/server/ee/ingestionMasking`,
+ `@langfuse/shared/src/server/ee/fips` (startup FIPS-mode enforcement; it
+ loads only env and the license check, so it can run before anything
+ connects),
`@langfuse/shared/src/server/llm/llmText`, and
`@langfuse/shared/src/utils/chatml`. The
`@langfuse/shared/src/utils/normalized-io` parser is client-safe and is the
diff --git a/packages/shared/package.json b/packages/shared/package.json
index ae95ab53af..1391c43a3d 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -83,6 +83,10 @@
"import": "./dist/src/server/ee/ingestionMasking/index.js",
"require": "./dist/src/server/ee/ingestionMasking/index.js"
},
+ "./src/server/ee/fips": {
+ "import": "./dist/src/server/ee/fips/index.js",
+ "require": "./dist/src/server/ee/fips/index.js"
+ },
"./src/server/ee/licenseCheck": {
"import": "./dist/src/server/ee/licenseCheck/index.js",
"require": "./dist/src/server/ee/licenseCheck/index.js"
diff --git a/packages/shared/prisma/schema.prisma b/packages/shared/prisma/schema.prisma
index b29f7da1e2..61fe8d0cc7 100644
--- a/packages/shared/prisma/schema.prisma
+++ b/packages/shared/prisma/schema.prisma
@@ -4,6 +4,10 @@
generator client {
provider = "prisma-client-js"
previewFeatures = ["views", "relationJoins", "metrics"]
+ // rhel-openssl-3.0.x: UBI9 runtime images (glibc); these engines link the
+ // system libcrypto.so.3, so Postgres TLS runs inside the platform's
+ // FIPS-validated OpenSSL module.
+ binaryTargets = ["native", "rhel-openssl-3.0.x"]
}
datasource db {
diff --git a/packages/shared/src/env.ts b/packages/shared/src/env.ts
index 7b8ba5331a..e4bb3bd8de 100644
--- a/packages/shared/src/env.ts
+++ b/packages/shared/src/env.ts
@@ -695,6 +695,10 @@ const EnvSchema = z.object({
// EE License
LANGFUSE_EE_LICENSE_KEY: z.string().optional(),
+ // FIPS mode (EE feature): refuse to start without an enterprise license or
+ // an active OpenSSL FIPS provider. See server/ee/fips.
+ LANGFUSE_REQUIRE_FIPS: z.enum(["true", "false"]).default("false"),
+
// Ingestion Masking (EE feature)
LANGFUSE_INGESTION_MASKING_CALLBACK_URL: z.url().optional(),
LANGFUSE_INGESTION_MASKING_CALLBACK_TIMEOUT_MS: z.coerce
diff --git a/packages/shared/src/server/ee/fips/assertFipsMode.test.ts b/packages/shared/src/server/ee/fips/assertFipsMode.test.ts
new file mode 100644
index 0000000000..0316321b2b
--- /dev/null
+++ b/packages/shared/src/server/ee/fips/assertFipsMode.test.ts
@@ -0,0 +1,70 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { SharedEnv } from "../../../env";
+import { assertFipsMode } from "./assertFipsMode";
+
+const { getFips } = vi.hoisted(() => ({ getFips: vi.fn() }));
+
+vi.mock("crypto", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("crypto")>()),
+ getFips,
+}));
+
+const createTestEnv = (overrides: Partial<SharedEnv> = {}): SharedEnv =>
+ ({
+ LANGFUSE_REQUIRE_FIPS: "true",
+ LANGFUSE_EE_LICENSE_KEY: "langfuse_ee_test",
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: undefined,
+ ...overrides,
+ }) as SharedEnv;
+
+describe("assertFipsMode", () => {
+ beforeEach(() => {
+ getFips.mockReset();
+ getFips.mockReturnValue(1);
+ });
+
+ it("does nothing when FIPS mode is not requested", () => {
+ getFips.mockReturnValue(0);
+
+ expect(() =>
+ assertFipsMode(
+ createTestEnv({
+ LANGFUSE_REQUIRE_FIPS: "false",
+ LANGFUSE_EE_LICENSE_KEY: undefined,
+ }),
+ ),
+ ).not.toThrow();
+ });
+
+ it("passes with an enterprise license and an active FIPS provider", () => {
+ expect(() => assertFipsMode(createTestEnv())).not.toThrow();
+ });
+
+ it("passes on Langfuse Cloud with an active FIPS provider", () => {
+ expect(() =>
+ assertFipsMode(
+ createTestEnv({
+ LANGFUSE_EE_LICENSE_KEY: undefined,
+ NEXT_PUBLIC_LANGFUSE_CLOUD_REGION: "EU",
+ }),
+ ),
+ ).not.toThrow();
+ });
+
+ it.each([
+ ["no license key", undefined],
+ ["a pro license key", "langfuse_pro_test"],
+ ])("refuses to start with %s, even on a FIPS host", (_label, licenseKey) => {
+ expect(() =>
+ assertFipsMode(createTestEnv({ LANGFUSE_EE_LICENSE_KEY: licenseKey })),
+ ).toThrow(/enterprise license/);
+ });
+
+ it("refuses to start with a license when the FIPS provider is inactive", () => {
+ getFips.mockReturnValue(0);
+
+ expect(() => assertFipsMode(createTestEnv())).toThrow(
+ /FIPS provider is not active/,
+ );
+ });
+});
diff --git a/packages/shared/src/server/ee/fips/assertFipsMode.ts b/packages/shared/src/server/ee/fips/assertFipsMode.ts
new file mode 100644
index 0000000000..d84057647e
--- /dev/null
+++ b/packages/shared/src/server/ee/fips/assertFipsMode.ts
@@ -0,0 +1,34 @@
+import { getFips } from "crypto";
+import { env, type SharedEnv } from "../../../env";
+import { isEnterpriseLicenseAvailable } from "../licenseCheck";
+
+/**
+ * Enforce FIPS mode (LANGFUSE_REQUIRE_FIPS=true), an enterprise feature.
+ *
+ * Fails closed: when FIPS mode is requested, startup must stop unless an
+ * enterprise license is available and Node's OpenSSL FIPS provider is active.
+ * The provider is only active when the host kernel runs in FIPS mode, so a
+ * container cannot switch it on by itself.
+ *
+ * Imports nothing that opens connections, so callers can run it before any
+ * other startup code.
+ */
+export function assertFipsMode(envOverride?: SharedEnv): void {
+ const e = envOverride ?? env;
+
+ if (e.LANGFUSE_REQUIRE_FIPS !== "true") {
+ return;
+ }
+
+ if (!isEnterpriseLicenseAvailable(e)) {
+ throw new Error(
+ "LANGFUSE_REQUIRE_FIPS=true requires a Langfuse enterprise license (LANGFUSE_EE_LICENSE_KEY).",
+ );
+ }
+
+ if (getFips() !== 1) {
+ throw new Error(
+ "LANGFUSE_REQUIRE_FIPS=true but Node's OpenSSL FIPS provider is not active (crypto.getFips() != 1). Run Langfuse on a host with FIPS mode enabled.",
+ );
+ }
+}
diff --git a/packages/shared/src/server/ee/fips/index.ts b/packages/shared/src/server/ee/fips/index.ts
new file mode 100644
index 0000000000..7f7d01771f
--- /dev/null
+++ b/packages/shared/src/server/ee/fips/index.ts
@@ -0,0 +1 @@
+export * from "./assertFipsMode";
diff --git a/packages/shared/src/server/redis/redis.ts b/packages/shared/src/server/redis/redis.ts
index 783dfaf095..d97f99227b 100644
--- a/packages/shared/src/server/redis/redis.ts
+++ b/packages/shared/src/server/redis/redis.ts
@@ -1,5 +1,5 @@
import Redis, { RedisOptions, Cluster, ClusterOptions } from "ioredis";
-import type { QueueBaseOptions } from "bullmq";
+import type { QueueBaseOptions, QueueOptions } from "bullmq";
import fs from "fs";
import { env } from "../../env";
import { logger } from "../logger";
@@ -76,10 +76,13 @@ export const redisQueueRetryOptions: Partial<RedisOptions> = {
},
};
+// "settings" comes from QueueOptions (not QueueBaseOptions); the same
+// AdvancedRepeatOptions shape is accepted by WorkerOptions["settings"].
type BullMQOptionsWithRedis = Pick<
QueueBaseOptions,
"connection" | "prefix" | "skipVersionCheck"
->;
+> &
+ Pick<QueueOptions, "settings">;
/**
* Parse Redis node definitions from environment variable
@@ -369,6 +372,15 @@ const getBullMQOptionsForRedisConnection = (
): BullMQOptionsWithRedis => ({
connection,
prefix: getQueuePrefix(queueName),
+ // bullmq hashes with md5 by default, and an OpenSSL FIPS provider refuses
+ // md5, so any call into the legacy repeat API throws
+ // ERR_OSSL_EVP_UNSUPPORTED on a FIPS image. Job scheduler registration does
+ // not hash and is unaffected, but the legacy-schedule cleanup in
+ // scheduleRecurringJob calls removeRepeatableByKey, which hashes before it
+ // reaches Redis. sha256 is FIPS-approved and keeps that call working; the
+ // digest itself is only used to build an identifier the removal script
+ // never reads.
+ settings: { repeatKeyHashAlgorithm: "sha256" },
...(env.LANGFUSE_BULLMQ_SKIP_REDIS_VERSION_CHECK === "true"
? { skipVersionCheck: true }
: {}),
diff --git a/packages/shared/src/server/redis/scheduleRecurringJob.test.ts b/packages/shared/src/server/redis/scheduleRecurringJob.test.ts
index 17c4cb4816..6fd8ad9d45 100644
--- a/packages/shared/src/server/redis/scheduleRecurringJob.test.ts
+++ b/packages/shared/src/server/redis/scheduleRecurringJob.test.ts
@@ -11,15 +11,46 @@ vi.mock("../logger", () => ({
},
}));
-const createQueueMock = () => {
- const removeRepeatable = vi.fn().mockResolvedValue(true);
+/** Legacy entries are keyed by a hash of the repeat options, never by name. */
+const legacyEntry = (name: string, pattern: string, key: string) => ({
+ key,
+ name,
+ pattern,
+ endDate: null,
+ tz: null,
+ every: null,
+ next: 0,
+});
+
+/** The job scheduler stores its entry under the scheduler id, i.e. the name. */
+const schedulerEntry = (name: string, pattern: string) => ({
+ key: name,
+ name,
+ pattern,
+ endDate: null,
+ tz: null,
+ every: null,
+ next: 0,
+});
+
+const createQueueMock = (
+ repeatables: ReturnType<typeof legacyEntry>[] = [],
+) => {
+ const getRepeatableJobs = vi.fn().mockResolvedValue(repeatables);
+ const removeRepeatableByKey = vi.fn().mockResolvedValue(true);
const upsertJobScheduler = vi.fn().mockResolvedValue({});
const queue = {
name: "test-queue",
- removeRepeatable,
+ getRepeatableJobs,
+ removeRepeatableByKey,
upsertJobScheduler,
} as unknown as Queue;
- return { queue, removeRepeatable, upsertJobScheduler };
+ return {
+ queue,
+ getRepeatableJobs,
+ removeRepeatableByKey,
+ upsertJobScheduler,
+ };
};
describe("scheduleRecurringJob", () => {
@@ -28,7 +59,11 @@ describe("scheduleRecurringJob", () => {
});
it("removes legacy schedules for the current and previous patterns before upserting", async () => {
- const { queue, removeRepeatable, upsertJobScheduler } = createQueueMock();
+ const { queue, removeRepeatableByKey, upsertJobScheduler } =
+ createQueueMock([
+ legacyEntry("my-job", "*/15 * * * *", "8f14e45fceea167a"),
+ legacyEntry("my-job", "25 * * * *", "45c48cce2e2d7fbd"),
+ ]);
await scheduleRecurringJob(queue, {
jobName: "my-job",
@@ -36,13 +71,9 @@ describe("scheduleRecurringJob", () => {
previousPatterns: ["25 * * * *"],
});
- expect(removeRepeatable).toHaveBeenCalledTimes(2);
- expect(removeRepeatable).toHaveBeenCalledWith("my-job", {
- pattern: "*/15 * * * *",
- });
- expect(removeRepeatable).toHaveBeenCalledWith("my-job", {
- pattern: "25 * * * *",
- });
+ expect(removeRepeatableByKey).toHaveBeenCalledTimes(2);
+ expect(removeRepeatableByKey).toHaveBeenCalledWith("8f14e45fceea167a");
+ expect(removeRepeatableByKey).toHaveBeenCalledWith("45c48cce2e2d7fbd");
expect(upsertJobScheduler).toHaveBeenCalledWith(
"my-job",
{ pattern: "*/15 * * * *" },
@@ -50,13 +81,67 @@ describe("scheduleRecurringJob", () => {
);
// The legacy cleanup must complete before the scheduler is upserted, so
// a concurrently created legacy entry cannot outlive the migration.
- for (const removeCall of removeRepeatable.mock.invocationCallOrder) {
+ for (const removeCall of removeRepeatableByKey.mock.invocationCallOrder) {
expect(removeCall).toBeLessThan(
upsertJobScheduler.mock.invocationCallOrder[0]!,
);
}
});
+ it("never removes the job scheduler's own entry", async () => {
+ // The scheduler shares the `repeat` sorted set with legacy entries and is
+ // keyed by the scheduler id. Matching on name alone would delete the live
+ // schedule this call is about to (re)create.
+ const { queue, removeRepeatableByKey } = createQueueMock([
+ schedulerEntry("my-job", "*/15 * * * *"),
+ ]);
+
+ await scheduleRecurringJob(queue, {
+ jobName: "my-job",
+ pattern: "*/15 * * * *",
+ });
+
+ expect(removeRepeatableByKey).not.toHaveBeenCalled();
+ });
+
+ it("leaves other jobs and unrelated patterns alone", async () => {
+ const { queue, removeRepeatableByKey } = createQueueMock([
+ legacyEntry("other-job", "*/15 * * * *", "aaa111"),
+ legacyEntry("my-job", "0 3 * * *", "bbb222"),
+ legacyEntry("my-job", "*/15 * * * *", "ccc333"),
+ ]);
+
+ await scheduleRecurringJob(queue, {
+ jobName: "my-job",
+ pattern: "*/15 * * * *",
+ });
+
+ expect(removeRepeatableByKey).toHaveBeenCalledTimes(1);
+ expect(removeRepeatableByKey).toHaveBeenCalledWith("ccc333");
+ });
+
+ it("does not hash to locate legacy entries, so it works under FIPS", async () => {
+ // Regression guard: Queue.removeRepeatable() recomputes an md5 digest to
+ // find the entry, which throws ERR_OSSL_EVP_UNSUPPORTED when OpenSSL runs
+ // in FIPS mode. The cleanup must go through the enumerate-then-remove-by-
+ // key path instead, which never derives a key.
+ const { queue, getRepeatableJobs, removeRepeatableByKey } = createQueueMock(
+ [legacyEntry("my-job", "*/15 * * * *", "ddd444")],
+ );
+ (queue as unknown as { removeRepeatable: unknown }).removeRepeatable =
+ vi.fn(() => {
+ throw new Error("ERR_OSSL_EVP_UNSUPPORTED");
+ });
+
+ await scheduleRecurringJob(queue, {
+ jobName: "my-job",
+ pattern: "*/15 * * * *",
+ });
+
+ expect(getRepeatableJobs).toHaveBeenCalled();
+ expect(removeRepeatableByKey).toHaveBeenCalledWith("ddd444");
+ });
+
it("passes template data through to the scheduler", async () => {
const { queue, upsertJobScheduler } = createQueueMock();
@@ -74,8 +159,8 @@ describe("scheduleRecurringJob", () => {
});
it("still upserts the scheduler when legacy cleanup fails", async () => {
- const { queue, removeRepeatable, upsertJobScheduler } = createQueueMock();
- removeRepeatable.mockRejectedValue(new Error("redis down"));
+ const { queue, getRepeatableJobs, upsertJobScheduler } = createQueueMock();
+ getRepeatableJobs.mockRejectedValue(new Error("redis down"));
await scheduleRecurringJob(queue, {
jobName: "my-job",
@@ -85,6 +170,25 @@ describe("scheduleRecurringJob", () => {
expect(upsertJobScheduler).toHaveBeenCalledTimes(1);
});
+ it("keeps removing the other legacy schedules when one removal fails", async () => {
+ const { queue, removeRepeatableByKey, upsertJobScheduler } =
+ createQueueMock([
+ legacyEntry("my-job", "*/15 * * * *", "eee555"),
+ legacyEntry("my-job", "25 * * * *", "fff666"),
+ ]);
+ removeRepeatableByKey.mockRejectedValueOnce(new Error("redis timeout"));
+
+ await scheduleRecurringJob(queue, {
+ jobName: "my-job",
+ pattern: "*/15 * * * *",
+ previousPatterns: ["25 * * * *"],
+ });
+
+ expect(removeRepeatableByKey).toHaveBeenCalledWith("eee555");
+ expect(removeRepeatableByKey).toHaveBeenCalledWith("fff666");
+ expect(upsertJobScheduler).toHaveBeenCalledTimes(1);
+ });
+
it("resolves instead of rejecting when the upsert fails", async () => {
const { queue, upsertJobScheduler } = createQueueMock();
upsertJobScheduler.mockRejectedValue(new Error("redis down"));
diff --git a/packages/shared/src/server/redis/scheduleRecurringJob.ts b/packages/shared/src/server/redis/scheduleRecurringJob.ts
index ef31ecff67..941c14deb1 100644
--- a/packages/shared/src/server/redis/scheduleRecurringJob.ts
+++ b/packages/shared/src/server/redis/scheduleRecurringJob.ts
@@ -7,7 +7,8 @@ import { logger } from "../logger";
// TRIGGERS FOREVER (bullmq v6 workers keep iterating stranded md5-keyed
// chains; only manual Redis cleanup stops them).
// HOLD THE BULLMQ v6 UPGRADE UNTIL LANGFUSE V5: v6 deletes
-// Queue.removeRepeatable(), which this cleanup requires.
+// Queue.getRepeatableJobs() and Queue.removeRepeatableByKey(), which this
+// cleanup requires.
/**
* Registers a recurring cron job on a queue via a BullMQ job scheduler.
@@ -23,11 +24,13 @@ import { logger } from "../logger";
* entirely.
*
* The scheduler id is the job name. Legacy schedules are keyed by
- * md5(name + pattern) instead, so this helper also removes the legacy
- * entries for the current pattern and any historical ones - without that
- * cleanup the old chain keeps firing in parallel with the scheduler. The
- * cleanup is idempotent and must stay in place for as long as a direct
- * upgrade from a Langfuse version that scheduled via the legacy API is
+ * md5(name:jobId:endDate:tz:pattern) instead, so this helper also removes the
+ * legacy entries for the current pattern and any historical ones - without
+ * that cleanup the old chain keeps firing in parallel with the scheduler. The
+ * legacy keys are read back from Redis rather than recomputed, because
+ * recomputing them requires md5 and that is unavailable when OpenSSL runs in
+ * FIPS mode. The cleanup is idempotent and must stay in place for as long as a
+ * direct upgrade from a Langfuse version that scheduled via the legacy API is
* supported.
*
* Never rejects: scheduling runs fire-and-forget from queue singleton
@@ -50,17 +53,50 @@ export const scheduleRecurringJob = async (
},
): Promise<void> => {
const { jobName, pattern, previousPatterns = [], data } = opts;
+ const legacyPatterns = new Set([pattern, ...previousPatterns]);
- for (const legacyPattern of [pattern, ...previousPatterns]) {
- try {
- // eslint-disable-next-line @typescript-eslint/no-deprecated -- Removes schedules created by the deprecated repeat API in earlier releases; keep while direct upgrades from those releases are supported.
- await queue.removeRepeatable(jobName, { pattern: legacyPattern });
- } catch (err) {
- logger.error(
- `Error removing legacy ${jobName} schedule (${legacyPattern}) on ${queue.name}`,
- err,
- );
+ try {
+ // Read the stored keys back rather than recomputing them.
+ // Queue.removeRepeatable() derives md5(name:jobId:endDate:tz:pattern) to
+ // locate the entry, and an OpenSSL FIPS provider refuses md5 outright, so
+ // on a FIPS image that call throws ERR_OSSL_EVP_UNSUPPORTED and the
+ // cleanup silently never happens - producing exactly the duplicate cron
+ // triggers this file exists to prevent. Enumerating hashes nothing, and
+ // removal by stored key is hash-free in the Lua script.
+ // eslint-disable-next-line @typescript-eslint/no-deprecated -- legacy repeat entries are invisible to getJobSchedulers; keep while direct upgrades from legacy-scheduling releases are supported.
+ const stored = await queue.getRepeatableJobs();
+
+ const staleKeys = stored
+ .filter(
+ (job) =>
+ job.name === jobName &&
+ // The scheduler keeps its own entry in this same sorted set, keyed
+ // by the scheduler id, which is the job name. Legacy entries are
+ // keyed by a hash, so an entry keyed by the job name is the live
+ // scheduler and must not be removed here.
+ job.key !== jobName &&
+ job.pattern !== null &&
+ legacyPatterns.has(job.pattern),
+ )
+ .map((job) => job.key);
+
+ // One failing key must not leave the remaining legacy chains firing.
+ for (const key of staleKeys) {
+ try {
+ // eslint-disable-next-line @typescript-eslint/no-deprecated -- pairs with getRepeatableJobs above; removeJobScheduler cannot address legacy entries.
+ await queue.removeRepeatableByKey(key);
+ } catch (err) {
+ logger.error(
+ `Error removing legacy ${jobName} schedule ${key} on ${queue.name}`,
+ err,
+ );
+ }
}
+ } catch (err) {
+ logger.error(
+ `Error listing legacy ${jobName} schedules on ${queue.name}`,
+ err,
+ );
}
try {
diff --git a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts
index b4e1130054..fc0fecb16b 100644
--- a/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts
+++ b/packages/shared/src/server/redis/v4LegacyApiUsageQueue.test.ts
@@ -13,13 +13,15 @@ describe("V4LegacyApiUsageQueue schedule", () => {
});
it("removes the legacy repeatable schedules and upserts a job scheduler", async () => {
- const removeRepeatable = vi.fn().mockResolvedValue(true);
+ const getRepeatableJobs = vi.fn();
+ const removeRepeatableByKey = vi.fn().mockResolvedValue(true);
const upsertJobScheduler = vi.fn().mockResolvedValue({});
const on = vi.fn();
vi.doMock("bullmq", () => ({
Queue: class {
- removeRepeatable = removeRepeatable;
+ getRepeatableJobs = getRepeatableJobs;
+ removeRepeatableByKey = removeRepeatableByKey;
upsertJobScheduler = upsertJobScheduler;
on = on;
},
@@ -43,6 +45,21 @@ describe("V4LegacyApiUsageQueue schedule", () => {
await import("./v4LegacyApiUsageQueue.js");
const { QueueJobs } = await import("../queues.js");
+ // Legacy repeatable entries are stored under a hash of their repeat
+ // options, one per pattern the job was ever scheduled with.
+ getRepeatableJobs.mockResolvedValue([
+ {
+ key: "legacy-current-pattern",
+ name: QueueJobs.V4LegacyApiUsageJob,
+ pattern: V4_LEGACY_API_USAGE_CRON_PATTERN,
+ },
+ {
+ key: "legacy-hourly-pattern",
+ name: QueueJobs.V4LegacyApiUsageJob,
+ pattern: "25 * * * *",
+ },
+ ]);
+
V4LegacyApiUsageQueue.getInstance();
// Scheduling is fire-and-forget from getInstance, so wait for the chain.
@@ -50,23 +67,20 @@ describe("V4LegacyApiUsageQueue schedule", () => {
expect(upsertJobScheduler).toHaveBeenCalledTimes(1);
});
- // Legacy repeatable entries are keyed by md5(name + pattern), so both the
- // current pattern and the pre-migration hourly pattern must be cleaned up.
- expect(removeRepeatable).toHaveBeenCalledWith(
- QueueJobs.V4LegacyApiUsageJob,
- { pattern: V4_LEGACY_API_USAGE_CRON_PATTERN },
- );
- expect(removeRepeatable).toHaveBeenCalledWith(
- QueueJobs.V4LegacyApiUsageJob,
- { pattern: "25 * * * *" },
+ // Both the current pattern and the pre-migration hourly pattern must be
+ // cleaned up, by their stored keys.
+ expect(removeRepeatableByKey).toHaveBeenCalledTimes(2);
+ expect(removeRepeatableByKey).toHaveBeenCalledWith(
+ "legacy-current-pattern",
);
+ expect(removeRepeatableByKey).toHaveBeenCalledWith("legacy-hourly-pattern");
expect(upsertJobScheduler).toHaveBeenCalledWith(
QueueJobs.V4LegacyApiUsageJob,
{ pattern: V4_LEGACY_API_USAGE_CRON_PATTERN },
{ name: QueueJobs.V4LegacyApiUsageJob, data: {} },
);
expect(V4_LEGACY_API_USAGE_CRON_PATTERN).toBe("*/15 * * * *");
- for (const removeCall of removeRepeatable.mock.invocationCallOrder) {
+ for (const removeCall of removeRepeatableByKey.mock.invocationCallOrder) {
expect(removeCall).toBeLessThan(
upsertJobScheduler.mock.invocationCallOrder[0]!,
);
diff --git a/packages/shared/src/server/services/StorageService.test.ts b/packages/shared/src/server/services/StorageService.test.ts
index d473c6dd96..f0652e3d6a 100644
--- a/packages/shared/src/server/services/StorageService.test.ts
+++ b/packages/shared/src/server/services/StorageService.test.ts
@@ -1,4 +1,4 @@
-import { createHash } from "crypto";
+import { createHash, getFips } from "crypto";
import { PassThrough, Readable } from "stream";
import { afterEach, describe, expect, it, vi } from "vitest";
@@ -525,27 +525,31 @@ describe("S3StorageService DeleteObjects checksum", () => {
expect(findHeader(request, "content-md5")).toBeUndefined();
});
- it("sends Content-MD5 on DeleteObjects when the algorithm is set to MD5", async () => {
- setChecksumAlgorithm("MD5");
- const { service, captured } = makeServiceWithCapture(
- EMPTY_DELETE_RESULT_XML,
- );
-
- await service.deleteFiles([
- "events/project-1/file-1.json",
- "media/project-1/file-2.png",
- ]);
-
- expect(captured).toHaveLength(1);
- const request = captured[0];
- expect(typeof request.body).toBe("string");
-
- const expectedMd5 = createHash("md5")
- .update(request.body as string)
- .digest("base64");
- expect(findHeader(request, "content-md5")).toBe(expectedMd5);
- expect(findHeader(request, "x-amz-checksum-crc32")).toBeUndefined();
- });
+ // MD5 checksums are unavailable when OpenSSL runs in FIPS mode.
+ it.skipIf(getFips() === 1)(
+ "sends Content-MD5 on DeleteObjects when the algorithm is set to MD5",
+ async () => {
+ setChecksumAlgorithm("MD5");
+ const { service, captured } = makeServiceWithCapture(
+ EMPTY_DELETE_RESULT_XML,
+ );
+
+ await service.deleteFiles([
+ "events/project-1/file-1.json",
+ "media/project-1/file-2.png",
+ ]);
+
+ expect(captured).toHaveLength(1);
+ const request = captured[0];
+ expect(typeof request.body).toBe("string");
+
+ const expectedMd5 = createHash("md5")
+ .update(request.body as string)
+ .digest("base64");
+ expect(findHeader(request, "content-md5")).toBe(expectedMd5);
+ expect(findHeader(request, "x-amz-checksum-crc32")).toBeUndefined();
+ },
+ );
});
describe("S3StorageService non-buffered upload part size", () => {
diff --git a/scripts/ci/fips-tests.sh b/scripts/ci/fips-tests.sh
new file mode 100755
index 0000000000..5f151f8cef
--- /dev/null
+++ b/scripts/ci/fips-tests.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+# Runs the web or worker test suites with OpenSSL's FIPS provider active, so a
+# code path that needs an algorithm a FIPS host refuses (md5, ...) fails here
+# with ERR_OSSL_EVP_UNSUPPORTED instead of in a customer deployment.
+#
+# Runs inside the UBI toolchain stage of worker/Dockerfile: Red Hat's Node links
+# the system OpenSSL, and OPENSSL_FORCE_FIPS_MODE=1 switches it into FIPS mode
+# without a FIPS host kernel. The databases, migrations and env files must
+# already be set up on the host, as the tests-fips CI job does.
+#
+# Local run, from the repo root with the dev containers up and migrated:
+# docker buildx build --load --target toolchain -f worker/Dockerfile -t langfuse-fips-toolchain .
+# docker run --rm --network host -e OPENSSL_FORCE_FIPS_MODE=1 -v "$PWD:/src:ro" \
+# langfuse-fips-toolchain bash /src/scripts/ci/fips-tests.sh worker [vitest filters...]
+
+set -euo pipefail
+
+readonly SUITE="${1:?usage: fips-tests.sh web|worker [vitest filters...]}"
+shift
+readonly SRC_DIR="${FIPS_SOURCE_DIR:-/src}"
+readonly WORK_DIR="/work"
+
+export HUSKY=0 TURBO_TELEMETRY_DISABLED=1
+
+group() { echo "::group::$*"; }
+endgroup() { echo "::endgroup::"; }
+
+if ! node -e 'process.exit(require("node:crypto").getFips() === 1 ? 0 : 1)'; then
+ echo "The OpenSSL FIPS provider is not active; run with OPENSSL_FORCE_FIPS_MODE=1." >&2
+ exit 1
+fi
+
+# Build on a private copy: the host's node_modules and build outputs were made
+# for the host, not for UBI. Env files written by the host come along.
+group "Copy sources"
+mkdir -p "$WORK_DIR"
+tar -C "$SRC_DIR" --exclude=.git --exclude=node_modules --exclude=.next \
+ --exclude=.turbo --exclude=dist --exclude=target -cf - . | tar -C "$WORK_DIR" -xf -
+cd "$WORK_DIR"
+endgroup
+
+group "Install dependencies"
+pnpm install --frozen-lockfile ${PNPM_STORE_DIR:+--store-dir "$PNPM_STORE_DIR"}
+pnpm --filter=shared run db:generate
+endgroup
+
+wait_for_health() {
+ local url="$1" pid="$2"
+ for _ in $(seq 1 150); do
+ if curl --silent --fail --output /dev/null "$url"; then
+ return 0
+ fi
+ if ! kill -0 "$pid" 2>/dev/null; then
+ echo "Langfuse exited before $url became healthy:" >&2
+ tail -n 100 /tmp/langfuse-start.log >&2
+ return 1
+ fi
+ sleep 2
+ done
+ echo "Timed out waiting for $url:" >&2
+ tail -n 100 /tmp/langfuse-start.log >&2
+ return 1
+}
+
+case "$SUITE" in
+ worker)
+ group "Build worker"
+ pnpm --filter=worker... run build
+ endgroup
+
+ NODE_ENV="test" pnpm --filter @langfuse/shared run test --passWithNoTests "$@"
+ pnpm --filter=worker run test:exclude-llm-connections --passWithNoTests "$@"
+ ;;
+
+ web)
+ # Mirrors tests-web: compile once, finalize for the runtime env, then run
+ # web and worker so the server tests can call them.
+ export NODE_ENV=test
+ group "Build web and worker"
+ NEXT_DISABLE_BUILD_CACHE=true NODE_OPTIONS=--max_old_space_size=8192 \
+ NEXT_IGNORE_BUILD_ERRORS=true pnpm turbo run build:test
+ cp .env.runtime .env
+ NEXT_DISABLE_BUILD_CACHE=true NODE_OPTIONS=--max_old_space_size=8192 \
+ NEXT_IGNORE_BUILD_ERRORS=true \
+ pnpm --filter web exec dotenv -e ../.env -- next build --experimental-build-mode generate
+ pnpm --filter web run build:otel-worker
+ endgroup
+
+ group "Start Langfuse"
+ # Docker sets HOSTNAME to the container's host name, and the worker binds
+ # to HOSTNAME; bind all interfaces so localhost reaches it.
+ HOSTNAME="0.0.0.0" \
+ LANGFUSE_INIT_ORG_ID="seed-org-id" \
+ LANGFUSE_INIT_ORG_NAME="Seed Org" \
+ LANGFUSE_INIT_ORG_CLOUD_PLAN="Team" \
+ LANGFUSE_INIT_PROJECT_ID="7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" \
+ LANGFUSE_INIT_PROJECT_NAME="Seed Project" \
+ LANGFUSE_INIT_PROJECT_PUBLIC_KEY="pk-lf-1234567890" \
+ LANGFUSE_INIT_PROJECT_SECRET_KEY="sk-lf-1234567890" \
+ LANGFUSE_INIT_USER_EMAIL="[email protected]" \
+ LANGFUSE_INIT_USER_NAME="Demo User" \
+ LANGFUSE_INIT_USER_PASSWORD="password" \
+ pnpm run start > /tmp/langfuse-start.log 2>&1 &
+ start_pid=$!
+ wait_for_health http://localhost:3000/api/public/health "$start_pid"
+ wait_for_health http://localhost:3030/api/health "$start_pid"
+ endgroup
+
+ cd web
+ NODE_COMPILE_CACHE=/tmp/node-compile-cache npx dotenv -e ../.env.test -e ../.env -- \
+ vitest run --project server --project server-isolated \
+ --project server-shared-source --project server-unit \
+ --project server-shared-source-unit \
+ --maxWorkers="${VITEST_MAX_WORKERS:-8}" --passWithNoTests "$@"
+ ;;
+
+ *)
+ echo "Unknown suite: $SUITE (expected web or worker)" >&2
+ exit 1
+ ;;
+esac
diff --git a/scripts/ci/write-web-test-env.sh b/scripts/ci/write-web-test-env.sh
new file mode 100755
index 0000000000..cd9735927b
--- /dev/null
+++ b/scripts/ci/write-web-test-env.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Writes the env file the web server tests run with: a dev env example plus
+# the test overrides. Used by the tests-web and tests-fips CI jobs.
+
+set -euo pipefail
+
+readonly SOURCE="${1:?usage: write-web-test-env.sh <env example> <output>}"
+readonly OUTPUT="${2:?usage: write-web-test-env.sh <env example> <output>}"
+
+{
+ grep -v -e '^LANGFUSE_S3_BATCH_EXPORT_ENABLED=' -e '^NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=' "$SOURCE"
+ echo "LANGFUSE_INGESTION_QUEUE_DELAY_MS=1"
+ echo "LANGFUSE_CACHE_PROMPT_ENABLED=false"
+ echo "LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=1"
+ echo "LANGFUSE_TRACE_DELETE_DELAY_MS=1"
+ echo "LANGFUSE_TRACE_DELETE_CONCURRENCY=100"
+ echo "ADMIN_API_KEY=admin-api-key"
+ echo "LANGFUSE_EE_LICENSE_KEY=langfuse_ee_test"
+ echo "LANGFUSE_SKIP_EVALUATOR_MODEL_CALL_VALIDATION=true"
+ echo "LANGFUSE_ENABLE_SCORES_V3_API=true"
+} > "$OUTPUT"
diff --git a/web/Dockerfile b/web/Dockerfile
index 19e246d35c..5eaa232d18 100644
--- a/web/Dockerfile
+++ b/web/Dockerfile
@@ -1,27 +1,111 @@
-# The node alpine image is available here: https://github.com/nodejs/docker-node
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:24-alpine AS alpine
-
-# It's important to update the index before installing packages to ensure you're getting the latest versions.
-# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client zlib
-
-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS build-base
+# FIPS 140-3 base: UBI 9 with the nodejs:24 module-stream RPM
+# instead of node:24-alpine. The RPM Node dynamically links the system
+# libcrypto.so.3, so node:crypto routes through RHEL's CMVP-validated OpenSSL
+# FIPS provider (/usr/lib64/ossl-modules/fips.so). Upstream node images
+# statically bundle their own non-validated OpenSSL and bypass the platform
+# module entirely.
+# The defaults are Red Hat's public UBI9 images, so the build needs no
+# credentials. Iron Bank serves the same content from an authenticated registry
+# with DoD provenance and scan attestations. Release builds opt in by overriding
+# both args after `docker login registry1.dso.mil` (free P1 account plus a CLI
+# secret):
+# --build-arg BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8
+# --build-arg MICRO_BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8
+ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.8
+ARG MICRO_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi-micro:9.8
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} ${BASE_IMAGE} AS base
+
+# nodejs:24 and openssl (incl. the FIPS provider) come from the freely
+# redistributable UBI repos, so no RHEL entitlement is required.
+# nodejs-full-i18n is required: the RPM Node is a small-ICU build and V8
+# segfaults in Intl.Segmenter (used e.g. by the prisma CLI's output
+# formatting) without the full ICU data.
+RUN dnf -y upgrade && \
+ dnf -y module enable nodejs:24 && \
+ dnf -y install --setopt=install_weak_deps=0 nodejs npm nodejs-full-i18n && \
+ dnf clean all
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS build-base
# Pin turbo to avoid nondeterministic prune output from future patch releases.
-RUN npm install [email protected] --global
-ENV PNPM_HOME="/pnpm"
-ENV PATH="$PNPM_HOME:$PATH"
+# pnpm is installed pinned via npm instead of corepack: the corepack shim would
+# fetch pnpm from an unpinned CDN on first use.
# Pass --build-arg NODE_USE_ENV_PROXY=1 when building behind a proxy.
ARG NODE_USE_ENV_PROXY
-RUN npm install --global [email protected] \
- && corepack enable \
- && corepack prepare [email protected] --activate
-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS runtime-base
-# Remove build-only package managers. npm stays here because the runner stage
-# uses it to install prisma and optional dd-trace before removing it.
-RUN rm -rf /usr/local/lib/node_modules/corepack \
- /root/.cache/node/corepack && \
- rm -f /usr/local/bin/corepack /usr/local/bin/yarn /usr/local/bin/yarnpkg
+# Runtime rootfs assembled with dnf --installroot and layered onto ubi9-micro:
+# the final image carries no package manager, python, or curl. It holds only
+# the packages listed below plus their deps, with an intact rpmdb for scanners.
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS rpm-installer
+
+ARG TARGETARCH
+ARG UID=1001
+ARG GID=1001
+
+# nodejs-full-i18n is required: the RPM Node is a small-ICU build and V8
+# segfaults in Intl.Segmenter (used e.g. by the prisma CLI's output
+# formatting) without the full ICU data. sed is used by entrypoint.sh.
+# wget keeps exec-style container health checks working (e.g. `wget --spider
+# http://127.0.0.1:<port>/...` in ECS task definitions); it links GnuTLS,
+# which follows the host's FIPS mode like OpenSSL does.
+RUN dnf -y --installroot /mnt/rootfs --releasever 9 --setopt=install_weak_deps=0 \
+ --disableexcludes=all \
+ module enable nodejs:24 && \
+ dnf -y --installroot /mnt/rootfs --releasever 9 \
+ --setopt=install_weak_deps=0 --setopt=tsflags=nodocs \
+ --disableexcludes=all \
+ install nodejs nodejs-full-i18n bash coreutils-single sed \
+ tzdata ca-certificates crypto-policies wget && \
+ dnf --installroot /mnt/rootfs clean all && \
+ rm -rf /mnt/rootfs/var/cache /mnt/rootfs/var/log/dnf*
+
+# The nodejs RPM drags npm in as a dependency; the runtime image must carry
+# no package manager, so remove it from the rootfs (rpmdb stays consistent).
+RUN if rpm --root /mnt/rootfs -q npm >/dev/null 2>&1; then \
+ rpm --root /mnt/rootfs -e --nodeps npm; \
+ fi
+
+# The FIPS crypto policy for OpenSSL ships as an opt-in config instead of
+# being activated system-wide, so installs without FIPS mode keep the default
+# TLS policy. entrypoint.sh points OPENSSL_CONF at it when
+# LANGFUSE_REQUIRE_FIPS=true; deployments that override the entrypoint set
+# OPENSSL_CONF=/etc/pki/tls/openssl-fips.cnf themselves. The FIPS provider
+# itself loads only when the host kernel runs in FIPS mode; with
+# LANGFUSE_REQUIRE_FIPS=true, startup fails unless it is active and an
+# enterprise license is configured.
+RUN sed 's#/etc/crypto-policies/back-ends/opensslcnf.config#/usr/share/crypto-policies/FIPS/opensslcnf.txt#' \
+ /mnt/rootfs/etc/pki/tls/openssl.cnf > /mnt/rootfs/etc/pki/tls/openssl-fips.cnf && \
+ grep -q '/usr/share/crypto-policies/FIPS/opensslcnf.txt' /mnt/rootfs/etc/pki/tls/openssl-fips.cnf
+
+# dumb-init is not packaged in UBI; install the pinned, checksum-verified
+# static release binary (preserves the semver-covered ENTRYPOINT contract).
+RUN set -eux; \
+ case "${TARGETARCH:-amd64}" in \
+ amd64) di_arch=x86_64; di_sha256=e874b55f3279ca41415d290c512a7ba9d08f98041b28ae7c2acb19a545f1c4df ;; \
+ arm64) di_arch=aarch64; di_sha256=b7d648f97154a99c539b63c55979cd29f005f88430fb383007fe3458340b795e ;; \
+ *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
+ esac; \
+ curl -fsSL -o /mnt/rootfs/usr/bin/dumb-init "https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_${di_arch}"; \
+ echo "${di_sha256} /mnt/rootfs/usr/bin/dumb-init" | sha256sum -c -; \
+ chmod 0755 /mnt/rootfs/usr/bin/dumb-init
+
+# Bake the non-root user into the rootfs (micro has no shadow-utils).
+RUN echo "nodejs:x:${GID}:" >> /mnt/rootfs/etc/group && \
+ echo "nextjs:x:${UID}:${GID}::/home/nextjs:/sbin/nologin" >> /mnt/rootfs/etc/passwd
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} ${MICRO_BASE_IMAGE} AS runtime-base
+COPY --from=rpm-installer /mnt/rootfs /
+
+# prisma CLI + optional dd-trace are npm-installed here and copied into the
+# package-manager-less runtime image.
+FROM --platform=${TARGETPLATFORM:-linux/amd64} build-base AS web-tools
+RUN npm install -g --prefix /opt/prisma-cli --no-package-lock --no-save [email protected]
+ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
+RUN mkdir -p /opt/dd/node_modules && \
+ if [ -n "$NEXT_PUBLIC_LANGFUSE_CLOUD_REGION" ]; then \
+ cd /opt/dd && npm install --no-package-lock --no-save [email protected]; \
+ fi
FROM --platform=${BUILDPLATFORM} golang:1.26 AS migrate-builder
ARG TARGETOS
@@ -30,6 +114,10 @@ ENV CGO_ENABLED=0 \
GOBIN=/out \
GOOS=${TARGETOS} \
GOARCH=${TARGETARCH}
+# Build against the validated Go Cryptographic Module snapshot (Go >= 1.24) so
+# the migrate CLI's ClickHouse TLS uses FIPS-approved crypto; GOFIPS140 also
+# defaults GODEBUG=fips140=on at runtime.
+ENV GOFIPS140=v1.0.0
# Build only the ClickHouse migrate CLI Langfuse uses at runtime.
# compile this ourselves instead of downloading the upstream release
# because prebuilt bins bundle many unused drivers and thus inherit CVEs
@@ -164,25 +252,17 @@ ENV DOCKER_BUILD 0
# Set NEXT_MANUAL_SIG_HANDLE for runtime
ENV NEXT_MANUAL_SIG_HANDLE true
-RUN apk add --no-cache dumb-init tzdata
+# The non-root user (nextjs, uid 1001) is baked into the runtime rootfs by
+# the rpm-installer stage, since micro has no shadow-utils.
-# Don't run production as root
-ARG UID=1001
-ARG GID=1001
-RUN addgroup --system --gid ${GID} nodejs
-RUN adduser --system --uid ${UID} nextjs
+# prisma CLI from the tools stage (bin/prisma symlink + lib/node_modules
+# structure lands under /usr/local); the runtime image has no npm.
+COPY --from=web-tools /opt/prisma-cli /usr/local
-RUN npm install -g --no-package-lock --no-save [email protected]
-
-# Install dd-trace only if NEXT_PUBLIC_LANGFUSE_CLOUD_REGION is configured
-ARG NEXT_PUBLIC_LANGFUSE_CLOUD_REGION
-RUN if [ -n "$NEXT_PUBLIC_LANGFUSE_CLOUD_REGION" ]; then \
- npm install --no-package-lock --no-save [email protected]; \
- fi
-
-# npm is only used for the installs above; remove it from the final runtime image.
-RUN rm -rf /usr/local/lib/node_modules/npm && \
- rm -f /usr/local/bin/npm /usr/local/bin/npx
+# dd-trace, npm-installed in the tools stage (node_modules is empty unless
+# NEXT_PUBLIC_LANGFUSE_CLOUD_REGION was set); the standalone output copied
+# below merges on top of it.
+COPY --from=web-tools --chown=nextjs:nodejs /opt/dd/node_modules ./node_modules
COPY --from=migrate-builder /out/migrate /usr/bin/migrate
@@ -202,7 +282,10 @@ COPY --chown=nextjs:nodejs ./web/entrypoint.sh ./web/entrypoint.sh
COPY --chown=nextjs:nodejs ./packages/shared/scripts/cleanup.sql ./packages/shared/scripts/cleanup.sql
RUN chmod +x ./web/entrypoint.sh
-USER nextjs
+# Numeric USER (uid of nextjs) so runAsNonRoot checks and Iron Bank hardening
+# validation can verify the user without resolving names.
+ARG UID=1001
+USER ${UID}
# Default port to 3000
ENV PORT 3000
diff --git a/web/entrypoint.sh b/web/entrypoint.sh
index aaf947916d..e2a3204b74 100644
--- a/web/entrypoint.sh
+++ b/web/entrypoint.sh
@@ -1,5 +1,20 @@
#!/bin/sh
+# Fail closed when FIPS mode is required but the OpenSSL FIPS provider is not
+# active for Node (e.g. a host without FIPS mode, or an image rebuilt on a
+# non-FIPS base). This runs before the migrations below open database
+# connections. The app additionally requires an enterprise license for FIPS
+# mode at startup (packages/shared/src/server/ee/fips).
+if [ "$LANGFUSE_REQUIRE_FIPS" = "true" ]; then
+ # Opt into the FIPS crypto policy for OpenSSL (see the Dockerfile). An
+ # OPENSSL_CONF set by the deployment takes precedence.
+ export OPENSSL_CONF="${OPENSSL_CONF:-/etc/pki/tls/openssl-fips.cnf}"
+ if ! node -e 'process.exit(require("node:crypto").getFips() === 1 ? 0 : 1)'; then
+ echo "Error: LANGFUSE_REQUIRE_FIPS=true but Node's OpenSSL FIPS provider is not active (crypto.getFips() != 1). Exiting..."
+ exit 1
+ fi
+fi
+
# Check whether a database URL's credentials contain characters that typically
# need percent-encoding for Prisma (@ : / % # ?). Best-effort heuristic —
# strips the scheme, extracts the authority (user:pass@host) before the first
diff --git a/web/src/instrumentation.ts b/web/src/instrumentation.ts
index 5d7a17fefd..5ef2f2d0a3 100644
--- a/web/src/instrumentation.ts
+++ b/web/src/instrumentation.ts
@@ -9,6 +9,19 @@ export async function register() {
const isNodeRuntime = process.env.NEXT_RUNTIME === "nodejs";
+ // Enforce LANGFUSE_REQUIRE_FIPS before any init script connects, also on
+ // secondary replicas that skip init and when entrypoint.sh is overridden.
+ if (isNodeRuntime) {
+ const { assertFipsMode } =
+ await import("@langfuse/shared/src/server/ee/fips");
+ try {
+ assertFipsMode();
+ } catch (error) {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exit(1);
+ }
+ }
+
if (isNodeRuntime && isInitLoadingEnabled) {
console.log("Running init scripts...");
await import("./observability.config");
diff --git a/worker/Dockerfile b/worker/Dockerfile
index 56c523d1ab..c0a4d011b4 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -1,26 +1,101 @@
-# The node alpine image is available here: https://github.com/nodejs/docker-node
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:24-alpine AS alpine
-
-# It's important to update the index before installing packages to ensure you're getting the latest versions.
-# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk update && apk upgrade --no-cache libcrypto3 libssl3 libc6-compat busybox ssl_client zlib
-
-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS build-base
+# FIPS 140-3 base: UBI 9 with the nodejs:24 module-stream RPM
+# instead of node:24-alpine. The RPM Node dynamically links the system
+# libcrypto.so.3, so node:crypto routes through RHEL's CMVP-validated OpenSSL
+# FIPS provider (/usr/lib64/ossl-modules/fips.so). Upstream node images
+# statically bundle their own non-validated OpenSSL and bypass the platform
+# module entirely.
+# The defaults are Red Hat's public UBI9 images, so the build needs no
+# credentials. Iron Bank serves the same content from an authenticated registry
+# with DoD provenance and scan attestations. Release builds opt in by overriding
+# both args after `docker login registry1.dso.mil` (free P1 account plus a CLI
+# secret):
+# --build-arg BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9:9.8
+# --build-arg MICRO_BASE_IMAGE=registry1.dso.mil/ironbank/redhat/ubi/ubi9-micro:9.8
+ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.8
+ARG MICRO_BASE_IMAGE=registry.access.redhat.com/ubi9/ubi-micro:9.8
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} ${BASE_IMAGE} AS base
+
+# nodejs:24 and openssl (incl. the FIPS provider) come from the freely
+# redistributable UBI repos, so no RHEL entitlement is required.
+# nodejs-full-i18n is required: the RPM Node is a small-ICU build and V8
+# segfaults in Intl.Segmenter (used e.g. by the prisma CLI's output
+# formatting) without the full ICU data.
+RUN dnf -y upgrade && \
+ dnf -y module enable nodejs:24 && \
+ dnf -y install --setopt=install_weak_deps=0 nodejs npm nodejs-full-i18n && \
+ dnf clean all
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS build-base
# Pin turbo to avoid nondeterministic prune output from future patch releases.
-RUN npm install [email protected] --global
-ENV PNPM_HOME="/pnpm"
-ENV PATH="$PNPM_HOME:$PATH"
+# pnpm is installed pinned via npm instead of corepack: the corepack shim would
+# fetch pnpm from an unpinned CDN on first use.
# Pass --build-arg NODE_USE_ENV_PROXY=1 when building behind a proxy.
ARG NODE_USE_ENV_PROXY
-RUN npm install --global [email protected] \
- && corepack enable \
- && corepack prepare [email protected] --activate
-FROM --platform=${TARGETPLATFORM:-linux/amd64} alpine AS runtime-base
-# package managers and build-only CLIs only increase exposure to CVEs -> remove them
-RUN rm -rf /usr/local/lib/node_modules/corepack /usr/local/lib/node_modules/npm \
- /root/.cache/node/corepack && \
- rm -f /usr/local/bin/corepack /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/yarn /usr/local/bin/yarnpkg
+# Runtime rootfs assembled with dnf --installroot and layered onto ubi9-micro:
+# the final image carries no package manager, python, or curl. It holds only
+# the packages listed below plus their deps, with an intact rpmdb for scanners.
+FROM --platform=${TARGETPLATFORM:-linux/amd64} base AS rpm-installer
+
+ARG TARGETARCH
+ARG UID=1001
+ARG GID=1001
+
+# nodejs-full-i18n is required: the RPM Node is a small-ICU build and V8
+# segfaults in Intl.Segmenter (used e.g. by the prisma CLI's output
+# formatting) without the full ICU data.
+# wget keeps exec-style container health checks working (e.g. `wget --spider
+# http://127.0.0.1:<port>/...` in ECS task definitions); it links GnuTLS,
+# which follows the host's FIPS mode like OpenSSL does.
+RUN dnf -y --installroot /mnt/rootfs --releasever 9 --setopt=install_weak_deps=0 \
+ --disableexcludes=all \
+ module enable nodejs:24 && \
+ dnf -y --installroot /mnt/rootfs --releasever 9 \
+ --setopt=install_weak_deps=0 --setopt=tsflags=nodocs \
+ --disableexcludes=all \
+ install nodejs nodejs-full-i18n bash coreutils-single sed \
+ tzdata ca-certificates crypto-policies wget && \
+ dnf --installroot /mnt/rootfs clean all && \
+ rm -rf /mnt/rootfs/var/cache /mnt/rootfs/var/log/dnf*
+
+# The nodejs RPM drags npm in as a dependency; the runtime image must carry
+# no package manager, so remove it from the rootfs (rpmdb stays consistent).
+RUN if rpm --root /mnt/rootfs -q npm >/dev/null 2>&1; then \
+ rpm --root /mnt/rootfs -e --nodeps npm; \
+ fi
+
+# The FIPS crypto policy for OpenSSL ships as an opt-in config instead of
+# being activated system-wide, so installs without FIPS mode keep the default
+# TLS policy. entrypoint.sh points OPENSSL_CONF at it when
+# LANGFUSE_REQUIRE_FIPS=true; deployments that override the entrypoint set
+# OPENSSL_CONF=/etc/pki/tls/openssl-fips.cnf themselves. The FIPS provider
+# itself loads only when the host kernel runs in FIPS mode; with
+# LANGFUSE_REQUIRE_FIPS=true, startup fails unless it is active and an
+# enterprise license is configured.
+RUN sed 's#/etc/crypto-policies/back-ends/opensslcnf.config#/usr/share/crypto-policies/FIPS/opensslcnf.txt#' \
+ /mnt/rootfs/etc/pki/tls/openssl.cnf > /mnt/rootfs/etc/pki/tls/openssl-fips.cnf && \
+ grep -q '/usr/share/crypto-policies/FIPS/opensslcnf.txt' /mnt/rootfs/etc/pki/tls/openssl-fips.cnf
+
+# dumb-init is not packaged in UBI; install the pinned, checksum-verified
+# static release binary (preserves the semver-covered ENTRYPOINT contract).
+RUN set -eux; \
+ case "${TARGETARCH:-amd64}" in \
+ amd64) di_arch=x86_64; di_sha256=e874b55f3279ca41415d290c512a7ba9d08f98041b28ae7c2acb19a545f1c4df ;; \
+ arm64) di_arch=aarch64; di_sha256=b7d648f97154a99c539b63c55979cd29f005f88430fb383007fe3458340b795e ;; \
+ *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
+ esac; \
+ curl -fsSL -o /mnt/rootfs/usr/bin/dumb-init "https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_${di_arch}"; \
+ echo "${di_sha256} /mnt/rootfs/usr/bin/dumb-init" | sha256sum -c -; \
+ chmod 0755 /mnt/rootfs/usr/bin/dumb-init
+
+# Bake the non-root user into the rootfs (micro has no shadow-utils).
+RUN echo "expressjs:x:${GID}:" >> /mnt/rootfs/etc/group && \
+ echo "expressjs:x:${UID}:${GID}::/home/expressjs:/sbin/nologin" >> /mnt/rootfs/etc/passwd
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} ${MICRO_BASE_IMAGE} AS runtime-base
+COPY --from=rpm-installer /mnt/rootfs /
FROM --platform=${TARGETPLATFORM:-linux/amd64} build-base AS pruner
@@ -29,24 +104,42 @@ WORKDIR /app
COPY . .
RUN turbo prune worker --docker
-FROM --platform=${TARGETPLATFORM:-linux/amd64} build-base AS builder
+# UBI build toolchain: RPM Node, pnpm, turbo, gcc and Rust. CI's tests-fips job
+# also runs the web and worker test suites on this stage with the OpenSSL FIPS
+# provider forced on (scripts/ci/fips-tests.sh).
+FROM --platform=${TARGETPLATFORM:-linux/amd64} build-base AS toolchain
+
+ARG TARGETARCH
+
+# @langfuse/native is a Rust addon (napi-rs) compiled by `turbo run build` for
+# the glibc (`*-unknown-linux-gnu`) target, linked against UBI9's glibc so it
+# loads in the runtime image. gcc provides the linker; tar and gzip unpack the
+# pinned toolchain; patch (with the base image's curl-minimal) prepares the
+# small clickhouse source overlay used by the native encoder.
+RUN dnf -y install --setopt=install_weak_deps=0 gcc tar gzip patch && \
+ dnf clean all
+
+# Pinned standalone Rust installer, the exact channel from
+# packages/native/rust-toolchain.toml (bump the two together), checksum-verified
+# like the dumb-init binary above. Only rustc, cargo and the target's std are
+# needed to build; clippy and rustfmt run outside Docker.
+RUN set -eux; \
+ case "${TARGETARCH:-amd64}" in \
+ amd64) triple=x86_64-unknown-linux-gnu; toolchain_sha256=aa30409afa67bd1ada244cefd82c7980e6a65bc113bb978e934b2413c75e3900 ;; \
+ arm64) triple=aarch64-unknown-linux-gnu; toolchain_sha256=5fbb4282403046d52a4672765c6761a809bf9f33e699b17e6eb7a93ab7770cc3 ;; \
+ *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
+ esac; \
+ curl -fsSL -o /tmp/rust.tar.gz "https://static.rust-lang.org/dist/rust-1.98.0-${triple}.tar.gz"; \
+ echo "${toolchain_sha256} /tmp/rust.tar.gz" | sha256sum -c -; \
+ tar -xzf /tmp/rust.tar.gz -C /tmp; \
+ /tmp/rust-1.98.0-${triple}/install.sh --prefix=/usr/local \
+ --components=rustc,cargo,rust-std-${triple}; \
+ rm -rf /tmp/rust.tar.gz /tmp/rust-1.98.0-${triple}
+
+FROM --platform=${TARGETPLATFORM:-linux/amd64} toolchain AS builder
WORKDIR /app
-# @langfuse/native is a Rust addon (napi-rs) compiled by `turbo run build`.
-# The toolchain comes from the official Rust image for the same Alpine release,
-# so the addon compiles with the exact version pinned in
-# packages/native/rust-toolchain.toml; bump the two together. gcc and musl-dev
-# provide the linker and C runtime objects the build needs; curl and patch
-# prepare the small clickhouse source overlay used by the native encoder.
-COPY --from=rust:1.98.0-alpine3.24 /usr/local/rustup /usr/local/rustup
-COPY --from=rust:1.98.0-alpine3.24 /usr/local/cargo /usr/local/cargo
-ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo PATH=/usr/local/cargo/bin:$PATH
-# rustc's musl targets default to a `<arch>-linux-musl-gcc` linker name that
-# Alpine does not ship; its gcc is reachable as `cc`.
-ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=cc CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=cc
-RUN apk add --no-cache gcc musl-dev curl patch
-
# First install the dependencies (as they change less often)
COPY --from=pruner /app/out/pnpm-lock.yaml ./pnpm-lock.yaml
COPY --from=pruner /app/out/pnpm-workspace.yaml ./pnpm-workspace.yaml
@@ -94,8 +187,6 @@ FROM --platform=${TARGETPLATFORM:-linux/amd64} runtime-base AS runner
ARG TARGETPLATFORM
ARG BUILDPLATFORM
-RUN apk add --no-cache dumb-init
-
WORKDIR /app
ARG NEXT_PUBLIC_BUILD_ID
@@ -104,16 +195,16 @@ ENV BUILD_ID=$NEXT_PUBLIC_BUILD_ID
ENV NODE_ENV production
ENV DOCKER_BUILD 0
-# Don't run production as root
-ARG UID=1001
-ARG GID=1001
-RUN addgroup --system --gid ${GID} expressjs
-RUN adduser --system --uid ${UID} expressjs
+# The non-root user (expressjs, uid 1001) is baked into the runtime rootfs by
+# the rpm-installer stage, since micro has no shadow-utils.
# Copy only production worker payload instead of full builder workspace (just /prod/worker not entire /app)
COPY --from=prod-deps --chown=expressjs:expressjs /prod/worker ./worker
RUN chmod +x ./worker/entrypoint.sh
-USER expressjs
+# Numeric USER (uid of expressjs) so runAsNonRoot checks and Iron Bank
+# hardening validation can verify the user without resolving names.
+ARG UID=1001
+USER ${UID}
EXPOSE 3030
ENV PORT=3030
diff --git a/worker/entrypoint.sh b/worker/entrypoint.sh
index 6c6cbce9ff..081e8bcf41 100644
--- a/worker/entrypoint.sh
+++ b/worker/entrypoint.sh
@@ -1,5 +1,20 @@
#!/bin/sh
+# Fail closed when FIPS mode is required but the OpenSSL FIPS provider is not
+# active for Node (e.g. a host without FIPS mode, or an image rebuilt on a
+# non-FIPS base). This runs before the migrations below open database
+# connections. The app additionally requires an enterprise license for FIPS
+# mode at startup (packages/shared/src/server/ee/fips).
+if [ "$LANGFUSE_REQUIRE_FIPS" = "true" ]; then
+ # Opt into the FIPS crypto policy for OpenSSL (see the Dockerfile). An
+ # OPENSSL_CONF set by the deployment takes precedence.
+ export OPENSSL_CONF="${OPENSSL_CONF:-/etc/pki/tls/openssl-fips.cnf}"
+ if ! node -e 'process.exit(require("node:crypto").getFips() === 1 ? 0 : 1)'; then
+ echo "Error: LANGFUSE_REQUIRE_FIPS=true but Node's OpenSSL FIPS provider is not active (crypto.getFips() != 1). Exiting..."
+ exit 1
+ fi
+fi
+
# Run cleanup script before running migrations
# Check if DATABASE_URL is not set
if [ -z "$DATABASE_URL" ]; then
diff --git a/worker/src/__tests__/bullmqVersionCheckOptions.test.ts b/worker/src/__tests__/bullmqVersionCheckOptions.test.ts
index 3840c0d1ae..4f1f2fefe6 100644
--- a/worker/src/__tests__/bullmqVersionCheckOptions.test.ts
+++ b/worker/src/__tests__/bullmqVersionCheckOptions.test.ts
@@ -124,6 +124,25 @@ describe("BullMQ Redis version check options", () => {
});
});
+ test("hashes repeatable-job keys with a FIPS-approved algorithm", async () => {
+ const { redisModule } = await importRedisModule();
+
+ // bullmq defaults repeatKeyHashAlgorithm to md5, which an OpenSSL FIPS
+ // provider refuses. Without this setting the legacy-schedule cleanup in
+ // scheduleRecurringJob throws ERR_OSSL_EVP_UNSUPPORTED on a FIPS image,
+ // leaving stale md5-keyed schedules firing alongside the job scheduler.
+ expect(
+ redisModule.createBullMQQueueOptionsWithRedis("test-queue"),
+ ).toMatchObject({
+ settings: { repeatKeyHashAlgorithm: "sha256" },
+ });
+ expect(
+ redisModule.createBullMQWorkerOptionsWithRedis("test-queue"),
+ ).toMatchObject({
+ settings: { repeatKeyHashAlgorithm: "sha256" },
+ });
+ });
+
test("uses worker Redis retry options without disabling offline queue", async () => {
const { redisInstances, redisModule } = await importRedisModule();
const options =
diff --git a/worker/src/ee/fipsMode.ts b/worker/src/ee/fipsMode.ts
new file mode 100644
index 0000000000..ebd164a080
--- /dev/null
+++ b/worker/src/ee/fipsMode.ts
@@ -0,0 +1,11 @@
+import { assertFipsMode } from "@langfuse/shared/src/server/ee/fips";
+
+// Side-effect module: index.ts imports it before anything that connects to
+// Redis, Postgres or ClickHouse, so FIPS mode also holds when a deployment
+// overrides the image's entrypoint.sh.
+try {
+ assertFipsMode();
+} catch (error) {
+ console.error(error instanceof Error ? error.message : String(error));
+ process.exit(1);
+}
diff --git a/worker/src/features/batch-data-retention-cleaner/index.ts b/worker/src/features/batch-data-retention-cleaner/index.ts
index 0b86a96254..0d39b89ab4 100644
--- a/worker/src/features/batch-data-retention-cleaner/index.ts
+++ b/worker/src/features/batch-data-retention-cleaner/index.ts
@@ -89,9 +89,11 @@ function summarizeBacklog(workloads: ProjectWorkload[]) {
/**
* Hash projectId to a short key for ClickHouse parameter names.
+ * sha256 rather than md5: an OpenSSL FIPS provider refuses md5. The key only
+ * names query parameters within one query, so it is never persisted.
*/
function toParamKey(projectId: string): string {
- return createHash("md5").update(projectId).digest("hex").slice(0, 8);
+ return createHash("sha256").update(projectId).digest("hex").slice(0, 8);
}
/**
diff --git a/worker/src/index.ts b/worker/src/index.ts
index 463ef272f5..5dd5112e1d 100644
--- a/worker/src/index.ts
+++ b/worker/src/index.ts
@@ -1,4 +1,5 @@
import "./instrumentation"; // instrumenting the application
+import "./ee/fipsMode"; // enforce LANGFUSE_REQUIRE_FIPS before anything connects
import type { Server } from "http";
import { initializeWorker } from "./initialize";
import { env } from "./env";