RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Revert "Provision native libs from versioned user cache (offline CI, drop /opt pre-bake)" (#22097)
xugangqianged7510638ee1
diff --git a/.dockerignore b/.dockerignoreindex d17b675cf1..4fa3cc2959 100644--- a/.dockerignore+++ b/.dockerignore@@ -38,10 +38,9 @@ target/# regardless of which resources are prepared for the dependency image.# The deps image is built from this directory with:# cd deps && docker build -f Dockerfile -t infiniflow/ragflow_deps .-# tools/scripts is NOT excluded here: the main Dockerfile COPYs it so build.sh-# can `source tools/scripts/native_lib_version.sh` (it does so at the top of the-# script) and reference tools/scripts/download_dev_deps.py in its remediation-# messages while building `./build.sh --go`.+# The native downloader tools/scripts/download_dev_deps.py is copied by the+# main Dockerfile (not excluded here) so build.sh can read its ORT version pin+# when running `./build.sh --go`.deps/*# ── IDE and editor config ──────────────────────────────────────────────────diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.ymlindex 45a00dfe11..558942850c 100644--- a/.github/workflows/sep-tests.yml+++ b/.github/workflows/sep-tests.yml@@ -243,16 +243,6 @@ jobs:export GOCACHE="${RUNNER_WORKSPACE_PREFIX}/gobuild"fi- # Fail fast if the native-lib user cache is not mounted: a missing or- # empty cache would otherwise surface as a cryptic cgo link error later.- NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"- for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do- if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then- echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2- exit 1- fi- done-BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')cleanup_builder() {if [[ -n "${BUILDER_CONTAINER:-}" ]]; then@@ -262,21 +252,11 @@ jobs:trap cleanup_builder EXITTZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}- # Mount the native static-lib user cache into the builder container.- # build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default- # ~/ragflow-native-libs); this pins the same root for the container and- # bind-mounts it read-only so the cgo link finds the versioned- # v<version>/ dirs without an /opt pre-bake. CI mounts the cache from- # the build host (populated by download_dev_deps.py); a missing mount is- # caught by the assert below, not by a cryptic cgo link error.- export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \-e TZ="${TZ}" \-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \- -e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \-v "${PWD}:/ragflow" \-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \- -v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \infiniflow/infinity_builder:ubuntu22_clang20sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'./build.sh --go@@ -830,16 +810,6 @@ jobs:export GOCACHE="${RUNNER_WORKSPACE_PREFIX}/gobuild"fi- # Fail fast if the native-lib user cache is not mounted: a missing or- # empty cache would otherwise surface as a cryptic cgo link error later.- NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"- for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do- if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then- echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2- exit 1- fi- done-BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_${DOC_ENGINE}_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')cleanup_builder() {if [[ -n "${BUILDER_CONTAINER:-}" ]]; then@@ -849,21 +819,11 @@ jobs:trap cleanup_builder EXITTZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}- # Mount the native static-lib user cache into the builder container.- # build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default- # ~/ragflow-native-libs); this pins the same root for the container and- # bind-mounts it read-only so the cgo link finds the versioned- # v<version>/ dirs without an /opt pre-bake. CI mounts the cache from- # the build host (populated by download_dev_deps.py); a missing mount is- # caught by the assert below, not by a cryptic cgo link error.- export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \-e TZ="${TZ}" \-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \- -e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \-v "${PWD}:/ragflow" \-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \- -v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \infiniflow/infinity_builder:ubuntu22_clang20sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'./build.sh --go@@ -1426,16 +1386,6 @@ jobs:# integration tests; the per-engine go jobs build the full server.run: |set -euo pipefail- # Fail fast if the native-lib user cache is not mounted: a missing or- # empty cache would otherwise surface as a cryptic cgo link error later.- NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"- for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do- if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then- echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2- exit 1- fi- done-BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_native_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')cleanup_builder() {if [[ -n "${BUILDER_CONTAINER:-}" ]]; then@@ -1445,21 +1395,11 @@ jobs:trap cleanup_builder EXITTZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}- # Mount the native static-lib user cache into the builder container.- # build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default- # ~/ragflow-native-libs); this pins the same root for the container and- # bind-mounts it read-only so the cgo link finds the versioned- # v<version>/ dirs without an /opt pre-bake. CI mounts the cache from- # the build host (populated by download_dev_deps.py); a missing mount is- # caught by the assert below, not by a cryptic cgo link error.- export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \-e TZ="${TZ}" \-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \- -e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \-v "${PWD}:/ragflow" \-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \- -v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \infiniflow/infinity_builder:ubuntu22_clang20sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'@@ -1546,16 +1486,6 @@ jobs:# starts only the dependency services, not the ragflow-cpu container.run: |set -euo pipefail- # Fail fast if the native-lib user cache is not mounted: a missing or- # empty cache would otherwise surface as a cryptic cgo link error later.- NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"- for lib in office_oxide pdfium-static pdf_oxide onnxruntime; do- if [ ! -d "${NATIVE_LIBS_DIR}/${lib}" ]; then- echo "ERROR: ${NATIVE_LIBS_DIR}/${lib} missing — native-lib cache not mounted on this runner." >&2- exit 1- fi- done-BUILDER_CONTAINER=ragflow_build_${GITHUB_RUN_ID}_go_int_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')cleanup_builder() {if [[ -n "${BUILDER_CONTAINER:-}" ]]; then@@ -1564,21 +1494,11 @@ jobs:}trap cleanup_builder EXITTZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}- # Mount the native static-lib user cache into the builder container.- # build.sh resolves the libs from RAGFLOW_NATIVE_LIBS_DIR (default- # ~/ragflow-native-libs); this pins the same root for the container and- # bind-mounts it read-only so the cgo link finds the versioned- # v<version>/ dirs without an /opt pre-bake. CI mounts the cache from- # the build host (populated by download_dev_deps.py); a missing mount is- # caught by the assert below, not by a cryptic cgo link error.- export RAGFLOW_NATIVE_LIBS_DIR="${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"sudo docker run --privileged -d --name "${BUILDER_CONTAINER}" \-e TZ="${TZ}" \-e UV_INDEX=https://mirrors.aliyun.com/pypi/simple \- -e "RAGFLOW_NATIVE_LIBS_DIR=${RAGFLOW_NATIVE_LIBS_DIR}" \-v "${PWD}:/ragflow" \-v "${PWD}/internal/binding/cpp/resource:/usr/share/infinity/resource" \- -v "${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}:ro" \infiniflow/infinity_builder:ubuntu22_clang20sudo docker exec "${BUILDER_CONTAINER}" bash -c 'git config --global safe.directory "*" && cd /ragflow && ./build.sh --cpp'diff --git a/Dockerfile b/Dockerfileindex 30acc382a0..0d15c07785 100644--- a/Dockerfile+++ b/Dockerfile@@ -227,21 +227,19 @@ RUN --mount=type=cache,id=ragflow_gomod,target=/root/.cache/gomod,sharing=lockedCOPY internal internalCOPY cmd cmdCOPY build.sh ./-COPY tools/scripts tools/scriptsCOPY Dockerfile ./-# ONNX Runtime static archives are provisioned by build.sh from the versioned-# user cache at ONNXRUNTIME_STATIC_PREFIX (${RAGFLOW_NATIVE_LIBS_DIR:-$HOME/-# ragflow-native-libs}/onnxruntime/v<version>). There is NO /opt pre-bake: on CI-# the host cache is mounted in at build time, and for the product image build-# (where no host cache is mounted) build.sh falls back to _ensure_native_libs_installed,-# which runs `uv run tools/scripts/download_dev_deps.py`. That downloader fetches-# the native archives on demand and self-creates its deps/ workdir when absent-# (see download_dev_deps.py::ensure_deps_dir), so it no longer crashes on a bare-# go-builder checkout. These .a files are consumed at link time only and do NOT-# enter the final image (only the compiled /ragflow/bin is COPY --from=go-builder).-# ARG ORT_VERSION stays because build.sh's check_ort_version_consistency greps it-# to cross-check the pinned ORT release against internal/common/environments.go.+# ONNX Runtime static archives: build.sh's _seed_from_system copies the ORT+# static libs from the runner's pre-baked /opt/ragflow-native-libs/onnxruntime+# (arch-correct per TARGETARCH) into the user-cache path ONNXRUNTIME_STATIC_PREFIX+# before linking. These .a files are consumed at link time only and do NOT enter+# the final image (only the compiled /ragflow/bin is COPY --from=go-builder).+# The Dockerfile no longer pre-copies them here: pre-creating the user-cache dir+# would make _seed_from_system treat it as "already cached" and skip the copy on+# the non-matching architecture (e.g. the arm64 runner has no x64 dir), so we+# rely on build.sh to seed ORT for both amd64 and arm64. ARG ORT_VERSION stays+# because build.sh's check_ort_version_consistency greps it to cross-check the+# pinned ORT release against internal/common/environments.go.ARG ORT_VERSION=1.29.0RUN git config --global safe.directory "*" && \diff --git a/build.sh b/build.shindex 37e0d3f3ac..e924517ce0 100755--- a/build.sh+++ b/build.sh@@ -15,18 +15,6 @@ PROJECT_ROOT="$SCRIPT_DIR"# untouched and merge conflicts against the EE branch are limited to this# single line plus the guarded hook call inside run_native_integration_tests.[[ -f "${SCRIPT_DIR}/build_ee.sh" ]] && source "${SCRIPT_DIR}/build_ee.sh"-# Version-keyed native-lib resolution: each native lib sits under a-# v<version>/ subdir of the user cache (~/ragflow-native-libs/<lib>/v<version>/),-# so a single mounted cache holds several versions at once. The version numbers-# come from go.mod (office_oxide / pdf_oxide), environments.go (ORT) and-# native_lib_version.sh (pdfium) — see that file for the single source of truth.-source "${SCRIPT_DIR}/tools/scripts/native_lib_version.sh"--# Native static-lib user-cache root. Overridable via RAGFLOW_NATIVE_LIBS_DIR so-# CI can mount the cache at a fixed host path (sep-tests.yml Slice 3); defaults-# to ~/ragflow-native-libs. Captured once here and reused by every <lib>/v<version>-# prefix below so the whole script agrees on a single cache root.-NATIVE_LIBS_DIR="$(ragflow_native_libs_dir)"# Build directoriesCPP_DIR="$PROJECT_ROOT/internal/binding/cpp"@@ -55,42 +43,31 @@ if [ -n "$EXTRA_GO_TAGS" ]; thenfi# Native static library settings. These are the user-cache paths (~/ragflow-native-libs/).-# Each lib lives under a version-keyed subdir (v<version>/) placed there by-# tools/scripts/download_dev_deps.py (idempotent, version-pinned) — run once by a-# developer, or mounted in from the build host on CI. There is NO /opt pre-bake and-# NO flat (version-less) fallback: a missing version dir means the cache is-# incomplete and build.sh triggers the downloader itself (see setup_cgo_env).--# office_oxide native library settings — static linking. Version is derived from-# go.mod (single source of truth, exactly as tools/scripts/download_dev_deps.py-# derives it). An empty value (module absent from go.mod) would produce a broken-# .../lib/v link path, so it is fatal. The prefix is set after the version, since-# it embeds the version.-OFFICE_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/office_oxide/go" "$PROJECT_ROOT")"+# If /opt/ragflow-native-libs/ exists (pre-seeded in CI runner image), it takes priority+# and skips the network (download_dev_deps.py) fallback.+SYSTEM_DEPS="/opt/ragflow-native-libs"++# office_oxide native library settings — static linking+OFFICE_OXIDE_PREFIX="${HOME}/ragflow-native-libs/office_oxide"+# Derived from go.mod (single source of truth: the Go module version). Matches+# tools/scripts/download_dev_deps.py's office_oxide_url derivation. The pipeline exits 0+# with an empty value when go.mod has no matching entry (no pipefail here), so+# the empty case needs its own check: an empty version would slip past the+# stale-lib comparison below and produce a broken .../lib/v link path.+OFFICE_OXIDE_VERSION="$(grep -m1 -E '^[[:space:]]*github.com/yfedoseev/office_oxide/go[[:space:]]+v' "$PROJECT_ROOT/go.mod" | sed -E 's/.*v([0-9][^[:space:]]*).*/\1/')"if [ -z "$OFFICE_OXIDE_VERSION" ]; thenecho -e "${RED}Error: could not read office_oxide version from go.mod${NC}" >&2; exit 1fi-OFFICE_OXIDE_PREFIX="${NATIVE_LIBS_DIR}/office_oxide/v${OFFICE_OXIDE_VERSION}"--# pdfium native library settings — static linking (kognitos/pdfium-static).-# Version is read from the single source of truth tools/scripts/native_lib_version.sh-# (pdfium_static_version), exactly as download_dev_deps.py does, so the bare build-# number lives in exactly one place. An empty value is fatal.-PDFIUM_STATIC_VERSION="$(pdfium_static_version "$PROJECT_ROOT")"-if [ -z "$PDFIUM_STATIC_VERSION" ]; then- echo -e "${RED}Error: could not read pdfium version from native_lib_version.sh${NC}" >&2; exit 1-fi-PDFIUM_STATIC_PREFIX="${NATIVE_LIBS_DIR}/pdfium-static/v${PDFIUM_STATIC_VERSION}"--# pdf_oxide native library settings — static linking (go-ffi tarball). Version-# is read from go.mod (single source of truth) instead of a hardcoded constant,-# so bumping the dependency needs no build.sh edit. The empty case is fatal for-# the same reason as office_oxide. The prefix is set after the version.-PDF_OXIDE_VERSION="$(go_module_version "github.com/yfedoseev/pdf_oxide/go" "$PROJECT_ROOT")"-if [ -z "$PDF_OXIDE_VERSION" ]; then- echo -e "${RED}Error: could not read pdf_oxide version from go.mod${NC}" >&2; exit 1-fi-PDF_OXIDE_PREFIX="${NATIVE_LIBS_DIR}/pdf_oxide/v${PDF_OXIDE_VERSION}"++# pdfium native library settings — static linking (kognitos/pdfium-static)+# Pinned here (not grepped from the dev downloader) so build.sh stays tied to the+# checked-out source tree and the Docker build needs no extra COPY of the script.+PDFIUM_STATIC_PREFIX="${HOME}/ragflow-native-libs/pdfium-static"+PDFIUM_STATIC_VERSION="7809"++# pdf_oxide native library settings — static linking (go-ffi tarball)+PDF_OXIDE_PREFIX="${HOME}/ragflow-native-libs/pdf_oxide"+PDF_OXIDE_VERSION="0.3.73"# onnxruntime native library settings — static linking for the in-process# (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary@@ -98,7 +75,7 @@ PDF_OXIDE_PREFIX="${NATIVE_LIBS_DIR}/pdf_oxide/v${PDF_OXIDE_VERSION}"# is then resolved via dlopen(NULL) (the process-global symbol table, not the# executable's own path), so no libonnxruntime.so is needed at runtime.# Downloaded by tools/scripts/download_dev_deps.py into onnxruntime/static_lib.-ONNXRUNTIME_STATIC_PREFIX="${NATIVE_LIBS_DIR}/onnxruntime/static_lib"+ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib"# ── target platform detection ──────────────────────────────────────────# Defaults to the build host (uname), overridable with RAGFLOW_TARGET_OS /@@ -178,53 +155,27 @@ ort_static_lib_prefix() {esac}-# ── native static library provisioning ──────────────────────────────────-# The native .a files live in the version-keyed user cache-# (~/ragflow-native-libs/<lib>/v<version>/), placed there by-# tools/scripts/download_dev_deps.py — idempotently and version-pinned — which a-# developer runs once, and which CI mounts in from the build host. There is NO-# /opt pre-bake and NO flat (version-less) fallback any more: a missing version-# dir means the cache is incomplete, so build.sh triggers the downloader itself-# (controlled auto-download) rather than silently linking a stale or wrong lib.--# True when every native lib the Go build links is already in the versioned user-# cache for the target platform. Used to skip the network entirely on an-# already-provisioned host.-_native_libs_present() {- [ -f "${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a" ] || return 1- [ -f "${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h" ] || return 1- [ -f "${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a" ] || return 1- [ -f "${PDF_OXIDE_PREFIX}/lib/${TARGET_OS}_${TARGET_ARCH}/libpdf_oxide.a" ] || return 1- # ORT is version+platform keyed under static_lib; any real .a there is- # authoritative (the version-stamped symlink dir created later holds only- # symlinks, so -type f skips them and matches the real archives).- [ -n "$(find "$ONNXRUNTIME_STATIC_PREFIX" -type f -name '*.a' 2>/dev/null | head -1)" ] || return 1- return 0-}+# Copy a dependency from the system pre-seed directory to the user cache.+# Returns 0 if the dep was copied or already exists in cache, 1 otherwise.+_seed_from_system() {+ local dep_name="$1" # e.g. "pdfium-static", "pdf_oxide", "office_oxide"+ local dep_dir="${HOME}/ragflow-native-libs/${dep_name}"+ local sys_dir="${SYSTEM_DEPS}/${dep_name}"-# Controlled auto-download: if the versioned native libs are not yet in the user-# cache, run the one-stop downloader ONCE. It is idempotent and version-pinned,-# so it can only land the exact pinned .a — never a wrong version. Fail loud only-# when the cache is still incomplete afterwards (a locked-down host with no-# network and no pre-placed libs).-_ensure_native_libs_installed() {- _native_libs_present && return 0- echo "One or more native static libraries are missing from the user cache;"- echo " running downloader: uv run tools/scripts/download_dev_deps.py"- ( cd "$PROJECT_ROOT" && uv run tools/scripts/download_dev_deps.py ) || {- echo -e "${RED}Error: native library downloader failed.${NC}" >&2- exit 1- }- if _native_libs_present; then+ echo "check if dep ${dep_name} exists in ${dep_dir} or ${sys_dir}"++ if [ -d "$dep_dir" ]; then+ echo " ${dep_name} → ${dep_dir} (user cache)"+ return 0 # already cached+ fi+ if [ -d "$sys_dir" ]; then+ echo " ${dep_name} → ${sys_dir} (system)"+ mkdir -p "$(dirname "$dep_dir")"+ cp -r "$sys_dir" "$dep_dir"return 0fi- echo -e "${RED}Error: native static libraries still missing after download.${NC}" >&2- echo " Pre-place the version-keyed dirs manually on a locked-down host:" >&2- echo " ${OFFICE_OXIDE_PREFIX}" >&2- echo " ${PDFIUM_STATIC_PREFIX}" >&2- echo " ${PDF_OXIDE_PREFIX}" >&2- echo " ${ONNXRUNTIME_STATIC_PREFIX}" >&2- exit 1+ echo " ${dep_name} not found in system or user cache"+ return 1}# ── cl100k_base BPE table ─────────────────────────────────────────────@@ -449,11 +400,6 @@ check_onnxruntime_deps() {# another OS/arch, and the Python side legitimately keeps another ORT# version in the same prefix. Matching both is why coexisting version# dirs need no separate fail-fast guard here.- # The ORT static archives are resolved from the version-keyed user cache- # (~/ragflow-native-libs/onnxruntime/static_lib, which the downloader- # populates and CI mounts in). There is no /opt pre-bake to reseed from; an- # incomplete cache is filled by the controlled auto-download in setup_cgo_env.-local ort_dir_prefixort_dir_prefix="$(ort_static_lib_prefix)"while IFS= read -r f; do@@ -486,7 +432,7 @@ check_onnxruntime_deps() {echo " available: ${avail:-<none>}" >&2echo " DeepDocORTVersion=${ort_version}; fetch it with:" >&2echo " uv run tools/scripts/download_dev_deps.py" >&2- echo " or pre-seed ${NATIVE_LIBS_DIR}/onnxruntime (CI mounts the cache from the build host)." >&2+ echo " or pre-seed /opt/ragflow-native-libs/onnxruntime (CI image)." >&2return 1fi@@ -549,9 +495,7 @@ check_onnxruntime_deps() {# Check office_oxide native librarycheck_office_oxide_deps() {print_section "Checking office_oxide native library"- # office_oxide is resolved from the version-keyed user cache- # (${OFFICE_OXIDE_PREFIX}). An incomplete cache is filled by the controlled- # auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.+ _seed_from_system "office_oxide" || truelocal lib_file="liboffice_oxide.a"local lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"@@ -572,12 +516,28 @@ check_office_oxide_deps() {if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; thenlocal found_versionfound_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)- echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"- echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"- echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"- echo " rm -rf ~/ragflow-native-libs/office_oxide deps/$(office_oxide_asset)"- echo " uv run tools/scripts/download_dev_deps.py"- exit 1+ echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}"+ rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/deps/$(office_oxide_asset)"+ if ! (cd "${PROJECT_ROOT}" && uv run tools/scripts/download_dev_deps.py); then+ echo -e "${RED}Error: office_oxide native lib version mismatch${NC}"+ echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"+ echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"+ echo " rm -rf ~/ragflow-native-libs/office_oxide deps/$(office_oxide_asset)"+ echo " uv run tools/scripts/download_dev_deps.py"+ exit 1+ fi+ lib_path="${OFFICE_OXIDE_PREFIX}/lib/${lib_file}"+ header_path="${OFFICE_OXIDE_PREFIX}/include/office_oxide_c/office_oxide.h"+ if [ ! -f "$lib_path" ] || [ ! -f "$header_path" ]; then+ echo -e "${RED}Error: office_oxide native library not found after refresh${NC}"+ exit 1+ fi+ if ! strings "$lib_path" 2>/dev/null | grep -Fxq "$OFFICE_OXIDE_VERSION"; then+ found_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)+ echo -e "${RED}Error: office_oxide native lib version mismatch after refresh${NC}"+ echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"+ exit 1+ fifiecho "✓ office_oxide v${OFFICE_OXIDE_VERSION} native library found at ${OFFICE_OXIDE_PREFIX}"@@ -586,9 +546,7 @@ check_office_oxide_deps() {# Check pdfium static library.check_pdfium_deps() {- # pdfium is resolved from the version-keyed user cache- # (${PDFIUM_STATIC_PREFIX}). An incomplete cache is filled by the controlled- # auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.+ _seed_from_system "pdfium-static" || truelocal lib_path="${PDFIUM_STATIC_PREFIX}/lib/libpdfium.a"if [ -f "$lib_path" ]; then@@ -646,9 +604,7 @@ pdf_oxide_validate_version() {}check_pdf_oxide_deps() {- # pdf_oxide is resolved from the version-keyed user cache- # (${PDF_OXIDE_PREFIX}). An incomplete cache is filled by the controlled- # auto-download in setup_cgo_env; there is no /opt pre-bake to recover from.+ _seed_from_system "pdf_oxide" || true# Map platform to tarball-internal subdirectory (lib/<os>_<arch>/). Reuses# TARGET_OS/TARGET_ARCH set by detect_target_platform instead of re-deriving# from uname, so it stays consistent with the other check_*_deps helpers.@@ -661,10 +617,10 @@ check_pdf_oxide_deps() {local lib_path="${PDF_OXIDE_PREFIX}/lib/${platform_subdir}/libpdf_oxide.a"if [ -f "$lib_path" ]; then- # Verify the on-disk lib matches the pinned version. The cache is- # version-keyed (${PDF_OXIDE_PREFIX}), so an earlier pin lives under a- # different v<version>/ dir and is never silently reused; this check is a- # defense-in-depth against a stale .a that somehow landed in this dir.+ # Verify the on-disk lib matches the pinned version. _seed_from_system+ # accepts an existing user-cache directory without inspecting it, so a+ # lib left over from an earlier pin is reused silently and the upgrade+ # becomes a no-op.## The version marker "pdf_oxide <version>" is merged into Rust's string# constant pool (unlike office_oxide's standalone "0.1.9" line), so we@@ -842,12 +798,12 @@ build_go() {echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2echo " Fetch the static libs with:" >&2echo " uv run tools/scripts/download_dev_deps.py" >&2- echo " or pre-seed them at ${NATIVE_LIBS_DIR}/onnxruntime (CI mounts the cache from the build host)." >&2+ echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2echo " This production binary must statically include ORT — there is no" >&2echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2echo " - the ORT static_lib dir was never seeded: run" >&2echo " 'uv run tools/scripts/download_dev_deps.py', or pre-seed" >&2- echo " ${NATIVE_LIBS_DIR}/onnxruntime as the CI mount does;" >&2+ echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2return 1fi@@ -881,22 +837,25 @@ build_go() {setup_cgo_env() {detect_target_platform- # Controlled auto-download: fill any missing version-keyed native lib from- # the user cache before resolving individual libs. On an already-provisioned- # host (the common CI case, where the cache is mounted in) this is a no-op;- # on a fresh host it runs the one-stop downloader once. There is no /opt- # pre-bake and no flat fallback to recover from.- _ensure_native_libs_installed-# ── office_oxide ──────────────────────────────────────────────────check_office_oxide_deps- # The office_oxide prefix is already version-keyed- # (${OFFICE_OXIDE_PREFIX} = .../office_oxide/v<version>), so the linked path- # carries the version and Go's build cache (which keys CGO_LDFLAGS as a- # string, not the .a contents) automatically relinks when the version bumps.+ # Go's build cache keys CGO_LDFLAGS as a string — it does NOT hash the+ # file content of referenced .a archives. So swapping the .a in-place+ # (same path) does NOT invalidate the cache, and `go build` silently+ # reuses a stale binary linked against the old .a.+ #+ # Create a version-stamped symlink directory so the flag string+ # includes the actual linked version. When the .a is upgraded, the+ # path changes → Go cache key changes → automatic relink.+ local office_oxide_lib_dir="${OFFICE_OXIDE_PREFIX}/lib"+ local versioned_lib_dir="${office_oxide_lib_dir}/v${OFFICE_OXIDE_VERSION}"+ mkdir -p "$versioned_lib_dir"+ ln -sf "${office_oxide_lib_dir}/liboffice_oxide.a" \+ "${versioned_lib_dir}/liboffice_oxide.a"+export CGO_CFLAGS="-I${OFFICE_OXIDE_PREFIX}/include/office_oxide_c${CGO_CFLAGS:+ $CGO_CFLAGS}"- export CGO_LDFLAGS="${OFFICE_OXIDE_PREFIX}/lib/liboffice_oxide.a"+ export CGO_LDFLAGS="${versioned_lib_dir}/liboffice_oxide.a"# ── pdfium ────────────────────────────────────────────────────────check_pdfium_deps || return 1@@ -935,11 +894,16 @@ setup_cgo_env() {linux_amd64|linux_arm64|darwin_amd64|darwin_arm64) ;;*) echo "pdf_oxide: unsupported platform: $pdf_oxide_subdir"; return 1 ;;esac- # The pdf_oxide prefix is already version-keyed- # (${PDF_OXIDE_PREFIX} = .../pdf_oxide/v<version>), so the linked path- # carries the version and Go's build cache relinks on a version bump.+ # Version-stamp the archive path so an in-place .a upgrade invalidates+ # Go's build cache. See the office_oxide block above for why the raw path+ # is not enough: the cache key hashes CGO_LDFLAGS as a string, not the+ # contents of the archives it names.local pdf_oxide_lib_dir="${PDF_OXIDE_PREFIX}/lib/${pdf_oxide_subdir}"- export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_lib_dir}/libpdf_oxide.a"+ local pdf_oxide_versioned_dir="${pdf_oxide_lib_dir}/v${PDF_OXIDE_VERSION}"+ mkdir -p "$pdf_oxide_versioned_dir"+ ln -sf "${pdf_oxide_lib_dir}/libpdf_oxide.a" \+ "${pdf_oxide_versioned_dir}/libpdf_oxide.a"+ export CGO_LDFLAGS="$CGO_LDFLAGS ${pdf_oxide_versioned_dir}/libpdf_oxide.a"# ── onnxruntime (static, resolved via dlopen(NULL)) ────────────────# Statically link libonnxruntime*.a into the binary. The org Go binding@@ -966,10 +930,11 @@ setup_cgo_env() {# all unused objects and missing transitive dependencies), and exports# it via -export_dynamic so runtime dlopen(NULL) resolves it.#- # The static ORT archives come from the version-keyed user cache (populated- # by download_dev_deps.py and mounted in on CI) — there is no /opt pre-bake.- # An incomplete cache is filled by the controlled auto-download at the top- # of setup_cgo_env.+ # Seed the static ORT archives from the system pre-bake (/opt, laid down+ # by the CI runner image) into the user cache before the link check+ # below. Mirrors the existing _seed_from_system calls for the other+ # native libs so CI never downloads ORT at build/test time.+ _seed_from_system "onnxruntime" || truecheck_onnxruntime_deps || return 1# Version-stamp the archive paths so an in-place .a upgrade invalidatesdiff --git a/internal/development.md b/internal/development.mdindex a563b25bae..a0b388b566 100644--- a/internal/development.md+++ b/internal/development.md@@ -214,11 +214,11 @@ The image contains Go DeepDoc weights, tokenizer assets, and the BPE table. Deep> Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from> `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If> you see `Error: ONNX Runtime static libraries are not linked`, run-> `uv run tools/scripts/download_dev_deps.py` (or pre-seed the versioned user-> cache at `${RAGFLOW_NATIVE_LIBS_DIR:-$HOME/ragflow-native-libs}/onnxruntime/-> v<version>`; CI mounts the host cache into the builder). There is no ORT-free-> production build path — if ORT is absent the binary fails at startup, so the-> remedy is always to seed the static lib above, never to build without it.+> `uv run tools/scripts/download_dev_deps.py` (or pre-seed+> `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is+> no ORT-free production build path — if ORT is absent the binary fails at+> startup, so the remedy is always to seed the static lib above, never to build+> without it.> **Note**: The ONNX Runtime native version is pinned in a few places that must> stay in sync. Bumping it in one spot and not the others fails the build withdiff --git a/tools/scripts/download_dev_deps.py b/tools/scripts/download_dev_deps.pyindex 3e72753473..d81d8758e2 100755--- a/tools/scripts/download_dev_deps.py+++ b/tools/scripts/download_dev_deps.py@@ -57,9 +57,7 @@ import hashlibimport osimport platformimport re-import shleximport shutil-import subprocessimport sysimport tarfileimport tempfile@@ -68,17 +66,11 @@ import zipfileimport requests# ── Dependency versions & URLs (edit here) ────────────────────────────────-# The native-library VERSIONS are NOT hardcoded in this script. They are derived-# from a single source of truth so build.sh and this downloader can never drift:-# - office_oxide / pdf_oxide -> go.mod (go_module_version, below)-# - ORT -> internal/common/environments.go DeepDocORTVersion-# (deepdoc_ort_version, below)-# - pdfium -> tools/scripts/native_lib_version.sh-# pdfium_static_version (a shared const; pdfium-# is not a Go module so go.mod does not apply)-# ORT_VERSION / PDFIUM_VERSION are bound to those derivations right after-# REPO_ROOT is resolved (see the block following _repo_root()), so bumping a-# version touches exactly one place upstream.+# This script's own copy of the native static-library versions and download URLs+# it fetches. build.sh does NOT grep these (it pins PDFIUM/PDF_OXIDE itself and+# reads ORT from internal/common/environments.go + the Dockerfile ARG, so it+# stays tied to the checked-out source tree and the Docker build needs no COPY+# of this script) — keep the two in sync when bumping a version.# ONNX Runtime (statically linked into the Go in-process DeepDoc backend).#@@ -91,6 +83,7 @@ import requests# patched content (e.g. the build that exports SessionGetInitializer*); the# downloader detects that via a `{asset}.sha256` sidecar and# re-downloads/re-extracts, so a stale local copy never silently lingers.+ORT_VERSION = "1.29.0"# Release tag on infiniflow/ragflow-build. v1.29.0's Linux x86_64 minimal build# reduced operators to required_operators.config, which omitted Clip@13 — so@@ -108,9 +101,9 @@ ORT_RELEASE_TAG = "onnxruntime-v1.29.0-patch2"PDF_OXIDE_VERSION = "0.3.73"# pdfium release, as the numeric build number after "chromium%2F" (the "v" and-# the URL-encoded slash are added by pdfium_url). Its bare value is bound to-# pdfium_static_version(REPO_ROOT) below (the single shared const, since pdfium-# is not a Go module).+# the URL-encoded slash are added by pdfium_url). Matches build.sh's+# PDFIUM_STATIC_VERSION.+PDFIUM_VERSION = "7809"def host_platform(target_os=None, target_arch=None):@@ -192,48 +185,6 @@ def go_module_version(module: str, repo_root: str) -> str:return match.group(1)-def deepdoc_ort_version(repo_root: str) -> str:- """Read the ONNX Runtime version the in-process (Go) DeepDoc backend links- from internal/common/environments.go's DeepDocORTVersion const.-- ORT is the one native lib whose version is owned by Go source (not go.mod),- so both build.sh and this script parse the SAME const — no second hardcoded- copy to drift."""- path = os.path.join(repo_root, "internal", "common", "environments.go")- if not os.path.isfile(path):- raise RuntimeError(f"environments.go not found at {path}")- with open(path, encoding="utf-8") as file:- match = re.search(r'DeepDocORTVersion\s*=\s*"([^"]+)"', file.read())- if match is None:- raise RuntimeError(f"DeepDocORTVersion const not found in {path}")- return match.group(1)---def pdfium_static_version(repo_root: str) -> str:- """Return the pinned pdfium build number from the single shared bash source- of truth tools/scripts/native_lib_version.sh.-- pdfium (kognitos/pdfium-static) is not a Go module, so unlike- office_oxide / pdf_oxide it cannot be derived from go.mod; its bare numeric- build number is the one constant exported by pdfium_static_version() and- sourced by both this script and build.sh."""- script = os.path.join(repo_root, "tools", "scripts", "native_lib_version.sh")- if not os.path.isfile(script):- raise RuntimeError(f"native_lib_version.sh not found at {script}")- result = subprocess.run(- ["bash", "-c", f"source {shlex.quote(script)} >/dev/null 2>&1; pdfium_static_version"],- capture_output=True,- text=True,- check=False,- )- if result.returncode != 0:- raise RuntimeError(f"pdfium_static_version failed: {result.stderr.strip()}")- version = result.stdout.strip()- if not version:- raise RuntimeError("pdfium_static_version produced no output")- return version--# ── URL builders ──────────────────────────────────────────────────────────# Every download URL is assembled from the constants above via one of these# helpers, so a version bump touches exactly one constant. Each takes the target@@ -318,15 +269,6 @@ def _repo_root() -> str:REPO_ROOT = _repo_root()--# Bind the native-library versions to their single source of truth (see the-# version block above). Keeping these as module-level constants means every-# downstream URL/dir builder still references ORT_VERSION / PDFIUM_VERSION, but-# the bare values now live in exactly one place upstream (environments.go and-# native_lib_version.sh) — bumping a version needs no edit here.-ORT_VERSION = deepdoc_ort_version(REPO_ROOT)-PDFIUM_VERSION = pdfium_static_version(REPO_ROOT)-# Downloaded archives are anchored under deps/ (ignored by .gitignore# via *.tgz / deps/*.zip) regardless of the caller's working directory,# so a move of this script does not scatter large files into tools/scripts/.@@ -334,28 +276,6 @@ RAGFLOW_DEPS_DIR = os.path.join(REPO_ROOT, "deps")NATIVE_LIBS_DIR = os.path.expanduser("~/ragflow-native-libs")-def native_libs_dir():- """Resolve where native static libraries are written/read.-- Honours RAGFLOW_NATIVE_LIBS_DIR so a CI builder can mount a pre-populated- user cache (e.g. ~/ragflow-native-libs) at a configurable path and the- downloader agrees with build.sh on that location. Falls back to the- in-home default when the variable is unset or empty.- """- return os.environ.get("RAGFLOW_NATIVE_LIBS_DIR") or NATIVE_LIBS_DIR---def ensure_deps_dir():- """Make sure RAGFLOW_DEPS_DIR exists before any chdir into it.-- In a product image build the go-builder stage does not carry a deps/- checkout, so a bare os.chdir(RAGFLOW_DEPS_DIR) would raise- FileNotFoundError before the auto-download fallback could run. Creating it- here lets the downloader proceed and fetch archives on demand.- """- os.makedirs(RAGFLOW_DEPS_DIR, exist_ok=True)--def _sha256_of(path):"""sha256 of a file, streamed in chunks so large archives don't blow memory."""h = hashlib.sha256()@@ -422,30 +342,10 @@ def _valid_download(filename):return True-def _stamp_present(data, version, stamp_format):- """True when the .a archive DATA carries VERSION as a known stamp form.-- Different native libs embed their version stamp differently:- - office_oxide embeds a null-delimited token (b"\\x00"+version+b"\\x00");- - pdf_oxide embeds a plain-text token (b"pdf_oxide "+version).- Accepting both keeps the extractor library-agnostic while still rejecting a- .a that does not actually contain the expected version."""- needle = version.encode()- if stamp_format == "null":- return b"\x00" + needle + b"\x00" in data- if stamp_format == "text":- return (b"pdf_oxide " + needle in data) or (b"\x00" + needle + b"\x00" in data)- return needle in data---def _extract_native_archive(archive_path, target, required_files, version=None, stamp_format="null"):+def _extract_native_archive(archive_path, target, required_files, version=None):"""Extract ARCHIVE_PATH into TARGET, verifying REQUIRED_FILES (and, for .a- files, an optional VERSION stamp) before promoting the staged tree.-- STAMP_FORMAT selects how the version is embedded in the .a:- - "null": office_oxide's b"\\x00"+version+b"\\x00" token;- - "text": pdf_oxide's plain b"pdf_oxide "+version token (also accepts a- null-delimited token, so a format switch does not silently break).+ files, an optional null-delimited VERSION stamp) before promoting the+ staged tree.Path-traversal / absolute / non-regular members are rejected beforeextraction, and an incomplete archive raises instead of replacing a@@ -458,9 +358,8 @@ def _extract_native_archive(archive_path, target, required_files, version=None,return Falseif version is not None and name.endswith(".a"):with open(path, "rb") as file:- data = file.read()- if not _stamp_present(data, version, stamp_format):- return False+ if b"\x00" + version.encode() + b"\x00" not in file.read():+ return Falsereturn Trueif complete(target):@@ -489,37 +388,6 @@ def _extract_native_archive(archive_path, target, required_files, version=None,print(f" ✓ extracted {os.path.basename(archive_path)} to {target}")-def extract_native_lib(archive_path, subdir, dir_version, required_files, stamp_version=None, stamp_format="null"):- """Extract a native static archive into the version-keyed cache dir- ~/ragflow-native-libs/<subdir>/<dir_version>/.-- The version-keyed dir (e.g. v7809) — NOT a flat layout — is what lets a- single mounted user cache hold several native-lib versions at once, so PRs- pinned to different go.mod versions all build offline. Idempotent: when the- target dir already holds the required files (matching stamp_version for- office_oxide) the extraction is skipped, so re-running the downloader is a- no-op for an already-provisioned cache.-- STAMP_FORMAT is forwarded to _extract_native_archive (see its docstring):- "null" for office_oxide, "text" for pdf_oxide. pdfium passes stamp_version- None and is not stamp-checked.-- Returns True when extraction ran or was skipped-successfully (the dir is in- place afterwards), False when the archive is absent (caller must then- download it). Re-raises the underlying tar/validation error after purging a- corrupt archive."""- if not os.path.isfile(archive_path):- print(f" Skipping extraction: {os.path.basename(archive_path)} not found")- return False- target = os.path.join(native_libs_dir(), subdir, dir_version)- try:- _extract_native_archive(archive_path, target, required_files, stamp_version, stamp_format)- except (tarfile.TarError, EOFError, RuntimeError):- os.unlink(archive_path)- raise- return True--def prune_stale_onnxruntime(static_lib_dir, expected_dir):"""Remove ONNX Runtime version dirs under static_lib that do NOT match`expected_dir`. Without this, a version bump (or a foreign-arch run on a@@ -627,7 +495,7 @@ def _download_ort_archive(archive_path, goos, goarch):if not _valid_download(archive_path):os.unlink(archive_path)raise RuntimeError(f"Invalid downloaded dependency: {os.path.basename(archive_path)}")- version_dir = os.path.join(native_libs_dir(), "onnxruntime", "static_lib", ort_normalized_dir(ORT_VERSION, goos, goarch))+ version_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib", ort_normalized_dir(ORT_VERSION, goos, goarch))if os.path.isdir(version_dir):print(f" Removing stale extracted ONNX Runtime dir: {version_dir}")shutil.rmtree(version_dir)@@ -833,7 +701,6 @@ def main():# Anchor the downloaded archives to deps/ regardless of the# caller's working directory (build.sh invokes this from the repo root).- ensure_deps_dir()os.chdir(RAGFLOW_DEPS_DIR)# ONNX Runtime archive (+ checksum sidecar), then the other native archives.@@ -854,22 +721,28 @@ def main():raise RuntimeError(f"Invalid downloaded dependency: {filename}")# Extract native static libraries to ~/ragflow-native-libs for the Go build.- # Ensures build.sh can find them without network access. Each lands under a- # version-keyed subdir (v<version>/) so a mounted cache can hold several- # versions at once; see extract_native_lib().+ # Ensures build.sh can find them without network access.office_version = go_module_version("github.com/yfedoseev/office_oxide/go", REPO_ROOT)extractions = [- ("pdfium-static", PDFIUM_STATIC_ASSETS[(goos, goarch)], "v" + PDFIUM_VERSION, ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None, "null"),- ("pdf_oxide", PDF_OXIDE_ASSETS[(goos, goarch)], "v" + PDF_OXIDE_VERSION, [f"lib/{goos}_{goarch}/libpdf_oxide.a", "include/pdf_oxide.h"], PDF_OXIDE_VERSION, "text"),- ("office_oxide", OFFICE_OXIDE_ASSETS[(goos, goarch)], "v" + office_version, ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version, "null"),+ (PDFIUM_STATIC_ASSETS[(goos, goarch)], "pdfium-static", ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"], None),+ (PDF_OXIDE_ASSETS[(goos, goarch)], "pdf_oxide", [f"lib/{goos}_{goarch}/libpdf_oxide.a", "include/pdf_oxide.h"], None),+ (OFFICE_OXIDE_ASSETS[(goos, goarch)], "office_oxide", ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"], office_version),]- for subdir, archive, dir_version, required_files, stamp_version, stamp_format in extractions:+ for archive, subdir, required_files, version in extractions:archive_path = os.path.join(RAGFLOW_DEPS_DIR, archive)- extract_native_lib(archive_path, subdir, dir_version, required_files, stamp_version, stamp_format)+ if not os.path.isfile(archive_path):+ print(f" Skipping extraction: {archive} not found")+ continue+ target = os.path.join(NATIVE_LIBS_DIR, subdir)+ try:+ _extract_native_archive(archive_path, target, required_files, version)+ except (tarfile.TarError, EOFError, RuntimeError):+ os.unlink(archive_path)+ raiseif not extract_onnxruntime(- os.path.join(native_libs_dir(), "onnxruntime", "static_lib"),+ os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib"),os.path.join(RAGFLOW_DEPS_DIR, ort_zip),ort_dir,):@@ -879,7 +752,7 @@ def main():# .a left here is exactly the "silent green" this split must prevent.print(f" ERROR: ONNX Runtime static archives for {ort_dir} were not "- f"extracted to {os.path.join(native_libs_dir(), 'onnxruntime', 'static_lib')}. "+ f"extracted to {os.path.join(NATIVE_LIBS_DIR, 'onnxruntime', 'static_lib')}. "f"Check the download above; build.sh will refuse to link without them.",file=sys.stderr,)@@ -890,7 +763,7 @@ def main():# find the archives — the .a files live under# ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend# resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback.- ort_static_dir = os.path.join(native_libs_dir(), "onnxruntime", "static_lib")+ ort_static_dir = os.path.join(NATIVE_LIBS_DIR, "onnxruntime", "static_lib")ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")]if ort_a_files:print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}")@@ -911,7 +784,7 @@ def main():download_tokenizer_assets(args.china_mirrors)print()- print(f"✓ Native build libraries ready under {native_libs_dir()}/")+ print(f"✓ Native build libraries ready under {NATIVE_LIBS_DIR}/")print(" Run: bash build.sh (it will find them without network access).")print(" Runtime resources (cl100k / DeepDoc .ort / tokenizer) are also in place,")print(" so the local Go server is ready to build, run, parse and count tokens.")diff --git a/tools/scripts/native_lib_version.sh b/tools/scripts/native_lib_version.shdeleted file mode 100644index 53f51f87f4..0000000000--- a/tools/scripts/native_lib_version.sh+++ /dev/null@@ -1,58 +0,0 @@-#!/usr/bin/env bash-# Single source of truth for the native static-library versions build.sh and-# tools/scripts/download_dev_deps.py agree on. Source this file from build.sh;-# it defines only functions (no top-level side effects) so it is safe to source.-#-# Each native lib resolves its version from exactly one place:-# - office_oxide / pdf_oxide -> go.mod (go_module_version)-# - ORT -> internal/common/environments.go-# DeepDocORTVersion (read by both build.sh and-# download_dev_deps.py directly — it is Go-owned)-# - pdfium -> pdfium_static_version (below; pdfium is not a-# Go module, so go.mod does not apply)-#-# There is NO /opt pre-bake and NO flat (version-less) cache layout any more: the-# libs land under ~/ragflow-native-libs/<lib>/v<version>/ (see download_dev_deps.py-# and build.sh), so a single mounted cache carries several versions at once.--# ragflow_native_libs_dir-# Root of the version-keyed native static-lib user cache-# (~/ragflow-native-libs by default). Overridable via RAGFLOW_NATIVE_LIBS_DIR so-# CI can mount the cache at a fixed host path (e.g. a RUNNER_WORKSPACE_PREFIX-# bind mount) instead of the build user's $HOME — see sep-tests.yml Slice 3. An-# EMPTY override falls back to the $HOME default (the ${VAR:-default} expansion),-# so a half-configured runner never resolves to an empty cache root. Echoes the-# resolved path and returns 0.-ragflow_native_libs_dir() {- printf '%s\n' "${RAGFLOW_NATIVE_LIBS_DIR:-${HOME}/ragflow-native-libs}"-}--# go_module_version <module> <repo_root>-# Reads the pinned version (the "vX.Y.Z" after the require line) of a Go module-# from <repo_root>/go.mod so build.sh and tools/scripts/download_dev_deps.py agree-# on the version (single source of truth). The module may appear either as a bare-# require line (`\t<module> vX.Y.Z`) inside a require block or on a single-line-# `require <module> vX.Y.Z`; an optional leading `require` keyword is allowed.-# Echoes the bare version (no "v" prefix) and returns 0, or returns 1 with empty-# output when go.mod is missing or the module is not required.-go_module_version() {- local module="$1" repo_root="$2"- local go_mod="${repo_root}/go.mod"- [ -f "$go_mod" ] || return 1- local version- version="$(grep -m1 -E "^[[:space:]]*(require[[:space:]]+)?${module}[[:space:]]+v" "$go_mod" \- | sed -E 's/.*v([0-9][^[:space:]]*).*/\1/')"- [ -n "$version" ] || return 1- printf '%s\n' "$version"-}--# pdfium_static_version-# pdfium (kognitos/pdfium-static) is NOT a Go module, so unlike office_oxide /-# pdf_oxide its version cannot be derived from go.mod. It is pinned here as the-# single shared constant; download_dev_deps.py and build.sh both source this-# function so the bare numeric build number lives in exactly one place. Echoes-# the constant and returns 0.-pdfium_static_version() {- # Single source of truth for the pdfium static build number. Bump here only.- printf '%s\n' "7809"-}diff --git a/tools/scripts/test_download_dev_deps.py b/tools/scripts/test_download_dev_deps.pyindex 8749be42fa..f4095ae8c0 100644--- a/tools/scripts/test_download_dev_deps.py+++ b/tools/scripts/test_download_dev_deps.py@@ -22,7 +22,6 @@import importlib.utilimport ioimport os-import subprocessimport tarfileimport threadingimport zipfile@@ -316,118 +315,6 @@ def test_native_archive_rejects_unsafe_members(tmp_path, kind):assert sorted(path.name for path in tmp_path.iterdir()) == ["native", "outside", "unsafe.tar.gz"]-# ── Version-stamped extraction (Slice 2): native .a must land under a-# version-keyed subdir (v<version>/) of the user cache, never a flat layout, so a-# single mounted cache can hold several versions at once (different PRs pinned to-# different go.mod versions all build offline). extract_native_lib() is the-# single extraction seam main() calls; it is idempotent (skips when the version-# dir already holds the required files). ─────────────────────────────────────---def _make_pdfium_archive(path, names):- with tarfile.open(path, "w:gz") as tf:- for name in names:- data = b"pdfium-payload-1234" # non-empty so complete() accepts it- info = tarfile.TarInfo(name)- info.size = len(data)- tf.addfile(info, io.BytesIO(data))---def test_extracts_pdfium_into_versioned_dir(tmp_path, monkeypatch):- monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))- archive = tmp_path / "pdfium-linux-x64-static.tgz"- _make_pdfium_archive(archive, ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"])- native.extract_native_lib(- str(archive),- "pdfium-static",- "v7809",- ["lib/libpdfium.a", "lib/libc++.a", "lib/libc++abi.a", "include/fpdfview.h"],- None,- )- target = tmp_path / "native-libs" / "pdfium-static" / "v7809"- assert (target / "lib" / "libpdfium.a").is_file()- assert (target / "include" / "fpdfview.h").is_file()---def test_extract_native_lib_idempotent_on_rerun(tmp_path, monkeypatch):- monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))- archive = tmp_path / "pdfium-linux-x64-static.tgz"- _make_pdfium_archive(archive, ["lib/libpdfium.a"])- native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)- before = sorted(os.listdir(tmp_path / "native-libs" / "pdfium-static" / "v7809" / "lib"))- native.extract_native_lib(str(archive), "pdfium-static", "v7809", ["lib/libpdfium.a"], None)- after = sorted(os.listdir(tmp_path / "native-libs" / "pdfium-static" / "v7809" / "lib"))- assert before == after---def _make_stamped_archive(path, lib_name, stamp, header_name):- """Build a tar.gz with the .a (carrying STAMP, a version-bearing byte- string) plus an include header, so complete()'s file-list AND stamp checks- can both be exercised per library format."""- with tarfile.open(path, "w:gz") as tf:- info = tarfile.TarInfo(lib_name)- payload = b"!<arch>\n" + stamp- info.size = len(payload)- tf.addfile(info, io.BytesIO(payload))- header = tarfile.TarInfo(header_name)- header.size = len(b"// header")- tf.addfile(header, io.BytesIO(b"// header"))---def test_pdf_oxide_stamp_text_accepts_matching_version(tmp_path, monkeypatch):- # pdf_oxide embeds a plain-text `pdf_oxide <version>` token (not the- # null-delimited token office_oxide uses). complete() must accept it when- # stamp_format="text", or auto-download would reject a good archive and- # build.sh's controlled download path would fail closed.- monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))- archive = tmp_path / "pdf_oxide-go-ffi-linux-amd64.tar.gz"- _make_stamped_archive(archive, "lib/linux_amd64/libpdf_oxide.a", b"pdf_oxide 0.3.73", "include/pdf_oxide.h")- native.extract_native_lib(- str(archive),- "pdf_oxide",- "v0.3.73",- ["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"],- "0.3.73",- "text",- )- assert (tmp_path / "native-libs" / "pdf_oxide" / "v0.3.73" / "lib" / "linux_amd64" / "libpdf_oxide.a").is_file()---def test_pdf_oxide_stamp_text_rejects_wrong_version(tmp_path, monkeypatch):- # A pdf_oxide .a stamped with a different version must be rejected, not- # silently accepted (the whole point of the stamp check is to catch a- # mismatched/wrong native build).- monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))- archive = tmp_path / "pdf_oxide-go-ffi-linux-amd64.tar.gz"- _make_stamped_archive(archive, "lib/linux_amd64/libpdf_oxide.a", b"pdf_oxide 0.9.9", "include/pdf_oxide.h")- with pytest.raises(RuntimeError, match="missing required native files or version 0.3.73"):- native.extract_native_lib(- str(archive),- "pdf_oxide",- "v0.3.73",- ["lib/linux_amd64/libpdf_oxide.a", "include/pdf_oxide.h"],- "0.3.73",- "text",- )---def test_office_oxide_stamp_null_accepts_matching_version(tmp_path, monkeypatch):- # Regression guard: office_oxide keeps its null-delimited stamp token; the- # "text" vs "null" split must not break it.- monkeypatch.setattr(native, "NATIVE_LIBS_DIR", str(tmp_path / "native-libs"))- archive = tmp_path / "native-linux-x86_64.tar.gz"- _make_stamped_archive(archive, "lib/liboffice_oxide.a", b"\x000.1.16\x00", "include/office_oxide_c/office_oxide.h")- native.extract_native_lib(- str(archive),- "office_oxide",- "v0.1.16",- ["lib/liboffice_oxide.a", "include/office_oxide_c/office_oxide.h"],- "0.1.16",- "null",- )- assert (tmp_path / "native-libs" / "office_oxide" / "v0.1.16" / "lib" / "liboffice_oxide.a").is_file()--# ── Local-dev one-stop: the dev downloader must also fetch the runtime# resources (cl100k / DeepDoc .ort / tokenizer assets) so a developer runs THIS# script ONCE and the Go server is ready to build, run, parse and count. ─────@@ -673,145 +560,3 @@ def test_arm64_docs_section_warns_that_lld18_links_but_hangs():section = _arm64_docs_section()assert "lld-18" in section, "lld-18 is no longer called out as insufficient"assert "hang" in section.lower() or "spin" in section.lower(), "lld-18 startup-hang failure mode missing"---# ── Single source of truth for native versions (Slice 1) ───────────────────────-# build.sh and download_dev_deps.py must agree on every native version. The-# authoritative values live in tools/scripts/native_lib_version.sh (office_oxide-# / pdf_oxide from go.mod, ORT from environments.go, pdfium as one shared const).-# These tests pin that contract: no second hardcoded copy may drift.---def test_deepdoc_ort_version_reads_environments_go(tmp_path):- # Fixture environments.go with a deliberately non-default version.- common = tmp_path / "internal" / "common"- common.mkdir(parents=True)- (common / "environments.go").write_text('package common\n\n// DeepDocORTVersion is ...\nconst DeepDocORTVersion = "9.9.9"\n')- assert native.deepdoc_ort_version(str(tmp_path)) == "9.9.9"---def test_deepdoc_ort_version_against_real_repo():- # Sanity: parses the checked-out repo's actual pin.- assert native.deepdoc_ort_version(native.REPO_ROOT) == "1.29.0"---def test_pdfium_static_version_reads_single_source(tmp_path):- # Fixture native_lib_version.sh exporting a sentinel (NOT the real 7809).- scripts = tmp_path / "tools" / "scripts"- scripts.mkdir(parents=True)- (scripts / "native_lib_version.sh").write_text("pdfium_static_version() { echo 1234; }\n")- assert native.pdfium_static_version(str(tmp_path)) == "1234"---def test_pdfium_static_version_against_real_repo():- # Sanity: the shared const on the checked-out repo.- assert native.pdfium_static_version(native.REPO_ROOT) == "7809"---# ── Slice 3: CI mounts the native-lib user cache (RAGFLOW_NATIVE_LIBS_DIR) ──-# build.sh resolves the native static libs from a single cache root-# (default ~/ragflow-native-libs, overridable via RAGFLOW_NATIVE_LIBS_DIR so CI-# can mount the cache at a fixed host path instead of the build user's $HOME).-# These tests pin that resolution contract (native_lib_version.sh) and the-# sep-tests.yml wiring that mounts the cache into every BUILDER_CONTAINER.---def _sh_eval(func_call, env):- """Source the real native_lib_version.sh and call FUNC_CALL in a throwaway- bash, returning the CompletedProcess. ENV is the full env (HOME/PATH must be- supplied), mirroring how build.sh sources the file."""- sh = os.path.join(native.REPO_ROOT, "tools", "scripts", "native_lib_version.sh")- assert os.path.isfile(sh), f"native_lib_version.sh not found at {sh}"- cmd = f'source "{sh}" && {func_call}'- return subprocess.run(["bash", "-c", cmd], env=env, capture_output=True, text=True, check=False)---def _env_with(home, override=None):- env = {"HOME": home, "PATH": os.environ.get("PATH", "")}- if override is not None:- env["RAGFLOW_NATIVE_LIBS_DIR"] = override- return env---def test_native_libs_dir_defaults_to_home():- # No override: resolve to $HOME/ragflow-native-libs.- res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice"))- assert res.returncode == 0, res.stderr- assert res.stdout.strip() == "/home/alice/ragflow-native-libs"---def test_native_libs_dir_honors_override():- # Override set: CI mounts the cache at a fixed host path; build.sh must use it.- res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice", "/mnt/cache"))- assert res.returncode == 0, res.stderr- assert res.stdout.strip() == "/mnt/cache"---def test_native_libs_dir_empty_override_falls_back():- # Adversarial: an explicitly EMPTY override must fall back to the HOME default- # (the ${VAR:-default} expansion), never resolve to an empty cache root.- res = _sh_eval("ragflow_native_libs_dir", _env_with("/home/alice", ""))- assert res.returncode == 0, res.stderr- assert res.stdout.strip() == "/home/alice/ragflow-native-libs"---SEP_TESTS_YML = os.path.join(native.REPO_ROOT, ".github", "workflows", "sep-tests.yml")---def test_sep_tests_mounts_native_libs_cache():- # Slice 3 contract: the native static-lib user cache is mounted into every- # BUILDER_CONTAINER (4 docker runs) and exposed via RAGFLOW_NATIVE_LIBS_DIR,- # replacing the old /opt pre-bake. The YAML must also stay parseable.- assert os.path.isfile(SEP_TESTS_YML), f"sep-tests.yml not found at {SEP_TESTS_YML}"- with open(SEP_TESTS_YML, encoding="utf-8") as f:- content = f.read()- # YAML must still parse (guards against a malformed edit); skip when pyyaml- # is unavailable in the test env (the CI runner has it).- try:- import yaml-- yaml.safe_load(content)- except ImportError:- pass- # The cache root is declared as a job env so runner and builder container agree.- assert "RAGFLOW_NATIVE_LIBS_DIR" in content- # Each of the 4 BUILDER_CONTAINER docker runs bind-mounts the cache.- assert content.count("${RAGFLOW_NATIVE_LIBS_DIR}:${RAGFLOW_NATIVE_LIBS_DIR}") >= 4- # A fail-fast guard catches a missing/partial mount before a cryptic cgo link error.- assert "native-lib cache not mounted" in content---# ── downloader root resolution must honour RAGFLOW_NATIVE_LIBS_DIR (P1#2) ─────-# build.sh resolves the cache root through native_lib_version.ragflow_native_libs_dir()-# (${RAGFLOW_NATIVE_LIBS_DIR:-~/ragflow-native-libs}); the downloader must agree so a-# runner that overrides the mount path does not silently download into the default.---def test_python_native_libs_dir_default(monkeypatch):- monkeypatch.delenv("RAGFLOW_NATIVE_LIBS_DIR", raising=False)- assert native.native_libs_dir() == os.path.expanduser("~/ragflow-native-libs")---def test_python_native_libs_dir_honors_override(monkeypatch):- # Adversarial: a custom mount path must be returned verbatim, NOT the default.- monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", "/srv/custom/native-libs")- assert native.native_libs_dir() == "/srv/custom/native-libs"---def test_python_native_libs_dir_empty_override_falls_back(monkeypatch):- # Adversarial: an empty override must fall back to the default, not resolve to "".- monkeypatch.setenv("RAGFLOW_NATIVE_LIBS_DIR", "")- assert native.native_libs_dir() == os.path.expanduser("~/ragflow-native-libs")---# ── downloader must not crash when deps/ is absent (P1#3) ─────────────────────---def test_ensure_deps_dir_creates_missing(monkeypatch, tmp_path):- # In the Docker go-builder stage nothing COPYs deps/, so the downloader's- # os.chdir(deps) used to raise FileNotFoundError and abort the image build.- # ensure_deps_dir() must create it first.- missing = tmp_path / "no_deps_dir"- monkeypatch.setattr(native, "RAGFLOW_DEPS_DIR", str(missing))- native.ensure_deps_dir()- assert missing.is_dir()