RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(deps): rename ragflow_deps directory to deps (#21345)
xugangqiang6a2ef7ec6b6c
diff --git a/.dockerignore b/.dockerignoreindex 55700b6609..028801cb52 100644--- a/.dockerignore+++ b/.dockerignore@@ -32,16 +32,16 @@ internal/binding/cpp/cmake-build-release/internal/binding/cpp/cmake-build-debug/target/-# ── ragflow_deps build context (built as a separate image, mounted ──+# ── deps build context (built as a separate image, mounted ──# ── from infiniflow/ragflow_deps:latest by the main Dockerfile) ──# Excluding the directory contents keeps the main build context small# regardless of which resources are prepared for the dependency image.# The deps image is built from this directory with:-# cd ragflow_deps && docker build -f Dockerfile -t infiniflow/ragflow_deps .+# cd deps && docker build -f Dockerfile -t infiniflow/ragflow_deps .# The native downloader tools/scripts/download_dev_deps.py is copied by the# main Dockerfile (not excluded here) so build.sh can read its ORT version pin# when running `./build.sh --go`.-ragflow_deps/*+deps/*# ── IDE and editor config ──────────────────────────────────────────────────.idea/diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.ymlindex 68b6066aa5..7386d2ade4 100644--- a/.github/workflows/sep-tests.yml+++ b/.github/workflows/sep-tests.yml@@ -300,7 +300,7 @@ jobs:set -euo pipefail# Provide the cl100k BPE table for the offline tokenizer loader.- # The loader reads <repo>/ragflow_deps/cl100k_base.tiktoken (or+ # The loader reads <repo>/deps/cl100k_base.tiktoken (or# TIKTOKEN_CACHE_DIR). Try to fetch it directly from the upstream# openai blob first (small ~1.6MB, reachable from the CI runner — the# pre-PR code downloaded it on the fly, so the network path is known@@ -311,36 +311,36 @@ jobs:# neither source works, the loader still fails loudly (no silent# 0-token degradation).# tiktoken is already installed in runner- mkdir -p ragflow_deps- cp /opt/ragflow_deps/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken || true- if [ ! -f ragflow_deps/cl100k_base.tiktoken ]; then- mkdir -p ragflow_deps+ mkdir -p deps+ cp /opt/ragflow_deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true+ if [ ! -f deps/cl100k_base.tiktoken ]; then+ mkdir -p depsif command -v curl >/dev/null 2>&1; then- if curl -fsSL -o ragflow_deps/cl100k_base.tiktoken \+ if curl -fsSL -o deps/cl100k_base.tiktoken \https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken; thenecho "tiktoken: cl100k table fetched via curl from openai blob"elseecho "tiktoken: curl fetch failed (network/GFW); will try cached image"fifi- if [ ! -f ragflow_deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; then+ if [ ! -f deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; thenCID=$(docker create infiniflow/ragflow_deps:latest true) || trueif [ -n "${CID:-}" ]; then- if docker cp "$CID":/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken 2>/dev/null; then+ if docker cp "$CID":/cl100k_base.tiktoken deps/cl100k_base.tiktoken 2>/dev/null; thenecho "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image"fidocker rm -f "$CID" >/dev/null 2>&1 || truefififi- if [ -f ragflow_deps/cl100k_base.tiktoken ]; then- echo "tiktoken: cl100k table provisioned ($(wc -c < ragflow_deps/cl100k_base.tiktoken) bytes)"+ if [ -f deps/cl100k_base.tiktoken ]; then+ echo "tiktoken: cl100k table provisioned ($(wc -c < deps/cl100k_base.tiktoken) bytes)"elseecho "tiktoken: cl100k table NOT provisioned — offline loader will fail loudly"fi# Provide the tokenizer assets the four non-cl100k counters load- # (ragflow_deps/huggingface.co/<repo>/<file>). Their only no-network source+ # (deps/huggingface.co/<repo>/<file>). Their only no-network source# is the infiniflow/ragflow_deps image — the runtime images read them from# exactly the same path — so this is the same "if the image is already# cached" fallback the table above uses (again, no `docker pull`). Without@@ -356,7 +356,7 @@ jobs:CID=$(docker create infiniflow/ragflow_deps:latest true) || trueif [ -n "${CID:-}" ]; thenfor asset in "${TOKENIZER_ASSET_PATHS[@]}"; do- target="ragflow_deps/huggingface.co/$asset"+ target="deps/huggingface.co/$asset"if [ -f "$target" ]; thencontinuefi@@ -846,7 +846,7 @@ jobs:set -euo pipefail# Provide the cl100k BPE table for the offline tokenizer loader.- # The loader reads <repo>/ragflow_deps/cl100k_base.tiktoken (or+ # The loader reads <repo>/deps/cl100k_base.tiktoken (or# TIKTOKEN_CACHE_DIR). Try to fetch it directly from the upstream# openai blob first (small ~1.6MB, reachable from the CI runner — the# pre-PR code downloaded it on the fly, so the network path is known@@ -857,36 +857,36 @@ jobs:# neither source works, the loader still fails loudly (no silent# 0-token degradation).# tiktoken is already installed in runner- mkdir -p ragflow_deps- cp /opt/ragflow_deps/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken || true- if [ ! -f ragflow_deps/cl100k_base.tiktoken ]; then- mkdir -p ragflow_deps+ mkdir -p deps+ cp /opt/ragflow_deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true+ if [ ! -f deps/cl100k_base.tiktoken ]; then+ mkdir -p depsif command -v curl >/dev/null 2>&1; then- if curl -fsSL -o ragflow_deps/cl100k_base.tiktoken \+ if curl -fsSL -o deps/cl100k_base.tiktoken \https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken; thenecho "tiktoken: cl100k table fetched via curl from openai blob"elseecho "tiktoken: curl fetch failed (network/GFW); will try cached image"fifi- if [ ! -f ragflow_deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; then+ if [ ! -f deps/cl100k_base.tiktoken ] && docker image inspect infiniflow/ragflow_deps:latest >/dev/null 2>&1; thenCID=$(docker create infiniflow/ragflow_deps:latest true) || trueif [ -n "${CID:-}" ]; then- if docker cp "$CID":/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken 2>/dev/null; then+ if docker cp "$CID":/cl100k_base.tiktoken deps/cl100k_base.tiktoken 2>/dev/null; thenecho "tiktoken: cl100k table copied from cached infiniflow/ragflow_deps image"fidocker rm -f "$CID" >/dev/null 2>&1 || truefififi- if [ -f ragflow_deps/cl100k_base.tiktoken ]; then- echo "tiktoken: cl100k table provisioned ($(wc -c < ragflow_deps/cl100k_base.tiktoken) bytes)"+ if [ -f deps/cl100k_base.tiktoken ]; then+ echo "tiktoken: cl100k table provisioned ($(wc -c < deps/cl100k_base.tiktoken) bytes)"elseecho "tiktoken: cl100k table NOT provisioned — offline loader will fail loudly"fi# Provide the tokenizer assets the four non-cl100k counters load- # (ragflow_deps/huggingface.co/<repo>/<file>). Their only no-network source+ # (deps/huggingface.co/<repo>/<file>). Their only no-network source# is the infiniflow/ragflow_deps image — the runtime images read them from# exactly the same path — so this is the same "if the image is already# cached" fallback the table above uses (again, no `docker pull`). Without@@ -902,7 +902,7 @@ jobs:CID=$(docker create infiniflow/ragflow_deps:latest true) || trueif [ -n "${CID:-}" ]; thenfor asset in "${TOKENIZER_ASSET_PATHS[@]}"; do- target="ragflow_deps/huggingface.co/$asset"+ target="deps/huggingface.co/$asset"if [ -f "$target" ]; thencontinuefi@@ -937,7 +937,7 @@ jobs:./build.sh --test -- $PKGSfi- - name: Run ragflow_deps regression guards+ - name: Run deps regression guards# Local, needs no API keys or model downloads; guards (a) the native# archive / ONNX Runtime extraction logic the in-process (Go) DeepDoc# backend build depends on (notably the version-bump path), and (b) the@@ -1813,15 +1813,15 @@ jobs:# (e.g. a non-standard runner). Mirrors the ragflow_native_backend job.run: |set -euo pipefail- mkdir -p ragflow_deps- cp /opt/ragflow_deps/cl100k_base.tiktoken ragflow_deps/cl100k_base.tiktoken || true- if [ ! -f ragflow_deps/cl100k_base.tiktoken ] && command -v curl >/dev/null 2>&1; then- curl -fsSL -o ragflow_deps/cl100k_base.tiktoken \+ mkdir -p deps+ cp /opt/ragflow_deps/cl100k_base.tiktoken deps/cl100k_base.tiktoken || true+ if [ ! -f deps/cl100k_base.tiktoken ] && command -v curl >/dev/null 2>&1; then+ curl -fsSL -o deps/cl100k_base.tiktoken \https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken || \echo "tiktoken: curl fetch failed; offline loader will fail loudly"fi- if [ -f ragflow_deps/cl100k_base.tiktoken ]; then- echo "tiktoken: cl100k table provisioned ($(wc -c < ragflow_deps/cl100k_base.tiktoken) bytes)"+ if [ -f deps/cl100k_base.tiktoken ]; then+ echo "tiktoken: cl100k table provisioned ($(wc -c < deps/cl100k_base.tiktoken) bytes)"elseecho "tiktoken: cl100k table NOT provisioned — offline loader will fail loudly"fidiff --git a/.gitignore b/.gitignoreindex 5cda1e7453..11a7336d2d 100644--- a/.gitignore+++ b/.gitignore@@ -33,7 +33,7 @@ docker/ragflow-logs//logsinternal/rag/res/deepdoc/# DeepDoc model weights and the HuggingFace download cache, fetched by-# ragflow_deps/download_deps.py. Same artifacts as internal/rag/res/deepdoc/,+# deps/download_deps.py. Same artifacts as internal/rag/res/deepdoc/,# under the legacy Python resource path.rag/res/@@ -272,8 +272,8 @@ internal/ingestion/task/tool/generate_dataflow_golden.pyinternal/ingestion/task/tool/README.mdinternal/cpp/cmake-build-release-# downloaded native static-lib artifacts (fetched by ragflow_deps/download_deps.py)-ragflow_deps/*.zip+# downloaded native static-lib artifacts (fetched by deps/download_deps.py)+deps/*.zip# external DeepDoc testdata, fetched on demand via scripts/fetch_deepdoc_testdata.sh# (symlinked into place from ~/.cache/ragflow-testdata; never committed)diff --git a/AGENTS.md b/AGENTS.mdindex 1d0f682f04..dc284eca91 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -53,7 +53,7 @@ Use this file as the local operating guide for the current codebase. Prefer the- `web/`: frontend application.- `docker/`: local and production compose files.- `sdk/python/`: legacy Python client SDK pending deletion.-- `tools/scripts/download_dev_deps.py` and `ragflow_deps/download_deps.py`: Python downloaders for the Go build's dependencies. `download_dev_deps.py` pulls the native static libraries (pdfium, pdf_oxide, office_oxide, onnxruntime) that `build.sh` links; `download_deps.py` pulls the runtime resources (cl100k BPE table, DeepDoc models, embedding tokenizer assets) and installs the stagehand server into the SDK cache. Versions and URLs are defined inline in each downloader (`tools/scripts/download_dev_deps.py` for native static libraries, `ragflow_deps/download_deps.py` for runtime resources); the tokenizer assets are inlined as `TOKENIZER_ASSETS` in each downloader. Replace these with non-Python preparation before removing Python dependencies.+- `tools/scripts/download_dev_deps.py` and `deps/download_deps.py`: Python downloaders for the Go build's dependencies. `download_dev_deps.py` pulls the native static libraries (pdfium, pdf_oxide, office_oxide, onnxruntime) that `build.sh` links; `download_deps.py` pulls the runtime resources (cl100k BPE table, DeepDoc models, embedding tokenizer assets) and installs the stagehand server into the SDK cache. Versions and URLs are defined inline in each downloader (`tools/scripts/download_dev_deps.py` for native static libraries, `deps/download_deps.py` for runtime resources); the tokenizer assets are inlined as `TOKENIZER_ASSETS` in each downloader. Replace these with non-Python preparation before removing Python dependencies.- `test/`: contains Python tests, including tests that import the legacy Python implementation directly. Migrate needed coverage to Go tests or frontend tests, then remove the Python tests; keep only tests for supported code.## Go-Specific Rules@@ -129,7 +129,7 @@ bash build.sh --test-manual # local opt-in onlybash build.sh --test-all # integration + e2e; excludes manual```-Native libraries, Go DeepDoc models, tokenizer assets, and `ragflow_deps/cl100k_base.tiktoken` come from `tools/scripts/download_dev_deps.py` (native static libs) and `ragflow_deps/download_deps.py` (models, cl100k, tokenizer assets); run them with `uv run`. Build the Go resource image with `cd ragflow_deps && docker build -f Dockerfile -t infiniflow/ragflow_deps:latest .` after downloading the resources.+Native libraries, Go DeepDoc models, tokenizer assets, and `deps/cl100k_base.tiktoken` come from `tools/scripts/download_dev_deps.py` (native static libs) and `deps/download_deps.py` (models, cl100k, tokenizer assets); run them with `uv run`. Build the Go resource image with `cd deps && docker build -f Dockerfile -t infiniflow/ragflow_deps:latest .` after downloading the resources.### Frontend```bashdiff --git a/Dockerfile b/Dockerfileindex 89e0efeaed..c4820f08f2 100644--- a/Dockerfile+++ b/Dockerfile@@ -55,16 +55,16 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co# Copy the cl100k_base BPE table used by the Go tokenizer (tiktoken-go# cl100k_base). The deps image ships it at its root; the Go image previously# mounted only /huggingface.co and omitted this file, so NumTokensFromString-# silently returned 0. localBpeLoader resolves it from <workdir>/ragflow_deps/,+# silently returned 0. localBpeLoader resolves it from <workdir>/deps/,# so dropping it here makes the table load offline at startup (see# InitCL100KEncoder fail-fast guard).RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/cl100k_base.tiktoken,target=/tmp/cl100k_base.tiktoken \- mkdir -p /ragflow/ragflow_deps && \- cp /tmp/cl100k_base.tiktoken /ragflow/ragflow_deps/cl100k_base.tiktoken+ mkdir -p /ragflow/deps && \+ cp /tmp/cl100k_base.tiktoken /ragflow/deps/cl100k_base.tiktoken# Embedding tokenizer assets for the Go counters (internal/tokenizer). Same story as# the cl100k table above: the deps image carries them and the Go image has to put them-# where the loaders search (<workdir>/ragflow_deps/huggingface.co/<repo>/<file>). A+# where the loaders search (<workdir>/deps/huggingface.co/<repo>/<file>). A# missing asset FAILS THE BUILD here rather than shipping an image where every tagged# model silently counts with the calibrated cl100k estimate. The tokenizer.json files# that exist only as cross-check oracles are deliberately not shipped.@@ -75,8 +75,8 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.coQwen/Qwen3-Embedding-0.6B/tokenizer.json \intfloat/e5-mistral-7b-instruct/tokenizer.json ; do \if [ -f "/huggingface.co/$asset" ]; then \- mkdir -p "/ragflow/ragflow_deps/huggingface.co/$(dirname "$asset")" && \- cp "/huggingface.co/$asset" "/ragflow/ragflow_deps/huggingface.co/$asset" ; \+ mkdir -p "/ragflow/deps/huggingface.co/$(dirname "$asset")" && \+ cp "/huggingface.co/$asset" "/ragflow/deps/huggingface.co/$asset" ; \else \echo "ERROR: tokenizer asset $asset is missing from the infiniflow/ragflow_deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \exit 1 ; \diff --git a/Dockerfile_base b/Dockerfile_baseindex ae2b0be2e2..a84ad82a55 100644--- a/Dockerfile_base+++ b/Dockerfile_base@@ -25,7 +25,7 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/,target=/depscp /deps/cl100k_base.tiktoken /ragflow/9b5ad71b2ce5302211f9c61530b329a4922fc6a4# Embedding tokenizer assets (internal/tokenizer/embedding_token_limits.md). The Go-# counters load them from ragflow_deps/huggingface.co/<repo>/<file>; an image without them+# counters load them from deps/huggingface.co/<repo>/<file>; an image without them# counts every tagged model with the calibrated cl100k estimate, the less precise path# these counters exist to replace - so a missing asset FAILS THE BUILD instead of shipping# a degraded counter nobody notices. The tokenizer.json files that download_deps.py@@ -37,8 +37,8 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.coQwen/Qwen3-Embedding-0.6B/tokenizer.json \intfloat/e5-mistral-7b-instruct/tokenizer.json ; do \if [ -f "/huggingface.co/$asset" ]; then \- mkdir -p "/ragflow/ragflow_deps/huggingface.co/$(dirname "$asset")" && \- cp "/huggingface.co/$asset" "/ragflow/ragflow_deps/huggingface.co/$asset" ; \+ mkdir -p "/ragflow/deps/huggingface.co/$(dirname "$asset")" && \+ cp "/huggingface.co/$asset" "/ragflow/deps/huggingface.co/$asset" ; \else \echo "ERROR: tokenizer asset $asset is missing from the infiniflow/ragflow_deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \exit 1 ; \@@ -146,7 +146,7 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \# deployments. CI cross-checks the two values.## The binary is pre-fetched by `download_deps.py` and shipped via-# the ragflow_deps image, then written directly to the stagehand-go+# the deps image, then written directly to the stagehand-go# cache path that `local.go:cacheDir()` constructs at runtime —# `/root/.cache/stagehand/lib/go_<ver>/stagehand-server-v3-<arch>`.ARG STAGEHAND_GO_VERSION=v3.21.0@@ -184,7 +184,7 @@ RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \-# Install Chrome and ChromeDriver for selenium (from ragflow_deps image)+# Install Chrome and ChromeDriver for selenium (from deps image)RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/chrome-linux64-121-0-6167-85,target=/chrome-linux64.zip \unzip /chrome-linux64.zip && \mv chrome-linux64 /opt/chrome && \diff --git a/README.md b/README.mdindex 76839dc994..fc8a1ea00a 100644--- a/README.md+++ b/README.md@@ -262,9 +262,9 @@ For document-engine changes, configuration updates, restarting services, and retuv run tools/scripts/download_dev_deps.py```- This is the local-dev one-stop: after one run the environment is ready, and already-present files are skipped so re-running never re-downloads. The image-facing counterpart `ragflow_deps/download_deps.py` fetches the same runtime resources PLUS the stagehand server binary (used by the Browser component) and skips the native `.a` files — run it only when building the dependency image or in CI, not for local development.+ This is the local-dev one-stop: after one run the environment is ready, and already-present files are skipped so re-running never re-downloads. The image-facing counterpart `deps/download_deps.py` fetches the same runtime resources PLUS the stagehand server binary (used by the Browser component) and skips the native `.a` files — run it only when building the dependency image or in CI, not for local development.- `download_dev_deps.py` pulls and unpacks pdfium, pdf_oxide, office_oxide, and onnxruntime under `~/ragflow-native-libs`, and writes the cl100k vocabulary, DeepDoc models, and embedding tokenizer assets under `ragflow_deps/`. It is a Python script; `uv run` installs its declared dependencies (`requests` and `huggingface-hub`) automatically. Skip this step if you have prepared the same resources by other means. When started from the repository root, Go services automatically find `internal/rag/res/deepdoc`; to start from another directory, set `DEEPDOC_MODEL_DIR` to its absolute path.+ `download_dev_deps.py` pulls and unpacks pdfium, pdf_oxide, office_oxide, and onnxruntime under `~/ragflow-native-libs`, and writes the cl100k vocabulary, DeepDoc models, and embedding tokenizer assets under `deps/`. It is a Python script; `uv run` installs its declared dependencies (`requests` and `huggingface-hub`) automatically. Skip this step if you have prepared the same resources by other means. When started from the repository root, Go services automatically find `internal/rag/res/deepdoc`; to start from another directory, set `DEEPDOC_MODEL_DIR` to its absolute path.3. Build the services:diff --git a/README_ar.md b/README_ar.mdindex 4a03eab99a..9ea8b15568 100644--- a/README_ar.md+++ b/README_ar.md@@ -216,10 +216,10 @@```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- يقوم `download_dev_deps.py` بتنزيل وفك pdfium وpdf_oxide وoffice_oxide وonnxruntime تحت `~/ragflow-native-libs`؛ ويقوم `download_deps.py` بتنزيل جدول cl100k ونماذج DeepDoc وأصول محلّلات التضمين تحت `ragflow_deps/`. كلاهما برنامج Python؛ ويثبّت `uv run` تبعياته المعلنة تلقائيًا (`requests`، بالإضافة إلى `huggingface-hub` لتنزيل النماذج والمحلّلات). تجاوز هذه الخطوة إذا كنت قد جهّزت الموارد نفسها بوسيلة أخرى. عند التشغيل من جذر المستودع تجد خدمات Go مجلد `internal/rag/res/deepdoc` تلقائيًا؛ وللتشغيل من مجلد آخر عيّن `DEEPDOC_MODEL_DIR` على مساره المطلق.+ يقوم `download_dev_deps.py` بتنزيل وفك pdfium وpdf_oxide وoffice_oxide وonnxruntime تحت `~/ragflow-native-libs`؛ ويقوم `download_deps.py` بتنزيل جدول cl100k ونماذج DeepDoc وأصول محلّلات التضمين تحت `deps/`. كلاهما برنامج Python؛ ويثبّت `uv run` تبعياته المعلنة تلقائيًا (`requests`، بالإضافة إلى `huggingface-hub` لتنزيل النماذج والمحلّلات). تجاوز هذه الخطوة إذا كنت قد جهّزت الموارد نفسها بوسيلة أخرى. عند التشغيل من جذر المستودع تجد خدمات Go مجلد `internal/rag/res/deepdoc` تلقائيًا؛ وللتشغيل من مجلد آخر عيّن `DEEPDOC_MODEL_DIR` على مساره المطلق.3. ابنِ الخدمات:diff --git a/README_fr.md b/README_fr.mdindex b33e1cc74a..20cd5db08d 100644--- a/README_fr.md+++ b/README_fr.md@@ -213,10 +213,10 @@ Docker utilise `docker/.env` et `docker/docker-compose.yml`. Suivez le [guide de```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- `download_dev_deps.py` télécharge et décompresse pdfium, pdf_oxide, office_oxide et onnxruntime sous `~/ragflow-native-libs` ; `download_deps.py` télécharge le vocabulaire cl100k, les modèles DeepDoc et les ressources de tokeniseurs d'embedding sous `ragflow_deps/`. Les deux sont des scripts Python ; `uv run` installe automatiquement leurs dépendances déclarées (`requests`, plus `huggingface-hub` pour les téléchargements de modèles et de tokeniseurs). Ignorez cette étape si vous avez déjà préparé les mêmes ressources autrement. Lors du démarrage depuis la racine du dépôt, les services Go trouvent automatiquement `internal/rag/res/deepdoc` ; pour démarrer depuis un autre répertoire, définissez `DEEPDOC_MODEL_DIR` sur son chemin absolu.+ `download_dev_deps.py` télécharge et décompresse pdfium, pdf_oxide, office_oxide et onnxruntime sous `~/ragflow-native-libs` ; `download_deps.py` télécharge le vocabulaire cl100k, les modèles DeepDoc et les ressources de tokeniseurs d'embedding sous `deps/`. Les deux sont des scripts Python ; `uv run` installe automatiquement leurs dépendances déclarées (`requests`, plus `huggingface-hub` pour les téléchargements de modèles et de tokeniseurs). Ignorez cette étape si vous avez déjà préparé les mêmes ressources autrement. Lors du démarrage depuis la racine du dépôt, les services Go trouvent automatiquement `internal/rag/res/deepdoc` ; pour démarrer depuis un autre répertoire, définissez `DEEPDOC_MODEL_DIR` sur son chemin absolu.3. Compilez les services :diff --git a/README_id.md b/README_id.mdindex 2995518e8b..09961eef4a 100644--- a/README_id.md+++ b/README_id.md@@ -213,10 +213,10 @@ Docker menggunakan `docker/.env` dan `docker/docker-compose.yml`. Ikuti [panduan```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- `download_dev_deps.py` mengunduh dan mengekstrak pdfium, pdf_oxide, office_oxide, dan onnxruntime ke `~/ragflow-native-libs`; `download_deps.py` mengunduh kosakata cl100k, model DeepDoc, dan aset tokenizer embedding ke `ragflow_deps/`. Keduanya adalah skrip Python; `uv run` memasang dependensi yang dideklarasikan secara otomatis (`requests`, plus `huggingface-hub` untuk unduhan model dan tokenizer). Lewati langkah ini jika Anda sudah menyiapkan sumber daya yang sama dengan cara lain. Saat dijalankan dari akar repositori, layanan Go secara otomatis menemukan `internal/rag/res/deepdoc`; untuk menjalankan dari direktori lain, tetapkan `DEEPDOC_MODEL_DIR` ke jalur absolutnya.+ `download_dev_deps.py` mengunduh dan mengekstrak pdfium, pdf_oxide, office_oxide, dan onnxruntime ke `~/ragflow-native-libs`; `download_deps.py` mengunduh kosakata cl100k, model DeepDoc, dan aset tokenizer embedding ke `deps/`. Keduanya adalah skrip Python; `uv run` memasang dependensi yang dideklarasikan secara otomatis (`requests`, plus `huggingface-hub` untuk unduhan model dan tokenizer). Lewati langkah ini jika Anda sudah menyiapkan sumber daya yang sama dengan cara lain. Saat dijalankan dari akar repositori, layanan Go secara otomatis menemukan `internal/rag/res/deepdoc`; untuk menjalankan dari direktori lain, tetapkan `DEEPDOC_MODEL_DIR` ke jalur absolutnya.3. Build layanan:diff --git a/README_ja.md b/README_ja.mdindex c4b8feb1a3..68be822d15 100644--- a/README_ja.md+++ b/README_ja.md@@ -223,10 +223,10 @@ Docker は迅速な評価、統合テスト、本番環境に、ソース起動```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- `download_dev_deps.py` は pdfium、pdf_oxide、office_oxide、onnxruntime を `~/ragflow-native-libs` にダウンロードして展開します。`download_deps.py` は cl100k 語彙、DeepDoc モデル、embedding トークナイザーリソースを `ragflow_deps/` にダウンロードします。どちらも Python スクリプトで、`uv run` が宣言された依存関係(`requests`、モデルとトークナイザーのダウンロードには `huggingface-hub`)を自動的にインストールします。同等のリソースを他の方法で用意済みの場合はこの手順をスキップしてください。リポジトリルートから起動する場合、Go サービスは `internal/rag/res/deepdoc` を自動的に見つけます。他のディレクトリから起動する場合は、`DEEPDOC_MODEL_DIR` にその絶対パスを設定してください。+ `download_dev_deps.py` は pdfium、pdf_oxide、office_oxide、onnxruntime を `~/ragflow-native-libs` にダウンロードして展開します。`download_deps.py` は cl100k 語彙、DeepDoc モデル、embedding トークナイザーリソースを `deps/` にダウンロードします。どちらも Python スクリプトで、`uv run` が宣言された依存関係(`requests`、モデルとトークナイザーのダウンロードには `huggingface-hub`)を自動的にインストールします。同等のリソースを他の方法で用意済みの場合はこの手順をスキップしてください。リポジトリルートから起動する場合、Go サービスは `internal/rag/res/deepdoc` を自動的に見つけます。他のディレクトリから起動する場合は、`DEEPDOC_MODEL_DIR` にその絶対パスを設定してください。3. サービスをビルドします:diff --git a/README_ko.md b/README_ko.mdindex 3adb208deb..7aff17a2ef 100644--- a/README_ko.md+++ b/README_ko.md@@ -223,10 +223,10 @@ Docker는 빠른 평가, 통합 테스트 및 프로덕션에 적합하고 소```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- `download_dev_deps.py`는 pdfium, pdf_oxide, office_oxide, onnxruntime을 `~/ragflow-native-libs`에 내려받아 압축 해제합니다. `download_deps.py`는 cl100k 어휘집, DeepDoc 모델, 임베딩 토크나이저 리소스를 `ragflow_deps/`에 내려받습니다. 두 스크립트 모두 Python 스크립트이며, `uv run`이 선언된 종속성(`requests`, 모델 및 토크나이저 다운로드에는 `huggingface-hub` 추가)을 자동으로 설치합니다. 같은 리소스를 다른 방법으로 준비했다면 이 단계를 건너뛰세요. 저장소 루트에서 시작하면 Go 서비스가 `internal/rag/res/deepdoc`을 자동으로 찾습니다. 다른 디렉터리에서 시작하려면 `DEEPDOC_MODEL_DIR`을 해당 디렉터리의 절대 경로로 설정하세요.+ `download_dev_deps.py`는 pdfium, pdf_oxide, office_oxide, onnxruntime을 `~/ragflow-native-libs`에 내려받아 압축 해제합니다. `download_deps.py`는 cl100k 어휘집, DeepDoc 모델, 임베딩 토크나이저 리소스를 `deps/`에 내려받습니다. 두 스크립트 모두 Python 스크립트이며, `uv run`이 선언된 종속성(`requests`, 모델 및 토크나이저 다운로드에는 `huggingface-hub` 추가)을 자동으로 설치합니다. 같은 리소스를 다른 방법으로 준비했다면 이 단계를 건너뛰세요. 저장소 루트에서 시작하면 Go 서비스가 `internal/rag/res/deepdoc`을 자동으로 찾습니다. 다른 디렉터리에서 시작하려면 `DEEPDOC_MODEL_DIR`을 해당 디렉터리의 절대 경로로 설정하세요.3. 서비스를 빌드합니다:diff --git a/README_pt_br.md b/README_pt_br.mdindex eca5247077..e4a0e14298 100644--- a/README_pt_br.md+++ b/README_pt_br.md@@ -214,10 +214,10 @@ Use `docker/.env` e `docker/docker-compose.yml`. Consulte a [configuração do D```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- `download_dev_deps.py` baixa e descompacta pdfium, pdf_oxide, office_oxide e onnxruntime em `~/ragflow-native-libs`; `download_deps.py` baixa o vocabulário cl100k, os modelos DeepDoc e os recursos de tokenizadores de embedding em `ragflow_deps/`. Ambos são scripts Python; o `uv run` instala automaticamente as dependências declaradas (`requests`, além de `huggingface-hub` para os downloads de modelos e tokenizadores). Pule esta etapa se você já preparou os mesmos recursos por outros meios. Quando iniciado a partir da raiz do repositório, os serviços Go encontram `internal/rag/res/deepdoc` automaticamente; para iniciar de outro diretório, defina `DEEPDOC_MODEL_DIR` com seu caminho absoluto.+ `download_dev_deps.py` baixa e descompacta pdfium, pdf_oxide, office_oxide e onnxruntime em `~/ragflow-native-libs`; `download_deps.py` baixa o vocabulário cl100k, os modelos DeepDoc e os recursos de tokenizadores de embedding em `deps/`. Ambos são scripts Python; o `uv run` instala automaticamente as dependências declaradas (`requests`, além de `huggingface-hub` para os downloads de modelos e tokenizadores). Pule esta etapa se você já preparou os mesmos recursos por outros meios. Quando iniciado a partir da raiz do repositório, os serviços Go encontram `internal/rag/res/deepdoc` automaticamente; para iniciar de outro diretório, defina `DEEPDOC_MODEL_DIR` com seu caminho absoluto.3. Compile os serviços:diff --git a/README_ru.md b/README_ru.mdindex af6651f49e..643e8ffe85 100644--- a/README_ru.md+++ b/README_ru.md@@ -220,10 +220,10 @@ Docker подходит для быстрой оценки, интеграцио```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- `download_dev_deps.py` загружает и распаковывает pdfium, pdf_oxide, office_oxide и onnxruntime в `~/ragflow-native-libs`; `download_deps.py` загружает словарь cl100k, модели DeepDoc и ресурсы токенизаторов эмбеддингов в `ragflow_deps/`. Оба скрипта — Python; `uv run` автоматически установит объявленные зависимости (`requests`, а для загрузки моделей и токенизаторов ещё и `huggingface-hub`). Пропустите этот шаг, если те же ресурсы уже подготовлены иным способом. При запуске из корня репозитория сервисы Go автоматически находят `internal/rag/res/deepdoc`; при запуске из другого каталога задайте `DEEPDOC_MODEL_DIR` с его абсолютным путём.+ `download_dev_deps.py` загружает и распаковывает pdfium, pdf_oxide, office_oxide и onnxruntime в `~/ragflow-native-libs`; `download_deps.py` загружает словарь cl100k, модели DeepDoc и ресурсы токенизаторов эмбеддингов в `deps/`. Оба скрипта — Python; `uv run` автоматически установит объявленные зависимости (`requests`, а для загрузки моделей и токенизаторов ещё и `huggingface-hub`). Пропустите этот шаг, если те же ресурсы уже подготовлены иным способом. При запуске из корня репозитория сервисы Go автоматически находят `internal/rag/res/deepdoc`; при запуске из другого каталога задайте `DEEPDOC_MODEL_DIR` с его абсолютным путём.3. Соберите сервисы:diff --git a/README_tr.md b/README_tr.mdindex b1a83d6bf7..e9980c1c9e 100644--- a/README_tr.md+++ b/README_tr.md@@ -223,10 +223,10 @@ Ayrıntılar için [hızlı başlangıç kılavuzuna](./docs/quickstart.mdx) bak```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- `download_dev_deps.py`, pdfium, pdf_oxide, office_oxide ve onnxruntime'ı `~/ragflow-native-libs` altına indirir ve açar; `download_deps.py`, cl100k sözlüğünü, DeepDoc modellerini ve gömme tokenizer varlıklarını `ragflow_deps/` altına indirir. İkisi de Python betiğidir; `uv run`, bildirilen bağımlılıkları (`requests` ve model ile tokenizer indirmeleri için `huggingface-hub`) otomatik olarak kurar. Aynı kaynakları başka şekilde hazırladıysanız bu adımı atlayın. Depo kökünden başlatıldığında Go hizmetleri `internal/rag/res/deepdoc` dizinini otomatik bulur; başka bir dizinden başlatmak için `DEEPDOC_MODEL_DIR` değişkenini o dizinin mutlak yoluna ayarlayın.+ `download_dev_deps.py`, pdfium, pdf_oxide, office_oxide ve onnxruntime'ı `~/ragflow-native-libs` altına indirir ve açar; `download_deps.py`, cl100k sözlüğünü, DeepDoc modellerini ve gömme tokenizer varlıklarını `deps/` altına indirir. İkisi de Python betiğidir; `uv run`, bildirilen bağımlılıkları (`requests` ve model ile tokenizer indirmeleri için `huggingface-hub`) otomatik olarak kurar. Aynı kaynakları başka şekilde hazırladıysanız bu adımı atlayın. Depo kökünden başlatıldığında Go hizmetleri `internal/rag/res/deepdoc` dizinini otomatik bulur; başka bir dizinden başlatmak için `DEEPDOC_MODEL_DIR` değişkenini o dizinin mutlak yoluna ayarlayın.3. Hizmetleri derleyin:diff --git a/README_tzh.md b/README_tzh.mdindex c030e78ee3..127eb52289 100644--- a/README_tzh.md+++ b/README_tzh.md@@ -267,10 +267,10 @@ Docker 部署使用 `docker/.env` 和 `docker/docker-compose.yml`。鏡像、端```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- `download_dev_deps.py` 拉取並解壓 pdfium、pdf_oxide、office_oxide、onnxruntime 等到 `~/ragflow-native-libs`;`download_deps.py` 拉取 cl100k 詞表、DeepDoc 模型與 embedding 分詞器資源到 `ragflow_deps/`。兩個腳本均為 Python 腳本,`uv run` 會自動安裝其聲明的依賴(`requests`,模型與分詞器下載另需 `huggingface-hub`)。如果已透過其他方式準備好相同資源,可跳過這一步。Go 服務從倉庫根目錄啓動時會自動查找 `internal/rag/res/deepdoc`;如需從其他目錄啓動,請將 `DEEPDOC_MODEL_DIR` 設為該目錄的絕對路徑。+ `download_dev_deps.py` 拉取並解壓 pdfium、pdf_oxide、office_oxide、onnxruntime 等到 `~/ragflow-native-libs`;`download_deps.py` 拉取 cl100k 詞表、DeepDoc 模型與 embedding 分詞器資源到 `deps/`。兩個腳本均為 Python 腳本,`uv run` 會自動安裝其聲明的依賴(`requests`,模型與分詞器下載另需 `huggingface-hub`)。如果已透過其他方式準備好相同資源,可跳過這一步。Go 服務從倉庫根目錄啓動時會自動查找 `internal/rag/res/deepdoc`;如需從其他目錄啓動,請將 `DEEPDOC_MODEL_DIR` 設為該目錄的絕對路徑。3. 編譯:diff --git a/README_zh.md b/README_zh.mdindex 6ef2239d1b..6995c3298f 100644--- a/README_zh.md+++ b/README_zh.md@@ -267,10 +267,10 @@ Docker 部署使用 `docker/.env` 和 `docker/docker-compose.yml`。镜像、端```bashuv run tools/scripts/download_dev_deps.py- uv run ragflow_deps/download_deps.py+ uv run deps/download_deps.py```- `download_dev_deps.py` 拉取并解压 pdfium、pdf_oxide、office_oxide、onnxruntime 等到 `~/ragflow-native-libs`;`download_deps.py` 拉取 cl100k 词表、DeepDoc 模型与 embedding 分词器资源到 `ragflow_deps/`。两个脚本均为 Python 脚本,`uv run` 会自动安装其声明的依赖(`requests`,模型与分词器下载还需 `huggingface-hub`)。如果已通过其他方式准备好相同资源,可跳过这一步。Go 服务从仓库根目录启动时会自动查找 `internal/rag/res/deepdoc`;如需从其他目录启动,请将 `DEEPDOC_MODEL_DIR` 设为该目录的绝对路径。+ `download_dev_deps.py` 拉取并解压 pdfium、pdf_oxide、office_oxide、onnxruntime 等到 `~/ragflow-native-libs`;`download_deps.py` 拉取 cl100k 词表、DeepDoc 模型与 embedding 分词器资源到 `deps/`。两个脚本均为 Python 脚本,`uv run` 会自动安装其声明的依赖(`requests`,模型与分词器下载还需 `huggingface-hub`)。如果已通过其他方式准备好相同资源,可跳过这一步。Go 服务从仓库根目录启动时会自动查找 `internal/rag/res/deepdoc`;如需从其他目录启动,请将 `DEEPDOC_MODEL_DIR` 设为该目录的绝对路径。3. 编译:diff --git a/build.sh b/build.shindex 3d4165b502..3189849209 100755--- a/build.sh+++ b/build.sh@@ -166,10 +166,10 @@ _seed_from_system() {# pre-seed first, then the network. A failure here is deliberately NOT fatal —# the loader stays the final gate, and its panic message names the file.CL100K_TABLE_URL="https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"-CL100K_TABLE_RELPATH="ragflow_deps/cl100k_base.tiktoken"+CL100K_TABLE_RELPATH="deps/cl100k_base.tiktoken"SYSTEM_DEPS_TOKENIZER="/opt/ragflow_deps"-# Provision the cl100k_base BPE table into ragflow_deps/ (no-op if present).+# Provision the cl100k_base BPE table into deps/ (no-op if present).# Returns 0 when the table is in place, 1 when it could not be provisioned._ensure_cl100k_table() {local dest="${PROJECT_ROOT}/${CL100K_TABLE_RELPATH}"@@ -344,7 +344,7 @@ check_ort_version_consistency() {# that lacks those custom slots links successfully and only fails at runtime.# The check below catches that by comparing the local release ZIP against the# published {asset}.sha256 sidecar (same source of truth the download scripts-# use). Devs who ran a download script have the zip under ragflow_deps/; CI+# use). Devs who ran a download script have the zip under deps/; CI# seeds from /opt and has no zip, where the bake is authoritative.check_onnxruntime_deps() {# Ensure TARGET_OS/TARGET_ARCH are resolved. ort_static_lib_prefix below@@ -434,7 +434,7 @@ check_onnxruntime_deps() {# macOS zip against the Linux digest would warn on every build, so an# unknown target verifies nothing instead.local asset="onnxruntime-v${ort_version}-$(ort_release_target).zip"- local zip_path="${PROJECT_ROOT}/ragflow_deps/${asset}"+ local zip_path="${PROJECT_ROOT}/deps/${asset}"local expected=""case "$asset" inonnxruntime-v*-linux-x86_64.zip)@@ -495,12 +495,12 @@ check_office_oxide_deps() {local found_versionfound_version=$(strings "$lib_path" 2>/dev/null | grep -E "^0\.[0-9]+\.[0-9]+$" | head -1)echo -e "${YELLOW}office_oxide native lib version mismatch (required v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}); refreshing from download${NC}"- rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/ragflow_deps/$(office_oxide_asset)"+ rm -rf "${OFFICE_OXIDE_PREFIX}" "${PROJECT_ROOT}/deps/$(office_oxide_asset)"if ! (cd "${PROJECT_ROOT}" && uv run tools/scripts/download_dev_deps.py); thenecho -e "${RED}Error: office_oxide native lib version mismatch${NC}"echo " Required: v${OFFICE_OXIDE_VERSION}; found: ${found_version:-unknown}"echo " A stale lib silently loses PPT97 (.ppt) slide content. Refresh:"- echo " rm -rf ~/ragflow-native-libs/office_oxide ragflow_deps/$(office_oxide_asset)"+ echo " rm -rf ~/ragflow-native-libs/office_oxide deps/$(office_oxide_asset)"echo " uv run tools/scripts/download_dev_deps.py"exit 1fi@@ -628,7 +628,7 @@ check_pdf_oxide_deps() {echo -e "${RED}Error: pdf_oxide native lib version mismatch${NC}"echo " Required: v${PDF_OXIDE_VERSION}; found: ${found_version:-unknown}"echo " A stale lib silently reverts PDF parsing fixes. Refresh:"- echo " rm -rf ${PDF_OXIDE_PREFIX} ragflow_deps/$(pdf_oxide_asset)"+ echo " rm -rf ${PDF_OXIDE_PREFIX} deps/$(pdf_oxide_asset)"echo " uv run tools/scripts/download_dev_deps.py"return 1;;diff --git a/ragflow_deps/Dockerfile b/deps/Dockerfilesimilarity index 77%rename from ragflow_deps/Dockerfilerename to deps/Dockerfileindex ade16bfd9b..e2ed64e5b7 100644--- a/ragflow_deps/Dockerfile+++ b/deps/Dockerfile@@ -1,4 +1,4 @@-# Build from ragflow_deps/ after running download_deps.py:+# Build from deps/ after running download_deps.py:# docker build -f Dockerfile -t infiniflow/ragflow_deps:latest .FROM scratchdiff --git a/ragflow_deps/download_deps.py b/deps/download_deps.pysimilarity index 96%rename from ragflow_deps/download_deps.pyrename to deps/download_deps.pyindex 5689e84420..69f3c2c97f 100755--- a/ragflow_deps/download_deps.py+++ b/deps/download_deps.py@@ -25,7 +25,7 @@## Typical workflow:#-# uv run ragflow_deps/download_deps.py+# uv run deps/download_deps.py## Go DeepDoc weights: in addition to the resources above, this script downloads# the five Go model files (internal/common.DeepDocModelFiles) from@@ -39,8 +39,8 @@## Embedding tokenizer assets: the Go token counters in internal/tokenizer load a# tokenizer file per family (XLM-R SentencePiece, BERT WordPiece, two byte-level BPE-# families) from `huggingface.co/<repo>/<file>` at the top of ragflow_deps/ - the path-# the loaders search and the one ragflow_deps/Dockerfile ships. This script fetches+# families) from `huggingface.co/<repo>/<file>` at the top of deps/ - the path+# the loaders search and the one deps/Dockerfile ships. This script fetches# them too, per the inlined TOKENIZER_ASSETS list. Models that declare a missing# tokenizer cannot be ingested (see internal/tokenizer/embedding_token_limits.md).@@ -217,7 +217,7 @@ def download_go_models(use_china_mirrors=False):`internal/rag/res/deepdoc/` (relative to the repo root, one level up fromthis script). The Go server auto-discovers that directory, and build.sh--test-native defaults MODEL_DIR there too. A copy also lands under- ragflow_deps/huggingface.co/InfiniFlow/deepdoc/ to keep the dependency+ deps/huggingface.co/InfiniFlow/deepdoc/ to keep the dependencyimage's existing build-context layout."""if use_china_mirrors:@@ -272,8 +272,8 @@ def download_go_models(use_china_mirrors=False):def download_tokenizer_assets(use_china_mirrors=False):"""Download the embedding tokenizer assets the Go counters load.- Written to `ragflow_deps/huggingface.co/<repo>/<file>`: the path internal/tokenizer- searches and the one ragflow_deps/Dockerfile builds its image from. Fetched per file+ Written to `deps/huggingface.co/<repo>/<file>`: the path internal/tokenizer+ searches and the one deps/Dockerfile builds its image from. Fetched per file(these repos also carry multi-GB weights we do not want), routed via hf-mirror.comwhen --china-mirrors is set.@@ -334,7 +334,7 @@ def main():parser.add_argument("--china-mirrors", action="store_true", help="Use China-accessible mirrors for downloads")args = parser.parse_args()- # Anchor archives, the BPE table, and tokenizer assets to ragflow_deps/.+ # Anchor archives, the BPE table, and tokenizer assets to deps/.os.chdir(SCRIPT_DIR)# ── cl100k_base BPE table ──────────────────────────────────────────────@@ -367,7 +367,7 @@ def main():download_go_models(args.china_mirrors)# Same one-stop idea for the token counters: their tokenizer files, in the layout- # the loaders search and ragflow_deps/Dockerfile ships.+ # the loaders search and deps/Dockerfile ships.print("== tokenizer assets ==")download_tokenizer_assets(args.china_mirrors)diff --git a/docker/launch_backend_service.sh b/docker/launch_backend_service.shindex 3ab864ad2f..6ae009a170 100755--- a/docker/launch_backend_service.sh+++ b/docker/launch_backend_service.sh@@ -69,7 +69,7 @@ PIDS=()# Set the path to the NLTK data directory# Use the repository's NLTK data directory when it is provisioned.-export NLTK_DATA="$(pwd)/ragflow_deps/nltk_data"+export NLTK_DATA="$(pwd)/deps/nltk_data"# Function to handle termination signalscleanup() {diff --git a/docs/develop/launch_ragflow_from_source.md b/docs/develop/launch_ragflow_from_source.mdindex bde892bb32..7ea948d9ba 100644--- a/docs/develop/launch_ragflow_from_source.md+++ b/docs/develop/launch_ragflow_from_source.md@@ -27,7 +27,7 @@ All long-running RAGFlow processes started below use `bin/ragflow_server`.- Go 1.27 or later, as declared in `go.mod` (check `go version`).- CMake 4.0 or later, Clang 20, LLD 20, and PCRE2 development headers.- Node.js 18.20.4 or later and npm for the frontend.-- Python 3.10 or later for `tools/scripts/download_dev_deps.py` and `ragflow_deps/download_deps.py`, which download the native libraries and model resources required by the Go build.+- Python 3.10 or later for `tools/scripts/download_dev_deps.py` and `deps/download_deps.py`, which download the native libraries and model resources required by the Go build.See the [Docker installation guide](https://docs.docker.com/engine/install/) if Docker is not installed. Use the Go version declared in `go.mod` and the compiler versions listed above when preparing the build environment.@@ -60,7 +60,7 @@ Download the native static libraries and the runtime model / tokenizer resourcesuv run tools/scripts/download_dev_deps.py # local-dev one-stop: native .a + cl100k, DeepDoc .ort, tokenizer assets```-`download_dev_deps.py` is the local-dev one-stop: it fetches the static libraries needed by `build.sh` (pdfium / pdf_oxide / office_oxide / onnxruntime → `~/ragflow-native-libs`) AND the runtime resources the backend loads at runtime — `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, `ocr.res` into `internal/rag/res/deepdoc/` (required by the in-process Go DeepDoc backend) and `ragflow_deps/cl100k_base.tiktoken` and the embedding tokenizer assets under `ragflow_deps/huggingface.co/`. Run it once and the local environment is ready; already-present files are skipped on re-run. The image-facing `ragflow_deps/download_deps.py` fetches the same runtime resources PLUS the stagehand server (installed into `${XDG_CACHE_HOME:-$HOME/.cache}/stagehand/lib/go_<SDK version>/`) and skips the native `.a`; run it only when building the dependency image or in CI. Native archives support Linux x86_64. Keep the server's working directory at the repository root so it can find the DeepDoc models automatically; if you launch it elsewhere, set `DEEPDOC_MODEL_DIR` to the absolute path of `internal/rag/res/deepdoc`.+`download_dev_deps.py` is the local-dev one-stop: it fetches the static libraries needed by `build.sh` (pdfium / pdf_oxide / office_oxide / onnxruntime → `~/ragflow-native-libs`) AND the runtime resources the backend loads at runtime — `det.ort`, `layout.ort`, `tsr.ort`, `rec.ort`, `ocr.res` into `internal/rag/res/deepdoc/` (required by the in-process Go DeepDoc backend) and `deps/cl100k_base.tiktoken` and the embedding tokenizer assets under `deps/huggingface.co/`. Run it once and the local environment is ready; already-present files are skipped on re-run. The image-facing `deps/download_deps.py` fetches the same runtime resources PLUS the stagehand server (installed into `${XDG_CACHE_HOME:-$HOME/.cache}/stagehand/lib/go_<SDK version>/`) and skips the native `.a`; run it only when building the dependency image or in CI. Native archives support Linux x86_64. Keep the server's working directory at the repository root so it can find the DeepDoc models automatically; if you launch it elsewhere, set `DEEPDOC_MODEL_DIR` to the absolute path of `internal/rag/res/deepdoc`.Skip either script if you have prepared the same resources by other means.@@ -70,10 +70,10 @@ Build the C++ bindings and Go binaries:bash build.sh --all```-For a smaller production binary, use `bash build.sh --strip --all` instead. The dependency downloader prepares `ragflow_deps/cl100k_base.tiktoken`. If the table is missing, download it before starting the server:+For a smaller production binary, use `bash build.sh --strip --all` instead. The dependency downloader prepares `deps/cl100k_base.tiktoken`. If the table is missing, download it before starting the server:```bash-curl -fsSL -o ragflow_deps/cl100k_base.tiktoken https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken+curl -fsSL -o deps/cl100k_base.tiktoken https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken```Check that the newly built executable can load and parse API arguments:@@ -86,15 +86,15 @@ It should print API usage information and may exit with status 1. This command oThe Go/C++ build requires LLD 20. Confirm that `ld.lld --version` reports LLD 20 before building; merely installing `lld-20` is insufficient when an older `ld.lld` remains the system default. If the binary builds successfully but exits or crashes before reaching Go `main`, verify the selected linker and rebuild with LLD 20.-To prepare the resource image used by the Go Dockerfile, first run `uv run ragflow_deps/download_deps.py` (above) to populate `ragflow_deps/`, then build from `ragflow_deps/`:+To prepare the resource image used by the Go Dockerfile, first run `uv run deps/download_deps.py` (above) to populate `deps/`, then build from `deps/`:```bash-cd ragflow_deps+cd depsdocker build -f Dockerfile -t infiniflow/ragflow_deps:latest .cd ..```-The downloader prepares the five Go DeepDoc files under `ragflow_deps/huggingface.co/InfiniFlow/deepdoc/` for the resource image, as well as `internal/rag/res/deepdoc/` for local use. The Go Dockerfile copies these models into the runtime model directory. Analyzer dictionaries and fonts are installed separately by the Go Dockerfile.+The downloader prepares the five Go DeepDoc files under `deps/huggingface.co/InfiniFlow/deepdoc/` for the resource image, as well as `internal/rag/res/deepdoc/` for local use. The Go Dockerfile copies these models into the runtime model directory. Analyzer dictionaries and fonts are installed separately by the Go Dockerfile.## 2. Start supporting services@@ -155,7 +155,7 @@ If you changed a dependency's published port or credentials, update `conf/servic`RAGFLOW_DEV_MODE=true` bypasses the code-versus-database version check. Use it only for development, not to run an older production binary against a newer database. The standalone `--migrate` action must still complete first.-If startup reports `no in-process DeepDoc backend serving`, check that all five model files above are in `internal/rag/res/deepdoc/`, then re-run the Go dependency downloader and rebuild if necessary. If the tokenizer reports a missing `cl100k_base.tiktoken`, ensure the BPE table is in `ragflow_deps/`.+If startup reports `no in-process DeepDoc backend serving`, check that all five model files above are in `internal/rag/res/deepdoc/`, then re-run the Go dependency downloader and rebuild if necessary. If the tokenizer reports a missing `cl100k_base.tiktoken`, ensure the BPE table is in `deps/`.## 4. Start the frontenddiff --git a/internal/common/environments.go b/internal/common/environments.goindex 473df68eff..068e32b7b6 100644--- a/internal/common/environments.go+++ b/internal/common/environments.go@@ -445,7 +445,7 @@ const (// do set-membership checks); keep it stable so logs and diffs stay readable.//// External consumers that re-list these names must stay in sync:-// - ragflow_deps/download_deps.py re-lists them as DEEPDOC_MODEL_FILES+// - deps/download_deps.py re-lists them as DEEPDOC_MODEL_FILES// (it fetches the files one by one, so it MUST be edited by hand when this// slice changes);var DeepDocModelFiles = []string{diff --git a/internal/common/model_assets.go b/internal/common/model_assets.goindex e36e0e9a59..cd0a0fe46c 100644--- a/internal/common/model_assets.go+++ b/internal/common/model_assets.go@@ -21,7 +21,7 @@ import ("strings")-// EnvModelAssetsDir points at a directory that plays the role of `ragflow_deps/`:+// EnvModelAssetsDir points at a directory that plays the role of `deps/`:// downloaded model assets stored in the HuggingFace repo layout, i.e.// `<dir>/huggingface.co/<repo>/<file>` (plus loose files that are not HF repos, such as// the cl100k BPE table). It is how a deployment keeps model assets outside the checkout -@@ -36,11 +36,11 @@ const EnvModelAssetsDir = "MODEL_ASSETS_DIR"// ModelAssetCandidates returns the paths to try, in priority order, for one asset when// MODEL_ASSETS_DIR is set. `relative` is the shipped name - either-// "ragflow_deps/huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model" or a bare+// "deps/huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model" or a bare// "cl100k_base.tiktoken" - and the candidates let an operator point at any of the three// sensible roots://-// <dir>/huggingface.co/<repo>/<file> the ragflow_deps-like root+// <dir>/huggingface.co/<repo>/<file> the deps-like root// <dir>/<repo>/<file> a huggingface.co-like root// <dir>/<file> a flat directory holding the files//@@ -50,7 +50,7 @@ func ModelAssetCandidates(relative string) []string {if root == "" {return nil}- rel := strings.TrimPrefix(filepath.ToSlash(relative), "ragflow_deps/")+ rel := strings.TrimPrefix(filepath.ToSlash(relative), "deps/")out := make([]string, 0, 3)seen := make(map[string]struct{}, 3)add := func(p string) {diff --git a/internal/common/model_assets_test.go b/internal/common/model_assets_test.goindex f45019f3aa..07b28c1639 100644--- a/internal/common/model_assets_test.go+++ b/internal/common/model_assets_test.go@@ -23,7 +23,7 @@ import (func TestModelAssetCandidatesOnlyAppliesWhenTheEnvIsSet(t *testing.T) {t.Setenv(EnvModelAssetsDir, "")- if got := ModelAssetCandidates("ragflow_deps/huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model"); got != nil {+ if got := ModelAssetCandidates("deps/huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model"); got != nil {t.Fatalf("expected no candidates without %s, got %v", EnvModelAssetsDir, got)}t.Setenv(EnvModelAssetsDir, " ")@@ -33,12 +33,12 @@ func TestModelAssetCandidatesOnlyAppliesWhenTheEnvIsSet(t *testing.T) {}// The three layouts an operator may have in mind when pointing MODEL_ASSETS_DIR at a-// tree: the ragflow_deps-like root, a huggingface.co-like root, or a flat directory.+// tree: the deps-like root, a huggingface.co-like root, or a flat directory.func TestModelAssetCandidatesCoverEverySensibleRoot(t *testing.T) {root := filepath.Join(string(filepath.Separator)+"assets", "models")t.Setenv(EnvModelAssetsDir, root)- got := ModelAssetCandidates("ragflow_deps/huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model")+ got := ModelAssetCandidates("deps/huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model")want := []string{filepath.Join(root, "huggingface.co", "BAAI", "bge-m3", "sentencepiece.bpe.model"),filepath.Join(root, "BAAI", "bge-m3", "sentencepiece.bpe.model"),@@ -54,7 +54,7 @@ func TestModelAssetCandidatesCoverEverySensibleRoot(t *testing.T) {}}- // The same asset named without the ragflow_deps/ prefix must resolve identically.+ // The same asset named without the deps/ prefix must resolve identically.bare := ModelAssetCandidates("huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model")if len(bare) != len(want) || bare[0] != want[0] {t.Errorf("prefix form differs: %v vs %v", bare, want)@@ -62,7 +62,7 @@ func TestModelAssetCandidatesCoverEverySensibleRoot(t *testing.T) {// A file that is not an HF repo (the cl100k table) sits at the root itself,// and its single candidate must not be repeated.- flat := ModelAssetCandidates("ragflow_deps/cl100k_base.tiktoken")+ flat := ModelAssetCandidates("deps/cl100k_base.tiktoken")if len(flat) != 1 || flat[0] != filepath.Join(root, "cl100k_base.tiktoken") {t.Errorf("flat asset candidates = %v, want exactly one root-level candidate", flat)}diff --git a/internal/development.md b/internal/development.mdindex f5a599cb4d..ca98f0c913 100644--- a/internal/development.md+++ b/internal/development.md@@ -72,17 +72,17 @@ uv run tools/scripts/download_dev_deps.py # local-dev one-stop: native .a + clInstall `uv` before running the downloaders; it installs each script's declared Python dependencies.-`download_dev_deps.py` is the local-dev one-stop: it provisions the Go native libraries under `~/ragflow-native-libs` AND the `.ort` weights, embedding tokenizer assets, and `cl100k_base.tiktoken` into `internal/rag/res/deepdoc/` and `ragflow_deps/huggingface.co/`. Run it once and the local environment is ready. The image-facing `ragflow_deps/download_deps.py` provisions the same runtime resources plus the stagehand driver (installed into the SDK's cache, using the SDK version from `go.mod`) and skips the native `.a`; run it only when building the dependency image or in CI. Native archives are fetched per target platform (linux/darwin on amd64/arm64; override with `RAGFLOW_TARGET_OS` / `RAGFLOW_TARGET_ARCH`).+`download_dev_deps.py` is the local-dev one-stop: it provisions the Go native libraries under `~/ragflow-native-libs` AND the `.ort` weights, embedding tokenizer assets, and `cl100k_base.tiktoken` into `internal/rag/res/deepdoc/` and `deps/huggingface.co/`. Run it once and the local environment is ready. The image-facing `deps/download_deps.py` provisions the same runtime resources plus the stagehand driver (installed into the SDK's cache, using the SDK version from `go.mod`) and skips the native `.a`; run it only when building the dependency image or in CI. Native archives are fetched per target platform (linux/darwin on amd64/arm64; override with `RAGFLOW_TARGET_OS` / `RAGFLOW_TARGET_ARCH`).-Build the Go resource image from `ragflow_deps/` after preparing these files:+Build the Go resource image from `deps/` after preparing these files:```shell-cd ragflow_deps+cd depsdocker build -f Dockerfile -t infiniflow/ragflow_deps:latest .cd ..```-The image contains Go DeepDoc weights, tokenizer assets, the BPE table, and stagehand binaries. DeepDoc weights are prepared under `ragflow_deps/huggingface.co/InfiniFlow/deepdoc/` for packaging and under `internal/rag/res/deepdoc/` for local use.+The image contains Go DeepDoc weights, tokenizer assets, the BPE table, and stagehand binaries. DeepDoc weights are prepared under `deps/huggingface.co/InfiniFlow/deepdoc/` for packaging and under `internal/rag/res/deepdoc/` for local use.> **Note**: If you use IDEs like GoLand to run/debug directly (via Run/Debug buttons), or run `go build` / `go run` from command line, set these CGO environment variables:>> ```bash@@ -193,7 +193,7 @@ The image contains Go DeepDoc weights, tokenizer assets, the BPE table, and stag> **Note**: `build.sh` also guards ONNX Runtime **archive integrity** and the> **link cache**, because a silently-stale `.a` is easy to miss:-> - `check_onnxruntime_deps` compares the local release ZIP under `ragflow_deps/`+> - `check_onnxruntime_deps` compares the local release ZIP under `deps/`> (`onnxruntime-v<ver>-<target>.zip`, e.g. `-linux-x86_64` / `-osx-arm64`)> against the SHA-256 digest pinned in> `build.sh`, without a network request. The downloader separately checks the@@ -273,7 +273,7 @@ missing from the model directory.If you instead see a fatal `no in-process DeepDoc backend serving`, it hastwo possible causes: ORT was not linked into the binary, or the modeldirectory is missing one of the five required files listed above. Check the- weights first, then re-run `uv run ragflow_deps/download_deps.py` and rebuild+ weights first, then re-run `uv run deps/download_deps.py` and rebuild(§1.4 explains the ORT link failure; `build.sh` fails fast with`Error: ONNX Runtime static libraries are not linked` before that happens).diff --git a/internal/ingestion/component/chunker/tool-py/capture_golden.py b/internal/ingestion/component/chunker/tool-py/capture_golden.pyindex 2b3cbe8182..76913efd46 100644--- a/internal/ingestion/component/chunker/tool-py/capture_golden.py+++ b/internal/ingestion/component/chunker/tool-py/capture_golden.py@@ -136,7 +136,7 @@ def _assert_tokenizer_alive() -> None:from rag.nlp import num_tokens_from_stringprobe = "RAGFlow chunker parity tokenizer liveness probe."- hint = "Fix the encoding first: bash build.sh --go provisions ragflow_deps/cl100k_base.tiktoken."+ hint = "Fix the encoding first: bash build.sh --go provisions deps/cl100k_base.tiktoken."# Two shapes to catch. common/token_utils.py builds the encoder on first use and# lets that failure propagate, so an unavailable BPE table raises here; a failure# to encode an otherwise valid string is still reported as 0.diff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.goindex 3d5744d0d8..387a123b0c 100644--- a/internal/ingestion/pipeline/real_storage_integration_test.go+++ b/internal/ingestion/pipeline/real_storage_integration_test.go@@ -271,7 +271,7 @@ func mustWriteTokenizerPOSDef(t *testing.T, dictPath, outPath string) {func mustPrepareTokenizerWordNet(t *testing.T, root string) {t.Helper()- zipPath := filepath.Join(repoRootFromPipelineTest(t), "ragflow_deps", "nltk_data", "corpora", "wordnet.zip")+ zipPath := filepath.Join(repoRootFromPipelineTest(t), "deps", "nltk_data", "corpora", "wordnet.zip")reader, err := zip.OpenReader(zipPath)if err != nil {t.Skipf("open wordnet zip %s: %v", zipPath, err)diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.goindex 8c94c9cdde..8c5ff04a95 100644--- a/internal/ingestion/task/embedder.go+++ b/internal/ingestion/task/embedder.go@@ -98,7 +98,7 @@ func (e *embedder) limiter() tokenizer.Limiter {}if id != "" && !tokenizer.CounterExact(id) {e.limiterErr = fmt.Errorf(- "embedding tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_deps.py`): refusing to count with the calibrated estimate",+ "embedding tokenizer %q is declared for %s but its asset is unavailable (check that deps/huggingface.co is present; run `uv run deps/download_deps.py`): refusing to count with the calibrated estimate",id, e.quotaKey())}e.limiterVal = tokenizer.LimiterFor(id, string(e.quotaKey()), tokenizer.DefaultCalibration())diff --git a/internal/ingestion/task/pipeline_real_integration_test.go b/internal/ingestion/task/pipeline_real_integration_test.goindex e69c936f1e..c1b447f074 100644--- a/internal/ingestion/task/pipeline_real_integration_test.go+++ b/internal/ingestion/task/pipeline_real_integration_test.go@@ -528,7 +528,7 @@ func taskMustWriteTokenizerPOSDef(t *testing.T, dictPath, outPath string) {func taskMustPrepareTokenizerWordNet(t *testing.T, root string) {t.Helper()- zipPath := filepath.Join(taskRepoRoot(t), "ragflow_deps", "nltk_data", "corpora", "wordnet.zip")+ zipPath := filepath.Join(taskRepoRoot(t), "deps", "nltk_data", "corpora", "wordnet.zip")reader, err := zip.OpenReader(zipPath)if err != nil {t.Skipf("open wordnet zip %s: %v", zipPath, err)diff --git a/internal/server/deepdoc/server.go b/internal/server/deepdoc/server.goindex de5791a1a4..22aa309885 100644--- a/internal/server/deepdoc/server.go+++ b/internal/server/deepdoc/server.go@@ -146,7 +146,7 @@ func RegisterNativeDeepDoc(arguments *bootstrap.ServerArgs) error {// resolveDeepDocModelDir picks the model directory: the explicit DEEPDOC_MODEL_DIR// env, else the RAGFlow default model dir (internal/rag/res/deepdoc),-// else the assets fetched by ragflow_deps/download_deps.py. The first+// else the assets fetched by deps/download_deps.py. The first// candidate that actually contains the required weights wins.func resolveDeepDocModelDir() string {if v := strings.TrimSpace(common.GetEnv(common.EnvDeepDocModelDir)); v != "" {diff --git a/internal/server/ingestor/server.go b/internal/server/ingestor/server.goindex dd4caa9d6c..0053a18af2 100644--- a/internal/server/ingestor/server.go+++ b/internal/server/ingestor/server.go@@ -113,7 +113,7 @@ func logTokenizerCounters() {if len(unavailable) > 0 {common.Warn("embedding tokenizers unavailable; models that declare them cannot be ingested until the asset is restored",zap.Strings("unavailable", unavailable),- zap.String("hint", "run `uv run ragflow_deps/download_deps.py`, or set "+common.EnvModelAssetsDir+" to a directory holding them"))+ zap.String("hint", "run `uv run deps/download_deps.py`, or set "+common.EnvModelAssetsDir+" to a directory holding them"))}}diff --git a/internal/tokenizer/asset_dir_test.go b/internal/tokenizer/asset_dir_test.goindex e95e020161..dff179eee0 100644--- a/internal/tokenizer/asset_dir_test.go+++ b/internal/tokenizer/asset_dir_test.go@@ -34,9 +34,9 @@ import (const assetDirChildEnv = "RAGFLOW_TOKENIZER_ASSET_DIR_CHILD"-// realSPMAsset is the asset shipped by ragflow_deps/download_deps.py, relative to the+// realSPMAsset is the asset shipped by deps/download_deps.py, relative to the// repository root.-var realSPMAsset = filepath.Join("ragflow_deps", "huggingface.co", "BAAI", "bge-m3", "sentencepiece.bpe.model")+var realSPMAsset = filepath.Join("deps", "huggingface.co", "BAAI", "bge-m3", "sentencepiece.bpe.model")func TestModelAssetsDirIsHonoured(t *testing.T) {if os.Getenv(assetDirChildEnv) != "" {@@ -44,7 +44,7 @@ func TestModelAssetsDirIsHonoured(t *testing.T) {}source := filepath.Join("..", "..", realSPMAsset)if _, err := os.Stat(source); err != nil {- t.Skipf("the SPM asset is not present here (%v); run ragflow_deps/download_deps.py", err)+ t.Skipf("the SPM asset is not present here (%v); run deps/download_deps.py", err)}dir := t.TempDir()diff --git a/internal/tokenizer/bpe.go b/internal/tokenizer/bpe.goindex d244267927..bf65b381a4 100644--- a/internal/tokenizer/bpe.go+++ b/internal/tokenizer/bpe.go@@ -901,12 +901,12 @@ const ()var llamaBPEAssetNames = []string{- "ragflow_deps/huggingface.co/intfloat/e5-mistral-7b-instruct/tokenizer.json",+ "deps/huggingface.co/intfloat/e5-mistral-7b-instruct/tokenizer.json","llama_bpe_tokenizer.json",}var qwenBPEAssetNames = []string{- "ragflow_deps/huggingface.co/Qwen/Qwen3-Embedding-0.6B/tokenizer.json",+ "deps/huggingface.co/Qwen/Qwen3-Embedding-0.6B/tokenizer.json","qwen_bpe_tokenizer.json",}@@ -961,7 +961,7 @@ const metaspaceCounterID = "metaspace-bpe"// LoadMetaspaceBPE loads a Metaspace-flavored BPE tokenizer.json from an// operator-provided path. Unlike the catalog counters this path is not pinned:// the file ships inside the Laya model directory (RAGFLOW_DECISION_DIR), not in-// ragflow_deps, so there is no shipped digest to check here.+// deps, so there is no shipped digest to check here.func LoadMetaspaceBPE(path string) (MetaspaceEncoder, error) {model, err := loadBPEModel(path, bpeFlavorMetaspace)if err != nil {diff --git a/internal/tokenizer/bpe_loader.go b/internal/tokenizer/bpe_loader.goindex 3a4dcc82bd..ad31cfca58 100644--- a/internal/tokenizer/bpe_loader.go+++ b/internal/tokenizer/bpe_loader.go@@ -20,7 +20,7 @@ package tokenizer//// RAGFlow ships the cl100k_base table on disk (Dockerfile drops it into the// working directory under its sha1 name; download_deps.py writes it to-// ragflow_deps/). tiktoken-go's stock loader instead downloads it over HTTP and+// deps/). tiktoken-go's stock loader instead downloads it over HTTP and// relies on TIKTOKEN_CACHE_DIR, which the Go server never inherits, so a// missing table degrades every token count to 0. This loader resolves the// table from disk only: it performs no network I/O, and when nothing is found@@ -56,7 +56,7 @@ func init() {// (223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7 for// cl100k_base); that value identifies the path, while the value below verifies// the bytes we actually load. Compute it from the table shipped by-// ragflow_deps/download_deps.py: `sha1sum cl100k_base.tiktoken`.+// deps/download_deps.py: `sha1sum cl100k_base.tiktoken`.var expectedBpeHashes = map[string]string{"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken": "6494e42d5aad2bbb441ea9793af9e7db335c8d9c",}@@ -122,7 +122,7 @@ func (localBpeLoader) LoadTiktokenBpe(bpeURL string) (map[string]int, error) {}err := fmt.Errorf(- "no local BPE table for %s; run `uv run ragflow_deps/download_deps.py` or set TIKTOKEN_CACHE_DIR to the directory holding the table; tried: %s",+ "no local BPE table for %s; run `uv run deps/download_deps.py` or set TIKTOKEN_CACHE_DIR to the directory holding the table; tried: %s",bpeURL, strings.Join(candidates, ", "))// Logged as well as returned: tiktoken-go propagates this to GetEncoding,// whose error NumTokensFromString discards to keep returning 0.@@ -170,10 +170,10 @@ func bpeCandidatePaths(bpeURL string) []string {// Same layout the Dockerfile creates: the table sits in the// installation root under its sha1 name.add(filepath.Join(root, cacheName))- // download_deps.py writes the table into ragflow_deps/ under its+ // download_deps.py writes the table into deps/ under its// download name; a developer checkout that has run it but never// started the Python side only has this copy.- add(filepath.Join(root, "ragflow_deps", bundledName))+ add(filepath.Join(root, "deps", bundledName))}return pathsdiff --git a/internal/tokenizer/bpe_loader_anchors_test.go b/internal/tokenizer/bpe_loader_anchors_test.goindex d4da6cf65b..e31af0da52 100644--- a/internal/tokenizer/bpe_loader_anchors_test.go+++ b/internal/tokenizer/bpe_loader_anchors_test.go@@ -34,7 +34,7 @@ import "testing"// structurally valid but wrong table.//// This test needs the real cl100k_base table on disk (TIKTOKEN_CACHE_DIR,-// the Dockerfile's /ragflow/<sha1> file, or ragflow_deps/cl100k_base.tiktoken),+// the Dockerfile's /ragflow/<sha1> file, or deps/cl100k_base.tiktoken),// so it is tagged `manual` and runs only under `build.sh --test-manual`,// which the docker builder provisions with /usr/share/infinity/resource.func TestNumTokensFromString_MatchesPythonAnchors(t *testing.T) {diff --git a/internal/tokenizer/bpe_loader_test.go b/internal/tokenizer/bpe_loader_test.goindex 3c3ca84867..7834feaea2 100644--- a/internal/tokenizer/bpe_loader_test.go+++ b/internal/tokenizer/bpe_loader_test.go@@ -121,7 +121,7 @@ func TestLocalBpeLoader_ReadsSha1FileFromWorkingDirectory(t *testing.T) {// its download name.func TestLocalBpeLoader_ReadsBundledVocabFromAncestor(t *testing.T) {dir := isolate(t)- writeBpeTable(t, filepath.Join(dir, "ragflow_deps", "cl100k_base.tiktoken"), 10)+ writeBpeTable(t, filepath.Join(dir, "deps", "cl100k_base.tiktoken"), 10)nested := filepath.Join(dir, "internal", "tokenizer")if err := os.MkdirAll(nested, 0o755); err != nil {t.Fatalf("mkdir: %v", err)@@ -139,7 +139,7 @@ func TestLocalBpeLoader_ReadsBundledVocabFromAncestor(t *testing.T) {func TestLocalBpeLoader_CacheDirWinsOverBundledVocab(t *testing.T) {dir := isolate(t)- writeBpeTable(t, filepath.Join(dir, "ragflow_deps", "cl100k_base.tiktoken"), 10)+ writeBpeTable(t, filepath.Join(dir, "deps", "cl100k_base.tiktoken"), 10)cache := t.TempDir()writeBpeTable(t, filepath.Join(cache, cacheFileName(testBpeURL)), 7)t.Setenv("TIKTOKEN_CACHE_DIR", cache)diff --git a/internal/tokenizer/counter_oracle_test.go b/internal/tokenizer/counter_oracle_test.goindex 2e9b22927e..90c8cff009 100644--- a/internal/tokenizer/counter_oracle_test.go+++ b/internal/tokenizer/counter_oracle_test.go@@ -118,7 +118,7 @@ func checkCounterAgainstOracle(t *testing.T, fx oracleFixture) {// asset it needs is not on disk.for _, status := range CounterStatuses() {if status.ID == fx.Counter {- t.Fatalf("fixture %q needs counter %q, which is declared but unavailable: its asset is missing from ragflow_deps/huggingface.co (run `uv run ragflow_deps/download_deps.py`)", fx.Model, fx.Counter)+ t.Fatalf("fixture %q needs counter %q, which is declared but unavailable: its asset is missing from deps/huggingface.co (run `uv run deps/download_deps.py`)", fx.Model, fx.Counter)}}t.Fatalf("fixture %q names counter %q, which this build does not implement", fx.Model, fx.Counter)diff --git a/internal/tokenizer/embedding_token_limits.md b/internal/tokenizer/embedding_token_limits.mdindex afe13a0278..3edd6d85d3 100644--- a/internal/tokenizer/embedding_token_limits.md+++ b/internal/tokenizer/embedding_token_limits.md@@ -180,7 +180,7 @@ that declares an unavailable counter fails instead of being calibrated silently.| id | family / models | asset (SHA-1 pinned) | implementation ||---|---|---|---|-| `cl100k_base` | OpenAI `text-embedding-3-*`, `ada-002` | `ragflow_deps/cl100k_base.tiktoken` (`download_deps.py`) | tiktoken-go + offline loader (`bpe_loader.go`) |+| `cl100k_base` | OpenAI `text-embedding-3-*`, `ada-002` | `deps/cl100k_base.tiktoken` (`download_deps.py`) | tiktoken-go + offline loader (`bpe_loader.go`) || `xlmr-spm` | `bge-m3`, `multilingual-e5-*`, `m3e`, `gte-multilingual`, `jina-embeddings-v3` | `…/BAAI/bge-m3/sentencepiece.bpe.model` `7e88c49f…` | `spm.go`: Unigram Viterbi, byte fallback, NFKC + whitespace + charmap-lite || `bert-wordpiece` | `bge-{large,base,small}-en-v1.5`, `bge-en-icl`, `e5-{base,large}-v2`, `gte-{base,large}`, `jina-embeddings-v2-*` | `…/BAAI/bge-large-en-v1.5/vocab.txt` `c3b41053…` | `wordpiece.go`: BertNormalizer + BertPreTokenizer + `##` longest-match || `qwen-bpe` | `Qwen3-Embedding-*` | `…/Qwen/Qwen3-Embedding-0.6B/tokenizer.json` `e6592f4d…` | `bpe.go`: hand-written Split scanner, GPT-2 byte alphabet, BPE |@@ -348,11 +348,11 @@ an unrelated `vocab.txt` there silently replaces the verified artifact; and audi`vocab.txt` / `sentencepiece.bpe.model`). If the vocabulary is byte-identical to afamily fixture, only step 4 is needed; if it differs, the model needs its own fixture.2. **Add the artifact** to the `TOKENIZER_ASSETS` list inlined in- `ragflow_deps/download_deps.py` (the downloader fetches individual files from) -+ `deps/download_deps.py` (the downloader fetches individual files from) -individual files, not `snapshot_download` of a multi-GB repo - pin its SHA-1 in theloader map so a mismatch fails at load instead of counting with the wrong table, and addit to the copy loop in `Dockerfile` / `Dockerfile_base` so every runtime image ships it- (the published `ragflow_deps` image also needs the asset, so refresh it -+ (the published `deps` image also needs the asset, so refresh it -see §Assets and the oracle for the whole chain).3. **Add the fixture line** to `ORACLES` in `scripts/gen_tokenizer_oracle.py`, generate it,and run `TestCountersMatchOracle`. Passing means every sample agrees, or sits inside the@@ -390,10 +390,10 @@ property tier green, `gofmt` clean.## Assets and the oracleAssets are fetched, never committed: the `TOKENIZER_ASSETS` list inlined in-`ragflow_deps/download_deps.py` is the source the downloader reads to download+`deps/download_deps.py` is the source the downloader reads to download**individual files** (not `snapshot_download` of a repo that also carries multi-GBweights) from, into-`ragflow_deps/huggingface.co/<repo>/<file>`, which+`deps/huggingface.co/<repo>/<file>`, which`.gitignore` already excludes (`huggingface.co/`). Loaders read them from disk with aSHA-1 pin, walk up from the working and executable directories like `bpe_loader.go`, andperform no network I/O. A missing asset fails loudly at every layer that can report one: the@@ -402,10 +402,10 @@ model declares a tokenizer it cannot load refuses to count rather than substitutcalibrated estimate, and the cl100k table fetched from the OpenAI blob has always been a startup panic when missing.**The whole chain, because a gap in it is invisible**: `download_deps.py` writes-`ragflow_deps/huggingface.co/…`; the published `ragflow_deps` image contains those+`deps/huggingface.co/…`; the published `deps` image contains thosefiles under `/huggingface.co`; the root`Dockerfile` and `Dockerfile_base` bind-mount that image and copy the four *runtime*-tokenizer assets into `/ragflow/ragflow_deps/huggingface.co/…`, which is where the+tokenizer assets into `/ragflow/deps/huggingface.co/…`, which is where thecounters look (the working directory is `/ragflow`). The `tokenizer.json` files thatexist only as cross-check oracles are deliberately **not** shipped. Two consequencesworth knowing: adding an asset means touching all three places (script, deps image,@@ -428,23 +428,23 @@ degraded to the calibrated path in the image.By default the loaders walk the working directory and the executable's directory (andtheir ancestors) for the shipped relative names. `MODEL_ASSETS_DIR` overrides that: it-names a directory playing the role of `ragflow_deps/` - downloaded model assets in the+names a directory playing the role of `deps/` - downloaded model assets in theHuggingFace repo layout - and it is **consulted first**. Three spellings are accepted, soan operator can point at whichever tree they have:| `MODEL_ASSETS_DIR` points at | candidate used ||---|---|-| the `ragflow_deps`-like root | `<dir>/huggingface.co/<repo>/<file>` |+| the `deps`-like root | `<dir>/huggingface.co/<repo>/<file>` || a `huggingface.co`-like root | `<dir>/<repo>/<file>` || a flat directory | `<dir>/<file>` |The variable is deliberately **not** embedding-specific: `resolveDeepDocModelDir`also checks `<dir>/huggingface.co/InfiniFlow/deepdoc` when `MODEL_ASSETS_DIR`points at `<dir>`. The downloader prepares the five Go DeepDoc files in that tree-under `ragflow_deps/`, as well as in `internal/rag/res/deepdoc/` for local use.-One mounted `ragflow_deps/` directory can therefore serve both asset types.-`ragflow_deps/Dockerfile` packages the tree under `/huggingface.co/` when built-from `ragflow_deps/`.+under `deps/`, as well as in `internal/rag/res/deepdoc/` for local use.+One mounted `deps/` directory can therefore serve both asset types.+`deps/Dockerfile` packages the tree under `/huggingface.co/` when built+from `deps/`.The layout and the precedence are pinned by tests:`internal/common/model_assets_test.go` for the candidate list, and`internal/tokenizer/asset_dir_test.go` for the end-to-end case (a child process with an@@ -455,7 +455,7 @@ Two things make the state of the assets observable:- each counter reports the file it loaded (`SourcePath`), so logs and failures can name it;- the ingestor logs an **availability report once at startup** -`embedding tokenizer counters {available: [...]}` plus, when something is missing, a- warning naming the unavailable ones and pointing at `ragflow_deps/download_deps.py` /+ warning naming the unavailable ones and pointing at `deps/download_deps.py` /`MODEL_ASSETS_DIR`. The report is diagnostics rather than the guard: startup is not fatal(untagged models count with the calibrated estimate by design, and they keep working), butingesting a model that declares a missing asset fails outright - so this report is what@@ -489,7 +489,7 @@ exactly one such spot, 18,182 tokens in).| counter | reference implementation | what is compared ||---|---|---|-| `cl100k_base` | OpenAI's own `tiktoken` (Python 0.12.0) over the table shipped in `ragflow_deps` | token **ids** (the numbering is shared - both read the same table) |+| `cl100k_base` | OpenAI's own `tiktoken` (Python 0.12.0) over the table shipped in `deps` | token **ids** (the numbering is shared - both read the same table) || `xlmr-spm` | the HuggingFace conversion of `BAAI/bge-m3` (the files SILICONFLOW serves), via `tokenizers` 0.22.2 reading the model's `tokenizer.json` | token **texts** - this family's ids are renumbered by the conversion (below) || `bert-wordpiece` | the same library reading `BAAI/bge-large-en-v1.5/tokenizer.json`; the algorithm itself is Google's `WordPiece` / `BasicTokenizer` | token texts (ids match as well) || `qwen-bpe`, `llama-bpe` | the same library reading each model's `tokenizer.json` | token texts (ids match as well) |@@ -615,7 +615,7 @@ against a different vocabulary:| counter | authoritative artifact (pinned) | format / algorithm reference | behavioural reference ||---|---|---|---|-| `cl100k_base` | `ragflow_deps/cl100k_base.tiktoken` (`expectedBpeHashes`, `bpe_loader.go`) | tiktoken's table format (`<base64 token> <rank>` per line) | the table itself, plus the Python anchors in `TestNumTokensFromString_MatchesPythonAnchors` |+| `cl100k_base` | `deps/cl100k_base.tiktoken` (`expectedBpeHashes`, `bpe_loader.go`) | tiktoken's table format (`<base64 token> <rank>` per line) | the table itself, plus the Python anchors in `TestNumTokensFromString_MatchesPythonAnchors` || `xlmr-spm` | `BAAI/bge-m3/sentencepiece.bpe.model` `7e88c49f…` | `sentencepiece/model.proto` wire format (decoded by the minimal protobuf reader in `spm.go`): `pieces`, `trainer_spec.model_type = UNIGRAM`, `normalizer_spec`; Unigram Viterbi; `BYTE` pieces for fallback | the HuggingFace conversion of the same repo (`tokenizer.json`) — it decides whitespace and pre-tokenization behaviour, and it **wins over the `.model` flags** when they disagree || `bert-wordpiece` | `BAAI/bge-large-en-v1.5/vocab.txt` `c3b41053…` | HuggingFace `BertNormalizer` / `BertPreTokenizer` / `WordPiece`, whose configuration is quoted from the model's `tokenizer.json` in `wordpiece.go`'s header; BERT's own `_is_punctuation` (category **P**, not S) | the same `tokenizer.json` || `qwen-bpe` | `Qwen/Qwen3-Embedding-0.6B/tokenizer.json` `e6592f4d…` | the artifact's `pre_tokenizer` regex, copied verbatim into `bpe.go`, plus GPT-2's `bytes_to_unicode` alphabet and canonical BPE merge order | the artifact's own `vocab` + `merges` (`NFC`, `byte_fallback=false`) |diff --git a/internal/tokenizer/embedlimit.go b/internal/tokenizer/embedlimit.goindex 45d753d639..2364071da0 100644--- a/internal/tokenizer/embedlimit.go+++ b/internal/tokenizer/embedlimit.go@@ -716,7 +716,7 @@ func RefuseUnavailableCounter(tokenizerID, calibrationKey string) error {return nil}return fmt.Errorf(- "model tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_deps.py`): refusing to count with the calibrated estimate",+ "model tokenizer %q is declared for %s but its asset is unavailable (check that deps/huggingface.co is present; run `uv run deps/download_deps.py`): refusing to count with the calibrated estimate",tokenizerID, calibrationKey)}@@ -740,7 +740,7 @@ func OwnTokenMax(texts []string, counter Counter) int {type cl100kCounter struct{ id string }// CountCL100K is the cl100k_base counter, backed by the BPE table shipped in-// ragflow_deps (see bpe_loader.go).+// deps (see bpe_loader.go).func CountCL100K() Counter { return cl100kCounter{id: CounterCL100K} }func (c cl100kCounter) ID() string { return c.id }diff --git a/internal/tokenizer/failfast_test.go b/internal/tokenizer/failfast_test.goindex 4bb8abaa58..ef6a7a81d4 100644--- a/internal/tokenizer/failfast_test.go+++ b/internal/tokenizer/failfast_test.go@@ -139,14 +139,14 @@ func findRealBpeTable(t *testing.T) string {}}}- // 2. repo-local ragflow_deps/cl100k_base.tiktoken (provisioned by download+ // 2. repo-local deps/cl100k_base.tiktoken (provisioned by download// scripts / CI), walking up from the package dir.pkgDir, err := os.Getwd()if err != nil {return ""}for dir := pkgDir; ; {- p := filepath.Join(dir, "ragflow_deps", "cl100k_base.tiktoken")+ p := filepath.Join(dir, "deps", "cl100k_base.tiktoken")if fileExists(p) {return p}diff --git a/internal/tokenizer/spm.go b/internal/tokenizer/spm.goindex 2e88499f8d..4878390f83 100644--- a/internal/tokenizer/spm.go+++ b/internal/tokenizer/spm.go@@ -21,7 +21,7 @@ package tokenizer// BAAI/bge-m3, intfloat/multilingual-e5-*, moka-ai/m3e-*, Alibaba-NLP/gte-// multilingual-base and jina-embeddings-v3 all tokenize with the same// XLM-RoBERTa SentencePiece model (Unigram, 250k pieces, byte_fallback), which-// is the vocabulary shipped in ragflow_deps - so one asset and one implementation+// is the vocabulary shipped in deps - so one asset and one implementation// cover the whole family. See internal/tokenizer/embedding_token_limits.md.//// What is faithful and what is approximate:@@ -784,7 +784,7 @@ var expectedSPMHashes = map[string]string{// xlmrSPMAssetNames are the file names the loader accepts, in priority order.var xlmrSPMAssetNames = []string{- "ragflow_deps/huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model",+ "deps/huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model","sentencepiece.bpe.model","xlmr_spm.model",}@@ -850,7 +850,7 @@ func readTokenizerAsset(names []string, pinKey string, pins map[string]string) (}return candidate, raw, nil}- return "", nil, fmt.Errorf("no tokenizer asset found; run `uv run ragflow_deps/download_deps.py` or set %s to a directory holding them; tried: %s",+ return "", nil, fmt.Errorf("no tokenizer asset found; run `uv run deps/download_deps.py` or set %s to a directory holding them; tried: %s",common.EnvModelAssetsDir, strings.Join(tried, ", "))}diff --git a/internal/tokenizer/tokenizer_close_race_test.go b/internal/tokenizer/tokenizer_close_race_test.goindex e53445c491..7f10ea4e78 100644--- a/internal/tokenizer/tokenizer_close_race_test.go+++ b/internal/tokenizer/tokenizer_close_race_test.go@@ -42,7 +42,7 @@ func prepareCloseRaceTest(t *testing.T) {dir = "/usr/share/infinity/resource"}if _, err := os.Stat(filepath.Join(dir, "rag", "huqie.trie")); err != nil {- t.Skipf("analyzer dictionary not present at %s (%v); run ragflow_deps/download_deps.py", dir, err)+ t.Skipf("analyzer dictionary not present at %s (%v); run deps/download_deps.py", dir, err)}}diff --git a/internal/tokenizer/wordpiece.go b/internal/tokenizer/wordpiece.goindex d645a1bbbc..e96fe49540 100644--- a/internal/tokenizer/wordpiece.go+++ b/internal/tokenizer/wordpiece.go@@ -72,7 +72,7 @@ var expectedWordPieceHashes = map[string]string{const wordPieceAssetPinKey = "BAAI/bge-large-en-v1.5/vocab.txt"var bertWordPieceAssetNames = []string{- "ragflow_deps/huggingface.co/BAAI/bge-large-en-v1.5/vocab.txt",+ "deps/huggingface.co/BAAI/bge-large-en-v1.5/vocab.txt","vocab.txt","bert_wordpiece_vocab.txt",}diff --git a/scripts/gen_tokenizer_oracle.py b/scripts/gen_tokenizer_oracle.pyindex 9dd793cf66..38e06caef5 100755--- a/scripts/gen_tokenizer_oracle.py+++ b/scripts/gen_tokenizer_oracle.py@@ -180,7 +180,7 @@ ORACLES = {# the served behaviour comes from tokenizer.json above.SPM_MODEL = "huggingface.co/BAAI/bge-m3/sentencepiece.bpe.model"-# The cl100k table as provisioned by download_deps.py or build.sh in ragflow_deps.+# The cl100k table as provisioned by download_deps.py or build.sh in deps.# The Go counter loads this file; tiktoken reads its own cached copy of the same blob, so# the check in main() is what keeps the check-out honest about what it ships.CL100K_TABLE = "cl100k_base.tiktoken"@@ -302,7 +302,7 @@ def main() -> int:parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)parser.add_argument("--model", choices=sorted(ORACLES), required=True)parser.add_argument("--out", required=True, help="fixture path to write")- parser.add_argument("--deps", default="ragflow_deps", help="directory holding huggingface.co/")+ parser.add_argument("--deps", default="deps", help="directory holding huggingface.co/")parser.add_argument("--corpus",required=True,@@ -323,13 +323,13 @@ def main() -> int:rows_src = samples(args.corpus) + (fuzz_samples(args.fuzz, args.seed) if args.fuzz else [])if rel is None:- # cl100k: OpenAI's own implementation on the table shipped in ragflow_deps.+ # cl100k: OpenAI's own implementation on the table shipped in deps.# tiktoken reads its own cached copy of the blob, so a checkout without the# shipped table would be checked against something else without a word -# fail loudly instead, the way the Go loader does.table = os.path.join(args.deps, CL100K_TABLE)if not os.path.isfile(table):- print(f"missing cl100k table: {table}\nrun `uv run ragflow_deps/download_deps.py`", file=sys.stderr)+ print(f"missing cl100k table: {table}\nrun `uv run deps/download_deps.py`", file=sys.stderr)return 1encoder = tiktoken.get_encoding("cl100k_base")rows = [{"label": label, "text": text, "ids": encoder.encode(text), "chars": len(text)} for label, text in rows_src]@@ -337,7 +337,7 @@ def main() -> int:else:tokenizer_path = os.path.join(args.deps, rel)if not os.path.isfile(tokenizer_path):- print(f"missing oracle tokenizer: {tokenizer_path}\nrun `uv run ragflow_deps/download_deps.py`", file=sys.stderr)+ print(f"missing oracle tokenizer: {tokenizer_path}\nrun `uv run deps/download_deps.py`", file=sys.stderr)return 1tokenizer = Tokenizer.from_file(tokenizer_path)vocab = tokenizer.get_vocab()@@ -379,7 +379,7 @@ def main() -> int:# --spm-crosscheck was asked for, so the asset it reads has to be there:# skipping quietly would turn "the second opinion agrees" into a claim# nobody made. Assets are not optional in this script - they are checked.- print(f"missing sentencepiece model: {model_file}\nrun `uv run ragflow_deps/download_deps.py`", file=sys.stderr)+ print(f"missing sentencepiece model: {model_file}\nrun `uv run deps/download_deps.py`", file=sys.stderr)return 1processor = spm.SentencePieceProcessor(model_file=model_file)divergent = 0diff --git a/tools/scripts/download_dev_deps.py b/tools/scripts/download_dev_deps.pyindex 5d73fee453..2781bae7c4 100755--- a/tools/scripts/download_dev_deps.py+++ b/tools/scripts/download_dev_deps.py@@ -19,7 +19,7 @@# and the embedding tokenizer assets). Run it ONCE and the local environment is# ready — you do NOT also need download_deps.py for local development.#-# The image-facing counterpart ragflow_deps/download_deps.py fetches the same+# The image-facing counterpart deps/download_deps.py fetches the same# runtime resources PLUS the stagehand server binary (used by the Browser# component) and skips the native `.a` files, because the build stage bakes# those into the server binary. Both scripts land the shared runtime resources@@ -262,10 +262,10 @@ def _repo_root() -> str:REPO_ROOT = _repo_root()-# Downloaded archives are anchored under ragflow_deps/ (ignored by .gitignore-# via *.tgz / ragflow_deps/*.zip) regardless of the caller's working directory,+# Downloaded archives are anchored under deps/ (ignored by .gitignore+# via *.tgz / deps/*.zip) regardless of the caller's working directory,# so a move of this script does not scatter large files into tools/scripts/.-RAGFLOW_DEPS_DIR = os.path.join(REPO_ROOT, "ragflow_deps")+RAGFLOW_DEPS_DIR = os.path.join(REPO_ROOT, "deps")NATIVE_LIBS_DIR = os.path.expanduser("~/ragflow-native-libs")@@ -498,7 +498,7 @@ def _download_ort_archive(archive_path, goos, goarch):# The build-facing native `.a` libs above let the server LINK; these let it# RUN, parse documents and count tokens. Fetched here too so a developer runs# THIS script once and the environment is ready. Landing paths match-# ragflow_deps/download_deps.py, so re-running that image script never+# deps/download_deps.py, so re-running that image script never# re-downloads an already-present asset. stagehand is deliberately NOT fetched# here (Browser component only) — that is download_deps.py's job for images.@@ -515,7 +515,7 @@ HF_DEEPDOC_MODEL_FILES = ["det.ort", "layout.ort", "tsr.ort", "rec.ort", "ocr.re# Embedding tokenizer assets the Go counters load. Inlined here (no separate# manifest file) as the single source of truth this script reads directly; the-# image script ragflow_deps/download_deps.py carries its own copy (no shared+# image script deps/download_deps.py carries its own copy (no shared# module, by design). One entry per tuple: (repo, file, kind).# runtime - the Go counters load it in production; the runtime images must ship# it and the loader must pin its SHA-1.@@ -541,7 +541,7 @@ def _hf_download(repo_id, filename, local_dir):def download_cl100k():- """Fetch the cl100k BPE table into ragflow_deps/ (skip if already present)."""+ """Fetch the cl100k BPE table into deps/ (skip if already present)."""target = os.path.join(RAGFLOW_DEPS_DIR, "cl100k_base.tiktoken")if os.path.isfile(target) and os.path.getsize(target) > 0:print(" ✓ cl100k_base.tiktoken already present")@@ -576,9 +576,9 @@ def download_go_models(use_china_mirrors=False):backend with no further setup.Written into the repo's canonical model directory- internal/rag/res/deepdoc/ (one level up from ragflow_deps/). The Go server+ internal/rag/res/deepdoc/ (one level up from deps/). The Go serverauto-discovers that directory, and build.sh --test-native defaults MODEL_DIR- there too. A copy also lands under ragflow_deps/huggingface.co/InfiniFlow/deepdoc/+ there too. A copy also lands under deps/huggingface.co/InfiniFlow/deepdoc/to keep the dependency image's existing build-context layout."""if use_china_mirrors:os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"@@ -621,8 +621,8 @@ def download_go_models(use_china_mirrors=False):def download_tokenizer_assets(use_china_mirrors=False):"""Download the embedding tokenizer assets the Go counters load, into- ragflow_deps/huggingface.co/<repo>/<file> (the path internal/tokenizer- searches and the one ragflow_deps/Dockerfile builds its image from).+ deps/huggingface.co/<repo>/<file> (the path internal/tokenizer+ searches and the one deps/Dockerfile builds its image from).A missing asset is fatal: models declaring its tokenizer refuse ingestion."""if use_china_mirrors:@@ -673,7 +673,7 @@ def main():This is the local-dev one-stop: one invocation readies the environment. TheBrowser component's server binary is intentionally NOT fetched here — that- is the image script's job. Run `uv run ragflow_deps/download_deps.py` for the+ is the image script's job. Run `uv run deps/download_deps.py` for thefull image resources (it adds that binary and skips the native .a, which thebuild stage bakes into the binary)."""parser = argparse.ArgumentParser(description="Download the native static libraries and runtime resources the local Go dev server needs")@@ -697,7 +697,7 @@ def main():ort_zip = ort_asset_name(ORT_VERSION, ort_target(goos, goarch))ort_dir = ort_normalized_dir(ORT_VERSION, goos, goarch)- # Anchor the downloaded archives to ragflow_deps/ regardless of the+ # Anchor the downloaded archives to deps/ regardless of the# caller's working directory (build.sh invokes this from the repo root).os.chdir(RAGFLOW_DEPS_DIR)diff --git a/tools/scripts/test_dep_rename_invariants.py b/tools/scripts/test_dep_rename_invariants.pynew file mode 100644index 0000000000..b88fa58d74--- /dev/null+++ b/tools/scripts/test_dep_rename_invariants.py@@ -0,0 +1,89 @@+"""Guard the two intentional exceptions of the ragflow_deps -> deps rename.++The rename PR decided to keep (a) the published image tag `infiniflow/ragflow_deps:latest`+and (b) the CI install path `/opt/ragflow_deps` unchanged. A blanket+`sed s/ragflow_deps/deps/g` accidentally rewrote those too, breaking the CI+offline fallback for the cl100k table and leaving misleading tag references.++This test fails while those exceptions are violated, and stays green once the+intentional names are restored. Run standalone: `python3 tools/scripts/test_dep_rename_invariants.py`.+"""++import os+import sys++REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++SEP_TESTS = os.path.join(REPO_ROOT, ".github", "workflows", "sep-tests.yml")+DOCKERIGNORE = os.path.join(REPO_ROOT, ".dockerignore")+DOCKERFILE = os.path.join(REPO_ROOT, "Dockerfile")+DOCKERFILE_BASE = os.path.join(REPO_ROOT, "Dockerfile_base")++# The bare, wrong forms that the over-broad rename introduced.+BARE_OPT_DEPS = "/opt/deps" # must stay /opt/ragflow_deps (CI install path)+BARE_IMAGE_TAG = "infiniflow/deps" # must stay infiniflow/ragflow_deps (image tag)++# The intentional, retained forms that must remain present.+KEPT_OPT_DEPS = "/opt/ragflow_deps"+KEPT_IMAGE_TAG = "infiniflow/ragflow_deps"+++def _read(path):+ with open(path, "r", encoding="utf-8") as f:+ return f.read()+++def test_sep_tests_keeps_opt_ragflow_deps_path():+ text = _read(SEP_TESTS)+ # The CI install path must NOT have been renamed to /opt/deps.+ assert BARE_OPT_DEPS not in text, f"{SEP_TESTS} still references '{BARE_OPT_DEPS}'; the CI install path must remain '{KEPT_OPT_DEPS}' (intentional rename exception)."+ # The retained path must actually be present as the cl100k source.+ assert KEPT_OPT_DEPS in text, f"{SEP_TESTS} is missing '{KEPT_OPT_DEPS}'; the CI cl100k source was lost."+++def test_sep_tests_keeps_ragflow_deps_image_tag():+ text = _read(SEP_TESTS)+ # Comments/echo strings must reference the real tag, not the bare wrong one.+ assert BARE_IMAGE_TAG not in text, f"{SEP_TESTS} still references bare '{BARE_IMAGE_TAG}'; the published image tag is '{KEPT_IMAGE_TAG}'."+ assert KEPT_IMAGE_TAG in text, f"{SEP_TESTS} is missing '{KEPT_IMAGE_TAG}'; the image tag reference was lost."+++def test_dockerignore_keeps_ragflow_deps_image_tag():+ text = _read(DOCKERIGNORE)+ assert BARE_IMAGE_TAG not in text, f"{DOCKERIGNORE} still references bare '{BARE_IMAGE_TAG}'; the published image tag is '{KEPT_IMAGE_TAG}'."+ assert KEPT_IMAGE_TAG in text, f"{DOCKERIGNORE} is missing '{KEPT_IMAGE_TAG}'; the image tag reference was lost."+++def test_dockerfiles_keep_ragflow_deps_image_tag():+ for path in (DOCKERFILE, DOCKERFILE_BASE):+ text = _read(path)+ # Error-message echo strings must reference the real tag, not the bare wrong one.+ assert BARE_IMAGE_TAG not in text, f"{path} still references bare '{BARE_IMAGE_TAG}'; the published image tag is '{KEPT_IMAGE_TAG}'."+ # The mount source (infiniflow/ragflow_deps:latest) must be present.+ assert KEPT_IMAGE_TAG in text, f"{path} is missing '{KEPT_IMAGE_TAG}'; the image tag reference was lost."+++def main():+ tests = [+ test_sep_tests_keeps_opt_ragflow_deps_path,+ test_sep_tests_keeps_ragflow_deps_image_tag,+ test_dockerignore_keeps_ragflow_deps_image_tag,+ test_dockerfiles_keep_ragflow_deps_image_tag,+ ]+ failed = 0+ for t in tests:+ try:+ t()+ print(f"PASS {t.__name__}")+ except AssertionError as e:+ failed += 1+ print(f"FAIL {t.__name__}: {e}")+ if failed:+ print(f"\n{failed} test(s) failed.")+ return 1+ print("\nAll rename-invariant checks passed.")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tools/scripts/test_download_dev_deps.py b/tools/scripts/test_download_dev_deps.pyindex a928c14708..3cd2a6800f 100644--- a/tools/scripts/test_download_dev_deps.py+++ b/tools/scripts/test_download_dev_deps.py@@ -319,7 +319,7 @@ def test_native_archive_rejects_unsafe_members(tmp_path, kind):def native_dirs(tmp_path, monkeypatch):"""Point the dev script's repo-anchored paths at a temp tree so theruntime-resource downloads written by tests never touch the real repo."""- deps = tmp_path / "ragflow_deps"+ deps = tmp_path / "deps"deps.mkdir()root = tmp_path / "repo"root.mkdir()@@ -372,7 +372,7 @@ def test_download_go_models_downloads_when_missing(native_dirs, monkeypatch):deepdoc_dir = root / "internal" / "rag" / "res" / "deepdoc"for fname in native.HF_DEEPDOC_MODEL_FILES:assert (deepdoc_dir / fname).is_file(), f"missing DeepDoc model {fname}"- # The image-layout copy under ragflow_deps/huggingface.co must also exist.+ # The image-layout copy under deps/huggingface.co must also exist.for fname in native.HF_DEEPDOC_MODEL_FILES:assert (deps / "huggingface.co" / native.HF_DEEPDOC_REPO / fname).is_file()@@ -390,7 +390,7 @@ def test_download_go_models_fails_loud(native_dirs, monkeypatch):def _seed_tokenizer_targets(deps, present):"""Create (present=True) or leave absent the on-disk targets the inlined- TOKENIZER_ASSETS list resolves to under ragflow_deps/huggingface.co/<repo>/<file>."""+ TOKENIZER_ASSETS list resolves to under deps/huggingface.co/<repo>/<file>."""for repo, filename, _kind in native.TOKENIZER_ASSETS:target = deps / "huggingface.co" / repo / filenameif present:@@ -483,7 +483,7 @@ def test_native_script_does_not_fetch_stagehand():def test_image_script_does_not_handle_native_archives():- deps_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "ragflow_deps", "download_deps.py")+ deps_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "deps", "download_deps.py")assert os.path.isfile(deps_path), f"download_deps.py not found at {deps_path}"spec = importlib.util.spec_from_file_location("download_deps_img", deps_path)img = importlib.util.module_from_spec(spec)diff --git a/uv.lock b/uv.lockindex f69c5fa654..ad30d4ba59 100644--- a/uv.lock+++ b/uv.lock@@ -190,7 +190,7 @@ name = "cffi"version = "2.0.0"source = { registry = "https://mirrors.aliyun.com/pypi/simple" }dependencies = [- { name = "pycparser", marker = "implementation_name != 'PyPy'" },+ { name = "pycparser", marker = "(implementation_name != 'PyPy' and platform_machine != 'aarch64' and sys_platform == 'linux') or (implementation_name != 'PyPy' and sys_platform != 'darwin' and sys_platform != 'linux')" },]sdist = { url = "https://mirrors.aliyun.com/pypi/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529" }wheels = [@@ -976,7 +976,7 @@ wheels = [[[package]]name = "ragflow"-version = "1.0.0rc1"+version = "1.0.0rc2"source = { virtual = "." }dependencies = [{ name = "aiohttp" },