Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(deepdoc): harden ORT build guard, OCR test contract (#19082)

xugangqiang0d1338d63e47
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index a32faca5a6..b4f4db16bb 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -396,6 +396,17 @@ jobs:
./build.sh --test -- $PKGS
fi
+ - name: Run ORT deps-download regression guard
+ if: env.API_PROXY_SCHEME == 'go'
+ # Pure-Python, local, needs no API keys or model downloads; guards the
+ # ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build
+ # depends on (notably the version-bump path C2 hardened). Run it in
+ # isolation so the test/testcases conftest (which requires
+ # SILICONFLOW_API_KEY) cannot fail the collection.
+ run: |
+ set -euo pipefail
+ uv sync --python 3.13 --group test --frozen
+ uv run pytest ragflow_deps/test_download_go_deps.py -q
- name: Build ragflow:nightly
run: |
@@ -1016,6 +1027,17 @@ jobs:
./build.sh --test -- $PKGS
fi
+ - name: Run ORT deps-download regression guard
+ if: env.API_PROXY_SCHEME == 'go'
+ # Pure-Python, local, needs no API keys or model downloads; guards the
+ # ONNX Runtime extraction logic the in-process (Go) DeepDoc backend build
+ # depends on (notably the version-bump path C2 hardened). Run it in
+ # isolation so the test/testcases conftest (which requires
+ # SILICONFLOW_API_KEY) cannot fail the collection.
+ run: |
+ set -euo pipefail
+ uv sync --python 3.13 --group test --frozen
+ uv run pytest ragflow_deps/test_download_go_deps.py -q
- name: Build ragflow:nightly
run: |
diff --git a/Dockerfile_go b/Dockerfile_go
index 1c0e89f838..75e2ff3a95 100644
--- a/Dockerfile_go
+++ b/Dockerfile_go
@@ -30,6 +30,16 @@ RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co
ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.manual.onnx && \
ln -s layout.onnx /ragflow/rag/res/deepdoc/layout.paper.onnx
+# Copy the cl100k_base BPE table used by the Go tokenizer (tiktoken-go
+# cl100k_base). The deps image ships it at its root; the Go image previously
+# mounted only /huggingface.co and omitted this file, so NumTokensFromString
+# silently returned 0. localBpeLoader resolves it from <workdir>/ragflow_deps/,
+# so dropping it here makes the table load offline at startup (see
+# InitCL100KEncoder fail-fast guard).
+RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/cl100k_base.tiktoken,target=/tmp/cl100k_base.tiktoken \
+ mkdir -p /ragflow/ragflow_deps && \
+ cp /tmp/cl100k_base.tiktoken /ragflow/ragflow_deps/cl100k_base.tiktoken
+
ENV DEBIAN_FRONTEND=noninteractive
# Setup apt
diff --git a/build.sh b/build.sh
index e88f92e090..888c8eb365 100755
--- a/build.sh
+++ b/build.sh
@@ -41,7 +41,8 @@ PDF_OXIDE_VERSION="0.3.73"
# (Go) DeepDoc backend. libonnxruntime*.a is linked into the server binary
# (--whole-archive + --export-dynamic); OrtGetApiBase is then resolved via
# dlopen(self), so no libonnxruntime.so is needed at runtime. Downloaded by
-# ragflow_deps/download_deps.py into onnxruntime/static_lib.
+# ragflow_deps/download_go_deps.py (and ragflow_deps/download_deps.py) into
+# onnxruntime/static_lib.
ONNXRUNTIME_STATIC_PREFIX="${HOME}/ragflow-native-libs/onnxruntime/static_lib"
# Copy a dependency from the system pre-seed directory to the user cache.
@@ -370,6 +371,33 @@ build_go() {
setup_cgo_env
+ # The in-process (Go) DeepDoc backend is statically linked against ONNX
+ # Runtime (--whole-archive + -Wl,--export-dynamic, see setup_cgo_env). The
+ # forked onnxruntime_go binding resolves OrtGetApiBase only at RUNTIME via
+ # dlopen(NULL), so a missing ORT archive still lets `go build` SUCCEED and
+ # yields a server binary that FAILS AT STARTUP with a fatal
+ # "no in-process DeepDoc backend serving". That is exactly the breakage
+ # colleagues hit when ORT was not present at build time and setup_cgo_env
+ # silently skipped it. Fail the build HERE instead of deferring the breakage
+ # to runtime: a server binary without ORT is unusable, not a degraded one.
+ if ! printf '%s' "$CGO_LDFLAGS" | grep -q 'libonnxruntime'; then
+ echo -e "${RED}Error: ONNX Runtime static libraries are not linked.${NC}" >&2
+ echo " The in-process DeepDoc backend requires libonnxruntime.a to be" >&2
+ echo " statically linked into ragflow_server (--whole-archive +" >&2
+ echo " -Wl,--export-dynamic). Without it the binary compiles but dies" >&2
+ echo " at startup with a fatal 'no in-process DeepDoc backend serving'." >&2
+ echo " Fetch the static libs with:" >&2
+ echo " uv run python3 ragflow_deps/download_go_deps.py" >&2
+ echo " or pre-seed them at /opt/ragflow-native-libs/onnxruntime (CI image)." >&2
+ echo " This production binary must statically include ORT — there is no" >&2
+ echo " ORT-free build path. ORT ends up unlinked only via one of:" >&2
+ echo " - the ORT static_lib dir was never seeded: run" >&2
+ echo " 'uv run python3 ragflow_deps/download_go_deps.py', or pre-seed" >&2
+ echo " /opt/ragflow-native-libs/onnxruntime as the CI image does;" >&2
+ echo " - setup_cgo_env did not add libonnxruntime to CGO_LDFLAGS." >&2
+ return 1
+ fi
+
local strip_flags=()
[ -n "$STRIP_SYMBOLS" ] && strip_flags=(-ldflags="-s -w")
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 01e316c5e1..1cd60c8074 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -568,6 +568,12 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg
}
defer tokenizer.Close()
+ // Fail fast if the cl100k_base BPE table is missing: without it
+ // NumTokensFromString silently returns 0, corrupting every token budget.
+ if err := tokenizer.InitCL100KEncoder(); err != nil {
+ common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err))
+ }
+
// The dataset-level post-processing consumer cluster (§11) is owned and run by
// the Ingestor: it is started inside ingestor.Start() and joined inside
// ingestor.Stop(), so its lifecycle matches the ingestor. The configured
@@ -712,6 +718,12 @@ func runAPI(ctx context.Context, args *serverArgs) error {
}
defer tokenizer.Close()
+ // Fail fast if the cl100k_base BPE table is missing: without it
+ // NumTokensFromString silently returns 0, corrupting every token budget.
+ if err := tokenizer.InitCL100KEncoder(); err != nil {
+ common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err))
+ }
+
// Initialize global QueryBuilder using tokenizer's DictPath
// This ensures the Synonym uses the same wordnet directory as tokenizer
if err := nlp.InitQueryBuilderFromTokenizer(tokenizerCfg.DictPath); err != nil {
diff --git a/internal/common/environments.go b/internal/common/environments.go
index 64111abd11..f461377289 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -254,17 +254,23 @@ func HasModelFiles(dir string) bool {
}
// DeepDocORTVersion is the onnxruntime native release the in-process (Go)
-// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is the
-// single source for the download URL and extracted dir name used across Go
-// and Python. The Go binding (github.com/yalue/onnxruntime_go, forked to
+// DeepDoc backend is built and tested against (e.g. "1.23.2"). It is ONE OF
+// THREE raw version declarations that must stay equal (the other two are
+// ORT_VERSION in ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py)
+// — NOT a single source of truth. The download URL and extracted dir name are
+// built from those ORT_VERSION constants, not from this one. The Go binding
+// (github.com/yalue/onnxruntime_go, forked to
// github.com/xugangqiang/onnxruntime_go) and the pip onnxruntime== pin must
// track this MINOR version: the binding uses its own release numbering
// (v1.23.0 <-> ORT 1.23.x) but is ABI-compatible with this native release on
// the same minor line. ONNX Runtime is linked statically (libonnxruntime.a),
// so there is no .so / SONAME at runtime.
//
-// To bump ORT: update DeepDocORTVersion (Go) AND ORT_VERSION in
-// ragflow_deps/download_deps.py AND the onnxruntime/onnxruntime-gpu pins in
-// pyproject.toml + .github/workflows/deepdoc-drift.yml, and refresh the
-// onnxruntime_go binding minor in go.mod.
+// To bump ORT, ALL of the following must change together (drift breaks the
+// static link or the runtime OrtGetApiBase lookup):
+// - DeepDocORTVersion (here, Go) AND ORT_VERSION in BOTH
+// ragflow_deps/download_go_deps.py and ragflow_deps/download_deps.py;
+// - the onnxruntime== pin in pyproject.toml and the onnxruntime /
+// onnxruntime-gpu pins in .github/workflows/deepdoc-drift.yml;
+// - the onnxruntime_go binding minor in go.mod.
const DeepDocORTVersion = "1.23.2"
diff --git a/internal/deepdoc/native/ocr_rec.go b/internal/deepdoc/native/ocr_rec.go
index a21f51abdb..e1e0b16c58 100644
--- a/internal/deepdoc/native/ocr_rec.go
+++ b/internal/deepdoc/native/ocr_rec.go
@@ -36,7 +36,7 @@ type OCRRecResult struct {
}
// RunOCRRec recognizes a single cropped text-line image. It is equivalent to a
-// one-line batch (see RunOCRRecBatch): the line is resized against its own
+// one-line batch (see RunOCRRecBatchReal): the line is resized against its own
// wh_ratio floored at 320/48, never the batch-max, so a caller recognizing
// lines independently gets the same result as a standalone Python
// TextRecognizer call.
@@ -55,28 +55,16 @@ func RunOCRRec(ctx context.Context, modelDir string, img *Image) (OCRRecResult,
return recognizeLine(ctx, modelDir, img, maxWhRatio, chars)
}
-// RunOCRRecBatch recognizes a batch of cropped text-line images the way
-// deepdoc's TextRecognizer.__call__ does: every line in the batch is resized to
-// the SAME width derived from the batch's maximum wh_ratio (floored at 320/48),
-// so a narrow line inside a wide batch is widened to the batch max rather than
-// its own ratio. This makes Go match production batch semantics, where a page's
-// lines are recognized together. Lines are recognized independently after the
-// shared resize, so order is preserved and results are deterministic per batch.
-//
-// DEPRECATED semantic-shape note: this variant still runs one ONNX Run PER
-// line (recMaxBatch=1). Prefer RunOCRRecBatchReal, which concatenates every
-// line's preprocessed blob into a single {N,3,48,imgW} tensor and performs ONE
-// ONNX Run — identical numerically (each line uses the same shared batch width)
-// but with the real throughput benefit of batched matmul. RunOCRRecBatchReal is
-// the production path.
// RunOCRRecBatchReal recognizes a batch of cropped text-line images with a
-// SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__ exactly: every
-// line is resized to the batch's shared width (imgW = 48*max_wh_ratio, floored
-// at 320/48) and zero-padded on the right, all blobs are concatenated into one
-// {N,3,48,imgW} tensor, and the model runs once. The output is split back into
-// per-line sequences and CTC-decoded in order, so the result is numerically
-// identical to calling RunOCRRec on each line (each line sees the same shared
-// batch width), but amortized over one forward pass instead of N.
+// SINGLE ONNX Run, mirroring deepdoc's TextRecognizer.__call__: each line is
+// resized to its own proportional width (recH * that line's wh_ratio), capped
+// by the batch-shared imgW (imgW = recH * max_wh_ratio, with max_wh_ratio
+// floored at 320/48), and zero-padded on the right out to imgW; all blobs are
+// concatenated into one {N,3,48,imgW} tensor, and the model runs once. The
+// output is split back into per-line sequences and CTC-decoded in order, so
+// the result is numerically identical to calling RunOCRRec on each line (each
+// line sees the same shared batch width), but amortized over one forward pass
+// instead of N.
//
// The shared batch width means a line is resized against the batch max wh_ratio,
// not its own — exactly what deepdoc does inside a batch. A standalone call to
diff --git a/internal/deepdoc/native/testdata_fetch.go b/internal/deepdoc/native/testdata_fetch.go
index b3d0a922ec..37fd37c1f6 100644
--- a/internal/deepdoc/native/testdata_fetch.go
+++ b/internal/deepdoc/native/testdata_fetch.go
@@ -2,8 +2,9 @@
// This file is compiled only with -tags fetch_testdata. Its init() downloads
// and symlinks the external DeepDoc testdata for this package (pkg = "native")
-// so the consuming tests can run. The download is best-effort: if it fails, the
-// guard in testdata_skip.go skips the tests instead of failing.
+// so the consuming tests can run. A fetch failure is fatal under this build tag:
+// it sets testdataFetchFailed so TestMain fails the package instead of silently
+// skipping — a missing fixture must never be a green CI run.
package native
import (
@@ -17,6 +18,7 @@ import (
func init() {
if err := fetchTestdata(); err != nil {
fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)
+ testdataFetchFailed = true
return
}
testdataFetchAttempted = true
diff --git a/internal/deepdoc/native/testdata_flag.go b/internal/deepdoc/native/testdata_flag.go
index 5f329abe5d..c01461a8ed 100644
--- a/internal/deepdoc/native/testdata_flag.go
+++ b/internal/deepdoc/native/testdata_flag.go
@@ -8,3 +8,11 @@ package native
// imports native, and _test.go files are excluded from dependency builds). In the
// default build it stays false, so TestMain skips the package.
var testdataFetchAttempted bool
+
+// testdataFetchFailed is set to true by testdata_fetch.go (compiled only with
+// -tags fetch_testdata) when the external testdata download fails. It is
+// declared here (always-compiled) so TestMain can read it. Under the
+// fetch_testdata build tag a failed fetch must FAIL the package rather than
+// silently skip — a missing fixture must never produce a green CI run. In the
+// default build this stays false, so the absence of fixtures is a benign skip.
+var testdataFetchFailed bool
diff --git a/internal/deepdoc/native/testdata_skip_test.go b/internal/deepdoc/native/testdata_skip_test.go
index 278455ef49..7126ec9c2f 100644
--- a/internal/deepdoc/native/testdata_skip_test.go
+++ b/internal/deepdoc/native/testdata_skip_test.go
@@ -1,9 +1,10 @@
// This file is always compiled (no build constraint) and provides the package
// TestMain that guards the whole test binary behind the presence of the
// package-level testdata directory. When the external testdata has not been
-// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed),
-// the tests are skipped rather than failed — keeping the default `go test ./...`
-// and `go test -tags cgo` runs green without the assets.
+// fetched (e.g. -tags fetch_testdata was not supplied), the tests are skipped
+// rather than failed — keeping the default `go test ./...` and `go test -tags cgo`
+// runs green without the assets. A fetch ATTEMPTED but FAILED (under
+// -tags fetch_testdata) is fatal: a missing fixture must never be a green run.
package native
import (
@@ -18,6 +19,10 @@ import (
// assets — which also makes a stale/partial leftover testdata directory on disk
// harmless: we skip regardless of what (if anything) is present.
func TestMain(m *testing.M) {
+ if testdataFetchFailed {
+ fmt.Fprintln(os.Stderr, "FAIL: DeepDoc native testdata fetch failed (see the fetch_deepdoc_testdata error above). CI must not pass with missing fixtures; run with -tags fetch_testdata and fix the download.")
+ os.Exit(1)
+ }
if !testdataFetchAttempted {
fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it")
os.Exit(0)
diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go
index cbd04e1345..0c2d9a8902 100644
--- a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go
+++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go
@@ -3,9 +3,10 @@
// This file is compiled only with -tags fetch_testdata. Its init() downloads
// and symlinks the external DeepDoc testdata. The native_analyzer tests read
// their fixtures from the sibling internal/deepdoc/native/testdata directory
-// (via a relative path), so we fetch with pkg = "native". The download is
-// best-effort: if it fails, the guard in testdata_skip.go skips the tests
-// instead of failing.
+// (via a relative path), so we fetch with pkg = "native". A fetch failure is
+// fatal under this build tag: it sets testdataFetchFailed so TestMain fails the
+// package instead of silently skipping — a missing fixture must never be a green
+// CI run.
package infnative
import (
@@ -19,6 +20,7 @@ import (
func init() {
if err := fetchTestdata(); err != nil {
fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)
+ testdataFetchFailed = true
return
}
testdataFetchAttempted = true
diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go
index 24a85df01b..4ba72471c7 100644
--- a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go
+++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_flag.go
@@ -7,3 +7,11 @@ package infnative
// this package is built as a dependency of another package's test binary. In the
// default build it stays false, so TestMain skips the package.
var testdataFetchAttempted bool
+
+// testdataFetchFailed is set to true by testdata_fetch.go (compiled only with
+// -tags fetch_testdata) when the external testdata download fails. Declared here
+// (always-compiled) so TestMain can read it. Under the fetch_testdata build tag a
+// failed fetch must FAIL the package rather than silently skip — a missing
+// fixture must never produce a green CI run. In the default build this stays
+// false, so the absence of fixtures is a benign skip.
+var testdataFetchFailed bool
diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go
index 7d37f03ac1..7ec26e5dcd 100644
--- a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go
+++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_skip_test.go
@@ -3,9 +3,10 @@
// DeepDoc testdata directory. The native_analyzer tests read their fixtures
// from the sibling internal/deepdoc/native/testdata directory (via a relative
// path), so that is what we check here. When the external testdata has not been
-// fetched (e.g. -tags fetch_testdata was not supplied, or the download failed),
-// the tests are skipped rather than failed — keeping the default `go test ./...`
-// and `go test -tags cgo` runs green without the assets.
+// fetched (e.g. -tags fetch_testdata was not supplied), the tests are skipped
+// rather than failed — keeping the default `go test ./...` and `go test -tags cgo`
+// runs green without the assets. A fetch ATTEMPTED but FAILED (under
+// -tags fetch_testdata) is fatal: a missing fixture must never be a green run.
package infnative
import (
@@ -20,6 +21,10 @@ import (
// assets — which also makes a stale/partial leftover testdata directory on disk
// harmless: we skip regardless of what (if anything) is present.
func TestMain(m *testing.M) {
+ if testdataFetchFailed {
+ fmt.Fprintln(os.Stderr, "FAIL: DeepDoc native testdata fetch failed (see the fetch_deepdoc_testdata error above). CI must not pass with missing fixtures; run with -tags fetch_testdata and fix the download.")
+ os.Exit(1)
+ }
if !testdataFetchAttempted {
fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it")
os.Exit(0)
diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.go
index 79ab8c4ead..7a137a85e3 100644
--- a/internal/deepdoc/parser/pdf/parser_ocr.go
+++ b/internal/deepdoc/parser/pdf/parser_ocr.go
@@ -105,24 +105,37 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,
allTexts := make([][]pdf.OCRText, len(cropAcc))
if p.docSupportsBatchOCR(doc) {
batch, berr := p.inferOCRRecognizeBatch(ctx, doc, cropAcc)
- if berr != nil {
- slog.Warn(logLabel+" OCR batch recognize failed", "page", pageNum, "err", berr)
- return nil
+ switch {
+ case berr != nil:
+ // A batch error must not abort the whole page: the canonical
+ // per-crop path below still produces correct results.
+ slog.Warn(logLabel+" OCR batch recognize failed; falling back to per-crop", "page", pageNum, "err", berr)
+ case len(batch) != len(cropAcc):
+ // Defensive: a count mismatch (or a nil result) would corrupt the
+ // per-box indexing further down. Fall back to per-crop instead of
+ // indexing out of range.
+ slog.Warn(logLabel+" OCR batch recognize returned unexpected count; falling back to per-crop", "page", pageNum, "got", len(batch), "want", len(cropAcc))
+ default:
+ allTexts = batch
}
- allTexts = batch
- } else {
- for ci, c := range cropAcc {
- // Stamp the detect-box index so a replay DocAnalyzer routes this
- // recognition back to the Python-dumped text for the same box. The
- // production analyzer ignores the key.
- recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, cropBoxIdx[ci])
- texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c)
- if rerr != nil {
- slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", rerr)
- return nil
- }
- allTexts[ci] = texts
+ }
+ // Fill any crop the batch path did not cover (unsupported doc, batch error,
+ // or count mismatch) with the per-crop canonical path. Stamping the
+ // detect-box index lets a replay DocAnalyzer route the recognition back to
+ // the Python-dumped text for the same box; the production analyzer ignores
+ // the key.
+ for ci := range cropAcc {
+ if allTexts[ci] != nil {
+ continue
+ }
+ c := cropAcc[ci]
+ recCtx := context.WithValue(ctx, ocrBoxIdxCtxKey, cropBoxIdx[ci])
+ texts, rerr := p.ocrRecognizeWithRotation(recCtx, doc, c)
+ if rerr != nil {
+ slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", rerr)
+ return nil
}
+ allTexts[ci] = texts
}
// Per box: pick the highest-confidence candidate (layer-2 winner) and
diff --git a/internal/development.md b/internal/development.md
index bfd506fa40..a0226c2146 100644
--- a/internal/development.md
+++ b/internal/development.md
@@ -39,14 +39,33 @@ go version
### 1.4 Install dependent library
```shell
sudo apt install libpcre2-dev
+# Native libs for the Go build (office_oxide, pdfium, pdf_oxide, onnxruntime).
+# download_go_deps.py is the Go-end script: it now fetches the ONNX Runtime
+# static lib too, so this single command covers every native dependency the
+# Go server binary needs (incl. the in-process DeepDoc backend).
python3 ragflow_deps/download_go_deps.py
+# Shared (Go + Python) deps. Retains ONNX Runtime for the ragflow_deps image /
+# backward-compat; also pulls Python-side artifacts (models, nltk, tika, ...).
+uv run python3 ragflow_deps/download_deps.py
```
> **Note**: If you use IDEs like GoLand to run/debug directly (via Run/Debug buttons), or run `go build` / `go run` from command line, set these CGO environment variables:
>
> ```bash
-> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by uv run ragflow_deps/download_deps.py
+> RAGFLOW_DEPS="${HOME}/ragflow-native-libs" # created by download_go_deps.py + download_deps.py
> PLATFORM="linux_amd64" # or darwin_amd64, linux_arm64, darwin_arm64
+> # NOTE: the ONNX Runtime static lib fetched by download_go_deps.py is
+> # linux-x64 ONLY (onnxruntime-linux-x64-static_lib-*). On darwin_* / non-amd64
+> # PLATFORM values the production DeepDoc backend cannot be linked, so those
+> # PLATFORM examples cover office_oxide/pdfium/pdf_oxide only — ORT is a
+> # Linux-amd64 link dependency here.
+>
+> # Resolve the version-stamped ORT archive path FIRST, in its own unquoted
+> # assignment: the shell does not expand `*` inside the double-quoted
+> # CGO_LDFLAGS below, and `--whole-archive` must stay a separate argument from
+> # the archive path. If this lists more than one match, delete the stale
+> # version dir — build.sh refuses to link two ORT versions.
+> ORT_A="$(ls ${RAGFLOW_DEPS}/onnxruntime/static_lib/*/lib/libonnxruntime.a)"
>
> export CGO_CFLAGS="-I${RAGFLOW_DEPS}/office_oxide/include/office_oxide_c"
> export CGO_LDFLAGS="\
@@ -55,11 +74,29 @@ python3 ragflow_deps/download_go_deps.py
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++.a \
> ${RAGFLOW_DEPS}/pdfium-static/lib/libc++abi.a \
> ${RAGFLOW_DEPS}/pdf_oxide/lib/${PLATFORM}/libpdf_oxide.a \
+> -Wl,--export-dynamic -Wl,--whole-archive ${ORT_A} -Wl,--no-whole-archive -lstdc++ \
> -fuse-ld=lld \
> -lm -lpthread -ldl -lrt -lgcc_s -lutil -lc"
> ```
>
-> All three native libraries are statically linked — no `LD_LIBRARY_PATH` or `-Wl,-rpath` needed.
+> All four native libraries are statically linked — no `LD_LIBRARY_PATH` or `-Wl,-rpath` needed.
+>
+> **ONNX Runtime is mandatory for the production binary.** The in-process (Go)
+> DeepDoc backend is statically linked against `libonnxruntime.a` via
+> `--whole-archive -Wl,--export-dynamic`, and `OrtGetApiBase` is resolved at
+> runtime through `dlopen(NULL)`. The forked `onnxruntime_go` binding only
+> needs `-ldl` to *compile*, so a binary built **without** ORT links
+> successfully but dies at startup with:
+> `Error looking up OrtGetApiBase in statically-linked ONNX Runtime` → fatal
+> `no in-process DeepDoc backend serving`.
+> Since `build.sh` (`build_go`) now **fails fast** when ORT is absent from
+> `CGO_LDFLAGS`, this breakage surfaces at build time instead of at runtime. If
+> you see `Error: ONNX Runtime static libraries are not linked`, run
+> `uv run python3 ragflow_deps/download_go_deps.py` (or pre-seed
+> `/opt/ragflow-native-libs/onnxruntime` as the CI runner image does). There is
+> no ORT-free production build path — if ORT is absent the binary fails at
+> startup, so the remedy is always to seed the static lib above, never to build
+> without it.
### 1.5 Build RAGFlow
@@ -77,6 +114,35 @@ python3 ragflow_deps/download_go_deps.py
./build.sh -s --go
```
+### 1.6 In-process (Go) DeepDoc backend
+
+The in-process DeepDoc backend is statically linked against ONNX Runtime
+(see §1.4). After a successful `./build.sh -s --go`, the `ragflow_server`
+binary carries it and registers the backend at startup.
+
+- **Confirm it is serving** — the server logs, at startup:
+ `in-process DeepDoc backend registered (production backend)`
+ If you instead see a fatal `no in-process DeepDoc backend serving`, ORT was
+ not linked into the binary. Re-run `uv run python3 ragflow_deps/download_go_deps.py`
+ and rebuild (§1.4 explains why; `build.sh` fails fast with
+ `Error: ONNX Runtime static libraries are not linked` before this happens).
+
+- **Run the binary directly (local dev)** — `./bin/ragflow_server --api`
+ (start `--admin` first, see §2) launches the Go server and registers the
+ backend. No extra environment variable is required.
+
+- **Run the Go Docker image** — the container entrypoint only starts the Go
+ server (`bin/ragflow_server --api/--ingestor/--admin`) when
+ `API_PROXY_SCHEME` is `go` or `hybrid`. With the variable unset (or `python`)
+ the Go server does NOT start, so the in-process backend is absent and the
+ image looks like a Python-only build. Start it with:
+ ```bash
+ docker run -e API_PROXY_SCHEME=go infiniflow/ragflow:go-test-1
+ ```
+
+- ORT is resolved via `dlopen(NULL)` at runtime, so no `LD_LIBRARY_PATH` /
+ `-rpath` is needed and no `libonnxruntime.so` ships with the image.
+
## 2. Start RAGFlow
- Start dependencies
diff --git a/internal/tokenizer/bpe_loader.go b/internal/tokenizer/bpe_loader.go
index 396a4e5732..80ad8eeb85 100644
--- a/internal/tokenizer/bpe_loader.go
+++ b/internal/tokenizer/bpe_loader.go
@@ -138,7 +138,7 @@ func bpeCandidatePaths(bpeURL string) []string {
}
}
- for _, root := range searchRoots() {
+ for _, root := range bpeSearchRoots() {
// Same layout the Dockerfile creates: the table sits in the
// installation root under its sha1 name.
add(filepath.Join(root, cacheName))
@@ -172,6 +172,25 @@ func searchRoots() []string {
return roots
}
+// bpeSearchRoots is the directory set the loader walks for the table. It
+// defaults to searchRoots (working + executable dirs and their ancestors) so
+// the production image finds the table dropped into the install root. Tests
+// override it with SetBpeSearchRootsForTest to isolate from the host
+// filesystem — without that, a tiktoken cache leaked into /tmp (an ancestor of
+// any temp working dir) would be picked up and make "missing table" tests
+// non-deterministic.
+var bpeSearchRoots = searchRoots
+
+// SetBpeSearchRootsForTest replaces the directory set the loader searches.
+// Pass nil to restore the default. Test-only; it mutates package state.
+func SetBpeSearchRootsForTest(roots []string) {
+ if roots == nil {
+ bpeSearchRoots = searchRoots
+ return
+ }
+ bpeSearchRoots = func() []string { return roots }
+}
+
func startingDirs() []string {
var dirs []string
if wd, err := os.Getwd(); err == nil {
diff --git a/internal/tokenizer/bpe_loader_test.go b/internal/tokenizer/bpe_loader_test.go
index 324bb38aae..cb00102155 100644
--- a/internal/tokenizer/bpe_loader_test.go
+++ b/internal/tokenizer/bpe_loader_test.go
@@ -41,8 +41,12 @@ func writeBpeTable(t *testing.T, path string, marker int) {
}
// Register this synthetic table's digest so the SHA-1 integrity gate in
// bpe_loader.go accepts it. This also exercises the verification path
- // instead of disabling it.
+ // instead of disabling it. Restore the prior value afterwards so the
+ // mutation does not leak into sibling tests (which would otherwise read a
+ // stale digest for the same URL).
+ prev := expectedBpeHashes[testBpeURL]
expectedBpeHashes[testBpeURL] = fmt.Sprintf("%x", sha1.Sum([]byte(line)))
+ t.Cleanup(func() { expectedBpeHashes[testBpeURL] = prev })
}
func cacheFileName(url string) string {
@@ -52,13 +56,17 @@ func cacheFileName(url string) string {
// isolate moves the process into an empty directory and clears both cache
// environment variables, so a test sees only the files it creates itself.
// Without this the real repository checkout — which does ship the table — would
-// satisfy every lookup and hide ordering bugs.
+// satisfy every lookup and hide ordering bugs. It also scopes the loader's
+// search roots to this directory so a tiktoken cache leaked into an ancestor
+// (e.g. /tmp) can't be picked up and make "missing table" tests flaky.
func isolate(t *testing.T) string {
t.Helper()
dir := t.TempDir()
t.Setenv("TIKTOKEN_CACHE_DIR", "")
t.Setenv("DATA_GYM_CACHE_DIR", "")
t.Chdir(dir)
+ SetBpeSearchRootsForTest([]string{dir})
+ t.Cleanup(func() { SetBpeSearchRootsForTest(nil) })
return dir
}
diff --git a/internal/tokenizer/failfast_test.go b/internal/tokenizer/failfast_test.go
new file mode 100644
index 0000000000..4567c66487
--- /dev/null
+++ b/internal/tokenizer/failfast_test.go
@@ -0,0 +1,147 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package tokenizer
+
+import (
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// TestInitCL100KEncoder_FailFast pins the contract that InitCL100KEncoder fails
+// fast on a missing cl100k_base table, and — the part the regression actually
+// guards — that a PRESENT table yields a working encoder (NumTokensFromString > 0)
+// rather than a silent 0.
+//
+// Background: NumTokensFromString swallows the loader error and returns 0, so a
+// Go image that forgot to bake cl100k_base.tiktoken silently zeroed every token
+// count while content_ltks (a separate C++ tokenizer) kept working. The fix is
+// to fail fast at startup via InitCL100KEncoder; this test must exercise BOTH
+// branches, otherwise it only re-asserts "missing → error" and leaves the
+// "present → counts" behaviour — the thing that regressed — uncovered.
+//
+// Both branches reset the encoder cache and scope bpeSearchRoots to a temp dir,
+// so neither depends on the host (no leaked tiktoken cache in an ancestor, no
+// success cached by a sibling test, no order dependence). "absent" runs first so
+// that the table "present" later copies under /tmp is not discoverable by it via
+// searchRoots() walking up to the shared /tmp ancestor.
+func TestInitCL100KEncoder_FailFast(t *testing.T) {
+ // fail-fast: an empty scoped dir must surface a hard error, not a silent 0.
+ t.Run("absent", func(t *testing.T) {
+ resetCL100KEncoderForTest()
+ dir := t.TempDir()
+ t.Setenv("TIKTOKEN_CACHE_DIR", "")
+ t.Setenv("DATA_GYM_CACHE_DIR", "")
+ SetBpeSearchRootsForTest([]string{dir})
+ t.Cleanup(func() { SetBpeSearchRootsForTest(nil) })
+
+ err := InitCL100KEncoder()
+ if err == nil {
+ t.Fatalf("InitCL100KEncoder returned nil with no table present; expected a fail-fast error")
+ }
+ if !strings.Contains(err.Error(), "cl100k") {
+ t.Fatalf("InitCL100KEncoder error does not mention cl100k: %v", err)
+ }
+ })
+
+ // present: copy a real table into an isolated dir (via TIKTOKEN_CACHE_DIR so
+ // the loader finds it without walking ancestors) and require that startup
+ // succeeds AND the encoder actually counts tokens (not just "loads").
+ t.Run("present", func(t *testing.T) {
+ src := findRealBpeTable(t)
+ if src == "" {
+ t.Skip("no cl100k_base.tiktoken on disk to exercise the present branch")
+ }
+ resetCL100KEncoderForTest()
+ dir := t.TempDir()
+ t.Setenv("TIKTOKEN_CACHE_DIR", dir)
+ t.Setenv("DATA_GYM_CACHE_DIR", "")
+ dst := filepath.Join(dir, cacheFileName(testBpeURL))
+ copyFile(t, src, dst)
+ SetBpeSearchRootsForTest([]string{dir})
+ t.Cleanup(func() { SetBpeSearchRootsForTest(nil) })
+
+ if err := InitCL100KEncoder(); err != nil {
+ t.Fatalf("InitCL100KEncoder with table present: unexpected error: %v", err)
+ }
+ if got := NumTokensFromString("hello world"); got <= 0 {
+ t.Fatalf("InitCL100KEncoder succeeded but NumTokensFromString(%q) = %d, want > 0 (silent-zero regression)", "hello world", got)
+ }
+ })
+}
+
+// findRealBpeTable returns a path to a cl100k_base.tiktoken on disk, or "" if
+// none is available. The loader's integrity gate already knows this table's
+// digest (expectedBpeHashes), so copying it into a scoped dir is enough.
+func findRealBpeTable(t *testing.T) string {
+ t.Helper()
+ // 1. Explicit cache dirs (tiktoken-go / data-gym layouts).
+ for _, env := range []string{"TIKTOKEN_CACHE_DIR", "DATA_GYM_CACHE_DIR"} {
+ if dir := strings.TrimSpace(os.Getenv(env)); dir != "" {
+ for _, name := range []string{cacheFileName(testBpeURL), "cl100k_base.tiktoken"} {
+ if p := filepath.Join(dir, name); fileExists(p) {
+ return p
+ }
+ }
+ }
+ }
+ // 2. repo-local ragflow_deps/cl100k_base.tiktoken (provisioned by download
+ // scripts / CI), walking up from the package dir.
+ pkgDir, err := os.Getwd()
+ if err != nil {
+ return ""
+ }
+ for dir := pkgDir; ; {
+ p := filepath.Join(dir, "ragflow_deps", "cl100k_base.tiktoken")
+ if fileExists(p) {
+ return p
+ }
+ parent := filepath.Dir(dir)
+ if parent == dir {
+ break
+ }
+ dir = parent
+ }
+ return ""
+}
+
+func fileExists(p string) bool {
+ info, err := os.Stat(p)
+ return err == nil && !info.IsDir()
+}
+
+func copyFile(t *testing.T, src, dst string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+ t.Fatalf("mkdir %s: %v", filepath.Dir(dst), err)
+ }
+ in, err := os.Open(src)
+ if err != nil {
+ t.Fatalf("open %s: %v", src, err)
+ }
+ defer in.Close()
+ out, err := os.Create(dst)
+ if err != nil {
+ t.Fatalf("create %s: %v", dst, err)
+ }
+ defer out.Close()
+ if _, err := io.Copy(out, in); err != nil {
+ t.Fatalf("copy %s -> %s: %v", src, dst, err)
+ }
+}
diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go
index fcf9acdd66..6e6fc51b52 100644
--- a/internal/tokenizer/tokenizer.go
+++ b/internal/tokenizer/tokenizer.go
@@ -560,9 +560,49 @@ func getCL100KEncoder() (*tiktoken.Tiktoken, error) {
return cl100kEncoder.enc, cl100kEncoder.err
}
+// resetCL100KEncoderForTest clears the cl100k encoder cache so a test can force a
+// fresh load under a different search-root scope. Test-only; it mutates package
+// state. A plain assignment replaces the embedded sync.Once with an un-fired one
+// and wipes any cached result, so the next getCL100KEncoder / InitCL100KEncoder
+// re-runs the loader rather than returning a result cached by a sibling test.
+func resetCL100KEncoderForTest() {
+ cl100kEncoder = struct {
+ sync.Once
+ enc *tiktoken.Tiktoken
+ err error
+ }{}
+}
+
+// InitCL100KEncoder loads the cl100k_base BPE encoder once and returns an
+// error if the table is unavailable. Call it during server startup (before any
+// request reaches NumTokensFromString) so a missing table fails fast instead
+// of silently zeroing every token count.
+//
+// Why fail-fast and not a retry/warning: NumTokensFromString intentionally
+// returns 0 on encoder error to stay cheap on the hot path, which means a
+// missing cl100k_base.tiktoken (the Go image used to omit it) degraded every
+// token budget to 0 while content_ltks — a separate offline C++ tokenizer —
+// kept working, and the divergence went unnoticed. Catching it at startup turns
+// that silent data corruption into a hard, loud failure.
+func InitCL100KEncoder() error {
+ enc, err := getCL100KEncoder()
+ if err != nil {
+ return fmt.Errorf("cl100k_base BPE table unavailable (NumTokensFromString would silently return 0): %w", err)
+ }
+ if enc == nil {
+ return fmt.Errorf("cl100k_base encoder unavailable: GetEncoding returned a nil encoder without error")
+ }
+ return nil
+}
+
// NumTokensFromString returns the number of tokens in s using the cl100k_base
// BPE encoding. Mirrors Python's num_tokens_from_string (common/token_utils.py):
// returns 0 on encoder error.
+//
+// Callers must ensure InitCL100KEncoder has succeeded at startup; otherwise a
+// missing table makes this return 0 for every non-empty string. The startup
+// check is the fail-fast guard — this cheap hot-path function stays silent by
+// design so it can be called per-token without error plumbing.
func NumTokensFromString(s string) int {
if s == "" {
return 0
diff --git a/ragflow_deps/download_go_deps.py b/ragflow_deps/download_go_deps.py
index b6ef0267b2..3819f6bc8c 100644
--- a/ragflow_deps/download_go_deps.py
+++ b/ragflow_deps/download_go_deps.py
@@ -33,10 +33,69 @@
import argparse
import os
+import shutil
import sys
+import zipfile
import requests
+# Mirrors internal/common.DeepDocORTVersion (Go in-process backend). ONE OF
+# THREE places (with that Go constant and the ORT_VERSION in
+# ragflow_deps/download_deps.py) that must carry the same ONNX Runtime native
+# release for the statically-linked Go DeepDoc backend. There is no single
+# source of truth — keep all three equal. The pip onnxruntime== pin
+# (pyproject.toml) and the onnxruntime_go binding minor (go.mod) must stay on
+# the same minor line.
+ORT_VERSION = "1.23.2"
+
+
+def prune_stale_onnxruntime(static_lib_dir, version):
+ """Remove ONNX Runtime version dirs under static_lib that do NOT match
+ `version`. Without this, a version bump leaves the stale dir next to
+ the new one and build.sh's `find ... -name '*.a'` links BOTH (duplicate
+ symbols / wrong version, silently)."""
+ if not os.path.isdir(static_lib_dir):
+ return
+ expected = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
+ for name in os.listdir(static_lib_dir):
+ if not name.startswith("onnxruntime-linux-x64-static_lib-"):
+ continue
+ if name == expected:
+ continue
+ stale = os.path.join(static_lib_dir, name)
+ print(f" Removing stale ONNX Runtime dir: {stale}")
+ shutil.rmtree(stale)
+
+
+def has_static_archives(directory):
+ """True when `directory` holds at least one static archive (.a)."""
+ return any(f.endswith(".a") for _, _, files in os.walk(directory) for f in files)
+
+
+def extract_onnxruntime(static_lib_dir, archive_path, version):
+ """Ensure the ONNX Runtime static archives for `version` sit under
+ `static_lib_dir`. Returns True when that version is available afterwards
+ (extracted now or already present), False when the archive is missing.
+
+ ORT ships a version-stamped top-level dir inside the zip
+ (onnxruntime-linux-x64-static_lib-<version>-glibc2_28/), so a present
+ `static_lib_dir` is NOT evidence that THIS version is extracted: after a
+ version bump the stale dir is pruned and the new one must be extracted.
+ """
+ if not os.path.isfile(archive_path):
+ print(f" Skipping extraction: {os.path.basename(archive_path)} not found")
+ return False
+ prune_stale_onnxruntime(static_lib_dir, version)
+ version_dir = os.path.join(static_lib_dir, f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28")
+ if os.path.isdir(version_dir) and has_static_archives(version_dir):
+ print(f" ✓ onnxruntime/static_lib ({version}) already extracted to {version_dir}")
+ return True
+ os.makedirs(static_lib_dir, exist_ok=True)
+ print(f" Extracting {os.path.basename(archive_path)} → {static_lib_dir}")
+ with zipfile.ZipFile(archive_path) as zf:
+ zf.extractall(static_lib_dir)
+ return True
+
def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
if use_china_mirrors:
@@ -57,12 +116,16 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
# compatibility contract.
"https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
"https://gh-proxy.com/https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
- # Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
- # Used by build.sh's check_*_deps functions — pre-downloaded to avoid
- # network access during CI.
+ # Native static libraries for Go build (pdfium, pdf_oxide,
+ # office_oxide, onnxruntime). Used by build.sh's check_*_deps
+ # functions — pre-downloaded to avoid network access during CI.
["https://gh-proxy.com/https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
["https://gh-proxy.com/https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
["https://gh-proxy.com/https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
+ [
+ f"https://gh-proxy.com/https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
+ f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
+ ],
]
else:
return [
@@ -82,12 +145,16 @@ def get_urls(use_china_mirrors=False) -> list[str | list[str]]:
# compatibility contract.
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-x64",
"https://github.com/browserbase/stagehand/releases/download/stagehand-server-v3/v3.7.2/stagehand-server-v3-linux-arm64",
- # Native static libraries for Go build (pdfium, pdf_oxide, office_oxide)
- # Used by build.sh's check_*_deps functions — pre-downloaded to avoid
- # network access during CI.
+ # Native static libraries for Go build (pdfium, pdf_oxide,
+ # office_oxide, onnxruntime). Used by build.sh's check_*_deps
+ # functions — pre-downloaded to avoid network access during CI.
["https://github.com/kognitos/pdfium-static/releases/download/chromium%2F7809/pdfium-linux-x64-static.tgz", "pdfium-linux-x64-static.tgz"],
["https://github.com/yfedoseev/pdf_oxide/releases/download/v0.3.73/pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide-go-ffi-linux-amd64.tar.gz"],
["https://github.com/yfedoseev/office_oxide/releases/download/v0.1.9/native-linux-x86_64.tar.gz", "office_oxide-linux-x86_64.tar.gz"],
+ [
+ f"https://github.com/csukuangfj/onnxruntime-libs/releases/download/v{ORT_VERSION}/onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
+ f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip",
+ ],
]
@@ -139,12 +206,13 @@ if __name__ == "__main__":
# Extract native static libraries to ~/ragflow-native-libs for Go build.
# Ensures build.sh can find them without network access.
native_deps_dir = os.path.expanduser("~/ragflow-native-libs")
+ import tarfile
+
extractions = [
("pdfium-linux-x64-static.tgz", "pdfium-static"),
("pdf_oxide-go-ffi-linux-amd64.tar.gz", "pdf_oxide"),
("office_oxide-linux-x86_64.tar.gz", "office_oxide"),
]
- import tarfile
for archive, subdir in extractions:
archive_path = os.path.join(os.getcwd(), archive)
@@ -159,3 +227,33 @@ if __name__ == "__main__":
print(f" Extracting {archive} → {target}")
with tarfile.open(archive_path) as tf:
tf.extractall(target)
+
+ if not extract_onnxruntime(
+ os.path.join(native_deps_dir, "onnxruntime", "static_lib"),
+ os.path.join(os.getcwd(), f"onnxruntime-linux-x64-static_lib-{ORT_VERSION}-glibc2_28.zip"),
+ ORT_VERSION,
+ ):
+ # The archive was not downloaded or failed to extract, so no .a landed.
+ # Fail loud instead of exiting 0: build.sh's ORT guard would otherwise
+ # reject the build later with a less actionable message, and a missing
+ # .a left here is exactly the "silent green" this PR is meant to prevent.
+ print(
+ f" ERROR: ONNX Runtime static archives for {ORT_VERSION} were not "
+ f"extracted to {os.path.join(native_deps_dir, 'onnxruntime', 'static_lib')}. "
+ f"Check the download above; build.sh will refuse to link without them.",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+
+ # ONNX Runtime is statically linked into the server binary, so there is no
+ # runtime .so to surface. Log where build.sh (ONNXRUNTIME_STATIC_PREFIX) will
+ # find the archives — the .a files live under
+ # ~/ragflow-native-libs/onnxruntime/static_lib. The in-process backend
+ # resolves OrtGetApiBase via dlopen(NULL); there is no dynamic .so fallback.
+ ort_static_dir = os.path.join(native_deps_dir, "onnxruntime", "static_lib")
+ ort_a_files = [os.path.join(root, f) for root, _, files in os.walk(ort_static_dir) for f in files if f.endswith(".a")]
+ if ort_a_files:
+ print(f" ✓ onnxruntime static archives ready: {len(ort_a_files)} .a under {ort_static_dir}")
+ else:
+ print(f" ERROR: ONNX Runtime .a files still missing under {ort_static_dir} after extraction; build.sh will refuse to link.", file=sys.stderr)
+ sys.exit(1)
diff --git a/ragflow_deps/test_download_go_deps.py b/ragflow_deps/test_download_go_deps.py
new file mode 100644
index 0000000000..6660797236
--- /dev/null
+++ b/ragflow_deps/test_download_go_deps.py
@@ -0,0 +1,71 @@
+# Tests for ragflow_deps/download_go_deps.py ONNX Runtime extraction.
+#
+# build.sh's build_go() fails fast when libonnxruntime.a is not linked, so these
+# tests must guarantee the archive is really landed on disk — above all after an
+# ORT version bump, where the zip's version-stamped top-level dir changes but
+# static_lib/ still exists from the previous run.
+
+import os
+import zipfile
+
+from download_go_deps import extract_onnxruntime, has_static_archives
+
+
+def make_ort_zip(path, version):
+ """Build a zip shaped like the upstream ORT release: a version-stamped
+ top-level dir holding lib/libonnxruntime.a."""
+ member = f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28/lib/libonnxruntime.a"
+ with zipfile.ZipFile(path, "w") as zf:
+ zf.writestr(member, b"!<arch>\nort-payload")
+ return path
+
+
+def test_extracts_on_first_run(tmp_path):
+ # static_lib/ does not exist yet: the plain first-run path.
+ static_lib = tmp_path / "onnxruntime" / "static_lib"
+ version = "1.23.2"
+ archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
+
+ assert extract_onnxruntime(str(static_lib), str(archive), version) is True
+
+ version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
+ assert version_dir.is_dir()
+ assert has_static_archives(str(version_dir))
+
+
+def test_extracts_after_version_bump_when_static_lib_exists(tmp_path):
+ """Regression: a static_lib/ left over from a previous version must not
+ suppress extraction of the bumped version. Pruning the stale dir leaves
+ static_lib/ in place; if extraction is then skipped no .a is ever landed and
+ build.sh's ORT guard rejects the build."""
+ static_lib = tmp_path / "onnxruntime" / "static_lib"
+ static_lib.mkdir(parents=True)
+ stale = static_lib / "onnxruntime-linux-x64-static_lib-1.23.1-glibc2_28"
+ (stale / "lib").mkdir(parents=True)
+ (stale / "lib" / "libonnxruntime.a").write_bytes(b"old-ort")
+
+ version = "1.23.2"
+ archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
+
+ assert extract_onnxruntime(str(static_lib), str(archive), version) is True
+
+ assert not stale.exists(), "stale ORT version should be pruned"
+ version_dir = static_lib / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28"
+ assert version_dir.is_dir(), "bumped ORT version was silently not extracted"
+ assert has_static_archives(str(version_dir)), "bumped ORT version landed no .a"
+
+
+def test_skips_when_archive_missing(tmp_path):
+ static_lib = tmp_path / "onnxruntime" / "static_lib"
+ assert extract_onnxruntime(str(static_lib), str(tmp_path / "missing.zip"), "1.23.2") is False
+
+
+def test_idempotent_when_matching_version_already_present(tmp_path):
+ static_lib = tmp_path / "onnxruntime" / "static_lib"
+ version = "1.23.2"
+ archive = make_ort_zip(tmp_path / f"onnxruntime-linux-x64-static_lib-{version}-glibc2_28.zip", version)
+
+ assert extract_onnxruntime(str(static_lib), str(archive), version) is True
+ before = sorted(os.listdir(static_lib))
+ assert extract_onnxruntime(str(static_lib), str(archive), version) is True
+ assert sorted(os.listdir(static_lib)) == before