RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(deepdoc): move testdata fetch into native and relocate script to tools/scripts (#20822)
xugangqiang1788528f8db2
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.ymlindex 091c40d6d4..6f99f148cf 100644--- a/.github/workflows/sep-tests.yml+++ b/.github/workflows/sep-tests.yml@@ -1355,7 +1355,7 @@ jobs:# Testdata is pre-seeded on the runner at /opt/ragflow-testdata# (asserted by the previous step). RAGFLOW_TESTDATA_DIR points the fetch# script at it so the network clone is skipped (see- # scripts/fetch_deepdoc_testdata.sh) — which is why no actions/cache step+ # tools/scripts/fetch_deepdoc_testdata.sh) — which is why no actions/cache step# is needed here. XDG_CACHE_HOME remains the location the script falls# back to when it runs outside this job (local dev, no mount).env:diff --git a/internal/deepdoc/native/testdata_fetch.go b/internal/deepdoc/native/testdata_fetch.goindex 37fd37c1f6..c6a20b0c8c 100644--- a/internal/deepdoc/native/testdata_fetch.go+++ b/internal/deepdoc/native/testdata_fetch.go@@ -5,6 +5,10 @@// so the consuming tests can run. A fetch failure is fatal under this build tag:// it sets testdataFetchFailed so TestMain fails the package instead of silently// skipping — a missing fixture must never be a green CI run.+//+// FetchTestdata is exported so the native_analyzer package — whose tests read+// native's testdata directory via a relative path — can reuse the same logic+// instead of duplicating it.package nativeimport (@@ -15,23 +19,33 @@ import ("runtime")-func init() {- if err := fetchTestdata(); err != nil {- fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)- testdataFetchFailed = true- return+// fetchScriptOverride is a test-only seam; empty means use the real script+// location. It lets tests exercise the error and happy paths deterministically.+var fetchScriptOverride string++// scriptPath returns the on-disk location of fetch_deepdoc_testdata.sh relative+// to the repository root.+func scriptPath(root string) string {+ return filepath.Join(root, "tools", "scripts", "fetch_deepdoc_testdata.sh")+}++// resolveScript returns the script path to invoke, honoring the test override.+func resolveScript(root string) string {+ if fetchScriptOverride != "" {+ return fetchScriptOverride}- testdataFetchAttempted = true+ return scriptPath(root)}-// fetchTestdata resolves the repo root, then invokes scripts/fetch_deepdoc_testdata.sh-// with this package's directory name as <pkg>.-func fetchTestdata() error {+// FetchTestdata resolves the repository root (via go.mod) and invokes+// tools/scripts/fetch_deepdoc_testdata.sh with the given <pkg> (the package+// directory that owns internal/deepdoc/<pkg>/testdata). The script resolves its+// own repo root, so cmd.Dir here is advisory.+func FetchTestdata(pkg string) error {_, thisFile, _, ok := runtime.Caller(0)if !ok {return fmt.Errorf("cannot resolve caller path")}- // thisFile = <root>/internal/deepdoc/native/testdata_fetch.godir := filepath.Dir(thisFile)root := dirfor {@@ -44,8 +58,7 @@ func fetchTestdata() error {}root = parent}- pkg := filepath.Base(dir)- script := filepath.Join(root, "scripts", "fetch_deepdoc_testdata.sh")+ script := resolveScript(root)if _, err := os.Stat(script); err != nil {return fmt.Errorf("fetch script not found at %s", script)}@@ -55,3 +68,12 @@ func fetchTestdata() error {cmd.Stderr = os.Stderrreturn cmd.Run()}++func init() {+ if err := FetchTestdata("native"); err != nil {+ fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)+ testdataFetchFailed = true+ return+ }+ testdataFetchAttempted = true+}diff --git a/internal/deepdoc/native/testdata_fetch_test.go b/internal/deepdoc/native/testdata_fetch_test.gonew file mode 100644index 0000000000..29bcbf9992--- /dev/null+++ b/internal/deepdoc/native/testdata_fetch_test.go@@ -0,0 +1,80 @@+//go:build fetch_testdata++package native++import (+ "os"+ "path/filepath"+ "runtime"+ "strings"+ "testing"+)++// testRepoRoot walks up from this test file to the directory containing go.mod.+func testRepoRoot(t *testing.T) string {+ t.Helper()+ _, thisFile, _, ok := runtime.Caller(0)+ if !ok {+ t.Fatal("cannot resolve caller path")+ }+ dir := filepath.Dir(thisFile)+ for {+ if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {+ return dir+ }+ parent := filepath.Dir(dir)+ if parent == dir {+ t.Fatal("repo root (go.mod) not found")+ }+ dir = parent+ }+}++func TestScriptPath(t *testing.T) {+ if got := scriptPath("/repo"); got != "/repo/tools/scripts/fetch_deepdoc_testdata.sh" {+ t.Fatalf("scriptPath = %q, want /repo/tools/scripts/fetch_deepdoc_testdata.sh", got)+ }+}++// TestFetchScriptResolvedToToolsScripts guards the move: the fetch script must+// live at tools/scripts/ (not the old scripts/) so the repo root derivation in+// the script itself stays correct.+func TestFetchScriptResolvedToToolsScripts(t *testing.T) {+ root := testRepoRoot(t)+ got := scriptPath(root)+ if _, err := os.Stat(got); err != nil {+ t.Fatalf("expected fetch script at %s: %v", got, err)+ }+ old := filepath.Join(root, "scripts", "fetch_deepdoc_testdata.sh")+ if _, err := os.Stat(old); err == nil {+ t.Fatalf("script still present at old location %s; it should have been moved to tools/scripts/", old)+ }+}++// TestFetchMissingScriptReturnsError covers the error path: a missing script+// must surface a descriptive error rather than a confusing exec failure.+func TestFetchMissingScriptReturnsError(t *testing.T) {+ root := testRepoRoot(t)+ fetchScriptOverride = filepath.Join(root, "does-not-exist.sh")+ defer func() { fetchScriptOverride = "" }()+ if err := FetchTestdata("native"); err == nil {+ t.Fatal("FetchTestdata returned nil error for missing script, want error")+ } else if !strings.Contains(err.Error(), "fetch script not found") {+ t.Fatalf("FetchTestdata error = %q, want it to mention 'fetch script not found'", err.Error())+ }+}++// TestFetchHappyPathWithFakeScript exercises the real exec path deterministically+// (no network clone) using a fake script that exits 0.+func TestFetchHappyPathWithFakeScript(t *testing.T) {+ tmp := t.TempDir()+ fake := filepath.Join(tmp, "fake-fetch.sh")+ if err := os.WriteFile(fake, []byte("#!/usr/bin/env bash\nexit 0\n"), 0o755); err != nil {+ t.Fatal(err)+ }+ fetchScriptOverride = fake+ defer func() { fetchScriptOverride = "" }()+ if err := FetchTestdata("native"); err != nil {+ t.Fatalf("FetchTestdata with fake success script = %v, want nil", err)+ }+}diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.goindex 0c2d9a8902..9688e9e60e 100644--- a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go+++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go@@ -3,55 +3,24 @@// This file is compiled only with -tags fetch_testdata. Its init() downloads// and symlinks the external DeepDoc testdata. The native_analyzer tests read// their fixtures from the sibling internal/deepdoc/native/testdata directory-// (via a relative path), so we fetch with pkg = "native". A fetch failure is-// fatal under this build tag: it sets testdataFetchFailed so TestMain fails the-// package instead of silently skipping — a missing fixture must never be a green-// CI run.+// (via a relative path), so we fetch with pkg = "native" by reusing native's+// FetchTestdata. A fetch failure is fatal under this build tag: it sets+// testdataFetchFailed so TestMain fails the package instead of silently+// skipping — a missing fixture must never be a green CI run.package infnativeimport ("fmt""os"- "os/exec"- "path/filepath"- "runtime"++ "ragflow/internal/deepdoc/native")func init() {- if err := fetchTestdata(); err != nil {+ if err := native.FetchTestdata("native"); err != nil {fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)testdataFetchFailed = truereturn}testdataFetchAttempted = true}--// fetchTestdata resolves the repo root, then invokes scripts/fetch_deepdoc_testdata.sh-// with pkg = "native" (the package that actually owns the testdata directory).-func fetchTestdata() error {- _, thisFile, _, ok := runtime.Caller(0)- if !ok {- return fmt.Errorf("cannot resolve caller path")- }- dir := filepath.Dir(thisFile)- root := dir- for {- if _, err := os.Stat(filepath.Join(root, "go.mod")); err == nil {- break- }- parent := filepath.Dir(root)- if parent == root {- return fmt.Errorf("repo root (go.mod) not found from %s", dir)- }- root = parent- }- script := filepath.Join(root, "scripts", "fetch_deepdoc_testdata.sh")- if _, err := os.Stat(script); err != nil {- return fmt.Errorf("fetch script not found at %s", script)- }- cmd := exec.Command("bash", script, "native")- cmd.Dir = root- cmd.Stdout = os.Stderr- cmd.Stderr = os.Stderr- return cmd.Run()-}diff --git a/scripts/fetch_deepdoc_testdata.sh b/tools/scripts/fetch_deepdoc_testdata.shsimilarity index 97%rename from scripts/fetch_deepdoc_testdata.shrename to tools/scripts/fetch_deepdoc_testdata.shindex 54c80ce821..285142faf8 100644--- a/scripts/fetch_deepdoc_testdata.sh+++ b/tools/scripts/fetch_deepdoc_testdata.sh@@ -1,7 +1,7 @@#!/usr/bin/env bash# Fetch external DeepDoc testdata into the repo on demand.#-# Usage: scripts/fetch_deepdoc_testdata.sh <pkg>+# Usage: tools/scripts/fetch_deepdoc_testdata.sh <pkg># <pkg> = the package directory that owns the testdata, e.g. "native"# (mirrors internal/deepdoc/<pkg>/testdata).#@@ -44,9 +44,9 @@ if [ $# -lt 1 ]; thenfiPKG="$1"-# Resolve repo root from this script's location (scripts/ -> repo root).+# Resolve repo root from this script's location (tools/scripts/ -> repo root).SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"-ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"+ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"TARGET="$ROOT/internal/deepdoc/$PKG/testdata"