Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(deepdoc): move testdata fetch into native and relocate script to tools/scripts (#20822)

xugangqiang1788528f8db2
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index 091c40d6d4..6f99f148cf 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -1355,7 +1355,7 @@ jobs:
# Testdata is pre-seeded on the runner at /opt/ragflow-testdata
# (asserted by the previous step). RAGFLOW_TESTDATA_DIR points the fetch
# script at it so the network clone is skipped (see
- # scripts/fetch_deepdoc_testdata.sh) — which is why no actions/cache step
+ # tools/scripts/fetch_deepdoc_testdata.sh) — which is why no actions/cache step
# is needed here. XDG_CACHE_HOME remains the location the script falls
# back to when it runs outside this job (local dev, no mount).
env:
diff --git a/internal/deepdoc/native/testdata_fetch.go b/internal/deepdoc/native/testdata_fetch.go
index 37fd37c1f6..c6a20b0c8c 100644
--- a/internal/deepdoc/native/testdata_fetch.go
+++ b/internal/deepdoc/native/testdata_fetch.go
@@ -5,6 +5,10 @@
// so the consuming tests can run. A fetch failure is fatal under this build tag:
// it sets testdataFetchFailed so TestMain fails the package instead of silently
// skipping — a missing fixture must never be a green CI run.
+//
+// FetchTestdata is exported so the native_analyzer package — whose tests read
+// native's testdata directory via a relative path — can reuse the same logic
+// instead of duplicating it.
package native
import (
@@ -15,23 +19,33 @@ import (
"runtime"
)
-func init() {
- if err := fetchTestdata(); err != nil {
- fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)
- testdataFetchFailed = true
- return
+// fetchScriptOverride is a test-only seam; empty means use the real script
+// location. It lets tests exercise the error and happy paths deterministically.
+var fetchScriptOverride string
+
+// scriptPath returns the on-disk location of fetch_deepdoc_testdata.sh relative
+// to the repository root.
+func scriptPath(root string) string {
+ return filepath.Join(root, "tools", "scripts", "fetch_deepdoc_testdata.sh")
+}
+
+// resolveScript returns the script path to invoke, honoring the test override.
+func resolveScript(root string) string {
+ if fetchScriptOverride != "" {
+ return fetchScriptOverride
}
- testdataFetchAttempted = true
+ return scriptPath(root)
}
-// fetchTestdata resolves the repo root, then invokes scripts/fetch_deepdoc_testdata.sh
-// with this package's directory name as <pkg>.
-func fetchTestdata() error {
+// FetchTestdata resolves the repository root (via go.mod) and invokes
+// tools/scripts/fetch_deepdoc_testdata.sh with the given <pkg> (the package
+// directory that owns internal/deepdoc/<pkg>/testdata). The script resolves its
+// own repo root, so cmd.Dir here is advisory.
+func FetchTestdata(pkg string) error {
_, thisFile, _, ok := runtime.Caller(0)
if !ok {
return fmt.Errorf("cannot resolve caller path")
}
- // thisFile = <root>/internal/deepdoc/native/testdata_fetch.go
dir := filepath.Dir(thisFile)
root := dir
for {
@@ -44,8 +58,7 @@ func fetchTestdata() error {
}
root = parent
}
- pkg := filepath.Base(dir)
- script := filepath.Join(root, "scripts", "fetch_deepdoc_testdata.sh")
+ script := resolveScript(root)
if _, err := os.Stat(script); err != nil {
return fmt.Errorf("fetch script not found at %s", script)
}
@@ -55,3 +68,12 @@ func fetchTestdata() error {
cmd.Stderr = os.Stderr
return cmd.Run()
}
+
+func init() {
+ if err := FetchTestdata("native"); err != nil {
+ fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)
+ testdataFetchFailed = true
+ return
+ }
+ testdataFetchAttempted = true
+}
diff --git a/internal/deepdoc/native/testdata_fetch_test.go b/internal/deepdoc/native/testdata_fetch_test.go
new file mode 100644
index 0000000000..29bcbf9992
--- /dev/null
+++ b/internal/deepdoc/native/testdata_fetch_test.go
@@ -0,0 +1,80 @@
+//go:build fetch_testdata
+
+package native
+
+import (
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+)
+
+// testRepoRoot walks up from this test file to the directory containing go.mod.
+func testRepoRoot(t *testing.T) string {
+ t.Helper()
+ _, thisFile, _, ok := runtime.Caller(0)
+ if !ok {
+ t.Fatal("cannot resolve caller path")
+ }
+ dir := filepath.Dir(thisFile)
+ for {
+ if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+ return dir
+ }
+ parent := filepath.Dir(dir)
+ if parent == dir {
+ t.Fatal("repo root (go.mod) not found")
+ }
+ dir = parent
+ }
+}
+
+func TestScriptPath(t *testing.T) {
+ if got := scriptPath("/repo"); got != "/repo/tools/scripts/fetch_deepdoc_testdata.sh" {
+ t.Fatalf("scriptPath = %q, want /repo/tools/scripts/fetch_deepdoc_testdata.sh", got)
+ }
+}
+
+// TestFetchScriptResolvedToToolsScripts guards the move: the fetch script must
+// live at tools/scripts/ (not the old scripts/) so the repo root derivation in
+// the script itself stays correct.
+func TestFetchScriptResolvedToToolsScripts(t *testing.T) {
+ root := testRepoRoot(t)
+ got := scriptPath(root)
+ if _, err := os.Stat(got); err != nil {
+ t.Fatalf("expected fetch script at %s: %v", got, err)
+ }
+ old := filepath.Join(root, "scripts", "fetch_deepdoc_testdata.sh")
+ if _, err := os.Stat(old); err == nil {
+ t.Fatalf("script still present at old location %s; it should have been moved to tools/scripts/", old)
+ }
+}
+
+// TestFetchMissingScriptReturnsError covers the error path: a missing script
+// must surface a descriptive error rather than a confusing exec failure.
+func TestFetchMissingScriptReturnsError(t *testing.T) {
+ root := testRepoRoot(t)
+ fetchScriptOverride = filepath.Join(root, "does-not-exist.sh")
+ defer func() { fetchScriptOverride = "" }()
+ if err := FetchTestdata("native"); err == nil {
+ t.Fatal("FetchTestdata returned nil error for missing script, want error")
+ } else if !strings.Contains(err.Error(), "fetch script not found") {
+ t.Fatalf("FetchTestdata error = %q, want it to mention 'fetch script not found'", err.Error())
+ }
+}
+
+// TestFetchHappyPathWithFakeScript exercises the real exec path deterministically
+// (no network clone) using a fake script that exits 0.
+func TestFetchHappyPathWithFakeScript(t *testing.T) {
+ tmp := t.TempDir()
+ fake := filepath.Join(tmp, "fake-fetch.sh")
+ if err := os.WriteFile(fake, []byte("#!/usr/bin/env bash\nexit 0\n"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ fetchScriptOverride = fake
+ defer func() { fetchScriptOverride = "" }()
+ if err := FetchTestdata("native"); err != nil {
+ t.Fatalf("FetchTestdata with fake success script = %v, want nil", err)
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go
index 0c2d9a8902..9688e9e60e 100644
--- a/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go
+++ b/internal/deepdoc/parser/pdf/inference/native_analyzer/testdata_fetch.go
@@ -3,55 +3,24 @@
// This file is compiled only with -tags fetch_testdata. Its init() downloads
// and symlinks the external DeepDoc testdata. The native_analyzer tests read
// their fixtures from the sibling internal/deepdoc/native/testdata directory
-// (via a relative path), so we fetch with pkg = "native". A fetch failure is
-// fatal under this build tag: it sets testdataFetchFailed so TestMain fails the
-// package instead of silently skipping — a missing fixture must never be a green
-// CI run.
+// (via a relative path), so we fetch with pkg = "native" by reusing native's
+// FetchTestdata. A fetch failure is fatal under this build tag: it sets
+// testdataFetchFailed so TestMain fails the package instead of silently
+// skipping — a missing fixture must never be a green CI run.
package infnative
import (
"fmt"
"os"
- "os/exec"
- "path/filepath"
- "runtime"
+
+ "ragflow/internal/deepdoc/native"
)
func init() {
- if err := fetchTestdata(); err != nil {
+ if err := native.FetchTestdata("native"); err != nil {
fmt.Fprintf(os.Stderr, "fetch_deepdoc_testdata: not fetched (%v)\n", err)
testdataFetchFailed = true
return
}
testdataFetchAttempted = true
}
-
-// fetchTestdata resolves the repo root, then invokes scripts/fetch_deepdoc_testdata.sh
-// with pkg = "native" (the package that actually owns the testdata directory).
-func fetchTestdata() error {
- _, thisFile, _, ok := runtime.Caller(0)
- if !ok {
- return fmt.Errorf("cannot resolve caller path")
- }
- dir := filepath.Dir(thisFile)
- root := dir
- for {
- if _, err := os.Stat(filepath.Join(root, "go.mod")); err == nil {
- break
- }
- parent := filepath.Dir(root)
- if parent == root {
- return fmt.Errorf("repo root (go.mod) not found from %s", dir)
- }
- root = parent
- }
- script := filepath.Join(root, "scripts", "fetch_deepdoc_testdata.sh")
- if _, err := os.Stat(script); err != nil {
- return fmt.Errorf("fetch script not found at %s", script)
- }
- cmd := exec.Command("bash", script, "native")
- cmd.Dir = root
- cmd.Stdout = os.Stderr
- cmd.Stderr = os.Stderr
- return cmd.Run()
-}
diff --git a/scripts/fetch_deepdoc_testdata.sh b/tools/scripts/fetch_deepdoc_testdata.sh
similarity index 97%
rename from scripts/fetch_deepdoc_testdata.sh
rename to tools/scripts/fetch_deepdoc_testdata.sh
index 54c80ce821..285142faf8 100644
--- a/scripts/fetch_deepdoc_testdata.sh
+++ b/tools/scripts/fetch_deepdoc_testdata.sh
@@ -1,7 +1,7 @@
#!/usr/bin/env bash
# Fetch external DeepDoc testdata into the repo on demand.
#
-# Usage: scripts/fetch_deepdoc_testdata.sh <pkg>
+# Usage: tools/scripts/fetch_deepdoc_testdata.sh <pkg>
# <pkg> = the package directory that owns the testdata, e.g. "native"
# (mirrors internal/deepdoc/<pkg>/testdata).
#
@@ -44,9 +44,9 @@ if [ $# -lt 1 ]; then
fi
PKG="$1"
-# Resolve repo root from this script's location (scripts/ -> repo root).
+# Resolve repo root from this script's location (tools/scripts/ -> repo root).
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
TARGET="$ROOT/internal/deepdoc/$PKG/testdata"