Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ci): read DeepDoc testdata from the pre-seeded dir instead of linking it in (#19594)

xugangqiang45e815cbc1c3
diff --git a/internal/deepdoc/native/testdata_dir_test.go b/internal/deepdoc/native/testdata_dir_test.go
new file mode 100644
index 0000000000..8cc82afd48
--- /dev/null
+++ b/internal/deepdoc/native/testdata_dir_test.go
@@ -0,0 +1,79 @@
+//go:build fetch_testdata
+
+// This package's TestMain skips everything unless the fixtures were fetched, so
+// these tests only run under the same tag — which is how CI executes them.
+package native
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+// packageDirName must match the real package dir name: resolveTestdataDir
+// derives the pre-seeded subtree from basename(pkgDir).
+const packageDirName = "native"
+
+// newPackageDir creates an empty <tmp>/native directory to stand in for the
+// package directory.
+func newPackageDir(t *testing.T) string {
+ t.Helper()
+ dir := filepath.Join(t.TempDir(), packageDirName)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatalf("mkdir %s: %v", dir, err)
+ }
+ return dir
+}
+
+// seedTestdata creates <dir>/testdata with one file so it counts as populated.
+func seedTestdata(t *testing.T, dir string) {
+ t.Helper()
+ target := filepath.Join(dir, "testdata")
+ if err := os.MkdirAll(target, 0o755); err != nil {
+ t.Fatalf("mkdir %s: %v", target, err)
+ }
+ if err := os.WriteFile(filepath.Join(target, "fixture.json"), []byte("{}"), 0o644); err != nil {
+ t.Fatalf("write fixture: %v", err)
+ }
+}
+
+// TestResolveTestdataDir_KeepsLocalCopy verifies that an inline testdata
+// directory always wins: it is what a developer fetched locally, and it must
+// keep working unchanged.
+func TestResolveTestdataDir_KeepsLocalCopy(t *testing.T) {
+ local := newPackageDir(t)
+ seedTestdata(t, local)
+ // A pre-seeded tree is configured too, but the local copy takes priority.
+ t.Setenv("RAGFLOW_TESTDATA_DIR", t.TempDir())
+
+ if got := resolveTestdataDir(local); got != "" {
+ t.Fatalf("want empty (stay in the package dir), got %q", got)
+ }
+}
+
+// TestResolveTestdataDir_FallsBackToPreseeded covers the read-only workspace
+// case: the fixtures cannot be linked into the package dir, and the pre-seeded
+// copy must be used from where it is instead of failing the package.
+func TestResolveTestdataDir_FallsBackToPreseeded(t *testing.T) {
+ local := newPackageDir(t) // no inline testdata
+ root := t.TempDir()
+ seedTestdata(t, filepath.Join(root, "deepdoc", packageDirName))
+ t.Setenv("RAGFLOW_TESTDATA_DIR", root)
+
+ want := filepath.Join(root, "deepdoc", packageDirName)
+ if got := resolveTestdataDir(local); got != want {
+ t.Fatalf("want %q, got %q", want, got)
+ }
+}
+
+// TestResolveTestdataDir_EmptyWhenNothingAvailable makes sure we do not chdir
+// into a directory that has no fixtures: that would turn a clear "missing
+// testdata" failure into a confusing file-not-found one.
+func TestResolveTestdataDir_EmptyWhenNothingAvailable(t *testing.T) {
+ local := newPackageDir(t)
+ t.Setenv("RAGFLOW_TESTDATA_DIR", t.TempDir()) // pre-seeded tree is empty
+
+ if got := resolveTestdataDir(local); got != "" {
+ t.Fatalf("want empty, got %q", got)
+ }
+}
diff --git a/internal/deepdoc/native/testdata_skip_test.go b/internal/deepdoc/native/testdata_skip_test.go
index 7126ec9c2f..50df278fb3 100644
--- a/internal/deepdoc/native/testdata_skip_test.go
+++ b/internal/deepdoc/native/testdata_skip_test.go
@@ -10,9 +10,51 @@ package native
import (
"fmt"
"os"
+ "path/filepath"
+ "runtime"
"testing"
)
+// packageDir returns this package's directory (the one holding testdata).
+func packageDir() string {
+ _, file, _, ok := runtime.Caller(0)
+ if !ok {
+ return ""
+ }
+ return filepath.Dir(file)
+}
+
+// dirHasFiles reports whether dir exists and holds at least one entry.
+func dirHasFiles(dir string) bool {
+ entries, err := os.ReadDir(dir)
+ return err == nil && len(entries) > 0
+}
+
+// resolveTestdataDir decides which directory the fixtures should be read from.
+//
+// Tests reach fixtures through the relative path "testdata/...", so the process
+// must either hold them inline or run from a directory that does. CI runners
+// pre-seed the assets at RAGFLOW_TESTDATA_DIR and mount the workspace
+// read-only, so linking the fixtures into the package dir — what the fetch
+// script used to do — fails there even though the data is present and readable.
+// Running from the pre-seeded directory instead keeps the workspace untouched.
+//
+// It returns "" when the package's own testdata should be used as-is.
+func resolveTestdataDir(pkgDir string) string {
+ if pkgDir == "" || dirHasFiles(filepath.Join(pkgDir, "testdata")) {
+ return ""
+ }
+ root := os.Getenv("RAGFLOW_TESTDATA_DIR")
+ if root == "" {
+ return ""
+ }
+ preseeded := filepath.Join(root, "deepdoc", filepath.Base(pkgDir))
+ if dirHasFiles(filepath.Join(preseeded, "testdata")) {
+ return preseeded
+ }
+ return ""
+}
+
// TestMain skips the entire package unless the testdata was fetched via the
// fetch_testdata build tag. In the default build (and any build without that
// tag) it stays false, so the tests are skipped rather than failing on absent
@@ -27,5 +69,13 @@ func TestMain(m *testing.M) {
fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it")
os.Exit(0)
}
+ // The fetch only guarantees the fixtures exist somewhere; with a pre-seeded
+ // runner they live outside the package, so run from their directory.
+ if dir := resolveTestdataDir(packageDir()); dir != "" {
+ if err := os.Chdir(dir); err != nil {
+ fmt.Fprintf(os.Stderr, "FAIL: cannot chdir to pre-seeded testdata %s: %v\n", dir, err)
+ os.Exit(1)
+ }
+ }
os.Exit(m.Run())
}
diff --git a/scripts/fetch_deepdoc_testdata.sh b/scripts/fetch_deepdoc_testdata.sh
index e1209f5af5..54c80ce821 100644
--- a/scripts/fetch_deepdoc_testdata.sh
+++ b/scripts/fetch_deepdoc_testdata.sh
@@ -62,7 +62,8 @@ mkdir -p "$LOCKDIR"
exec 9>"$LOCKDIR/$PKG.lock"
flock 9
-# Determine whether we need a writable copy (regeneration) or a symlink.
+# Determine whether we need a writable copy (regeneration) or can use the
+# pre-seeded copy directly.
NEED_WRITE=0
for v in "${!GEN_@}"; do
if [ -n "${!v:-}" ]; then NEED_WRITE=1; break; fi
@@ -75,42 +76,23 @@ done
if [ -n "${RAGFLOW_TESTDATA_DIR:-}" ]; then
PRESET="$RAGFLOW_TESTDATA_DIR/deepdoc/$PKG/testdata"
if [ -d "$PRESET" ] && [ -n "$(ls -A "$PRESET" 2>/dev/null)" ]; then
- if [ -L "$TARGET" ] && [ "$(readlink -f "$TARGET")" = "$(readlink -f "$PRESET")" ] && [ -n "$(ls -A "$PRESET" 2>/dev/null)" ]; then
- echo "fetch_deepdoc_testdata: $PKG already linked to pre-seeded $PRESET"
- exit 0
- fi
- if [ -d "$TARGET" ] && [ ! -L "$TARGET" ] && [ -n "$(ls -A "$TARGET" 2>/dev/null)" ]; then
- echo "fetch_deepdoc_testdata: $PKG testdata already present inline at $TARGET"
- exit 0
- fi
- # Remove any stale TARGET before linking. TARGET may be a wrong symlink, an
- # empty directory, or a leftover real directory from a prior GEN_* copy or a
- # dereferenced symlink on a persistent runner. `ln -sfn` below also unlinks
- # TARGET first, but removing it here keeps intent explicit. A partial
- # removal failure is non-fatal: ln -sfn still replaces TARGET, and the copy
- # fallback below covers the rare case where even that is blocked.
- #
- # Use `ln -sfn`, never `ln -s`: when TARGET already exists as a directory,
- # plain `ln -s` descends INTO it and creates <TARGET>/testdata (the exact
- # "Permission denied" failure this job hit), because the existing dir is not
- # writable. `-f` unlinks TARGET first; `-n`/`--no-dereference` stops ln from
- # treating TARGET as a container.
- rm -rf -- "$TARGET" 2>/dev/null || true
if [ "$NEED_WRITE" -eq 1 ]; then
- echo "fetch_deepdoc_testdata: copying writable pre-seeded testdata for regeneration ($PKG)"
- rm -rf -- "$TARGET" 2>/dev/null || true
- cp -r "$PRESET" "$TARGET"
- elif ln -sfn "$PRESET" "$TARGET" 2>/dev/null; then
- echo "fetch_deepdoc_testdata: linked $TARGET -> $PRESET (pre-seeded)"
- else
- # Symlink creation failed (e.g. the parent directory is not writable on a
- # shared runner). The pre-seeded fixtures are authoritative and readable,
- # so copy them in place instead of reddening CI over a filesystem quirk.
- echo "fetch_deepdoc_testdata: symlink failed; copying pre-seeded testdata for $PKG" >&2
- mkdir -p "$TARGET"
- cp -r "$PRESET"/. "$TARGET"/.
+ # Regeneration rewrites the fixtures, so it needs a writable copy next to
+ # the tests — which a pre-seeded runner cannot provide. Say so plainly
+ # instead of failing later with a confusing permission error.
+ if ! (rm -rf -- "$TARGET" && cp -r "$PRESET" "$TARGET") 2>/dev/null; then
+ echo "fetch_deepdoc_testdata: GEN_* regeneration needs a writable $TARGET, but the workspace is read-only. Run it where $TARGET is writable, or drop RAGFLOW_TESTDATA_DIR to clone the fixtures." >&2
+ exit 1
+ fi
+ echo "fetch_deepdoc_testdata: copied writable pre-seeded testdata for regeneration ($PKG)"
+ exit 0
fi
- echo "fetch_deepdoc_testdata: done ($PKG, pre-seeded)"
+ # Leave the workspace untouched: the consuming test binary reads the
+ # fixtures from this directory directly (see testdata_skip_test.go).
+ # Linking or copying them into $TARGET used to be required, and failed on
+ # runners that mount the workspace read-only — which was then reported as a
+ # missing fixture even though the data was right here.
+ echo "fetch_deepdoc_testdata: using pre-seeded testdata at $PRESET (workspace untouched)"
exit 0
fi
echo "fetch_deepdoc_testdata: RAGFLOW_TESTDATA_DIR set but deepdoc/$PKG/testdata is missing; falling back to clone" >&2