RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ci): read DeepDoc testdata from the pre-seeded dir instead of linking it in (#19594)
xugangqiang45e815cbc1c3
diff --git a/internal/deepdoc/native/testdata_dir_test.go b/internal/deepdoc/native/testdata_dir_test.gonew file mode 100644index 0000000000..8cc82afd48--- /dev/null+++ b/internal/deepdoc/native/testdata_dir_test.go@@ -0,0 +1,79 @@+//go:build fetch_testdata++// This package's TestMain skips everything unless the fixtures were fetched, so+// these tests only run under the same tag — which is how CI executes them.+package native++import (+ "os"+ "path/filepath"+ "testing"+)++// packageDirName must match the real package dir name: resolveTestdataDir+// derives the pre-seeded subtree from basename(pkgDir).+const packageDirName = "native"++// newPackageDir creates an empty <tmp>/native directory to stand in for the+// package directory.+func newPackageDir(t *testing.T) string {+ t.Helper()+ dir := filepath.Join(t.TempDir(), packageDirName)+ if err := os.MkdirAll(dir, 0o755); err != nil {+ t.Fatalf("mkdir %s: %v", dir, err)+ }+ return dir+}++// seedTestdata creates <dir>/testdata with one file so it counts as populated.+func seedTestdata(t *testing.T, dir string) {+ t.Helper()+ target := filepath.Join(dir, "testdata")+ if err := os.MkdirAll(target, 0o755); err != nil {+ t.Fatalf("mkdir %s: %v", target, err)+ }+ if err := os.WriteFile(filepath.Join(target, "fixture.json"), []byte("{}"), 0o644); err != nil {+ t.Fatalf("write fixture: %v", err)+ }+}++// TestResolveTestdataDir_KeepsLocalCopy verifies that an inline testdata+// directory always wins: it is what a developer fetched locally, and it must+// keep working unchanged.+func TestResolveTestdataDir_KeepsLocalCopy(t *testing.T) {+ local := newPackageDir(t)+ seedTestdata(t, local)+ // A pre-seeded tree is configured too, but the local copy takes priority.+ t.Setenv("RAGFLOW_TESTDATA_DIR", t.TempDir())++ if got := resolveTestdataDir(local); got != "" {+ t.Fatalf("want empty (stay in the package dir), got %q", got)+ }+}++// TestResolveTestdataDir_FallsBackToPreseeded covers the read-only workspace+// case: the fixtures cannot be linked into the package dir, and the pre-seeded+// copy must be used from where it is instead of failing the package.+func TestResolveTestdataDir_FallsBackToPreseeded(t *testing.T) {+ local := newPackageDir(t) // no inline testdata+ root := t.TempDir()+ seedTestdata(t, filepath.Join(root, "deepdoc", packageDirName))+ t.Setenv("RAGFLOW_TESTDATA_DIR", root)++ want := filepath.Join(root, "deepdoc", packageDirName)+ if got := resolveTestdataDir(local); got != want {+ t.Fatalf("want %q, got %q", want, got)+ }+}++// TestResolveTestdataDir_EmptyWhenNothingAvailable makes sure we do not chdir+// into a directory that has no fixtures: that would turn a clear "missing+// testdata" failure into a confusing file-not-found one.+func TestResolveTestdataDir_EmptyWhenNothingAvailable(t *testing.T) {+ local := newPackageDir(t)+ t.Setenv("RAGFLOW_TESTDATA_DIR", t.TempDir()) // pre-seeded tree is empty++ if got := resolveTestdataDir(local); got != "" {+ t.Fatalf("want empty, got %q", got)+ }+}diff --git a/internal/deepdoc/native/testdata_skip_test.go b/internal/deepdoc/native/testdata_skip_test.goindex 7126ec9c2f..50df278fb3 100644--- a/internal/deepdoc/native/testdata_skip_test.go+++ b/internal/deepdoc/native/testdata_skip_test.go@@ -10,9 +10,51 @@ package nativeimport ("fmt""os"+ "path/filepath"+ "runtime""testing")+// packageDir returns this package's directory (the one holding testdata).+func packageDir() string {+ _, file, _, ok := runtime.Caller(0)+ if !ok {+ return ""+ }+ return filepath.Dir(file)+}++// dirHasFiles reports whether dir exists and holds at least one entry.+func dirHasFiles(dir string) bool {+ entries, err := os.ReadDir(dir)+ return err == nil && len(entries) > 0+}++// resolveTestdataDir decides which directory the fixtures should be read from.+//+// Tests reach fixtures through the relative path "testdata/...", so the process+// must either hold them inline or run from a directory that does. CI runners+// pre-seed the assets at RAGFLOW_TESTDATA_DIR and mount the workspace+// read-only, so linking the fixtures into the package dir — what the fetch+// script used to do — fails there even though the data is present and readable.+// Running from the pre-seeded directory instead keeps the workspace untouched.+//+// It returns "" when the package's own testdata should be used as-is.+func resolveTestdataDir(pkgDir string) string {+ if pkgDir == "" || dirHasFiles(filepath.Join(pkgDir, "testdata")) {+ return ""+ }+ root := os.Getenv("RAGFLOW_TESTDATA_DIR")+ if root == "" {+ return ""+ }+ preseeded := filepath.Join(root, "deepdoc", filepath.Base(pkgDir))+ if dirHasFiles(filepath.Join(preseeded, "testdata")) {+ return preseeded+ }+ return ""+}+// TestMain skips the entire package unless the testdata was fetched via the// fetch_testdata build tag. In the default build (and any build without that// tag) it stays false, so the tests are skipped rather than failing on absent@@ -27,5 +69,13 @@ func TestMain(m *testing.M) {fmt.Println("SKIP: DeepDoc native testdata not fetched; run with -tags fetch_testdata to fetch and run it")os.Exit(0)}+ // The fetch only guarantees the fixtures exist somewhere; with a pre-seeded+ // runner they live outside the package, so run from their directory.+ if dir := resolveTestdataDir(packageDir()); dir != "" {+ if err := os.Chdir(dir); err != nil {+ fmt.Fprintf(os.Stderr, "FAIL: cannot chdir to pre-seeded testdata %s: %v\n", dir, err)+ os.Exit(1)+ }+ }os.Exit(m.Run())}diff --git a/scripts/fetch_deepdoc_testdata.sh b/scripts/fetch_deepdoc_testdata.shindex e1209f5af5..54c80ce821 100644--- a/scripts/fetch_deepdoc_testdata.sh+++ b/scripts/fetch_deepdoc_testdata.sh@@ -62,7 +62,8 @@ mkdir -p "$LOCKDIR"exec 9>"$LOCKDIR/$PKG.lock"flock 9-# Determine whether we need a writable copy (regeneration) or a symlink.+# Determine whether we need a writable copy (regeneration) or can use the+# pre-seeded copy directly.NEED_WRITE=0for v in "${!GEN_@}"; doif [ -n "${!v:-}" ]; then NEED_WRITE=1; break; fi@@ -75,42 +76,23 @@ doneif [ -n "${RAGFLOW_TESTDATA_DIR:-}" ]; thenPRESET="$RAGFLOW_TESTDATA_DIR/deepdoc/$PKG/testdata"if [ -d "$PRESET" ] && [ -n "$(ls -A "$PRESET" 2>/dev/null)" ]; then- if [ -L "$TARGET" ] && [ "$(readlink -f "$TARGET")" = "$(readlink -f "$PRESET")" ] && [ -n "$(ls -A "$PRESET" 2>/dev/null)" ]; then- echo "fetch_deepdoc_testdata: $PKG already linked to pre-seeded $PRESET"- exit 0- fi- if [ -d "$TARGET" ] && [ ! -L "$TARGET" ] && [ -n "$(ls -A "$TARGET" 2>/dev/null)" ]; then- echo "fetch_deepdoc_testdata: $PKG testdata already present inline at $TARGET"- exit 0- fi- # Remove any stale TARGET before linking. TARGET may be a wrong symlink, an- # empty directory, or a leftover real directory from a prior GEN_* copy or a- # dereferenced symlink on a persistent runner. `ln -sfn` below also unlinks- # TARGET first, but removing it here keeps intent explicit. A partial- # removal failure is non-fatal: ln -sfn still replaces TARGET, and the copy- # fallback below covers the rare case where even that is blocked.- #- # Use `ln -sfn`, never `ln -s`: when TARGET already exists as a directory,- # plain `ln -s` descends INTO it and creates <TARGET>/testdata (the exact- # "Permission denied" failure this job hit), because the existing dir is not- # writable. `-f` unlinks TARGET first; `-n`/`--no-dereference` stops ln from- # treating TARGET as a container.- rm -rf -- "$TARGET" 2>/dev/null || trueif [ "$NEED_WRITE" -eq 1 ]; then- echo "fetch_deepdoc_testdata: copying writable pre-seeded testdata for regeneration ($PKG)"- rm -rf -- "$TARGET" 2>/dev/null || true- cp -r "$PRESET" "$TARGET"- elif ln -sfn "$PRESET" "$TARGET" 2>/dev/null; then- echo "fetch_deepdoc_testdata: linked $TARGET -> $PRESET (pre-seeded)"- else- # Symlink creation failed (e.g. the parent directory is not writable on a- # shared runner). The pre-seeded fixtures are authoritative and readable,- # so copy them in place instead of reddening CI over a filesystem quirk.- echo "fetch_deepdoc_testdata: symlink failed; copying pre-seeded testdata for $PKG" >&2- mkdir -p "$TARGET"- cp -r "$PRESET"/. "$TARGET"/.+ # Regeneration rewrites the fixtures, so it needs a writable copy next to+ # the tests — which a pre-seeded runner cannot provide. Say so plainly+ # instead of failing later with a confusing permission error.+ if ! (rm -rf -- "$TARGET" && cp -r "$PRESET" "$TARGET") 2>/dev/null; then+ echo "fetch_deepdoc_testdata: GEN_* regeneration needs a writable $TARGET, but the workspace is read-only. Run it where $TARGET is writable, or drop RAGFLOW_TESTDATA_DIR to clone the fixtures." >&2+ exit 1+ fi+ echo "fetch_deepdoc_testdata: copied writable pre-seeded testdata for regeneration ($PKG)"+ exit 0fi- echo "fetch_deepdoc_testdata: done ($PKG, pre-seeded)"+ # Leave the workspace untouched: the consuming test binary reads the+ # fixtures from this directory directly (see testdata_skip_test.go).+ # Linking or copying them into $TARGET used to be required, and failed on+ # runners that mount the workspace read-only — which was then reported as a+ # missing fixture even though the data was right here.+ echo "fetch_deepdoc_testdata: using pre-seeded testdata at $PRESET (workspace untouched)"exit 0fiecho "fetch_deepdoc_testdata: RAGFLOW_TESTDATA_DIR set but deepdoc/$PKG/testdata is missing; falling back to clone" >&2