Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix ci unit test not found in installed package 'wordnet' (#19251)

maoyifeng5d0e2253cf7a
diff --git a/api/validation.py b/api/validation.py
index 65f65d991b..65f11b2957 100644
--- a/api/validation.py
+++ b/api/validation.py
@@ -33,8 +33,16 @@ python_version_validation()
# Download nltk data
def download_nltk_data():
+ import os
+
+ # NLTK >=3.10 refuses proxied downloads (SSRF guard) unless opted in.
+ os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")
+
import nltk
+ # NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` package, so
+ # both must be present or tokenization-backed paths raise LookupError.
+ nltk.download("omw-1.4", halt_on_error=False, quiet=True)
nltk.download("wordnet", halt_on_error=False, quiet=True)
nltk.download("punkt_tab", halt_on_error=False, quiet=True)
diff --git a/ragflow_deps/download_deps.py b/ragflow_deps/download_deps.py
index 8be9822aa0..d599a1f14e 100644
--- a/ragflow_deps/download_deps.py
+++ b/ragflow_deps/download_deps.py
@@ -35,6 +35,10 @@ import os
import shutil
import urllib.request
+# NLTK >=3.10 refuses proxied downloads (SSRF guard) unless opted in; the
+# runners sit behind a proxy, so allow proxied fetches before importing nltk.
+os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")
+
import nltk
from huggingface_hub import snapshot_download
@@ -254,7 +258,9 @@ if __name__ == "__main__":
print(f" Skipping onnxruntime static check: no .a found under {ort_static_dir}")
local_dir = os.path.abspath("nltk_data")
- for data in ["wordnet", "punkt", "punkt_tab"]:
+ # NLTK >=3.8.2 gates `wordnet` behind `omw-1.4`; both must be provisioned
+ # or tokenization-backed paths raise LookupError at runtime.
+ for data in ["omw-1.4", "wordnet", "punkt", "punkt_tab"]:
print(f"Downloading nltk {data}...")
nltk.download(data, download_dir=local_dir)
diff --git a/run_tests.py b/run_tests.py
index dc3cad603a..c15f9399ca 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -181,7 +181,10 @@ EXAMPLES:
def build_pytest_command(self) -> List[str]:
"""Build the pytest command arguments"""
- cmd = ["pytest"]
+ # Use the current interpreter (the venv python) to run pytest as a
+ # module so the run works even when the `pytest` console script is not
+ # on PATH (e.g. the venv was not explicitly activated).
+ cmd = [self.python, "-m", "pytest"]
if self.test_path:
test_target = Path(self.test_path)
if not test_target.is_absolute():
diff --git a/test/unit_test/conftest.py b/test/unit_test/conftest.py
index 4547f073f2..512398c591 100644
--- a/test/unit_test/conftest.py
+++ b/test/unit_test/conftest.py
@@ -29,22 +29,42 @@ when present, and download only what is still missing.
import os
+# NLTK >=3.10 refuses proxied downloads (SSRF guard, CWE-918) by default. The
+# CI runners and some dev boxes sit behind a proxy, so opt in to proxied fetches
+# before importing nltk; otherwise `nltk.download` fails with a Security
+# Violation. Must be set before the `import nltk` below so pathsec reads it.
+os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")
+
import nltk
+import warnings
# Reuse data already fetched by download_deps.py (the directory the app exports
-# as NLTK_DATA) so provisioned environments do not download it again.
+# as NLTK_DATA) so provisioned environments do not download it again. Create it
+# if absent so the fallback download below lands in a repo-local, reproducible
+# location instead of a shared home directory.
_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
_LOCAL_NLTK_DATA = os.path.join(_REPO_ROOT, "ragflow_deps", "nltk_data")
-if os.path.isdir(_LOCAL_NLTK_DATA) and _LOCAL_NLTK_DATA not in nltk.data.path:
+os.makedirs(_LOCAL_NLTK_DATA, exist_ok=True)
+if _LOCAL_NLTK_DATA not in nltk.data.path:
nltk.data.path.insert(0, _LOCAL_NLTK_DATA)
# (download name, resource path used by nltk.data.find)
+# NOTE: NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` data
+# package. Downloading `wordnet` alone leaves a stub `wordnet.zip` that raises
+# LookupError at load time; `omw-1.4` must also be present.
_REQUIRED_NLTK_DATA = (
("punkt_tab", "tokenizers/punkt_tab"),
("wordnet", "corpora/wordnet"),
+ ("omw-1.4", "corpora/omw-1.4"),
)
for _name, _find_path in _REQUIRED_NLTK_DATA:
try:
nltk.data.find(_find_path)
except LookupError:
- nltk.download(_name, quiet=True)
+ # On shared CI runners the download dir is often group-writable, which
+ # makes NLTK emit a "non-private download directory" UserWarning. pytest
+ # escalates warnings to errors (filterwarnings = error), so suppress it
+ # for the duration of the download and fetch into the repo-local dir.
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", UserWarning)
+ nltk.download(_name, download_dir=_LOCAL_NLTK_DATA, quiet=True)