RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix ci unit test not found in installed package 'wordnet' (#19251)
maoyifeng5d0e2253cf7a
diff --git a/api/validation.py b/api/validation.pyindex 65f65d991b..65f11b2957 100644--- a/api/validation.py+++ b/api/validation.py@@ -33,8 +33,16 @@ python_version_validation()# Download nltk datadef download_nltk_data():+ import os++ # NLTK >=3.10 refuses proxied downloads (SSRF guard) unless opted in.+ os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")+import nltk+ # NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` package, so+ # both must be present or tokenization-backed paths raise LookupError.+ nltk.download("omw-1.4", halt_on_error=False, quiet=True)nltk.download("wordnet", halt_on_error=False, quiet=True)nltk.download("punkt_tab", halt_on_error=False, quiet=True)diff --git a/ragflow_deps/download_deps.py b/ragflow_deps/download_deps.pyindex 8be9822aa0..d599a1f14e 100644--- a/ragflow_deps/download_deps.py+++ b/ragflow_deps/download_deps.py@@ -35,6 +35,10 @@ import osimport shutilimport urllib.request+# NLTK >=3.10 refuses proxied downloads (SSRF guard) unless opted in; the+# runners sit behind a proxy, so allow proxied fetches before importing nltk.+os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")+import nltkfrom huggingface_hub import snapshot_download@@ -254,7 +258,9 @@ if __name__ == "__main__":print(f" Skipping onnxruntime static check: no .a found under {ort_static_dir}")local_dir = os.path.abspath("nltk_data")- for data in ["wordnet", "punkt", "punkt_tab"]:+ # NLTK >=3.8.2 gates `wordnet` behind `omw-1.4`; both must be provisioned+ # or tokenization-backed paths raise LookupError at runtime.+ for data in ["omw-1.4", "wordnet", "punkt", "punkt_tab"]:print(f"Downloading nltk {data}...")nltk.download(data, download_dir=local_dir)diff --git a/run_tests.py b/run_tests.pyindex dc3cad603a..c15f9399ca 100755--- a/run_tests.py+++ b/run_tests.py@@ -181,7 +181,10 @@ EXAMPLES:def build_pytest_command(self) -> List[str]:"""Build the pytest command arguments"""- cmd = ["pytest"]+ # Use the current interpreter (the venv python) to run pytest as a+ # module so the run works even when the `pytest` console script is not+ # on PATH (e.g. the venv was not explicitly activated).+ cmd = [self.python, "-m", "pytest"]if self.test_path:test_target = Path(self.test_path)if not test_target.is_absolute():diff --git a/test/unit_test/conftest.py b/test/unit_test/conftest.pyindex 4547f073f2..512398c591 100644--- a/test/unit_test/conftest.py+++ b/test/unit_test/conftest.py@@ -29,22 +29,42 @@ when present, and download only what is still missing.import os+# NLTK >=3.10 refuses proxied downloads (SSRF guard, CWE-918) by default. The+# CI runners and some dev boxes sit behind a proxy, so opt in to proxied fetches+# before importing nltk; otherwise `nltk.download` fails with a Security+# Violation. Must be set before the `import nltk` below so pathsec reads it.+os.environ.setdefault("NLTK_ALLOW_PROXIED_URLOPEN", "1")+import nltk+import warnings# Reuse data already fetched by download_deps.py (the directory the app exports-# as NLTK_DATA) so provisioned environments do not download it again.+# as NLTK_DATA) so provisioned environments do not download it again. Create it+# if absent so the fallback download below lands in a repo-local, reproducible+# location instead of a shared home directory._REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))_LOCAL_NLTK_DATA = os.path.join(_REPO_ROOT, "ragflow_deps", "nltk_data")-if os.path.isdir(_LOCAL_NLTK_DATA) and _LOCAL_NLTK_DATA not in nltk.data.path:+os.makedirs(_LOCAL_NLTK_DATA, exist_ok=True)+if _LOCAL_NLTK_DATA not in nltk.data.path:nltk.data.path.insert(0, _LOCAL_NLTK_DATA)# (download name, resource path used by nltk.data.find)+# NOTE: NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` data+# package. Downloading `wordnet` alone leaves a stub `wordnet.zip` that raises+# LookupError at load time; `omw-1.4` must also be present._REQUIRED_NLTK_DATA = (("punkt_tab", "tokenizers/punkt_tab"),("wordnet", "corpora/wordnet"),+ ("omw-1.4", "corpora/omw-1.4"),)for _name, _find_path in _REQUIRED_NLTK_DATA:try:nltk.data.find(_find_path)except LookupError:- nltk.download(_name, quiet=True)+ # On shared CI runners the download dir is often group-writable, which+ # makes NLTK emit a "non-private download directory" UserWarning. pytest+ # escalates warnings to errors (filterwarnings = error), so suppress it+ # for the duration of the download and fetch into the repo-local dir.+ with warnings.catch_warnings():+ warnings.simplefilter("ignore", UserWarning)+ nltk.download(_name, download_dir=_LOCAL_NLTK_DATA, quiet=True)