Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix: verify MinerU repository names in ModelScope cache

Bingxi Zhao (Frank)c129e9a0c5d0
diff --git a/deeptutor/services/parsing/engines/mineru/readiness.py b/deeptutor/services/parsing/engines/mineru/readiness.py
index 13664e0c0..6c022b290 100644
--- a/deeptutor/services/parsing/engines/mineru/readiness.py
+++ b/deeptutor/services/parsing/engines/mineru/readiness.py
@@ -16,8 +16,19 @@ from pathlib import Path
from ...base import ReadinessReport
-# Substrings of HF/ModelScope cache dir names that indicate MinerU's weights.
-_MODEL_DIR_HINTS = ("opendatalab", "mineru", "pdf-extract")
+# MinerU's ModelScope/Hugging Face repositories are published by OpenDataLab.
+# A populated OpenDataLab namespace alone can contain unrelated models.
+_MODEL_REPO_PREFIXES = ("mineru", "pdf-extract-kit")
+
+
+def _is_mineru_model_dir(path: Path) -> bool:
+ name = path.name.casefold()
+ if path.parent.name.casefold() == "opendatalab":
+ return name.startswith(_MODEL_REPO_PREFIXES)
+ for prefix in ("models--opendatalab--", "opendatalab--"):
+ if name.startswith(prefix):
+ return name[len(prefix) :].startswith(_MODEL_REPO_PREFIXES)
+ return False
def _hf_hub_dir() -> Path:
@@ -49,13 +60,17 @@ def mineru_models_ready(_source: str = "huggingface") -> bool:
if not root.is_dir():
continue
for child in root.iterdir():
- name = child.name.lower()
- if (
- child.is_dir()
- and any(hint in name for hint in _MODEL_DIR_HINTS)
- and any(child.iterdir())
- ):
- return True
+ if not child.is_dir():
+ continue
+ # ModelScope may use <cache>/models/OpenDataLab/<repo>.
+ candidates = child.iterdir() if child.name.casefold() == "opendatalab" else (child,)
+ for candidate in candidates:
+ if (
+ candidate.is_dir()
+ and _is_mineru_model_dir(candidate)
+ and any(candidate.iterdir())
+ ):
+ return True
except Exception:
continue
return False
diff --git a/tests/services/parsing/test_engines.py b/tests/services/parsing/test_engines.py
index 13514309a..84d6edfe5 100644
--- a/tests/services/parsing/test_engines.py
+++ b/tests/services/parsing/test_engines.py
@@ -184,6 +184,14 @@ def test_mineru_models_ready_supports_modelscope_cache_layouts(tmp_path, monkeyp
assert rd.mineru_models_ready() is True
+ namespaced_cache = tmp_path / "modelscope-namespaced"
+ namespaced_model = namespaced_cache / "models" / "OpenDataLab" / "PDF-Extract-Kit-1.0"
+ namespaced_model.mkdir(parents=True)
+ (namespaced_model / "config.json").write_text("{}", encoding="utf-8")
+ monkeypatch.setenv("MODELSCOPE_CACHE", str(namespaced_cache))
+
+ assert rd.mineru_models_ready() is True
+
flat_cache = tmp_path / "modelscope-flat"
flat_model = flat_cache / "OpenDataLab--PDF-Extract-Kit-1.0"
flat_model.mkdir(parents=True)
@@ -193,6 +201,24 @@ def test_mineru_models_ready_supports_modelscope_cache_layouts(tmp_path, monkeyp
assert rd.mineru_models_ready() is True
+def test_mineru_models_ready_ignores_unrelated_modelscope_repositories(
+ tmp_path, monkeypatch
+) -> None:
+ from deeptutor.services.parsing.engines.mineru import readiness as rd
+
+ monkeypatch.setenv("HF_HOME", str(tmp_path / "empty-huggingface"))
+ cache = tmp_path / "modelscope"
+ unrelated = cache / "models" / "OpenDataLab" / "unrelated-model"
+ unrelated.mkdir(parents=True)
+ (unrelated / "model.safetensors").write_bytes(b"weights")
+ misleading = cache / "models" / "AnotherPublisher--MinerU-demo"
+ misleading.mkdir()
+ (misleading / "model.safetensors").write_bytes(b"weights")
+ monkeypatch.setenv("MODELSCOPE_CACHE", str(cache))
+
+ assert rd.mineru_models_ready() is False
+
+
def test_mineru_cloud_readiness_needs_token() -> None:
from deeptutor.services.parsing.engines.mineru.config import MinerUConfig
from deeptutor.services.parsing.engines.mineru.readiness import mineru_readiness