Fully automatic censorship removal for language models
fix: recognize root Hugging Face repo IDs (#325)
iuyua9551db26bb75e
diff --git a/src/heretic/utils.py b/src/heretic/utils.pyindex 27dd697..32f78b7 100644--- a/src/heretic/utils.py+++ b/src/heretic/utils.py@@ -22,6 +22,7 @@ from datasets import DatasetDict, ReadInstruction, load_dataset, load_from_diskfrom datasets.config import DATASET_STATE_JSON_FILENAMEfrom datasets.download.download_manager import DownloadModefrom datasets.utils.info_utils import VerificationMode+from huggingface_hub.utils import validate_repo_idfrom optuna import Trialfrom psutil import Processfrom questionary import Choice, Style@@ -172,13 +173,13 @@ def format_duration(seconds: float) -> str:def is_hf_path(path: str) -> bool:"""Checks whether a path likely refers to a Hugging Face repository."""- return (- not path.startswith("/")- and not path.endswith("/")- and path.count("/") == 1- and "\\" not in path- and not Path(path).exists()- )+ # Match Transformers: existing local paths take precedence over Hub lookup,+ # even if the path string is also a valid repository ID.+ if Path(path).exists():+ return False++ validate_repo_id(path)+ return True@dataclass