PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix CWE-22 (path traversal) in checkpoint loading issues (#2389)
Liang Lvfe973526982d
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex 478bd29e0..d8de7c624 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -152,6 +152,7 @@ from auto_round.utils.model_free_utils import (is_model_free_supported_scheme,preprocess_model_type_source_tensors,)+from auto_round.utils.path_safety import resolve_within_directory# Backward-compat aliases for internal/private helper names used in tests and# downstream imports.@@ -326,7 +327,7 @@ def _prefetch_shard(# colliding with quantized output shard names in output_dir.shard_cache_dir = os.path.join(work_dir, ".cache", "model_free_source_shards")return _download_single_shard(model_name_or_path, shard_name, shard_cache_dir)- path = os.path.join(source_dir, shard_name)+ path = str(resolve_within_directory(source_dir, shard_name, origin="shard list"))return path if os.path.exists(path) else Noneexcept Exception as e: # pragma: no coverlogger.warning(f"Prefetch failed for {shard_name}: {e}")diff --git a/auto_round/export/export_to_gguf/conversion/base.py b/auto_round/export/export_to_gguf/conversion/base.pyindex db916b51e..b726465a4 100644--- a/auto_round/export/export_to_gguf/conversion/base.py+++ b/auto_round/export/export_to_gguf/conversion/base.py@@ -28,6 +28,8 @@ if 'NO_LOCAL_GGUF' not in os.environ:import gguffrom gguf.vocab import MistralTokenizerType, MistralVocab+from auto_round.utils.path_safety import resolve_within_directory, validate_weight_map+try:from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found, ty:unresolved-import]from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # type: ignore[import-not-found, ty:unresolved-import]@@ -240,6 +242,7 @@ class ModelBase:weight_map = index.get("weight_map")if weight_map is None or not isinstance(weight_map, dict):raise ValueError(f"Can't load 'weight_map' from {index_name!r}")+ weight_map = validate_weight_map(weight_map, self.dir_model, index_path=index_file)tensor_names_from_index.update(weight_map.keys())part_dict: dict[str, None] = dict.fromkeys(weight_map.values(), None) # ty: ignore[invalid-assignment]part_names = sorted(part_dict.keys())@@ -250,11 +253,12 @@ class ModelBase:for part_name in part_names:logger.debug(f"gguf: indexing model part '{part_name}'")+ part_path = resolve_within_directory(self.dir_model, part_name)ctx: ContextManager[Any]if is_safetensors:- ctx = cast(ContextManager[Any], gguf.utility.SafetensorsLocal(self.dir_model / part_name))+ ctx = cast(ContextManager[Any], gguf.utility.SafetensorsLocal(part_path))else:- ctx = contextlib.nullcontext(torch.load(str(self.dir_model / part_name), map_location="cpu", mmap=True, weights_only=True))+ ctx = contextlib.nullcontext(torch.load(str(part_path), map_location="cpu", mmap=True, weights_only=True))with ctx as model_part:assert model_part is not Nonediff --git a/auto_round/export/export_to_gguf/conversion/minimax.py b/auto_round/export/export_to_gguf/conversion/minimax.pyindex 53a9ff60f..ee78dcd27 100644--- a/auto_round/export/export_to_gguf/conversion/minimax.py+++ b/auto_round/export/export_to_gguf/conversion/minimax.py@@ -8,6 +8,7 @@ if TYPE_CHECKING:from torch import Tensorfrom .base import ModelBase, TextModel, MmprojModel, gguf, logger+from auto_round.utils.path_safety import resolve_within_directory@ModelBase.register("MiniMaxText01ForCausalLM")@@ -32,7 +33,8 @@ class MiniMaxText01Model(TextModel):embeddings_tensor_name = "model.embed_tokens.weight"embeddings_shard_name = weight_map[embeddings_tensor_name]- with gguf.utility.SafetensorsLocal(self.dir_model / embeddings_shard_name) as model_shard:+ embeddings_shard_path = resolve_within_directory(self.dir_model, embeddings_shard_name)+ with gguf.utility.SafetensorsLocal(embeddings_shard_path) as model_shard:embeddings_data = model_shard[embeddings_tensor_name]embeddings_weights_dtype = LazyTorchTensor._dtype_str_map[embeddings_data.dtype]diff --git a/auto_round/export/export_to_gguf/special_handle.py b/auto_round/export/export_to_gguf/special_handle.pyindex f93b6f008..513db51bb 100644--- a/auto_round/export/export_to_gguf/special_handle.py+++ b/auto_round/export/export_to_gguf/special_handle.py@@ -21,6 +21,7 @@ from safetensors import safe_openfrom torch import Tensorfrom auto_round.utils import LazyImport, download_or_get_path+from auto_round.utils.path_safety import resolve_within_directorygguf = LazyImport("gguf")@@ -103,7 +104,7 @@ def get_tensor_from_file(dir_path, tensor_name):filename = "model.safetensors"# get tensor- f = safe_open(os.path.join(dir_path, filename), framework="pt")+ f = safe_open(str(resolve_within_directory(dir_path, filename, origin=INDEX_FILE)), framework="pt")return f.get_tensor(tensor_name)diff --git a/auto_round/export/utils.py b/auto_round/export/utils.pyindex b834ea155..12cc289ee 100644--- a/auto_round/export/utils.py+++ b/auto_round/export/utils.py@@ -26,6 +26,7 @@ from auto_round.utils import (restore_fp32_tensors_from_source,unsupported_meta_device,)+from auto_round.utils.path_safety import UnsafeCheckpointPathError, resolve_within_directorydef save_pretrained_artifact(artifact, output_dir: str, artifact_name: str = "artifact") -> bool:@@ -156,10 +157,12 @@ def _resolve_model_source_dir(model: nn.Module) -> str | None:def _copy_pipeline_artifact(model_dir: str, relative_path: str, output_dir: str) -> None:- target_path = os.path.join(output_dir, relative_path)+ # ``relative_path`` comes from the pipeline's own model_index.json, so it must+ # stay inside output_dir (write side) and model_dir (read side).+ target_path = str(resolve_within_directory(output_dir, relative_path, origin="model_index.json"))os.makedirs(os.path.dirname(target_path), exist_ok=True)if is_local_pipeline_model_dir(model_dir):- source_path = os.path.join(model_dir, relative_path)+ source_path = str(resolve_within_directory(model_dir, relative_path, origin="model_index.json"))else:from huggingface_hub import hf_hub_download@@ -204,6 +207,11 @@ def _copy_pipeline_artifacts(source_dir: str, output_dir: str, exclude_componentfor component_name in component_dirs:if component_name in exclude_components:continue+ # Keys of model_index.json are pipeline kwargs; anything else would be joined as a path.+ if not component_name.isidentifier():+ raise UnsafeCheckpointPathError(+ f"model_index.json: component name {component_name!r} is not a plain directory name"+ )if is_local:src = os.path.join(source_dir, component_name)dst = os.path.join(output_dir, component_name)diff --git a/auto_round/utils/bagel_loader.py b/auto_round/utils/bagel_loader.pyindex a4fb42599..9f3b7271f 100644--- a/auto_round/utils/bagel_loader.py+++ b/auto_round/utils/bagel_loader.py@@ -33,6 +33,7 @@ from transformers import AutoTokenizer, PretrainedConfigfrom transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM, Qwen2MLP, Qwen2RMSNormfrom auto_round.logger import logger+from auto_round.utils.path_safety import resolve_within_directory, validate_weight_mapclass BagelConfig(PretrainedConfig):@@ -133,7 +134,7 @@ def _load_safetensors_weights(model_path):with open(index_path, "r", encoding="utf-8") as f:index = json.load(f)- weight_map = index.get("weight_map", {})+ weight_map = validate_weight_map(index.get("weight_map", {}), model_path, index_path=index_path)# Determine which shard files contain non-VAE weights# VAE weights: decoder.*, encoder.* (in ae.safetensors)@@ -150,7 +151,7 @@ def _load_safetensors_weights(model_path):for shard_file in lm_shard_files:if shard_file in loaded_files:continue- sf_path = os.path.join(model_path, shard_file)+ sf_path = str(resolve_within_directory(model_path, shard_file))if os.path.exists(sf_path):weights = load_file(sf_path, device="cpu")# Only keep non-VAE weights from this filediff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.pyindex 0a96312db..1ea98c06e 100644--- a/auto_round/utils/disk_stream_util.py+++ b/auto_round/utils/disk_stream_util.py@@ -22,6 +22,8 @@ import torch.nn as nnfrom accelerate.utils import set_module_tensor_to_devicefrom safetensors import SafetensorError, safe_open+from auto_round.utils.path_safety import resolve_within_directory, validate_weight_map+logger = logging.getLogger(__name__)@@ -43,7 +45,12 @@ class SafetensorsIndex:index_path = self.checkpoint_dir / "model.safetensors.index.json"if index_path.exists():with open(index_path) as f:- self.weight_map: Dict[str, str] = json.load(f)["weight_map"]+ # The shard names come from the checkpoint's own index, i.e. from the+ # artifact being loaded -- validate them against the directory before+ # anything downstream gets a chance to join and open them.+ self.weight_map: Dict[str, str] = validate_weight_map(+ json.load(f)["weight_map"], self.checkpoint_dir, index_path=index_path+ )else:# Small, unsharded checkpoint: one model.safetensors file.single_file = self.checkpoint_dir / "model.safetensors"@@ -59,7 +66,8 @@ class SafetensorsIndex:def tensor_shape(self, name: str) -> tuple[int, ...]:"""Return a tensor's shape from its safetensors header without reading its payload."""shard_name = self.weight_map[name]- with safe_open(str(self.checkpoint_dir / shard_name), framework="pt") as f:+ shard_path = resolve_within_directory(self.checkpoint_dir, shard_name)+ with safe_open(str(shard_path), framework="pt") as f:return tuple(f.get_slice(name).get_shape())def read_tensors(self, names: list[str], device: str = "cpu") -> Dict[str, torch.Tensor]:@@ -71,7 +79,8 @@ class SafetensorsIndex:result: Dict[str, torch.Tensor] = {}for shard_name, shard_tensor_names in by_shard.items():- with safe_open(str(self.checkpoint_dir / shard_name), framework="pt") as f:+ shard_path = resolve_within_directory(self.checkpoint_dir, shard_name)+ with safe_open(str(shard_path), framework="pt") as f:for name in shard_tensor_names:tensor = f.get_tensor(name)if device != "cpu":diff --git a/auto_round/utils/missing_tensors.py b/auto_round/utils/missing_tensors.pyindex 832aae485..82a912503 100644--- a/auto_round/utils/missing_tensors.py+++ b/auto_round/utils/missing_tensors.py@@ -50,6 +50,7 @@ from auto_round.utils.model_free_utils import (quantize_weight_rtn,split_fused_expert_tensors,)+from auto_round.utils.path_safety import resolve_within_directory, validate_weight_mapfrom auto_round.utils.weight_handler import _dequant_fp8_linear_weight@@ -167,8 +168,12 @@ def _resolve_tensor_shard_maps(if os.path.exists(source_index_file):with open(source_index_file) as f:src_index = json.load(f)- for tensor_name, shard_file in src_index["weight_map"].items():- source_tensor_to_file[tensor_name] = os.path.join(source_dir, shard_file)+ # Shard names come from the source checkpoint's own index: validate them+ # against source_dir, then resolve at the point of use below.+ for tensor_name, shard_file in validate_weight_map(+ src_index["weight_map"], source_dir, index_path=source_index_file+ ).items():+ source_tensor_to_file[tensor_name] = str(resolve_within_directory(source_dir, shard_file))elif os.path.exists(source_single_file):with safe_open(source_single_file, framework="pt", device="cpu") as f:for key in f.keys():diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 2f874158e..ae982495f 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -29,6 +29,7 @@ from auto_round import envsfrom auto_round.export.export_to_gguf.config import ModelTypefrom auto_round.logger import loggerfrom auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, monkey_patch_model+from auto_round.utils.path_safety import UnsafeCheckpointPathErrorfrom auto_round.utils.weight_handler import (_dequant_fp8_linear_weight,check_and_mark_quantized_module,@@ -605,6 +606,9 @@ def _find_pipeline_model_subfolder(model_dir_or_repo: str, file_list: list = Nonfor name, value in model_index.items():if name.startswith("_") or not isinstance(value, list) or len(value) < 2:continue+ # The chosen name becomes a read subfolder and later an output subfolder.+ if not name.isidentifier():+ raise UnsafeCheckpointPathError(f"model_index.json: component name {name!r} is not a plain directory name")# Load component config.jsonif is_local:cfg_path = os.path.join(model_dir_or_repo, name, "config.json")diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex 4e0379fe2..afd811b32 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -37,6 +37,11 @@ from auto_round.logger import loggerfrom auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_schemefrom auto_round.utils.common import _normalize_tensor_name_for_warning, to_standard_regexfrom auto_round.utils.device import clear_memory, compile_func+from auto_round.utils.path_safety import (+ resolve_within_directory,+ sanitize_shard_name,+ validate_weight_map,+)_NVFP4_E5M3_DATA_TYPE = "nvfp4_v2"_BLOCK_NAME_TO_IGNORE = ("shared_expert_gate.", ".gate.", "embed", "conv")@@ -1595,7 +1600,7 @@ def _hydrate_missing_fp8_scales_from_index(hydrated = 0shard_prefix = f"[{shard_name}] " if shard_name else ""for target_shard, scale_names in scales_by_shard.items():- target_path = os.path.join(donor_dir, target_shard)+ target_path = str(resolve_within_directory(donor_dir, target_shard, origin="weight_map"))if not os.path.exists(target_path):logger.warning(f"{shard_prefix}Donor shard '{target_shard}' not found in '{donor_dir}' while hydrating "@@ -1773,7 +1778,7 @@ def _hydrate_and_clean_modelopt_nvfp4_aux(hydrated = 0for target_shard, aux_names in wanted_by_shard.items():- target_path = os.path.join(donor_dir, target_shard)+ target_path = str(resolve_within_directory(donor_dir, target_shard, origin="weight_map"))if not os.path.exists(target_path):logger.warning(f"{shard_prefix}Donor shard '{target_shard}' not found in '{donor_dir}' while hydrating "@@ -2096,12 +2101,8 @@ def _process_shard(quantize_func = compile_func(quantize_weight_rtn, device) if enable_torch_compile else quantize_weight_rtnif shard_path.endswith(".bin"):- # PyTorch pickle checkpoint — load with weights_only where supported.- try:- raw_tensors = torch.load(shard_path, map_location="cpu", weights_only=True)- except TypeError:- # weights_only not available in older PyTorch versions- raw_tensors = torch.load(shard_path, map_location="cpu") # nosec+ # No unrestricted fallback: this pickle comes from an untrusted artifact.+ raw_tensors = torch.load(shard_path, map_location="cpu", weights_only=True)# Flatten nested state-dict wrappers if present.if not isinstance(raw_tensors, dict):raise ValueError(f"Expected a dict from {shard_path}, got {type(raw_tensors)}")@@ -2385,9 +2386,12 @@ def _list_weight_shards(source_dir: str) -> list[str]:def _shards_from_index(index_path: str) -> list[str]:with open(index_path) as f:index = json.load(f)+ # Shard names are declared by the checkpoint's own index and are later+ # joined onto source_dir / a shard cache dir, so contain them here.+ weight_map = validate_weight_map(index["weight_map"], source_dir, index_path=index_path)seen: set[str] = set()shards: list[str] = []- for shard_file in index["weight_map"].values():+ for shard_file in weight_map.values():if shard_file not in seen:seen.add(shard_file)shards.append(shard_file)@@ -2451,13 +2455,13 @@ def _download_single_shard() -> str:"""Download a single safetensors shard file. Returns the local path."""os.makedirs(local_dir, exist_ok=True)- local_path = os.path.join(local_dir, shard_filename)+ local_path = os.path.join(local_dir, sanitize_shard_name(shard_filename, origin="shard list"))if os.path.exists(local_path):logger.info(f"Shard '{shard_filename}' already exists at '{local_path}', skipping download.")return local_pathif os.path.isdir(model_name_or_path):- src = os.path.join(model_name_or_path, shard_filename)+ src = str(resolve_within_directory(model_name_or_path, shard_filename, origin="shard list"))if os.path.exists(src):shutil.copy2(src, local_path)return local_pathdiff --git a/auto_round/utils/offload.py b/auto_round/utils/offload.pyindex b5143a8e4..35d1f9ae2 100644--- a/auto_round/utils/offload.py+++ b/auto_round/utils/offload.py@@ -57,6 +57,7 @@ import torchfrom auto_round.logger import loggerfrom auto_round.utils.model import get_module+from auto_round.utils.path_safety import resolve_within_directory, validate_weight_map__all__ = ["OffloadManager"]@@ -226,7 +227,9 @@ def _build_weight_map(model_dir: str) -> dict[str, str]:index_path = os.path.join(model_dir, custom_indexes[0])if os.path.exists(index_path):with open(index_path) as f:- return json.load(f)["weight_map"]+ # Shard names are declared by the checkpoint itself; validate them+ # before any caller joins one onto model_dir.+ return validate_weight_map(json.load(f)["weight_map"], model_dir, index_path=index_path)single_path = os.path.join(model_dir, "model.safetensors")if os.path.exists(single_path):@@ -242,11 +245,12 @@ def _build_weight_map(model_dir: str) -> dict[str, str]:bin_index_path = os.path.join(model_dir, custom_indexes[0])if os.path.exists(bin_index_path):with open(bin_index_path) as f:- return json.load(f)["weight_map"]+ return validate_weight_map(json.load(f)["weight_map"], model_dir, index_path=bin_index_path)single_bin = os.path.join(model_dir, "pytorch_model.bin")if os.path.exists(single_bin):- state_dict = torch.load(single_bin, map_location="cpu")+ # No unrestricted fallback: this pickle comes from an untrusted artifact.+ state_dict = torch.load(single_bin, map_location="cpu", weights_only=True)return {k: "pytorch_model.bin" for k in state_dict.keys()}raise FileNotFoundError(@@ -301,7 +305,7 @@ def load_block_from_model_files(model_dir: str, block_name: str, block: torch.nnstate_dict = {}for shard_file, tensor_names in shard_to_tensors.items():- shard_path = os.path.join(model_dir, shard_file)+ shard_path = str(resolve_within_directory(model_dir, shard_file))if shard_file.endswith(".safetensors"):from safetensors import safe_open@@ -309,7 +313,7 @@ def load_block_from_model_files(model_dir: str, block_name: str, block: torch.nnfor name in tensor_names:state_dict[name[len(prefix) :]] = f.get_tensor(name)else:- full_state = torch.load(shard_path, map_location="cpu")+ full_state = torch.load(shard_path, map_location="cpu", weights_only=True)for name in tensor_names:if name in full_state:state_dict[name[len(prefix) :]] = full_state[name]diff --git a/auto_round/utils/path_safety.py b/auto_round/utils/path_safety.pynew file mode 100644index 000000000..c337aaf01--- /dev/null+++ b/auto_round/utils/path_safety.py@@ -0,0 +1,151 @@+# Copyright (C) 2026 Intel Corporation+# SPDX-License-Identifier: Apache-2.0++"""Containment checks for file paths declared by model artifacts.++A sharded checkpoint ships its own ``*.index.json`` mapping tensor names to shard+file names. That mapping is *attacker-controlled input*: a model directory or a+Hugging Face repo is a downloaded artifact, not trusted local state. Joining such+a shard name onto a directory and opening it verbatim lets the artifact escape its+own directory (CWE-22): ``dir / "../../secret.safetensors"`` climbs out of ``dir``,+and ``dir / "/abs/path.safetensors"`` *replaces* ``dir`` entirely -- for+``pathlib``'s ``/`` and ``os.path.join`` alike. The same artifact can also point a+shard at a FIFO or device node and block the reader forever.++Every read of an artifact-declared checkpoint file therefore goes through+:func:`resolve_within_directory` (or :func:`validate_weight_map` for a whole+``weight_map``), which enforces:++* the declared name may not be absolute,+* after lexical normalisation it must stay inside the base directory, and+* if it exists it must be a regular file (symlinks followed), so a FIFO / device+ node declared by the artifact cannot stall or divert the reader.++Containment is checked on the declared *name*, not on symlink targets: the+Hugging Face cache links every snapshot file to ``../../blobs/<sha>``, so+rejecting out-of-directory link targets would reject every Hub download. A+symlink the artifact ships is trusted exactly like any other file it ships,+which is also what ``transformers`` / ``safetensors`` do when loading it.++Names whose target does not exist are returned as-is, so callers keep their+existing missing-file handling. Validation is deliberately paired with+resolution *at the point of use*: validating the map fails fast with a clear+error, and resolving again at open time means a call site cannot accidentally+bypass the check by joining the raw name itself.+"""++from __future__ import annotations++import os+from pathlib import Path, PureWindowsPath+from typing import Any, Dict++__all__ = [+ "UnsafeCheckpointPathError",+ "sanitize_shard_name",+ "resolve_within_directory",+ "validate_weight_map",+]+++class UnsafeCheckpointPathError(ValueError):+ """Raised when a checkpoint artifact declares a path outside its own directory."""+++def _looks_absolute(relative_path: str) -> bool:+ """True for POSIX-absolute paths and for Windows drive/UNC prefixes.++ ``C:/weights/x.safetensors`` is not absolute on POSIX, but joining it would+ still be interpreted as a drive path by a Windows consumer of the same+ checkpoint, so both spellings are rejected.+ """+ return os.path.isabs(relative_path) or PureWindowsPath(relative_path).is_absolute()+++def sanitize_shard_name(shard_name: Any, *, origin: str = "weight_map") -> str:+ """Validate one artifact-declared shard name without touching the filesystem.++ Rejects the cases where a join stops being a lookup *inside* the directory:+ non-strings, empty names, embedded NUL bytes, and absolute paths. ``..``+ escapes are caught by :func:`resolve_within_directory`, which is the+ only place that knows the directory to compare against.+ """+ if not isinstance(shard_name, str) or not shard_name:+ raise UnsafeCheckpointPathError(f"{origin}: shard name must be a non-empty string, got {shard_name!r}")+ if "\x00" in shard_name:+ raise UnsafeCheckpointPathError(f"{origin}: shard name contains a NUL byte: {shard_name!r}")+ if _looks_absolute(shard_name):+ raise UnsafeCheckpointPathError(+ f"{origin}: shard name {shard_name!r} is an absolute path; shard references must be "+ "relative to the checkpoint directory"+ )+ return shard_name+++def resolve_within_directory(+ base_dir: str | os.PathLike,+ relative_path: Any,+ *,+ origin: str = "weight_map",+) -> Path:+ """Resolve an artifact-declared path to a file contained in *base_dir*.++ Args:+ base_dir: Directory the artifact is allowed to read from (the checkpoint+ or output directory the reference is relative to).+ relative_path: The path as declared by the artifact.+ origin: Human-readable label used in error messages, e.g.+ ``"model.safetensors.index.json[model.layers.0.weight]"``.++ Returns:+ The resolved absolute path. Sub-directories inside *base_dir* are+ allowed; anything that escapes it is not.++ Raises:+ UnsafeCheckpointPathError: The path is absolute, escapes *base_dir*, or+ points at something that is not a regular file.+ """+ name = sanitize_shard_name(relative_path, origin=origin)+ base = Path(os.path.abspath(base_dir))+ # Callers open the returned normalised path, so ``link/..`` cannot be re-expanded by the OS.+ resolved = Path(os.path.normpath(base / name))+ if resolved != base and not resolved.is_relative_to(base):+ raise UnsafeCheckpointPathError(+ f"{origin}: path {name!r} resolves to {str(resolved)!r}, outside {str(base)!r}; " "refusing to open it"+ )+ # A FIFO (or device node) is not a weight file, and opening one blocks the+ # reader until a writer shows up -- a trivially reachable denial of service+ # from a crafted index. Names that do not exist are left for the caller.+ if resolved.exists() and not resolved.is_file():+ raise UnsafeCheckpointPathError(+ f"{origin}: path {name!r} resolves to {str(resolved)!r}, which is not a regular file; "+ "refusing to open it"+ )+ return resolved+++def validate_weight_map(+ weight_map: Any,+ base_dir: str | os.PathLike,+ *,+ index_path: str | os.PathLike | None = None,+) -> Dict[str, str]:+ """Validate every shard reference in an artifact-provided ``weight_map``.++ Returns a new dict with the same keys and the same relative shard names, so it+ is a drop-in replacement for ``json.load(...)["weight_map"]``. Raises+ :class:`UnsafeCheckpointPathError` naming the offending tensor, which turns a+ malicious artifact into one clear error at load time instead of a confusing+ ``FileNotFoundError`` -- or, worse, a silent success -- at open time.+ """+ label = Path(index_path).name if index_path is not None else "weight_map"+ if not isinstance(weight_map, dict):+ raise UnsafeCheckpointPathError(+ f"{label}: 'weight_map' must be a JSON object mapping tensor names to shards, "+ f"got {type(weight_map).__name__}"+ )+ validated: Dict[str, str] = {}+ for tensor_name, shard_name in weight_map.items():+ resolve_within_directory(base_dir, shard_name, origin=f"{label}[{tensor_name}]")+ validated[tensor_name] = shard_name+ return validateddiff --git a/test/unit/common/utils/test_path_safety.py b/test/unit/common/utils/test_path_safety.pynew file mode 100644index 000000000..0159c5bcb--- /dev/null+++ b/test/unit/common/utils/test_path_safety.py@@ -0,0 +1,377 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Tests for checkpoint shard-path containment (CWE-22).++A sharded checkpoint declares its own shard file names in+``model.safetensors.index.json``. Those names are attacker-controlled input, so+they must never be joined onto the checkpoint directory and opened verbatim: a+``../`` value climbs out of the directory and an absolute value replaces it,+which lets the model read (and inject) any file on the victim's filesystem.+"""++import json+import os+from pathlib import Path++import pytest+import torch+import torch.nn as nn+from safetensors.torch import save_file++from auto_round.utils.disk_stream_util import SafetensorsIndex+from auto_round.utils.offload import load_block_from_model_files+from auto_round.utils.path_safety import (+ UnsafeCheckpointPathError,+ resolve_within_directory,+ sanitize_shard_name,+ validate_weight_map,+)+++def _write_index(model_dir: Path, weight_map: dict) -> None:+ (model_dir / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map}))+++def _make_outside_secrets(root: Path) -> Path:+ """Create weight files *outside* any model directory (the attack target)."""+ secret_dir = root / "SECRET_outside_model"+ secret_dir.mkdir(parents=True, exist_ok=True)+ save_file({"stolen.weight": torch.full((4,), 1337.0)}, str(secret_dir / "secret.safetensors"))+ torch.save({"blk.layer.weight": torch.full((1, 4), 999.0)}, str(secret_dir / "secret.bin"))+ return secret_dir+++class _Block(nn.Module):+ def __init__(self):+ super().__init__()+ self.layer = nn.Linear(4, 1)+++# ---------------------------------------------------------------------------+# sanitize_shard_name+# ---------------------------------------------------------------------------+++ "name",+ [+ "model-00001-of-00002.safetensors",+ "model.safetensors",+ "./model.safetensors",+ "nested/shard.safetensors",+ ],+)+def test_sanitize_shard_name_accepts_relative_names(name):+ assert sanitize_shard_name(name) == name+++ "name",+ [+ "",+ None,+ 42,+ "/etc/passwd",+ "/abs/model.safetensors",+ "C:\\weights\\model.safetensors",+ "\\\\server\\share\\model.safetensors",+ "model\x00.safetensors",+ ],+)+def test_sanitize_shard_name_rejects_unsafe_values(name):+ with pytest.raises(UnsafeCheckpointPathError):+ sanitize_shard_name(name)+++# ---------------------------------------------------------------------------+# resolve_within_directory+# ---------------------------------------------------------------------------+++def test_resolve_allows_file_inside_directory(tmp_path):+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ shard = model_dir / "model-00001-of-00002.safetensors"+ shard.write_bytes(b"")++ assert resolve_within_directory(model_dir, "model-00001-of-00002.safetensors") == shard+++def test_resolve_allows_subdirectory_inside_directory(tmp_path):+ model_dir = tmp_path / "model"+ (model_dir / "nested").mkdir(parents=True)+ shard = model_dir / "nested" / "shard.safetensors"+ shard.write_bytes(b"")++ assert resolve_within_directory(model_dir, "nested/shard.safetensors") == shard+++def test_resolve_allows_missing_file_for_caller_side_handling(tmp_path):+ """A shard that is simply absent must not raise: callers keep their own+ missing-file handling (warnings, partial downloads, shard scheduling)."""+ model_dir = tmp_path / "model"+ model_dir.mkdir()++ resolved = resolve_within_directory(model_dir, "model-00002-of-00002.safetensors")++ assert not resolved.exists()+ assert resolved.is_relative_to(model_dir)+++ "shard_name",+ ["..", "../secret.safetensors", "../../secret.safetensors", "nested/../../secret.safetensors"],+)+def test_resolve_rejects_relative_traversal(tmp_path, shard_name):+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ (tmp_path / "secret.safetensors").write_bytes(b"")++ with pytest.raises(UnsafeCheckpointPathError):+ resolve_within_directory(model_dir, shard_name)+++def test_resolve_rejects_absolute_path(tmp_path):+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ outside = tmp_path / "secret.safetensors"+ outside.write_bytes(b"")++ with pytest.raises(UnsafeCheckpointPathError):+ resolve_within_directory(model_dir, str(outside))+++def test_resolve_allows_hf_cache_snapshot_symlinks(tmp_path):+ """HF cache snapshots link every file to ``../../blobs/<sha>``; those links+ point outside the snapshot directory and must still be accepted."""+ repo = tmp_path / "models--org--name"+ snapshot = repo / "snapshots" / "abc123"+ snapshot.mkdir(parents=True)+ (repo / "blobs").mkdir()+ (repo / "blobs" / "deadbeef").write_bytes(b"")+ try:+ (snapshot / "model-00001-of-00001.safetensors").symlink_to("../../blobs/deadbeef")+ except (OSError, NotImplementedError):+ pytest.skip("symlinks are not available on this platform")++ resolved = resolve_within_directory(snapshot, "model-00001-of-00001.safetensors")++ assert resolved == snapshot / "model-00001-of-00001.safetensors"+ assert validate_weight_map({"w": "model-00001-of-00001.safetensors"}, snapshot)+++def test_resolve_allows_symlink_pointing_inside(tmp_path):+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ target = model_dir / "real.safetensors"+ target.write_bytes(b"")+ try:+ (model_dir / "link.safetensors").symlink_to(target)+ except (OSError, NotImplementedError):+ pytest.skip("symlinks are not available on this platform")++ assert resolve_within_directory(model_dir, "link.safetensors") == model_dir / "link.safetensors"+++def test_resolve_rejects_non_regular_file(tmp_path):+ """A FIFO declared as a shard would block the reader indefinitely, so it is+ rejected before anything tries to open it."""+ if not hasattr(os, "mkfifo"):+ pytest.skip("mkfifo is POSIX-only")+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ os.mkfifo(model_dir / "shard.safetensors")++ with pytest.raises(UnsafeCheckpointPathError, match="not a regular file"):+ resolve_within_directory(model_dir, "shard.safetensors")+++def test_resolve_rejects_directory(tmp_path):+ model_dir = tmp_path / "model"+ (model_dir / "shard.safetensors").mkdir(parents=True)++ with pytest.raises(UnsafeCheckpointPathError, match="not a regular file"):+ resolve_within_directory(model_dir, "shard.safetensors")+++# ---------------------------------------------------------------------------+# validate_weight_map+# ---------------------------------------------------------------------------+++def test_validate_weight_map_rejects_non_dict(tmp_path):+ with pytest.raises(UnsafeCheckpointPathError):+ validate_weight_map(["not", "a", "mapping"], tmp_path)+++def test_validate_weight_map_rejects_traversal_and_names_tensor(tmp_path):+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ (tmp_path / "secret.safetensors").write_bytes(b"")++ with pytest.raises(UnsafeCheckpointPathError, match="model.layers.0.self_attn.q_proj.weight"):+ validate_weight_map(+ {"model.layers.0.self_attn.q_proj.weight": "../secret.safetensors"},+ model_dir,+ index_path=model_dir / "model.safetensors.index.json",+ )+++def test_validate_weight_map_is_a_drop_in_replacement(tmp_path):+ """Callers join the returned names themselves, so the relative names have to+ survive validation unchanged."""+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ weight_map = {"a.weight": "model-00001-of-00002.safetensors", "b.weight": "nested/b.safetensors"}++ assert validate_weight_map(weight_map, model_dir) == weight_map+++# ---------------------------------------------------------------------------+# SafetensorsIndex -- the disk-streaming shard reader+# ---------------------------------------------------------------------------++[email protected]("shard_value", ["relative", "absolute"])+def test_safetensors_index_rejects_traversal(tmp_path, shard_value):+ secret_dir = _make_outside_secrets(tmp_path)+ model_dir = tmp_path / "malicious_model"+ model_dir.mkdir()+ if shard_value == "relative":+ shard_name = "../SECRET_outside_model/secret.safetensors"+ else:+ shard_name = str(secret_dir / "secret.safetensors")+ _write_index(model_dir, {"stolen.weight": shard_name})++ with pytest.raises(UnsafeCheckpointPathError):+ SafetensorsIndex(str(model_dir))+++def test_safetensors_index_reads_legit_shard(tmp_path):+ """Positive control: an ordinary sharded checkpoint must still load."""+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ value = torch.full((4,), 7.0)+ save_file({"model.embed_tokens.weight": value}, str(model_dir / "model-00001-of-00001.safetensors"))+ _write_index(model_dir, {"model.embed_tokens.weight": "model-00001-of-00001.safetensors"})++ index = SafetensorsIndex(str(model_dir))+ tensor = index.read_tensor("model.embed_tokens.weight")++ assert torch.equal(tensor, value)+++def test_safetensors_index_reads_unsharded_checkpoint(tmp_path):+ """Positive control for the single-file path, whose weight map is built from+ the file's own contents rather than from an index."""+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ value = torch.full((4,), 3.0)+ save_file({"model.embed_tokens.weight": value}, str(model_dir / "model.safetensors"))++ assert torch.equal(SafetensorsIndex(str(model_dir)).read_tensor("model.embed_tokens.weight"), value)+++def test_safetensors_index_tensor_shape_is_contained(tmp_path):+ """``tensor_shape`` must go through the same containment as ``read_tensors``."""+ _make_outside_secrets(tmp_path)+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ save_file({"a.weight": torch.zeros(2, 3)}, str(model_dir / "shard.safetensors"))+ _write_index(model_dir, {"a.weight": "shard.safetensors"})+ index = SafetensorsIndex(str(model_dir))++ assert index.tensor_shape("a.weight") == (2, 3)++ index.weight_map["stolen.weight"] = "../SECRET_outside_model/secret.safetensors"+ with pytest.raises(UnsafeCheckpointPathError):+ index.tensor_shape("stolen.weight")+++# ---------------------------------------------------------------------------+# load_block_from_model_files -- the offloaded-reload reader+# ---------------------------------------------------------------------------+++def test_load_block_from_model_files_rejects_traversal(tmp_path):+ _make_outside_secrets(tmp_path)+ model_dir = tmp_path / "malicious_model"+ model_dir.mkdir()+ _write_index(model_dir, {"blk.layer.weight": "../SECRET_outside_model/secret.bin"})++ block = _Block()+ before = block.layer.weight.detach().clone()++ with pytest.raises(UnsafeCheckpointPathError):+ load_block_from_model_files(str(model_dir), "blk.layer", block.layer)++ assert torch.equal(block.layer.weight.detach(), before), "weights must not be replaced from outside the model dir"+++def test_load_block_from_model_files_loads_legit_shard(tmp_path):+ """Positive control: reloading a block from its own shard still works."""+ model_dir = tmp_path / "model"+ model_dir.mkdir()+ value = torch.full((1, 4), 5.0)+ save_file({"blk.layer.weight": value}, str(model_dir / "shard.safetensors"))+ _write_index(model_dir, {"blk.layer.weight": "shard.safetensors"})++ block = _Block()+ load_block_from_model_files(str(model_dir), "blk.layer", block.layer)++ assert torch.equal(block.layer.weight.detach(), value)+++# ---------------------------------------------------------------------------+# GGUF export and pipeline artifact copy+# ---------------------------------------------------------------------------+++def test_gguf_get_tensor_from_file_rejects_traversal(tmp_path):+ from auto_round.export.export_to_gguf.special_handle import get_tensor_from_file++ _make_outside_secrets(tmp_path)+ model_dir = tmp_path / "malicious_model"+ model_dir.mkdir()+ _write_index(model_dir, {"stolen.weight": "../SECRET_outside_model/secret.safetensors"})++ with pytest.raises(UnsafeCheckpointPathError):+ get_tensor_from_file(str(model_dir), "stolen.weight")++[email protected]("component_name", ["../../outside", "/abs/outside", "a/b", ".."])+def test_copy_pipeline_artifacts_rejects_unsafe_component_name(tmp_path, component_name):+ from auto_round.export.utils import _copy_pipeline_artifacts++ source_dir = tmp_path / "pipe"+ source_dir.mkdir()+ (source_dir / "model_index.json").write_text(json.dumps({component_name: ["diffusers", "X"]}))++ with pytest.raises(UnsafeCheckpointPathError):+ _copy_pipeline_artifacts(str(source_dir), str(tmp_path / "out"))+++def test_find_pipeline_model_subfolder_rejects_traversal(tmp_path):+ from auto_round.utils.model import _find_pipeline_model_subfolder++ outside = tmp_path / "outside"+ outside.mkdir()+ (outside / "config.json").write_text(json.dumps({"architectures": ["EvilForCausalLM"]}))+ source_dir = tmp_path / "pipe"+ source_dir.mkdir()+ (source_dir / "model_index.json").write_text(json.dumps({"../outside": ["transformers", "X"]}))++ with pytest.raises(UnsafeCheckpointPathError):+ _find_pipeline_model_subfolder(str(source_dir))