FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(engine): raise WeightLoadError when the checkpoint cannot be read (#518)
Xiaoze Fan6410c97e15d7
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 06a74ee5..cc14cf53 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -1,5 +1,7 @@from __future__ import annotations+import contextlib+import errnoimport gcimport mathimport os@@ -272,6 +274,31 @@ def _make_dummy_weight_state_dict(return state_dict+class WeightLoadError(RuntimeError):+ """The checkpoint itself could not be read. Resource and config failures keep their own type."""+++def _is_resource_failure(exc: Exception) -> bool:+ if isinstance(exc, (torch.OutOfMemoryError, MemoryError, PinFailed)):+ return True+ # an anonymous mmap that does not fit raises ENOMEM, not MemoryError+ if isinstance(exc, OSError) and exc.errno == errno.ENOMEM:+ return True+ # torch has no type for a failed CPU allocation+ return isinstance(exc, RuntimeError) and "DefaultCPUAllocator" in str(exc)+++def _weight_load_context():+ """Wrap a checkpoint read so the startup failure reason (log and /health) starts with WeightLoadError."""+ try:+ yield+ except Exception as exc:+ if _is_resource_failure(exc):+ raise+ raise WeightLoadError(f"{type(exc).__name__}: {exc}") from exc++def _materialize_loaded_weight_state_dict(model_state: Dict[str, torch.Tensor],weights: Iterable[Tuple[str, torch.Tensor]],@@ -330,8 +357,7 @@ class Engine:set_rope_device(self.device)with torch.device("meta"), torch_dtype(config.dtype):self.model = create_model(config.model_config)- self.model.load_state_dict(self._load_weight_state_dict(config))- finalize_quant(self.model)+ self._load_weights(config)if config.active_encoders:from freetoken.models.blocks import SupportsMultimodal@@ -357,7 +383,8 @@ class Engine:# planning sees the pin quota the table already spent.self._host_tables_bytes = 0if hasattr(self.model, "load_host_tables"):- self._host_tables_bytes = int(self.model.load_host_tables(config) or 0)+ with _weight_load_context():+ self._host_tables_bytes = int(self.model.load_host_tables(config) or 0)if is_offload_moe_strategy(config.moe_strategy):self._init_offload_moe_cache(config)if hasattr(self.model, "prepare_for_runtime"):@@ -497,16 +524,21 @@ class Engine:assert tp_cpu_group is not Nonereturn tp_cpu_group- def _load_weight_state_dict(self, config: EngineConfig) -> Dict[str, torch.Tensor]:- model_state = self.model.state_dict()- if config.use_dummy_weight:- return _make_dummy_weight_state_dict(model_state, device=self.device)- if config.active_encoders and ftw_lacks_vision(config.model_path):+ def _load_weights(self, config: EngineConfig) -> None:+ if config.active_encoders and not config.use_dummy_weight and ftw_lacks_vision(config.model_path):raise ValueError(f"{config.model_path} holds no vision encoder tensors: it was converted by a build before this ""family served images. Reconvert it with `ft checkpoint`, add the encoder in place with ""scripts/ftw_hotfix.py (docs/ftw-hotfix.md), or start with --text-model-only")+ with _weight_load_context():+ self.model.load_state_dict(self._load_weight_state_dict(config))+ finalize_quant(self.model)++ def _load_weight_state_dict(self, config: EngineConfig) -> Dict[str, torch.Tensor]:+ model_state = self.model.state_dict()+ if config.use_dummy_weight:+ return _make_dummy_weight_state_dict(model_state, device=self.device)# _materialize casts each loaded tensor to its model-param dtype (model_state), so# models declaring per-tensor dtypes (e.g. DSV4's mixed fp8/fp32/bf16) are preserved;# offload models exclude experts (served from the offload cache, not dense weights).@@ -635,17 +667,18 @@ class Engine:for i in range(config.model_config.num_moe_layers)]try:- banks = load_expert_banks(- config.model_path,- config.model_config,- method=method,- device=self.device,- dtype=self.dtype,- dummy=config.use_dummy_weight,- parallel=expert_parallel,- decode_target=("cpu" if decode_target in ("cpu", "hybrid") else "gpu"),- layer_residency=requested_residency,- )+ with _weight_load_context():+ banks = load_expert_banks(+ config.model_path,+ config.model_config,+ method=method,+ device=self.device,+ dtype=self.dtype,+ dummy=config.use_dummy_weight,+ parallel=expert_parallel,+ decode_target=("cpu" if decode_target in ("cpu", "hybrid") else "gpu"),+ layer_residency=requested_residency,+ )except PinFailed as exc:raise RuntimeError(f"{exc}; {_pin_hint(self._host_tables_bytes)}") from excif config.moe_cache_auto:@@ -1368,7 +1401,8 @@ def _adjust_ftw_quant_backend(model_path: str, quant_backend: QuantBackend) -> Qfrom freetoken.checkpoint.ftw import ftw_quant_formatfrom freetoken.moe.legacy_format import kind_kernel_for- fmt = ftw_quant_format(model_path) if model_path else None+ with _weight_load_context():+ fmt = ftw_quant_format(model_path) if model_path else Noneif fmt is None:return quant_backendtry: