Python
refactored linear splitting to support any qtype
DeepBeepMeep5d48f3e378ab
diff --git a/models/qwen/qwen_main.py b/models/qwen/qwen_main.pyindex c52a269..808d225 100644--- a/models/qwen/qwen_main.py+++ b/models/qwen/qwen_main.py@@ -26,10 +26,6 @@ _QWEN_FUSED_SPLIT_MAP = {}-from shared.qtypes import nunchaku_int4 as _nunchaku_int4-_split_nunchaku_fused_qkv = _nunchaku_int4.make_nunchaku_splitter(_QWEN_FUSED_SPLIT_MAP)--def stitch_images(img1, img2):# Resize img2 to match img1's heightwidth1, height1 = img1.size@@ -91,9 +87,9 @@ class model_factory():source = model_def.get("source", None)if source is not None:- offload.load_model_data(transformer, source, preprocess_sd=_split_nunchaku_fused_qkv)+ offload.load_model_data(transformer, source, fused_split_map=_QWEN_FUSED_SPLIT_MAP)else:- offload.load_model_data(transformer, transformer_filename, preprocess_sd=_split_nunchaku_fused_qkv)+ offload.load_model_data(transformer, transformer_filename, fused_split_map=_QWEN_FUSED_SPLIT_MAP)# transformer = offload.fast_load_transformers_model("transformer_quanto.safetensors", writable_tensors= True , modelClass=QwenImageTransformer2DModel, defaultConfigPath="transformer_config.json")if not source is None:diff --git a/models/z_image/z_image_main.py b/models/z_image/z_image_main.pyindex cef551c..435fbf7 100644--- a/models/z_image/z_image_main.py+++ b/models/z_image/z_image_main.py@@ -20,8 +20,6 @@ def conv_state_dict(sd: dict) -> dict:if "x_embedder.weight" not in sd and "model.diffusion_model.x_embedder.weight" not in sd:return sd- from shared.qtypes.gguf import GGUFSourceTensor-inverse_replace = {"final_layer.": "all_final_layer.2-1.","x_embedder.": "all_x_embedder.2-1.",@@ -36,60 +34,22 @@ def conv_state_dict(sd: dict) -> dict:for key, tensor in sd.items():key = key.replace("model.diffusion_model.", "")- if key.endswith(".attention.qkv.weight"):- base = key[: -len(".attention.qkv.weight")]-- orig_shape = getattr(tensor, "tensor_shape", None)- total_dim = orig_shape[0] if orig_shape is not None else tensor.shape[0]- if total_dim % 3 != 0:- raise ValueError(- f"{key}: qkv first dimension ({total_dim}) not divisible by 3"- )- d = total_dim // 3- if isinstance(tensor, GGUFSourceTensor) or getattr(tensor, "tensor_type", None) is not None:- chunks = torch.split(tensor, d, dim=0)- tensor_type = getattr(tensor, "tensor_type", None)- tensor_shape = (d, orig_shape[1]) if orig_shape is not None else (d, tensor.shape[1])- q, k_w, v = [- GGUFSourceTensor.wrap(chunk, tensor_type=tensor_type, tensor_shape=tensor_shape)- for chunk in chunks- ]- else:- q, k_w, v = torch.split(tensor, d, dim=0)-- out_sd[base + ".attention.to_q.weight"] = q- out_sd[base + ".attention.to_k.weight"] = k_w- out_sd[base + ".attention.to_v.weight"] = v- continue-new_key = key- for comfy_sub, orig_sub in inverse_replace.items():- new_key = new_key.replace(comfy_sub, orig_sub)+ for ori_sub, orig_sub in inverse_replace.items():+ new_key = new_key.replace(ori_sub, orig_sub)out_sd[new_key] = tensor- to_add = {}- for key, tensor in out_sd.items():- if key.endswith(".attention.to_out.0.weight"):- prefix = key[: -len(".attention.to_out.0.weight")]- bias_key = prefix + ".attention.to_out.0.bias"- if bias_key not in out_sd:- to_add[bias_key] = torch.zeros(tensor.shape[0], dtype=tensor.dtype)-- out_sd.update(to_add)return out_sd_ZIMAGE_FUSED_SPLIT_MAP = {"attention.to_qkv": {"mapped_modules": ("attention.to_q", "attention.to_k", "attention.to_v")},+ "attention.qkv": {"mapped_modules": ("attention.to_q", "attention.to_k", "attention.to_v")},"feed_forward.net.0.proj": {"mapped_modules": ("feed_forward.w3", "feed_forward.w1")},"feed_forward.net.2": {"mapped_modules": ("feed_forward.w2",)},}-from shared.qtypes import nunchaku_int4 as _nunchaku_int4-_split_nunchaku_fused = _nunchaku_int4.make_nunchaku_splitter(_ZIMAGE_FUSED_SPLIT_MAP)--class model_factory:def __init__(self,@@ -124,12 +84,11 @@ class model_factory:default_transformer_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), "configs", f"{base_model_type}.json")def preprocess_sd(state_dict):- state_dict = conv_state_dict(state_dict)- return _split_nunchaku_fused(state_dict)+ return conv_state_dict(state_dict)model_class = ZImageTransformer2DModel- kwargs_light= { "writable_tensors": False, "preprocess_sd": preprocess_sd }+ kwargs_light= { "writable_tensors": False, "preprocess_sd": preprocess_sd, "fused_split_map": _ZIMAGE_FUSED_SPLIT_MAP }# model_filename contains all files to load (transformer + modules merged by loader)import jsonimport accelerate@@ -181,7 +140,7 @@ class model_factory:# VAEvae_filename = fl.locate_file("ZImageTurbo_VAE_bf16.safetensors")- vae_config_path = os.path.join(os.path.dirname(vae_filename), "ZImageTurbo_VAE_bf16_config.json")+ vae_config_path = fl.locate_file("ZImageTurbo_VAE_bf16_config.json")vae = offload.fast_load_transformers_model(vae_filename,diff --git a/shared/qtypes/gguf.py b/shared/qtypes/gguf.pyindex 6e5ab58..be290a5 100644--- a/shared/qtypes/gguf.py+++ b/shared/qtypes/gguf.py@@ -288,6 +288,43 @@ class GGUFSourceTensor(torch.Tensor):torch.utils.swap_tensors(self, data)+def _split_gguf_tensor(src, *, dim, split_sizes, context):+ if not torch.is_tensor(src):+ return None+ tensor_type = getattr(src, "tensor_type", None)+ if tensor_type is None:+ return None+ tensor_shape = getattr(src, "tensor_shape", None) or src.shape+ total = sum(split_sizes)+ if dim >= len(tensor_shape) or tensor_shape[dim] != total:+ return None+ chunks = torch.split(src, split_sizes, dim=dim)+ out = []+ for chunk, size in zip(chunks, split_sizes):+ new_shape = list(tensor_shape)+ new_shape[dim] = size+ wrapped = GGUFSourceTensor.wrap(chunk, tensor_type=tensor_type, tensor_shape=tuple(new_shape))+ if hasattr(src, "_gguf_bias_orig_dtype"):+ wrapped._gguf_bias_orig_dtype = getattr(src, "_gguf_bias_orig_dtype")+ if hasattr(src, "_gguf_bias_orig_tensor_type"):+ wrapped._gguf_bias_orig_tensor_type = getattr(src, "_gguf_bias_orig_tensor_type")+ out.append(wrapped)+ return out+++def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):+ from mmgp import offload+ return offload.sd_split_linear(+ state_dict,+ fused_split_map,+ split_fields={"weight": 0, "bias": 0},+ split_handlers={"weight": _split_gguf_tensor, "bias": _split_gguf_tensor},+ verboseLevel=verboseLevel,+ allowed_bases=allowed_bases,+ return_split_bases=True,+ )++def _is_gguf_qtype(qtype_obj):if gguf is None:return Falsediff --git a/shared/qtypes/nunchaku_fp4.py b/shared/qtypes/nunchaku_fp4.pyindex 471b1bf..0907a41 100644--- a/shared/qtypes/nunchaku_fp4.py+++ b/shared/qtypes/nunchaku_fp4.py@@ -73,6 +73,19 @@ def make_nunchaku_splitter(split_map):return _split+def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):+ from mmgp import offload+ split_kwargs = get_nunchaku_split_kwargs()+ return offload.sd_split_linear(+ state_dict,+ fused_split_map,+ verboseLevel=verboseLevel,+ allowed_bases=allowed_bases,+ return_split_bases=True,+ **split_kwargs,+ )++def _install_nunchaku_shim(candidate_root):candidate_pkg = candidate_root / "nunchaku"if not candidate_pkg.exists():diff --git a/shared/qtypes/nunchaku_int4.py b/shared/qtypes/nunchaku_int4.pyindex 11327c6..77d998d 100644--- a/shared/qtypes/nunchaku_int4.py+++ b/shared/qtypes/nunchaku_int4.py@@ -73,6 +73,19 @@ def make_nunchaku_splitter(split_map):return _split+def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):+ from mmgp import offload+ split_kwargs = get_nunchaku_split_kwargs()+ return offload.sd_split_linear(+ state_dict,+ fused_split_map,+ verboseLevel=verboseLevel,+ allowed_bases=allowed_bases,+ return_split_bases=True,+ **split_kwargs,+ )++def _install_nunchaku_shim(candidate_root):candidate_pkg = candidate_root / "nunchaku"if not candidate_pkg.exists():diff --git a/shared/qtypes/nvfp4.py b/shared/qtypes/nvfp4.pyindex 5fdb7b1..59b194c 100644--- a/shared/qtypes/nvfp4.py+++ b/shared/qtypes/nvfp4.py@@ -61,6 +61,19 @@ _NVFP4_KERNEL_CHECKED = False_NVFP4_KERNEL_BACKEND = None_NVFP4_ACT_SCALE_CACHE = {}+_NVFP4_SPLIT_FIELDS = {+ "weight": 0,+ "bias": 0,+ "weight_scale": 0,+ "weight_scale_2": 0,+ "input_scale": 0,+ "input_global_scale": 0,+ "alpha": 0,+ "input_absmax": 0,+ "weight_global_scale": 0,+ "output_scale": 0,+}+_NVFP4_BACKEND = os.environ.get("WGP_NVFP4_BACKEND", _NVFP4_BACKEND_AUTO).strip().lower()_NVFP4_BACKEND = _NVFP4_BACKEND_LIGHTX2V@@ -79,6 +92,41 @@ def _normalize_nvfp4_backend(name):return norm+def _split_or_share_nvfp4_scale(src, *, dim, split_sizes, context):+ if src is None or not torch.is_tensor(src):+ return None+ total = sum(split_sizes)+ if src.numel() == 1:+ return [src] * len(split_sizes)+ if src.dim() > dim and src.size(dim) == total:+ return torch.split(src, split_sizes, dim=dim)+ if src.ndim > 1 and src.size(1) == total:+ return torch.split(src, split_sizes, dim=1)+ return [src] * len(split_sizes)+++def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):+ from mmgp import offload+ return offload.sd_split_linear(+ state_dict,+ fused_split_map,+ split_fields=dict(_NVFP4_SPLIT_FIELDS),+ split_handlers={+ "weight_scale": _split_or_share_nvfp4_scale,+ "weight_scale_2": _split_or_share_nvfp4_scale,+ "input_scale": _split_or_share_nvfp4_scale,+ "input_global_scale": _split_or_share_nvfp4_scale,+ "alpha": _split_or_share_nvfp4_scale,+ "input_absmax": _split_or_share_nvfp4_scale,+ "weight_global_scale": _split_or_share_nvfp4_scale,+ "output_scale": _split_or_share_nvfp4_scale,+ },+ verboseLevel=verboseLevel,+ allowed_bases=allowed_bases,+ return_split_bases=True,+ )++_NVFP4_BACKEND = _normalize_nvfp4_backend(_NVFP4_BACKEND)diff --git a/shared/qtypes/scaled_fp8.py b/shared/qtypes/scaled_fp8.pyindex 6079e59..f801bef 100644--- a/shared/qtypes/scaled_fp8.py+++ b/shared/qtypes/scaled_fp8.py@@ -50,6 +50,14 @@ _FP8_MM_SUPPORT = {_FP8_MM_PROBED = False_SCALED_FP8_DEFAULT_DTYPE = None+_SCALED_FP8_SPLIT_FIELDS = {+ "weight": 0,+ "bias": 0,+ "scale_weight": 0,+ "weight_scale": 0,+}+_SCALED_FP8_SHARE_FIELDS = ("input_scale", "output_scale")+def _is_float8_dtype(dtype):return dtype in _SCALED_FP8_QTYPE_BY_DTYPE@@ -75,6 +83,36 @@ def _normalize_default_dtype(dtype):return dtype+def _split_fp8_scale(src, *, dim, split_sizes, context):+ if src is None or not torch.is_tensor(src):+ return None+ total = sum(split_sizes)+ if src.numel() == 1:+ return [src] * len(split_sizes)+ if src.dim() > dim and src.size(dim) == total:+ return torch.split(src, split_sizes, dim=dim)+ if src.ndim > 1 and src.size(1) == total:+ return torch.split(src, split_sizes, dim=1)+ return [src] * len(split_sizes)+++def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):+ from mmgp import offload+ return offload.sd_split_linear(+ state_dict,+ fused_split_map,+ split_fields=dict(_SCALED_FP8_SPLIT_FIELDS),+ share_fields=_SCALED_FP8_SHARE_FIELDS,+ split_handlers={+ "scale_weight": _split_fp8_scale,+ "weight_scale": _split_fp8_scale,+ },+ verboseLevel=verboseLevel,+ allowed_bases=allowed_bases,+ return_split_bases=True,+ )++def _scaled_mm_available(dtype):if os.environ.get("WAN2GP_FORCE_FP8_FALLBACK", "").strip().lower() in ("1", "true", "yes", "on"):