Skip to content
Python

refactored linear splitting to support any qtype

DeepBeepMeep5d48f3e378ab
diff --git a/models/qwen/qwen_main.py b/models/qwen/qwen_main.py
index c52a269..808d225 100644
--- a/models/qwen/qwen_main.py
+++ b/models/qwen/qwen_main.py
@@ -26,10 +26,6 @@ _QWEN_FUSED_SPLIT_MAP = {
}
-from shared.qtypes import nunchaku_int4 as _nunchaku_int4
-_split_nunchaku_fused_qkv = _nunchaku_int4.make_nunchaku_splitter(_QWEN_FUSED_SPLIT_MAP)
-
-
def stitch_images(img1, img2):
# Resize img2 to match img1's height
width1, height1 = img1.size
@@ -91,9 +87,9 @@ class model_factory():
source = model_def.get("source", None)
if source is not None:
- offload.load_model_data(transformer, source, preprocess_sd=_split_nunchaku_fused_qkv)
+ offload.load_model_data(transformer, source, fused_split_map=_QWEN_FUSED_SPLIT_MAP)
else:
- offload.load_model_data(transformer, transformer_filename, preprocess_sd=_split_nunchaku_fused_qkv)
+ offload.load_model_data(transformer, transformer_filename, fused_split_map=_QWEN_FUSED_SPLIT_MAP)
# transformer = offload.fast_load_transformers_model("transformer_quanto.safetensors", writable_tensors= True , modelClass=QwenImageTransformer2DModel, defaultConfigPath="transformer_config.json")
if not source is None:
diff --git a/models/z_image/z_image_main.py b/models/z_image/z_image_main.py
index cef551c..435fbf7 100644
--- a/models/z_image/z_image_main.py
+++ b/models/z_image/z_image_main.py
@@ -20,8 +20,6 @@ def conv_state_dict(sd: dict) -> dict:
if "x_embedder.weight" not in sd and "model.diffusion_model.x_embedder.weight" not in sd:
return sd
- from shared.qtypes.gguf import GGUFSourceTensor
-
inverse_replace = {
"final_layer.": "all_final_layer.2-1.",
"x_embedder.": "all_x_embedder.2-1.",
@@ -36,60 +34,22 @@ def conv_state_dict(sd: dict) -> dict:
for key, tensor in sd.items():
key = key.replace("model.diffusion_model.", "")
- if key.endswith(".attention.qkv.weight"):
- base = key[: -len(".attention.qkv.weight")]
-
- orig_shape = getattr(tensor, "tensor_shape", None)
- total_dim = orig_shape[0] if orig_shape is not None else tensor.shape[0]
- if total_dim % 3 != 0:
- raise ValueError(
- f"{key}: qkv first dimension ({total_dim}) not divisible by 3"
- )
- d = total_dim // 3
- if isinstance(tensor, GGUFSourceTensor) or getattr(tensor, "tensor_type", None) is not None:
- chunks = torch.split(tensor, d, dim=0)
- tensor_type = getattr(tensor, "tensor_type", None)
- tensor_shape = (d, orig_shape[1]) if orig_shape is not None else (d, tensor.shape[1])
- q, k_w, v = [
- GGUFSourceTensor.wrap(chunk, tensor_type=tensor_type, tensor_shape=tensor_shape)
- for chunk in chunks
- ]
- else:
- q, k_w, v = torch.split(tensor, d, dim=0)
-
- out_sd[base + ".attention.to_q.weight"] = q
- out_sd[base + ".attention.to_k.weight"] = k_w
- out_sd[base + ".attention.to_v.weight"] = v
- continue
-
new_key = key
- for comfy_sub, orig_sub in inverse_replace.items():
- new_key = new_key.replace(comfy_sub, orig_sub)
+ for ori_sub, orig_sub in inverse_replace.items():
+ new_key = new_key.replace(ori_sub, orig_sub)
out_sd[new_key] = tensor
- to_add = {}
- for key, tensor in out_sd.items():
- if key.endswith(".attention.to_out.0.weight"):
- prefix = key[: -len(".attention.to_out.0.weight")]
- bias_key = prefix + ".attention.to_out.0.bias"
- if bias_key not in out_sd:
- to_add[bias_key] = torch.zeros(tensor.shape[0], dtype=tensor.dtype)
-
- out_sd.update(to_add)
return out_sd
_ZIMAGE_FUSED_SPLIT_MAP = {
"attention.to_qkv": {"mapped_modules": ("attention.to_q", "attention.to_k", "attention.to_v")},
+ "attention.qkv": {"mapped_modules": ("attention.to_q", "attention.to_k", "attention.to_v")},
"feed_forward.net.0.proj": {"mapped_modules": ("feed_forward.w3", "feed_forward.w1")},
"feed_forward.net.2": {"mapped_modules": ("feed_forward.w2",)},
}
-from shared.qtypes import nunchaku_int4 as _nunchaku_int4
-_split_nunchaku_fused = _nunchaku_int4.make_nunchaku_splitter(_ZIMAGE_FUSED_SPLIT_MAP)
-
-
class model_factory:
def __init__(
self,
@@ -124,12 +84,11 @@ class model_factory:
default_transformer_config = os.path.join(os.path.dirname(os.path.abspath(__file__)), "configs", f"{base_model_type}.json")
def preprocess_sd(state_dict):
- state_dict = conv_state_dict(state_dict)
- return _split_nunchaku_fused(state_dict)
+ return conv_state_dict(state_dict)
model_class = ZImageTransformer2DModel
- kwargs_light= { "writable_tensors": False, "preprocess_sd": preprocess_sd }
+ kwargs_light= { "writable_tensors": False, "preprocess_sd": preprocess_sd, "fused_split_map": _ZIMAGE_FUSED_SPLIT_MAP }
# model_filename contains all files to load (transformer + modules merged by loader)
import json
import accelerate
@@ -181,7 +140,7 @@ class model_factory:
# VAE
vae_filename = fl.locate_file("ZImageTurbo_VAE_bf16.safetensors")
- vae_config_path = os.path.join(os.path.dirname(vae_filename), "ZImageTurbo_VAE_bf16_config.json")
+ vae_config_path = fl.locate_file("ZImageTurbo_VAE_bf16_config.json")
vae = offload.fast_load_transformers_model(
vae_filename,
diff --git a/shared/qtypes/gguf.py b/shared/qtypes/gguf.py
index 6e5ab58..be290a5 100644
--- a/shared/qtypes/gguf.py
+++ b/shared/qtypes/gguf.py
@@ -288,6 +288,43 @@ class GGUFSourceTensor(torch.Tensor):
torch.utils.swap_tensors(self, data)
+def _split_gguf_tensor(src, *, dim, split_sizes, context):
+ if not torch.is_tensor(src):
+ return None
+ tensor_type = getattr(src, "tensor_type", None)
+ if tensor_type is None:
+ return None
+ tensor_shape = getattr(src, "tensor_shape", None) or src.shape
+ total = sum(split_sizes)
+ if dim >= len(tensor_shape) or tensor_shape[dim] != total:
+ return None
+ chunks = torch.split(src, split_sizes, dim=dim)
+ out = []
+ for chunk, size in zip(chunks, split_sizes):
+ new_shape = list(tensor_shape)
+ new_shape[dim] = size
+ wrapped = GGUFSourceTensor.wrap(chunk, tensor_type=tensor_type, tensor_shape=tuple(new_shape))
+ if hasattr(src, "_gguf_bias_orig_dtype"):
+ wrapped._gguf_bias_orig_dtype = getattr(src, "_gguf_bias_orig_dtype")
+ if hasattr(src, "_gguf_bias_orig_tensor_type"):
+ wrapped._gguf_bias_orig_tensor_type = getattr(src, "_gguf_bias_orig_tensor_type")
+ out.append(wrapped)
+ return out
+
+
+def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):
+ from mmgp import offload
+ return offload.sd_split_linear(
+ state_dict,
+ fused_split_map,
+ split_fields={"weight": 0, "bias": 0},
+ split_handlers={"weight": _split_gguf_tensor, "bias": _split_gguf_tensor},
+ verboseLevel=verboseLevel,
+ allowed_bases=allowed_bases,
+ return_split_bases=True,
+ )
+
+
def _is_gguf_qtype(qtype_obj):
if gguf is None:
return False
diff --git a/shared/qtypes/nunchaku_fp4.py b/shared/qtypes/nunchaku_fp4.py
index 471b1bf..0907a41 100644
--- a/shared/qtypes/nunchaku_fp4.py
+++ b/shared/qtypes/nunchaku_fp4.py
@@ -73,6 +73,19 @@ def make_nunchaku_splitter(split_map):
return _split
+def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):
+ from mmgp import offload
+ split_kwargs = get_nunchaku_split_kwargs()
+ return offload.sd_split_linear(
+ state_dict,
+ fused_split_map,
+ verboseLevel=verboseLevel,
+ allowed_bases=allowed_bases,
+ return_split_bases=True,
+ **split_kwargs,
+ )
+
+
def _install_nunchaku_shim(candidate_root):
candidate_pkg = candidate_root / "nunchaku"
if not candidate_pkg.exists():
diff --git a/shared/qtypes/nunchaku_int4.py b/shared/qtypes/nunchaku_int4.py
index 11327c6..77d998d 100644
--- a/shared/qtypes/nunchaku_int4.py
+++ b/shared/qtypes/nunchaku_int4.py
@@ -73,6 +73,19 @@ def make_nunchaku_splitter(split_map):
return _split
+def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):
+ from mmgp import offload
+ split_kwargs = get_nunchaku_split_kwargs()
+ return offload.sd_split_linear(
+ state_dict,
+ fused_split_map,
+ verboseLevel=verboseLevel,
+ allowed_bases=allowed_bases,
+ return_split_bases=True,
+ **split_kwargs,
+ )
+
+
def _install_nunchaku_shim(candidate_root):
candidate_pkg = candidate_root / "nunchaku"
if not candidate_pkg.exists():
diff --git a/shared/qtypes/nvfp4.py b/shared/qtypes/nvfp4.py
index 5fdb7b1..59b194c 100644
--- a/shared/qtypes/nvfp4.py
+++ b/shared/qtypes/nvfp4.py
@@ -61,6 +61,19 @@ _NVFP4_KERNEL_CHECKED = False
_NVFP4_KERNEL_BACKEND = None
_NVFP4_ACT_SCALE_CACHE = {}
+_NVFP4_SPLIT_FIELDS = {
+ "weight": 0,
+ "bias": 0,
+ "weight_scale": 0,
+ "weight_scale_2": 0,
+ "input_scale": 0,
+ "input_global_scale": 0,
+ "alpha": 0,
+ "input_absmax": 0,
+ "weight_global_scale": 0,
+ "output_scale": 0,
+}
+
_NVFP4_BACKEND = os.environ.get("WGP_NVFP4_BACKEND", _NVFP4_BACKEND_AUTO).strip().lower()
_NVFP4_BACKEND = _NVFP4_BACKEND_LIGHTX2V
@@ -79,6 +92,41 @@ def _normalize_nvfp4_backend(name):
return norm
+def _split_or_share_nvfp4_scale(src, *, dim, split_sizes, context):
+ if src is None or not torch.is_tensor(src):
+ return None
+ total = sum(split_sizes)
+ if src.numel() == 1:
+ return [src] * len(split_sizes)
+ if src.dim() > dim and src.size(dim) == total:
+ return torch.split(src, split_sizes, dim=dim)
+ if src.ndim > 1 and src.size(1) == total:
+ return torch.split(src, split_sizes, dim=1)
+ return [src] * len(split_sizes)
+
+
+def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):
+ from mmgp import offload
+ return offload.sd_split_linear(
+ state_dict,
+ fused_split_map,
+ split_fields=dict(_NVFP4_SPLIT_FIELDS),
+ split_handlers={
+ "weight_scale": _split_or_share_nvfp4_scale,
+ "weight_scale_2": _split_or_share_nvfp4_scale,
+ "input_scale": _split_or_share_nvfp4_scale,
+ "input_global_scale": _split_or_share_nvfp4_scale,
+ "alpha": _split_or_share_nvfp4_scale,
+ "input_absmax": _split_or_share_nvfp4_scale,
+ "weight_global_scale": _split_or_share_nvfp4_scale,
+ "output_scale": _split_or_share_nvfp4_scale,
+ },
+ verboseLevel=verboseLevel,
+ allowed_bases=allowed_bases,
+ return_split_bases=True,
+ )
+
+
_NVFP4_BACKEND = _normalize_nvfp4_backend(_NVFP4_BACKEND)
diff --git a/shared/qtypes/scaled_fp8.py b/shared/qtypes/scaled_fp8.py
index 6079e59..f801bef 100644
--- a/shared/qtypes/scaled_fp8.py
+++ b/shared/qtypes/scaled_fp8.py
@@ -50,6 +50,14 @@ _FP8_MM_SUPPORT = {
_FP8_MM_PROBED = False
_SCALED_FP8_DEFAULT_DTYPE = None
+_SCALED_FP8_SPLIT_FIELDS = {
+ "weight": 0,
+ "bias": 0,
+ "scale_weight": 0,
+ "weight_scale": 0,
+}
+_SCALED_FP8_SHARE_FIELDS = ("input_scale", "output_scale")
+
def _is_float8_dtype(dtype):
return dtype in _SCALED_FP8_QTYPE_BY_DTYPE
@@ -75,6 +83,36 @@ def _normalize_default_dtype(dtype):
return dtype
+def _split_fp8_scale(src, *, dim, split_sizes, context):
+ if src is None or not torch.is_tensor(src):
+ return None
+ total = sum(split_sizes)
+ if src.numel() == 1:
+ return [src] * len(split_sizes)
+ if src.dim() > dim and src.size(dim) == total:
+ return torch.split(src, split_sizes, dim=dim)
+ if src.ndim > 1 and src.size(1) == total:
+ return torch.split(src, split_sizes, dim=1)
+ return [src] * len(split_sizes)
+
+
+def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):
+ from mmgp import offload
+ return offload.sd_split_linear(
+ state_dict,
+ fused_split_map,
+ split_fields=dict(_SCALED_FP8_SPLIT_FIELDS),
+ share_fields=_SCALED_FP8_SHARE_FIELDS,
+ split_handlers={
+ "scale_weight": _split_fp8_scale,
+ "weight_scale": _split_fp8_scale,
+ },
+ verboseLevel=verboseLevel,
+ allowed_bases=allowed_bases,
+ return_split_bases=True,
+ )
+
+
def _scaled_mm_available(dtype):
if os.environ.get("WAN2GP_FORCE_FP8_FALLBACK", "").strip().lower() in ("1", "true", "yes", "on"):