Skip to content
Python
55.7 KB
Raw Blame
import ast
import os
import re

import torch
from torch.utils import _pytree as pytree

from optimum.quanto import QModuleMixin, register_qmodule
from optimum.quanto.tensor.qtensor import QTensor
from optimum.quanto.tensor.qtype import qtype as _quanto_qtype, qtypes as _quanto_qtypes
from collections import OrderedDict
from mmgp.offload import QEmbedding as BaseQEmbedding


HANDLER_NAME = "gguf"

try:
    import gguf
except Exception:
    gguf = None


_GGUF_QTYPE_NAME = "gguf"
if _GGUF_QTYPE_NAME not in _quanto_qtypes:
    _quanto_qtypes[_GGUF_QTYPE_NAME] = _quanto_qtype(
        _GGUF_QTYPE_NAME,
        is_floating_point=False,
        bits=6,
        dtype=torch.uint8,
        qmin=-32.0,
        qmax=31.0,
    )
_GGUF_QTYPE = _quanto_qtypes[_GGUF_QTYPE_NAME]

_GGUF_DEFAULT_DTYPE = None
_GGUF_LABEL_CACHE = {}
_GGUF_RUNTIME_LOGGED = set()
_GGUF_CUDA_KERNELS_ENV = "WGP_GGUF_LLAMACPP_CUDA"
_GGUF_CUDA_KERNELS_ENABLED_CACHE = None
_GGUF_CUDA_MODULE = None
_GGUF_CUDA_LOAD_ERROR = None


def _normalize_gguf_path(file_path):
    try:
        return os.path.normcase(os.path.abspath(file_path))
    except Exception:
        return str(file_path).lower()


def normalize(file_path):
    return _normalize_gguf_path(file_path)


def _set_default_dtype_from_loader(dtype):
    global _GGUF_DEFAULT_DTYPE
    if dtype is None:
        return
    _GGUF_DEFAULT_DTYPE = dtype


def _resolve_default_dtype(dtype, fallback=None):
    if dtype is None:
        return _GGUF_DEFAULT_DTYPE or fallback
    if _GGUF_DEFAULT_DTYPE is not None and fallback is not None and dtype == fallback:
        return _GGUF_DEFAULT_DTYPE
    return dtype


def _gguf_log_once(key, message):
    if key in _GGUF_RUNTIME_LOGGED:
        return
    _GGUF_RUNTIME_LOGGED.add(key)
    print(message)


def _gguf_cuda_env_enabled():
    raw = str(os.environ.get(_GGUF_CUDA_KERNELS_ENV, "1")).strip().lower()
    return raw in ("1", "true", "yes", "y", "on")


def _validate_gguf_cuda_module(module):
    import numpy as np

    if gguf is None:
        raise RuntimeError("gguf package is unavailable")
    if not torch.cuda.is_available():
        raise RuntimeError("CUDA is not available")
    if not module.may_support_linear_qtype_name("Q4_0"):
        raise RuntimeError("Q4_0 linear is not supported by the installed GGUF CUDA package")

    weight = np.linspace(-1.0, 1.0, 32, dtype=np.float32).reshape(1, 32)
    quantized_weight = gguf.quants.quantize(weight, gguf.GGMLQuantizationType.Q4_0)
    raw_weight = torch.from_numpy(quantized_weight).contiguous().cuda()
    input_tensor = torch.linspace(-0.5, 0.5, 64, dtype=torch.float32, device="cuda").reshape(2, 32)
    fast_out = module.linear(raw_weight, "Q4_0", [1, 32], input_tensor, None, torch.float32)
    dense_weight = torch.from_numpy(gguf.quants.dequantize(quantized_weight, gguf.GGMLQuantizationType.Q4_0)).to(device=input_tensor.device, dtype=torch.float32)
    ref_out = torch.nn.functional.linear(input_tensor, dense_weight, None)
    torch.cuda.synchronize()
    if fast_out.shape != ref_out.shape:
        raise RuntimeError(f"Q4 self-test shape mismatch: {tuple(fast_out.shape)} vs {tuple(ref_out.shape)}")
    if not torch.allclose(fast_out, ref_out, atol=2e-2, rtol=2e-2):
        max_diff = float((fast_out - ref_out).abs().max().item())
        raise RuntimeError(f"Q4 self-test output mismatch (max diff {max_diff:.6f})")


def _probe_gguf_cuda_runtime(force=False):
    global _GGUF_CUDA_KERNELS_ENABLED_CACHE, _GGUF_CUDA_MODULE, _GGUF_CUDA_LOAD_ERROR
    if not force and not _gguf_cuda_env_enabled():
        _GGUF_CUDA_KERNELS_ENABLED_CACHE = False
        _GGUF_CUDA_MODULE = None
        _GGUF_CUDA_LOAD_ERROR = None
        return False
    try:
        import llamacpp_gguf_cuda as gguf_cuda_module
        _validate_gguf_cuda_module(gguf_cuda_module)
    except Exception as exc:
        _GGUF_CUDA_KERNELS_ENABLED_CACHE = False
        _GGUF_CUDA_MODULE = None
        _GGUF_CUDA_LOAD_ERROR = exc
        _gguf_log_once("gguf_cuda_probe_failed", f"[GGUF][llama.cpp CUDA] kernels unavailable, using fallback: {exc}")
        return False
    _GGUF_CUDA_KERNELS_ENABLED_CACHE = True
    _GGUF_CUDA_MODULE = gguf_cuda_module
    _GGUF_CUDA_LOAD_ERROR = None
    _gguf_log_once("gguf_cuda_probe_ok", "[GGUF][llama.cpp CUDA] kernels available.")
    return True


def _gguf_cuda_kernels_enabled():
    global _GGUF_CUDA_KERNELS_ENABLED_CACHE
    if _GGUF_CUDA_KERNELS_ENABLED_CACHE is not None:
        return _GGUF_CUDA_KERNELS_ENABLED_CACHE
    return _probe_gguf_cuda_runtime()


def _gguf_cuda_module():
    if not _gguf_cuda_kernels_enabled():
        return None
    return _GGUF_CUDA_MODULE


def set_gguf_cuda_kernels_enabled(enabled=None):
    global _GGUF_CUDA_KERNELS_ENABLED_CACHE, _GGUF_CUDA_MODULE, _GGUF_CUDA_LOAD_ERROR
    if enabled is False:
        _GGUF_CUDA_KERNELS_ENABLED_CACHE = False
        _GGUF_CUDA_MODULE = None
        _GGUF_CUDA_LOAD_ERROR = None
        return
    _GGUF_CUDA_KERNELS_ENABLED_CACHE = None
    _GGUF_CUDA_MODULE = None
    _GGUF_CUDA_LOAD_ERROR = None
    if enabled is None:
        return _probe_gguf_cuda_runtime()
    return _probe_gguf_cuda_runtime(force=True)


_probe_gguf_cuda_runtime()

def get_file_metadata(file_path):
    if gguf is None:
        raise RuntimeError("GGUF support requires the 'gguf' package.")
    reader = gguf.GGUFReader(file_path)
    metadata = {}
    field = reader.get_field("config")
    if field is not None:
        try:
            metadata["config"] = field.contents() if callable(getattr(field, "contents", None)) else field.contents
        except Exception:
            pass
    return OrderedDict(), metadata


def _filter_state_dict_basic(state_dict, base_model_prefix, keep_prefix=False):
    new_state_dict = {}
    start = -1
    if keep_prefix:
        for k, v in state_dict.items():
            if k.startswith(base_model_prefix):
                new_state_dict[k] = v
    else:
        for k, v in state_dict.items():
            if k.startswith(base_model_prefix):
                new_start = len(base_model_prefix)
            else:
                pos = k.find("." + base_model_prefix)
                if pos < 0:
                    continue
                new_start = pos + len(base_model_prefix) + 1
            if start != -1 and start != new_start:
                new_state_dict = state_dict
                break
            start = new_start
            new_state_dict[k[start:]] = v
    return new_state_dict


def _gguf_get_orig_shape(reader, tensor_name):
    if gguf is None:
        raise RuntimeError("GGUF support requires the 'gguf' package.")
    field_key = f"comfy.gguf.orig_shape.{tensor_name}"
    field = reader.get_field(field_key)
    if field is None:
        return None
    if len(field.types) != 2 or field.types[0] != gguf.GGUFValueType.ARRAY or field.types[1] != gguf.GGUFValueType.INT32:
        raise TypeError(f"Bad GGUF shape metadata for {field_key}: {field.types}")
    return torch.Size(tuple(int(field.parts[part_idx][0]) for part_idx in field.data))


def _gguf_resolve_prefix(tensor_names, prefixes):
    for prefix in prefixes:
        if any(name.startswith(prefix) for name in tensor_names):
            return prefix
    return None


def load_gguf_state_dict(
    file_path,
    filters=None,
    keep_prefixes=False,
    writable_tensors=True,
    verboseLevel=1,
    default_dtype=None,
    pin_to_memory=False,
):
    if gguf is None:
        raise RuntimeError("GGUF support requires the 'gguf' package.")
    if pin_to_memory:
        raise Exception("Pinning to memory while loading GGUF files is not supported")

    import warnings

    def _cast_plain_tensor(torch_tensor, tensor_type):
        if tensor_type == gguf.GGMLQuantizationType.F16:
            if torch_tensor.dtype in (torch.uint8, torch.uint16):
                torch_tensor = torch_tensor.view(torch.float16)
            elif torch_tensor.dtype != torch.float16:
                torch_tensor = torch_tensor.to(torch.float16)
        elif tensor_type == gguf.GGMLQuantizationType.BF16:
            if torch_tensor.dtype in (torch.uint8, torch.uint16):
                torch_tensor = torch_tensor.view(torch.bfloat16)
            elif torch_tensor.dtype != torch.bfloat16:
                torch_tensor = torch_tensor.to(torch.bfloat16)
        elif tensor_type == gguf.GGMLQuantizationType.F32:
            if torch_tensor.dtype in (torch.uint8, torch.uint16, torch.uint32):
                torch_tensor = torch_tensor.view(torch.float32)
            elif torch_tensor.dtype != torch.float32:
                torch_tensor = torch_tensor.to(torch.float32)
        return torch_tensor

    def _tensor_type_from_dtype(dtype):
        if dtype == torch.float16:
            return gguf.GGMLQuantizationType.F16
        if dtype == torch.bfloat16:
            return gguf.GGMLQuantizationType.BF16
        if dtype == torch.float32:
            return gguf.GGMLQuantizationType.F32
        return None

    reader = gguf.GGUFReader(file_path)
    if verboseLevel >= 2:
        try:
            from mmgp import safetensors2
            safetensors2.verboseLevel = verboseLevel
            tracker = safetensors2.MmapTracker(file_path)
            tracker.register(reader.data, 0, 0, int(reader.data.nbytes))
        except Exception:
            tracker = None
    tensor_names = [tensor.name for tensor in reader.tensors]
    prefix = _gguf_resolve_prefix(tensor_names, ("model.diffusion_model.", "diffusion_model."))

    state_dict = {}
    qtype_counts = {}
    for tensor in reader.tensors:
        name = tensor.name
        if prefix and name.startswith(prefix):
            name = name[len(prefix):]

        with warnings.catch_warnings():
            warnings.filterwarnings("ignore", message="The given NumPy array is not writable")
            torch_tensor = torch.from_numpy(tensor.data)

        shape = _gguf_get_orig_shape(reader, tensor.name)
        if shape is None:
            shape = torch.Size(tuple(int(v) for v in reversed(tensor.shape)))
        if tensor.tensor_type in (
            gguf.GGMLQuantizationType.F32,
            gguf.GGMLQuantizationType.F16,
            gguf.GGMLQuantizationType.BF16,
        ):
            torch_tensor = _cast_plain_tensor(torch_tensor, tensor.tensor_type)
            torch_tensor = torch_tensor.view(*shape)
        wrapped = GGUFSourceTensor.wrap(torch_tensor, tensor_type=tensor.tensor_type, tensor_shape=shape)
        if name.endswith(".bias"):
            wrapped._gguf_bias_orig_dtype = wrapped.dtype
            wrapped._gguf_bias_orig_tensor_type = tensor.tensor_type
        state_dict[name] = wrapped
        type_name = getattr(tensor.tensor_type, "name", str(tensor.tensor_type))
        qtype_counts[type_name] = qtype_counts.get(type_name, 0) + 1

    if verboseLevel >= 2 and qtype_counts:
        print("GGUF qtypes: " + ", ".join(f"{k} ({v})" for k, v in qtype_counts.items()))

    if filters is not None:
        if not isinstance(filters, list):
            filters = [filters]
        new_sd = {}
        for one_filter in filters:
            new_sd.update(_filter_state_dict_basic(state_dict, one_filter, keep_prefixes))
        state_dict = new_sd

    if state_dict:
        for name, bias in list(state_dict.items()):
            if not name.endswith(".bias") or not torch.is_tensor(bias):
                continue
            weight_name = name[:-5] + ".weight"
            weight = state_dict.get(weight_name)
            target_dtype = None
            weight_type = getattr(weight, "tensor_type", None) if torch.is_tensor(weight) else None
            if torch.is_tensor(weight) and weight_type is None:
                target_dtype = weight.dtype if weight.dtype.is_floating_point else default_dtype
            elif weight_type in (
                gguf.GGMLQuantizationType.F16,
                gguf.GGMLQuantizationType.BF16,
                gguf.GGMLQuantizationType.F32,
            ):
                target_dtype = weight.dtype
            else:
                target_dtype = default_dtype
            if target_dtype is None or bias.dtype == target_dtype:
                continue
            casted = bias.to(target_dtype)
            if isinstance(casted, GGUFSourceTensor):
                casted._gguf_bias_orig_dtype = getattr(bias, "_gguf_bias_orig_dtype", bias.dtype)
                casted._gguf_bias_orig_tensor_type = getattr(
                    bias, "_gguf_bias_orig_tensor_type", getattr(bias, "tensor_type", None)
                )
                new_tensor_type = _tensor_type_from_dtype(target_dtype)
                if new_tensor_type is not None:
                    casted.tensor_type = new_tensor_type
                casted.tensor_shape = getattr(bias, "tensor_shape", casted.shape)
            state_dict[name] = casted

    return state_dict, None, None


def load_state_dict(*args, **kwargs):
    return load_gguf_state_dict(*args, **kwargs)


class GGUFSourceTensor(torch.Tensor):
    @staticmethod
    def wrap(tensor, *, tensor_type, tensor_shape):
        wrapped = tensor.as_subclass(GGUFSourceTensor)
        wrapped.tensor_type = tensor_type
        wrapped.tensor_shape = tensor_shape
        return wrapped

    def to(self, *args, **kwargs):
        new = super().to(*args, **kwargs)
        new.tensor_type = getattr(self, "tensor_type", None)
        new.tensor_shape = getattr(self, "tensor_shape", new.shape)
        return new

    def clone(self, *args, **kwargs):
        cloned = super().clone(*args, **kwargs).as_subclass(GGUFSourceTensor)
        cloned.tensor_type = getattr(self, "tensor_type", None)
        cloned.tensor_shape = getattr(self, "tensor_shape", cloned.shape)
        return cloned

    def detach(self, *args, **kwargs):
        detached = super().detach(*args, **kwargs).as_subclass(GGUFSourceTensor)
        detached.tensor_type = getattr(self, "tensor_type", None)
        detached.tensor_shape = getattr(self, "tensor_shape", detached.shape)
        return detached

    def get_quantized_subtensors(self):
        return [("data", self)]

    def set_quantized_subtensors(self, sub_tensors):
        if isinstance(sub_tensors, dict):
            data = sub_tensors.get("data")
        else:
            data = dict(sub_tensors).get("data")
        if data is None or data is self:
            return
        torch.utils.swap_tensors(self, data)


def materialize_source_tensor(tensor: torch.Tensor) -> torch.Tensor:
    if not isinstance(tensor, GGUFSourceTensor):
        return tensor
    with torch._C.DisableTorchFunctionSubclass():
        plain = torch.empty_strided(
            tuple(tensor.shape),
            tuple(tensor.stride()),
            dtype=tensor.dtype,
            device=tensor.device,
        )
        plain.copy_(tensor)
    return plain


def materialize_module_source_tensors(module: torch.nn.Module) -> int:
    converted = 0
    for submodule in module.modules():
        for name, param in list(submodule._parameters.items()):
            if param is None or not isinstance(param, GGUFSourceTensor):
                continue
            submodule._parameters[name] = torch.nn.Parameter(
                materialize_source_tensor(param),
                requires_grad=param.requires_grad,
            )
            converted += 1
        for name, buf in list(submodule._buffers.items()):
            if buf is None or not isinstance(buf, GGUFSourceTensor):
                continue
            submodule._buffers[name] = materialize_source_tensor(buf)
            converted += 1
    return converted


def _split_gguf_tensor(src, *, dim, split_sizes, context):
    if not torch.is_tensor(src):
        return None
    tensor_type = getattr(src, "tensor_type", None)
    if tensor_type is None:
        return None
    tensor_shape = getattr(src, "tensor_shape", None) or src.shape
    total = sum(split_sizes)
    if dim >= len(tensor_shape) or tensor_shape[dim] != total:
        return None
    chunks = torch.split(src, split_sizes, dim=dim)
    out = []
    for chunk, size in zip(chunks, split_sizes):
        new_shape = list(tensor_shape)
        new_shape[dim] = size
        wrapped = GGUFSourceTensor.wrap(chunk, tensor_type=tensor_type, tensor_shape=tuple(new_shape))
        if hasattr(src, "_gguf_bias_orig_dtype"):
            wrapped._gguf_bias_orig_dtype = getattr(src, "_gguf_bias_orig_dtype")
        if hasattr(src, "_gguf_bias_orig_tensor_type"):
            wrapped._gguf_bias_orig_tensor_type = getattr(src, "_gguf_bias_orig_tensor_type")
        out.append(wrapped)
    return out


def split_fused_weights(state_dict, fused_split_map, quantization_map=None, allowed_bases=None, default_dtype=None, verboseLevel=1):
    from mmgp import offload
    return offload.sd_split_linear(
        state_dict,
        fused_split_map,
        split_fields={"weight": 0, "bias": 0},
        split_handlers={"weight": _split_gguf_tensor, "bias": _split_gguf_tensor},
        verboseLevel=verboseLevel,
        allowed_bases=allowed_bases,
        return_split_bases=True,
    )


def _is_gguf_qtype(qtype_obj):
    if gguf is None:
        return False
    if qtype_obj is None:
        return False
    return qtype_obj not in (
        gguf.GGMLQuantizationType.F32,
        gguf.GGMLQuantizationType.F16,
        gguf.GGMLQuantizationType.BF16,
    )


def _gguf_qtype_name(qtype_obj):
    if qtype_obj is None:
        return None
    return getattr(qtype_obj, "name", None) or str(qtype_obj)


def _try_llamacpp_cuda_linear(weight_tensor, input_tensor, bias, target_dtype):
    if not _gguf_cuda_kernels_enabled():
        return None
    if not torch.is_tensor(input_tensor) or input_tensor.device.type != "cuda":
        return None
    raw = getattr(weight_tensor, "_data", None)
    if not torch.is_tensor(raw) or raw.device.type != "cuda" or not raw.is_contiguous():
        return None
    qtype_name = _gguf_qtype_name(getattr(weight_tensor, "_tensor_type", None))
    try:
        gguf_llamacpp_cuda = _gguf_cuda_module()
        if gguf_llamacpp_cuda is None or not gguf_llamacpp_cuda.may_support_linear_qtype_name(qtype_name):
            return None
        return gguf_llamacpp_cuda.linear(raw, qtype_name, tuple(getattr(weight_tensor, "_tensor_shape", weight_tensor.shape)), input_tensor, bias, target_dtype)
    except Exception as exc:
        _gguf_log_once(f"llamacpp_cuda_linear_{qtype_name}", f"[GGUF][llama.cpp CUDA] linear GGUF CUDA kernels failed for {qtype_name}, using fallback: {exc}")
        return None


def _try_llamacpp_cuda_embedding(weight_tensor, index_tensor, target_dtype):
    if not _gguf_cuda_kernels_enabled():
        return None
    if not torch.is_tensor(index_tensor) or index_tensor.device.type != "cuda":
        return None
    raw = getattr(weight_tensor, "_data", None)
    if not torch.is_tensor(raw) or raw.device.type != "cuda" or not raw.is_contiguous():
        return None
    qtype_name = _gguf_qtype_name(getattr(weight_tensor, "_tensor_type", None))
    try:
        gguf_llamacpp_cuda = _gguf_cuda_module()
        if gguf_llamacpp_cuda is None or not gguf_llamacpp_cuda.may_support_embedding_qtype_name(qtype_name):
            return None
        return gguf_llamacpp_cuda.embedding(raw, qtype_name, tuple(getattr(weight_tensor, "_tensor_shape", weight_tensor.shape)), index_tensor, target_dtype)
    except Exception as exc:
        _gguf_log_once(f"llamacpp_cuda_embedding_{qtype_name}", f"[GGUF][llama.cpp CUDA] embedding GGUF CUDA kernels failed for {qtype_name}, using fallback: {exc}")
        return None


def _may_try_llamacpp_cuda_linear(weight_tensor, input_tensor):
    if not _gguf_cuda_kernels_enabled():
        return False
    if not torch.is_tensor(input_tensor) or input_tensor.device.type != "cuda":
        return False
    raw = getattr(weight_tensor, "_data", None)
    if not torch.is_tensor(raw) or raw.device.type != "cuda" or not raw.is_contiguous():
        return False
    qtype_name = _gguf_qtype_name(getattr(weight_tensor, "_tensor_type", None))
    try:
        gguf_llamacpp_cuda = _gguf_cuda_module()
        return gguf_llamacpp_cuda is not None and gguf_llamacpp_cuda.may_support_linear_qtype_name(qtype_name)
    except Exception:
        return False


def _may_try_llamacpp_cuda_embedding(weight_tensor, index_tensor):
    if not _gguf_cuda_kernels_enabled():
        return False
    if not torch.is_tensor(index_tensor) or index_tensor.device.type != "cuda":
        return False
    raw = getattr(weight_tensor, "_data", None)
    if not torch.is_tensor(raw) or raw.device.type != "cuda" or not raw.is_contiguous():
        return False
    qtype_name = _gguf_qtype_name(getattr(weight_tensor, "_tensor_type", None))
    try:
        gguf_llamacpp_cuda = _gguf_cuda_module()
        return gguf_llamacpp_cuda is not None and gguf_llamacpp_cuda.may_support_embedding_qtype_name(qtype_name)
    except Exception:
        return False


def _guess_variant_from_filename(filename):
    base = os.path.basename(str(filename))
    match = re.search(r"(?i)(?:^|[_-])(Q\d+_K|Q\d+_\d|Q\d+|IQ\d+_\w+)(?:$|[_.-])", base)
    if match:
        return match.group(1).upper()
    return None


def detect_gguf_quantization_variant(file_path, verboseLevel=1):
    if gguf is None:
        return None
    try:
        reader = gguf.GGUFReader(file_path)
    except Exception:
        return None
    counts = {}
    for tensor in reader.tensors:
        qtype = getattr(tensor, "tensor_type", None)
        if qtype in (
            gguf.GGMLQuantizationType.F32,
            gguf.GGMLQuantizationType.F16,
            gguf.GGMLQuantizationType.BF16,
        ):
            continue
        name = _gguf_qtype_name(qtype)
        if not name:
            continue
        counts[name] = counts.get(name, 0) + 1
    if not counts:
        return None
    return max(counts, key=counts.get)


def detect_quantization_kind_for_file(file_path, verboseLevel=1):
    if not file_path or str(file_path).lower().endswith(".gguf") is False:
        return None
    if gguf is None:
        return None
    return "gguf"


def detect_quantization_label_from_filename(filename, verboseLevel=1):
    if not filename or str(filename).lower().endswith(".gguf") is False:
        return ""
    key = _normalize_gguf_path(filename)
    cached = _GGUF_LABEL_CACHE.get(key)
    if cached:
        return cached
    variant = _guess_variant_from_filename(filename)
    if not variant and os.path.isfile(filename):
        variant = detect_gguf_quantization_variant(filename, verboseLevel=verboseLevel)
    if variant:
        label = f"GGUF-{variant}"
    else:
        label = "GGUF"
    _GGUF_LABEL_CACHE[key] = label
    return label


def _gguf_qfallback(callable, *args, **kwargs):
    args, kwargs = pytree.tree_map_only(GGUFWeightTensor, lambda x: x.dequantize(), (args, kwargs or {}))
    return callable(*args, **kwargs)


def _reshape_scale(scale, weight):
    if scale.ndim == 0 or scale.numel() == 1:
        return scale
    if scale.ndim == 1 and scale.shape[0] == weight.shape[0]:
        return scale.view(weight.shape[0], *([1] * (weight.ndim - 1)))
    return scale


def _gguf_dequantize_tensor(raw, qtype_obj, oshape, dtype=None):
    if gguf is None:
        raise RuntimeError("gguf package is required to dequantize GGUF weights.")
    if qtype_obj in (
        gguf.GGMLQuantizationType.F32,
        gguf.GGMLQuantizationType.F16,
        gguf.GGMLQuantizationType.BF16,
    ):
        out = raw.view(*oshape)
        return out.to(dtype) if dtype is not None else out
    if qtype_obj not in _DEQUANTIZE_FUNCTIONS:
        out = gguf.quants.dequantize(raw.cpu().numpy(), qtype_obj)
        out = torch.from_numpy(out)
        return out.to(dtype) if dtype is not None else out
    block_size, type_size = gguf.GGML_QUANT_SIZES[qtype_obj]
    dequantize_blocks = _DEQUANTIZE_FUNCTIONS[qtype_obj]
    rows = raw.reshape((-1, raw.shape[-1])).view(torch.uint8)
    n_blocks = rows.numel() // type_size
    blocks = rows.reshape((n_blocks, type_size))
    blocks = dequantize_blocks(blocks, block_size, type_size, dtype)
    return blocks.reshape(oshape)


def _maybe_cast_bias(bias, target_dtype):
    if bias is None or not torch.is_tensor(bias) or target_dtype is None:
        return bias
    if bias.dtype == target_dtype:
        return bias
    if isinstance(bias, GGUFSourceTensor):
        tensor_type = getattr(bias, "tensor_type", None)
        tensor_shape = getattr(bias, "tensor_shape", bias.shape)
        if _is_gguf_qtype(tensor_type):
            return _gguf_dequantize_tensor(bias, tensor_type, tensor_shape, dtype=target_dtype)
    return bias.to(target_dtype)


def _to_uint32(x):
    x = x.view(torch.uint8).to(torch.int32)
    return (x[:, 0] | x[:, 1] << 8 | x[:, 2] << 16 | x[:, 3] << 24).unsqueeze(1)


def _to_uint16(x):
    x = x.view(torch.uint8).to(torch.int32)
    return (x[:, 0] | x[:, 1] << 8).unsqueeze(1)


def _const_like(ref, values, dtype):
    device = ref.device if torch.is_tensor(ref) else None
    count = len(values)
    if count == 0:
        return torch.empty((0,), device=device, dtype=dtype)
    if count == 1:
        return torch.full((1,), values[0], device=device, dtype=dtype)
    step = values[1] - values[0]
    if all(values[idx] - values[idx - 1] == step for idx in range(1, count)):
        end = values[0] + step * count
        return torch.arange(values[0], end, step, device=device, dtype=dtype)
    raise ValueError("Unsupported constant pattern for GGUF dequantization.")


def _split_block_dims(blocks, *args):
    n_max = blocks.shape[1]
    dims = list(args) + [n_max - sum(args)]
    return torch.split(blocks, dims, dim=1)


def _dequantize_blocks_Q8_0(blocks, block_size, type_size, dtype=None):
    d, x = _split_block_dims(blocks, 2)
    d = d.view(torch.float16).to(dtype)
    x = x.view(torch.int8)
    return d * x


def _dequantize_blocks_Q5_1(blocks, block_size, type_size, dtype=None):
    n_blocks = blocks.shape[0]
    d, m, qh, qs = _split_block_dims(blocks, 2, 2, 4)
    d = d.view(torch.float16).to(dtype)
    m = m.view(torch.float16).to(dtype)
    qh = _to_uint32(qh)
    qh = qh.reshape((n_blocks, 1)) >> torch.arange(32, device=d.device, dtype=torch.int32).reshape(1, 32)
    ql = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> _const_like(d, [0, 4], torch.uint8).reshape(1, 1, 2, 1)
    qh = (qh & 1).to(torch.uint8)
    ql = (ql & 0x0F).reshape((n_blocks, -1))
    qs = ql | (qh << 4)
    return d * qs + m


def _dequantize_blocks_Q5_0(blocks, block_size, type_size, dtype=None):
    n_blocks = blocks.shape[0]
    d, qh, qs = _split_block_dims(blocks, 2, 4)
    d = d.view(torch.float16).to(dtype)
    qh = _to_uint32(qh)
    qh = qh.reshape(n_blocks, 1) >> torch.arange(32, device=d.device, dtype=torch.int32).reshape(1, 32)
    ql = qs.reshape(n_blocks, -1, 1, block_size // 2) >> _const_like(d, [0, 4], torch.uint8).reshape(1, 1, 2, 1)
    qh = (qh & 1).to(torch.uint8)
    ql = (ql & 0x0F).reshape(n_blocks, -1)
    qs = (ql | (qh << 4)).to(torch.int8) - 16
    return d * qs


def _dequantize_blocks_Q4_1(blocks, block_size, type_size, dtype=None):
    n_blocks = blocks.shape[0]
    d, m, qs = _split_block_dims(blocks, 2, 2)
    d = d.view(torch.float16).to(dtype)
    m = m.view(torch.float16).to(dtype)
    qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> _const_like(d, [0, 4], torch.uint8).reshape(1, 1, 2, 1)
    qs = (qs & 0x0F).reshape((n_blocks, -1))
    return d * qs + m


def _dequantize_blocks_Q4_0(blocks, block_size, type_size, dtype=None):
    n_blocks = blocks.shape[0]
    d, qs = _split_block_dims(blocks, 2)
    d = d.view(torch.float16).to(dtype)
    qs = qs.reshape((n_blocks, -1, 1, block_size // 2)) >> _const_like(d, [0, 4], torch.uint8).reshape(1, 1, 2, 1)
    qs = (qs & 0x0F).reshape((n_blocks, -1))
    qs = qs.to(torch.int8) - 8
    return d * qs


QK_K = 256
K_SCALE_SIZE = 12


def _get_scale_min(scales):
    n_blocks = scales.shape[0]
    scales = scales.view(torch.uint8)
    scales = scales.reshape((n_blocks, 3, 4))
    d, m, m_d = torch.split(scales, scales.shape[-2] // 3, dim=-2)
    sc = torch.cat([d & 0x3F, (m_d & 0x0F) | ((d >> 2) & 0x30)], dim=-1)
    mn = torch.cat([m & 0x3F, (m_d >> 4) | ((m >> 2) & 0x30)], dim=-1)
    return sc.reshape((n_blocks, 8)), mn.reshape((n_blocks, 8))


def _dequantize_blocks_Q6_K(blocks, block_size, type_size, dtype=None):
    n_blocks = blocks.shape[0]
    ql, qh, scales, d = _split_block_dims(blocks, QK_K // 2, QK_K // 4, QK_K // 16)
    scales = scales.view(torch.int8).to(dtype)
    d = d.view(torch.float16).to(dtype)
    d = (d * scales).reshape((n_blocks, QK_K // 16, 1))
    ql = ql.reshape((n_blocks, -1, 1, 64)) >> _const_like(d, [0, 4], torch.uint8).reshape((1, 1, 2, 1))
    ql = (ql & 0x0F).reshape((n_blocks, -1, 32))
    qh = qh.reshape((n_blocks, -1, 1, 32)) >> _const_like(d, [0, 2, 4, 6], torch.uint8).reshape((1, 1, 4, 1))
    qh = (qh & 0x03).reshape((n_blocks, -1, 32))
    q = (ql | (qh << 4)).to(torch.int8) - 32
    q = q.reshape((n_blocks, QK_K // 16, -1))
    return (d * q).reshape((n_blocks, QK_K))


def _dequantize_blocks_Q5_K(blocks, block_size, type_size, dtype=None):
    n_blocks = blocks.shape[0]
    d, dmin, scales, qh, qs = _split_block_dims(blocks, 2, 2, K_SCALE_SIZE, QK_K // 8)
    d = d.view(torch.float16).to(dtype)
    dmin = dmin.view(torch.float16).to(dtype)
    sc, m = _get_scale_min(scales)
    d = (d * sc).reshape((n_blocks, -1, 1))
    dm = (dmin * m).reshape((n_blocks, -1, 1))
    ql = qs.reshape((n_blocks, -1, 1, 32)) >> _const_like(d, [0, 4], torch.uint8).reshape((1, 1, 2, 1))
    qh = qh.reshape((n_blocks, -1, 1, 32)) >> _const_like(d, list(range(8)), torch.uint8).reshape((1, 1, 8, 1))
    ql = (ql & 0x0F).reshape((n_blocks, -1, 32))
    qh = (qh & 0x01).reshape((n_blocks, -1, 32))
    q = ql | (qh << 4)
    return (d * q - dm).reshape((n_blocks, QK_K))


def _dequantize_blocks_Q4_K(blocks, block_size, type_size, dtype=None):
    n_blocks = blocks.shape[0]
    d, dmin, scales, qs = _split_block_dims(blocks, 2, 2, K_SCALE_SIZE)
    d = d.view(torch.float16).to(dtype)
    dmin = dmin.view(torch.float16).to(dtype)
    sc, m = _get_scale_min(scales)
    d = (d * sc).reshape((n_blocks, -1, 1))
    dm = (dmin * m).reshape((n_blocks, -1, 1))
    qs = qs.reshape((n_blocks, -1, 1, 32)) >> _const_like(d, [0, 4], torch.uint8).reshape((1, 1, 2, 1))
    qs = (qs & 0x0F).reshape((n_blocks, -1, 32))
    return (d * qs - dm).reshape((n_blocks, QK_K))


def _dequantize_blocks_Q3_K(blocks, block_size, type_size, dtype=None):
    n_blocks = blocks.shape[0]
    hmask, qs, scales, d = _split_block_dims(blocks, QK_K // 8, QK_K // 4, 12)
    d = d.view(torch.float16).to(dtype)
    lscales, hscales = scales[:, :8], scales[:, 8:]
    lscales = lscales.reshape((n_blocks, 1, 8)) >> _const_like(d, [0, 4], torch.uint8).reshape((1, 2, 1))
    lscales = lscales.reshape((n_blocks, 16))
    hscales = hscales.reshape((n_blocks, 1, 4)) >> _const_like(d, [0, 2, 4, 6], torch.uint8).reshape((1, 4, 1))
    hscales = hscales.reshape((n_blocks, 16))
    scales = (lscales & 0x0F) | ((hscales & 0x03) << 4)
    scales = (scales.to(torch.int8) - 32)
    dl = (d * scales).reshape((n_blocks, 16, 1))
    ql = qs.reshape((n_blocks, -1, 1, 32)) >> _const_like(d, [0, 2, 4, 6], torch.uint8).reshape((1, 1, 4, 1))
    qh = hmask.reshape(n_blocks, -1, 1, 32) >> _const_like(d, list(range(8)), torch.uint8).reshape((1, 1, 8, 1))
    ql = ql.reshape((n_blocks, 16, QK_K // 16)) & 3
    qh = (qh.reshape((n_blocks, 16, QK_K // 16)) & 1) ^ 1
    q = (ql.to(torch.int8) - (qh << 2).to(torch.int8))
    return (dl * q).reshape((n_blocks, QK_K))


def _dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None):
    n_blocks = blocks.shape[0]
    scales, qs, d, dmin = _split_block_dims(blocks, QK_K // 16, QK_K // 4, 2)
    d = d.view(torch.float16).to(dtype)
    dmin = dmin.view(torch.float16).to(dtype)
    dl = (d * (scales & 0xF)).reshape((n_blocks, QK_K // 16, 1))
    ml = (dmin * (scales >> 4)).reshape((n_blocks, QK_K // 16, 1))
    shift = _const_like(d, [0, 2, 4, 6], torch.uint8).reshape((1, 1, 4, 1))
    qs = (qs.reshape((n_blocks, -1, 1, 32)) >> shift) & 3
    qs = qs.reshape((n_blocks, QK_K // 16, 16))
    qs = dl * qs - ml
    return qs.reshape((n_blocks, -1))


if gguf is not None:
    _DEQUANTIZE_FUNCTIONS = {
        gguf.GGMLQuantizationType.Q8_0: _dequantize_blocks_Q8_0,
        gguf.GGMLQuantizationType.Q5_1: _dequantize_blocks_Q5_1,
        gguf.GGMLQuantizationType.Q5_0: _dequantize_blocks_Q5_0,
        gguf.GGMLQuantizationType.Q4_1: _dequantize_blocks_Q4_1,
        gguf.GGMLQuantizationType.Q4_0: _dequantize_blocks_Q4_0,
        gguf.GGMLQuantizationType.Q6_K: _dequantize_blocks_Q6_K,
        gguf.GGMLQuantizationType.Q5_K: _dequantize_blocks_Q5_K,
        gguf.GGMLQuantizationType.Q4_K: _dequantize_blocks_Q4_K,
        gguf.GGMLQuantizationType.Q3_K: _dequantize_blocks_Q3_K,
        gguf.GGMLQuantizationType.Q2_K: _dequantize_blocks_Q2_K,
    }
else:
    _DEQUANTIZE_FUNCTIONS = {}


class GGUFWeightTensor(QTensor):
    @staticmethod
    def create(raw_tensor, size, stride, dtype, device=None, requires_grad=False, tensor_type=None, tensor_shape=None):
        if tensor_type is None:
            tensor_type = getattr(raw_tensor, "tensor_type", None)
        if tensor_shape is None:
            tensor_shape = getattr(raw_tensor, "tensor_shape", None) or size
        device = raw_tensor.device if device is None else device
        if raw_tensor.device != device:
            raw_tensor = raw_tensor.to(device)
        return GGUFWeightTensor(
            qtype=_GGUF_QTYPE,
            axis=0,
            size=size,
            stride=stride,
            raw=raw_tensor,
            tensor_type=tensor_type,
            tensor_shape=tensor_shape,
            dtype=dtype,
            requires_grad=requires_grad,
        )

    @staticmethod
    def __new__(cls, qtype, axis, size, stride, raw, tensor_type, tensor_shape, dtype, requires_grad=False):
        return torch.Tensor._make_wrapper_subclass(
            cls,
            size,
            strides=stride,
            dtype=dtype,
            device=raw.device,
            requires_grad=requires_grad,
        )

    def __init__(self, qtype, axis, size, stride, raw, tensor_type, tensor_shape, dtype, requires_grad=False):
        super().__init__(qtype, axis)
        self._data = raw
        self._tensor_type = tensor_type
        self._tensor_shape = torch.Size(tensor_shape)
        self._gguf_default_dtype = dtype

    def __repr__(self):
        cls_name = self.__class__.__name__
        try:
            shape = tuple(self.shape)
        except Exception:
            shape = "<?>"
        try:
            dtype = str(self.dtype).replace("torch.", "")
        except Exception:
            dtype = "<?>"
        try:
            device = str(self.device)
        except Exception:
            device = "<?>"
        qtype = getattr(self, "_qtype", None)
        qtype_name = getattr(qtype, "name", None) or str(qtype) if qtype is not None else "<?>"
        tensor_type = _gguf_qtype_name(getattr(self, "_tensor_type", None)) or "<?>"
        return (
            f"{cls_name}(shape={shape}, dtype={dtype}, device={device}, "
            f"qtype={qtype_name}, tensor_type={tensor_type})"
        )

    __str__ = __repr__

    def dequantize(self, dtype=None, device=None):
        if dtype is None:
            dtype = self.dtype
        if device is None:
            device = self.device
        raw = self._data if self._data.device == device else self._data.to(device)
        return _gguf_dequantize_tensor(raw, self._tensor_type, self._tensor_shape, dtype=dtype)

    def linear(self, input, bias=None):
        if torch.is_tensor(input):
            target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=input.dtype)
            target_device = input.device
        else:
            target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=self.dtype)
            target_device = self.device
        if _may_try_llamacpp_cuda_linear(self, input):
            fast_out = _try_llamacpp_cuda_linear(self, input, bias, target_dtype)
            if fast_out is not None:
                return fast_out
        weight = self.dequantize(dtype=target_dtype, device=target_device)
        if torch.is_tensor(input) and input.dtype != weight.dtype:
            input = input.to(weight.dtype)
        bias = _maybe_cast_bias(bias, weight.dtype)
        return torch.nn.functional.linear(input, weight, bias)

    def get_quantized_subtensors(self):
        return [("data", self._data)]

    def set_quantized_subtensors(self, sub_tensors):
        if isinstance(sub_tensors, dict):
            sub_map = sub_tensors
        else:
            sub_map = {name: tensor for name, tensor in sub_tensors}
        data = sub_map.get("data", None)
        if data is not None:
            old_data = self._data
            if torch.is_tensor(old_data):
                try:
                    torch.utils.swap_tensors(old_data, data)
                    self._data = old_data
                except Exception:
                    self._data = data
            else:
                self._data = data
            if hasattr(self, "_ggml_raw_cpu"):
                self._ggml_raw_cpu = None

    def __tensor_flatten__(self):
        inner_tensors = ["_data"]
        meta = {
            "qtype": self._qtype.name,
            "axis": str(self._axis),
            "size": str(list(self.size())),
            "stride": str(list(self.stride())),
            "dtype": str(self.dtype),
            "tensor_type": _gguf_qtype_name(self._tensor_type) or "",
            "tensor_shape": str(list(self._tensor_shape)),
        }
        return inner_tensors, meta

    @staticmethod
    def __tensor_unflatten__(inner_tensors, meta, outer_size, outer_stride):
        qtype = _quanto_qtypes[meta["qtype"]]
        axis = ast.literal_eval(meta["axis"])
        size = ast.literal_eval(meta["size"])
        stride = ast.literal_eval(meta["stride"])
        dtype_str = meta.get("dtype", "torch.float16")
        if dtype_str.startswith("torch."):
            dtype_name = dtype_str.split(".", 1)[1]
            dtype = getattr(torch, dtype_name, torch.float16)
        else:
            dtype = getattr(torch, dtype_str, torch.float16)
        tensor_shape = ast.literal_eval(meta.get("tensor_shape", str(list(size))))
        tensor_type = None
        if gguf is not None and meta.get("tensor_type"):
            tensor_type = getattr(gguf.GGMLQuantizationType, meta["tensor_type"], None)
        return GGUFWeightTensor(
            qtype=qtype,
            axis=axis,
            size=size,
            stride=stride,
            raw=inner_tensors["_data"],
            tensor_type=tensor_type,
            tensor_shape=tensor_shape,
            dtype=dtype,
        )

    @classmethod
    def __torch_function__(cls, func, types, args=(), kwargs=None):
        kwargs = kwargs or {}
        if func is torch.nn.functional.linear:
            input = args[0] if len(args) > 0 else kwargs.get("input", None)
            weight = args[1] if len(args) > 1 else kwargs.get("weight", None)
            bias = args[2] if len(args) > 2 else kwargs.get("bias", None)
            if isinstance(weight, GGUFWeightTensor):
                return weight.linear(input, bias=bias)
        with torch._C.DisableTorchFunctionSubclass():
            return func(*args, **kwargs)

    @classmethod
    def __torch_dispatch__(cls, op, types, args, kwargs=None):
        op = op.overloadpacket
        kwargs = kwargs or {}
        if op is torch.ops.aten.linear:
            input = args[0]
            weight = args[1]
            bias = args[2] if len(args) > 2 else None
            if isinstance(weight, GGUFWeightTensor):
                return weight.linear(input, bias=bias)
        if op is torch.ops.aten.detach:
            t = args[0]
            return GGUFWeightTensor.create(
                raw_tensor=op(t._data),
                size=t.size(),
                stride=t.stride(),
                dtype=t.dtype,
                device=t.device,
                requires_grad=t.requires_grad,
                tensor_type=getattr(t, "_tensor_type", None),
                tensor_shape=getattr(t, "_tensor_shape", None),
            )
        if op in (torch.ops.aten._to_copy, torch.ops.aten.to):
            t = args[0]
            dtype = kwargs.pop("dtype", t.dtype) if kwargs else t.dtype
            device = kwargs.pop("device", t.device) if kwargs else t.device
            if dtype != t.dtype:
                return t.dequantize(dtype=dtype, device=device)
            out_data = op(t._data, device=device, **(kwargs or {}))
            return GGUFWeightTensor.create(
                raw_tensor=out_data,
                size=t.size(),
                stride=t.stride(),
                dtype=t.dtype,
                device=device,
                requires_grad=t.requires_grad,
                tensor_type=getattr(t, "_tensor_type", None),
                tensor_shape=getattr(t, "_tensor_shape", None),
            )
        return _gguf_qfallback(op, *args, **(kwargs or {}))


class QLinearGGUF(QModuleMixin, torch.nn.Linear):
    def __init__(
        self,
        in_features,
        out_features,
        bias=True,
        device=None,
        dtype=None,
        weights=None,
        activations=None,
        optimizer=None,
        quantize_input=True,
    ):
        super().__init__(
            in_features,
            out_features,
            bias=bias,
            device=device,
            dtype=dtype,
            weights=weights,
            activations=activations,
            optimizer=optimizer,
            quantize_input=quantize_input,
        )
        self._gguf_default_dtype = dtype

    @classmethod
    def qcreate(cls, module, weights, activations=None, optimizer=None, device=None):
        if torch.is_tensor(module.weight) and module.weight.dtype.is_floating_point:
            weight_dtype = module.weight.dtype
        elif torch.is_tensor(getattr(module, "bias", None)) and module.bias.dtype.is_floating_point:
            weight_dtype = module.bias.dtype
        else:
            weight_dtype = torch.float16
        return cls(
            module.in_features,
            module.out_features,
            module.bias is not None,
            device=device,
            dtype=weight_dtype,
            weights=weights,
            activations=activations,
            optimizer=optimizer,
            quantize_input=True,
        )

    def set_default_dtype(self, dtype):
        self._gguf_default_dtype = dtype

    @property
    def qweight(self):
        if self.weight_qtype == _GGUF_QTYPE:
            return self.weight
        return super().qweight

    def forward(self, input: torch.Tensor) -> torch.Tensor:
        qweight = self.qweight
        if isinstance(qweight, GGUFWeightTensor):
            return qweight.linear(input, bias=self.bias)
        return torch.nn.functional.linear(input, qweight, bias=self.bias)

    def _load_from_state_dict(
        self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
    ):
        if self.weight_qtype != _GGUF_QTYPE:
            return super()._load_from_state_dict(
                state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
            )

        weight_key = prefix + "weight"
        bias_key = prefix + "bias"
        input_scale_key = prefix + "input_scale"
        output_scale_key = prefix + "output_scale"

        weight_raw = state_dict.pop(weight_key, None)
        bias = state_dict.pop(bias_key, None)
        input_scale = state_dict.pop(input_scale_key, None)
        output_scale = state_dict.pop(output_scale_key, None)

        if weight_raw is None:
            missing_keys.append(weight_key)

        target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=self.weight.dtype)
        if weight_raw is not None:
            gguf_weight = GGUFWeightTensor.create(
                raw_tensor=weight_raw,
                size=self.weight.size(),
                stride=self.weight.stride(),
                dtype=target_dtype,
                device=weight_raw.device,
                requires_grad=False,
            )
            self.weight = torch.nn.Parameter(gguf_weight, requires_grad=False)

        if bias is not None:
            self.bias = torch.nn.Parameter(bias, requires_grad=False)

        if torch.is_tensor(weight_raw):
            scale_device = weight_raw.device
        elif torch.is_tensor(self.weight):
            scale_device = self.weight.device
        elif torch.is_tensor(bias):
            scale_device = bias.device
        else:
            scale_device = torch.device("cpu")

        if input_scale is not None:
            self.input_scale = input_scale.to(scale_device)
        else:
            if not hasattr(self, "input_scale") or self.input_scale.is_meta:
                scale_dtype = self.input_scale.dtype if hasattr(self, "input_scale") else torch.float32
                self.input_scale = torch.ones((), dtype=scale_dtype, device=scale_device)

        if output_scale is not None:
            self.output_scale = output_scale.to(scale_device)
        else:
            if not hasattr(self, "output_scale") or self.output_scale.is_meta:
                scale_dtype = self.output_scale.dtype if hasattr(self, "output_scale") else torch.float32
                self.output_scale = torch.ones((), dtype=scale_dtype, device=scale_device)

        return


@register_qmodule(torch.nn.Conv1d)
class QConv1dGGUF(QModuleMixin, torch.nn.Conv1d):
    def __init__(
        self,
        in_channels,
        out_channels,
        kernel_size,
        stride=1,
        padding=0,
        dilation=1,
        groups=1,
        bias=True,
        padding_mode="zeros",
        device=None,
        dtype=None,
        weights=None,
        activations=None,
        optimizer=None,
        quantize_input=True,
    ):
        super().__init__(
            in_channels,
            out_channels,
            kernel_size,
            stride=stride,
            padding=padding,
            dilation=dilation,
            groups=groups,
            bias=bias,
            padding_mode=padding_mode,
            device=device,
            dtype=dtype,
            weights=weights,
            activations=activations,
            optimizer=optimizer,
            quantize_input=quantize_input,
        )
        self._gguf_default_dtype = dtype

    @classmethod
    def qcreate(cls, module, weights, activations=None, optimizer=None, device=None):
        if torch.is_tensor(module.weight) and module.weight.dtype.is_floating_point:
            weight_dtype = module.weight.dtype
        elif torch.is_tensor(getattr(module, "bias", None)) and module.bias.dtype.is_floating_point:
            weight_dtype = module.bias.dtype
        else:
            weight_dtype = torch.float16
        return cls(
            module.in_channels,
            module.out_channels,
            module.kernel_size,
            stride=module.stride,
            padding=module.padding,
            dilation=module.dilation,
            groups=module.groups,
            bias=module.bias is not None,
            padding_mode=module.padding_mode,
            device=device,
            dtype=weight_dtype,
            weights=weights,
            activations=activations,
            optimizer=optimizer,
            quantize_input=True,
        )

    def set_default_dtype(self, dtype):
        self._gguf_default_dtype = dtype

    @property
    def qweight(self):
        if self.weight_qtype == _GGUF_QTYPE:
            return self.weight
        return super().qweight

    def forward(self, input: torch.Tensor) -> torch.Tensor:
        qweight = self.qweight
        if isinstance(qweight, GGUFWeightTensor):
            target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=input.dtype)
            weight = qweight.dequantize(dtype=target_dtype, device=input.device)
            bias = _maybe_cast_bias(self.bias, weight.dtype)
            if input.dtype != weight.dtype:
                input = input.to(weight.dtype)
            return torch.nn.functional.conv1d(
                input,
                weight,
                bias=bias,
                stride=self.stride,
                padding=self.padding,
                dilation=self.dilation,
                groups=self.groups,
            )
        return torch.nn.functional.conv1d(
            input,
            qweight,
            bias=self.bias,
            stride=self.stride,
            padding=self.padding,
            dilation=self.dilation,
            groups=self.groups,
        )

    def _load_from_state_dict(
        self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
    ):
        if self.weight_qtype != _GGUF_QTYPE:
            return super()._load_from_state_dict(
                state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
            )

        weight_key = prefix + "weight"
        bias_key = prefix + "bias"
        input_scale_key = prefix + "input_scale"
        output_scale_key = prefix + "output_scale"

        weight_raw = state_dict.pop(weight_key, None)
        bias = state_dict.pop(bias_key, None)
        input_scale = state_dict.pop(input_scale_key, None)
        output_scale = state_dict.pop(output_scale_key, None)

        if weight_raw is None:
            missing_keys.append(weight_key)

        target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=self.weight.dtype)
        if weight_raw is not None:
            gguf_weight = GGUFWeightTensor.create(
                raw_tensor=weight_raw,
                size=self.weight.size(),
                stride=self.weight.stride(),
                dtype=target_dtype,
                device=weight_raw.device,
                requires_grad=False,
            )
            self.weight = torch.nn.Parameter(gguf_weight, requires_grad=False)

        if bias is not None:
            self.bias = torch.nn.Parameter(bias, requires_grad=False)

        if torch.is_tensor(weight_raw):
            scale_device = weight_raw.device
        elif torch.is_tensor(self.weight):
            scale_device = self.weight.device
        elif torch.is_tensor(bias):
            scale_device = bias.device
        else:
            scale_device = torch.device("cpu")

        if input_scale is not None:
            self.input_scale = input_scale.to(scale_device)
        elif not hasattr(self, "input_scale") or self.input_scale.is_meta:
            scale_dtype = self.input_scale.dtype if hasattr(self, "input_scale") else torch.float32
            self.input_scale = torch.ones((), dtype=scale_dtype, device=scale_device)

        if output_scale is not None:
            self.output_scale = output_scale.to(scale_device)
        elif not hasattr(self, "output_scale") or self.output_scale.is_meta:
            scale_dtype = self.output_scale.dtype if hasattr(self, "output_scale") else torch.float32
            self.output_scale = torch.ones((), dtype=scale_dtype, device=scale_device)

        return


@register_qmodule(torch.nn.Embedding)
class QEmbeddingGGUF(BaseQEmbedding):
    def __init__(
        self,
        num_embeddings,
        embedding_dim,
        padding_idx=None,
        max_norm=None,
        norm_type=2.0,
        scale_grad_by_freq=False,
        sparse=False,
        device=None,
        dtype=None,
        weights=None,
        activations=None,
        optimizer=None,
        quantize_input=True,
    ):
        super().__init__(
            num_embeddings,
            embedding_dim,
            padding_idx=padding_idx,
            max_norm=max_norm,
            norm_type=norm_type,
            scale_grad_by_freq=scale_grad_by_freq,
            sparse=sparse,
            device=device,
            dtype=dtype,
            weights=weights,
            activations=activations,
            optimizer=optimizer,
            quantize_input=quantize_input,
        )
        self._gguf_default_dtype = dtype

    def set_default_dtype(self, dtype):
        self._gguf_default_dtype = dtype

    @property
    def qweight(self):
        if self.weight_qtype == _GGUF_QTYPE:
            return self.weight
        return super().qweight

    def forward(self, input: torch.Tensor) -> torch.Tensor:
        qweight = self.qweight
        if isinstance(qweight, GGUFWeightTensor):
            target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=torch.float16)
            if self.max_norm is None and not self.scale_grad_by_freq and not self.sparse:
                if _may_try_llamacpp_cuda_embedding(qweight, input):
                    fast_out = _try_llamacpp_cuda_embedding(qweight, input, target_dtype)
                    if fast_out is not None:
                        return fast_out
            weight = qweight.dequantize(dtype=target_dtype, device=input.device)
            return torch.nn.functional.embedding(
                input,
                weight,
                self.padding_idx,
                self.max_norm,
                self.norm_type,
                self.scale_grad_by_freq,
                self.sparse,
            )
        return super().forward(input)

    def _load_from_state_dict(
        self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
    ):
        if self.weight_qtype != _GGUF_QTYPE:
            return super()._load_from_state_dict(
                state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs
            )

        weight_key = prefix + "weight"
        weight_raw = state_dict.pop(weight_key, None)
        if weight_raw is None:
            missing_keys.append(weight_key)
            return

        target_dtype = _resolve_default_dtype(self._gguf_default_dtype, fallback=self.weight.dtype)
        gguf_weight = GGUFWeightTensor.create(
            raw_tensor=weight_raw,
            size=self.weight.size(),
            stride=self.weight.stride(),
            dtype=target_dtype,
            device=weight_raw.device,
            requires_grad=False,
        )
        self.weight = torch.nn.Parameter(gguf_weight, requires_grad=False)
        scale_device = weight_raw.device if torch.is_tensor(weight_raw) else torch.device("cpu")
        if not hasattr(self, "input_scale") or self.input_scale is None or self.input_scale.is_meta:
            self.input_scale = torch.ones((), dtype=torch.float32, device=scale_device)
        else:
            self.input_scale = self.input_scale.to(scale_device)
        if not hasattr(self, "output_scale") or self.output_scale is None or self.output_scale.is_meta:
            self.output_scale = torch.ones((), dtype=torch.float32, device=scale_device)
        else:
            self.output_scale = self.output_scale.to(scale_device)


def _collect_gguf_specs(state_dict):
    specs = []
    for key, tensor in state_dict.items():
        if not key.endswith(".weight"):
            continue
        if not _is_gguf_qtype(getattr(tensor, "tensor_type", None)):
            continue
        specs.append({"name": key[:-7], "tensor": tensor})
    return specs


def detect(state_dict, verboseLevel=1):
    if gguf is None:
        return {"matched": False, "kind": "none", "details": {"error": "gguf not installed"}}
    specs = _collect_gguf_specs(state_dict)
    if not specs:
        return {"matched": False, "kind": "none", "details": {}}
    names = [spec["name"] for spec in specs][:8]
    return {"matched": True, "kind": "gguf", "details": {"count": len(specs), "names": names}}


def convert_to_quanto(state_dict, default_dtype, verboseLevel=1, detection=None):
    if gguf is None:
        return {"state_dict": state_dict, "quant_map": {}}
    if detection is not None and not detection.get("matched", False):
        return {"state_dict": state_dict, "quant_map": {}}
    specs = _collect_gguf_specs(state_dict)
    if not specs:
        return {"state_dict": state_dict, "quant_map": {}}
    _set_default_dtype_from_loader(default_dtype)
    quant_map = {spec["name"]: {"weights": _GGUF_QTYPE_NAME, "activations": "none"} for spec in specs}
    return {"state_dict": state_dict, "quant_map": quant_map}


def apply_pre_quantization(model, state_dict, quantization_map, default_dtype=None, verboseLevel=1):
    if default_dtype is None or model is None or not quantization_map:
        return quantization_map or {}, []
    _set_default_dtype_from_loader(default_dtype)
    quantized = set(quantization_map.keys())
    for name, module in model.named_modules():
        if name in quantized and isinstance(module, torch.nn.Linear):
            module._router_default_dtype = default_dtype
    return quantization_map or {}, []