Skip to content

SUPRA-Tex: 3D Generation with Super-Aligned Texture

26.1 KB
Raw Blame
from dataclasses import dataclass, field, fields
from typing import Any, Callable, Optional
import torch
from .base import QUANT_BACKENDS


QUANT_METHODS = {}


@dataclass
class QuantMethodSpec:
    backend: str
    config_factory: Callable[[dict], Any]
    label: str = ""


def register_quant_method(name, backend, config_factory, label=""):
    QUANT_METHODS[name] = QuantMethodSpec(backend=backend, config_factory=config_factory, label=label)


def describe_quant_method(name):
    """Print a method's backend, label, and the accepted `backend_config_kwargs` with defaults."""
    from . import backends
    backends.load_all_backends()
    if name not in QUANT_METHODS:
        raise ValueError(f"Unknown quantization method: {name}. Available methods:\n{_available_methods()}")
    spec = QUANT_METHODS[name]
    lines = [f"method: {name}", f"backend: {spec.backend}", f"detail: {spec.label}"]
    try:
        config = spec.config_factory({})
    except Exception as error:
        lines.append(f"backend config: could not be built with default kwargs: {error}")
        print("\n".join(lines))
        return
    cls = type(config)
    lines.append(f"backend config: {cls.__module__}.{cls.__qualname__}")
    all_fields = fields(config)
    user_fields = [f for f in all_fields if f.init]
    pinned_fields = [f for f in all_fields if not f.init]
    if user_fields:
        lines.append("backend_config_kwargs (user-tunable):")
        width = max(len(f.name) for f in user_fields)
        lines += [f"  {f.name:<{width}} = {getattr(config, f.name)!r}" for f in user_fields]
    if pinned_fields:
        lines.append("pinned by method (not overridable):")
        lines += [f"  {f.name} = {getattr(config, f.name)!r}" for f in pinned_fields]
    print("\n".join(lines))


def _available_methods():
    from . import backends
    backends.load_all_backends()
    methods = sorted(QUANT_METHODS.items())
    if len(methods) == 0:
        return "  (no method registered)"
    name_width = max(len(name) for name, _ in methods)
    backend_width = max(len(spec.backend) for _, spec in methods)
    lines = [f"method: {name:<{name_width}}  backend: {spec.backend:<{backend_width}}  detail: {spec.label}"
             for name, spec in methods]
    width = max(len(line) for line in lines)
    body = "\n".join(f"│ {line:<{width}} │" for line in lines)
    return f"┌{'─' * (width + 2)}┐\n{body}\n└{'─' * (width + 2)}┘"


def _format_error_report(reports):
    if not reports:
        return "Quantization error report: no targeted nn.Linear layers found."
    methods = sorted({report["method"] for report in reports})
    lines = [f"Quantization error report ({' + '.join(methods)}, {len(reports)} layers, sorted by relative_error, worst first):"]
    mixed = len(methods) > 1
    name_width = max(len(report["name"]) for report in reports)
    header = f"  {'relative_error':>14}  {'max_abs':>12}  {'shape':<16}  {'name':<{name_width}}"
    if mixed:
        header += "  method"
    lines.append(header)
    for report in reports:
        line = (
            f"  {report['relative_error'] * 100:>13.2f}%  {report['max_abs']:>12.3e}"
            f"  {str(report['shape']):<16}  {report['name']:<{name_width}}"
        )
        if mixed:
            line += f"  {report['method']}"
        lines.append(line)
    return "\n".join(lines)


@dataclass
class QuantizeConfig:
    """
    Quantization config and entry points for any `nn.Module`.

    Online (dynamic) quantization -- quantize an fp model whose weights are loaded:

        cfg = QuantizeConfig(method="bitsandbytes_nf4")
        model.load_state_dict(fp_state_dict)
        cfg.quantize_model(model, compute_device="cuda")

    Loading a pre-quantized checkpoint -- swap in quantized shells before loading:

        cfg = QuantizeConfig(method="bitsandbytes_nf4", load_prequantized=True)
        cfg.prepare_for_prequantized_load(model, compute_dtype=torch.bfloat16)
        state_dict = cfg.unflatten_state_dict(state_dict, metadata)
        model.load_state_dict(state_dict, assign=True)

    Dequantize-once -- after either flow above, back to plain fp Linears that carry
    the quantization error:

        cfg.dequantize_model(model, compute_dtype=torch.bfloat16)

    Fields:
        method: name from QUANT_METHODS, determines backend + scheme + backend config.
        mode: "dynamic" keeps the backend-native quantized Linear, which dequantizes
            its weight on every forward; "dequant_once" dequantizes to a plain fp
            `nn.Linear` right after the weights are quantized or loaded.
        target_modules / exclude_modules: list of module names; a layer is matched if
            its full dotted name equals an entry, or ends with "." + entry
            (e.g. "img_mod.1" matches "transformer_blocks.0.img_mod.1").
        backend_config_kwargs: passed through to the method's backend config factory
            (e.g. nf4 blocksize); `describe_quant_method(name)` prints what a method
            accepts.
        load_prequantized: the checkpoint already holds quantized weights; load them
            directly instead of quantizing fp weights online.
    """
    method: str = None
    mode: str = "dynamic"
    target_modules: Optional[list] = None
    exclude_modules: Optional[list] = None
    backend_config_kwargs: dict = field(default_factory=dict)
    load_prequantized: bool = False

    def __post_init__(self):
        if self.method is None:
            raise ValueError(f"`QuantizeConfig.method` is required. Available methods:\n{_available_methods()}")
        if self.mode not in ("dynamic", "dequant_once"):
            raise ValueError(f"`QuantizeConfig.mode` should be `dynamic` or `dequant_once`, but got `{self.mode}`.")
        for field_name in ("target_modules", "exclude_modules"):
            value = getattr(self, field_name)
            if value is not None and not isinstance(value, list):
                raise ValueError(f"`QuantizeConfig.{field_name}` should be a list of module names, but got `{type(value).__name__}`.")
        self.backend = self._build_backend()

    def _build_backend(self):
        from . import backends
        backends.load_backend_for_method(self.method)
        if self.method not in QUANT_METHODS:
            raise ValueError(f"Unknown quantization method: {self.method}. Available methods:\n{_available_methods()}")
        spec = QUANT_METHODS[self.method]
        if spec.backend not in QUANT_BACKENDS:
            raise ValueError(f"Quantization backend `{spec.backend}` (required by method `{self.method}`) is not registered.")
        backend = QUANT_BACKENDS[spec.backend]()
        backend.validate_environment()
        backend.announce_environment()
        backend.config = spec.config_factory(dict(self.backend_config_kwargs))
        return backend

    def quantize_model(self, model: torch.nn.Module, compute_device=None, model_device=None):
        """
        Quantize the targeted `nn.Linear` layers of an fp model in place, keeping each
        layer's existing dtype. Call AFTER `load_state_dict`. Does nothing when
        `load_prequantized` is set, since such a checkpoint is already quantized.

        Parameters:
            model: the fp model whose targeted layers are replaced by quantized ones.
            compute_device: where quantization runs; `None` quantizes each layer in place.
            model_device: where each quantized layer is stored afterwards; `None` leaves it
                on `compute_device`. Combining an fp model on the CPU with
                `compute_device="cuda", model_device="cpu"` streams the work layer by
                layer, so the accelerator only ever holds one layer at a time.
        """
        if self.load_prequantized:
            return model

        def quantize(linear):
            return self.backend.create_quantized_linear(linear, compute_device=compute_device, model_device=model_device)

        replaced = self._replace_target_linears(model, quantize)
        print(f"{len(replaced)} nn.Linear layers quantized (method: {self.method}).")
        return model

    def dequantize_model(self, model: torch.nn.Module, compute_dtype: torch.dtype = torch.bfloat16, compute_device=None, model_device=None):
        """
        Replace every quantized Linear in the model by a plain fp `nn.Linear`.
        Does nothing unless `mode` is `"dequant_once"`.

        Parameters:
            model: the quantized model to restore in place.
            compute_dtype: dtype of the restored fp weights.
            compute_device: where dequantization runs; `None` restores each layer in place.
            model_device: where each restored layer is stored afterwards; `None` leaves it
                on `compute_device`.
        """
        if self.mode != "dequant_once":
            return model
        self._dequantize_linears(model, compute_dtype, compute_device, model_device)
        return model

    def is_quantized_linear(self, module) -> bool:
        """Whether `module` is one of this config's backend-native quantized Linears."""
        return self.backend.is_quantized_linear(module)

    def build_quantized_shell(self, module, compute_dtype: torch.dtype, **kwargs):
        """
        Build an empty quantized Linear matching `module`, used to release a layer's
        weights while keeping it routable, and to stage a transient copy on the
        computation device.

        Parameters:
            module: the quantized layer whose shape and bias presence are mirrored.
            compute_dtype: dtype the shell dequantizes to at forward time.
        """
        return self.backend.create_quantized_linear_shell(module, compute_dtype)

    def prepare_for_prequantized_load(self, model: torch.nn.Module, compute_dtype: torch.dtype = torch.bfloat16):
        """
        Replace the targeted `nn.Linear` layers with empty quantized Linears matching a
        pre-quantized checkpoint. Call BEFORE `load_state_dict(assign=True)`.

        Parameters:
            model: the freshly constructed model whose targeted layers become shells.
            compute_dtype: dtype the quantized layers dequantize to at forward time.
        """
        def build_shell(linear):
            return self.backend.create_quantized_linear_shell(linear, compute_dtype)

        replaced = self._replace_target_linears(model, build_shell)
        print(f"{len(replaced)} nn.Linear layers replaced for loading the pre-quantized checkpoint (method: {self.method}).")
        return model

    def unflatten_state_dict(self, state_dict: dict, metadata: dict):
        """
        Rebuild composite quantized tensors from a flat (safetensors) state dict, so the
        result can be given to `load_state_dict(assign=True)`.

        Parameters:
            state_dict: the flat tensors read from the checkpoint.
            metadata: the string-only metadata stored alongside them.
        """
        return self.backend.unflatten_state_dict(state_dict, metadata)

    def flatten_state_dict(self, state_dict: dict):
        """
        Flatten a quantized model's state dict into plain tensors and string-only metadata.
        Returns `(tensors, metadata)`, ready for
        `safetensors.torch.save_file(tensors, path, metadata=metadata)`.

        Parameters:
            state_dict: the quantized model's state dict, holding composite tensors.
        """
        if not self.backend.capabilities().get("is_serializable", False):
            raise NotImplementedError(
                f"Backend `{self.backend.name}` (method `{self.method}`) does not declare "
                'serialization support (`capabilities()["is_serializable"]`), so its quantized '
                "state dict cannot be flattened for saving."
            )
        tensors, metadata = self.backend.flatten_state_dict(state_dict)
        tensors = {key: value.contiguous() for key, value in tensors.items()}
        metadata = {"format": "pt", **{key: value if isinstance(value, str) else str(value) for key, value in metadata.items()}}
        return tensors, metadata

    def measure_quantization_error(self, model: torch.nn.Module, compute_device="cuda", verbose=True) -> list:
        """
        Measure the weight error this config's quantization introduces.

        Parameters:
            model: the fp model to analyze.
            compute_device: where the quantize-dequantize round trip runs.
            verbose: print the per-layer report, sorted by `relative_error`, worst first.

        Returns:
            A list of per-layer dicts: `name`, `method`, `shape`, `relative_error` and `max_abs` (worst-element absolute error).
        """
        reports = [
            self._measure_layer_error(full_name, module, compute_device)
            for full_name, module in model.named_modules()
            if full_name != "" and self._should_quantize(full_name, module)
        ]
        reports.sort(key=lambda report: report["relative_error"], reverse=True)
        if verbose:
            print(_format_error_report(reports))
        return reports

    def _measure_layer_error(self, full_name, module, compute_device):
        fp_weight = module.weight.detach().to(device=compute_device, dtype=torch.float32)
        linear = torch.nn.Linear(
            module.in_features, module.out_features,
            bias=module.bias is not None, dtype=module.weight.dtype, device=compute_device,
        )
        linear.weight.data.copy_(module.weight.detach())
        if module.bias is not None:
            linear.bias.data.copy_(module.bias.detach().to(device=compute_device))
        quantized = self.backend.create_quantized_linear(linear, compute_device=compute_device, model_device=compute_device)
        restored = self.backend.dequantize_to_linear(quantized, compute_dtype=torch.float32, compute_device=compute_device, model_device=compute_device)
        quant_weight = restored.weight.detach()
        error = quant_weight - fp_weight
        report = {
            "name": full_name,
            "method": self.method,
            "shape": tuple(fp_weight.shape),
            "relative_error": (torch.linalg.vector_norm(error) / torch.linalg.vector_norm(fp_weight)).item(),
            "max_abs": error.abs().max().item(),
        }
        del fp_weight, linear, quantized, restored, quant_weight, error
        return report

    @staticmethod
    def _name_matches(full_name, patterns):
        if patterns is None:
            return False
        if full_name in patterns:
            return True
        return any(full_name.endswith(f".{pattern}") for pattern in patterns)

    def _should_quantize(self, full_name, module):
        if not isinstance(module, torch.nn.Linear):
            return False
        if self.is_quantized_linear(module):
            return False
        if self.target_modules is not None and not self._name_matches(full_name, self.target_modules):
            return False
        if self._name_matches(full_name, self.exclude_modules):
            return False
        return True

    def _replace_target_linears(self, model, transform):
        replaced = [
            full_name
            for full_name, module in model.named_modules()
            if full_name != "" and self._should_quantize(full_name, module)
        ]
        for full_name in replaced:
            parent_name, _, leaf_name = full_name.rpartition(".")
            parent = model.get_submodule(parent_name) if parent_name else model
            setattr(parent, leaf_name, transform(getattr(parent, leaf_name)))
        return replaced

    def _dequantize_linears(self, model, compute_dtype, compute_device=None, model_device=None):
        restored = [
            full_name
            for full_name, module in model.named_modules()
            if full_name != "" and self.backend.is_quantized_linear(module)
        ]
        for full_name in restored:
            parent_name, _, leaf_name = full_name.rpartition(".")
            parent = model.get_submodule(parent_name) if parent_name else model
            restored_linear = self.backend.dequantize_to_linear(
                getattr(parent, leaf_name), compute_dtype, compute_device=compute_device, model_device=model_device
            )
            setattr(parent, leaf_name, restored_linear)
        return restored


@dataclass
class MixedQuantizeConfig:
    """
    Combines multiple `QuantizeConfig` passes, one per disjoint layer set, behind the
    same interface as a single `QuantizeConfig`. Mixed quantization assigns different
    methods to different layers, e.g. nf4 for attention/MLP and int8 for the
    quantization-sensitive modulation layers:

        mod_layers = ["img_mod.1", "txt_mod.1", ...]
        cfg = MixedQuantizeConfig(configs=[
            QuantizeConfig(method="bitsandbytes_nf4", exclude_modules=mod_layers),
            QuantizeConfig(method="torchao_int8_w8a16", target_modules=mod_layers),
        ])
        cfg.quantize_model(model, compute_device="cuda")

    The passes run in `configs` order. Their matched layer sets must be pairwise
    disjoint; `quantize_model` and `prepare_for_prequantized_load` verify this on the
    fp model and raise before touching it.

    Fields:
        configs: list of `QuantizeConfig`, one per layer set; all must share the same
            `mode`, and their `load_prequantized` must stay False.
        load_prequantized: set on this wrapper (not on the children) to load a mixed
            pre-quantized checkpoint.
    """
    configs: list = None
    load_prequantized: bool = False

    def __post_init__(self):
        if not self.configs:
            raise ValueError("`MixedQuantizeConfig.configs` requires at least one `QuantizeConfig`.")
        for config in self.configs:
            if not isinstance(config, QuantizeConfig):
                raise ValueError(f"`MixedQuantizeConfig.configs` should contain `QuantizeConfig` instances, but got `{type(config).__name__}`.")
            if config.load_prequantized:
                raise ValueError(
                    "Set `load_prequantized=True` on `MixedQuantizeConfig` itself, not on its "
                    "child configs; the children only describe method + layer set."
                )
        modes = {config.mode for config in self.configs}
        if len(modes) > 1:
            raise ValueError(f"All configs in `MixedQuantizeConfig` should share the same `mode`, but got {sorted(modes)}.")
        self._ownership = {}

    @property
    def method(self):
        return " + ".join(config.method for config in self.configs)

    @property
    def mode(self):
        return self.configs[0].mode

    def quantize_model(self, model: torch.nn.Module, compute_device=None, model_device=None):
        """
        Run every config's quantization pass in order, after verifying their layer sets
        are disjoint. Does nothing when `load_prequantized` is set.

        Parameters:
            model: the fp model whose targeted layers are replaced by quantized ones.
            compute_device: where quantization runs; `None` quantizes each layer in place.
            model_device: where each quantized layer is stored afterwards; `None` leaves it
                on `compute_device`.
        """
        if self.load_prequantized:
            return model
        self._build_ownership(model)
        for config in self.configs:
            config.quantize_model(model, compute_device=compute_device, model_device=model_device)
        return model

    def dequantize_model(self, model: torch.nn.Module, compute_dtype: torch.dtype = torch.bfloat16, compute_device=None, model_device=None):
        """
        Replace every quantized Linear of every config's backend by a plain fp `nn.Linear`.
        Does nothing unless `mode` is `"dequant_once"`.

        Parameters:
            model: the quantized model to restore in place.
            compute_dtype: dtype of the restored fp weights.
            compute_device: where dequantization runs; `None` restores each layer in place.
            model_device: where each restored layer is stored afterwards; `None` leaves it
                on `compute_device`.
        """
        if self.mode != "dequant_once":
            return model
        for config in self.configs:
            config.dequantize_model(model, compute_dtype=compute_dtype, compute_device=compute_device, model_device=model_device)
        return model

    def is_quantized_linear(self, module) -> bool:
        """Whether `module` is a quantized Linear of any config's backend."""
        return any(config.is_quantized_linear(module) for config in self.configs)

    def build_quantized_shell(self, module, compute_dtype: torch.dtype, layer_name: str = None):
        """
        Build an empty quantized Linear for `module` using the config that owns it.

        Parameters:
            module: the quantized layer whose shape and bias presence are mirrored.
            compute_dtype: dtype the shell dequantizes to at forward time.
            layer_name: its dotted name inside the model; required to pick the right config
                when several configs share one backend, since their quantized Linears are
                then the same class.
        """
        return self._owning_config(module, layer_name).build_quantized_shell(module, compute_dtype)

    def _owning_config(self, module, layer_name=None):
        config = self._ownership.get(layer_name)
        if config is not None:
            return config
        for config in self.configs:
            if config.is_quantized_linear(module):
                return config
        raise ValueError(
            f"`{type(module).__name__}` is not a quantized Linear of any config in this "
            f"`MixedQuantizeConfig` (methods: {self.method})."
        )

    def prepare_for_prequantized_load(self, model: torch.nn.Module, compute_dtype: torch.dtype = torch.bfloat16):
        """
        Replace each config's targeted Linears by that backend's shells, after verifying
        the layer sets are disjoint.

        Parameters:
            model: the freshly constructed model whose targeted layers become shells.
            compute_dtype: dtype the quantized layers dequantize to at forward time.
        """
        self._build_ownership(model)
        for config in self.configs:
            config.prepare_for_prequantized_load(model, compute_dtype=compute_dtype)
        return model

    def unflatten_state_dict(self, state_dict: dict, metadata: dict):
        """
        Rebuild composite quantized tensors from a flat (safetensors) state dict.
        """
        if not self._ownership:
            for config in self._distinct_backend_configs():
                state_dict = config.unflatten_state_dict(state_dict, metadata)
            return state_dict
        rebuilt = {}
        for config, keys in self._grouped_by_backend(state_dict):
            rebuilt.update(config.unflatten_state_dict(keys, metadata) if config else keys)
        return rebuilt

    def flatten_state_dict(self, state_dict: dict):
        """
        Flatten the mixed model's state dict into plain tensors and string-only metadata.
        Returns (state_dict, metadata), ready for `safetensors.torch.save_file(tensors,
        path, metadata=metadata)`.
        """
        for config in self.configs:
            if not config.backend.capabilities().get("is_serializable", False):
                raise NotImplementedError(
                    f"Backend `{config.backend.name}` (method `{config.method}`) does not declare "
                    'serialization support (`capabilities()["is_serializable"]`), so the mixed '
                    "state dict cannot be flattened for saving."
                )
        merged_metadata = {}
        if not self._ownership:
            flattened = state_dict
            for config in self._distinct_backend_configs():
                flattened, metadata = config.backend.flatten_state_dict(flattened)
                merged_metadata.update(metadata)
        else:
            flattened = {}
            for config, keys in self._grouped_by_backend(state_dict):
                if config is None:
                    flattened.update(keys)
                    continue
                tensors, metadata = config.backend.flatten_state_dict(keys)
                flattened.update(tensors)
                merged_metadata.update(metadata)
        tensors = {key: value.contiguous() for key, value in flattened.items()}
        metadata = {"format": "pt", **{key: value if isinstance(value, str) else str(value) for key, value in merged_metadata.items()}}
        return tensors, metadata

    def measure_quantization_error(self, model: torch.nn.Module, compute_device="cuda", verbose=True) -> list:
        """
        Measure the weight error of the mixed quantization.
        """
        reports = []
        for config in self.configs:
            reports.extend(config.measure_quantization_error(model, compute_device=compute_device, verbose=False))
        reports.sort(key=lambda report: report["relative_error"], reverse=True)
        if verbose:
            print(_format_error_report(reports))
        return reports

    def _distinct_backend_configs(self):
        seen = set()
        distinct = []
        for config in self.configs:
            if config.backend.name not in seen:
                seen.add(config.backend.name)
                distinct.append(config)
        return distinct

    def _grouped_by_backend(self, state_dict):
        groups = {}
        for key, value in state_dict.items():
            parts = key.split(".")
            config = next((owner for end in range(len(parts) - 1, 0, -1)
                           if (owner := self._ownership.get(".".join(parts[:end]))) is not None), None)
            groups.setdefault(config.backend.name if config else None, (config, {}))[1][key] = value
        return groups.values()

    def _build_ownership(self, model):
        ownership = {}
        for index, config in enumerate(self.configs):
            names = [name for name, module in model.named_modules() if name and config._should_quantize(name, module)]
            overlap = sorted(set(names) & ownership.keys())
            if overlap:
                owner = self.configs.index(ownership[overlap[0]])
                raise ValueError(
                    f"Configs {owner} (`{self.configs[owner].method}`) and {index} (`{config.method}`) "
                    f"in `MixedQuantizeConfig` both match {len(overlap)} layers (e.g. {', '.join(overlap[:5])}). "
                )
            ownership.update({name: config for name in names})
        self._ownership.update(ownership)