Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

10.2 KB
Raw Blame
import pytest
import torch

from auto_round.export.svdquant_adapters.flux import FluxSVDQuantNunchakuAdapter
from auto_round.export.svdquant_nunchaku import (
    SourceLinearRecord,
    SVDQuantExportConfig,
    SVDQuantLinearScheme,
    collect_svdquant_tensors,
    save_svdquant_nunchaku_safetensors,
)

SCHEME = SVDQuantLinearScheme("mx_fp4", 4, 32, True, "mx_fp4", 4, 32, True, True)


def _source(name, out_features=8, in_features=8, rank=2, seed=0, bias=True):
    generator = torch.Generator().manual_seed(seed)
    return SourceLinearRecord(
        name=name,
        residual_weight=torch.randn(out_features, in_features, generator=generator),
        lora_down=torch.randn(rank, in_features, generator=generator),
        lora_up=torch.randn(out_features, rank, generator=generator),
        smooth=torch.linspace(0.5, 1.5, in_features),
        smooth_orig=torch.linspace(0.75, 1.75, in_features),
        bias=torch.randn(out_features, generator=generator) if bias else None,
        scheme=SCHEME,
    )


def _effective(source):
    return (source.residual_weight + source.lora_up @ source.lora_down) * source.smooth


def _model(config=None):
    model = torch.nn.Module()
    model.config = config or {
        "num_layers": 1,
        "num_single_layers": 1,
        "num_attention_heads": 2,
        "attention_head_dim": 4,
    }
    return model


def test_double_qkv_reconstructs_effective_sources_at_fixed_rank_in_order():
    sources = tuple(
        _source(f"transformer_blocks.0.attn.to_{name}", seed=index + 1) for index, name in enumerate(("q", "k", "v"))
    )
    adapter = FluxSVDQuantNunchakuAdapter(require_complete_model=False)

    (record,) = tuple(adapter.map_modules(_model(), sources))

    assert record.prefix == "transformer_blocks.0.qkv_proj"
    assert record.sources == sources
    assert record.lora_down.shape[0] == 2
    assert torch.equal(record.smooth, torch.ones(8))
    expected = torch.cat([_effective(source) for source in sources])
    actual = record.residual_weight + record.lora_up @ record.lora_down
    torch.testing.assert_close(actual, expected, atol=2e-5, rtol=2e-5)
    torch.testing.assert_close(record.bias, torch.cat([source.bias for source in sources]))
    assert all(
        tensor.device.type == "cpu" and tensor.is_contiguous()
        for tensor in (record.residual_weight, record.lora_down, record.lora_up)
    )


def test_double_qkv_preserves_shared_low_rank_and_smooth_without_redecomposition():
    shared_down = torch.randn(2, 8)
    shared_smooth = torch.linspace(0.5, 1.5, 8)
    shared_smooth_orig = torch.linspace(0.75, 1.75, 8)
    sources = tuple(
        _source(f"transformer_blocks.0.attn.to_{name}", seed=index + 21) for index, name in enumerate(("q", "k", "v"))
    )
    sources = tuple(
        SourceLinearRecord(
            name=source.name,
            residual_weight=source.residual_weight,
            lora_down=shared_down.clone(),
            lora_up=source.lora_up,
            smooth=shared_smooth.clone(),
            smooth_orig=shared_smooth_orig.clone(),
            bias=source.bias,
            scheme=source.scheme,
        )
        for source in sources
    )

    (record,) = tuple(FluxSVDQuantNunchakuAdapter(require_complete_model=False).map_modules(_model(), sources))

    torch.testing.assert_close(record.residual_weight, torch.cat([source.residual_weight for source in sources]))
    torch.testing.assert_close(record.lora_down, shared_down)
    torch.testing.assert_close(record.lora_up, torch.cat([source.lora_up for source in sources]))
    torch.testing.assert_close(record.smooth, shared_smooth)
    torch.testing.assert_close(record.smooth_orig, shared_smooth_orig)


def test_single_proj_out_splits_input_columns_and_keeps_bias_only_on_mlp():
    source = _source("single_transformer_blocks.0.proj_out", out_features=8, in_features=16, seed=10)
    model = _model({"num_layers": 0, "num_single_layers": 1, "num_attention_heads": 2, "attention_head_dim": 4})
    adapter = FluxSVDQuantNunchakuAdapter(require_complete_model=False)

    out_proj, mlp_fc2 = tuple(adapter.map_modules(model, (source,)))

    assert (out_proj.prefix, mlp_fc2.prefix) == (
        "single_transformer_blocks.0.out_proj",
        "single_transformer_blocks.0.mlp_fc2",
    )
    expected = _effective(source)
    out_effective = (out_proj.residual_weight + out_proj.lora_up @ out_proj.lora_down) * out_proj.smooth
    mlp_effective = (mlp_fc2.residual_weight + mlp_fc2.lora_up @ mlp_fc2.lora_down) * mlp_fc2.smooth
    torch.testing.assert_close(out_effective, expected[:, :8], atol=2e-5, rtol=2e-5)
    torch.testing.assert_close(mlp_effective, expected[:, 8:], atol=2e-5, rtol=2e-5)
    torch.testing.assert_close(out_proj.lora_down, source.lora_down[:, :8])
    torch.testing.assert_close(mlp_fc2.lora_down, source.lora_down[:, 8:])
    torch.testing.assert_close(out_proj.smooth, source.smooth[:8])
    torch.testing.assert_close(mlp_fc2.smooth, source.smooth[8:])
    assert out_proj.bias is None
    torch.testing.assert_close(mlp_fc2.bias, source.bias)


def _install(root, path, module):
    current = root
    parts = path.split(".")
    for part in parts[:-1]:
        if not hasattr(current, part):
            current.add_module(part, torch.nn.Module())
        current = getattr(current, part)
    current.add_module(parts[-1], module)


def _install_parameter(root, name, tensor):
    module_name, parameter_name = name.rsplit(".", 1)
    current = root
    for part in module_name.split("."):
        if not hasattr(current, part):
            current.add_module(part, torch.nn.Module())
        current = getattr(current, part)
    current.register_parameter(parameter_name, torch.nn.Parameter(tensor))


def _small_complete_top_level_tensors():
    from auto_round.export.svdquant_adapters.flux import FLUX_TOP_LEVEL_TENSOR_KEYS

    return {
        key: torch.ones(2, dtype=torch.bfloat16) if key.endswith(".bias") else torch.ones(2, 2, dtype=torch.bfloat16)
        for key in FLUX_TOP_LEVEL_TENSOR_KEYS
    }


def _small_linear_tensors(prefix):
    return {
        f"{prefix}.qweight": torch.ones(1, 1, dtype=torch.int8),
        f"{prefix}.wscales": torch.ones(1, 1, dtype=torch.uint8),
        f"{prefix}.smooth": torch.ones(1, dtype=torch.bfloat16),
        f"{prefix}.smooth_orig": torch.ones(1, dtype=torch.bfloat16),
        f"{prefix}.lora_down": torch.ones(1, 1, dtype=torch.bfloat16),
        f"{prefix}.lora_up": torch.ones(1, 1, dtype=torch.bfloat16),
        f"{prefix}.bias": torch.ones(1, dtype=torch.bfloat16),
    }


def _small_adanorm_tensors(prefix):
    return {
        f"{prefix}.qweight": torch.ones(1, 1, dtype=torch.int32),
        f"{prefix}.wscales": torch.ones(1, 1, dtype=torch.bfloat16),
        f"{prefix}.wzeros": torch.ones(1, 1, dtype=torch.bfloat16),
        f"{prefix}.bias": torch.ones(1, dtype=torch.bfloat16),
    }


def _small_standard_complete_tensors():
    tensors = _small_complete_top_level_tensors()
    for index in range(19):
        block = f"transformer_blocks.{index}"
        for linear in (
            "qkv_proj",
            "qkv_proj_context",
            "out_proj",
            "out_proj_context",
            "mlp_fc1",
            "mlp_fc2",
            "mlp_context_fc1",
            "mlp_context_fc2",
        ):
            tensors.update(_small_linear_tensors(f"{block}.{linear}"))
        tensors.update(_small_adanorm_tensors(f"{block}.norm1.linear"))
        tensors.update(_small_adanorm_tensors(f"{block}.norm1_context.linear"))
        for norm in ("norm_q", "norm_k", "norm_added_q", "norm_added_k"):
            tensors[f"{block}.{norm}.weight"] = torch.ones(1, dtype=torch.bfloat16)
    for index in range(38):
        block = f"single_transformer_blocks.{index}"
        for linear in ("qkv_proj", "out_proj", "mlp_fc1", "mlp_fc2"):
            tensors.update(_small_linear_tensors(f"{block}.{linear}"))
        tensors.update(_small_adanorm_tensors(f"{block}.norm.linear"))
        for norm in ("norm_q", "norm_k"):
            tensors[f"{block}.{norm}.weight"] = torch.ones(1, dtype=torch.bfloat16)
    return tensors


def test_standard_complete_schema_validates_2604_tiny_tensors_without_model_allocation():
    model = _model({"num_layers": 19, "num_single_layers": 38})
    adapter = FluxSVDQuantNunchakuAdapter(require_complete_model=True)
    tensors = _small_standard_complete_tensors()

    assert len(tensors) == 2604
    adapter.validate(tensors, adapter.metadata(model, 32))


def test_complete_extra_collection_requires_and_copies_exact_top_level_parameters():
    model = _model({"num_layers": 0, "num_single_layers": 0})
    expected = _small_complete_top_level_tensors()
    for key, tensor in expected.items():
        _install_parameter(model, key, tensor)

    actual = FluxSVDQuantNunchakuAdapter(require_complete_model=True).extra_tensors(model)

    assert set(actual) == set(expected)
    assert all(tensor.dtype == torch.bfloat16 for tensor in actual.values())


def test_partial_flux_collect_and_save_roundtrip(tmp_path):
    from safetensors import safe_open

    from auto_round.algorithms.transforms.svdquant.wrapper import SVDQuantLinear

    residual = torch.nn.Linear(32, 8)
    residual.data_type, residual.bits, residual.group_size, residual.sym = "mx_fp4", 4, 32, True
    residual.act_data_type, residual.act_bits, residual.act_group_size = "mx_fp4", 4, 32
    residual.act_sym, residual.act_dynamic = True, True
    wrapped = SVDQuantLinear(
        residual,
        torch.nn.Linear(32, 2, bias=False),
        torch.nn.Linear(2, 8, bias=False),
        torch.linspace(0.5, 1.5, 32),
    )
    model = _model({"num_layers": 1, "num_single_layers": 0})
    _install(model, "transformer_blocks.0.attn.to_out.0", wrapped)
    _install(model, "x_embedder", torch.nn.Linear(8, 8))
    adapter = FluxSVDQuantNunchakuAdapter(require_complete_model=False)
    config = SVDQuantExportConfig(runtime_loadable=True)

    collected = collect_svdquant_tensors(model, adapter=adapter, config=config)
    path = tmp_path / "flux.safetensors"
    save_svdquant_nunchaku_safetensors(model, str(path), adapter=adapter, config=config)

    with safe_open(path, framework="pt") as handle:
        assert set(handle.keys()) == set(collected)
        assert handle.metadata()["model_class"] == "NunchakuFluxTransformer2dModel"
        assert handle.get_tensor("x_embedder.weight").dtype == torch.bfloat16