Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[refactor] decouple calibaration code (#1765)

Heng Guo71fb3506343d
diff --git a/.claude/skills/adapt-new-diffusion-model/SKILL.md b/.claude/skills/adapt-new-diffusion-model/SKILL.md
index e24350c11..89b36c274 100644
--- a/.claude/skills/adapt-new-diffusion-model/SKILL.md
+++ b/.claude/skills/adapt-new-diffusion-model/SKILL.md
@@ -7,11 +7,13 @@ description: "Adapt AutoRound to support a new diffusion model architecture (DiT
## Overview
-AutoRound's DiffusionCompressor works with standard diffusers pipelines
-(e.g., FLUX). This skill covers what code changes are needed when a new
-diffusion model doesn't work out-of-the-box. Common reasons for adaptation:
+AutoRound's new diffusion path uses `auto_round/compressors/diffusion_mixin.py`,
+`auto_round/calibration/diffusion.py`, and the quantizer implementations under
+`auto_round/algorithms/quantization/`. This skill covers what code changes are
+needed when a new diffusion model doesn't work out-of-the-box. Common reasons
+for adaptation:
-- Transformer block type not registered in `output_configs`
+- Transformer block type not registered in `DIFFUSION_OUTPUT_CONFIGS`
- Non-standard pipeline API (not compatible with `pipe(prompts, ...)`)
- Hybrid architecture with both AR and diffusion components
- Model not detected as a diffusion model
@@ -49,16 +51,16 @@ def is_diffusion_model(model_or_path):
# Checks for model_index.json presence
```
-If your model doesn't have `model_index.json`, either:
-- Create one in the model directory
-- Force diffusion mode via `ExtraConfig`:
+If your model doesn't have `model_index.json`, either create one in the model
+directory or pass diffusion-specific options through new-architecture
+`ExtraConfig` / `AutoRound` kwargs:
```python
-from auto_round.compressors import ExtraConfig
+from auto_round.compressors.config import ExtraConfig
ar = AutoRound(
model,
- extra_config=ExtraConfig(diffusion_config=DiffusionConfig(...)),
+ extra_config=ExtraConfig(num_inference_steps=5),
)
```
@@ -71,7 +73,7 @@ different attribute (e.g., `pipe.unet`), this needs adjustment in
## Step 2: Register Transformer Block Output Config
-This is the **most common** adaptation needed. The `output_configs` dict maps
+This is the **most common** adaptation needed. `DIFFUSION_OUTPUT_CONFIGS` maps
transformer block class names to their output tensor names. Without this,
calibration crashes because AutoRound doesn't know how to collect activations.
@@ -86,17 +88,18 @@ for name, module in pipe.transformer.named_modules():
print(f"{name}: {type(module).__name__}")
```
-### Register in `output_configs`
+### Register in `DIFFUSION_OUTPUT_CONFIGS`
-Edit `auto_round/compressors/diffusion/compressor.py`:
+Edit `auto_round/algorithms/quantization/base.py`:
```python
-output_configs = {
- "FluxTransformerBlock": ["encoder_hidden_states", "hidden_states"],
- "FluxSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],
- # Add your block type:
- "YourTransformerBlock": ["hidden_states"], # output tensor names in order
-}
+class BaseQuantizers:
+ DIFFUSION_OUTPUT_CONFIGS = {
+ "FluxTransformerBlock": ["encoder_hidden_states", "hidden_states"],
+ "FluxSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],
+ # Add your block type:
+ "YourTransformerBlock": ["hidden_states"], # output tensor names in order
+ }
```
The list must match the **exact order** of tensors returned by the block's
@@ -111,12 +114,12 @@ The list must match the **exact order** of tensors returned by the block's
**Example**: If `forward()` returns `(hidden_states, encoder_hidden_states)`:
```python
-output_configs["YourBlock"] = ["hidden_states", "encoder_hidden_states"]
+BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS["YourBlock"] = ["hidden_states", "encoder_hidden_states"]
```
**Example**: If `forward()` returns just `hidden_states`:
```python
-output_configs["YourBlock"] = ["hidden_states"]
+BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS["YourBlock"] = ["hidden_states"]
```
## Step 3: Handle Non-Standard Pipeline API
@@ -125,55 +128,60 @@ If your model's inference API differs from the standard
`pipe(prompts, guidance_scale=..., num_inference_steps=...)`, provide a custom
pipeline function.
-### Option A: Pass `pipeline_fn` parameter (no code changes)
+### Option A: Add a custom pipeline dispatch in `DiffusionCalibrator`
+
+Update `auto_round/calibration/diffusion.py` so `DiffusionCalibrator.calib()`
+dispatches through a small helper instead of calling `pipe(...)` directly:
```python
-def your_model_pipeline_fn(pipe, prompts, guidance_scale=7.5, num_inference_steps=28, generator=None, **kwargs):
- """Custom pipeline function for YourModel."""
- for prompt in (prompts if isinstance(prompts, list) else [prompts]):
- pipe.generate(
- prompt=prompt,
- cfg_scale=guidance_scale,
- steps=num_inference_steps,
+class DiffusionCalibrator(LLMCalibrator):
+ ...
+
+ def _run_pipeline(self, pipe, prompts, generator):
+ if getattr(pipe, "_autoround_pipeline_fn", None) is not None:
+ pipe._autoround_pipeline_fn(
+ pipe,
+ prompts,
+ guidance_scale=self.compressor.guidance_scale,
+ num_inference_steps=self.compressor.num_inference_steps,
+ generator=generator,
+ )
+ return
+ pipe(
+ prompts,
+ guidance_scale=self.compressor.guidance_scale,
+ num_inference_steps=self.compressor.num_inference_steps,
generator=generator,
)
-
-
-ar = AutoRound(
- "your-model",
- pipeline_fn=your_model_pipeline_fn,
- num_inference_steps=28,
- guidance_scale=7.5,
-)
```
-### Option B: Attach to pipe object
+### Option B: Attach a model-specific function during model loading
-If using `diffusion_load_model()` directly:
+For a known model family, attach `_autoround_pipeline_fn` in
+`auto_round/utils/model.py` or `auto_round/special_model_handler.py`:
```python
pipe._autoround_pipeline_fn = your_model_pipeline_fn
```
-### Option C: Subclass DiffusionCompressor
+### Option C: Add a dedicated branch in `DiffusionCalibrator`
-For full control, override `_run_pipeline()`:
+For full control, update `auto_round/calibration/diffusion.py` so
+`DiffusionCalibrator.calib()` dispatches through your custom pipeline function:
```python
-from auto_round.compressors.diffusion.compressor import DiffusionCompressor
-
+class DiffusionCalibrator(LLMCalibrator):
+ ...
-class YourModelCompressor(DiffusionCompressor):
- def _run_pipeline(self, prompts):
+ def _run_pipeline(self, pipe, prompts):
+ c = self.compressor
generator = (
- None
- if self.generator_seed is None
- else torch.Generator(device=self.pipe.device).manual_seed(self.generator_seed)
+ None if c.generator_seed is None else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)
)
- self.pipe.your_custom_generate(
+ pipe.your_custom_generate(
prompts,
- steps=self.num_inference_steps,
- cfg=self.guidance_scale,
+ steps=c.num_inference_steps,
+ cfg=c.guidance_scale,
generator=generator,
)
```
@@ -184,7 +192,10 @@ For models with both autoregressive and diffusion components (e.g., GLM-Image).
### 4a. Register AR component
-Edit `auto_round/compressors/diffusion/hybrid.py`:
+Add hybrid routing through the new architecture. Start with
+`auto_round/autoround.py`, `auto_round/compressors/entry.py`, and
+`auto_round/compressors/diffusion_mixin.py`.
+If a reusable AR-component registry is needed, place it near the new routing code:
```python
HYBRID_AR_COMPONENTS = [
@@ -198,10 +209,10 @@ The attribute name must match what exists on the diffusers pipeline object
### 4b. Register DiT block output config
-Also in `hybrid.py`, add the DiT-specific output config:
+Add the DiT-specific output config in `BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS`:
```python
-output_configs["YourDiTBlock"] = ["hidden_states", "encoder_hidden_states"]
+BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS["YourDiTBlock"] = ["hidden_states", "encoder_hidden_states"]
```
### 4c. Register AR block handler
@@ -223,7 +234,7 @@ SPECIAL_MULTIMODAL_BLOCK["your_model_type"] = _get_your_hybrid_multimodal_block
### Hybrid quantization flow
-The `HybridCompressor` runs two phases:
+The new hybrid flow should run two phases:
1. **Phase 1 (AR)**: Quantizes the AR component using text calibration data
(MLLM-style)
2. **Phase 2 (DiT)**: Quantizes the DiT component using diffusion pipeline
@@ -243,7 +254,10 @@ ar = AutoRound(
If your model needs a specific dataset format:
-Edit `auto_round/compressors/diffusion/dataset.py`:
+Edit the diffusion calibration path used by the new architecture:
+
+- `auto_round/calibration/diffusion.py` for how diffusion prompts are loaded and consumed
+- `auto_round/calib_dataset.py` for reusable dataset registration helpers
```python
def get_diffusion_dataloader(dataset_name, nsamples, ...):
@@ -288,12 +302,12 @@ ar = AutoRound(
- [ ] `is_diffusion_model()` detects model (or forced via extra_config)
- [ ] Transformer block class name identified
-- [ ] `output_configs` entry added with correct output tensor names and order
+- [ ] `DIFFUSION_OUTPUT_CONFIGS` entry added with correct output tensor names and order
- [ ] Pipeline runs without errors during calibration
-- [ ] Custom `pipeline_fn` provided if non-standard API
+- [ ] Custom pipeline dispatch added in `DiffusionCalibrator` if non-standard API
- [ ] For hybrid: AR component registered in `HYBRID_AR_COMPONENTS`
- [ ] For hybrid: AR block handler in `SPECIAL_MULTIMODAL_BLOCK`
-- [ ] For hybrid: DiT output config in `hybrid.py`
+- [ ] For hybrid: DiT output config in `BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS`
- [ ] Quantization produces valid layers (check "Quantized X/Y layers" log)
- [ ] Export to `fake` format works
- [ ] README.md + README_CN.md updated
@@ -302,9 +316,10 @@ ar = AutoRound(
| File | Purpose |
|------|---------|
-| `auto_round/compressors/diffusion/compressor.py` | `DiffusionCompressor`, `output_configs` dict |
-| `auto_round/compressors/diffusion/hybrid.py` | `HybridCompressor`, `HYBRID_AR_COMPONENTS` |
-| `auto_round/compressors/diffusion/dataset.py` | Calibration dataset loading |
+| `auto_round/algorithms/quantization/base.py` | `BaseQuantizers.DIFFUSION_OUTPUT_CONFIGS` |
+| `auto_round/calibration/diffusion.py` | `DiffusionCalibrator`, pipeline-driving calibration logic |
+| `auto_round/compressors/diffusion_mixin.py` | Diffusion compressor mixin and calibrator routing |
+| `auto_round/compressors/entry.py` | New-architecture `AutoRoundCompatible` factory routing |
| `auto_round/utils/model.py` | `is_diffusion_model()`, `diffusion_load_model()` |
| `auto_round/special_model_handler.py` | AR block handlers for hybrid models |
| `auto_round/autoround.py` | Model type routing (diffusion vs hybrid vs LLM) |
@@ -313,6 +328,6 @@ ar = AutoRound(
| Model | Type | What Was Adapted |
|-------|------|-----------------|
-| FLUX.1-dev | Pure DiT | `output_configs` for `FluxTransformerBlock`/`FluxSingleTransformerBlock` |
-| GLM-Image | Hybrid AR+DiT | `HYBRID_AR_COMPONENTS` + `SPECIAL_MULTIMODAL_BLOCK` + DiT `output_configs` |
-| NextStep | Custom pipeline | `pipeline_fn` parameter for non-standard inference API |
+| FLUX.1-dev | Pure DiT | `DIFFUSION_OUTPUT_CONFIGS` for `FluxTransformerBlock`/`FluxSingleTransformerBlock` |
+| GLM-Image | Hybrid AR+DiT | AR routing + `SPECIAL_MULTIMODAL_BLOCK` + DiT `DIFFUSION_OUTPUT_CONFIGS` |
+| NextStep | Custom pipeline | model-specific pipeline function attached by model handler / loader |
diff --git a/.claude/skills/adapt-new-llm/SKILL.md b/.claude/skills/adapt-new-llm/SKILL.md
index 21d50ee9a..3c9de24e6 100644
--- a/.claude/skills/adapt-new-llm/SKILL.md
+++ b/.claude/skills/adapt-new-llm/SKILL.md
@@ -291,5 +291,7 @@ def test_your_model_quantization():
| `auto_round/modeling/fused_moe/replace_modules.py` | MoE unfusing registry (`BUILTIN_MODULES`) |
| `auto_round/utils/common.py` | `SUPPORTED_LAYER_TYPES`, `INNER_SUPPORTED_LAYER_TYPES` |
| `auto_round/utils/model.py` | `get_block_names()`, `is_mllm_model()`, model loading |
-| `auto_round/compressors/base.py` | Core quantization loop, `calib()`, `_quantize_blocks()` |
+| `auto_round/compressors/data_driven.py` | New-architecture quantization loop and block scheduling |
+| `auto_round/algorithms/quantization/base.py` | Quantizer block execution, sampling, and diffusion output configs |
+| `auto_round/calibration/llm.py` | LLM calibration data collection and `calib()` flow |
| `auto_round/autoround.py` | `AutoRound` factory — model type routing logic |
diff --git a/.claude/skills/add-vlm-model/SKILL.md b/.claude/skills/add-vlm-model/SKILL.md
index 1dd5eff4e..8d91c2af2 100644
--- a/.claude/skills/add-vlm-model/SKILL.md
+++ b/.claude/skills/add-vlm-model/SKILL.md
@@ -14,7 +14,7 @@ multi-modal data.
The integration involves three parts:
1. **Multimodal Block Handler** — Tell AutoRound how to find quantizable blocks
-2. **Calibration Template** — Define how to build calibration prompts
+2. **MLLM Calibration Path** — Ensure `MLLMCalibrator` can build and feed calibration samples
3. **Special Model Handler** — Handle model-specific forward pass quirks
## Prerequisites
@@ -94,11 +94,29 @@ mllms_with_limited_bs = (
)
```
-## Step 2: Add Calibration Template
+## Step 2: Wire MLLM Calibration
-### 2a. Create template JSON
+The new architecture routes multimodal calibration through:
-Create `auto_round/compressors/mllm/templates/your_vlm.json`:
+- `auto_round/compressors/mllm_mixin.py` for compressor construction and calibrator selection
+- `auto_round/calibration/mllm.py` for template selection, dataloader creation, and calibration forward calls
+- `auto_round/special_model_handler.py` for multimodal block discovery and special forwards
+
+If your model works with an existing template/processor, prefer passing
+`template=...`, `processor=...`, or `image_processor=...` through `AutoRound` /
+`ExtraConfig` instead of adding compressor code.
+
+## Step 3: Add Calibration Template
+
+The built-in MLLM template and processor registries live in
+`auto_round/compressors/mllm/` and are consumed by the new architecture through
+`MLLMCalibrator`. When adding a new built-in template, keep the
+new-architecture caller in mind: `auto_round/calibration/mllm.py` will load it
+via `get_template()`.
+
+### 3a. Create template JSON
+
+Create a template JSON file in `auto_round/compressors/mllm/templates/`:
```json
{
@@ -116,9 +134,10 @@ Create `auto_round/compressors/mllm/templates/your_vlm.json`:
Adjust the template fields to match your model's chat format. Check the model's
`tokenizer_config.json` or documentation for the correct chat template.
-### 2b. Register the template
+### 3b. Register the template
-Edit `auto_round/compressors/mllm/template.py`:
+Register it in the MLLM template registry loaded by
+`auto_round/calibration/mllm.py`:
```python
_register_template(
@@ -128,10 +147,11 @@ _register_template(
)
```
-### 2c. Add a custom processor (if needed)
+### 3c. Add a custom processor (if needed)
If your model requires special image/prompt processing for calibration, create a
-processor in `auto_round/compressors/mllm/template.py`:
+processor in `auto_round/compressors/mllm/processor.py`, which is used by
+`MLLMCalibrator`:
```python
def _your_vlm_processor(raw_data, model_path, seqlen, processor=None, **kwargs):
@@ -155,7 +175,7 @@ Register it:
PROCESSORS["your_vlm"] = _your_vlm_processor
```
-## Step 3: Handle Special Forward Pass (If Needed)
+## Step 4: Handle Special Forward Pass (If Needed)
If your VLM's `forward()` method is non-standard (e.g., requires special
kwargs, has multiple model components that need separate handling), add a
@@ -181,7 +201,7 @@ def _handle_special_model(model):
return model
```
-## Step 4: Add Custom Calibration Dataset (Optional)
+## Step 5: Add Custom Calibration Dataset (Optional)
If your model needs a specialized calibration dataset loader, create one in
`auto_round/calib_dataset.py` using the `@register_dataset` decorator:
@@ -199,7 +219,7 @@ class YourVLMDataset:
yield sample
```
-## Step 5: Test
+## Step 6: Test
```python
def test_your_vlm_quantization():
@@ -226,7 +246,7 @@ ar = AutoRound(
)
```
-## Step 6: Update Documentation
+## Step 7: Update Documentation
1. Add your model to the supported VLM list in `README.md`
2. Update `README_CN.md` with the same changes (Chinese translation required)
@@ -251,7 +271,9 @@ ar = AutoRound(
| Block handler | `special_model_handler.py` | `SPECIAL_MULTIMODAL_BLOCK[model_type]` |
| Text-only support | `special_model_handler.py` | `SUPPORT_ONLY_TEXT_MODELS` list |
| Batch limit | `special_model_handler.py` | `mllms_with_limited_bs` tuple |
-| Template | `compressors/mllm/templates/*.json` | `_register_template()` |
-| Processor | `compressors/mllm/template.py` | `PROCESSORS` dict |
+| MLLM routing | `compressors/mllm_mixin.py` | `_get_calibrator_kind() -> "mllm"` |
+| MLLM calibration | `calibration/mllm.py` | `MLLMCalibrator.calib()` |
+| Template | `compressors/mllm/template.py` | `_register_template()` |
+| Processor | `compressors/mllm/processor.py` | `PROCESSORS` dict |
| Custom forward | `special_model_handler.py` | `_handle_special_model()` |
| Dataset loader | `calib_dataset.py` | `@register_dataset()` |
diff --git a/README.md b/README.md
index a57da7e71..ed97a394f 100644
--- a/README.md
+++ b/README.md
@@ -88,7 +88,7 @@ Use `--iters 0` for fast quantization with some accuracy drop for 4 bits. Detail
Quantize 7B models in about 10 minutes on a single GPU. Details are shown in [quantization costs](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#quantization-costs)
✅ **10+ VLMs Support**
-Out-of-the-box quantization for 10+ vision-language models [example models](https://huggingface.co/collections/OPEA/vlms-autoround-675bc712fdd6a55ebaf11bfa), [support matrix](https://github.com/intel/auto-round/tree/main/auto_round/mllm#support-matrix)
+Out-of-the-box quantization for 10+ vision-language models [example models](https://huggingface.co/collections/OPEA/vlms-autoround-675bc712fdd6a55ebaf11bfa), [support matrix](https://github.com/intel/auto-round/tree/main/auto_round/compressors/mllm#vlm-support-matrix)
✅ **Multiple Recipes**
Choose from `auto-round-best`, `auto-round`, and `auto-round-light` to suit your needs. Details are shown in [quantization recipes](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#recipe-recommendation)
diff --git a/README_CN.md b/README_CN.md
index cb2645f95..a1e3879ac 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -77,7 +77,7 @@ AutoRound 是专为大语言模型(LLMs)和视觉-语言模型(VLMs)设
✅ **可接受的量化成本** 在单张 GPU 上量化一个 7B 的模型只需约十分钟。详见:[量化成本](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#quantization-costs)
-✅ **支持十余种 VLM 模型** 已支持十余种视觉语言模型,让用户有“开盖即食”般的量化体验。详见:[示例模型](https://huggingface.co/collections/OPEA/vlms-autoround-675bc712fdd6a55ebaf11bfa),[支持矩阵](https://github.com/intel/auto-round/tree/main/auto_round/mllm#support-matrix)
+✅ **支持十余种 VLM 模型** 已支持十余种视觉语言模型,让用户有“开盖即食”般的量化体验。详见:[示例模型](https://huggingface.co/collections/OPEA/vlms-autoround-675bc712fdd6a55ebaf11bfa),[支持矩阵](https://github.com/intel/auto-round/tree/main/auto_round/compressors/mllm#vlm-support-matrix)
✅ **多种量化方案可选** 提供 `auto-round-best`​、`auto-round`​、`auto-round-light`​ 等多种预设方案,能够满足多样化需求。详见:[量化方案](https://github.com/intel/auto-round/blob/main/docs/step_by_step.md#recipe-recommendation)
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 42a230804..09385061d 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -20,7 +20,7 @@ import sys
import torch
from auto_round.auto_scheme import AutoScheme
-from auto_round.compressors import BaseCompressor
+from auto_round.compressors.base import BaseCompressor
from auto_round.eval.eval_cli import EvalArgumentParser, eval, eval_task_by_task
from auto_round.eval.evaluation import run_model_evaluation
from auto_round.schemes import PRESET_SCHEMES, preset_name_to_scheme
@@ -799,7 +799,7 @@ def tune(args):
)
rot_config = None
if args.rotation_type:
- from auto_round.experimental.transform.rotation_config import RotationConfig
+ from auto_round.algorithms.transforms.rotation.config import RotationConfig
rot_config = RotationConfig(hadamard_type=args.rotation_type)
diff --git a/auto_round/algorithms/quantization/__init__.py b/auto_round/algorithms/quantization/__init__.py
index 6a727f31b..3d8ca932b 100644
--- a/auto_round/algorithms/quantization/__init__.py
+++ b/auto_round/algorithms/quantization/__init__.py
@@ -16,6 +16,7 @@ from auto_round.algorithms.quantization.base import BaseQuantizers
from auto_round.algorithms.quantization.config import QuantizationConfig
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
+from auto_round.algorithms.quantization.sign_roundv2 import SignRoundV2Quantizer
from auto_round.algorithms.quantization.adam_round.adam import AdamRoundQuantizer
from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.algorithms.quantization.rtn.quantizer import RTNQuantizer, OptimizedRTNQuantizer
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.py
index 29c0a70f3..fd39f547e 100644
--- a/auto_round/algorithms/quantization/base.py
+++ b/auto_round/algorithms/quantization/base.py
@@ -19,7 +19,7 @@ from typing import Union
import torch
from auto_round.algorithms.quantization.config import QuantizationConfig
-from auto_round.compressors_new.utils import (
+from auto_round.compressors.utils import (
block_forward,
check_need_act_calibration,
)
@@ -73,11 +73,14 @@ class BaseQuantizers:
self.ignore_layers = config.ignore_layers
self.quant_lm_head = config.quant_lm_head
self.to_quant_block_names = config.to_quant_block_names
- # Calibration / sampling attrs – synced from compressor in post_init.
- self.seqlen = 2048
- self.nsamples = 128
- self.batch_size = getattr(config, "batch_size", 8)
- self.batch_dim = getattr(config, "batch_dim", None)
+ # Calibration-time state lives on a shared
+ # :class:`~auto_round.calibration.state.CalibrationState` instance.
+ # The compressor wires its own instance here in ``_resolve_scheme``;
+ # until then we own a private placeholder so property-based reads /
+ # writes during construction don't blow up.
+ from auto_round.calibration.state import CalibrationState
+
+ self._calibration_state = CalibrationState()
self.infer_bs_coeff = getattr(config, "infer_bs_coeff", 1)
# Whether to feed quantized-block outputs as inputs to the next block.
# Subclasses that support cascaded quantized-input (e.g. SignRoundQuantizer)
@@ -85,6 +88,48 @@ class BaseQuantizers:
# (RTN) where activations are not used during weight optimization.
self.enable_quanted_input = getattr(config, "enable_quanted_input", False)
+ # ── Shared CalibrationState forwarders ───────────────────────────────────────
+ @property
+ def calibration_state(self):
+ return self._calibration_state
+
+ @calibration_state.setter
+ def calibration_state(self, new_state) -> None:
+ # Compressor-supplied shared instance; just rebind.
+ self._calibration_state = new_state
+
+ @property
+ def attention_mask(self) -> list:
+ return self._calibration_state.attention_mask
+
+ @attention_mask.setter
+ def attention_mask(self, value) -> None:
+ self._calibration_state.attention_mask = value if value is not None else []
+
+ @property
+ def batch_dim(self):
+ return self._calibration_state.batch_dim
+
+ @batch_dim.setter
+ def batch_dim(self, value) -> None:
+ self._calibration_state.batch_dim = value
+
+ @property
+ def batch_size(self) -> int:
+ return self._calibration_state.batch_size
+
+ @batch_size.setter
+ def batch_size(self, value) -> None:
+ self._calibration_state.batch_size = value
+
+ @property
+ def nsamples(self) -> int:
+ return self._calibration_state.nsamples
+
+ @property
+ def seqlen(self) -> int:
+ return self._calibration_state.seqlen
+
@classmethod
def from_config(cls, config: QuantizationConfig):
if cls.__name__ == config._alg_cls:
@@ -94,6 +139,24 @@ class BaseQuantizers:
alg_cls = getattr(module, config._alg_cls)
return alg_cls(config)
+ def bind(self, compressor) -> None:
+ """Wire shared state from the owning compressor.
+
+ The compressor owns the authoritative ``model_context`` /
+ ``compress_context`` / ``CalibrationState`` and resolves
+ ``scale_dtype`` (string → torch dtype). All quantizer fields that
+ merely mirror the compressor are pulled from here in one place.
+ """
+ self.model_context = compressor.model_context
+ self.compress_context = compressor.compress_context
+ self.scale_dtype = compressor.scale_dtype
+ # Share the compressor's CalibrationState instance.
+ self._calibration_state = compressor._calibration_state
+
+ @property
+ def model(self):
+ return self.model_context.model if self.model_context is not None else None
+
@property
def formats(self):
return getattr(self.compress_context, "formats", None)
diff --git a/auto_round/algorithms/quantization/rtn/config.py b/auto_round/algorithms/quantization/rtn/config.py
index 6afc41b0f..5e27dbccb 100644
--- a/auto_round/algorithms/quantization/rtn/config.py
+++ b/auto_round/algorithms/quantization/rtn/config.py
@@ -24,19 +24,14 @@ class RTNConfig(QuantizationConfig):
self,
*,
disable_opt_rtn: bool = None,
- # for opt-rtn
- batch_size: int = 8,
**kwargs,
):
# pop before super().__init__ so it doesn't leak into QuantizationConfig as an unknown kwarg
enable_opt_rtn = kwargs.pop("enable_opt_rtn", None)
super().__init__(**kwargs)
- self.batch_size = batch_size
-
# Some helpers
self.infer_bs_coeff = 1
- self.batch_dim = None
if enable_opt_rtn:
disable_opt_rtn = False
diff --git a/auto_round/algorithms/quantization/rtn/quantizer.py b/auto_round/algorithms/quantization/rtn/quantizer.py
index 07aaf7697..af8312f26 100644
--- a/auto_round/algorithms/quantization/rtn/quantizer.py
+++ b/auto_round/algorithms/quantization/rtn/quantizer.py
@@ -21,8 +21,8 @@ import torch
from auto_round.algorithms.quantization.base import BaseQuantizers
from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
-from auto_round.compressors_new.shard_writer import ShardWriter
-from auto_round.compressors_new.utils import (
+from auto_round.compressors.shard_writer import ShardWriter
+from auto_round.compressors.utils import (
IndexSampler,
block_forward,
check_need_act_calibration,
@@ -210,8 +210,6 @@ class OptimizedRTNQuantizer(RTNQuantizer):
def __init__(self, config: RTNConfig):
BaseQuantizers.__init__(self, config)
- self.batch_size = config.batch_size
- self.batch_dim = config.batch_dim
self.data_type = config.data_type
self.group_size = config.group_size
self.infer_bs_coeff = config.infer_bs_coeff
diff --git a/auto_round/algorithms/quantization/sign_round/config.py b/auto_round/algorithms/quantization/sign_round/config.py
index 90b7d86b1..2aad5e08e 100644
--- a/auto_round/algorithms/quantization/sign_round/config.py
+++ b/auto_round/algorithms/quantization/sign_round/config.py
@@ -25,7 +25,6 @@ class SignRoundConfig(QuantizationConfig):
lr (float): The learning rate (default is 0.005).
minmax_lr (float): The learning rate for min-max tuning (default is None).
lr_scheduler: The learning rate scheduler to be used.
- batch_size (int): Batch size for training (default is 8).
enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
"""
@@ -40,7 +39,6 @@ class SignRoundConfig(QuantizationConfig):
minmax_lr: float = None,
lr_scheduler=None,
momentum: float = 0.0,
- batch_size: int = 8,
nblocks: int = 1,
enable_minmax_tuning: bool = True,
enable_norm_bias_tuning: bool = False,
@@ -54,6 +52,7 @@ class SignRoundConfig(QuantizationConfig):
**kwargs,
):
super().__init__(**kwargs)
+ self.gradient_accumulate_steps = gradient_accumulate_steps
self.iters = iters
if self.iters < 0:
logger.warning("`iters` must be non-negative, reset it to 200")
@@ -71,14 +70,12 @@ class SignRoundConfig(QuantizationConfig):
self.minmax_lr = minmax_lr or self.lr
self.lr_scheduler = lr_scheduler
- self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps
self.nblocks = nblocks
self.momentum = momentum
self.enable_alg_ext = enable_alg_ext
# Some helpers
self.infer_bs_coeff = 1
- self.batch_dim = None
self.enable_minmax_tuning = enable_minmax_tuning
self.enable_norm_bias_tuning = enable_norm_bias_tuning
@@ -92,6 +89,8 @@ class SignRoundConfig(QuantizationConfig):
if self.enable_adam:
self._alg_cls = "AdamRoundQuantizer"
+ elif self.enable_alg_ext:
+ self._alg_cls = "SignRoundV2Quantizer"
def check_configs(self) -> None:
"""Checks if the configurations are valid.
@@ -101,8 +100,6 @@ class SignRoundConfig(QuantizationConfig):
"""
super().check_config()
- if self.batch_size <= 0:
- raise ValueError("`batch_size` must be positive")
if self.iters < 0:
raise ValueError("`iters` must be non-negative")
if self.nblocks <= 0:
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index 1f7b5b539..e0859d260 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -24,7 +24,7 @@ from torch import autocast
from auto_round.algorithms.quantization.base import BaseQuantizers
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
from auto_round.algorithms.quantization.sign_round.sign_sgd import SignSGD
-from auto_round.compressors_new.utils import (
+from auto_round.compressors.utils import (
IndexSampler,
block_forward,
check_need_act_calibration,
@@ -58,14 +58,10 @@ class SignRoundQuantizer(BaseQuantizers):
def __init__(self, config: SignRoundConfig):
super().__init__(config)
- self.attention_mask = []
-
self.iters = config.iters
self.lr = config.lr
self.minmax_lr = config.minmax_lr
self.lr_scheduler = config.lr_scheduler
- self.batch_size = config.batch_size
- self.batch_dim = config.batch_dim
self.momentum = config.momentum
self.infer_bs_coeff = config.infer_bs_coeff
self.enable_minmax_tuning = config.enable_minmax_tuning
@@ -79,15 +75,6 @@ class SignRoundQuantizer(BaseQuantizers):
self.optimizer = self._get_optimizer(optimizer=config.optimizer)
self.wrapper_block = wrapper_block
- if self.enable_alg_ext:
- try:
- logger.info("using algorithm extension for quantization.")
- from auto_round.alg_ext import wrapper_autoround
-
- wrapper_autoround(self)
- except (ImportError, ModuleNotFoundError):
- logger.error("algorithm extension import error, fallback to default mode")
-
def _get_current_output(self, output: list[torch.Tensor], indices: list[int]) -> torch.Tensor:
if self.model_context.is_diffusion:
assert "hidden_states" in output
diff --git a/auto_round/algorithms/quantization/sign_roundv2/__init__.py b/auto_round/algorithms/quantization/sign_roundv2/__init__.py
new file mode 100644
index 000000000..4eb376f53
--- /dev/null
+++ b/auto_round/algorithms/quantization/sign_roundv2/__init__.py
@@ -0,0 +1,25 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.algorithms.quantization.sign_roundv2.quantizer import (
+ SignRoundDQWrapperLinear,
+ SignRoundOptimizedWrapperLinear,
+ SignRoundV2Quantizer,
+)
+
+__all__ = [
+ "SignRoundDQWrapperLinear",
+ "SignRoundOptimizedWrapperLinear",
+ "SignRoundV2Quantizer",
+]
diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
new file mode 100644
index 000000000..36e5ab9dc
--- /dev/null
+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py
@@ -0,0 +1,224 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from contextlib import nullcontext
+from functools import partial
+from typing import Callable, Union
+
+import torch
+import transformers
+from torch import autocast
+
+from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
+from auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer
+from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq, quant_tensor_gguf_sym_dq
+from auto_round.data_type.int import quant_tensor_asym, quant_tensor_sym, search_scales
+from auto_round.data_type.mxfp import quant_mx, search_mx_scale
+from auto_round.data_type.nvfp import nv_fp4, search_nvfp4_scale
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+from auto_round.logger import logger
+from auto_round.utils import check_to_quantized, compile_func
+from auto_round.wrapper import WrapperLinear, wrapper_block
+
+
+def _named_wrapper_block(wrapper_cls, name: str):
+ wrapped = partial(wrapper_block, wrapper_cls=wrapper_cls)
+ wrapped.__name__ = name
+ return wrapped
+
+
+class SignRoundOptimizedWrapperLinear(WrapperLinear):
+ minmax_scale_bound = (0.0, 2.0)
+
+ def _init_tuning_params_and_quant_func(self):
+ super()._init_tuning_params_and_quant_func()
+
+ orig_weight = getattr(self.orig_layer, "get_weight", lambda: self.orig_layer.weight)()
+ if type(self.orig_layer) == transformers.pytorch_utils.Conv1D:
+ orig_weight = orig_weight.t()
+ weight_reshape, _, _ = reshape_pad_tensor_by_group_size(orig_weight.data, self.orig_layer.group_size)
+ if hasattr(self.orig_layer, "imatrix"):
+ imatrix = self.orig_layer.imatrix.reshape(1, -1)
+ imatrix = reshape_pad_tensor_by_group_size(imatrix, self.orig_layer.group_size, val=1e-5)[0].view(1, -1)
+ imatrix = imatrix.expand(weight_reshape.numel() // imatrix.numel(), -1)
+ imatrix = imatrix.reshape(weight_reshape.shape).to(orig_weight.device)
+ else:
+ imatrix = 1.0
+
+ if self.orig_layer.data_type.startswith("int"):
+ self.init_scale = search_scales(weight_reshape, self.orig_layer.bits, imatrix)
+ self.init_scale = torch.where(
+ self.init_scale < 0,
+ torch.clamp(self.init_scale, max=-self.q_scale_thresh),
+ torch.clamp(self.init_scale, min=self.q_scale_thresh),
+ )
+ self.weight_quant_func = quant_tensor_sym
+ elif self.orig_layer.data_type.startswith("mx"):
+ self.init_scale = search_mx_scale(weight_reshape, self.orig_layer.bits, imatrix)
+ self.weight_quant_func = quant_mx
+ elif self.orig_layer.data_type.startswith("nv"):
+ self.init_scale = search_nvfp4_scale(weight_reshape, self.orig_layer.bits, imatrix)
+ self.weight_quant_func = nv_fp4
+ else:
+ raise ValueError(f"unsupported SignRound optimized data type: {self.orig_layer.data_type}")
+
+ self.data_type = self.orig_layer.data_type
+ if hasattr(self.orig_layer, "imatrix"):
+ delattr(self.orig_layer, "imatrix")
+ if self.enable_torch_compile:
+ self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
+
+
+class SignRoundDQWrapperLinear(WrapperLinear):
+ minmax_scale_bound = (0.5, 1.5)
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.prev_scale = None
+ self.prev_wmin = None
+ self.prev_d_scale = None
+ self.prev_d_wmin = None
+
+ def _init_tuning_params_and_quant_func(self):
+ super()._init_tuning_params_and_quant_func()
+ if hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:
+ self.weight_quant_func = (
+ quant_tensor_gguf_asym_dq if self.orig_layer.data_type == "int_asym_dq" else quant_tensor_gguf_sym_dq
+ )
+ elif self.orig_layer.sym:
+ self.weight_quant_func = quant_tensor_sym
+ else:
+ self.weight_quant_func = quant_tensor_asym
+ self.data_type = self.orig_layer.data_type
+ if self.enable_act_quant:
+ self.act_quant_func = (
+ quant_tensor_gguf_asym_dq
+ if self.orig_layer.act_data_type == "int_asym_dq"
+ else quant_tensor_gguf_sym_dq
+ )
+ if self.enable_torch_compile:
+ self.act_quant_func = compile_func(self.act_quant_func, self.device)
+ self._init_params("act_max_scale", torch.float32, (1), 1.0, not self.orig_layer.act_dynamic)
+ if self.enable_torch_compile:
+ self.weight_quant_func = compile_func(self.weight_quant_func, self.device)
+
+ def _qdq_weight(self, value, min_scale, max_scale):
+ weight_q, scale, zp = super()._qdq_weight(value, min_scale, max_scale)
+ if isinstance(scale, dict) and "d_scale" in scale and self.prev_scale is None:
+ self.prev_scale = scale["scale"]
+ self.prev_d_scale = scale["d_scale"]
+ if isinstance(zp, dict):
+ self.prev_wmin = zp["wmin"]
+ self.prev_d_wmin = zp["d_wmin"]
+ elif self.prev_scale is None:
+ self.prev_scale = scale
+ return weight_q, scale, zp
+
+ def _extra_quant_kwargs(self):
+ return {
+ "prev_scale": self.prev_scale,
+ "prev_wmin": self.prev_wmin,
+ "prev_d_scale": self.prev_d_scale,
+ "prev_d_wmin": self.prev_d_wmin,
+ "iter": getattr(self, "cur_iter", None),
+ }
+
+
+class SignRoundV2Quantizer(SignRoundQuantizer):
+ """SignRound variant using the open algorithm-extension path in the new architecture."""
+
+ def __init__(self, config: SignRoundConfig):
+ super().__init__(config)
+ self._use_outlier_suppressed_loss = False
+ logger.info("using algorithm extension for quantization.")
+
+ if (
+ self.sym
+ and self.super_group_size is None
+ and (self.data_type.startswith("int") or self.data_type.startswith("mx") or self.data_type.startswith("nv"))
+ ):
+ if self.bits > 2 and not (self.data_type.startswith("mx") or self.data_type.startswith("nv")):
+ logger.warning_once(
+ "algorithm extension has only undergone limited validation on "
+ "W2A16,INT4, MXFP4 and NVFP4; use with caution."
+ )
+ self._use_outlier_suppressed_loss = True
+ self.wrapper_block = _named_wrapper_block(SignRoundOptimizedWrapperLinear, "wrapper_block")
+
+ if self.data_type.endswith("dq"):
+ self.wrapper_block = _named_wrapper_block(SignRoundDQWrapperLinear, "dq_wrapper_block")
+
+ def _get_loss(
+ self,
+ output_q: torch.Tensor,
+ current_output: torch.Tensor,
+ indices: torch.Tensor,
+ mse_loss: Callable,
+ device: Union[str, torch.device] = "cpu",
+ ):
+ if self._use_outlier_suppressed_loss:
+ loss_diff = torch.abs(output_q - current_output)
+ flat_diff = loss_diff.view(-1)
+ topk = max(1, int(flat_diff.numel() / 1000))
+ _, top_indices = torch.topk(torch.abs(flat_diff), topk)
+ mask = torch.zeros_like(flat_diff, dtype=torch.bool)
+ mask[top_indices] = True
+ mask = (~mask).view_as(loss_diff)
+
+ autocast_ctx = (
+ autocast(device_type=str(device).split(":")[0], dtype=self.amp_dtype) if self.amp else nullcontext()
+ )
+ if self.attention_mask:
+ tmp_attention_mask = [self.attention_mask[i] for i in indices]
+ tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(device)
+ tmp_attention_mask.unsqueeze_(-1)
+ with autocast_ctx:
+ return torch.mean(
+ (
+ torch.abs(output_q.to(torch.float32) - current_output.to(torch.float32))
+ * tmp_attention_mask
+ * mask
+ )
+ ** 2
+ )
+
+ with autocast_ctx:
+ return torch.mean(
+ (torch.abs(output_q.to(torch.float32) - current_output.to(torch.float32)) * mask) ** 2
+ )
+ return super()._get_loss(output_q, current_output, indices, mse_loss, device)
+
+ def _register_act_max_hook(self, model):
+ hook_handles = super()._register_act_max_hook(model)
+
+ is_wint4aint4 = ("int4" in self.act_data_type or ("int" in self.act_data_type and self.act_bits == 4)) and (
+ "int4" in self.data_type or ("int" in self.data_type and self.bits == 4)
+ )
+ if is_wint4aint4:
+ return hook_handles
+
+ def get_imatrix_hook(module, input, output):
+ input = input[0] if isinstance(input, (tuple, list)) else input
+ flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
+ squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)
+
+ if not hasattr(module, "imatrix"):
+ module.imatrix = squared
+ else:
+ module.imatrix += squared.to(module.imatrix.device)
+
+ for _, module in model.named_modules():
+ if isinstance(module, self.supported_types) and check_to_quantized(module):
+ hook_handles.append(module.register_forward_hook(get_imatrix_hook))
+ return hook_handles
diff --git a/auto_round/algorithms/transforms/rotation/config.py b/auto_round/algorithms/transforms/rotation/config.py
index 2a8c50697..629714557 100644
--- a/auto_round/algorithms/transforms/rotation/config.py
+++ b/auto_round/algorithms/transforms/rotation/config.py
@@ -14,13 +14,12 @@
"""Rotation/transform configuration (canonical, unified).
This module is the **single source of truth** for the ``RotationConfig``
-schema. The legacy location
-``auto_round.experimental.transform.rotation_config`` re-exports from here.
+schema.
Two implementation backends share this one schema (method B):
* ``backend="inplace"`` – QuaRot-style residual-stream rotation, implemented
- under :mod:`auto_round.experimental.rotation_inplace`. Works for any
+ under :mod:`auto_round.algorithms.transforms.rotation.inplace`. Works for any
weight/activation dtype and can optionally fuse the online Hadamard into
weights (``fuse_online_to_weight=True``).
@@ -50,7 +49,6 @@ __all__ = [
"dump_group_size_to_rotation_config",
]
-
# Supported Hadamard transform types (also used by HadamardTransform registry).
HADAMARD_TYPES: frozenset[str] = frozenset({"hadamard", "random_hadamard", "quarot_hadamard"})
_SUPPORTED_BACKENDS: frozenset[str] = frozenset({"auto", "inplace", "transform"})
diff --git a/auto_round/algorithms/transforms/rotation/dispatcher.py b/auto_round/algorithms/transforms/rotation/dispatcher.py
index ecae6652f..ef47c2e56 100644
--- a/auto_round/algorithms/transforms/rotation/dispatcher.py
+++ b/auto_round/algorithms/transforms/rotation/dispatcher.py
@@ -5,7 +5,6 @@
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
-
"""Unified entry point for Hadamard rotation/transform.
Two backend implementations exist:
@@ -14,7 +13,7 @@ Two backend implementations exist:
QuaRot-style residual-stream rotation. Works for any weight/activation
dtype. Optionally fuses the online Hadamard into weights
(``fuse_online_to_weight=True``).
-* ``transform`` – :mod:`auto_round.experimental.transform`
+* ``transform`` – :mod:`auto_round.algorithms.transforms.rotation`
Per-Linear weight + activation Hadamard with a fused triton kernel.
Only supports MXFP4 / NVFP4 and **cannot** fuse online to weight.
diff --git a/auto_round/algorithms/transforms/rotation/inplace/__init__.py b/auto_round/algorithms/transforms/rotation/inplace/__init__.py
index 9bef07da9..d7b85d9b1 100644
--- a/auto_round/algorithms/transforms/rotation/inplace/__init__.py
+++ b/auto_round/algorithms/transforms/rotation/inplace/__init__.py
@@ -2,8 +2,7 @@
# # SPDX-License-Identifier: Apache-2.0
"""Inplace (QuaRot-style) Hadamard rotation backend.
-Canonical home of the residual-stream Hadamard rotation implementation
-(formerly under :mod:`auto_round.experimental.rotation_inplace`).
+Canonical home of the residual-stream Hadamard rotation implementation.
"""
from auto_round.algorithms.transforms.rotation.inplace.apply import apply_rotation_transform # noqa: F401
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 7a4271647..12b841ace 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -17,26 +17,13 @@ from typing import TYPE_CHECKING, Optional, Union
import torch
-import auto_round.envs as envs
-from auto_round.compressors import (
- AdamCompressor,
- BaseCompressor,
- DiffusionCompressor,
- ExtraConfig,
- LLMCompressor,
- MLLMCompressor,
-)
-from auto_round.compressors.diffusion.hybrid import HybridCompressor, is_hybrid_diffusion_model
-from auto_round.compressors.model_free import ModelFreeCompressor
from auto_round.logger import deprecated, logger
from auto_round.schemes import QuantizationScheme
-from auto_round.utils import is_diffusion_model, is_mllm_model, is_model_free_route
if TYPE_CHECKING:
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
-
-# Default to new architecture; set AR_DISABLE_NEW_ARCH=true/1 to force old architecture.
-NEW_ARCH = not envs.AR_DISABLE_NEW_ARCH
+ from auto_round.compressors.base import BaseCompressor
+ from auto_round.compressors.config import ExtraConfig
class AutoRound:
@@ -91,12 +78,12 @@ class AutoRound:
enable_torch_compile: bool = False,
seed: int = 42,
enable_adam: bool = False,
- extra_config: ExtraConfig = None,
+ extra_config: "ExtraConfig" = None,
enable_alg_ext: bool = False,
disable_opt_rtn: bool | None = None,
low_cpu_mem_usage: bool = True,
**kwargs,
- ) -> BaseCompressor:
+ ) -> "BaseCompressor":
"""Initialize AutoRound with quantization and tuning configuration.
Args:
@@ -167,73 +154,9 @@ class AutoRound:
if extra_config is not None:
local_args.update({k: v for k, v in extra_config.to_dict().items() if k in local_args and v is not None})
- if NEW_ARCH:
- from auto_round.compressors_new.entry import AutoRoundCompatible
-
- return AutoRoundCompatible(**local_args, **kwargs)
-
- # ---- Model-free fast-path detection --------------------------------
- if is_model_free_route(model, scheme, iters, disable_opt_rtn, kwargs):
- if not isinstance(model, str):
- raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")
- if not bool(kwargs.get("model_free", False)):
- logger.info(
- "Auto-routing to model-free quantization "
- "(iters=0, disable_opt_rtn=True, supported scheme). "
- "Pass disable_model_free=True to use the regular flow."
- )
- if extra_config is not None:
- local_args.update(extra_config.to_dict())
- local_args["model_name_or_path"] = local_args.pop("model")
- return ModelFreeCompressor(**local_args, **kwargs)
- # --------------------------------------------------------------------
-
- model_cls = []
-
- has_multimodal_assets = kwargs.get("processor") is not None or kwargs.get("image_processor") is not None
-
- if is_hybrid_diffusion_model(model):
- logger.info("using Hybrid AR+Diffusion mode for hybrid model.")
- model_cls.append(HybridCompressor)
- if extra_config:
- extra_config.mllm_config = None
- extra_config.diffusion_config = None
- elif (
- (extra_config and not extra_config.mllm_config.is_default())
- or has_multimodal_assets
- or is_mllm_model(model, platform=platform)
- ):
- logger.info("using MLLM mode for multimodal model.")
- model_cls.append(MLLMCompressor)
- if extra_config:
- extra_config.diffusion_config = None
- elif (extra_config and not extra_config.diffusion_config.is_default()) or is_diffusion_model(model):
- logger.info("using Diffusion mode for diffusion model.")
- model_cls.append(DiffusionCompressor)
- if extra_config:
- extra_config.mllm_config = None
- else:
- if extra_config:
- extra_config.mllm_config = None
- extra_config.diffusion_config = None
- model_cls.append(LLMCompressor)
-
- if enable_adam:
- model_cls.append(AdamCompressor)
- dynamic_compressor = type("AutoRound", tuple(model_cls), {})
- if "fp_layers" in kwargs:
- logger.warning_once(
- "'fp_layers' is deprecated, please use 'ignore_layers' to set layers not to be quantized."
- )
- kwargs["ignore_layers"] = kwargs.pop("fp_layers")
+ from auto_round.compressors.entry import AutoRoundCompatible
- if extra_config is not None:
- local_args.update(extra_config.to_dict())
- ar = dynamic_compressor(
- **local_args,
- **kwargs,
- )
- return ar
+ return AutoRoundCompatible(**local_args, **kwargs)
@classmethod
@torch.no_grad()
@@ -288,347 +211,29 @@ class AutoRound:
@deprecated("AutoRound")
-class AutoRoundLLM(LLMCompressor):
- """Class for LLM quantization
-
- Args:
- model (torch.nn.Module | str): Model object or model name to load.
- tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
- scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
- bits (int, optional): Weight quantization bits. Defaults to 4.
- group_size (int or tuple, optional): Weight quantization group size. Defaults to 128.
- sym (bool, optional): Symmetric weight quantization. Defaults to True.
- layer_config (dict, optional): Layer-wise quantization config. Defaults to None.
- batch_size (int, optional): Calibration batch size. Defaults to 8.
- amp (bool, optional): Use AMP for tuning. Defaults to True.
- device (str | torch.device | int, optional): Compute device. Defaults to 0.
- dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".
- enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
- lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
- minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
- low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
- iters (int, optional): Optimization iterations. Defaults to 200.
- seqlen (int, optional): Calibration sequence length. Defaults to 2048.
- nsamples (int, optional): Number of calibration samples. Defaults to 128.
- seed (int, optional): Random seed. Defaults to 42.
- gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.
- data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
- act_bits (int, optional): Activation quantization bits. Defaults to 16.
- act_group_size (int, optional): Activation group size. Defaults to None.
- act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
- act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
- act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
- enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.
- device_map (str | dict, optional): Device placement map. Defaults to None.
- disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to None.
- enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
- **kwargs: Backward compatible options:
- - enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,
- super_group_size, super_bits, scale_dtype ("fp16" etc.),
- nblocks, to_quant_block_names,
- enable_norm_bias_tuning, enable_quanted_input,
- disable_deterministic_algorithms, mllm, static_kv_dtype
- Raises:
- ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.
- RuntimeError: If model parameters are on meta device.
- Example:
- Layer-wise configuration structure:
-
- >>> layer_config = {
- ... "layer1": {
- ... "data_type": "int",
- ... "bits": 4,
- ... "group_size": 128,
- ... "sym": True,
- ... "act_data_type": None,
- ... "act_bits": 16,
- ... "act_group_size": None,
- ... "act_sym": None,
- ... },
- ... # ...
- ... }
- """
-
- bits: int | None
- group_size: int | tuple | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
+class AutoRoundLLM:
- def __init__(
- self,
- model: Union[torch.nn.Module, str],
- tokenizer=None,
- platform: str = "hf",
- scheme: Union[str, dict, QuantizationScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- **kwargs,
- ):
- local_args = {k: v for k, v in locals().items() if k not in ("local_args", "kwargs", "self")}
- super().__init__(
- **local_args,
- **kwargs,
- )
+ def __new__(cls, *args, **kwargs):
+ return AutoRound(*args, **kwargs)
@deprecated("AutoRound")
-class AutoRoundAdam(AdamCompressor):
- """Class for quantization with optimizers like adamw of a PyTorch model.
-
- Args:
- model: The PyTorch model to be quantized.
- tokenizer: An optional tokenizer for processing input data.
- platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
- scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
- bits (int): Number of bits for quantization (default is 4).
- group_size (int or tuple): Size of the quantization group (default is 128).
- sym (bool): Whether sym to be used (default is True).
- layer_config (dict): Configuration for weight quantization (default is None).
- batch_size (int): Batch size for training (default is 8).
- amp (bool): Whether to use automatic mixed precision (default is True).
- device: The device to be used for training (default is "auto").
- lr_scheduler: The learning rate scheduler to be used.
- dataset: The default dataset name (default is "NeelNanda/pile-10k").
- enable_quanted_input (bool): Whether to use quantized input data (default is True).
- enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
- lr (float): The learning rate (default is 0.005).
- minmax_lr (float): The learning rate for min-max tuning (default is None).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
- iters (int): Number of iterations (default is 200).
- seqlen (int): Length of the sequence.
- nsamples (int): Number of samples (default is 128).
- sampler (str): The sampling method (default is "rand").
- seed (int): The random seed (default is 42).
- nblocks (int): Number of blocks (default is 1).
- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- not_use_best_mse (bool): Whether to use mean squared error (default is False).
- dynamic_max_gap (int): The dynamic maximum gap (default is -1).
- data_type (str): The data type to be used (default is "int").
- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
- have different choices.
- act_bits (int): Number of bits for activation quantization. Default is 16.
- act_group_size (int): Group size for activation quantization. Default is None.
- act_sym (bool): Whether to use symmetric activation quantization. Default is None.
- act_data_type (str): Specifies the data type for activations.
- Defaults to None, in which case it inherits the weight data type.
- act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- to_quant_block_names (str|list): A string or list whose elements are list of
- block's layer names to be quantized.
- enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function
- **kwargs: Additional keyword arguments.
-
- Returns:
- The quantized model.
- """
-
- bits: int | None
- group_size: int | tuple | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
+class AutoRoundAdam:
- def __init__(
- self,
- model: Union[torch.nn.Module, str],
- tokenizer=None,
- platform: str = "hf",
- scheme: Union[str, dict, QuantizationScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, int, torch.device, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- optimizer="AdamW",
- **kwargs,
- ):
- local_args = {k: v for k, v in locals().items() if k not in ("local_args", "kwargs", "self")}
- super().__init__(
- **local_args,
- **kwargs,
- )
+ def __new__(cls, *args, **kwargs):
+ kwargs.setdefault("enable_adam", True)
+ return AutoRound(*args, **kwargs)
@deprecated("AutoRound")
-class AutoRoundMLLM(MLLMCompressor):
- """Class for automatic rounding-based quantization with MLLMs.
-
- Args:
- model: The PyTorch model to be quantized.
- tokenizer: An optional tokenizer for processing input data.
- platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
- processor: Any multi-modal model will require an object to encode or
- decode the data that groups several modalities (among text, vision and audio).
- image_processor: Image processor for special model like llava.
- bits (int): Number of bits for quantization (default is 4).
- group_size (int or tuple): Size of the quantization group (default is 128).
- sym (bool): Whether sym to be used (default is True).
- layer_config (dict): Configuration for weight quantization (default is None).
- batch_size (int): Batch size for training (default is 8).
- amp (bool): Whether to use automatic mixed precision (default is True).
- device: The device to be used for training (default is "auto").
- lr_scheduler: The learning rate scheduler to be used.
- dataset: The path or name of the calib dataset.
- extra_data_dir: The path of extra data such as images, audio and videos.
- template: The path or name of template used to specify process for different MLLMs.
- quant_nontext_module: Whether to quantize nontext module.
- enable_quanted_input (bool): Whether to use quantized input data (default is True).
- enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
- lr (float): The learning rate (default is 0.005).
- minmax_lr (float): The learning rate for min-max tuning (default is None).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
- iters (int): Number of iterations (default is 200).
- seqlen (int): Length of the sequence.
- nsamples (int): Number of samples (default is 128).
- sampler (str): The sampling method (default is "rand").
- seed (int): The random seed (default is 42).
- nblocks (int): Number of blocks (default is 1).
- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- not_use_best_mse (bool): Whether to use mean squared error (default is False).
- dynamic_max_gap (int): The dynamic maximum gap (default is -1).
- data_type (str): The data type to be used (default is "int").
- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
- have different choices.
- act_bits (int): Number of bits for activation quantization. Default is 32.
- act_group_size (int): Group size for activation quantization. Default is None.
- act_sym (bool): Whether to use symmetric activation quantization. Default is None.
- act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- to_quant_block_names (str|list): A string or list whose elements are list of
- block's layer names to be quantized.
- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer
- **kwargs: Additional keyword arguments.
- """
-
- bits: int | None
- group_size: int | tuple | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
+class AutoRoundMLLM:
- def __init__(
- self,
- model: Union[torch.nn.Module, str],
- tokenizer=None,
- platform: str = "hf",
- processor=None,
- image_processor=None,
- scheme: Union[str, dict, QuantizationScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- quant_nontext_module: bool = False,
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- **kwargs,
- ):
- local_args = {k: v for k, v in locals().items() if k not in ("local_args", "kwargs", "self")}
- super().__init__(
- **local_args,
- **kwargs,
- )
+ def __new__(cls, *args, **kwargs):
+ return AutoRound(*args, **kwargs)
@deprecated("AutoRound")
-class AutoRoundDiffusion(DiffusionCompressor):
- """Class for automatic rounding-based quantization with Diffusion models.
-
- Args:
- model: The PyTorch model to be quantized.
- tokenizer: An optional tokenizer for processing input data, is not used for diffusion models.
- platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
- guidance_scale (float): Control how much the image generation process follows the text prompt.
- The more it is, the more closely it follows the prompt (default is 7.5).
- num_inference_steps (int): The reference number of denoising steps (default is 50).
- generator_seed (int): A sees that controls the initial noise from which an image is generated (default is None).
- scheme: (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations.
- layer_config (dict): Configuration for weight quantization (default is None).
- dataset: The path or name of the calib dataset.
- iters (int): Number of iterations (default is 200).
- seqlen (int): Length of the sequence.
- nsamples (int): Number of samples (default is 128).
- batch_size (int): Batch size for training (default is 8).
- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
- device_map (str | dict | int | torch.device, optional): Device placement map. Defaults to 0.
- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer
- **kwargs: Additional keyword arguments.
- """
-
- bits: int | None
- group_size: int | tuple | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
+class AutoRoundDiffusion:
- def __init__(
- self,
- model: Union[object, str],
- tokenizer=None,
- guidance_scale: float = 7.5,
- num_inference_steps: int = 50,
- generator_seed: int = None,
- scheme: Union[str, dict, QuantizationScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "coco2014",
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- **kwargs,
- ):
- local_args = {k: v for k, v in locals().items() if k not in ("local_args", "kwargs", "self")}
- super().__init__(
- **local_args,
- **kwargs,
- )
+ def __new__(cls, *args, **kwargs):
+ return AutoRound(*args, **kwargs)
diff --git a/auto_round/calibration/__init__.py b/auto_round/calibration/__init__.py
index 14a492441..e784a8c6c 100644
--- a/auto_round/calibration/__init__.py
+++ b/auto_round/calibration/__init__.py
@@ -11,3 +11,21 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+"""Calibration sub-package."""
+
+from auto_round.calibration.base import Calibrator
+from auto_round.calibration.state import CalibrationState
+from auto_round.calibration.register import CALIBRATORS, get_calibrator, register_calibrator
+
+# Importing the strategy modules triggers their ``@register_calibrator`` decorators.
+from auto_round.calibration import llm as _llm # noqa: F401
+from auto_round.calibration import mllm as _mllm # noqa: F401
+from auto_round.calibration import diffusion as _diffusion # noqa: F401
+
+__all__ = [
+ "Calibrator",
+ "CalibrationState",
+ "CALIBRATORS",
+ "get_calibrator",
+ "register_calibrator",
+]
diff --git a/auto_round/calibration/base.py b/auto_round/calibration/base.py
new file mode 100644
index 000000000..a32f03a30
--- /dev/null
+++ b/auto_round/calibration/base.py
@@ -0,0 +1,86 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Calibrator abstract base class.
+
+A ``Calibrator`` owns the *block-input collection* phase of quantization.
+It holds a back-reference to the owning ``BaseCompressor`` so it can
+read/write shared state (``model_context`` / ``compress_context`` /
+``quantizer`` / ``inputs`` / ``dataloader`` / ...).
+
+Extension points for new calibration strategies:
+
+- :meth:`calib` — how the model is driven so hooks fire.
+- :meth:`should_stop` — early-stop policy (e.g. diffusion never stops).
+- :meth:`wrap_block_forward` — block-forward wrapping (e.g. positional → kwargs).
+"""
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from auto_round.compressors.base import BaseCompressor
+
+
+class Calibrator(ABC):
+ """Abstract base for all calibration strategies."""
+
+ def __init__(self, compressor: "BaseCompressor") -> None:
+ self.compressor = compressor
+
+ # ── Public API ──────────────────────────────────────────────────────────
+
+ @abstractmethod
+ def collect(self, block_names, nsamples, layer_names=None, last_cache_name=None):
+ """Run calibration and return ``inputs`` dict for the given blocks/layers."""
+
+ @abstractmethod
+ def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):
+ """Replace forward, run :meth:`calib`, return cached ``inputs``."""
+
+ @abstractmethod
+ def calib(self, nsamples: int, bs: int) -> None:
+ """Drive the model so block-forward hooks fire.
+
+ Subclasses (LLM / MLLM / Diffusion / ...) implement their own data
+ loading and forward driver here.
+ """
+
+ # ── Optional hooks (sane defaults) ─────────────────────────────────────
+
+ def should_stop(self, name: str) -> bool:
+ """Default early-stop policy: delegate to the module helper.
+
+ Subclasses (e.g. ``DiffusionCalibrator``) may override to always
+ return ``False`` so the pipeline runs every denoising step.
+ """
+ from auto_round.calibration.hooks import should_stop_cache_forward
+
+ return should_stop_cache_forward(self.compressor, name)
+
+ def wrap_block_forward(self, forward_fn):
+ """Optionally wrap the block-forward function. Default: passthrough.
+
+ Subclasses (e.g. ``DiffusionCalibrator``) may override to convert
+ positional → kwargs so diffusion blocks can be captured uniformly.
+ """
+ return forward_fn
+
+ def __getattr__(self, name: str) -> Any:
+ # Anything not defined on the calibrator is read off the compressor.
+ # Calibrator code can use ``self.model_context`` / ``self.quantizer`` /
+ # ``self.dataset`` / etc. as if it were still a method on DataDrivenCompressor.
+ compressor = self.__dict__.get("compressor", None)
+ if compressor is None:
+ raise AttributeError(name)
+ return getattr(compressor, name)
diff --git a/auto_round/calibration/diffusion.py b/auto_round/calibration/diffusion.py
new file mode 100644
index 000000000..2476c0f12
--- /dev/null
+++ b/auto_round/calibration/diffusion.py
@@ -0,0 +1,143 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Diffusion calibration strategy.
+
+Inherits :class:`LLMCalibrator` for ``collect`` / ``cache_inter_data``,
+and customises:
+
+- :meth:`calib` — drive the diffusion ``pipe`` with prompts / inference steps.
+- :meth:`should_stop` — always ``False`` so all denoising steps execute.
+- :meth:`wrap_block_forward` — convert positional → kwargs for diffusion blocks.
+"""
+
+import torch
+from tqdm import tqdm
+
+from auto_round.calibration.llm import LLMCalibrator
+from auto_round.calibration.register import register_calibrator
+from auto_round.logger import logger
+from auto_round.utils.model import wrap_block_forward_positional_to_kwargs
+
+
+@register_calibrator("diffusion")
+class DiffusionCalibrator(LLMCalibrator):
+ """Calibrator for diffusion models (Stable Diffusion / FLUX / ...)."""
+
+ # ── Overrides for diffusion-specific behaviour ─────────────────────────
+
+ def should_stop(self, name: str) -> bool:
+ """Diffusion models must run *all* denoising steps to collect enough inputs.
+
+ Mirrors the legacy ``DiffusionMixin._should_stop_cache_forward`` which
+ always returns ``False`` so the pipeline never exits early.
+ """
+ return False
+
+ def wrap_block_forward(self, forward_fn):
+ """Wrap positional-arg block forward into kwargs form for diffusion blocks."""
+ return wrap_block_forward_positional_to_kwargs(forward_fn)
+
+ @torch.no_grad()
+ def calib(self, nsamples: int, bs: int) -> None:
+ """Drive the diffusion pipeline so block-forward hooks fire.
+
+ Verbatim port of the legacy ``DiffusionMixin.calib``.
+ """
+ from auto_round.compressors.diffusion.dataset import get_diffusion_dataloader
+
+ c = self.compressor
+ pipe = c.model_context.pipe
+ if pipe is None:
+ raise ValueError(
+ "Diffusion pipeline not found in model_context. " "Ensure the model was loaded as a diffusion model."
+ )
+
+ logger.warning(
+ "Diffusion model will catch nsamples * num_inference_steps inputs, "
+ "you can reduce nsamples or num_inference_steps if OOM or take too much time."
+ )
+ if isinstance(c.dataset, str):
+ dataset = c.dataset.replace(" ", "")
+ c.dataloader, c.batch_size, c.gradient_accumulate_steps = get_diffusion_dataloader(
+ dataset=dataset,
+ bs=c.batch_size,
+ seed=c.seed,
+ nsamples=c.nsamples,
+ gradient_accumulate_steps=c.gradient_accumulate_steps,
+ )
+ else:
+ c.dataloader = c.dataset
+ total_cnt = 0
+
+ total = nsamples if not hasattr(c.dataloader, "len") else min(nsamples, len(c.dataloader))
+
+ if (
+ hasattr(c.model, "hf_device_map")
+ and len(c.model.hf_device_map) > 1
+ and pipe.device != c.model.device
+ and torch.device(c.model.device).type in ["cuda", "xpu"]
+ ):
+ logger.error(
+ "Diffusion model is activated sequential model offloading, it will crash during moving to GPU/XPU. "
+ "Please use model path for quantization or "
+ "move the pipeline object to GPU/XPU before passing them into API."
+ )
+ exit(-1)
+
+ if pipe.device != c.model.device:
+ pipe.to(c.model.device)
+ with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:
+ for ids, prompts in c.dataloader:
+ if isinstance(prompts, tuple):
+ prompts = list(prompts)
+ try:
+ pipe(
+ prompts,
+ guidance_scale=c.guidance_scale,
+ num_inference_steps=c.num_inference_steps,
+ generator=(
+ None
+ if c.generator_seed is None
+ else torch.Generator(device=pipe.device).manual_seed(c.generator_seed)
+ ),
+ )
+ except NotImplementedError:
+ pass
+ except Exception as error:
+ raise error
+ step = len(prompts)
+ total_cnt += step
+ pbar.update(step)
+ if total_cnt >= nsamples:
+ break
+ if total_cnt == 0:
+ logger.error(
+ f"no data has been cached, please provide more data with sequence length >={c.seqlen} in the "
+ f"dataset or decease the sequence length"
+ )
+ exit(-1)
+ elif total_cnt < nsamples:
+ logger.warning(
+ f"Insufficient number of samples collected may affect the quantization. "
+ f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
+ )
+ if total_cnt < c.batch_size:
+ raise ValueError(
+ f"valid samples is less than batch_size({c.batch_size})," " please adjust c.batch_size or seqlen."
+ )
+ max_len = (total_cnt // c.batch_size) * c.batch_size
+ for k, v in c.inputs.items():
+ for key in v:
+ if isinstance(v[key], list) and len(v[key]) == total_cnt:
+ c.inputs[k][key] = v[key][:max_len]
diff --git a/auto_round/calibration/hooks.py b/auto_round/calibration/hooks.py
new file mode 100644
index 000000000..ff4c6ddd9
--- /dev/null
+++ b/auto_round/calibration/hooks.py
@@ -0,0 +1,211 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Hook factories for collecting block-level inputs during calibration.
+
+These module-level factories can be reused by any ``Calibrator`` subclass
+without inheriting from a particular Compressor.
+
+A ``state`` object (typically the Compressor / Calibrator ``self``) is
+passed in as the mutable holder of the following attributes:
+
+- ``inputs`` : dict[str, dict | list]
+- ``quantizer`` : has ``batch_size`` / ``batch_dim`` / ``attention_mask``
+- ``has_variable_block_shape`` : bool
+- ``blocks_requiring_input_ids`` : list[str]
+- ``model_context`` : ``ModelContext`` (uses ``shared_cache_keys`` / ``replace_forward``)
+- ``_should_stop_cache_forward`` : callable(name) -> bool
+ (kept on ``state`` so subclasses can override it; e.g. DiffusionMixin always returns False)
+- ``to_cached_layers`` : list[str] (only required by ``replace_forward``)
+"""
+
+from functools import partial
+from typing import Callable
+
+import torch
+
+from auto_round.compressors.utils import check_skippable_keywords, init_cache, reset_params
+from auto_round.logger import logger
+from auto_round.utils import SUPPORTED_LAYER_TYPES, to_device
+
+
+def make_block_forward_func(state, name: str) -> Callable:
+ """Build a ``forward`` replacement that captures inputs for *block* ``name``.
+
+ Mirrors the legacy ``DataDrivenCompressor._get_block_forward_func`` exactly.
+ The returned function expects to be bound as ``module.forward = partial(fn, module)``.
+ """
+
+ def post_process_cache_data(batch_size, data, data_name):
+ new_data = data
+ if data_name in state.model_context.shared_cache_keys:
+ return None
+ if batch_size <= 1:
+ return new_data
+ if "alibi" in data_name:
+ if isinstance(data, torch.Tensor):
+ alibi = data
+ alibi = alibi.reshape(batch_size, -1, alibi.shape[1], alibi.shape[2])
+ new_data = alibi
+ return new_data
+
+ def forward(m, hidden_states=None, *positional_inputs, **kwargs):
+ if name not in state.inputs:
+ state.inputs[name] = {}
+ init_cache(positional_inputs, state.inputs[name])
+
+ if state.quantizer.batch_dim is None:
+ state.quantizer.batch_dim = 0
+ if hidden_states is not None and state.quantizer.batch_size > 1:
+ if hidden_states.shape[0] > state.quantizer.batch_size:
+ state.quantizer.batch_dim = 1
+ if len(hidden_states.shape) > 1 and hidden_states.shape[1] > state.quantizer.batch_size:
+ logger.error(
+ "this model has not been supported, "
+ "please raise an issue in https://github.com/intel/auto-round/issues"
+ " or try to set the `batch_size` to 1 and "
+ "`gradient_accumulate_steps` to your current batch size."
+ )
+ exit(-1)
+
+ if hidden_states is not None:
+ kwargs["hidden_states"] = hidden_states
+
+ for key in kwargs.keys():
+ if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) or isinstance(kwargs[key], tuple):
+ if (
+ state.has_variable_block_shape
+ and name not in state.blocks_requiring_input_ids
+ and key == "hidden_states"
+ ):
+ continue
+ if key not in state.inputs[name].keys(): # initialization
+ data = to_device(kwargs[key], device=torch.device("cpu"))
+ if data is None or key in state.model_context.shared_cache_keys:
+ state.inputs[name][key] = data
+ continue
+ if state.quantizer.batch_size <= 1:
+ state.inputs[name][key] = [data]
+ else:
+ data = post_process_cache_data(state.quantizer.batch_size, data, key)
+ if isinstance(data, torch.Tensor):
+ state.inputs[name][key] = list(torch.split(data, 1, dim=state.quantizer.batch_dim))
+ else:
+ state.inputs[name][key] = [data]
+ else: # append cache inputs
+ new_data = post_process_cache_data(state.quantizer.batch_size, kwargs[key], key)
+ if new_data is None: # shareable args or NoneType
+ continue
+ new_data = to_device(new_data, device=torch.device("cpu"))
+ if state.quantizer.batch_size <= 1:
+ state.inputs[name][key].append(new_data)
+ else:
+ if isinstance(new_data, torch.Tensor):
+ state.inputs[name][key].extend(
+ list(torch.split(new_data, 1, dim=state.quantizer.batch_dim))
+ )
+ else:
+ state.inputs[name][key].append(new_data)
+ elif isinstance(kwargs[key], (str, bool, type(None))):
+ if key not in state.inputs[name].keys():
+ state.inputs[name][key] = kwargs[key]
+ else:
+ # Parameters not to be cached
+ if check_skippable_keywords(key):
+ logger.warning_once(
+ f"Please note that '{key}' key" " is not currently used in quantization fine-tuning."
+ )
+ reset_params(state.inputs[name])
+
+ if state._should_stop_cache_forward(name):
+ raise NotImplementedError
+ else:
+ if hidden_states is not None:
+ kwargs.pop("hidden_states")
+ return m.orig_forward(hidden_states, *positional_inputs, **kwargs)
+ else:
+ # Currently only for Llama-3.2-Vision-Instruct Series
+ return m.orig_forward(*positional_inputs, **kwargs)
+
+ return forward
+
+
+def make_layer_cache_hook(state, name: str) -> Callable:
+ """Build a forward-hook that captures inputs for *layer* ``name``.
+
+ Mirrors the legacy ``DataDrivenCompressor._get_cache_data_hook_for_layer`` exactly.
+ """
+
+ def cache_input_hook(module, inputs, outputs):
+ input = inputs
+ if isinstance(inputs, tuple) or isinstance(input, list):
+ input = inputs[0]
+ if name in state.inputs:
+ state.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))
+ else:
+ state.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))
+
+ if state._should_stop_cache_forward(name):
+ raise NotImplementedError
+
+ return cache_input_hook
+
+
+def replace_forward_with_hooks(state) -> None:
+ """Install block-forward replacements and layer hooks via ``model_context.replace_forward``.
+
+ Mirrors the legacy ``DataDrivenCompressor._replace_forward`` exactly. The
+ ``state`` is expected to expose ``to_cached_layers`` / ``hook_handles`` /
+ ``model_context`` and the two factory methods on its class
+ (``_get_block_forward_func`` / ``_get_cache_data_hook_for_layer``) so
+ that subclass overrides (e.g. ``DiffusionMixin``) still take effect.
+ """
+
+ def register_hook(n, m, hook_handles):
+ if n in state.to_cached_layers and type(m) not in SUPPORTED_LAYER_TYPES: # block
+ m.orig_forward = m.forward
+ m.forward = partial(state._get_block_forward_func(n), m)
+ elif n in state.to_cached_layers: # linear / conv1d layer
+ hook_func = state._get_cache_data_hook_for_layer(n)
+ hook_handle = m.register_forward_hook(hook_func)
+ hook_handles.append(hook_handle)
+
+ state.model_context.replace_forward(register_hook)
+
+
+def should_stop_cache_forward(state, name: str) -> bool:
+ """Default early-stop policy for block input collection.
+
+ Mirrors the legacy ``DataDrivenCompressor._should_stop_cache_forward`` exactly.
+ Subclasses (e.g. ``DiffusionMixin``) override the method on the Compressor
+ class to always return ``False``; this helper is only used by the default
+ LLM path.
+ """
+ if name == state.last_cache_name:
+ return True
+
+ if state.last_cache_name is not None:
+ return False
+
+ if not hasattr(state, "_cache_target_set") or not hasattr(state, "_cache_seen_targets"):
+ return False
+
+ if name in state._cache_target_set:
+ state._cache_seen_targets.add(name)
+
+ if not state._cache_target_set.issubset(state._cache_seen_targets):
+ return False
+
+ # Lock the last cache name after the first full forward pass.
+ state.last_cache_name = name
+ return True
diff --git a/auto_round/calibration/inputs.py b/auto_round/calibration/inputs.py
new file mode 100644
index 000000000..6893ce5d5
--- /dev/null
+++ b/auto_round/calibration/inputs.py
@@ -0,0 +1,85 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Pure helpers for shaping cached block inputs."""
+
+from typing import Tuple
+
+import torch
+
+from auto_round.utils import clear_memory, to_device, to_dtype
+
+__all__ = ["split_inputs", "preprocess_block_inputs"]
+
+
+def split_inputs(
+ inputs: dict,
+ first_input_name: str,
+ *,
+ is_diffusion: bool,
+) -> Tuple[object, dict]:
+ """Split a captured ``inputs`` dict into ``(input_ids, input_others)``.
+
+ Mirrors the original ``DataDrivenCompressor._split_inputs`` exactly:
+
+ - For diffusion models, every key containing ``"hidden_state"`` is pulled
+ out into a dict and returned as ``input_ids``; the remaining kwargs are
+ returned as ``input_others``.
+ - Otherwise, ``inputs[first_input_name]`` is popped and returned as
+ ``input_ids`` (may be ``None``); the remainder is ``input_others``.
+
+ The function mutates ``inputs`` (pops keys) — this matches the legacy
+ behaviour that downstream code relies on.
+ """
+ if is_diffusion:
+ input_id_str = [key for key in inputs.keys() if "hidden_state" in key]
+ input_ids = {k: inputs.pop(k, None) for k in input_id_str}
+ input_others = inputs
+ return input_ids, input_others
+ input_ids = inputs.get(first_input_name, None)
+ inputs.pop(first_input_name, None)
+ input_others = inputs
+ return input_ids, input_others
+
+
+def preprocess_block_inputs(
+ inputs: dict,
+ *,
+ model_context,
+ compress_context,
+ first_input_name: str = "input_ids",
+) -> Tuple[object, dict]:
+ """Move/cast cached block inputs onto the calibration cache device.
+
+ Mirrors the original ``DataDrivenCompressor._preprocess_block_inputs`` exactly.
+ Parameterized on ``model_context`` (for ``amp`` / ``amp_dtype`` /
+ ``is_diffusion``) and ``compress_context`` (for ``cache_device`` /
+ ``device_list``) so it does not require a Compressor ``self``.
+ """
+ input_ids, input_others = split_inputs(inputs, first_input_name, is_diffusion=model_context.is_diffusion)
+ clear_memory(device_list=compress_context.device_list)
+ tmp_dtype = model_context.amp_dtype if model_context.amp else torch.float32
+ if input_ids is not None:
+ input_ids = to_device(input_ids, compress_context.cache_device)
+ input_ids = to_dtype(input_ids, tmp_dtype)
+ input_others = to_device(input_others, compress_context.cache_device)
+
+ for key in input_others.keys():
+ if isinstance(input_others[key], torch.Tensor) and (
+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
+ ):
+ input_others[key] = input_others[key].to(tmp_dtype)
+ elif isinstance(input_others[key], list):
+ for i in range(len(input_others[key])):
+ to_dtype(input_others[key][i], tmp_dtype)
+ return input_ids, input_others
diff --git a/auto_round/calibration/llm.py b/auto_round/calibration/llm.py
new file mode 100644
index 000000000..75306c486
--- /dev/null
+++ b/auto_round/calibration/llm.py
@@ -0,0 +1,404 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""LLM (text-only) calibration strategy.
+
+Implements ``try_cache_inter_data_gpucpu`` / ``cache_inter_data`` /
+``calib`` for the plain-text path. Compressor state is accessed via
+``self.compressor.X``.
+"""
+
+import traceback
+
+import accelerate
+import torch
+from accelerate.big_modeling import dispatch_model, infer_auto_device_map
+from accelerate.utils import get_balanced_memory, get_max_memory
+
+from auto_round import envs
+from auto_round.calibration.base import Calibrator
+from auto_round.calibration.register import register_calibrator
+from auto_round.calibration.utils import _infer_last_cache_name
+from auto_round.logger import logger
+from auto_round.modeling.fused_moe.replace_modules import materialize_model_
+from auto_round.utils import (
+ check_seqlen_compatible,
+ clear_memory,
+ flatten_list,
+ hook_ngram_embeddings_on_cpu,
+ is_quantized_input_module,
+ mv_module_from_gpu,
+ to_device,
+ to_dtype,
+)
+from auto_round.utils.device import parse_available_devices
+
+
+@register_calibrator("llm")
+class LLMCalibrator(Calibrator):
+ """Calibrator for plain text / LLM models."""
+
+ # ── Public API ──────────────────────────────────────────────────────────
+
+ @torch.no_grad()
+ def collect(self, block_names, nsamples, layer_names=None, last_cache_name=None):
+ """Attempts to cache intermediate data on GPU; on OOM, falls back to CPU.
+
+ Verbatim port of the legacy ``DataDrivenCompressor.try_cache_inter_data_gpucpu``.
+ """
+ c = self.compressor
+ if is_quantized_input_module(c.model_context.model):
+ layer_names = []
+ if layer_names is None:
+ layer_names = []
+
+ block_names = flatten_list(block_names)
+ c.blocks_requiring_input_ids = [data if isinstance(data, str) else data[0] for data in block_names]
+
+ calibrate_on_cpu = False
+ cannot_calibrate_on_cpu = False
+ if c.compress_context.low_gpu_mem_usage or (
+ len(block_names) == 1
+ and len(layer_names) == 0
+ and not c.quantizer.has_qlayer_outside_block
+ and (last_cache_name is None or last_cache_name in block_names)
+ and not getattr(c, "mllm", False)
+ ):
+ # low_gpu_mem_usage or calibrate only the embedding layer (also fast on CPU)
+ calibrate_on_cpu = True
+ try:
+ all_inputs = self.cache_inter_data(
+ block_names, nsamples, layer_names=[], last_cache_name=last_cache_name
+ )
+ except NotImplementedError as error:
+ error_msg = str(error)
+ if "flash_attn::" in error_msg and "CPU" in error_msg:
+ cannot_calibrate_on_cpu = True
+ else:
+ raise error
+
+ if not calibrate_on_cpu or cannot_calibrate_on_cpu:
+ try:
+ if any(p.device.type == "meta" for p in c.model_context.model.parameters()):
+ materialize_model_(c.model_context.model)
+
+ if hasattr(c.model_context.model, "hf_device_map") and len(c.model_context.model.hf_device_map) > 1:
+ c.model_context.model = dispatch_model(
+ c.model_context.model, device_map=c.model_context.model.hf_device_map
+ )
+ else:
+ if str(c.model_context.model.device) == "cpu" and (not c.compress_context.device.startswith("hpu")):
+ no_split_modules = list(getattr(c.model_context.model, "_no_split_modules", []))
+ devices = parse_available_devices(c.compress_context.device_map)
+
+ max_memory = get_max_memory()
+ new_max_memory = {}
+ if "cpu" not in devices:
+ devices.append("cpu")
+ for device in devices:
+ if ":" in device:
+ device = int(device.split(":")[-1])
+ elif device == "cpu":
+ device = "cpu"
+ elif isinstance(device, str):
+ device = 0
+ else:
+ raise ValueError(
+ f"Unsupported device {device} in device_map: {c.compress_context.device_map}"
+ )
+ if device not in max_memory:
+ continue
+ new_max_memory[device] = max_memory[device] * 0.9
+
+ requested_non_cpu = any((d != "cpu") for d in devices)
+ has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)
+ if requested_non_cpu and not has_non_cpu_memory:
+ raise torch.OutOfMemoryError(
+ "No non-CPU device available in accelerate's reported memory. "
+ "Falling back to CPU caching."
+ )
+
+ has_ngram_embeddings, raw_ngram_embeddings = hook_ngram_embeddings_on_cpu(c.model_context.model)
+ new_max_memory = get_balanced_memory(
+ c.model_context.model,
+ max_memory=new_max_memory,
+ no_split_module_classes=no_split_modules,
+ )
+ if hasattr(c.model_context.model, "tie_weights"):
+ c.model_context.model.tie_weights()
+ device_map = infer_auto_device_map(
+ c.model_context.model,
+ max_memory=new_max_memory,
+ no_split_module_classes=no_split_modules,
+ )
+ if len(devices) > 1 and "cpu" in device_map.values():
+ logger.warning(
+ "Some layers are offloaded to cpu, which may severely impact calibration speed."
+ " Please consider using more cards."
+ )
+
+ try:
+ c.model_context.model = dispatch_model(c.model_context.model, device_map=device_map)
+ if has_ngram_embeddings:
+ c.model_context.model.model.ngram_embeddings = raw_ngram_embeddings
+ except ValueError as e:
+ if "offload_dir" in e.__str__():
+ logger.warning(
+ f"Due to insufficient resources, disk is used to store the model."
+ f" `offload_dir={envs.AR_WORK_SPACE}`"
+ )
+ c.model_context.model = dispatch_model(
+ c.model_context.model,
+ device_map=device_map,
+ offload_dir=envs.AR_WORK_SPACE,
+ )
+ else:
+ raise
+ else:
+ c.model_context.model = c.model_context.model.to(c.compress_context.device)
+
+ all_inputs = self.cache_inter_data(
+ block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
+ )
+ if hasattr(c.model_context.model, "hf_device_map") and len(c.model_context.model.hf_device_map) > 1:
+ accelerate.hooks.remove_hook_from_submodules(c.model_context.model)
+
+ except torch.OutOfMemoryError as e:
+ if cannot_calibrate_on_cpu:
+ raise e
+ cuda_error_msg = traceback.format_exc()
+ try:
+ logger.info("switch to cpu to cache block inputs")
+ c.compress_context.cache_device = torch.device("cpu")
+ if c.quantizer.has_qlayer_outside_block or c.__class__.__name__ == "AutoRoundMLLM":
+ logger.warning(
+ "we recommend using more GPUs in calibration."
+ " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."
+ )
+ accelerate.hooks.remove_hook_from_submodules(c.model_context.model)
+ c.model_context.model = mv_module_from_gpu(c.model_context.model)
+ clear_memory(device_list=c.compress_context.device_list)
+ # On cpu, we use rtn mode for layers in layer_names (post v0.51).
+ all_inputs = self.cache_inter_data(
+ block_names, nsamples, layer_names=[], last_cache_name=last_cache_name
+ )
+ except Exception:
+ logger.error(cuda_error_msg)
+ raise
+ return all_inputs
+
+ @torch.no_grad()
+ def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):
+ """Replace forward, run :meth:`calib`, return cached ``inputs``.
+
+ Verbatim port of the legacy ``DataDrivenCompressor.cache_inter_data``.
+ """
+ c = self.compressor
+ if layer_names is None:
+ layer_names = []
+
+ if not c._post_init_done:
+ c.post_init()
+
+ if hasattr(c, "quantizer") and hasattr(c.quantizer, "attention_mask"):
+ c.quantizer.attention_mask = []
+
+ c.inputs = {}
+ block_names = flatten_list(block_names)
+ c.to_cached_layers = block_names + layer_names
+
+ tmp_dtype = None # TODO delete this as most model is not fp32 now
+ ## have bug if block name is not the first block
+ if (len(block_names) > 1 or len(layer_names) > 0) and c.compress_context.low_gpu_mem_usage:
+ tmp_dtype = c.model_context.model.dtype
+ if c.model_context.amp:
+ if c.model_context.model.dtype != c.model_context.model.dtype:
+ c.model_context.model = c.model_context.model.to(torch.bfloat16)
+ else:
+ c.model_context.model = c.model_context.model.to(torch.float32) # model on cpu
+
+ c.last_cache_name = _infer_last_cache_name(block_names, layer_names, last_cache_name)
+ c._cache_target_set = set(c.to_cached_layers)
+ c._cache_seen_targets = set()
+ calib_bs = c.quantizer.batch_size
+ c.hook_handles = []
+ c._replace_forward()
+ try:
+ # Dispatch via the Compressor so that MLLMMixin / DiffusionMixin overrides
+ # of ``calib`` are honoured; if neither override applies, the Compressor's
+ # ``calib`` thin-wrapper routes back into ``self.calib`` below.
+ c.calib(nsamples, calib_bs)
+ finally:
+ # Use finally to recover_forward and delattr in case calib raises
+ # NotImplementedError, e.g. flash_attn on CPU.
+ c.model_context.recover_forward()
+ for attr in ("last_cache_name", "_cache_target_set", "_cache_seen_targets", "to_cached_layers"):
+ if hasattr(c, attr):
+ delattr(c, attr)
+ # Release calibration dataloader to free tokenized sample tensors.
+ if hasattr(c, "dataloader"):
+ del c.dataloader
+ res = c.inputs
+ if tmp_dtype is not None:
+ c.model_context.model = c.model_context.model.to(tmp_dtype)
+
+ return res
+
+ @torch.no_grad()
+ def calib(self, nsamples: int, bs: int) -> None:
+ """Drive the model with text data so block hooks fire.
+
+ Verbatim port of the legacy ``DataDrivenCompressor.calib`` (LLM path only).
+ """
+ from auto_round.calib_dataset import get_dataloader
+
+ c = self.compressor
+ need_attention_mask = True
+ if isinstance(c.dataset, str):
+ need_attention_mask = False # all supported datasets do not use pad
+ dataset = c.dataset.replace(" ", "") # remove all whitespaces
+
+ # slow here
+ c.dataloader = get_dataloader(
+ c.model_context.tokenizer,
+ c.seqlen,
+ dataset,
+ c.seed,
+ bs,
+ c.nsamples,
+ )
+ else:
+ c.dataloader = c.dataset
+ total_cnt = 0
+ if c.dataloader.__class__.__name__ == "BatchEncoding":
+ c.dataloader = [c.dataloader.data]
+
+ for data in c.dataloader:
+ if data.__class__.__name__ == "BatchEncoding":
+ data = data.data
+ if data is None:
+ continue
+ if isinstance(data, torch.Tensor):
+ input_ids = data.to(c.model.device)
+ data_new = input_ids
+ elif isinstance(data, str):
+ if c.model_context.tokenizer is None:
+ logger.error("please provide tokenizer for string input")
+ exit(-1)
+ data = c.model_context.tokenizer(data, truncation=True, max_length=c.seqlen, return_tensors="pt").data
+ data_new = {}
+ for key in data.keys():
+ data_new[key] = data[key].to(c.model.device)
+ input_ids = data_new["input_ids"]
+ elif isinstance(data, tuple) or isinstance(data, list):
+ data_new = to_device(data, c.model.device)
+ input_ids = data_new[0]
+ else:
+ data_new = {}
+ for key in data.keys():
+ data_new[key] = to_device(data[key], c.model.device)
+ if key == "images":
+ data_new[key] = to_dtype(data_new[key], c.model.dtype)
+ input_ids = data_new["input_ids"]
+ if input_ids.shape[-1] < c.seqlen:
+ continue
+ if need_attention_mask:
+ if (
+ isinstance(data_new, dict)
+ and "attention_mask" in data_new
+ and data_new["attention_mask"] is not None
+ ):
+ new_attention_mask = data_new["attention_mask"]
+ elif (
+ c.model_context.tokenizer is not None
+ and hasattr(c.model_context.tokenizer, "pad_token")
+ and c.model_context.tokenizer.pad_token is not None
+ ):
+ new_attention_mask = (input_ids != c.model_context.tokenizer.pad_token_id).to(torch.long)
+ else:
+ # Default all ones
+ new_attention_mask = torch.ones_like(input_ids, dtype=torch.long)
+
+ # For each sample, check if there are trailing repeated tokens; if
+ # so, set the mask of the last token to 0.
+ batch_size, seq_len = input_ids.shape
+ for i in range(batch_size):
+ last_token = input_ids[i, -1]
+ j = seq_len - 2
+ repeated = False
+ while j >= 0 and input_ids[i, j] == last_token:
+ repeated = True
+ new_attention_mask[i, j] = 0
+ j -= 1
+ if repeated:
+ new_attention_mask[i, -1] = 0
+
+ # Workaround: some models treat an all-1 attention mask as equivalent to
+ # None and will internally replace it with None for block inputs, which
+ # can cause tensor concatenation / shape-mismatch issues downstream.
+ # Force the last token in each sequence to be masked out so the mask is
+ # never "all ones".
+ new_attention_mask[:, -1] = 0
+
+ if not hasattr(c.quantizer, "attention_mask"):
+ c.quantizer.attention_mask = []
+ c.quantizer.attention_mask.extend(list(torch.split(new_attention_mask, 1, dim=0)))
+ else:
+ new_attention_mask = None
+ try:
+ kwargs = {"use_cache": False}
+ if new_attention_mask is not None and not (isinstance(data_new, dict) and "attention_mask" in data_new):
+ kwargs["attention_mask"] = new_attention_mask
+
+ if isinstance(data_new, torch.Tensor):
+ c.model(data_new, **kwargs)
+ elif isinstance(data_new, tuple) or isinstance(data_new, list):
+ c.model(*data_new, **kwargs)
+ else:
+ c.model(**data_new, **kwargs)
+ except NotImplementedError as error:
+ error_msg = str(error)
+ # Re-raise to fallback to CUDA when flash_attn does not support CPU.
+ if "flash_attn::" in error_msg and "CPU" in error_msg:
+ raise NotImplementedError(
+ "Could not run 'flash_attn::_flash_attn_varlen_forward'"
+ " with arguments from the 'CPU' backend."
+ )
+ else:
+ pass
+ except RuntimeError as error:
+ error_msg = str(error)
+ if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:
+ check_seqlen_compatible(c.seqlen, c.model_context.tokenizer, c.model)
+ logger.warning(
+ "When quantization encounters tensor shape mismatch error, "
+ "you can try to avoid it with batch_size=1"
+ )
+ raise error
+ except Exception as error:
+ raise error
+
+ total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1
+ if total_cnt >= nsamples:
+ break
+ if total_cnt == 0:
+ logger.error(
+ f"no data has been cached, please provide more data with sequence length "
+ f">={c.seqlen} in the dataset or decease the sequence length"
+ )
+ exit(-1)
+ elif total_cnt < nsamples:
+ logger.warning_once(
+ f"An insufficient number of samples likely reduces the accuracy of the quantized model. "
+ f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"
+ )
diff --git a/auto_round/calibration/mllm.py b/auto_round/calibration/mllm.py
new file mode 100644
index 000000000..62dfd2448
--- /dev/null
+++ b/auto_round/calibration/mllm.py
@@ -0,0 +1,177 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""MLLM (vision-language) calibration strategy.
+
+Inherits :class:`LLMCalibrator` to reuse ``collect`` / ``cache_inter_data``
+and overrides :meth:`calib` to drive the model with multimodal data.
+
+MLLM-specific runtime state (``template`` / ``extra_data_dir`` /
+``quant_nontext_module`` / ``template_obj``) lives on the *Compressor*
+(populated by ``MLLMMixin.__init__``) and is read here through
+``self.compressor``.
+"""
+
+import torch
+
+from auto_round.calibration.llm import LLMCalibrator
+from auto_round.calibration.register import register_calibrator
+from auto_round.logger import logger
+from auto_round.utils import to_device
+
+
+@register_calibrator("mllm")
+class MLLMCalibrator(LLMCalibrator):
+ """Calibrator for multimodal (vision-language) models."""
+
+ @torch.no_grad()
+ def calib(self, nsamples: int, bs: int) -> None:
+ """Drive the multimodal model so block-forward hooks fire.
+
+ Verbatim port of the legacy ``MLLMMixin.calib``.
+ """
+ from transformers import PreTrainedModel
+
+ from auto_round.compressors.mllm.dataset import get_mllm_dataloader
+ from auto_round.compressors.mllm.template import get_template
+ from auto_round.special_model_handler import MISTRAL_3_2_MODELS
+
+ c = self.compressor
+ mc = c.model_context
+ processor = mc.processor
+ image_processor = mc.image_processor
+ tokenizer = mc.tokenizer
+
+ # Handle template selection
+ if isinstance(mc.model, PreTrainedModel):
+ model_type = getattr(mc.model.config, "model_type", None)
+ if model_type == "llava" and c.template is None:
+ c.template = "default"
+
+ if hasattr(mc.model, "name_or_path"):
+ name = mc.model.name_or_path
+ if any([m in name for m in MISTRAL_3_2_MODELS]):
+ c.template = "mistral3_2"
+
+ template_name = c.template
+ if template_name is None and hasattr(mc.model.config, "model_type"):
+ template_name = mc.model.config.model_type
+ if template_name is None:
+ template_name = "default"
+
+ c.template_obj = get_template(
+ template_name,
+ model=mc.model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ use_rtn=getattr(c.quantize_config, "iters", None) == 0,
+ quiet=not c.quant_nontext_module,
+ )
+
+ logger.info(f"Using MLLM template: {template_name}")
+
+ dataset = c.dataset.replace(" ", "") if isinstance(c.dataset, str) else c.dataset
+ if dataset is None:
+ dataset = c.template_obj.default_dataset
+
+ if isinstance(c.dataset, str):
+ dataset = c.dataset.replace(" ", "")
+ # Switch text-only dataset to MLLM dataset when quant_nontext_module=True,
+ # as text datasets cannot calibrate vision modules.
+ from auto_round.calib_dataset import CALIB_DATASETS
+
+ if c.quant_nontext_module and dataset in CALIB_DATASETS:
+ logger.warning(
+ "Text only dataset cannot be used for calibrating non-text modules,"
+ " switching to liuhaotian/llava_conv_58k"
+ )
+ dataset = "liuhaotian/llava_conv_58k"
+ (
+ c.dataloader,
+ c.batch_size,
+ c.seqlen,
+ c.gradient_accumulate_steps,
+ ) = get_mllm_dataloader(
+ template=c.template_obj,
+ model=mc.model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ dataset=dataset,
+ extra_data_dir=c.extra_data_dir,
+ seqlen=c.seqlen,
+ bs=bs,
+ seed=c.seed,
+ nsamples=nsamples,
+ quant_nontext_module=c.quant_nontext_module,
+ )
+ else:
+ c.dataloader = c.dataset
+
+ # Process data through the model for calibration
+ total_cnt = 0
+ for data in c.dataloader:
+ if data is None:
+ continue
+
+ try:
+ if isinstance(data, str):
+ # List-of-strings dataset: process through template → model inputs
+ processed = c.template_obj.processor.get_input(
+ text=data, images=None, max_length=c.seqlen, squeeze=False
+ )
+ data_new = {k: to_device(v, mc.model.device) for k, v in processed.items()}
+ elif isinstance(data, dict) and "text" in data:
+ # FakeDataLoader-style {"text": ..., "image": ...}: process through template
+ text = data["text"]
+ if isinstance(text, dict):
+ text = [text]
+ input_text = c.template_obj._encode(text)
+ processed = c.template_obj.processor.get_input(
+ text=input_text,
+ images=data.get("image", None),
+ max_length=c.seqlen,
+ squeeze=False,
+ )
+ data_new = {}
+ for key, value in processed.items():
+ tensor_val = value if isinstance(value, torch.Tensor) else torch.as_tensor(value)
+ data_new[key] = to_device(tensor_val, mc.model.device)
+ elif isinstance(data, dict):
+ data_new = {
+ key: value.to(mc.model.device) if isinstance(value, torch.Tensor) else value
+ for key, value in data.items()
+ }
+ else:
+ data_new = data
+
+ if isinstance(data_new, dict):
+ mc.model(**data_new)
+ else:
+ mc.model(data_new)
+ except NotImplementedError:
+ pass
+ except Exception as e:
+ logger.warning(f"Calibration forward pass failed: {e}")
+ continue
+
+ total_cnt += bs
+ if total_cnt >= nsamples:
+ break
+
+ if total_cnt == 0:
+ logger.error("no data has been cached, please provide more data")
+ exit(-1)
+ elif total_cnt < nsamples:
+ logger.warning(f"Insufficient number of samples: required {nsamples}, but only {total_cnt} were processed.")
diff --git a/auto_round/calibration/register.py b/auto_round/calibration/register.py
new file mode 100644
index 000000000..cd8dd4900
--- /dev/null
+++ b/auto_round/calibration/register.py
@@ -0,0 +1,47 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Registry for ``Calibrator`` strategies.
+
+Mirrors the same pattern used elsewhere in the codebase (e.g.
+``auto_round/data_type/register.py``). Add a new calibration strategy by
+decorating its class with ``@register_calibrator("my_kind")``.
+"""
+
+from typing import Type
+
+from auto_round.calibration.base import Calibrator
+
+CALIBRATORS: dict[str, Type[Calibrator]] = {}
+
+
+def register_calibrator(name: str):
+ """Class decorator: register a ``Calibrator`` subclass under ``name``."""
+
+ def _wrap(cls: Type[Calibrator]) -> Type[Calibrator]:
+ if not issubclass(cls, Calibrator):
+ raise TypeError(f"{cls.__name__} must subclass auto_round.calibration.base.Calibrator")
+ cls.name = name
+ if name in CALIBRATORS and CALIBRATORS[name] is not cls:
+ raise ValueError(f"Calibrator '{name}' already registered as {CALIBRATORS[name].__name__}")
+ CALIBRATORS[name] = cls
+ return cls
+
+ return _wrap
+
+
+def get_calibrator(name: str) -> Type[Calibrator]:
+ """Look up a registered calibrator class by name."""
+ if name not in CALIBRATORS:
+ raise KeyError(f"No calibrator registered under '{name}'. " f"Known: {sorted(CALIBRATORS.keys())}")
+ return CALIBRATORS[name]
diff --git a/auto_round/calibration/state.py b/auto_round/calibration/state.py
new file mode 100644
index 000000000..85f28d67d
--- /dev/null
+++ b/auto_round/calibration/state.py
@@ -0,0 +1,148 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""``CalibrationState`` — single source of truth for calibration-time state.
+
+This dataclass owns every per-run calibration field shared between
+:class:`~auto_round.compressors.base.BaseCompressor` and
+:class:`~auto_round.algorithms.quantization.base.BaseQuantizers`:
+
+- Cache state ``(inputs, to_cached_layers, last_cache_name, blocks_requiring_input_ids)``
+- Per-batch shape state ``(attention_mask, batch_dim)``
+- Calibration parameters ``(batch_size, nsamples, seqlen, dataset, dataloader)``
+
+Both the compressor and the quantizer hold a reference to the same
+``CalibrationState`` instance (wired in ``BaseCompressor._resolve_scheme``).
+All legacy attribute reads/writes are routed here through ``@property``
+forwarders, so existing call sites need no changes.
+
+The dataclass also provides two behavioural helpers that previously lived
+inline in :class:`~auto_round.compressors.data_driven.DataDrivenCompressor`:
+
+- :meth:`clamp_seqlen` — clamps ``self.seqlen`` to model / tokenizer limits.
+- :meth:`ensure_dataloader` — builds ``self.dataloader`` from
+ ``self.dataset`` (string name or pre-built loader).
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+from auto_round.logger import logger
+
+__all__ = ["CalibrationState"]
+
+
+@dataclass
+class CalibrationState:
+ """Authoritative runtime store for calibration state.
+
+ See module docstring for field semantics and design rationale.
+ """
+
+ # ── Capture buffers ────────────────────────────────────────────────────
+ inputs: dict = field(default_factory=dict)
+ to_cached_layers: list = field(default_factory=list)
+ last_cache_name: Optional[str] = None
+ blocks_requiring_input_ids: list = field(default_factory=list)
+
+ # ── Per-batch shape state ──────────────────────────────────────────────
+ attention_mask: list = field(default_factory=list)
+ batch_dim: Optional[int] = None
+
+ # ── Calibration parameters ─────────────────────────────────────────────
+ batch_size: int = 8
+ nsamples: int = 128
+ seqlen: int = 2048
+ dataset: Any = None
+ dataloader: Any = None
+
+ # ── Compressor / quantizer round-tripping ──────────────────────────────
+
+ @classmethod
+ def from_compressor(cls, compressor: Any) -> "CalibrationState":
+ """Return the live shared instance held by ``compressor``.
+
+ The compressor always owns a ``_calibration_state`` after
+ ``BaseCompressor.__init__``, so the legacy "snapshot" fallback is
+ no longer required. We still allow it for safety in case a custom
+ subclass forgets to call ``super().__init__``.
+ """
+ live = getattr(compressor, "_calibration_state", None)
+ if isinstance(live, cls):
+ return live
+ # Legacy fallback (no shared instance wired yet).
+ return cls(
+ inputs=getattr(compressor, "inputs", {}) or {},
+ to_cached_layers=getattr(compressor, "to_cached_layers", []) or [],
+ last_cache_name=getattr(compressor, "last_cache_name", None),
+ blocks_requiring_input_ids=getattr(compressor, "blocks_requiring_input_ids", []) or [],
+ batch_size=getattr(compressor, "batch_size", 8) or 8,
+ nsamples=getattr(compressor, "nsamples", 128) or 128,
+ seqlen=getattr(compressor, "seqlen", 2048) or 2048,
+ dataset=getattr(compressor, "dataset", None),
+ dataloader=getattr(compressor, "dataloader", None),
+ )
+
+ # ── Behavioural helpers ────────────────────────────────────────────────
+
+ def clamp_seqlen(self, model_context: Any) -> None:
+ """Clamp :attr:`seqlen` to model / tokenizer maximum lengths.
+
+ Migrated verbatim from ``DataDrivenCompressor._check_compatibility``.
+ Safe to call multiple times; warns on each clamp.
+ """
+ if self.seqlen is None:
+ return
+ model = getattr(model_context, "model", None)
+ max_pos = getattr(getattr(model, "config", None), "max_position_embeddings", None)
+ if max_pos is not None and max_pos < self.seqlen:
+ logger.warning(f"Change sequence length to {max_pos} due to the limitation of max_position_embeddings")
+ self.seqlen = min(self.seqlen, max_pos)
+
+ tokenizer = getattr(model_context, "tokenizer", None)
+ tok_max = getattr(tokenizer, "model_max_length", None)
+ if tok_max is not None and tok_max < self.seqlen:
+ logger.warning(
+ f"Change sequence length to {tok_max} due to the limitation of model_max_length. "
+ "You can also try to increase the model_max_length to avoid this issue."
+ )
+ self.seqlen = min(self.seqlen, tok_max)
+
+ def ensure_dataloader(self, model_context: Any, seed: int) -> Any:
+ """Resolve :attr:`dataset` into :attr:`dataloader` and return it.
+
+ - If ``self.dataset`` is a string, builds a tokenized dataloader via
+ :func:`auto_round.calib_dataset.get_dataloader`.
+ - Otherwise, treats ``self.dataset`` as an already-iterable loader.
+
+ Mirrors the inline logic that previously lived in
+ ``DataDrivenCompressor._compute_imatrix`` and the calibrator subclasses.
+ """
+ if isinstance(self.dataset, str):
+ tokenizer = getattr(model_context, "tokenizer", None)
+ if tokenizer is None:
+ raise ValueError("A tokenizer must be set for the model when using a dataset string.")
+ from auto_round.calib_dataset import get_dataloader
+
+ dataset_name = self.dataset.replace(" ", "")
+ self.dataloader = get_dataloader(
+ tokenizer,
+ self.seqlen,
+ dataset_name,
+ seed,
+ self.batch_size,
+ self.nsamples,
+ )
+ else:
+ self.dataloader = self.dataset
+ return self.dataloader
diff --git a/auto_round/compressors/__init__.py b/auto_round/compressors/__init__.py
index 07c763482..531fc237f 100644
--- a/auto_round/compressors/__init__.py
+++ b/auto_round/compressors/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025 Intel Corporation
+# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,17 +12,82 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.compressors.adam import AdamCompressor
-from auto_round.compressors.base import BaseCompressor
-from auto_round.compressors.base import LLMCompressor
-from auto_round.compressors.mllm.compressor import MLLMCompressor
-from auto_round.compressors.diffusion.compressor import DiffusionCompressor
-from auto_round.compressors.diffusion.hybrid import HybridCompressor
-from auto_round.compressors.model_free import ModelFreeCompressor
-from auto_round.compressors.config import (
- DiffusionExtraConfig,
- ExtraConfig,
- MLLMExtraConfig,
- SchemeExtraConfig,
- TuningExtraConfig,
-)
+# Lazy imports to avoid circular dependencies
+# Users should import from specific modules instead of this __init__.py
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from auto_round.compressors.base import BaseCompressor
+ from auto_round.compressors.config import (
+ DiffusionExtraConfig,
+ ExtraConfig,
+ MLLMExtraConfig,
+ SchemeExtraConfig,
+ TuningExtraConfig,
+ )
+ from auto_round.compressors.data_driven import CalibratedRTNCompressor, DataDrivenCompressor
+ from auto_round.compressors.entry import AutoRoundCompatible, AutoRound
+ from auto_round.compressors.model_free import ModelFreeCompressor
+ from auto_round.compressors.zero_shot import ZeroShotCompressor
+
+__all__ = [
+ "AutoRound",
+ "BaseCompressor",
+ "DataDrivenCompressor",
+ "CalibratedRTNCompressor",
+ "ZeroShotCompressor",
+ "AutoRoundCompatible",
+ "ModelFreeCompressor",
+ "ExtraConfig",
+ "TuningExtraConfig",
+ "SchemeExtraConfig",
+ "MLLMExtraConfig",
+ "DiffusionExtraConfig",
+]
+
+
+def __getattr__(name):
+ """Lazy import to avoid circular dependencies."""
+ if name == "AutoRound" or name == "AutoRoundCompatible":
+ from auto_round.compressors.entry import AutoRound, AutoRoundCompatible
+
+ if name == "AutoRound":
+ return AutoRound
+ return AutoRoundCompatible
+ elif name == "BaseCompressor":
+ from auto_round.compressors.base import BaseCompressor
+
+ return BaseCompressor
+ elif name in ("DataDrivenCompressor", "CalibratedRTNCompressor"):
+ from auto_round.compressors.data_driven import DataDrivenCompressor, CalibratedRTNCompressor
+
+ return {
+ "DataDrivenCompressor": DataDrivenCompressor,
+ "CalibratedRTNCompressor": CalibratedRTNCompressor,
+ }[name]
+ elif name == "ZeroShotCompressor":
+ from auto_round.compressors.zero_shot import ZeroShotCompressor
+
+ return ZeroShotCompressor
+ elif name == "ModelFreeCompressor":
+ from auto_round.compressors.model_free import ModelFreeCompressor
+
+ return ModelFreeCompressor
+ elif name in ("ExtraConfig", "TuningExtraConfig", "SchemeExtraConfig", "MLLMExtraConfig", "DiffusionExtraConfig"):
+ from auto_round.compressors.config import (
+ DiffusionExtraConfig,
+ ExtraConfig,
+ MLLMExtraConfig,
+ SchemeExtraConfig,
+ TuningExtraConfig,
+ )
+
+ return {
+ "ExtraConfig": ExtraConfig,
+ "TuningExtraConfig": TuningExtraConfig,
+ "SchemeExtraConfig": SchemeExtraConfig,
+ "MLLMExtraConfig": MLLMExtraConfig,
+ "DiffusionExtraConfig": DiffusionExtraConfig,
+ }[name]
+ raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
diff --git a/auto_round/compressors/adam.py b/auto_round/compressors/adam.py
deleted file mode 100644
index fb79cf39a..000000000
--- a/auto_round/compressors/adam.py
+++ /dev/null
@@ -1,164 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from typing import Union
-
-import torch
-
-from auto_round.compressors.base import BaseCompressor
-from auto_round.schemes import QuantizationScheme
-from auto_round.utils import check_is_cpu, htcore, is_hpex_available
-
-
-class AdamCompressor(BaseCompressor):
- """Class for quantization with optimizers like adamw of a PyTorch model.
-
- Args:
- model: The PyTorch model to be quantized.
- tokenizer: An optional tokenizer for processing input data.
- platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
- scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
- bits (int): Number of bits for quantization (default is 4).
- group_size (int): Size of the quantization group (default is 128).
- sym (bool): Whether sym to be used (default is True).
- layer_config (dict): Configuration for weight quantization (default is None).
- batch_size (int): Batch size for training (default is 8).
- amp (bool): Whether to use automatic mixed precision (default is True).
- device: The device to be used for training (default is "auto").
- lr_scheduler: The learning rate scheduler to be used.
- dataset: The default dataset name (default is "NeelNanda/pile-10k").
- enable_quanted_input (bool): Whether to use quantized input data (default is True).
- enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
- lr (float): The learning rate (default is 0.005).
- minmax_lr (float): The learning rate for min-max tuning (default is None).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
- iters (int): Number of iterations (default is 200).
- seqlen (int): Length of the sequence.
- nsamples (int): Number of samples (default is 128).
- sampler (str): The sampling method (default is "rand").
- seed (int): The random seed (default is 42).
- nblocks (int): Number of blocks (default is 1).
- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- not_use_best_mse (bool): Whether to use mean squared error (default is False).
- dynamic_max_gap (int): The dynamic maximum gap (default is -1).
- data_type (str): The data type to be used (default is "int").
- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
- have different choices.
- act_bits (int): Number of bits for activation quantization. Default is 16.
- act_group_size (int): Group size for activation quantization. Default is None.
- act_sym (bool): Whether to use symmetric activation quantization. Default is None.
- act_data_type (str): Specifies the data type for activations.
- Defaults to None, in which case it inherits the weight data type.
- act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- to_quant_block_names (str|list): A string or list whose elements are list of
- block's layer names to be quantized.
- enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning
- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer function
- **kwargs: Additional keyword arguments.
-
- Returns:
- The quantized model.
- """
-
- bits: int | None
- group_size: int | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
-
- def __init__(
- self,
- model: Union[torch.nn.Module, str],
- tokenizer=None,
- platform="hf",
- scheme: Union[str, dict, QuantizationScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, int, torch.device, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- optimizer="AdamW",
- **kwargs,
- ):
- super(AdamCompressor, self).__init__(
- model=model,
- tokenizer=tokenizer,
- platform=platform,
- scheme=scheme,
- layer_config=layer_config,
- batch_size=batch_size,
- dataset=dataset,
- low_gpu_mem_usage=low_gpu_mem_usage,
- iters=iters,
- seqlen=seqlen,
- nsamples=nsamples,
- seed=seed,
- gradient_accumulate_steps=gradient_accumulate_steps,
- enable_torch_compile=enable_torch_compile,
- device_map=device_map,
- **kwargs,
- )
-
- self.optimizer = self._get_optimizer(optimizer)
-
- def _get_optimizer(self, optimizer):
- if optimizer is None:
- optimizer = torch.optim.AdamW
- elif isinstance(optimizer, str):
- optimizer = getattr(torch.optim, optimizer)
- else:
- optimizer = optimizer
- return optimizer
-
- def _get_scaler(self):
- scaler = None
- if self.amp and not check_is_cpu(self.device):
- from torch.cuda.amp import GradScaler
-
- scaler = GradScaler(init_scale=1024, growth_interval=100000)
- return scaler
-
- def _scale_loss_and_backward(self, scaler, loss):
- if scaler is not None:
- loss = scaler.scale(loss)
-
- loss.backward()
- if is_hpex_available():
- htcore.mark_step()
- return loss
-
- def _step(self, scaler, optimizer, lr_schedule):
- if scaler is not None:
- scaler.step(optimizer)
- optimizer.zero_grad()
- lr_schedule.step()
- scaler.update()
- else:
- optimizer.step()
- optimizer.zero_grad()
- lr_schedule.step()
- if is_hpex_available():
- htcore.mark_step()
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index c60bc2849..d70010dc1 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2023 Intel Corporation
+# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,385 +11,218 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
import copy
+import gc
import os
import sys
-import time
-import traceback
-from collections import defaultdict
-from contextlib import nullcontext
-from dataclasses import asdict, fields
-from functools import partial
-from typing import Any, Callable, Optional, Union
-
-import accelerate
+from dataclasses import asdict, dataclass, fields
+from typing import Any, Optional, Union
+
import torch
-from accelerate.big_modeling import dispatch_model, infer_auto_device_map
-from accelerate.utils import get_balanced_memory, get_max_memory
-from packaging import version
-from torch import autocast
-from tqdm import tqdm
-from transformers import AutoConfig, set_seed
-
-from auto_round import envs
-from auto_round.algorithms.quantization.sign_round.sign_sgd import SignSGD
-from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
-from auto_round.compressors.shard_writer import shard_writer
-from auto_round.compressors.utils import (
- IndexSampler,
- block_forward,
- check_need_act_calibration,
- check_skippable_keywords,
- collect_best_params,
- get_shared_keys,
- infer_bits_by_data_type,
- init_cache,
- is_block_wfp8,
- is_dynamic_afp8,
- is_dynamic_wint8aint8,
- is_mx_fp,
- is_nv_fp,
- is_static_wfp8afp8,
- is_wfp8afp8,
- is_wint4aint4,
- reset_params,
- set_layer_config,
+from transformers import set_seed
+
+from auto_round.algorithms.alg_config import AlgConfig
+from auto_round.algorithms.quantization import BaseQuantizers, QuantizationConfig
+from auto_round.algorithms.transforms import (
+ BaseRotationConfig,
+ apply_rotation,
)
-from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, update_block_global_scale_if_needed
-from auto_round.experimental.transform.rotation_config import RotationConfig
-from auto_round.experimental.utils import dump_group_size_to_rotation_config, normalize_rotation_config
-from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG
+from auto_round.compressors.shard_writer import ShardWriter
+from auto_round.compressors.utils import _get_save_folder_name, is_mx_fp, is_nv_fp, set_layer_config
+from auto_round.context.compress import CompressContext
+from auto_round.context.model import ModelContext
from auto_round.formats import OutputFormat, get_formats
from auto_round.logger import logger
-from auto_round.modeling.fused_moe.replace_modules import materialize_model_, safe_to_cpu_
-from auto_round.modeling.unfused_moe import apply_model_monkey_patches
from auto_round.schemes import (
QuantizationScheme,
_handle_special_schemes,
+ _parse_scheme,
get_gguf_scheme,
preset_name_to_scheme,
- scheme_to_preset_name,
)
-from auto_round.special_model_handler import get_predefined_fixed_attr, get_predefined_ignore_layers, update_module
+from auto_round.special_model_handler import get_predefined_fixed_attr, get_predefined_ignore_layers
from auto_round.utils import (
INNER_SUPPORTED_LAYER_TYPES,
- SUPPORTED_DTYPES,
SUPPORTED_LAYER_TYPES,
TORCH_VERSION_AT_LEAST_2_6,
- CpuInfo,
- check_and_mark_quantized_module,
- check_seqlen_compatible,
- check_to_quantized,
- clear_memory,
- compile_func,
compress_layer_names,
convert_dtype_str2torch,
- convert_module_to_hp_if_necessary,
- detect_device,
+ extract_block_names_to_str,
find_matching_blocks,
- flatten_list,
get_block_names,
- get_layer_names_in_block,
- get_lm_head_name,
- get_module,
get_reverse_checkpoint_conversion_mapping,
- global_state,
- hook_ngram_embeddings_on_cpu,
- htcore,
- is_auto_device_mapping,
is_debug_mode,
is_hpex_available,
- is_moe_model,
- is_moe_model_via_config,
is_quantized_input_module,
- llm_load_model,
memory_monitor,
- mv_module_from_gpu,
- normalize_no_split_modules,
revert_checkpoint_conversion_mapping,
- set_amax_for_all_moe_layers,
- set_module,
- to_device,
- to_dtype,
- unsupported_meta_device,
)
from auto_round.utils.device import (
- clear_memory_if_reached_threshold,
+ _force_trim_malloc,
get_major_device,
- parse_available_devices,
patch_xpu_sdpa_drop_causal_mask,
- set_auto_device_map_for_block_with_tuning,
set_non_auto_device_map,
)
-from auto_round.utils.distributed import setup_ddp_if_needed_
from auto_round.utils.offload import OffloadManager
-from auto_round.wrapper import WrapperLinear, WrapperMultiblock, unwrapper_block, unwrapper_layer, wrapper_block
-
-SERIALIZATION_KEYS = (
- "bits",
- "act_bits",
- "data_type",
- "act_data_type",
- "group_size",
- "act_group_size",
- "sym",
- "act_sym",
- "act_dynamic",
- "amp",
- "batch_size",
- "enable_minmax_tuning",
- "enable_norm_bias_tuning",
- "enable_quanted_input",
- "gradient_accumulate_steps",
- "iters",
- "lr",
- "low_gpu_mem_usage",
- "minmax_lr",
- "nsamples",
- "quant_block_list",
- "regex_config",
- "scale_dtype",
- "seqlen",
- "supported_types",
- "static_attention_dtype",
- "static_kv_dtype",
- "super_bits",
- "super_group_size",
- "to_quant_block_names",
- "rotation_config",
-)
+
+
+@dataclass
+class SerializedCompressorConfig:
+ bits: Optional[int] = None
+ act_bits: Optional[int] = None
+ data_type: Optional[str] = None
+ act_data_type: Optional[str] = None
+ group_size: Optional[int] = None
+ act_group_size: Optional[int] = None
+ sym: Optional[bool] = None
+ act_sym: Optional[bool] = None
+ act_dynamic: Optional[bool] = None
+ amp: Optional[bool] = None
+ batch_size: Optional[int] = None
+ enable_minmax_tuning: Optional[bool] = True
+ enable_norm_bias_tuning: Optional[bool] = False
+ enable_quanted_input: Optional[bool] = True
+ gradient_accumulate_steps: Optional[int] = None
+ iters: Optional[int] = None
+ lr: Optional[float] = None
+ low_gpu_mem_usage: Optional[bool] = None
+ minmax_lr: Optional[float] = None
+ nsamples: Optional[int] = None
+ quant_block_list: Optional[list[str]] = None
+ regex_config: Optional[dict[str, Any]] = None
+ scale_dtype: Optional[str] = None
+ seqlen: Optional[int] = None
+ supported_types: Optional[list[str]] = SUPPORTED_LAYER_TYPES
+ static_attention_dtype: Optional[str] = None
+ static_kv_dtype: Optional[str] = None
+ super_bits: Optional[int] = None
+ super_group_size: Optional[int] = None
+ to_quant_block_names: Optional[list[str]] = None
+ rotation_configs: Optional[list[dict[str, Any]]] = None
+
+
+SERIALIZATION_KEYS = tuple(field.name for field in fields(SerializedCompressorConfig))
class BaseCompressor(object):
- """Base compressor for LLM quantization
-
- Attributes:
- model (torch.nn.Module): The loaded PyTorch model in eval mode.
- tokenizer: Tokenizer used to prepare input text for calibration/tuning.
- platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
- bits (int): Weight quantization bits.
- group_size (int): Per-group size for weight quantization.
- sym (bool): Whether to use symmetric weight quantization.
- layer_config (dict): Per-layer quantization configuration.
- nsamples (int): Number of calibration samples.
- enable_torch_compile (bool): Whether to enable compile_func for quant blocks/layers.
- """
-
- bits: int | None
- group_size: int | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
+ need_calib: bool = True
+ compress_context: CompressContext = None
+ model_context: ModelContext = None
+ shard_writer: ShardWriter = None
+ supported_types = SUPPORTED_LAYER_TYPES
+ inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
+
+ # ── Scheme state (populated during resolve_scheme / _scheme_post_init) ──
+ is_auto_scheme: bool = False
+ orig_scheme = None
+ scheme = None
+ scale_dtype = None
+ layer_config = None
+ has_qlayer_outside_block: bool = False
+ regex_config: dict = None
+ quant_block_list: list = None
+ to_quant_block_names = None
+ ignore_layers: str = ""
+ quant_lm_head: bool = False
+ _scheme_resolved: bool = False
+ scheme_generator = None
def __init__(
self,
+ config: Union[AlgConfig, list[AlgConfig]],
model: Union[torch.nn.Module, str],
tokenizer=None,
platform="hf",
- scheme: Union[str, dict, QuantizationScheme, AutoScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
+ format=None,
+ scheme="W4A16",
low_gpu_mem_usage: bool = False,
device_map: Union[str, torch.device, int, dict] = 0,
enable_torch_compile: bool = False,
- enable_alg_ext: bool = False,
- disable_opt_rtn: bool | None = None,
seed: int = 42,
low_cpu_mem_usage: bool = True,
- rotation_config: str | dict | RotationConfig | None = None,
+ layer_config=None,
+ nsamples: int = None,
+ seqlen: int = None,
**kwargs,
):
- """Initialize AutoRound with quantization and tuning configuration.
-
- Args:
- model (torch.nn.Module | str): Model object or model name to load.
- tokenizer: Tokenizer for text processing. Required if `model` is not a string and `iters > 0`.
- scheme (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations
- bits (int, optional): Weight quantization bits. Defaults to 4.
- group_size (int, optional): Weight quantization group size. Defaults to 128.
- sym (bool, optional): Symmetric weight quantization. Defaults to True.
- layer_config (dict, optional): Layer-wise quantization config. Defaults to None.
- batch_size (int, optional): Calibration batch size. Defaults to 8.
- amp (bool, optional): Use AMP for tuning. Defaults to True.
- device (str | torch.device | int, optional): Compute device. Defaults to 0.
- dataset (str | list | tuple | DataLoader, optional): Calibration data. Defaults to "NeelNanda/pile-10k".
- enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
- lr (float, optional): Learning rate; if None, set to 1.0 / iters except when iters==0.
- minmax_lr (float, optional): Learning rate for min-max tuning; defaults to `lr`.
- low_gpu_mem_usage (bool, optional): Lower GPU memory mode. Defaults to False.
- low_cpu_mem_usage (bool, optional): Lower CPU memory mode. Defaults to False.
- iters (int, optional): Optimization iterations. Defaults to 200.
- seqlen (int, optional): Calibration sequence length. Defaults to 2048.
- nsamples (int, optional): Number of calibration samples. Defaults to 128.
- seed (int, optional): Random seed. Defaults to 42.
- gradient_accumulate_steps (int, optional): Gradient accumulation steps. Defaults to 1.
- data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
- act_bits (int, optional): Activation quantization bits. Defaults to 16.
- act_group_size (int, optional): Activation group size. Defaults to None.
- act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
- act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
- act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
- enable_torch_compile (bool, optional): Enable torch.compile for quant blocks/layers. Defaults to False.
- device_map (str | dict, optional): Device placement map. Defaults to None.
- disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to None.
- enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
- **kwargs: Backward compatible options:
- - enable_alg_ext, quant_lm_head, lr, lr_scheduler, not_use_best_mse, dynamic_max_gap,
- super_group_size, super_bits, scale_dtype ("fp16" etc.),
- nblocks, to_quant_block_names,
- enable_norm_bias_tuning, enable_quanted_input, enable_opt_rtn,
- disable_deterministic_algorithms, mllm, static_kv_dtype,enable_deterministic_algorithms,momentum
- Raises:
- ValueError: If invalid device is provided or tokenizer is missing for non-str model with iters > 0.
- RuntimeError: If model parameters are on meta device.
- Example:
- Layer-wise configuration structure:
-
- >>> layer_config = {
- ... "layer1": {
- ... "bits": 3,
- ... "group_size": 128,
- ... "sym": True,
- ... },
- ... "layer2": {
- ... "W8A16"
- ... }
- ... # ...
- ... }
- """
- scheme_fields = [f.name for f in fields(QuantizationScheme)]
-
- # 1. Pre-extract user-specified overrides from kwargs
- # This ensures we know exactly what the user wants to "force"
- self.user_scheme_overrides = {}
- for k in scheme_fields:
- if k in kwargs:
- value = kwargs.pop(k)
- if value is not None:
- self.user_scheme_overrides[k] = value
- # Model related
- model_dtype = kwargs.pop("model_dtype", None)
- self.mllm = kwargs.pop("mllm") if "mllm" in kwargs else False
- self.trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else True
- self.diffusion = kwargs.pop("diffusion") if "diffusion" in kwargs else False
- self.quantized = False
- self.is_model_patched = False
- if isinstance(model, str):
- config: Optional[AutoConfig] = None
- try:
- config = AutoConfig.from_pretrained(model, trust_remote_code=self.trust_remote_code)
- except (OSError, EnvironmentError) as e:
- logger.debug(
- "Failed to load config via AutoConfig.from_pretrained for %s: %s. "
- "Proceeding without config-based checks.",
- model,
- e,
- )
-
- self.is_model_patched = apply_model_monkey_patches(
- model_name=model, trust_remote_code=self.trust_remote_code
- )
- import transformers
-
- if (
- not self.is_model_patched
- and config is not None
- and is_moe_model_via_config(config)
- and version.parse(transformers.__version__) >= version.parse("5.0.0")
- ):
- from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES
-
- model_type = getattr(config, "model_type", None)
- if model_type is not None and model_type not in BUILTIN_MODULES:
- logger.warning(
- "This MoE model has not been optimized by AutoRound yet, which may result in high RAM usage, "
- "Please consider submitting an issue to https://github.com/intel/auto-round/issues"
- )
-
- model, tokenizer = llm_load_model(
- model,
- platform=platform,
- device="cpu", # always load cpu first
- model_dtype=model_dtype,
- trust_remote_code=self.trust_remote_code,
- )
- elif tokenizer is None and not self.diffusion and iters > 0:
- raise ValueError("A tokenizer must be set for non-str model input")
- if unsupported_meta_device(model):
- raise RuntimeError(
- "AutoRound does not support parameters on meta device. "
- "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
- )
- check_and_mark_quantized_module(model)
- self.model = model.eval()
- self.tokenizer = tokenizer
- self.shared_cache_keys = get_shared_keys(self.model)
+ # ``CalibrationState`` is the single source of truth for calibration
+ # runtime state. Seed every calibration field here in one block so
+ # the rest of ``__init__`` only ever interacts with the state object
+ # via property forwarders. ``_resolve_scheme`` later wires this same
+ # instance onto the quantizer so the two share state.
+ from auto_round.calibration.state import CalibrationState
+
+ self._calibration_state = CalibrationState(
+ nsamples=nsamples if nsamples is not None else 128,
+ seqlen=seqlen if seqlen is not None else 2048,
+ batch_size=kwargs.pop("batch_size", 8),
+ )
+ # ``dataset`` is not a named __init__ parameter – it arrives via
+ # **kwargs from the compatibility layer. Pop it early and route
+ # through the property setter so CalibrationState owns it.
+ _dataset = kwargs.pop("dataset", None)
+ if _dataset is not None:
+ self.dataset = _dataset
+
+ self.quantize_config = None
+ self.rotation_configs: list[BaseRotationConfig] = []
+ _config_list = config if isinstance(config, list) else [config]
+ for _cfg in _config_list:
+ if isinstance(_cfg, QuantizationConfig):
+ self.quantize_config = _cfg
+ elif isinstance(_cfg, BaseRotationConfig):
+ self.rotation_configs.append(_cfg)
+ assert self.quantize_config is not None, "QuantizationConfig is required for Compressor"
+
+ # Compressor-level layer params (do not live in QuantizationConfig).
+ # Calibration params (nsamples/seqlen/batch_size) are owned by
+ # ``self._calibration_state`` (seeded above) and exposed via
+ # ``@property`` forwarders.
self.layer_config = layer_config
+ # ``post_init()`` may run before ``quantize_and_save()`` in tests and
+ # compatibility paths, so seed the same default used by
+ # ``quantize_and_save(..., inplace=True)`` here.
+ self.inplace = True
- self.supported_types = SUPPORTED_LAYER_TYPES
- self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
- self.quant_lm_head = kwargs.pop("quant_lm_head", False)
+ # Scheme is passed directly to the compressor, not stored in QuantizationConfig.
+ self.scheme = scheme
- # should be set after loading model and set layer_config, cause some special scheme need these.
- # Preserve the original, unparsed scheme for later use in auto scheme generation
- # within `configure_layer_config` (which may need the raw value instead of `self.scheme`).
- default_scheme, self.is_auto_scheme = self._parse_and_set_scheme(scheme, self.user_scheme_overrides)
- self.orig_scheme = copy.deepcopy(scheme)
- self.scheme = default_scheme
+ # Calibrator strategy (auto_round.calibration.base.Calibrator). Constructed
+ # lazily by ``DataDrivenCompressor.post_init`` based on ``_get_calibrator_kind()``;
+ # remains ``None`` for ``ZeroShotCompressor`` (RTN does not need data).
+ self.calibration = None
- gguf_scheme_name = get_gguf_scheme(self.scheme)
- # GGUF uses fp32 scale dtype as default
- scale_dtype = kwargs.pop("scale_dtype", None)
- if scale_dtype is None:
- scale_dtype = "fp32" if gguf_scheme_name else "fp16"
+ self.formats = format
# Extra/legacy kwargs for backward compatibility
# Major version releases may pack them with extra configuration options
amp = kwargs.pop("amp", True)
- lr = kwargs.pop("lr", None)
- enable_minmax_tuning = kwargs.pop("enable_minmax_tuning", True)
- minmax_lr = kwargs.pop("minmax_lr", None)
- lr_scheduler = kwargs.pop("lr_scheduler", None)
- not_use_best_mse = kwargs.pop("not_use_best_mse", False)
- dynamic_max_gap = kwargs.pop("dynamic_max_gap", -1)
nblocks = kwargs.pop("nblocks", 1)
- to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)
- enable_norm_bias_tuning: bool = kwargs.pop("enable_norm_bias_tuning", False)
- enable_quanted_input: bool = kwargs.pop("enable_quanted_input", True)
disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)
enable_deterministic_algorithms = kwargs.pop("enable_deterministic_algorithms", False)
- self.momentum = kwargs.pop("momentum", 0.0)
- static_kv_dtype = kwargs.pop("static_kv_dtype", None)
- static_attention_dtype = kwargs.pop("static_attention_dtype", None)
- enable_opt_rtn = kwargs.pop("enable_opt_rtn", None)
- device = kwargs.pop("device", None)
- if envs.AR_USE_MODELSCOPE:
- platform = "model_scope"
- self.platform = platform
-
- self.ignore_layers = kwargs.pop("ignore_layers", "")
- self.supported_types = SUPPORTED_LAYER_TYPES
- self.inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
- self.scale_dtype = convert_dtype_str2torch(scale_dtype)
- self.low_cpu_mem_usage = low_cpu_mem_usage
+
self._offloader = OffloadManager(enabled=low_cpu_mem_usage, mode="offload", offload_dir_prefix="compressor")
+ # Model related
+ model_dtype = kwargs.pop("model_dtype", None)
+ trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else True
+ quant_nontext_module = kwargs.pop("quant_nontext_module", False)
+
+ self.static_attention_dtype = kwargs.pop("static_attention_dtype", None)
+ # Attention static dtype
+ if self.static_attention_dtype is not None:
+ logger.warning("The static attention dtype is experimental and currently has limited support.")
+ # KV cache, this one does not affect tuning but will collect some infos during tuning
+ self.static_kv_dtype = kwargs.pop("static_kv_dtype", None)
+ if self.static_kv_dtype is not None:
+ logger.warning("The static kv is experimental and currently has limited support.")
+
if kwargs:
- logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
+ logger.warning_once(
+ f"unrecognized keys {list(kwargs.keys())} were passed. "
+ "Please check them. If you use old api, just ignore this warning."
+ )
if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
# Deprecated, default not to use torch.use_deterministic_algorithms
@@ -409,202 +242,155 @@ class BaseCompressor(object):
# instead of MATH (avoids ~10x peak-VRAM blow-up during block tuning).
patch_xpu_sdpa_drop_causal_mask()
- self.to_quant_block_names = to_quant_block_names
- if not hasattr(self, "quant_block_list"):
- all_blocks = get_block_names(model)
- self.quant_block_list = find_matching_blocks(model, all_blocks, self.to_quant_block_names)
-
+ device = kwargs.pop("device", None)
if device is not None:
logger.warning("`device` is deprecated, please use `device_map` instead")
- if device_map is None:
- device_map = 0
-
- self.device_map = device_map
- if isinstance(self.device_map, str):
- self.device_map = self.device_map.replace(" ", "")
-
- self.device_list = parse_available_devices(device_map)
-
- # Set device, must place after model loading
- self.device = get_major_device(device_map)
- set_non_auto_device_map(self.model, self.device_map)
-
# Tuning hyperparameters
self.seed = seed
set_seed(self.seed)
- self.amp = amp
- self.enable_quanted_input = enable_quanted_input
- self.enable_minmax_tuning = enable_minmax_tuning
- self.nsamples = nsamples
- self.enable_norm_bias_tuning = enable_norm_bias_tuning
- self.low_gpu_mem_usage = low_gpu_mem_usage
- self.seqlen = seqlen
- self.batch_size, self.gradient_accumulate_steps = batch_size, gradient_accumulate_steps
- self.nblocks = nblocks
- self.dataset = dataset
- self.iters = iters
- if self.iters < 0:
- logger.warning("`iters` must be non-negative, reset it to 200")
- self.iters = 200
- if self.iters == 0:
- self.lr = 5e-3
- else:
- if not lr:
- # TODO need to check 4 bits lr setting for auto-round-best, 3bits only validate on small models
- if self.iters >= 1000 and self.bits <= 3:
- self.lr = 2.0 / self.iters
- logger.info("set the lr to 2.0/iters for better accuracy")
- else:
- self.lr = 1.0 / self.iters
- else:
- self.lr = lr
-
- # Automatically adjust the disable_opt_rtn option if the user does not explicitly set it.
- # To avoid None issue, we keep a copy though it's a little ugly
- if enable_opt_rtn and disable_opt_rtn:
- raise ValueError("`enable_opt_rtn` and `disable_opt_rtn` are mutually exclusive; " "only one can be set.")
- if enable_opt_rtn:
- disable_opt_rtn = False
- self.orig_disable_opt_rtn = disable_opt_rtn
-
- if self.iters != 0 and self.orig_disable_opt_rtn is not None:
- logger.warning("`disable_opt_rtn` only works when `iters` is set to 0, ignore it now.")
- disable_opt_rtn = True
- if (
- self.bits >= 8
- and self.act_bits >= 8
- and self.iters == 0
- and self.data_type in ["int", "fp"]
- and disable_opt_rtn is None
- ):
- logger.warning("`disable_opt_rtn` is turned on for W8A16/W8A8 quantization to improve efficiency.")
- disable_opt_rtn = True
- if disable_opt_rtn is None and self.iters == 0:
- logger.info(
- "`enable_opt_rtn` is turned on, set `--disable_opt_rtn` for higher speed at the cost of accuracy."
- )
- disable_opt_rtn = False
-
- if self.iters > 0 and is_block_wfp8(self):
- logger.warning(
- "RTN is recommended as it achieves accuracy comparable to tuning for block-wise FP8 quantization "
- "while being significantly faster. You can set `--iters 0 --disable_opt_rtn` to enable RTN mode."
- )
- # Important Note! This is not very robust, do NOT rely on it to do high risky thing
- self.is_moe_model = is_moe_model(self.model)
+ self.nblocks = nblocks
- self.minmax_lr = minmax_lr or self.lr
- self.enable_alg_ext = enable_alg_ext
- self.not_use_best_mse = not_use_best_mse
- self.dynamic_max_gap = dynamic_max_gap
- self.lr_scheduler = lr_scheduler
- self.optimizer = self._get_optimizer(None)
- self.disable_opt_rtn = disable_opt_rtn
+ self.enable_torch_compile = enable_torch_compile
# Whether to pack the layer immediately after tuning
- self.is_immediate_packing = False
- self.is_immediate_saving = False
+ # Managed via self.compress_context.is_immediate_packing / is_immediate_saving
- # KV cache, this one does not affect tuning but will collect some infos during tuning
- self.static_kv_dtype = static_kv_dtype
- if self.static_kv_dtype is not None:
- logger.warning("The static kv is experimental and currently has limited support.")
+ torch.set_printoptions(precision=3, sci_mode=True)
- # Attention static dtype
- self.static_attention_dtype = static_attention_dtype
- if self.static_attention_dtype is not None:
- logger.warning("The static attention dtype is experimental and currently has limited support.")
+ if is_hpex_available():
+ logger.info("habana_frameworks is available, import htcore explicitly.")
+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401
- self._set_amp_dtype()
- self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
- if self.act_bits <= 8 and self.amp_dtype == torch.float16:
- logger.warning("force to use bf16 to for quantization tuning when enabling activation quantization")
- self.amp_dtype = torch.bfloat16
- if self.model.dtype != torch.bfloat16: # keep the model's buffer dtype unchanged
- self.model = self.model.to(torch.bfloat16)
- else:
- logger.info(f"using {self.model.dtype} for quantization tuning")
+ # Reset both context singletons before creating fresh instances so that
+ # consecutive AutoRound creations don't inherit stale config from earlier ones.
+ CompressContext.reset_context()
+ ModelContext.reset_context()
+
+ # Resolve the device eagerly so ModelContext can be created before
+ # CompressContext. Creating ModelContext first places the large model
+ # allocation early in the heap, matching the OLD arch allocation order
+ # and reducing C-heap fragmentation (which is amplified on HPU).
+ _device = get_major_device(device_map if device_map is not None else 0)
+
+ self.model_context = ModelContext(
+ model,
+ tokenizer=tokenizer,
+ platform=platform,
+ model_dtype=model_dtype,
+ trust_remote_code=trust_remote_code,
+ amp=amp,
+ need_calib=self.need_calib,
+ device=_device,
+ formats=self.formats,
+ is_act_quantize=self.quantize_config.is_act_quantize,
+ quant_nontext_module=quant_nontext_module,
+ )
+ # Alternatively, you can use CompressContext.create_context
+ self.compress_context = CompressContext(
+ low_cpu_mem_usage,
+ low_gpu_mem_usage,
+ device_map,
+ enable_torch_compile,
+ formats=self.formats,
+ static_kv_dtype=self.static_kv_dtype,
+ static_attention_dtype=self.static_attention_dtype,
+ )
+ self.shard_writer = None
- # Some helpers
- self.batch_dim = None
- self.infer_bs_coeff = 1
+ # scale_dtype is resolved in quantizer.resolve_scheme() after scheme resolution,
+ # so it is not initialized here to avoid premature evaluation with an unresolved scheme.
- # after setting iters
- self.enable_torch_compile = enable_torch_compile
+ # Flag for post_init idempotency. Set to False here so post_init() can be called
+ # either via quantize_and_save() (preferred, outside inference_mode) or directly
+ # from quantize() as a fallback for non-AutoScheme cases.
+ self._post_init_done = False
+
+ # Apply torch compile adjustments eagerly so that ar.enable_torch_compile
+ # reflects the correct value immediately after construction (not only after post_init).
self._adjust_torch_compile(enable_torch_compile)
+ self.compress_context.enable_torch_compile = self.enable_torch_compile
- if (
- (self.act_bits < 16 and (not self.act_dynamic or self.data_type == "nvfp")) # have hooks
- or self.enable_alg_ext # Use imatrix
- or not self.disable_opt_rtn # Use imatrix
- ):
- self.block_forward = block_forward
- else:
- # TODO FIXME
- # This function could not be compiled, causing a large accuracy drop when `enable_alg_ext` is used.
- # To avoid issues, remove it in all scenarios except WOQ.
- self.block_forward = (
- compile_func(block_forward, self.device) if self.enable_torch_compile else block_forward
- )
+ # ``self._calibration_state`` was created at the top of __init__ so
+ # all calibration-related property writes above (nsamples / seqlen /
+ # batch_size from kwargs) have already routed through it.
- self._check_configs()
- torch.set_printoptions(precision=3, sci_mode=True)
+ self.has_variable_block_shape = False
+ fixed_attr = get_predefined_fixed_attr(self.model_context.model) or {}
+ for key, value in fixed_attr.items():
+ setattr(self, key, value)
- if is_hpex_available():
- logger.info("habana_frameworks is available, import htcore explicitly.")
- import habana_frameworks.torch.core as htcore # pylint: disable=E0401
+ # ── Scheme resolution ─────────────────────────────────────────────────────
- self.attention_mask = []
+ def resolve_scheme(self, model_context=None, compress_context=None, dataset: str = None) -> None:
+ """Phase-1 init: resolve scheme and bind config attrs (no model structure needed).
- self.wrapper_block = wrapper_block
- if self.enable_alg_ext:
- try:
- logger.info("using algorithm extension for quantization.")
- from auto_round.alg_ext import wrapper_autoround
-
- wrapper_autoround(self)
- except (ImportError, ModuleNotFoundError):
- logger.error("algorithm extension import error, fallback to default mode")
-
- # Hadamard rotation / transform – unified entry that auto-dispatches
- # between the "inplace" (QuaRot-style residual rotation; supports any
- # dtype incl. INT4/INT8/FPx; can fuse online Had into weights) and
- # "transform" (per-Linear weight+activation Had with fused triton
- # kernel; only MXFP4 / NVFP4; cannot fuse) backends.
- # See :class:`RotationConfig` for routing rules.
- if rotation_config is not None:
- from auto_round.experimental.apply_rotation_transform import apply_hadamard_rotation
-
- default_block_size = self.group_size
- if self.act_bits < 16:
- default_block_size = self.act_group_size
- rotation_config = dump_group_size_to_rotation_config(rotation_config, default_block_size)
-
- self.model, _ = apply_hadamard_rotation(
- self.model,
- rotation_config,
- data_type=self.data_type,
- compute_device=self.device,
+ Must be called BEFORE ``get_formats()`` and BEFORE ``_scheme_post_init()``.
+ Idempotent: safe to call multiple times.
+ """
+ if self._scheme_resolved:
+ return
+
+ if model_context is not None:
+ self.model_context = model_context
+ if compress_context is not None:
+ self.compress_context = compress_context
+ if dataset is not None:
+ self.dataset = dataset
+
+ scheme_fields = {f.name for f in fields(QuantizationScheme)}
+ user_scheme_overrides = {
+ k: getattr(self.quantize_config, k)
+ for k in scheme_fields
+ if getattr(self.quantize_config, k, None) is not None
+ }
+ default_scheme, self.is_auto_scheme, final_attrs = _parse_scheme(self.scheme, user_scheme_overrides)
+
+ for key, value in final_attrs.items():
+ setattr(self.quantize_config, key, value)
+ if hasattr(self, key):
+ setattr(self, key, value)
+ self.quantize_config.check_config()
+
+ self.orig_scheme = copy.deepcopy(self.scheme)
+ self.scheme = default_scheme
+
+ gguf_scheme_name = get_gguf_scheme(self.scheme)
+ if self.scale_dtype is None:
+ self.scale_dtype = "fp32" if gguf_scheme_name else "fp16"
+ self.scale_dtype = convert_dtype_str2torch(self.scale_dtype)
+
+ self._scheme_resolved = True
+
+ def _scheme_post_init(self) -> None:
+ """Phase-4 init: build layer config on the patched model.
+
+ Requires ``resolve_scheme()`` to have been called first.
+ Must be called AFTER ``model_context.apply_patches()``.
+ """
+ assert self._scheme_resolved, (
+ "resolve_scheme() must be called before _scheme_post_init(). "
+ "BaseCompressor.post_init() does this automatically."
+ )
+
+ enable_gguf_official_mixed = not self.is_auto_scheme
+
+ if self.quant_block_list is None:
+ quant_nontext_module = getattr(self.model_context, "quant_nontext_module", False)
+ all_blocks = get_block_names(self.model_context.model, quant_vision=quant_nontext_module)
+ self.quant_block_list = find_matching_blocks(
+ self.model_context.model, all_blocks, self.to_quant_block_names
)
- stored = getattr(self.model, "rotation_config", rotation_config)
- # Normalize to a plain dict so it round-trips through JSON
- # (HF saves quantization_config to config.json on save_pretrained).
- if hasattr(stored, "model_dump"):
- stored = stored.model_dump()
- self.rotation_config = stored
- # Mirror the JSON-safe form back onto the model.
- setattr(self.model, "rotation_config", stored)
-
- self.blocks_requiring_input_ids = []
- self.has_variable_block_shape = False
- fixed_attr = get_predefined_fixed_attr(self.model) or {}
- for key, value in fixed_attr.items():
- setattr(self, key, value)
+ if self.to_quant_block_names is None and self.quant_block_list:
+ self.to_quant_block_names = extract_block_names_to_str(self.quant_block_list)
+ self.quantize_config.to_quant_block_names = self.to_quant_block_names
+
+ self.configure_layer_config(enable_gguf_official_mixed=enable_gguf_official_mixed)
def _gen_auto_scheme(self) -> dict[str, dict]:
- if self.mllm:
+ """Generate per-layer config via AutoScheme delta-loss selection."""
+ if self.model_context.is_mllm:
# AutoScheme on a VLM only scores the language tower (the block
# walker in delta_loss already skips vision/audio sub-trees) and
# uses a pure-text calibration dataset by default, falling back to
@@ -615,21 +401,17 @@ class BaseCompressor(object):
"as a fallback if needed)."
)
- if is_quantized_input_module(self.model):
- logger.info("AutoScheme does not currently support quantized input models (e.g., FP8).")
- sys.exit(-1)
+ if is_quantized_input_module(self.model_context.model):
+ raise NotImplementedError("AutoScheme does not currently support quantized input models (e.g., FP8).")
all_dtypes = []
all_gguf = True
for option in self.orig_scheme.options:
- # Resolve the quantization scheme or data type
dtype = "int"
if isinstance(option, str):
if not option.lower().startswith("gguf"):
all_gguf = False
-
option = preset_name_to_scheme(option)
-
else:
all_gguf = False
@@ -640,7 +422,6 @@ class BaseCompressor(object):
all_dtypes.append(dtype)
- # Check for mixed data types
unique_dtypes = set(all_dtypes)
if len(unique_dtypes) > 1 and not all_gguf:
logger.warning(
@@ -650,7 +431,7 @@ class BaseCompressor(object):
)
layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
- self.model,
+ self.model_context.model,
self.layer_config,
self.scheme,
self.scale_dtype,
@@ -660,34 +441,23 @@ class BaseCompressor(object):
self.ignore_layers,
self.quant_lm_head,
enable_gguf_official_mixed=False,
- is_mllm=self.mllm,
+ is_mllm=self.model_context.is_mllm,
)
quant_layer_names = layer_config.keys()
# ---- VLM: peel non-text sub-trees AutoScheme should not score ---- #
- # Behavior matches ``--quant_nontext_module``:
- # * default (False): peel BOTH vision and audio; AutoScheme only
- # scores the language tower (vision/audio kept at their original
- # 16-bit configuration).
- # * True: peel only audio; AutoScheme also scores the vision tower
- # (audio is always peeled because we don't ship an audio
- # calibration dataset, so any score for it would be unreliable).
nontext_skipped_layers: dict[str, dict] = {}
- if self.mllm:
+ if self.model_context.is_mllm:
from auto_round.utils import get_block_names
quant_nontext = getattr(self, "quant_nontext_module", False)
- # ``_get_vlm_block_names`` always drops audio blocks regardless of
- # ``quant_vision``.
- scoreable_blocks = get_block_names(self.model, quant_vision=quant_nontext)
+ scoreable_blocks = get_block_names(self.model_context.model, quant_vision=quant_nontext)
scoreable_block_prefixes = tuple(blk for group in scoreable_blocks for blk in group)
if quant_nontext:
- # vision stays in; only audio gets peeled.
peel_markers = ("audio", "speech", "wav", "waveform")
tower_label = "language+vision"
peel_label = "audio/speech"
else:
- # default: peel both vision and audio.
peel_markers = (
"vision",
"visual",
@@ -702,10 +472,8 @@ class BaseCompressor(object):
peel_label = "vision/audio"
def _is_scoreable_layer(name: str) -> bool:
- # Quick prefix test against detected scoreable blocks.
if any(name == p or name.startswith(p + ".") for p in scoreable_block_prefixes):
return True
- # Fallback: drop anything matching a peel marker.
lname = name.lower()
return not any(marker in lname for marker in peel_markers)
@@ -735,31 +503,28 @@ class BaseCompressor(object):
if v.get("fixed_by_user", False)
}
- # mainly using quant_layers and fixed by users
from auto_round.auto_scheme.gen_auto_scheme import GenScheme
- if not self.enable_torch_compile and self.super_bits is None and not self.orig_scheme.low_gpu_mem_usage:
+ if (
+ not self.compress_context.enable_torch_compile
+ and self.quantize_config.super_bits is None
+ and not self.orig_scheme.low_gpu_mem_usage
+ ):
logger.warning("we strongly recommend to set `enable_torch_compile` to True for AutoScheme to save VRAM")
self.scheme_generator = GenScheme(
self.orig_scheme,
- self.model,
+ self.model_context.model,
quant_layer_names,
fixed_layer_scheme_new,
self.dataset,
- device_map=self.device_map,
- tokenizer=self.tokenizer,
- enable_torch_compile=self.enable_torch_compile,
- processor=getattr(self, "processor", None),
+ device_map=self.compress_context.device_map,
+ tokenizer=self.model_context.tokenizer,
+ enable_torch_compile=self.compress_context.enable_torch_compile,
+ processor=self.model_context.processor,
)
layer_config = self.scheme_generator.get_layer_config()
# Re-attach vision/audio-tower layers we peeled off earlier so the
# downstream quantization pipeline sees the complete layer map.
- # NOTE: ``nontext_skipped_layers`` was populated *after* the first
- # ``set_layer_config`` call, so each cfg already carries internal-only
- # keys such as ``in_blocks`` that the second ``set_layer_config`` call
- # in ``configure_layer_config`` would reject via its ``normalize_item``
- # validator. Strip those keys (keep only the allowed scheme fields)
- # before merging.
if nontext_skipped_layers:
allowed_keys = {f.name for f in fields(QuantizationScheme)} | {
"fixed_by_user",
@@ -771,188 +536,100 @@ class BaseCompressor(object):
layer_config.setdefault(name, clean_cfg)
return layer_config
- def _set_device(self, device_map: Union[str, torch.device, int, dict]) -> None:
- if hasattr(self, "device") and self.device is not None:
- return
- if isinstance(device_map, (str, torch.device, int)):
- self.device = detect_device(device_map)
-
- elif isinstance(device_map, dict) and device_map:
- tmp_devices = []
- for val in device_map.values():
- if isinstance(val, (str, torch.device, int)): # could optimize
- tmp_device = detect_device(val)
- tmp_device = tmp_device.split(":")[0]
- tmp_devices.append(tmp_device)
- tmp_devices = list(set(tmp_devices))
- if len(tmp_devices) > 1:
- logger.warning(
- f"there are multiple device types in the device_map, "
- f"please make sure they are correct,use the first device {tmp_devices[0]} as the core device."
- )
+ def configure_layer_config(self, enable_gguf_official_mixed: bool | None = True) -> None:
+ """Build ``self.layer_config`` from the resolved scheme on the patched model."""
+ is_gguf_format = (f := getattr(self.compress_context, "formats", None)) is not None and "gguf" in f
+ predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)
+ compressed_predefined_ignore_layers = compress_layer_names(predefined_ignore_layers)
+ if not is_gguf_format:
+ predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)
+ if predefined_ignore_layers:
+ logger.info(f"Using predefined ignore_layers: {compressed_predefined_ignore_layers}")
+ tmp_str = ",".join(predefined_ignore_layers)
+ if self.ignore_layers == "":
+ self.ignore_layers = tmp_str
+ else:
+ self.ignore_layers += "," + tmp_str
- self.device = tmp_devices[0]
+ if self.is_auto_scheme:
+ self.layer_config = self._gen_auto_scheme()
else:
- raise TypeError(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}")
-
- def _reconcile_bits_and_dtype(self, config: dict, prefix: str = ""):
- """
- Harmonizes 'bits' and 'data_type' for weights or activations.
- Ensures internal consistency by prioritizing data_type inference.
- """
- dt_key = f"{prefix}data_type"
- bits_key = f"{prefix}bits"
-
- if config.get(dt_key) is None:
- return
-
- # Infer the correct bit-width based on the data_type string
- inferred_bits = infer_bits_by_data_type(config[dt_key])
-
- if inferred_bits is not None and inferred_bits < 16:
- # Check for conflict between user-specified bits and inferred bits
- if inferred_bits != config.get(bits_key):
- logger.warning(
- f"'{dt_key}' does not match '{bits_key}'. " f"Resetting '{bits_key}' to {inferred_bits}."
- )
- config[bits_key] = inferred_bits
-
- # Normalize data_type (e.g., 'mx_fp4' -> 'mx')
- for supported in SUPPORTED_DTYPES:
- if config[dt_key] == f"{supported}{inferred_bits}":
- config[dt_key] = supported
- break
-
- def _override_scheme_with_user_specify(
- self, scheme: Union[str, dict, QuantizationScheme], user_scheme_overrides: dict[str, Any], return_str=True
- ) -> Union[str, QuantizationScheme]:
- """
- Updates a base quantization scheme with user-provided overrides.
- Handles GGUF formatting and synchronizes weight/activation parameters.
- """
- # 1. GGUF special handling: map data_type suffix to GGUF scheme names
- dt_override = user_scheme_overrides.get("data_type", "")
- if (
- isinstance(scheme, QuantizationScheme) or (isinstance(scheme, str) and not scheme.startswith("gguf"))
- ) and dt_override.endswith("_dq"):
- if "bits" not in user_scheme_overrides:
- raise KeyError(f"Must specify 'bits' when using data_type={dt_override}")
-
- bits = user_scheme_overrides["bits"]
- suffix = "k" if bits == 6 else "k_s"
- scheme = f"gguf:q{bits}_{suffix}"
-
- # 2. Convert input scheme to a dictionary for processing
- if isinstance(scheme, QuantizationScheme):
- scheme_dict = asdict(scheme)
- elif isinstance(scheme, str):
- normalized_name = scheme.strip("'\" ").upper()
- if normalized_name.startswith("GGUF") and len(user_scheme_overrides) > 0:
- logger.warning_once(
- "When using GGUF scheme, user-specified overrides will be ignored to ensure format compatibility."
+ self.layer_config = _handle_special_schemes(
+ self.orig_scheme,
+ self.layer_config,
+ self.model_context.model,
+ supported_types=SUPPORTED_LAYER_TYPES,
+ inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,
+ quant_lm_head=self.quant_lm_head,
+ mllm=self.model_context.is_mllm,
+ )
+ _gguf_orig_fmt = getattr(self, "_gguf_original_format_name", None)
+ if _gguf_orig_fmt and "_MIXED" in _gguf_orig_fmt.upper():
+ self.layer_config = _handle_special_schemes(
+ _gguf_orig_fmt.lower(),
+ self.layer_config,
+ self.model_context.model,
+ supported_types=SUPPORTED_LAYER_TYPES,
+ inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,
+ quant_lm_head=self.quant_lm_head,
+ mllm=self.model_context.is_mllm,
)
- user_scheme_overrides = {}
- # If no overrides exist, return the normalized string immediately
- if not user_scheme_overrides and return_str:
- return normalized_name
- scheme_dict = asdict(preset_name_to_scheme(normalized_name))
- else:
- scheme_dict = scheme.copy()
-
- # 3. Apply overrides and define default behaviors
- scheme_dict.update(user_scheme_overrides)
-
- if scheme_dict.get("act_dynamic") is None:
- scheme_dict["act_dynamic"] = True
-
- # 4. Reconcile weight settings (bits vs data_type)
- self._reconcile_bits_and_dtype(scheme_dict)
- # 5. Fallback logic: Inherit activation settings from weight settings
- scheme_dict["act_group_size"] = (
- scheme_dict.get("act_group_size")
- if scheme_dict.get("act_group_size") is not None
- else scheme_dict.get("group_size")
- )
- scheme_dict["act_bits"] = scheme_dict.get("act_bits") or 16
- scheme_dict["act_sym"] = (
- scheme_dict.get("act_sym") if scheme_dict.get("act_sym") is not None else scheme_dict.get("sym")
+ fill_default_value = not self.is_auto_scheme
+ self.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
+ self.model_context.model,
+ self.layer_config,
+ self.scheme,
+ self.scale_dtype,
+ SUPPORTED_LAYER_TYPES,
+ INNER_SUPPORTED_LAYER_TYPES,
+ self.quant_block_list,
+ self.ignore_layers,
+ self.quant_lm_head,
+ enable_gguf_official_mixed=enable_gguf_official_mixed,
+ is_mllm=self.model_context.is_mllm,
+ fill_default_value=fill_default_value,
+ gguf_format_name=getattr(self, "_gguf_format_name", None),
)
- # 6. Activation data_type logic
- if scheme_dict.get("act_data_type") is None:
- is_supported = scheme_dict["data_type"] in SUPPORTED_DTYPES
- if is_supported and scheme_dict["act_bits"] < 16:
- scheme_dict["act_data_type"] = scheme_dict["data_type"]
- logger.info(f"Activation adopting weight data_type: {scheme_dict['data_type']}")
- else:
- scheme_dict["act_data_type"] = "float"
-
- # 7. Reconcile activation settings
- self._reconcile_bits_and_dtype(scheme_dict, prefix="act_")
+ # ─────────────────────────────────────────────────────────────────────────
- return QuantizationScheme.from_dict(scheme_dict)
+ @property
+ def mllm(self):
+ return self.model_context.is_mllm
- def _parse_and_set_scheme(
- self, scheme: Union[str, dict, QuantizationScheme, AutoScheme], user_scheme_overrides: dict[str, Any]
- ) -> tuple[Union[str, QuantizationScheme], bool]:
- """
- Parses the final scheme and binds all resulting attributes to 'self'.
- """
-
- is_auto_scheme = isinstance(scheme, AutoScheme)
-
- if is_auto_scheme:
- if not scheme.options:
- raise ValueError("AutoScheme options cannot be empty")
- else:
- for option in scheme.options:
- if isinstance(option, str):
- if "mixed" in option:
- raise ValueError(f"Mixed option {option} is not supported")
-
- # Map user overrides across all auto-scheme options
- scheme.options = [
- self._override_scheme_with_user_specify(opt, user_scheme_overrides) for opt in scheme.options
- ]
-
- # Select the primary scheme for attribute binding (skipping BF16)
- default_scheme = scheme.options[0]
- for opt in scheme.options:
- if opt == "BF16":
- continue
- if isinstance(opt, QuantizationScheme):
- if opt.bits < 16 or (opt.act_bits and opt.act_bits < 16):
- default_scheme = opt
- break
- else:
- default_scheme = self._override_scheme_with_user_specify(scheme, user_scheme_overrides)
-
- # Extract attributes from the chosen default_scheme
- if isinstance(default_scheme, str):
- final_attrs = self._override_scheme_with_user_specify(
- default_scheme, user_scheme_overrides, return_str=False
- )
- else:
- final_attrs = asdict(default_scheme)
-
- # Bind attributes to self for easy instance-level access
- for key, value in final_attrs.items():
- setattr(self, key, value)
-
- return default_scheme, is_auto_scheme
+ @property
+ def diffusion(self):
+ return self.model_context.is_diffusion
def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
"""Sets the torch compile configuration for the tuning."""
self.enable_torch_compile = enable_torch_compile
+
+ # Determine fp8 / nvfp4 intent from raw config before scheme resolution.
+ cfg = self.quantize_config
+ raw_scheme = self.scheme if isinstance(self.scheme, str) else ""
+ raw_dt = (cfg.data_type or "").lower()
+ raw_adt = (cfg.act_data_type or "").lower()
+ raw_scheme_upper = raw_scheme.upper()
+
+ is_raw_nv_fp = "nv_fp" in raw_dt or "nv_fp" in raw_adt or "NVFP" in raw_scheme_upper
+ is_raw_fp8 = (
+ "fp8" in raw_dt
+ or "fp8" in raw_adt
+ or "FP8" in raw_scheme_upper
+ or ("fp" in raw_dt and getattr(cfg, "bits", 16) == 8)
+ or ("fp" in raw_adt and getattr(cfg, "act_bits", 16) == 8)
+ )
+
+ act_bits = getattr(cfg, "act_bits", 16) or 16
if (
not self.enable_torch_compile
and TORCH_VERSION_AT_LEAST_2_6
- and self.act_bits > 8
+ and act_bits > 8
and not is_debug_mode()
- and "fp8" not in self.data_type
- and "fp8" not in self.act_data_type
- and self.iters > 0
+ and not is_raw_fp8
+ and self.need_calib
):
logger.info(
"%s",
@@ -960,2609 +637,611 @@ class BaseCompressor(object):
"Enabling it can reduce tuning cost by 20%, but it might throw an exception.",
)
# On HPU, we rely on torch.compile to speed up the model execution.
- if self.enable_torch_compile and is_wfp8afp8(self) and not is_hpex_available():
+ if self.enable_torch_compile and is_raw_fp8 and not is_hpex_available():
self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as fp8 is enabled")
+ logger.warning_once("reset enable_torch_compile to `False` as fp8 is enabled")
# TODO: fix https://github.com/intel/auto-round/issues/1109
- if self.enable_torch_compile and is_nv_fp(self.act_data_type):
+ if self.enable_torch_compile and is_raw_nv_fp:
self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as nvfp4 is enabled")
- if self.enable_torch_compile and self.super_group_size is not None and self.enable_alg_ext:
+ logger.warning_once("reset enable_torch_compile to `False` as nvfp4 is enabled")
+ super_group_size = getattr(cfg, "super_group_size", None)
+ enable_alg_ext = getattr(cfg, "enable_alg_ext", False)
+ if self.enable_torch_compile and super_group_size is not None and enable_alg_ext:
self.enable_torch_compile = False
- logger.warning("reset enable_torch_compile to `False` as super_group_size is set for algorithm extension")
-
- def _dq_check(self) -> None:
- """Reset the default value of super_bits and super_group_size"""
- if self.data_type.endswith("_dq"):
- gguf_config = GGUF_INNER_CONFIG[f"gguf:q{self.bits}_k"]
- self.super_bits = gguf_config.get("super_bits", None) if self.super_bits is None else self.super_bits
- self.super_group_size = (
- gguf_config.get("super_group_size", None) if self.super_group_size is None else self.super_group_size
+ logger.warning_once(
+ "reset enable_torch_compile to `False` as super_group_size is set for algorithm extension"
)
- def _check_configs(self) -> None:
- """Checks if the configurations are valid.
+ def _get_calibration_dataset(self) -> str:
+ """Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""
+ dataset = self._calibration_state.dataset
+ if dataset is not None:
+ return dataset
+ from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
+
+ scheme = self.scheme
+ if isinstance(scheme, AutoScheme) and scheme.dataset:
+ return scheme.dataset
+ return "NeelNanda/pile-10k"
+
+ def post_init(self) -> None:
+ """One-time initialization that requires a loaded model.
+
+ Call this OUTSIDE any ``@torch.inference_mode()`` context when using
+ AutoScheme – delta-loss selection needs autograd (backward pass).
+ ``quantize_and_save()`` does this automatically before entering the
+ inference-mode quantize loop.
+
+ Delegates to ordered pipeline phases; see each ``_resolve_scheme``,
+ ``_resolve_formats``, ``_build_quantizer``, ``_patch_model``,
+ ``_build_layer_config``, and ``_hardware_setup`` for the precise
+ preconditions and postconditions.
+ """
+ if self._post_init_done:
+ return
- Raises:
- ValueError, TypeError: If any of the configurations are invalid.
+ self._resolve_scheme()
+
+ # After scheme resolution, is_act_quantize is known. When activation
+ # quantization is enabled and the model is in float16, convert to
+ # bfloat16 to match the old arch. This also detaches any parameter
+ # tensors that are still backed by safetensors' mmap, preventing
+ # per-block RSS growth (~14 MB/block) when .to(device) page-faults
+ # the underlying file pages into physical memory.
+ if self.quantize_config.is_act_quantize and self.model_context.amp_dtype == torch.float16:
+ logger.warning("force to use bf16 for quantization tuning when enabling activation quantization")
+ self.model_context.amp_dtype = torch.bfloat16
+ if self.model_context.model.dtype != torch.bfloat16:
+ self.model_context.model = self.model_context.model.to(torch.bfloat16)
+
+ self._resolve_formats()
+ self._build_quantizer()
+ self._patch_model()
+ self._build_layer_config()
+ self._apply_rotations()
+
+ # Reclaim temporaries from Phases 1-4 (scheme resolution, format
+ # parsing, model patching, layer-config walk) before Phase 5
+ # allocates hardware/compile objects. This compacts the heap so that
+ # the fragmentation gap between live and freed blocks is minimised.
+ gc.collect()
+ _force_trim_malloc()
+
+ self._hardware_setup()
+
+ # Final trim after all init phases.
+ gc.collect()
+ _force_trim_malloc()
+
+ self._post_init_done = True
+
+ # ── Pipeline phase methods ────────────────────────────────────────────────
+
+ def _resolve_scheme(self) -> None:
+ """Phase 1 – Scheme resolution.
+
+ Preconditions:
+ - ``self.quantize_config`` is a valid :class:`QuantizationConfig`.
+
+ Work performed:
+ - Seeds scheme-related attrs (``scale_dtype``, ``ignore_layers``,
+ ``quant_lm_head``, ``to_quant_block_names``) from ``quantize_config``.
+ - Calls :meth:`resolve_scheme` to derive ``data_type``, ``bits``,
+ ``sym``, ``scale_dtype`` etc. and write them back to both ``self``
+ and ``self.quantize_config``.
+
+ Postconditions:
+ - ``self.scheme`` and ``self.quantize_config`` carry resolved scheme attrs.
"""
- if not isinstance(self.model, torch.nn.Module):
- raise TypeError("model must be an instance of torch.nn.Module")
- if self.bits <= 0:
- raise ValueError("`bits` must be positive")
- if self.act_bits <= 0:
- raise ValueError("`act_bits` must be positive")
- if not isinstance(self.group_size, tuple) and not (self.group_size == -1 or self.group_size >= 0):
- raise ValueError(
- "`group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer or a tuple of length 2"
- )
- if isinstance(self.act_group_size, tuple) or not (self.act_group_size == -1 or self.act_group_size >= 0):
- raise ValueError("`act_group_size` must be -1 (per channel) or 0 (per-tensor) or a positive integer")
- if isinstance(self.group_size, tuple) and len(self.group_size) != 2:
- raise ValueError("`group_size` must be a tuple of length 2")
- if self.batch_size <= 0:
- raise ValueError("`batch_size` must be positive")
- if self.iters < 0:
- raise ValueError("`iters` must be non-negative")
- if self.seqlen <= 0:
- raise ValueError("`seqlen` must be positive")
- if self.nblocks <= 0:
- raise ValueError("`nblocks` must be positive")
- if self.gradient_accumulate_steps <= 0:
- raise ValueError("`gradient_accumulate_steps` must be positive")
+ cfg = self.quantize_config
+ self.scale_dtype = cfg.scale_dtype
+ # self.layer_config is already set from __init__ (direct compressor param).
+ self.ignore_layers = cfg.ignore_layers
+ self.quant_lm_head = cfg.quant_lm_head
+ self.to_quant_block_names = cfg.to_quant_block_names
+
+ # Resolve the scheme (pure config work: sets data_type / bits / sym /
+ # scale_dtype etc. on both self and self.quantize_config).
+ self.resolve_scheme(
+ model_context=self.model_context,
+ compress_context=self.compress_context,
+ dataset=self._get_calibration_dataset(),
+ )
- if (
- self.act_bits <= 8
- and (not is_nv_fp(self.act_data_type) or "static_gs" not in self.act_data_type)
- and not is_mx_fp(self.act_data_type)
- and not is_dynamic_wint8aint8(self)
- and not is_static_wfp8afp8(self.act_data_type)
- and not is_dynamic_afp8(self)
- ):
- logger.warning(
- "activation quantization is an experimental feature with limited support and a complex API. "
- "And please save the quantized model to fake format as real deployment is not supported currently"
+ def _build_quantizer(self) -> None:
+ """Phase 1b – Quantizer construction and wiring.
+
+ Preconditions:
+ - :meth:`_resolve_scheme` complete: ``self.quantize_config`` carries
+ resolved scheme attrs.
+ - :meth:`_resolve_formats` complete: format-driven overrides have
+ been synced back to ``self.quantize_config``.
+
+ Work performed:
+ - Constructs ``self.quantizer`` from the resolved config.
+ - Calls ``quantizer.bind(self)`` so the quantizer pulls
+ ``model_context`` / ``compress_context`` / ``scale_dtype`` /
+ ``CalibrationState`` from this compressor. ``quantizer.model``
+ is a property that reads ``model_context.model``.
+
+ Postconditions:
+ - ``self.quantizer`` is ready and shares ``CalibrationState`` with
+ the compressor.
+ """
+ self.quantizer = BaseQuantizers.from_config(self.quantize_config)
+ self.quantizer.bind(self)
+
+ def _resolve_formats(self) -> None:
+ """Phase 2 – Format resolution and config attr sync.
+
+ Preconditions:
+ - Phase 1 complete: the scheme is resolved (``data_type``, ``bits``,
+ ``sym`` etc. are set on both ``self`` and ``self.quantize_config``).
+
+ Work performed:
+ - Converts a string ``self.formats`` to a list of
+ :class:`~auto_round.formats.OutputFormat` objects via
+ :func:`~auto_round.formats.get_formats`.
+ - Initialises :class:`~auto_round.compressors.shard_writer.ShardWriter`
+ when formats are present.
+ - **(2b)** Detects format-driven attribute mutations (``bits``, ``sym``,
+ ``data_type``, ``group_size``, etc.) that ``gguf_args_check`` may
+ have written onto ``self`` inside ``get_formats``, syncs them back
+ to ``self.quantize_config``, and rebuilds ``self.scheme`` accordingly.
+ - Merges any format-injected entries into ``self.layer_config``.
+
+ Postconditions:
+ - ``self.formats`` is a list (or ``None``).
+ - ``self.compress_context.formats`` mirrors ``self.formats``.
+ - ``self.quantize_config`` and ``self.scheme`` reflect the final attrs.
+ """
+ # get_formats() inspects data_type / bits etc. that were just resolved.
+ if isinstance(self.formats, str):
+ self.formats = get_formats(self.formats, self)
+ if self.formats is not None:
+ self.compress_context.formats = self.formats
+ ShardWriter.reset()
+ # Defer ShardWriter construction to _ensure_shard_writer() to avoid
+ # heap fragmentation during post_init (parameter iteration).
+
+ # Snapshot the user-specified layer_config before format processing may
+ # inject extra per-layer entries (e.g. GGUF embedding / lm_head).
+ _pre_gguf_layer_config = copy.copy(self.layer_config) or {}
+
+ # ── 2b: propagate format-adjusted attrs back to quantize_config ─────
+ # gguf_args_check (called inside get_formats) may have overridden
+ # bits / sym / data_type / super_bits / super_group_size / group_size
+ # on *this* BaseCompressor object via setattr(self, ...). Sync those
+ # changes to self.quantize_config before creating the quantizer so it is
+ # constructed with the definitive final values.
+ _gguf_forwarded_attrs = (
+ "bits",
+ "sym",
+ "data_type",
+ "super_bits",
+ "super_group_size",
+ "group_size",
+ "act_bits",
+ "scale_dtype",
+ )
+ _any_gguf_attr_changed = False
+ for _attr in _gguf_forwarded_attrs:
+ if _attr not in self.__dict__:
+ continue
+ config_val = getattr(self.quantize_config, _attr, None)
+ self_val = self.__dict__[_attr]
+ if _attr not in ("scale_dtype", "act_bits") and config_val != self_val:
+ _any_gguf_attr_changed = True
+ if config_val != self_val:
+ setattr(self.quantize_config, _attr, self_val)
+ # If format resolution changed scheme attrs, rebuild self.scheme so that
+ # configure_layer_config() / set_layer_config() see the correct values.
+ if _any_gguf_attr_changed:
+ from auto_round.schemes import PRESET_SCHEMES
+ from auto_round.schemes import QuantizationScheme as _QS
+
+ # Prefer to derive the scheme directly from the gguf format name to
+ # avoid ambiguity (e.g. Q4_K_S and Q4_K_M share identical weight attrs).
+ _gguf_preset_scheme = None
+ _gguf_fmt_name = None
+ _gguf_original_fmt_name = None
+ for _fmt in self.formats or []:
+ # GGUFFormat (outer) has output_format="gguf" but backend.output_format="gguf:q4_k_m"
+ # GGUFFormat (inner/standalone) has output_format="gguf:q4_k_m"
+ _of = getattr(_fmt, "output_format", "")
+ if "gguf" in str(_of):
+ if str(_of) == "gguf":
+ # outer GGUFFormat: full format in _original_format (e.g. "gguf:q2_k_mixed")
+ # or backend.output_format (e.g. "gguf:q2_k_s" after _mixed → _s conversion)
+ _orig = getattr(_fmt, "_original_format", None)
+ if _orig:
+ _gguf_original_fmt_name = str(_orig).upper()
+ _backend = getattr(_fmt, "backend", None)
+ _of = getattr(_backend, "output_format", _of) if _backend is not None else _of
+ _preset_key = str(_of).upper()
+ if _preset_key in PRESET_SCHEMES:
+ _gguf_preset_scheme = PRESET_SCHEMES[_preset_key]
+ _gguf_fmt_name = _preset_key
+ break
+ if _gguf_preset_scheme is not None:
+ # Update scheme on both compressor and quantizer.
+ self.scheme = _gguf_preset_scheme
+ # Store the exact gguf format name so configure_layer_config /
+ # set_layer_config can use it directly, avoiding Q4_K_S / Q4_K_M ambiguity.
+ self._gguf_format_name = _gguf_fmt_name
+ # Store original format name (may include _mixed) for _handle_special_schemes
+ if _gguf_original_fmt_name:
+ self._gguf_original_format_name = _gguf_original_fmt_name
+ else:
+ _new_scheme_dict = {f.name: getattr(self, f.name, None) for f in fields(_QS)}
+ _new_scheme = _QS.from_dict({k: v for k, v in _new_scheme_dict.items() if v is not None})
+ self.scheme = _new_scheme
+
+ _gguf_layer_cfg = {
+ k: v for k, v in (self.__dict__.get("layer_config") or {}).items() if k not in (_pre_gguf_layer_config)
+ }
+ if _gguf_layer_cfg:
+ if self.layer_config is None:
+ self.layer_config = {}
+ for _lname, _lval in _gguf_layer_cfg.items():
+ self.layer_config.setdefault(_lname, _lval)
+
+ def _apply_rotations(self) -> None:
+ """Phase 4.5 – Apply Hadamard / rotation transforms to the model.
+
+ Preconditions:
+ - Phase 3 complete: model topology is final (``apply_patches`` has
+ replaced / merged layers, e.g. MoE experts), so rotation operates
+ on the same modules that quantization will later see.
+ - Phase 4 complete: ``self.layer_config`` is built; rotation only
+ transforms weights and does not change layer names, so this
+ ordering matches the old arch where rotation ran after
+ ``configure_layer_config``.
+ - ``self.quantize_config.data_type`` is final (rotation backend
+ dispatch depends on it).
+
+ Work performed:
+ - Iterates ``self.rotation_configs`` and calls
+ :func:`~auto_round.algorithms.transforms.apply_rotation` on the
+ model for each config.
+
+ Postconditions:
+ - ``self.model_context.model`` carries the rotated weights and any
+ inserted online-Hadamard hooks.
+ """
+ if not self.rotation_configs:
+ return
+ logger.info("Applying Hadamard transform to the model.")
+ for rotation_cfg in self.rotation_configs:
+ self.model_context.model = apply_rotation(
+ self.model_context.model,
+ rotation_cfg,
+ data_type=self.quantize_config.data_type,
)
- if is_mx_fp(self.data_type) and self.group_size != 32:
- logger.warning("dtype mx_fp should only support group_size of 32 in real deployment")
+ def _patch_model(self) -> None:
+ """Phase 3 – Model structure patching.
- if is_nv_fp(self.data_type) and (self.group_size != 16):
- logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")
+ Preconditions:
+ - Phase 2 complete: ``self.formats`` is resolved so that
+ ``apply_patches`` can inspect format-specific requirements.
- if isinstance(self.group_size, tuple):
- if not is_block_wfp8(self):
- raise NotImplementedError("only support block-wise quantization for fp8 weight quantization.")
- if not is_dynamic_afp8(self):
- raise NotImplementedError("only support dynamic fp8 activation for fp8 weight quantization.")
+ Work performed:
+ - Delegates to :meth:`~auto_round.context.model.ModelContext.apply_patches`
+ which may replace or merge layers (e.g. MoE expert merging, adding
+ static-kv wrappers) to produce the final model topology.
- if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
- if self.batch_size > self.nsamples:
- if self.iters > 0: # GGUF should log this warning, but we don't know the format here
- logger.warning(
- f"reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"
- f" is smaller than batch_size({self.batch_size})"
- )
- self.batch_size = self.nsamples
- if self.gradient_accumulate_steps > self.nsamples // self.batch_size:
- self.gradient_accumulate_steps = self.nsamples // self.batch_size
- logger.warning(
- f"reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"
- f" as nsamples must equal or greater"
- f" than gradient_accumulate_steps * batch_size"
- )
+ Postconditions:
+ - ``self.model_context.model`` reflects the definitive topology that
+ :meth:`_build_layer_config` will walk.
+ """
+ # apply_patches() may replace layers (e.g. MoE expert merging); must
+ # happen before configure_layer_config() so it sees the final topology.
+ self.model_context.apply_patches(self.formats)
+
+ def _build_layer_config(self) -> None:
+ """Phase 4 – Layer-config construction and quantizer sync.
+
+ Preconditions:
+ - Phase 3 complete: model topology is final.
+ - ``self.scheme`` and all scheme-resolved attrs are consistent with
+ the (possibly GGUF-adjusted) values set in Phase 2.
+
+ Work performed:
+ - Calls :meth:`_scheme_post_init` which walks the patched model to
+ build ``self.layer_config``, ``self.quant_block_list``, etc.
+ On the AutoScheme path this also runs delta-loss forward/backward
+ passes to select per-layer schemes.
+ - Syncs the fully-resolved ``layer_config`` and related attrs to
+ ``self.quantizer`` so quantization methods have the complete view.
+
+ Postconditions:
+ - ``self.layer_config`` is fully populated.
+ - ``self.quantizer`` mirrors ``layer_config``, ``has_qlayer_outside_block``,
+ ``regex_config``, ``quant_block_list``, ``to_quant_block_names``,
+ ``scale_dtype``, and ``ignore_layers``.
+ """
+ # configure_layer_config() walks the patched model; _gen_auto_scheme()
+ # (AutoScheme path) runs delta-loss forward+backward passes.
+ self._scheme_post_init()
+
+ # Sync the fully-resolved scheme state to the quantizer so that
+ # quantization methods (quantize_block, quantize_layer, etc.) have
+ # access to layer_config, scale_dtype, quant_block_list, etc.
+ self.quantizer.layer_config = self.layer_config
+ self.quantizer.has_qlayer_outside_block = self.has_qlayer_outside_block
+ self.quantizer.regex_config = self.regex_config
+ self.quantizer.quant_block_list = self.quant_block_list
+ self.quantizer.to_quant_block_names = self.to_quant_block_names
+ self.quantizer.scale_dtype = self.scale_dtype
+ self.quantizer.ignore_layers = self.ignore_layers
+
+ def _hardware_setup(self) -> None:
+ """Phase 5 – Hardware and compile configuration.
+
+ Preconditions:
+ - Phase 4 complete: ``layer_config`` is built and
+ ``has_qlayer_outside_block`` is known.
+ - ``self.quantize_config.data_type`` is the final resolved value
+ (needed by :meth:`_adjust_torch_compile`).
+
+ Work performed:
+ - Applies the device map via :func:`~auto_round.utils.device.set_non_auto_device_map`.
+ - Re-evaluates ``torch.compile`` eligibility now that ``data_type`` is
+ resolved and writes the result back to ``compress_context``.
+ - Resets the offload manager when ``low_cpu_mem_usage`` is active.
+ - Disables ``self.inplace`` when quantized layers live outside
+ transformer blocks (incompatible with in-place rewriting).
+ - Calls :meth:`_adjust_immediate_packing_and_saving` to decide whether
+ layers should be packed / written immediately after each block.
+
+ Postconditions:
+ - ``compress_context.enable_torch_compile`` is final.
+ - ``self.inplace`` and ``compress_context.is_immediate_packing`` /
+ ``compress_context.is_immediate_saving`` are set to their definitive values.
+ """
+ set_non_auto_device_map(self.model_context.model, self.compress_context.device_map)
+ # Re-evaluate torch.compile eligibility now that data_type is resolved.
+ self._adjust_torch_compile(self.enable_torch_compile)
+ self.compress_context.enable_torch_compile = self.enable_torch_compile
+ if self.compress_context.low_cpu_mem_usage:
+ self._offloader.reset()
- if self.enable_norm_bias_tuning:
- logger.warning("the `enable_norm_bias_tuning` feature is experimental and currently has limited support.")
+ # Disable inplace when quantized layers live outside transformer blocks.
+ if self.has_qlayer_outside_block and self.need_calib:
+ self.inplace = False
- self._dq_check()
+ if not hasattr(self, "formats"):
+ logger.warning("this API is deprecated, please use `quantize_and_save` instead")
+ else:
+ self._adjust_immediate_packing_and_saving()
- def _check_compatibility(self) -> None:
- """Checks compatibility of the configurations and model."""
- if (
- self.seqlen is not None
- and hasattr(self.model, "config")
- and hasattr(self.model.config, "max_position_embeddings")
- ):
- if self.model.config.max_position_embeddings < self.seqlen:
- logger.warning(
- f"Change sequence length to {self.model.config.max_position_embeddings} "
- "due to the limitation of max_position_embeddings"
- )
- self.seqlen = min(self.seqlen, self.model.config.max_position_embeddings)
+ # backward compatible with the legacy API
+ def __getattr__(self, name: str) -> Any:
+ if name in self.__dict__:
+ return self.__dict__[name]
- if self.seqlen is not None and hasattr(self.tokenizer, "model_max_length"):
- if self.tokenizer.model_max_length < self.seqlen:
- logger.warning(
- f"Change sequence length to {self.tokenizer.model_max_length} "
- "due to the limitation of model_max_length. "
- "You can also try to increase the model_max_length to avoid this issue."
- )
- self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)
+ for obj in ["quantizer", "quantize_config", "model_context", "compress_context"]:
+ if obj not in self.__dict__:
+ continue
+ obj = object.__getattribute__(self, obj)
+ try:
+ return object.__getattribute__(obj, name)
+ except AttributeError:
+ continue
- if self.group_size == 0 and "fp8" not in self.data_type:
- logger.warning("`group_size==0` is not supported for data_type other than fp8 ")
+ raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
- if self.bits <= 2 and (self.iters < 1000 or not self.enable_alg_ext) and self.super_group_size is None:
- logger.warning(
- "for bits <= 2, it is recommended to enable `auto-round-best` " "and turn on `--enable_alg_ext` "
- )
+ # ── Forwarding properties to ``self._calibration_state`` ──────────────────
+ @property
+ def calibration_state(self):
+ return self._calibration_state
+
+ @calibration_state.setter
+ def calibration_state(self, value) -> None:
+ self._calibration_state = value
+ # Re-wire quantizer if it already exists so they keep sharing.
+ q = self.__dict__.get("quantizer")
+ if q is not None:
+ q.calibration_state = value
+
+ @property
+ def inputs(self) -> dict:
+ return self._calibration_state.inputs
+
+ @inputs.setter
+ def inputs(self, value: dict) -> None:
+ self._calibration_state.inputs = value if value is not None else {}
+
+ @property
+ def to_cached_layers(self) -> list:
+ return self._calibration_state.to_cached_layers
+
+ @to_cached_layers.setter
+ def to_cached_layers(self, value: list) -> None:
+ self._calibration_state.to_cached_layers = value if value is not None else []
+
+ @to_cached_layers.deleter
+ def to_cached_layers(self) -> None:
+ self._calibration_state.to_cached_layers = []
+
+ @property
+ def last_cache_name(self):
+ return self._calibration_state.last_cache_name
+
+ @last_cache_name.setter
+ def last_cache_name(self, value) -> None:
+ self._calibration_state.last_cache_name = value
+
+ @last_cache_name.deleter
+ def last_cache_name(self) -> None:
+ self._calibration_state.last_cache_name = None
+
+ @property
+ def blocks_requiring_input_ids(self) -> list:
+ return self._calibration_state.blocks_requiring_input_ids
+
+ @blocks_requiring_input_ids.setter
+ def blocks_requiring_input_ids(self, value: list) -> None:
+ self._calibration_state.blocks_requiring_input_ids = value if value is not None else []
+
+ @property
+ def batch_size(self) -> int:
+ return self._calibration_state.batch_size
+
+ @batch_size.setter
+ def batch_size(self, value: int) -> None:
+ self._calibration_state.batch_size = value
+
+ @property
+ def nsamples(self) -> int:
+ return self._calibration_state.nsamples
+
+ @nsamples.setter
+ def nsamples(self, value: int) -> None:
+ if value is not None:
+ self._calibration_state.nsamples = value
+
+ @property
+ def seqlen(self) -> int:
+ return self._calibration_state.seqlen
+
+ @seqlen.setter
+ def seqlen(self, value: int) -> None:
+ if value is not None:
+ self._calibration_state.seqlen = value
+
+ @property
+ def dataset(self):
+ return self._calibration_state.dataset
+
+ @dataset.setter
+ def dataset(self, value) -> None:
+ self._calibration_state.dataset = value
+
+ @property
+ def dataloader(self):
+ return self._calibration_state.dataloader
+
+ @dataloader.setter
+ def dataloader(self, value) -> None:
+ self._calibration_state.dataloader = value
+
+ @dataloader.deleter
+ def dataloader(self) -> None:
+ self._calibration_state.dataloader = None
+
+ @property
+ def optimizer(self):
+ """Return the actual optimizer class, converting string to class for backward compat.
+
+ Old API stored ``self.optimizer = torch.optim.AdamW`` (the class itself).
+ New arch stores the optimizer name as a string in ``quantize_config.optimizer``.
+ This property converts it so that ``ar.optimizer == torch.optim.AdamW`` works.
+ """
+ if self.quantize_config is None:
+ return None
+ opt = getattr(self.quantize_config, "optimizer", None)
+ if opt is None:
+ # Default to AdamW when enable_adam=True and no explicit optimizer was set
+ if getattr(self.quantize_config, "enable_adam", False):
+ return torch.optim.AdamW
+ return None
+ if isinstance(opt, str):
+ return getattr(torch.optim, opt, None)
+ return opt
- def quantize_and_save(
- self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace: bool = True, **kwargs
- ) -> tuple[torch.nn.Module, dict[str, Any]]:
- """Quantizes the model and saves it in the specified format(s).
+ def _adjust_immediate_packing_and_saving(self):
+ from auto_round.algorithms.quantization.rtn.config import RTNConfig
- This function checks the validity of the requested format(s), quantizes
- the model accordingly, and saves it to the specified output directory.
- If multiple formats are provided, the model is saved separately for each format.
+ if self.formats is None:
+ return
- Args:
- output_dir (str, optional): The directory where the quantized model
- will be saved. Defaults to "tmp_autoround".
- format (str, optional): The quantization format(s) to use, separated
- by commas if multiple. Defaults to "auto_round".
- inplace (bool, optional): Whether to modify the model in place if only
- one format is used. Defaults to True.
- **kwargs: Additional arguments for the quantization and saving process.
+ formats = getattr(self, "formats", [])
+ if len(formats) == 1 and not formats[0].is_fake() and self.inplace:
+ self.compress_context.is_immediate_packing = True
- Returns:
- model: A qdq model or packed model based on the configurations
- folders: The folder paths where the quantized models are saved.
+ if self.has_qlayer_outside_block and self.need_calib:
+ self.compress_context.is_immediate_packing = False
- Raises:
- ValueError: If an unsupported format is specified.
- """
- # Validate and process the specified formats
- self.orig_output_dir = output_dir
+ if not ("causallm" in self.model_context.model.__class__.__name__.lower() and not self.model_context.is_mllm):
+ # TODO For tied keys, there may some issues, we haven't not verified this
+ tied_weight_keys = getattr(self.model_context.model, "_tied_weight_keys", {})
+ if len(tied_weight_keys) > 1:
+ self.compress_context.is_immediate_saving = False
+ if self.compress_context.low_cpu_mem_usage:
+ logger.warning("reset low_cpu_mem_usage to False due to tied weights")
+ return
+ if len(tied_weight_keys) == 1:
+ key = list(tied_weight_keys.keys())[0]
+ if "lm_head" not in key:
+ self.compress_context.is_immediate_saving = False
+ if self.compress_context.low_cpu_mem_usage:
+ logger.warning("reset low_cpu_mem_usage to False due to tied weights")
+ return
- # check and update the format based on the current configuration
- format_list = get_formats(format, self)
- self.formats = format_list
+ if self.compress_context.low_cpu_mem_usage and self.compress_context.is_immediate_packing:
+ self.compress_context.is_immediate_saving = True
- # If multiple formats are specified, enforce inplace=False
- if len(format_list) > 1:
- inplace = False
- self.inplace = kwargs.get("inplace", inplace)
- kwargs.pop("inplace", None)
-
- # Perform model quantization
- if self.static_attention_dtype is not None:
- from auto_round.experimental.attention import attention_quant_ctx
-
- with attention_quant_ctx(self.model, static_attention_dtype=self.static_attention_dtype):
- model, _ = self.quantize()
- elif self.static_kv_dtype is not None:
- from auto_round.experimental.kv_cache import kvcache_quant_context
-
- with kvcache_quant_context(self.model, static_kv_dtype=self.static_kv_dtype):
- model, _ = self.quantize()
- else:
- model, _ = self.quantize()
- # Save the quantized model in the specified format_list
- model, folders = self.save_quantized(
- output_dir, format=self.formats, inplace=inplace, return_folders=True, **kwargs
- )
- memory_monitor.log_summary()
-
- return model, folders
-
- def _get_save_folder_name(self, format: OutputFormat) -> str:
- """Generates the save folder name based on the provided format string.
-
- If there are multiple formats to handle, the function creates a subfolder
- named after the format string with special characters replaced. If there's
- only one format, it returns the original output directory directly.
-
- Args:
- format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
-
- Returns:
- str: The path to the folder where results should be saved.
- """
- # Replace special characters to make the folder name filesystem-safe
- sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")
-
- # Use a subfolder only if there are multiple formats
- if len(self.formats) > 1:
- return os.path.join(self.orig_output_dir, sanitized_format)
-
- return self.orig_output_dir
-
- @torch.inference_mode()
- def _quantize_embedding_layer(self):
- """Quantizes embedding layers in the model according to the configuration.
-
- This method iterates through all modules in the model, identifies embedding
- layers specified in `self.layer_config`, and applies the appropriate quantization
- function based on bit precision, grouping strategy, and dtype.
-
- Returns:
- bool: True if the quantization process completes without critical errors.
- """
- is_quantized = False
- for name, module in self.model.named_modules():
- # Skip non-Embedding modules or layers not in config
- if not isinstance(module, torch.nn.Embedding) or name not in self.layer_config:
- continue
-
- config = self.layer_config[name]
-
- # Skip layers that are not marked for quantization
- if not check_to_quantized(config):
- continue
- is_quantized = True
- config["scale_dtype"] = self.scale_dtype
- dtype = config["data_type"]
-
- # Determine quantization function key with symmetry/asymmetry
- if dtype not in QUANT_FUNC_WITH_DTYPE:
- dtype = f"{dtype}_{'sym' if config['sym'] else 'asym'}"
-
- # Optionally use optimized rounding (RTN) variant
- if not self.disable_opt_rtn and f"opt_rtn_{dtype}" in QUANT_FUNC_WITH_DTYPE:
- dtype = f"opt_rtn_{dtype}"
- elif f"rtn_{dtype}" in QUANT_FUNC_WITH_DTYPE:
- dtype = f"rtn_{dtype}"
-
- quant_func = QUANT_FUNC_WITH_DTYPE[dtype]
- dtype = module.weight.dtype
- # As typically float32 are used in RTN to search scale zp,
- # to avoid cache a bf16 copy we'd better use float32
- if config.get("super_group_size", None) is not None:
- dtype = torch.float32
-
- # Attempt quantization on GPU, fall back to CPU if OOM
- try:
- weight, scale, zp = quant_func(
- module.weight.to(dtype=dtype, device=self.device),
- **{
- k: config.get(k, None)
- for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]
- },
- )
- except torch.OutOfMemoryError:
- cuda_error_msg = traceback.format_exc()
- try:
- logger.error(cuda_error_msg)
- logger.warning("falling back to CPU")
- weight, scale, zp = quant_func(
- module.weight.to("cpu"),
- **{
- k: config.get(k, None)
- for k in ["bits", "group_size", "super_bits", "super_group_size", "scale_dtype"]
- },
- )
- except Exception as e:
- raise
-
- # Overwrite the module's weights with the quantized version
- module.weight.data.copy_(weight.cpu())
-
- # Attach scale and zero point (zp) to the module
- for param_name, value in zip(["scale", "zp"], [scale, zp]):
- if isinstance(value, dict):
- for k, v in value.items():
- setattr(module, k if k == "scale" else f"w_{k}", v.cpu())
- elif isinstance(value, torch.Tensor):
- setattr(module, param_name, value.cpu())
- else:
- setattr(module, param_name, value)
-
- # Update config
- self.layer_config.setdefault(name, {}).update(config)
- del weight
- del scale
- del zp
- clear_memory(device_list=self.device_list)
-
- return is_quantized
-
- def _quant_rtn_with_imatrix(self, all_to_quantized_module_names: list[str]) -> None:
- """Performs RTN quantization using input activation statistics (imatrix).
-
- This method accumulates per-channel second-moment activation statistics (imatrix)
- via forward hooks and uses them to perform RTN quantization. If CUDA memory runs out,
- it falls back to CPU-based blockwise quantization.
-
- Args:
- all_to_quantized_module_names (list[str]):
- A list of module names (e.g., 'model.layers.0.self_attn.q_proj') to be quantized.
-
- Returns:
- None
- """
- logger.info("start to compute imatrix")
-
- # Load dataset
- from auto_round.calib_dataset import get_dataloader
-
- if isinstance(self.dataset, str):
- if self.tokenizer is None:
- raise ValueError("A tokenizer must be set for the model when using a dataset string.")
- dataset_name = self.dataset.replace(" ", "")
- self.dataloader = get_dataloader(
- self.tokenizer, self.seqlen, dataset_name, self.seed, self.batch_size, self.nsamples
- )
- else:
- self.dataloader = self.dataset
-
- model = self.model
-
- # Dispatch multi-GPU model if necessary
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- dispatch_model(model, model.hf_device_map)
-
- def register_act_hook(model):
- """Registers hooks to accumulate activation squared norms into `imatrix`."""
-
- def get_imatrix_hook(module, input, output):
- input = input[0] if isinstance(input, (tuple, list)) else input
- flattened = input.reshape(-1, input.shape[-1]).to(torch.float32)
- squared = torch.sum(torch.pow(flattened, 2), dim=0).to(torch.float32)
-
- if not hasattr(module, "imatrix"):
- module.imatrix = squared
- module.imatrix_cnt = input.shape[0]
- else:
- module.imatrix += squared.to(module.imatrix.device)
- module.imatrix_cnt += input.shape[0]
-
- hook_handles = []
- for name, module in model.named_modules():
- if type(module) in self.supported_types and check_to_quantized(module):
- hook = module.register_forward_hook(get_imatrix_hook)
- hook_handles.append(hook)
- return hook_handles
-
- if not is_wint4aint4(self): # INT4 no imatrix is much better
- hooks = register_act_hook(model)
- else:
- hooks = []
-
- try:
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- import accelerate
-
- accelerate.hooks.remove_hook_from_submodules(model)
- safe_to_cpu_(model)
- clear_memory(device_list=self.device_list)
- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
- except torch.OutOfMemoryError:
- cuda_error_msg = traceback.format_exc()
- try:
- logger.error(cuda_error_msg)
- # Final fallback: warn and use CPU-only quantization
- logger.warning(
- "Fallback to CPU. "
- "Consider enabling `low_gpu_mem_usage` or using more GPUs via `--device 0,1,2,3`."
- )
- safe_to_cpu_(model)
- clear_memory(device_list=self.device_list)
- if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
- import accelerate
-
- accelerate.hooks.remove_hook_from_submodules(model)
-
- orig_device = self.device
- self.device = "cpu"
- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
- self.device = orig_device
- except Exception as e:
- raise
- finally:
- # Always remove hooks
- for hook in hooks:
- hook.remove()
-
- def _quantize_layer_via_rtn(self, name: str, dtype: torch.dtype = None, to_cpu=True) -> None:
- """Quantizes a layer using RTN (Round-To-Nearest) if available.
-
- This function attempts to quantize a layer by switching its data type to a
- `rtn_*` version if supported, then wraps and unwraps the module to apply
- quantization. If GPU memory is insufficient, it falls back to CPU.
-
- If packing is enabled (`immediate_packing`), the function will also export
- the quantized layer to the appropriate backend format.
-
- Args:
- name (str): Name of the layer to quantize.
-
- Raises:
- RuntimeError: If quantization fails for reasons unrelated to memory.
- """
- m = get_module(self.model, name)
- if dtype is not None:
- m = m.to(dtype)
-
- m = convert_module_to_hp_if_necessary(m, self.amp_dtype, self.device)
- set_module(self.model, name, m)
- tuning_device = m.tuning_device if hasattr(m, "tuning_device") else self.device
- # Step 1: let gguf merge layers or rename module first and we will handle the RTN is gguf specific logic
- if self.is_immediate_packing and self.iters == 0 and self.formats[0].is_gguf() and not self.disable_opt_rtn:
- m = m.to(tuning_device)
- m.scale = None
- m.zp = None
- else:
- try:
- disable_opt_rtn = self.disable_opt_rtn
- if (
- not disable_opt_rtn
- and self.orig_disable_opt_rtn is None
- and self.is_moe_model
- and "expert" in m.global_name
- and "shared_expert" not in m.global_name
- and self.super_bits is None # GGUF still uses the optimized RTN for MoE layers
- ):
- disable_opt_rtn = True
- logger.warning_once(
- "MoE layer detected: optimized RTN is disabled for efficiency. "
- "Use `--enable_opt_rtn` to force-enable it for MoE layers."
- )
- m = m.to(tuning_device)
- m = WrapperLinear(
- m,
- device=tuning_device,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- enable_round_tuning=False,
- enable_torch_compile=self.enable_torch_compile,
- disable_opt_rtn=disable_opt_rtn,
- iters=self.iters,
- )
- m = m.unwrapper({})
- except torch.OutOfMemoryError:
- cuda_error_msg = traceback.format_exc()
- m = m.orig_layer if hasattr(m, "orig_layer") else m
- try:
- logger.error(cuda_error_msg)
- logger.warning("falling back to CPU.")
- m.to("cpu")
- m = WrapperLinear(
- m,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- enable_round_tuning=False,
- enable_torch_compile=self.enable_torch_compile,
- iters=self.iters,
- )
- m = m.unwrapper({})
- except Exception as e:
- raise
-
- # Step 2: Optional immediate packing/export
- if self.is_immediate_packing: # For gguf, packing conducts on block level
- self._immediate_pack(name)
- if to_cpu:
- m = m.to("cpu")
- packed_m = get_module(self.model, name)
- set_module(self.model, name, packed_m.to("cpu"))
- else:
- if to_cpu:
- m = m.to("cpu")
- set_module(self.model, name, m)
- if self.is_immediate_saving:
- m = get_module(self.model, name)
- m.to("cpu")
- shard_writer(self, m, name, False)
- # Free RAM immediately: the data is now in the shard-writer buffer
- # (and will be flushed to disk). Keeping it also in the model tree
- # causes linear RAM growth for large models.
- m.to("meta")
-
- def _immediate_pack(self, name: str):
- if not self.is_immediate_packing:
- return
- self.formats[0].immediate_pack(
- name=name,
- model=self.model,
- device=self.device,
- output_dir=self._get_save_folder_name(self.formats[0]),
- layer_config=self.layer_config,
- tokenizer=self.tokenizer,
- )
-
- # Use no_grad instead of inference mode
- # https://github.com/intel/auto-round/issues/1620
- @torch.no_grad()
- def _quantize_rtn(self) -> tuple[torch.nn.Module, dict[str, Any]]:
- """Quantize all modules in the model using RTN (Round-To-Nearest) strategy.
-
- If the target format includes GGUF with `k`, and optimized RTN is enabled,
- blockwise quantization with input caching and imatrix is used.
-
- Returns:
- tuple[nn.Module, Dict[str, Any]]: The quantized model and the layer configuration.
- """
- if self.amp and self.model.dtype != self.amp_dtype:
- self.model.to(self.amp_dtype)
-
- all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
- self.all_to_quantized_module_names = all_to_quantized_module_names
-
- if not (any(fmt.is_gguf() for fmt in getattr(self, "formats", [])) or self.super_bits is not None):
- self._quantize_embedding_layer() # leave to gguf itself to handle
-
- # Release memory
- clear_memory(device_list=self.device_list)
-
- enable_imatrix = False
- if not self.disable_opt_rtn:
- has_gguf_k = (
- any(fmt.is_gguf() and "k" in fmt.output_format for fmt in getattr(self, "formats", []))
- or self.super_bits is not None
- )
- if has_gguf_k:
- enable_imatrix = True
- elif self.data_type == "int" and self.sym:
- enable_imatrix = True
- if enable_imatrix:
- self._quant_rtn_with_imatrix(all_to_quantized_module_names)
- elif self.act_bits <= 8 and check_need_act_calibration(
- self.act_dynamic,
- self.act_data_type,
- self.act_bits,
- self.static_kv_dtype,
- self.static_attention_dtype,
- ): # TODO, mixed datatype has bug
- hook_handles = self._register_act_max_hook(self.model)
- try:
- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
- except torch.OutOfMemoryError:
- logger.warning("Fallback to CPU. Consider using more GPUs via `--device 0,1,2,3`.")
- self.model = self.model.to("cpu")
- clear_memory(device_list=self.device_list)
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- import accelerate
-
- accelerate.hooks.remove_hook_from_submodules(self.model)
- orig_device = self.device
- self.device = "cpu"
- self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
- self.device = orig_device
- for handle in hook_handles:
- handle.remove()
- else:
- # By default, we go with layer-wise way if no replacement happened.
- # In RTN mode (iters == 0), force blockwise quantization to avoid
- # full-model materialization and linear CPU RAM growth.
- use_blockwise_quantization = global_state.replaced_module_count > 0
- if self.iters == 0 and not use_blockwise_quantization:
- logger.info(
- "RTN mode detected (iters=0): force blockwise quantization to avoid "
- "layer-wise full-model materialization."
- )
- use_blockwise_quantization = True
- tied_weights_keys = getattr(self.model, "_tied_weights_keys", [])
- if tied_weights_keys is None:
- tied_weights_keys = []
- if isinstance(tied_weights_keys, dict):
- tied_weights_values = list(tied_weights_keys.values())
- else:
- tied_weights_values = list(tied_weights_keys)
- tied_weights_layers = [".".join(val.split(".")[:-1]) for val in tied_weights_values] # rm weight/bias
- # In fact, we should detect whether it is is_separate_lm_head, to simplify, we don't do it
- if hasattr(self, "formats") and self.formats[0].is_gguf():
- lm_head_name = get_lm_head_name(self.model)
- if lm_head_name is not None:
- tied_weights_layers.append(lm_head_name)
-
- if use_blockwise_quantization: # The ram usage is a little higher
- all_to_quantized_module_names = list(dict.fromkeys(all_to_quantized_module_names))
-
- all_blocks = self.quant_block_list or get_block_names(self.model)
- pbar = tqdm(range(sum(len(block) for block in all_blocks)))
- for block_names in all_blocks:
- for block_name in block_names:
- pbar.set_description(f"Quantizing {block_name}")
- block = get_module(self.model, block_name)
-
- materialize_model_(block)
-
- for name, m in block.named_modules():
- if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:
- self._quantize_layer_via_rtn(m.global_name, to_cpu=self.low_gpu_mem_usage)
- all_to_quantized_module_names.remove(m.global_name)
- elif (
- not any(m.children())
- and len(m.state_dict()) > 0
- and m.global_name not in tied_weights_layers
- and self.is_immediate_saving
- ):
- set_module(self.model, m.global_name, copy.deepcopy(m))
- if self.is_immediate_saving:
- shard_writer(self, name=m.global_name)
- copied_m = get_module(self.model, m.global_name)
- copied_m.to("meta")
- m.to("meta")
- # Move remaining GPU tensors to CPU; offload to disk if low_cpu_mem_usage.
- # This mirrors _quantize_via_rtn_blockwise's post-block cleanup.
- if not self.is_immediate_saving:
- mv_module_from_gpu(block)
- else:
- # Save once at block scope to capture tensors that are not saved
- # in per-layer branch (e.g., custom module-level params/buffers).
- shard_writer(self, name=block_name)
- block.to("meta")
- if self.low_cpu_mem_usage and not self.is_immediate_saving:
- self._offloader(self.model, block_name)
- clear_memory(device_list=self.device_list)
- memory_monitor.log_summary()
- pbar.update(1)
- cnt = 1
- for name in all_to_quantized_module_names:
- logger.info(f"Quantizing remaining layer {name} on CPU.")
- self._quantize_layer_via_rtn(name, to_cpu=True)
- cnt += 1
- if cnt % 10 == 0:
- clear_memory(device_list=self.device_list)
- memory_monitor.log_summary()
- else:
- materialize_model_(self.model)
- self.model.to("cpu")
- block_names_cnt = len(flatten_list(get_block_names(self.model, True)))
- clear_mem_freq = len(all_to_quantized_module_names) // block_names_cnt
- cnt = 0
- pbar = tqdm(all_to_quantized_module_names)
-
- for n, m in self.model.named_modules():
- if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:
- pbar.set_description(f"Quantizing {m.global_name}")
- self._quantize_layer_via_rtn(m.global_name)
- cnt += 1
- pbar.update()
- if cnt % clear_mem_freq == 0:
- clear_memory(device_list=self.device_list)
- memory_monitor.log_summary()
-
- elif (
- not any(m.children())
- and len(m.state_dict()) > 0
- and n not in tied_weights_layers
- and self.is_immediate_saving
- ):
- set_module(self.model, n, copy.deepcopy(m))
- shard_writer(self, name=n)
- m.to("meta")
-
- # Convert remaining fp8
- convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device)
- if self.low_cpu_mem_usage:
- self._offloader.reload(self.model)
- if self.is_immediate_saving:
- shard_writer(self, is_finalize=True)
-
- self.quantized = True
- return self.model, self.layer_config
-
- def _quantize_via_rtn_blockwise(self, all_to_quantized_module_names: list[str]) -> None:
- """Quantize model layers block by block using cached inputs and imatrix.
-
- Args:
- all_to_quantized_module_names (list[str]): Names of layers to be quantized.
- """
- all_to_quantized_module_names = list(set(all_to_quantized_module_names))
-
- all_blocks = self.quant_block_list or get_block_names(self.model)
- if not all_blocks:
- raise ValueError("Could not find any blocks. Check the model or quant_block_list.")
-
- if not self.has_variable_block_shape:
- to_cache_block_names = [block[0] for block in all_blocks]
- else:
- to_cache_block_names = flatten_list(all_blocks)
- layer_names = self._get_quantized_layer_names_outside_blocks()
- if self.act_bits < 16 and (not self.act_dynamic or len(layer_names) > 0) or self.has_variable_block_shape:
- if len(layer_names) > 0:
- logger.warning(
- "quantize layers outside blocks for static activation quantizaiton"
- " will significantly increase calibration time"
- )
- all_inputs = self.try_cache_inter_data_gpucpu(to_cache_block_names, self.nsamples, layer_names)
- else:
- all_inputs = self.cache_inter_data(to_cache_block_names, self.nsamples)
-
- # Clear hooks for multi-GPU setups
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(self.model)
-
- pbar = tqdm(range(sum(len(block) for block in all_blocks)))
-
- for block_names in all_blocks:
- first_block = block_names[0]
- inputs = all_inputs.pop(first_block)
- input_keys = [k for k in inputs if k.startswith("hidden_state")]
- if len(input_keys) != 1:
- raise RuntimeError(
- "hidden_states arg mismatch. Please file an issue at https://github.com/intel/auto-round/issues"
- )
- inputs["input_ids"] = inputs.pop(input_keys[0])
-
- clear_memory(self.inputs, device_list=self.device_list)
-
- total_samples = len(inputs["input_ids"])
- if total_samples < self.batch_size:
- self.batch_size = total_samples
- logger.warning(f"Forcing batch size to {total_samples}")
- tmp_dtype = self.amp_dtype if self.amp else torch.float32
-
- input_ids = to_device(inputs.pop("input_ids"), self.cache_device)
- input_ids = [id_.to(tmp_dtype) for id_ in input_ids]
-
- def process_input_others(input_others):
-
- input_others = to_device(input_others, self.cache_device)
- for key, val in input_others.items():
- if isinstance(val, torch.Tensor) and val.dtype in (torch.float16, torch.bfloat16):
- input_others[key] = val.to(tmp_dtype)
- elif isinstance(val, list):
- input_others[key] = [to_dtype(v, tmp_dtype) for v in val]
- return input_others
-
- input_others = inputs
- input_others = process_input_others(input_others)
- for block_name in block_names:
- if block_name in all_inputs.keys():
- input_others = all_inputs[block_name]
- input_others = process_input_others(input_others)
- all_inputs.pop(block_name)
- pbar.set_description(f"Quantizing {block_name}")
- block = get_module(self.model, block_name)
- materialize_model_(block)
- block.to("cpu")
-
- block = convert_module_to_hp_if_necessary(block, dtype=self.amp_dtype, device=self.device)
- update_block_global_scale_if_needed(block, self.data_type, self.group_size)
- self._register_act_max_hook(block)
- if (
- is_auto_device_mapping(self.device_map)
- and len(self.device_list) > 1
- and not getattr(self, "is_diffusion", False)
- ):
- set_auto_device_map_for_block_with_tuning(
- block, self.device_map, input_ids, self.low_gpu_mem_usage, self.batch_size, self.device
- )
- # Dispatch model if needed
- if len(self.device_list) > 1:
- from accelerate.hooks import AlignDevicesHook, add_hook_to_module
-
- for _, m in block.named_modules():
- if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
- continue
- hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
- add_hook_to_module(m, hook, True)
- else:
- block = block.to(self.device)
- input_ids = self._get_block_outputs(
- block,
- input_ids,
- input_others,
- self.batch_size * self.infer_bs_coeff,
- self.device,
- self.cache_device,
- )
-
- if len(self.device_list) > 1:
- accelerate.hooks.remove_hook_from_submodules(block)
-
- if is_nv_fp(self.act_data_type) or is_static_wfp8afp8(self):
- # enable moe experts act_max automatic generation for Linear
- set_amax_for_all_moe_layers(block, attr_name="act_max")
- # Normalize imatrix and quantize layers
- if self.low_gpu_mem_usage:
- block.to("cpu")
- clear_memory(device_list=self.device_list)
-
- for name, m in block.named_modules():
- # fix issue: Ling-flash-2.0-q2_k_s fail infer on cuda but well on cpu
- # https://huggingface.co/Intel/Ling-flash-2.0-gguf-q2ks-mixed-AutoRound/discussions/1
- if hasattr(m, "imatrix"):
- m.imatrix /= m.imatrix_cnt
- if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:
- self._quantize_layer_via_rtn(m.global_name, to_cpu=self.low_gpu_mem_usage)
- all_to_quantized_module_names.remove(m.global_name)
-
- mv_module_from_gpu(block)
- if self.low_cpu_mem_usage and not self.is_immediate_saving:
- self._offloader(self.model, block_name)
- if block_name == block_names[-1]:
- clear_memory(input_ids, device_list=self.device_list)
- else:
- clear_memory(device_list=self.device_list)
-
- memory_monitor.log_summary()
- pbar.update(1)
- pbar.close()
- # Process remaining layers not in blocks
- for name in all_to_quantized_module_names:
- dtype = None
- if self.super_group_size is not None:
- dtype = torch.float32
- self._quantize_layer_via_rtn(name, dtype=dtype)
- # clear_memory(device_list=self.device_list)
- # if self.is_immediate_saving:
- # shard_writer(self, is_finalize=True)
-
- def _update_inputs(self, inputs: dict, q_inputs: dict) -> tuple[dict, torch.Tensor]:
- keys = inputs.keys()
- input_id_str = [key for key in keys if key.startswith("hidden_state")]
- if len(input_id_str) != 1:
- raise RuntimeError(
- "hidden_states arg mismatch error,"
- "please raise an issue in https://github.com/intel/auto-round/issues"
- )
- inputs["input_ids"] = inputs.pop(input_id_str[0], None)
- if q_inputs is not None:
- q_inputs = q_inputs.pop(input_id_str[0], None)
- return inputs, q_inputs
-
- def configure_layer_config(self, enable_gguf_official_mixed: None | bool = True):
- is_gguf_format = (f := getattr(self, "formats", None)) is not None and len(f) > 0 and f[0].is_gguf()
- if not is_gguf_format:
- predefined_ignore_layers = get_predefined_ignore_layers(self.model)
- compressed_predefined_ignore_layers = compress_layer_names(predefined_ignore_layers)
- if predefined_ignore_layers:
- logger.info(f"Using predefined ignore_layers: {compressed_predefined_ignore_layers}")
- tmp_str = ",".join(predefined_ignore_layers)
- if self.ignore_layers == "":
- self.ignore_layers = tmp_str
- else:
- self.ignore_layers += "," + tmp_str
-
- if self.is_auto_scheme:
- self.layer_config = self._gen_auto_scheme()
- else:
- self.layer_config = _handle_special_schemes(
- self.orig_scheme,
- self.layer_config,
- self.model,
- supported_types=self.supported_types,
- inner_supported_types=self.inner_supported_types,
- quant_lm_head=self.quant_lm_head,
- mllm=getattr(self, "mllm", False),
- )
-
- fill_default_value = True
- if self.is_auto_scheme:
- fill_default_value = False
- self.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
- self.model,
- self.layer_config,
- self.scheme,
- self.scale_dtype,
- self.supported_types,
- self.inner_supported_types,
- self.quant_block_list,
- self.ignore_layers,
- self.quant_lm_head,
- enable_gguf_official_mixed=enable_gguf_official_mixed,
- is_mllm=self.mllm,
- fill_default_value=fill_default_value,
- )
-
- def _adjust_immediate_packing_and_saving(self):
- formats = getattr(self, "formats", [])
- if len(formats) == 1 and not formats[0].is_fake() and self.inplace:
- self.is_immediate_packing = True
-
- if self.has_qlayer_outside_block and self.iters != 0:
- self.is_immediate_packing = False
-
- if not ("causallm" in self.model.__class__.__name__.lower() and not self.mllm):
- # TODO For tied keys, there may some issues, we haven't not verified this
- tied_weight_keys = getattr(self.model, "_tied_weight_keys", {})
- if len(tied_weight_keys) > 1:
- self.is_immediate_saving = False
- if self.low_cpu_mem_usage:
- logger.warning("reset low_cpu_mem_usage to False due to tied weights")
- return
- if len(tied_weight_keys) == 1:
- key = tied_weight_keys.keys[0]
- if "lm_head" not in key:
- self.is_immediate_saving = False
- if self.low_cpu_mem_usage:
- logger.warning("reset low_cpu_mem_usage to False due to tied weights")
- return
-
- if self.low_cpu_mem_usage and self.is_immediate_packing:
- self.is_immediate_saving = True
-
- if self.low_cpu_mem_usage and not self.is_immediate_packing:
- logger.info(
- "`low_cpu_mem_usage` is only supported when `immediate_packing` is True. "
- "Setting `low_cpu_mem_usage` to False."
- )
- self.low_cpu_mem_usage = False
- self.is_immediate_saving = False
-
- if self.low_cpu_mem_usage and self.is_immediate_packing:
+ if self.compress_context.low_cpu_mem_usage and self.compress_context.is_immediate_packing:
if formats[0].is_gguf():
logger.warning(
"`low_cpu_mem_usage` is not fully supported for gguf format. "
"Setting `low_cpu_mem_usage` to False."
)
- self.low_cpu_mem_usage = False
- self.is_immediate_saving = False
- elif self.has_qlayer_outside_block and self.disable_opt_rtn and self.iters == 0:
+ self.compress_context.low_cpu_mem_usage = False
+ self.compress_context.is_immediate_saving = False
+ elif (
+ self.has_qlayer_outside_block
+ and getattr(self, "disable_opt_rtn", None)
+ and isinstance(self.quantize_config, RTNConfig)
+ ):
logger.info(
"Keeping `low_cpu_mem_usage` enabled in RTN mode (iters=0): "
"RTN path uses blockwise quantization and supports per-block offloading."
)
- elif self.has_qlayer_outside_block and self.iters > 0:
+ elif self.has_qlayer_outside_block and not isinstance(self.quantize_config, RTNConfig):
logger.warning(
"`low_cpu_mem_usage` is not fully supported "
"when there are quantized layers outside blocks and optimized RTN is disabled. "
"Setting low_cpu_mem_usage to False."
)
- self.low_cpu_mem_usage = False
- self.is_immediate_saving = False
+ self.compress_context.low_cpu_mem_usage = False
+ self.compress_context.is_immediate_saving = False
- if self.is_immediate_saving and not (
- "int" in self.data_type or is_nv_fp(self.data_type) or is_mx_fp(self.data_type)
+ if self.compress_context.is_immediate_saving and not (
+ "int" in self.quantize_config.data_type
+ or is_nv_fp(self.quantize_config.data_type)
+ or is_mx_fp(self.quantize_config.data_type)
):
logger.warning("immediate_saving is only supported for int/nv_fp/mx_fp quantization, set to False")
- self.is_immediate_saving = False
+ self.compress_context.is_immediate_saving = False
- if self.orig_output_dir is None:
- self.is_immediate_saving = False
+ if self.output_dir is None:
+ self.compress_context.is_immediate_saving = False
+
+ # Create ShardWriter eagerly only when immediate saving is active
+ # (it interleaves with the quantize loop). Otherwise keep it deferred
+ # until save_quantized() to avoid heap fragmentation during init.
+ if self.compress_context.is_immediate_saving:
+ self._ensure_shard_writer()
+
+ def _ensure_shard_writer(self):
+ """Lazily create ShardWriter if it hasn't been created yet."""
+ if self.shard_writer is None and self.formats is not None:
+ self.shard_writer = ShardWriter(self.model_context.model, bits=8)
def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:
"""Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
Returns:
The quantized model and layer configurations.
"""
-
- self._check_compatibility()
- formats = self.formats if hasattr(self, "formats") else None
- # It is best to modify the model structure in the quantize function and check the format,
- # because it may cause the gguf format to not be exported normally.
- self.model = update_module(
- self.model, formats=formats, trust_remote_code=self.trust_remote_code, cleanup_original=False
- )
-
- # Temporary names must be assigned after handle_moe_model;
- # placing them earlier would cause them to be removed when the module is replaced.
- for n, m in self.model.named_modules():
- m.global_name = n
-
- if not self.is_auto_scheme:
- enable_gguf_official_mixed = True
- else:
- enable_gguf_official_mixed = False
-
- self.configure_layer_config(enable_gguf_official_mixed=enable_gguf_official_mixed)
-
- if self.low_cpu_mem_usage:
- self._offloader.reset()
-
- def _should_disable_inplace_due_to_layers_outside_block() -> bool:
- return self.has_qlayer_outside_block and (self.iters != 0 or (self.iters == 0 and not self.disable_opt_rtn))
-
- # Disable inplace mode when there are quantized layers outside blocks
- # under specific iteration/optimization settings.
- if _should_disable_inplace_due_to_layers_outside_block():
- self.inplace = False
- if not hasattr(self, "formats"):
- logger.warning("this API is deprecated, please use `quantize_and_save` instead")
- else:
- # Determine if immediate packing is required
- self._adjust_immediate_packing_and_saving()
-
- if self.iters == 0:
- return self._quantize_rtn()
-
- if bool(self.quant_block_list):
- all_blocks = self.quant_block_list
- else:
- all_blocks = get_block_names(self.model)
-
- if len(all_blocks) == 0:
- logger.warning("could not find blocks, exit with original model")
- return self.model, self.layer_config
-
- if self.amp and self.model.dtype != self.amp_dtype:
- self.model = self.model.to(self.amp_dtype)
-
- layer_names = self._get_quantized_layer_names_outside_blocks()
- if not self.has_variable_block_shape:
- to_cache_block_names = [block[0] for block in all_blocks]
- else:
- to_cache_block_names = flatten_list(all_blocks)
- if len(layer_names) > 0:
- logger.info(
- "Starting to cache block inputs. This may be slow due to external block layers: %s", layer_names
- )
- else:
- logger.info("start to cache block inputs")
- all_inputs = self.try_cache_inter_data_gpucpu(to_cache_block_names, self.nsamples, layer_names=layer_names)
- is_quantized_embedding = self._quantize_embedding_layer()
- clear_memory(device_list=self.device_list)
- all_q_inputs = None
- if is_quantized_embedding:
- all_inputs = copy.deepcopy(self.inputs)
- clear_memory(self.inputs, device_list=self.device_list)
- all_q_inputs = self.try_cache_inter_data_gpucpu(
- to_cache_block_names, self.nsamples, layer_names=layer_names
- )
- # Remove accelerate dispatch hooks before moving parameters.
- # hf_device_map is kept for reference but hooks are no longer needed.
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(self.model)
- self.model = mv_module_from_gpu(self.model)
- clear_memory(device_list=self.device_list)
- logger.info("caching done")
- if self.low_cpu_mem_usage:
- if self.is_model_patched and not self.is_immediate_saving:
- self._offloader(self.model, all_blocks, clear_memory=True, device_list=self.device_list)
- if not self._offloader.enabled:
- self.low_cpu_mem_usage = False
- else:
- self.low_cpu_mem_usage = False
- if len(all_blocks) > 1:
- pbar = tqdm(range(0, sum([len(i) for i in all_blocks]), self.nblocks))
- else:
- pbar = tqdm(range(0, len(all_blocks[0]), self.nblocks)) # move the alg warning outside pbar
-
- start_time = time.time()
- for block_names in all_blocks:
- inputs = all_inputs[block_names[0]]
- all_inputs.pop(block_names[0])
- q_inputs = None
- if all_q_inputs is not None:
- q_inputs = all_q_inputs[block_names[0]]
- all_q_inputs.pop(block_names[0])
-
- inputs, q_inputs = self._update_inputs(inputs, q_inputs)
-
- clear_memory(self.inputs, device_list=self.device_list)
-
- if "input_ids" in inputs.keys():
- total_samples = len(inputs["input_ids"])
- if total_samples < self.batch_size:
- self.batch_size = total_samples
- logger.warning(f"force the train batch size to {total_samples}")
-
- self._quantize_blocks(
- self.model,
- inputs,
- block_names,
- q_input=q_inputs if q_inputs is not None else None,
- nblocks=self.nblocks,
- device=self.device,
- pbar=pbar,
- input_others_extra_blocks=all_inputs,
- )
- if self.is_immediate_packing and len(self.formats) != 1:
- raise ValueError(
- f"Expected exactly one packing format when 'immediate_packing' is True, "
- f"but got {len(self.formats)} formats."
- )
- pbar.set_description("Quantizing done")
- pbar.close()
- if self.low_cpu_mem_usage:
- self._offloader.reload(self.model)
- self._quantize_layers(layer_names, all_inputs)
-
- convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device, to_cpu=True)
- if self.is_immediate_saving:
- shard_writer(self, is_finalize=True)
-
- end_time = time.time()
- cost_time = end_time - start_time
- logger.info(f"quantization tuning time {cost_time}")
-
- # Dump a summary
- quantized_layers = []
- unquantized_layers = []
- for n, m in self.model.named_modules():
- if isinstance(m, tuple(self.supported_types)):
- if check_to_quantized(m):
- quantized_layers.append(n)
- else:
- unquantized_layers.append(n)
- elif hasattr(m, "scales") or hasattr(m, "scale"): # packing_immediately
- quantized_layers.append(n)
- summary_info = (
- f"Summary: quantized {len(quantized_layers)}/{len(quantized_layers) + len(unquantized_layers)} in the model"
- )
- if len(unquantized_layers) > 0:
- compressed_unquantized_layers = compress_layer_names(unquantized_layers)
- summary_info += f", unquantized layers: {compressed_unquantized_layers}"
- logger.info(summary_info)
-
- self.quantized = True
- return self.model, self.layer_config
-
- def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None:
- """Quantizes specified layers based on inputs and configuration.
-
- Args:
- layer_names (list): list of layer names to quantize.
- layer_inputs (dict): Dictionary mapping layer names to input data.
-
- Returns:
- None
- """
- # TODO currently we take all the layers outside blocks as post block layers which is not optimal
- # if there is no input for layer, we use rtn
-
- for layer_name in copy.deepcopy(layer_names):
- if layer_name not in layer_inputs:
- if self.act_bits < 16 and not self.act_dynamic:
- if "lm_head" in layer_name:
- logger.warning_once(
- "Static activation quantization for lm_head is not fully supported yet. "
- "If lm_head calibration inputs are missing, activation scale may fall back to unit scale "
- "or quantization may be skipped."
- )
- # Activation quantization requires collected inputs
- msg_prefix = (
- f"Activation max hook for layer '{layer_name}' is unavailable due to "
- f"insufficient collected inputs. "
- )
- if "fp8_e5m2" in self.act_data_type:
- logger.warning(msg_prefix + "Please notes that unit scale is used for this layer.")
- else:
- logger.warning(
- msg_prefix + "Static activation quantization is not supported or ineffective, "
- "Skipping quantization for this layer."
- )
- layer_names.remove(layer_name)
- continue
- logger.info(f"using rtn to quantize {layer_name}")
- from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
-
- layer = get_module(self.model, layer_name)
- layer = layer.to(self.device)
- layer = convert_module_to_hp_if_necessary(layer, self.amp_dtype, self.device)
- set_module(self.model, layer_name, layer)
-
- wrapper_layer = WrapperLinear(
- layer,
- enable_round_tuning=False,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- enable_torch_compile=self.enable_torch_compile,
- device=self.device,
- disable_opt_rtn=self.disable_opt_rtn,
- iters=self.iters,
- )
- new_layer = wrapper_layer.unwrapper({})
- set_module(self.model, layer_name, new_layer)
- layer.cpu()
- layer_names.remove(layer_name)
- if len(layer_names) == 0:
- memory_monitor.update()
- memory_monitor.log_summary()
- return
- q_layer_inputs = None
- enable_quanted_input = self.enable_quanted_input
- has_gguf = False
-
- if hasattr(self, "formats"):
- has_gguf = any(format_.is_gguf() for format_ in self.formats)
- if has_gguf and self.is_immediate_packing:
- enable_quanted_input = False
-
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1 and enable_quanted_input:
- dispatch_model(self.model, self.model.hf_device_map)
-
- if enable_quanted_input:
- logger.info("starting to cache layer inputs for %s, this may be quite slow ", layer_names)
- q_layer_inputs = self.try_cache_inter_data_gpucpu([], self.nsamples, layer_names=layer_names)
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(
- self.model
- ) # self.model.hf_device_map has not been changed
- if not self.is_immediate_saving:
- self.model = mv_module_from_gpu(self.model)
- clear_memory(device_list=self.device_list)
- quant_layer = self._quantize_layer
- for layer_name in layer_names:
- layer_input = layer_inputs[layer_name]
- layer_input = to_device(layer_input, self.cache_device)
- q_layer_input = q_layer_inputs.get(layer_name, None) if q_layer_inputs is not None else None
- q_layer_input = to_device(q_layer_input, self.cache_device)
- quant_layer(layer_name, layer_input, q_layer_input, device=self.device)
- if self.is_immediate_packing:
- self._immediate_pack(layer_name)
-
- if self.is_immediate_saving:
- m = get_module(self.model, layer_name)
- shard_writer(self, m, name=layer_name, is_finalize=False)
- del layer_input
- clear_memory(q_layer_input, device_list=self.device_list)
- memory_monitor.log_summary()
-
- @torch.no_grad()
- def _get_block_outputs(
- self,
- block: torch.nn.Module,
- input_ids: torch.Tensor | list[torch.Tensor],
- input_others: torch.Tensor | dict,
- bs: int,
- device: Union[str, torch.device],
- cache_device: Union[str, torch.device],
- save_output: bool = True,
- ):
- """Compute the output of a given block of the model for a given input.
-
- Args:
- block: The block of the model.
- input_ids: The input tensor containing tokenized input ids.
- input_others: A dictionary containing additional input data.
- bs: The batch size for computing the output.
- device: The device for computation.
- cache_device: The device for storing the output.
- batch_dim: The batch dimension of the output tensor.
-
- Returns:
- The output tensor of the block.
- """
- output = []
- nsamples = len(input_ids)
- for i in range(0, nsamples, bs):
- end_index = min(nsamples, i + bs)
- indices = torch.arange(i, end_index).to(torch.long)
- tmp_input_ids, tmp_input_others = self._sampling_inputs(
- input_ids, input_others, indices, self.seqlen, self.batch_dim, share_cache_keys=self.shared_cache_keys
- )
- tmp_output = self.block_forward(
- block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device
- ).to(cache_device)
- if save_output:
- if self.batch_size == 1:
- output.append(tmp_output)
- else:
- output.extend(list(torch.split(tmp_output, 1, dim=self.batch_dim)))
- if self.low_gpu_mem_usage:
- clear_memory(device_list=self.device_list)
-
- return output
-
- def normalize_decoding_layer_inputs_(self, decoding_layer_inputs: list[tuple[tuple[Any, dict[str, Any]]]]):
- """
- Processes and stores decoding layer inputs for block quantization.
-
- This function iterates through a list of captured decoding layer calls,
- replaying them through a fake decoding layer to extract and store the
- inputs required for the decoding block in `self.inputs`. This effectively
- "normalizes" the inputs by making them accessible in a consistent format
- for subsequent quantization steps.
-
- Args:
- decoding_layer_inputs:
- A list of entries captured by a forward hook on the decoding layer.
- Each element is expected to be a tuple whose first item is
- `(args, kwargs)`, where `args` are the positional arguments and
- `kwargs` are the keyword arguments seen during the original
- forward pass.
-
- The capture hook look like:
-
- def input_capture_hook(module, *args, **kwargs):
- _all_module_input[module._global_name].append((args, kwargs))
- """
- first_block_name = self.quant_block_list[0][0]
-
- class _FakeDecodingLayer(torch.nn.Module):
-
- def forward(self, *args, **kwargs):
- return args, kwargs
-
- fake_layer = _FakeDecodingLayer()
- fake_layer.orig_forward = fake_layer.forward
- fake_layer.forward = partial(self._get_block_forward_func(first_block_name), fake_layer)
-
- self.inputs = {}
- self.last_cache_name = None
- for step_input in decoding_layer_inputs:
- args, kwargs = step_input[0]
- fake_layer(*args, **kwargs)
-
- @torch.no_grad()
- def calib(self, nsamples, bs):
- """Perform calibration for quantization.
-
- This method calibrates the model for quantization by processing a specified
- number of samples from the calibration dataset. It ensures that the data is
- properly formatted and feeds it to the model. If the number of samples processed
- is less than the specified number, it logs a warning. If no samples are processed,
- it logs an error and exits.
- Args:
- nsamples (int): The number of samples to use for calibration.
- bs (int): The number of samples to use for calibration
- """
- from auto_round.calib_dataset import get_dataloader
-
- need_attention_mask = True
- if isinstance(self.dataset, str):
- need_attention_mask = False # all supported datasets does not use pad
- dataset = self.dataset.replace(" ", "") ##remove all whitespaces
-
- # slow here
- self.dataloader = get_dataloader(
- self.tokenizer,
- self.seqlen,
- dataset,
- self.seed,
- bs,
- self.nsamples,
- )
- else:
- self.dataloader = self.dataset
- total_cnt = 0
- if self.dataloader.__class__.__name__ == "BatchEncoding":
- self.dataloader = [self.dataloader.data]
-
- for data in self.dataloader:
- if data.__class__.__name__ == "BatchEncoding":
- data = data.data
- if data is None:
- continue
- if isinstance(data, torch.Tensor):
- input_ids = data.to(self.model.device)
- data_new = input_ids
- elif isinstance(data, str):
- if self.tokenizer is None:
- logger.error("please provide tokenizer for string input")
- exit(-1)
- data = self.tokenizer(data, truncation=True, max_length=self.seqlen, return_tensors="pt").data
- data_new = {}
- for key in data.keys():
- data_new[key] = data[key].to(self.model.device)
- input_ids = data_new["input_ids"]
- elif isinstance(data, tuple) or isinstance(data, list):
- data_new = to_device(data, self.model.device)
- input_ids = data_new[0]
- else:
- data_new = {}
- for key in data.keys():
- data_new[key] = to_device(data[key], self.model.device)
- if key == "images":
- data_new[key] = to_dtype(data_new[key], self.model.dtype)
- input_ids = data_new["input_ids"]
- if input_ids.shape[-1] < self.seqlen:
- continue
- if need_attention_mask:
- if (
- isinstance(data_new, dict)
- and "attention_mask" in data_new
- and data_new["attention_mask"] is not None
- ):
- new_attention_mask = data_new["attention_mask"]
- elif (
- self.tokenizer is not None
- and hasattr(self.tokenizer, "pad_token")
- and self.tokenizer.pad_token is not None
- ):
- new_attention_mask = (input_ids != self.tokenizer.pad_token_id).to(torch.long)
- else:
- # Default all ones
- new_attention_mask = torch.ones_like(input_ids, dtype=torch.long)
-
- # For each sample, check if there are trailing repeated tokens
- # If so, set the mask of the last token to 0
- batch_size, seq_len = input_ids.shape
- for i in range(batch_size):
- last_token = input_ids[i, -1]
- # Check for trailing repeats
- j = seq_len - 2
- repeated = False
- while j >= 0 and input_ids[i, j] == last_token:
- repeated = True
- new_attention_mask[i, j] = 0
- j -= 1
- # If there was at least one repeat, set last token mask to 0
- if repeated:
- new_attention_mask[i, -1] = 0
-
- # Workaround: some models treat an all-1 attention mask as equivalent to None and
- # will internally replace it with None for block inputs, which can cause tensor
- # concatenation / shape-mismatch issues in downstream code. To avoid providing an
- # all-1 mask, we force the last token in each sequence to be masked out (set to 0)
- # so that the mask is never "all ones". This means the model will not attend to the
- # last position, so the impact on accuracy is minimal as basically equivalent to dropping a single token
- new_attention_mask[:, -1] = 0
-
- self.attention_mask.extend(list(torch.split(new_attention_mask, 1, dim=0)))
- else:
- new_attention_mask = None
- try:
- kwargs = {"use_cache": False}
- if new_attention_mask is not None and not (isinstance(data_new, dict) and "attention_mask" in data_new):
- kwargs["attention_mask"] = new_attention_mask
-
- if isinstance(data_new, torch.Tensor):
- self.model(data_new, **kwargs)
- elif isinstance(data_new, tuple) or isinstance(data_new, list):
- self.model(*data_new, **kwargs)
- else:
- self.model(**data_new, **kwargs)
- except NotImplementedError as error:
- error_msg = str(error)
- # Raise NotImplementedError to fallback to CUDA device
- if "flash_attn::" in error_msg and "CPU" in error_msg:
- raise NotImplementedError(
- "Could not run 'flash_attn::_flash_attn_varlen_forward' with arguments from the 'CPU' backend."
- )
- else:
- pass
- except RuntimeError as error:
- error_msg = str(error)
- if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:
- check_seqlen_compatible(self.seqlen, self.tokenizer, self.model)
- logger.warning(
- "When quantization encounters tensor shape mismatch error, "
- "you can try to avoid it with batch_size=1"
- )
- raise error
- except Exception as error:
- raise error
-
- total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1
- if total_cnt >= nsamples:
- break
- if total_cnt == 0:
- logger.error(
- f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
- f"dataset or decease the sequence length"
- )
- exit(-1)
- elif total_cnt < nsamples:
- logger.warning_once(
- f"An insufficient number of samples likely reduces the accuracy of the quantized model. "
- f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"
- )
-
- @torch.no_grad()
- def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):
- """Attempts to cache intermediate data on GPU, if failed, then using CPU.
-
- Args:
- block_names (list): List of block names to cache data for.
- nsamples (int): Number of samples to use for caching.
- layer_names (list, optional): List of layer names to cache data for. Defaults to [].
- last_cache_name (str, optional): Name of the last cache. Defaults to None.
-
- Returns:
- all_inputs: Cached intermediate data.
-
- Raises:
- Exception: If caching on GPU fails, switches to CPU and caches there.
- """
- block_names = flatten_list(block_names)
- if is_quantized_input_module(self.model):
- layer_names = []
- if layer_names is None:
- layer_names = []
- self.blocks_requiring_input_ids = [data if isinstance(data, str) else data[0] for data in block_names]
-
- calibrate_on_cpu = False
- cannot_calibrate_on_cpu = False
- if self.low_gpu_mem_usage or (
- len(block_names) == 1
- and len(layer_names) == 0
- and not self.has_qlayer_outside_block
- and (last_cache_name is None or last_cache_name in block_names)
- and not getattr(self, "mllm", False)
- ):
- # low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU
- calibrate_on_cpu = True
- try:
- all_inputs = self.cache_inter_data(
- block_names, nsamples, layer_names=[], last_cache_name=last_cache_name
- )
- except NotImplementedError as error:
- error_msg = str(error)
- if "flash_attn::" in error_msg and "CPU" in error_msg:
- cannot_calibrate_on_cpu = True # fallback to GPU when flash attention is not supported on CPU
- else:
- raise error
-
- if not calibrate_on_cpu or cannot_calibrate_on_cpu:
- try:
- if any(p.device.type == "meta" for p in self.model.parameters()):
- materialize_model_(self.model)
-
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)
- else:
- # Change this if new device is supported
- if str(self.model.device) == "cpu" and (not self.device.startswith("hpu")):
- # type(self.model._no_split_modules) changes from list to set when transformers > 5.0
- no_split_modules = list(getattr(self.model, "_no_split_modules", []))
- devices = parse_available_devices(self.device_map)
-
- max_memory = get_max_memory()
- new_max_memory = {}
- if "cpu" not in devices:
- devices.append("cpu")
- for device in devices:
- if ":" in device:
- device = int(device.split(":")[-1])
- elif device == "cpu":
- device = "cpu"
- elif isinstance(device, str):
- device = 0
- else:
- raise ValueError(f"Unsupported device {device} in device_map: {self.device_map}")
- if device not in max_memory:
- # Skip devices that are not reported by accelerate's max_memory.
- # This is expected when a device is unavailable or cannot provide memory info.
- continue
- # Use 90% of the reported max memory to leave headroom for activations,
- # temporary tensors, other processes, and allocator fragmentation, reducing
- # the chance of runtime OOM while still utilizing most available memory.
- new_max_memory[device] = max_memory[device] * 0.9
-
- # If non-CPU devices were requested but none survived, fall back to CPU caching
- # via the OOM handler below, avoiding unnecessary dispatch overhead.
- requested_non_cpu = any((d != "cpu") for d in devices)
- has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)
- if requested_non_cpu and not has_non_cpu_memory:
- raise torch.OutOfMemoryError(
- "No non-CPU device available in accelerate's reported memory. "
- "Falling back to CPU caching."
- )
- # Keep ngram_embeddings on CPU
- has_ngram_embeddings, raw_ngram_embeddings = hook_ngram_embeddings_on_cpu(self.model)
- new_max_memory = get_balanced_memory(
- self.model,
- max_memory=new_max_memory,
- no_split_module_classes=no_split_modules,
- )
- if hasattr(self.model, "tie_weights") and callable(self.model.tie_weights):
- self.model.tie_weights()
- device_map = infer_auto_device_map(
- self.model, max_memory=new_max_memory, no_split_module_classes=no_split_modules
- )
- if len(devices) > 1 and "cpu" in device_map.values():
- logger.warning(
- "Some layers are offloaded to cpu, which may severely impact calibration speed."
- " Please consider using more cards."
- )
-
- try:
- self.model = dispatch_model(self.model, device_map=device_map)
- if has_ngram_embeddings:
- self.model.model.ngram_embeddings = raw_ngram_embeddings
- except ValueError as e:
- if "offload_dir" in e.__str__():
- logger.warning(
- f"Due to insufficient resources, disk is used to store the model."
- f" `offload_dir={envs.AR_WORK_SPACE}`"
- )
- self.model = dispatch_model(
- self.model, device_map=device_map, offload_dir=envs.AR_WORK_SPACE
- )
- else:
- raise
- else:
- self.model = self.model.to(self.device)
-
- all_inputs = self.cache_inter_data(
- block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
- )
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- accelerate.hooks.remove_hook_from_submodules(self.model)
-
- except torch.OutOfMemoryError as e:
- if cannot_calibrate_on_cpu:
- raise e
- cuda_error_msg = traceback.format_exc()
- try:
- logger.info("switch to cpu to cache block inputs")
- self.cache_device = torch.device("cpu")
- if self.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":
- logger.warning(
- "we recommend using more GPUs in calibration."
- " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."
- )
- accelerate.hooks.remove_hook_from_submodules(self.model)
- self.model = mv_module_from_gpu(self.model)
- clear_memory(device_list=self.device_list)
- # Important change after v0.51, on cpu, we use rtn mode for layers in layer_names
- all_inputs = self.cache_inter_data(
- block_names, nsamples, layer_names=[], last_cache_name=last_cache_name
- )
- except Exception as e:
- logger.error(cuda_error_msg)
- raise
- return all_inputs
-
- @torch.no_grad()
- def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):
- """Save the inputs of block_name for calibration.
-
- This method temporarily replaces the forward method of the model to capture
- the inputs passing through the specified block. It then calibrates the model
- using a specified number of samples. Finally, it restores the original forward
- method and returns the inputs for the specified block.
- Args:
- block_names (list): The names of the blocks for which inputs are to be saved.
- layer_names (list):The names of the layers for which inputs are to be saved.
- nsamples (int): The number of samples to use for calibration.
- last_cache_name (str, optional): The name of the last layer to be cached,
- we could break the forward in this layer to save time
-
- Returns:
- dict: A dictionary containing the inputs for the specified block.
- """
- if layer_names is None:
- layer_names = []
- self.inputs = {}
- block_names = flatten_list(block_names)
- self.to_cached_layers = block_names + layer_names
-
- tmp_dtype = None # TODO delete this as most model is not fp32 now
- # There is a bug if the block name is not the first block
- if (len(block_names) > 1 or len(layer_names) > 0) and self.low_gpu_mem_usage:
- tmp_dtype = self.model.dtype
- if self.amp:
- if self.model.dtype != self.model.dtype:
- self.model = self.model.to(torch.bfloat16)
- else:
- self.model = self.model.to(torch.float32) # model on cpu
-
- self.last_cache_name = self._infer_last_cache_name(block_names, layer_names, last_cache_name)
- self._cache_target_set = set(self.to_cached_layers)
- self._cache_seen_targets = set()
- calib_bs = self.batch_size
- self.hook_handles = []
- self._replace_forward()
- try:
- self.calib(nsamples, calib_bs)
- finally:
- # Use finally to recover_forward and delattr in case of that
- # self.calib raises NotImplementedError, such as: flash_attn on CPU.
- self._recover_forward()
- for attr in ("last_cache_name", "_cache_target_set", "_cache_seen_targets", "to_cached_layers"):
- if hasattr(self, attr):
- delattr(self, attr)
- res = self.inputs
- if tmp_dtype is not None:
- self.model = self.model.to(tmp_dtype)
-
- return res
-
- def _infer_last_cache_name(self, block_names, layer_names=None, requested_last_cache_name=None):
- """The latest required cache layer for early-stop forward.
-
- If there are multiple cache targets, return ``None`` and let runtime
- hooks stop only after all targets are observed in real forward execution.
- """
- if layer_names is None:
- layer_names = []
-
- if requested_last_cache_name is not None:
- return requested_last_cache_name
-
- cache_targets = list(block_names) + list(layer_names)
- if len(cache_targets) == 1:
- return cache_targets[0]
-
- # return None here to enable the logic in _should_stop_cache_forward
- return None
-
- def _should_stop_cache_forward(self, name: str) -> bool:
- """Determine whether current forward pass can stop after caching `name`."""
- if name == self.last_cache_name:
- return True
-
- if self.last_cache_name is not None:
- return False
-
- if not hasattr(self, "_cache_target_set") or not hasattr(self, "_cache_seen_targets"):
- return False
-
- if name in self._cache_target_set:
- self._cache_seen_targets.add(name)
-
- if not self._cache_target_set.issubset(self._cache_seen_targets):
- return False
-
- # Lock the last cache name after the first full forward pass.
- self.last_cache_name = name
- return True
-
- @torch.no_grad()
- def _get_block_forward_func(self, name: str) -> Callable:
- """Gets the forward function.
-
- Args:
- name (str): The name of the function.
- Returns:
- function: The forward function.
- """
-
- def post_process_cache_data(batch_size, data, data_name):
- """
- Processes store data for batch handling, reshaping if necessary.
-
- Args:
- batch_size (int): The size of the batch.
- data: The data value to store, potentially for caching.
- data_name (str): Name of the data.
-
- Returns:
- Processed data or None
- """
- new_data = data
- if data_name in self.shared_cache_keys:
- return None
- if batch_size <= 1:
- return new_data
- if "alibi" in data_name:
- if isinstance(data, torch.Tensor):
- alibi = data
- alibi = alibi.reshape(batch_size, -1, alibi.shape[1], alibi.shape[2])
- new_data = alibi
- return new_data
-
- def forward(m, hidden_states=None, *positional_inputs, **kwargs):
- """Rewrite forward function, process and collect input data.
-
- Args:
- hidden_states (torch.Tensor): The hidden states tensor.
- *positional_inputs: Variable number of positional arguments.
- **kwargs: Variable number of keyword arguments.
-
- Returns:
- NotImplementedError: Getting the first layer inputs and then raise the error to save runtime.
- """
- if name not in self.inputs:
- self.inputs[name] = {}
- init_cache(positional_inputs, self.inputs[name])
-
- if self.batch_dim is None:
- self.batch_dim = 0
- if hidden_states is not None and self.batch_size > 1:
- if hidden_states.shape[0] > self.batch_size:
- self.batch_dim = 1
- if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.batch_size:
- logger.error(
- "this model has not been supported, "
- "please raise an issue in https://github.com/intel/auto-round/issues"
- " or try to set the `batch_size` to 1 and "
- "`gradient_accumulate_steps` to your current batch size."
- )
- exit(-1)
-
- if hidden_states is not None:
- kwargs["hidden_states"] = hidden_states
-
- for key in kwargs.keys():
- if (
- isinstance(kwargs[key], torch.Tensor)
- or isinstance(kwargs[key], list)
- or isinstance(kwargs[key], tuple)
- ):
- if (
- self.has_variable_block_shape
- and name not in self.blocks_requiring_input_ids
- and key == "hidden_states"
- ):
- continue
- if key not in self.inputs[name].keys(): # initialization
- data = to_device(kwargs[key], device=torch.device("cpu"))
- if data is None or key in self.shared_cache_keys:
- self.inputs[name][key] = data
- continue
- if self.batch_size <= 1:
- self.inputs[name][key] = [data]
- else:
- data = post_process_cache_data(self.batch_size, data, key)
- self.inputs[name][key] = list(torch.split(data, 1, dim=self.batch_dim))
- else: # append cache inputs
- new_data = post_process_cache_data(self.batch_size, kwargs[key], key)
- if new_data is None: # shareable args or NoneType
- continue
- new_data = to_device(new_data, device=torch.device("cpu"))
- if self.batch_size <= 1:
- self.inputs[name][key].append(new_data)
- else:
- self.inputs[name][key].extend(list(torch.split(new_data, 1, dim=self.batch_dim)))
- elif isinstance(kwargs[key], (str, bool, type(None))):
- if key not in self.inputs[name].keys():
- self.inputs[name][key] = kwargs[key]
- else:
- # Parameters not to be cached
- if check_skippable_keywords(key):
- logger.warning_once(
- f"Please note that '{key}' key" " is not currently used in quantization fine-tuning."
- )
- reset_params(self.inputs[name])
-
- if self._should_stop_cache_forward(name):
- raise NotImplementedError
- else:
- if hidden_states is not None:
- kwargs.pop("hidden_states")
- return m.orig_forward(hidden_states, *positional_inputs, **kwargs)
- else:
- # Currently only for Llama-3.2-Vision-Instruct Series
- return m.orig_forward(*positional_inputs, **kwargs)
-
- return forward
-
- @torch.no_grad()
- def _get_cache_data_hook_for_layer(self, name):
- """A forward hook to save input max of a module
- :param name: the module name
- :return: A hook function."""
-
- def cache_input_hook(module, inputs, outputs):
- input = inputs
- if isinstance(inputs, tuple) or isinstance(input, list):
- input = inputs[0]
- if name in self.inputs:
- self.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))
- else:
- self.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))
-
- if self._should_stop_cache_forward(name):
- raise NotImplementedError
-
- return cache_input_hook
-
- def _recover_forward(self):
- """Recovers the forward function."""
- for n, m in self.model.named_modules():
- if hasattr(m, "orig_forward"):
- m.forward = m.orig_forward
- delattr(m, "orig_forward")
- for hook_handle in self.hook_handles:
- hook_handle.remove()
- self.hook_handles = []
-
- def _replace_forward(self):
- """Replaces the forward function."""
- for n, m in self.model.named_modules():
- if n in self.to_cached_layers and type(m) not in self.supported_types: ##block
- m.orig_forward = m.forward
- m.forward = partial(self._get_block_forward_func(n), m)
- elif n in self.to_cached_layers: ##linear layer or conv1d layer
- hook_func = self._get_cache_data_hook_for_layer(n)
- hook_handle = m.register_forward_hook(hook_func)
- self.hook_handles.append(hook_handle)
-
- def _register_act_max_hook(self, model):
-
- def get_act_max_hook(module, input, output):
- if isinstance(input, (tuple, list)):
- input = input[0]
- if input.numel() == 0:
- return # as no needs for act_max update
- input, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)
- act_max = torch.max(torch.abs(input), dim=-1).values
- if not hasattr(module, "act_max") or module.act_max.numel() == 0:
- module.act_max = act_max
- if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage
- max_val = act_max.max()
- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- else:
- act_max = act_max.to(module.act_max.device)
- if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usage
- max_val = torch.max(act_max.max(), module.act_max.max())
- module.act_max = max_val.unsqueeze(0) if max_val.dim() == 0 else max_val
- else:
- module.act_max = torch.max(act_max, module.act_max)
-
- hook_handles = []
- # for single layers out of blocks, like lm_head
- if isinstance(model, SUPPORTED_LAYER_TYPES):
- m = model
- if (
- hasattr(m, "act_dynamic")
- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)
- and check_to_quantized(m)
- ):
- hook = m.register_forward_hook(get_act_max_hook)
- hook_handles.append(hook)
- return hook_handles
-
- for n, m in model.named_modules():
- if (
- hasattr(m, "act_dynamic")
- and check_need_act_calibration(m.act_dynamic, m.act_data_type, m.act_bits)
- and check_to_quantized(m)
- ):
- hook = m.register_forward_hook(get_act_max_hook)
- hook_handles.append(hook)
- continue
-
- # for whole model, RTN
- if n in self.layer_config:
- config = self.layer_config[n]
- act_dynamic = config.get("act_dynamic", True)
- act_data_type = config.get("act_data_type", None)
- act_bits = config.get("act_bits", 16)
- if (
- config["bits"] <= 8
- and check_need_act_calibration(act_dynamic, act_data_type, act_bits)
- and check_to_quantized(config)
- ):
- hook = m.register_forward_hook(get_act_max_hook)
- hook_handles.append(hook)
- continue
- return hook_handles
-
- def _quantize_layer(
- self, layer_name: str, inputs: torch.Tensor, q_inputs: torch.Tensor = None, device: str = "cpu"
- ):
- """Quantize a specific layer of the model using the provided inputs.
-
- Args:
- layer_name (str): The name of the layer to quantize.
- inputs (torch.Tensor): Input data for quantization.
- q_inputs (torch.Tensor, optional): Quantized input data. Defaults to None.
- device (torch.device, optional): The device to use for quantization. Defaults to torch.device("cpu").
-
- Returns:
- None
- """
- logger.info(f"quantizing layer {layer_name}")
- layer = get_module(self.model, layer_name)
- if hasattr(layer, "tuning_device"):
- device = layer.tuning_device
-
- layer = layer.to(device)
- for i in range(len(inputs)):
- inputs[i] = inputs[i].to(layer.weight.dtype)
- if q_inputs is not None:
- q_inputs[i] = q_inputs[i].to(layer.weight.dtype)
-
- if self.act_bits <= 8 and check_need_act_calibration(
- self.act_dynamic,
- self.act_data_type,
- self.act_bits,
- self.static_kv_dtype,
- self.static_attention_dtype,
- ):
- tmp_inputs = q_inputs if q_inputs is not None else inputs
- hook_handles = self._register_act_max_hook(layer)
- with torch.no_grad():
- for input in tmp_inputs:
- layer(input)
- for handle in hook_handles:
- handle.remove()
-
- wrapper_linear = WrapperLinear(
- layer,
- enable_minmax_tuning=self.enable_minmax_tuning,
- enable_torch_compile=self.enable_torch_compile,
- device=device,
- iters=self.iters,
- ).to(device)
- round_params = []
- minmax_params = []
- for key in wrapper_linear.params.keys():
- if "min" in key or "max" in key:
- minmax_params.append(wrapper_linear.params[key])
- else:
- round_params.append(wrapper_linear.value)
- if len(round_params) + len(minmax_params) <= 0:
- dump_info = f"quantized {layer_name}"
- logger.info(dump_info)
- with torch.no_grad():
- unwrapper_layer(self.model, wrapper_linear, layer_name, {})
- mv_module_from_gpu(layer)
-
- lr = torch.tensor(self.lr)
- minmax_lr = torch.tensor(self.minmax_lr)
- if self.enable_minmax_tuning:
- optimizer = self.optimizer(
- [{"params": round_params}, {"params": minmax_params, "lr": minmax_lr}], lr=lr, weight_decay=0
- )
- else:
- optimizer = self.optimizer(round_params, lr=lr, weight_decay=0)
-
- if self.lr_scheduler is None:
- lr_schedule = torch.optim.lr_scheduler.LinearLR(
- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters
- )
- else:
- lr_schedule = copy.deepcopy(self.lr_scheduler)
- nsamples = len(inputs)
- last_best_iter = 0
- best_loss = torch.finfo(torch.float).max
- scaler = self._get_scaler() # pylint: disable=assignment-from-none
- init_loss = None
- gradient_accumulate_steps = self.batch_size # Force to low gpu
-
- total_loss = 0
- num_elm = 1
- mse_reduction = "mean"
- if gradient_accumulate_steps != 1:
- mse_reduction = "sum"
- mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
- batch_size = 1 # Force to low gpu
- global_batch_size = self.batch_size * gradient_accumulate_steps
- global_batch_size = min(nsamples, global_batch_size)
- if gradient_accumulate_steps != 1 and not self.attention_mask:
- whole_indices = torch.arange(global_batch_size)
- if q_inputs is not None:
- num_elm = self._get_current_num_elm(q_inputs, whole_indices)
- else:
- num_elm = self._get_current_num_elm(inputs, whole_indices)
-
- index_sampler = IndexSampler(nsamples, global_batch_size)
-
- for i in range(self.iters):
- total_loss = 0
- global_indices = index_sampler.next_batch()
- if self.attention_mask:
- num_elm = self._get_non_zero_cnt(self.attention_mask, global_indices)
-
- for tmp_step in range(gradient_accumulate_steps):
- indices = global_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]
- if q_inputs is not None:
- current_input = [q_inputs[i] for i in indices]
- current_input = torch.cat(current_input, dim=0).to(device)
- org_input = [inputs[i] for i in indices]
- org_input = torch.cat(org_input, dim=0).to(device)
- else:
- current_input = [inputs[i] for i in indices]
- current_input = torch.cat(current_input, dim=0).to(device)
- org_input = current_input
- with torch.no_grad():
- current_output = layer(org_input)
- autocast_ctx = (
- nullcontext()
- if not self.amp
- else autocast(device_type=str(device).split(":")[0], dtype=self.amp_dtype)
- )
- if self.attention_mask:
- tmp_attention_mask = [self.attention_mask[i] for i in indices]
- tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(device)
- tmp_attention_mask.unsqueeze_(-1)
-
- with autocast_ctx:
- output_q = wrapper_linear(current_input) # pylint: disable=not-callable
- loss = mse_loss( # pylint: disable=not-callable
- (output_q * tmp_attention_mask).to(torch.float32),
- (current_output * tmp_attention_mask).to(torch.float32),
- )
-
- else:
- with autocast_ctx:
- output_q = wrapper_linear(current_input) # pylint: disable=not-callable
- loss = mse_loss( # pylint: disable=not-callable
- output_q.to(torch.float32),
- current_output.to(torch.float32), # mul 1.0 will copy the output
- )
-
- num_elm = 1 if num_elm <= 0 else num_elm
- total_loss += loss.item() / num_elm
-
- self._scale_loss_and_backward(scaler, loss)
- if i == 0:
- init_loss = total_loss
-
- if total_loss < best_loss:
- best_loss = total_loss
- if not self.not_use_best_mse:
- best_params = collect_best_params(wrapper_linear, self.cache_device)
- last_best_iter = i
- if self.not_use_best_mse and i == self.iters - 1:
- best_params = collect_best_params(wrapper_linear, self.cache_device)
-
- if not self.not_use_best_mse:
- if 0 < self.dynamic_max_gap <= i - last_best_iter:
- break
- self._step(scaler, optimizer, lr_schedule)
-
- last_loss = total_loss
- best_iter = self.iters
- if not self.not_use_best_mse:
- last_loss = best_loss
- best_iter = last_best_iter
- with torch.no_grad():
- unwrapper_layer(self.model, wrapper_linear, layer_name, best_params)
- mv_module_from_gpu(layer)
- dump_info = f"quantized {layer_name}, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
- logger.info(dump_info)
-
- def _get_current_output(self, output: list[torch.Tensor], indices: list[int]) -> torch.Tensor:
- current_output = [output[x] for x in indices]
- current_output = torch.cat(current_output, dim=self.batch_dim)
- return current_output
-
- def _get_current_q_output(
- self,
- block: torch.nn.Module,
- input_ids: list[torch.Tensor],
- input_others: dict,
- indices: list[int],
- device: str,
- cache_device: str = "cpu",
- ) -> torch.Tensor:
- current_input_ids, current_input_others = self._sampling_inputs(
- input_ids,
- input_others,
- indices,
- seqlen=self.seqlen,
- batch_dim=self.batch_dim,
- share_cache_keys=self.shared_cache_keys,
- )
- output_q = self.block_forward(block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device)
- return output_q.to(cache_device)
-
- def _get_current_num_elm(
- self,
- input_ids: list[torch.Tensor],
- indices: list[int],
- ) -> int:
- current_input_ids = [input_ids[i] for i in indices]
- return sum(id.numel() for id in current_input_ids)
-
- def _get_non_zero_cnt(self, tensor: list[torch.Tensor], indices: list[int]) -> int:
- current_tensors = [tensor[i] for i in indices]
- non_zero_cnt = 0
- for t in current_tensors:
- non_zero_cnt += torch.count_nonzero(t).item()
- return non_zero_cnt
-
- def quantize_block(
- self,
- block: torch.nn.Module,
- inputs: tuple[Union[list[torch.Tensor], dict, Any], Optional[dict]],
- q_input: Union[torch.Tensor, dict, None] = None,
- device: Union[str, torch.device] = "cpu",
- auto_offload=True,
- ):
- """
- This function quantizes a specific decoded block of a model.
- It is primarily used by LLM-Compressor. For more details, please refer to the following PR:
- https://github.com/vllm-project/llm-compressor/pull/1994
- """
-
- # TODO: release below assertion after supporting MLLM and diffusion model quantization with quantize_block
- assert self.__class__.__name__ not in [
- "DiffusionCompressor",
- "MLLMCompressor",
- ], f"Currently, {self.__class__.__name__} does not support support quantize block with this function."
- self.normalize_decoding_layer_inputs_(inputs)
- block_inputs = self.inputs[self.quant_block_list[0][0]]
- decoding_layer_first_input_name = "hidden_states"
- input_ids, input_others = self._preprocess_block_inputs(block_inputs, decoding_layer_first_input_name)
- return self._quantize_block(block, input_ids, input_others, q_input, device, auto_offload)
-
- def _get_loss(
- self,
- output_q: torch.Tensor,
- current_output: torch.Tensor,
- indices: torch.Tensor,
- mse_loss: Callable,
- device: Union[str, torch.device] = "cpu",
- ):
- autocast_ctx = (
- nullcontext() if not self.amp else autocast(device_type=str(device).split(":")[0], dtype=self.amp_dtype)
- )
- if self.attention_mask:
- tmp_attention_mask = [self.attention_mask[i] for i in indices]
- tmp_attention_mask = torch.cat(tmp_attention_mask, dim=0).to(device)
- tmp_attention_mask.unsqueeze_(-1)
-
- with autocast_ctx:
- loss = mse_loss( # pylint: disable=not-callable
- (output_q * tmp_attention_mask).to(torch.float32),
- (current_output * tmp_attention_mask).to(torch.float32),
- )
- else:
- with autocast_ctx:
- loss = mse_loss( # pylint: disable=not-callable
- output_q.to(torch.float32), current_output.to(torch.float32)
- )
-
- return loss
-
- def _quantize_block(
- self,
- block: torch.nn.Module,
- input_ids: Union[list[torch.Tensor], dict],
- input_others: dict,
- q_input: Union[torch.Tensor, dict, None] = None,
- device: Union[str, torch.device] = "cpu",
- auto_offload=True,
- ):
- """Quantize the weights of a given block of the model.
-
- Args:
- block: The block of the model to be quantized.
- input_ids: The input tensor containing tokenized input ids.
- input_others: A dictionary containing additional input data.
- q_input: The quantized input tensor.
- device: The device for quantization.
-
- Returns:
- Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
- """
- materialize_model_(block)
- convert_module_to_hp_if_necessary(block, self.amp_dtype, device)
-
- if auto_offload:
- # card_0_in_high_risk indicates that card_0 memory is already in high usage (90%) w/o any weights
- # loss_device is used to calculate loss on the second device if available and card_0_in_high_risk
- if (
- is_auto_device_mapping(self.device_map)
- and len(self.device_list) > 1
- and not getattr(self, "is_diffusion", False)
- ):
- card_0_in_high_risk, loss_device = set_auto_device_map_for_block_with_tuning(
- block, self.device_map, input_ids, self.low_gpu_mem_usage, self.batch_size, device
- )
- else:
- block = block.to(device)
- card_0_in_high_risk, loss_device = False, device
- else:
- card_0_in_high_risk, loss_device = False, device
-
- if len(self.device_list) > 1 and auto_offload:
- for n, m in block.named_modules():
- if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
- continue
- from accelerate.hooks import AlignDevicesHook, add_hook_to_module
-
- hook = AlignDevicesHook(m.tuning_device, io_same_device=True)
- add_hook_to_module(m, hook, True)
-
- if q_input is None:
- hook_handles = self._register_act_max_hook(block)
-
- output = self._get_block_outputs(
- block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device
- )
-
- for handle in hook_handles:
- handle.remove()
- else:
- output = self._get_block_outputs(
- block, input_ids, input_others, self.batch_size * self.infer_bs_coeff, device, self.cache_device
- )
- hook_handles = self._register_act_max_hook(block)
- if hook_handles:
- self._get_block_outputs(
- block,
- q_input if q_input is not None else input_ids,
- input_others,
- self.batch_size * self.infer_bs_coeff,
- device,
- self.cache_device,
- save_output=False,
- )
-
- for handle in hook_handles:
- handle.remove()
-
- if q_input is not None:
- if input_ids is not q_input:
- clear_memory(input_ids, device_list=self.device_list)
- else:
- clear_memory(device_list=self.device_list)
- input_ids = q_input
-
- quantized_layer_names, unquantized_layer_names = self.wrapper_block(
- block,
- self.enable_minmax_tuning,
- self.enable_norm_bias_tuning,
- enable_torch_compile=self.enable_torch_compile,
- device=device,
- iters=self.iters,
- )
- # Call this before quantization and after applying the block wrapper.
- if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse.
- from auto_round.data_type.utils import update_fused_layer_global_scales
-
- modules = block.modules()
- for module in modules:
- update_fused_layer_global_scales(module)
- round_params = []
- minmax_params = []
- for n, m in block.named_modules():
- if hasattr(m, "orig_layer"):
- for key in m.params.keys():
- if "min" in key or "max" in key:
- minmax_params.append(m.params[key])
- else:
- round_params.append(m.params[key])
-
- lr = torch.tensor(self.lr)
- minmax_lr = torch.tensor(self.minmax_lr)
- is_adam = "adam" in self.__class__.__name__.lower()
-
- extra_kwargs = {} if is_adam else {"momentum": self.momentum}
-
- if self.enable_minmax_tuning:
- params = [
- {"params": round_params},
- {"params": minmax_params, "lr": minmax_lr},
- ]
- else:
- params = round_params
-
- optimizer = self.optimizer(
- params,
- lr=lr,
- weight_decay=0,
- **extra_kwargs,
- )
-
- if len(round_params) + len(minmax_params) <= 0:
- dump_info = (
- f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
- f"layers in the block"
- )
- logger.info(dump_info)
- unwrapper_block(block, {})
- mv_module_from_gpu(block)
- return output, output
-
- if self.lr_scheduler is None:
- lr_schedule = torch.optim.lr_scheduler.LinearLR(
- optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters
- )
- else:
- lr_schedule = copy.deepcopy(self.lr_scheduler)
-
- if isinstance(input_ids, dict): # input_ids of Flux is dict
- nsamples = len(input_ids["hidden_states"])
- else:
- nsamples = len(input_ids)
- last_best_iter = 0
- best_loss = torch.finfo(torch.float).max
- num_elm = 1
- mse_reduction = "mean"
- if self.gradient_accumulate_steps != 1:
- mse_reduction = "sum"
- mse_loss = torch.nn.MSELoss(reduction=mse_reduction).to(device)
- scaler = self._get_scaler() # pylint: disable=assignment-from-none
- init_loss = None
- best_params = {}
- total_loss = 0
- global_batch_size = self.batch_size * self.gradient_accumulate_steps
- global_batch_size = min(nsamples, global_batch_size)
- # We assume the block input and output shape is same
- if self.gradient_accumulate_steps != 1 and not self.attention_mask:
- whole_indices = torch.arange(global_batch_size)
- num_elm = self._get_current_num_elm(input_ids, whole_indices)
- setup_ddp_if_needed_(self, block, self.device_list)
- index_sampler = IndexSampler(nsamples, global_batch_size)
- batch_size = self.batch_size
- for i in range(self.iters):
- if self.enable_alg_ext and self.data_type.endswith("dq"):
- for n, m in block.named_modules():
- m.cur_iter = i
- total_loss = 0
- global_indices = index_sampler.next_batch()
- if self.attention_mask:
- num_elm = self._get_non_zero_cnt(self.attention_mask, global_indices)
-
- for tmp_step in range(self.gradient_accumulate_steps):
- indices = global_indices[tmp_step * batch_size : (tmp_step + 1) * batch_size]
- current_output = self._get_current_output(output, indices)
- current_output = to_device(current_output, loss_device)
- output_q = self._get_current_q_output(block, input_ids, input_others, indices, device, loss_device)
- loss = self._get_loss(output_q, current_output, indices, mse_loss, device)
- num_elm = 1 if num_elm <= 0 else num_elm
- total_loss += loss.item() / num_elm
-
- if self.low_gpu_mem_usage and card_0_in_high_risk:
- # clear memory to avoid OOM due to memory fragmentation
- clear_memory_if_reached_threshold(threshold=0.5, device_list=self.device_list)
-
- self._scale_loss_and_backward(scaler, loss)
-
- if self.low_gpu_mem_usage and card_0_in_high_risk:
- # clear memory to avoid OOM due to memory fragmentation
- clear_memory_if_reached_threshold(threshold=0.8, device_list=self.device_list)
-
- if i == 0:
- init_loss = total_loss
-
- if total_loss < best_loss:
- best_loss = total_loss
- if not self.not_use_best_mse:
- best_params = collect_best_params(block, self.cache_device)
- # print(f"get better result at iter {i}, the loss is {total_loss}", flush=True)
-
- last_best_iter = i
- if self.not_use_best_mse and i == self.iters - 1:
- best_params = collect_best_params(block, self.cache_device)
-
- if not self.not_use_best_mse:
- if 0 < self.dynamic_max_gap <= i - last_best_iter:
- break
- self._step(scaler, optimizer, lr_schedule)
-
- last_loss = total_loss
- best_iter = self.iters
- if not self.not_use_best_mse:
- last_loss = best_loss
- best_iter = last_best_iter
- if self.iters > 0:
- dump_info = (
- f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
- f"layers in the block, loss iter 0: {init_loss:.6f} -> iter {best_iter}: {last_loss:.6f}"
- )
- else:
- dump_info = (
- f"quantized {len(quantized_layer_names)}/{(len(quantized_layer_names) + len(unquantized_layer_names))} "
- "layers in the block"
- )
-
- if self.low_gpu_mem_usage:
- clear_memory(device_list=self.device_list) # clear cached memory during training
- if len(unquantized_layer_names) != 0:
- logger.info(f"Unquantized layers: {unquantized_layer_names}")
- with torch.no_grad():
- unwrapper_block(block, best_params)
-
- if is_nv_fp(self.act_data_type):
- # enable moe experts act_max automatic generation for WrapperWALayer
- set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")
-
- if self.enable_quanted_input:
- q_outputs = self._get_block_outputs(
- block,
- input_ids,
- input_others,
- self.batch_size * self.infer_bs_coeff,
- device,
- cache_device=self.cache_device,
- )
-
- if len(self.device_list) > 1 and auto_offload:
- accelerate.hooks.remove_hook_from_submodules(block)
- if auto_offload:
- mv_module_from_gpu(block)
-
- clear_memory(input_ids, device_list=self.device_list)
- memory_info_summary = memory_monitor.get_summary()
- logger.infoclean(dump_info + "," + memory_info_summary)
-
- return q_outputs, output
- else:
- if len(self.device_list) > 1 and auto_offload:
- accelerate.hooks.remove_hook_from_submodules(block)
- if auto_offload:
- mv_module_from_gpu(block)
- clear_memory(input_ids, device_list=self.device_list)
- memory_info_summary = memory_monitor.get_summary()
- logger.infoclean(dump_info + "," + memory_info_summary)
-
- return None, output
-
- def _split_inputs(self, inputs: dict, first_input_name: str) -> tuple[torch.Tensor, dict]:
- input_ids = inputs.get(first_input_name, None)
- inputs.pop(first_input_name, None)
- input_others = inputs
- return input_ids, input_others
-
- def _preprocess_block_inputs(self, inputs, first_input_name="input_ids"):
- input_ids, input_others = self._split_inputs(inputs, first_input_name)
- clear_memory(device_list=self.device_list)
- tmp_dtype = self.amp_dtype if self.amp else torch.float32
- if input_ids is not None:
- input_ids = to_device(input_ids, self.cache_device)
- input_ids = to_dtype(input_ids, tmp_dtype)
- input_others = to_device(input_others, self.cache_device)
- # As in calibration phase, we may use bf16 for calibration due to low_gpu_memory usage
-
- for key in input_others.keys():
- if isinstance(input_others[key], torch.Tensor) and (
- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
- ):
- input_others[key] = input_others[key].to(tmp_dtype)
- elif isinstance(input_others[key], list):
- for i in range(len(input_others[key])):
- to_dtype(input_others[key][i], tmp_dtype)
- return input_ids, input_others
-
- def _quantize_blocks(
- self,
- model: torch.nn.Module,
- inputs: dict,
- block_names: list,
- q_input: torch.Tensor = None,
- nblocks: int = 1,
- device: str = "cpu",
- pbar: tqdm = None,
- input_others_extra_blocks: dict = None,
- ):
- """Quantize and dequantize the weights of the specified blocks in the model.
-
- Args:
- model: The PyTorch model to be quantized.
- inputs: The input data for quantization.
- block_names: The names of the blocks to be quantized and dequantized.
- nblocks: The number of blocks to quantize and dequantize.
- device: The device for quantization and dequantization.
-
- Returns:
- None
- """
- clear_memory(device_list=self.device_list)
- for n, m in model.named_parameters():
- m.requires_grad_(False)
-
- input_ids, input_others = self._preprocess_block_inputs(inputs)
-
- if pbar is None:
- pbar = tqdm(range(0, len(block_names), nblocks))
-
- for i in range(0, len(block_names), nblocks):
- if input_others_extra_blocks and block_names[i] in input_others_extra_blocks:
- input_others = input_others_extra_blocks[block_names[i]]
- _, input_others = self._preprocess_block_inputs(input_others)
- input_others_extra_blocks.pop(block_names[i])
- if i != 0:
- pbar.update(1)
- if nblocks == 1:
- n = block_names[i]
- pbar.set_description(f"Quantizing {n}")
- m = get_module(model, n)
-
- else:
- names = block_names[i : min(i + nblocks, len(block_names))]
- pbar.set_description(f"Quantizing [{i + 1}-{min(i + nblocks, len(block_names))}]/{len(block_names)}")
- modules = [get_module(model, n) for n in names]
- m = WrapperMultiblock(modules)
-
- if self.low_cpu_mem_usage:
- if nblocks == 1:
- self._offloader.reload(model, n)
- else:
- self._offloader.reload(model, names)
-
- m.config = model.config if hasattr(model, "config") else None
- q_input, input_ids = self._quantize_block(
- m,
- input_ids,
- input_others,
- q_input=q_input,
- device=device,
- )
- if hasattr(model, "config"):
- del m.config
- if self.enable_torch_compile:
- torch._dynamo.reset()
- if self.is_immediate_packing:
- for n, tmp_m in m.named_modules():
- if not (hasattr(tmp_m, "bits") and check_to_quantized(tmp_m)):
- continue
- self._immediate_pack(tmp_m.global_name)
-
- if self.is_immediate_saving:
- shard_writer(self, m, is_finalize=False)
-
- if self.low_cpu_mem_usage and not self.is_immediate_saving:
- if nblocks == 1:
- self._offloader(model, n, overwrite=True)
- else:
- for name in names:
- self._offloader(model, name, overwrite=True)
- if pbar is not None:
- pbar.update(1)
-
- if not self.is_immediate_saving:
- self.model = mv_module_from_gpu(self.model)
- for n, m in self.model.named_modules():
- if hasattr(m, "name"):
- delattr(m, "name")
-
- del q_input
- del input_ids
- del input_others
- del inputs
-
- clear_memory(device_list=self.device_list)
+ raise NotImplementedError("quantize method must be implemented in subclass")
def save_quantized(
self,
output_dir: str = None,
- format: Union[str, list[OutputFormat]] = "auto_round",
+ format: Union[str, list[OutputFormat]] = None,
inplace: bool = True,
return_folders=False,
**kwargs,
@@ -3578,31 +1257,44 @@ class BaseCompressor(object):
Returns:
object: The compressed model object.
"""
- self.orig_output_dir = output_dir
- if isinstance(format, str) and getattr(self, "formats", None) is None:
- formats = get_formats(format, self)
- if not hasattr(self, "formats"):
- self.formats = formats
+ self.output_dir = output_dir
+ if output_dir is not None:
+ self.compress_context.output_dir = output_dir
+ if format is not None:
+ if isinstance(format, str) and getattr(self, "formats", None) is None:
+ logger.warning(
+ f"save_quantized with format is deprecated and will be deleted in auto_round version 1.0."
+ f" Please use AutoRound(format='{format}' instead)."
+ )
+ self.formats = get_formats(format, self)
+ self.compress_context.formats = self.formats
- if not self.quantized:
+ if not self.model_context.quantized:
logger.warning("please run autoround.quantize first")
return
folders = []
+ if self.formats is None:
+ logger.info("format is not set, using default auto_round format.")
+ self.formats = "auto_round"
+ if isinstance(self.formats, str):
+ self.formats = get_formats(self.formats, self)
+ self.compress_context.formats = self.formats
for format in self.formats:
- save_folder = self._get_save_folder_name(format)
+ save_folder = _get_save_folder_name(format)
if self.act_bits <= 8 and format.is_fake():
logger.warning(
"Support for exporting activation quantization is limited. "
"Please ensure that your configuration is supported."
)
- serialization_dict = {}
- for key in SERIALIZATION_KEYS:
- serialization_dict[key] = getattr(self, key)
-
+ serialization_dict = asdict(SerializedCompressorConfig())
+ for key in serialization_dict:
+ serialization_dict[key] = getattr(self, key, serialization_dict[key])
from auto_round.version import __version__
serialization_dict["autoround_version"] = __version__
+ if serialization_dict.get("to_quant_block_names") is None and self.quantizer.quant_block_list:
+ serialization_dict["to_quant_block_names"] = extract_block_names_to_str(self.quantizer.quant_block_list)
if "scale_dtype" in serialization_dict.keys():
serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])
@@ -3622,11 +1314,11 @@ class BaseCompressor(object):
compressed_model = format.save_quantized(
save_folder,
- model=self.model,
- layer_config=self.layer_config,
+ model=self.model_context.model,
+ layer_config=self.quantizer.layer_config,
inplace=inplace,
- tokenizer=self.tokenizer,
- device=self.device,
+ tokenizer=self.model_context.tokenizer,
+ device=self.compress_context.device,
serialization_dict=serialization_dict,
**kwargs,
)
@@ -3639,157 +1331,144 @@ class BaseCompressor(object):
else:
return compressed_model
- def _get_quantized_layer_names_outside_blocks(self) -> list:
- """Gets the names of quantized layers outside blocks in the model.
+ def _get_export_dir(self, output_dir: str, format_str: str) -> str:
+ """Derive a descriptive export directory from model name and quantization config.
- Returns:
- list: List of layer names outside blocks.
- """
- if self.layer_config is None or len(self.layer_config) == 0:
- return []
+ Must be called after ``post_init()`` so that scheme-resolved attrs
+ (bits, group_size, data_type, etc.) are available on ``self.quantize_config``.
- layer_names = []
- all_layers_in_block = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
-
- for key in self.layer_config.keys():
- if key in all_layers_in_block:
- continue
- layer = get_module(self.model, key)
- if layer is None:
- logger.warning_once(f"could not find layer {key} in the model, skipping")
- continue
- if type(layer) in self.supported_types and check_to_quantized(self.layer_config[key]):
- layer_names.append(key)
-
- return layer_names
-
- def _set_amp_dtype(self) -> None:
- """Sets the automatic mixed precision (AMP) data type for the model based on the device and configuration."""
- self.amp_dtype = torch.bfloat16
- if self.model.dtype != torch.float32:
- self.amp_dtype = self.model.dtype
- if self.device == "cpu" or "hpu" in self.device:
- self.amp_dtype = torch.bfloat16
- if self.amp:
- if self.device == "cpu" and not CpuInfo().bf16:
- self.amp = False
- self.amp_dtype = torch.float32
- self.model = self.model.to(torch.float32)
- logger.warning(
- f"amp is set to FALSE as the current {self.device} device does not support the 'bf16' data type."
- )
+ Mirrors the logic previously in ``__main__.py`` so callers only need to
+ pass the base ``output_dir`` and the format string.
+ """
+ # Diffusion models use save_quantized from DiffusionMixin which manages its own
+ # directory layout (model_index.json + per-component subdirs). Appending a
+ # scheme-derived suffix here would place files one level too deep.
+ if getattr(self, "diffusion", False):
+ return output_dir
+
+ model_name = (getattr(self.model_context.model, "name_or_path", "") or "").rstrip("/")
+ cfg = self.quantize_config
+ group_size = cfg.group_size
+ bits = cfg.bits
+ data_type = cfg.data_type or "int"
+ act_bits = cfg.act_bits or 16
+ act_data_type = cfg.act_data_type or "float"
+
+ is_gguf = "gguf" in (format_str or "")
+ last = model_name.split("/")[-1].strip(".")
+
+ if last == "" and not is_gguf:
+ # model path is just '.' or './' – put inside output_dir with suffix
+ if group_size <= 0:
+ suffix = f"afp{act_bits}" if "fp" in act_data_type else f"a{act_bits}"
else:
- if self.model.dtype != self.amp_dtype:
- self.model = self.model.to(self.amp_dtype)
- else:
- self.amp_dtype = torch.float32
- self.model = self.model.to(torch.float32)
+ suffix = f"g{group_size}"
+ return os.path.join(output_dir, f"w{bits}{suffix}")
- def _get_optimizer(self, optimizer: Any):
- """Returns the specified optimizer. In SignRound, we fix the optimizer.
+ if last == "" and is_gguf:
+ return output_dir
- Args:
- optimizer: The optimizer to be used.
+ if is_gguf:
+ return os.path.join(output_dir, model_name.split("/")[-1] + "-gguf")
- Returns:
- The specified optimizer.
- """
- return SignSGD
+ # Normal case: derive suffix from group_size / act config
+ if isinstance(group_size, tuple):
+ assert len(group_size) == 2, f"Only support 2D group_size, but got {group_size}"
+ suffix = f"g{group_size[0]}x{group_size[1]}"
+ elif group_size <= 0:
+ suffix = f"afp{act_bits}" if "fp" in act_data_type else f"a{act_bits}"
+ else:
+ suffix = f"g{group_size}"
- def _get_scaler(self):
- """Returns scaler, in SignRound, no need to use scaler."""
- return None
+ prefix = data_type.lower().replace("_", "") if "int" not in data_type or "mx" in data_type else ""
+ return os.path.join(
+ output_dir,
+ model_name.split("/")[-1] + (f"-{prefix}" if prefix else "") + f"-w{bits}{suffix}",
+ )
- def _scale_loss_and_backward(self, scaler: Any, loss: torch.Tensor) -> torch.Tensor:
- """Scales the loss and performs backward pass.
+ def quantize_and_save(
+ self, output_dir: str = "tmp_autoround", format: str = None, inplace: bool = True, **kwargs
+ ) -> tuple[torch.nn.Module, dict[str, Any]]:
+ """Quantizes the model and saves it in the specified format(s).
+
+ This function checks the validity of the requested format(s), quantizes
+ the model accordingly, and saves it to the specified output directory.
+ If multiple formats are provided, the model is saved separately for each format.
Args:
- scaler: The scaler to be used.
- loss: The loss to be scaled.
+ output_dir (str, optional): The directory where the quantized model
+ will be saved. Defaults to "tmp_autoround".
+ format (str, optional): The quantization format(s) to use, separated
+ by commas if multiple. Defaults to "auto_round".
+ inplace (bool, optional): Whether to modify the model in place if only
+ one format is used. Defaults to True.
+ **kwargs: Additional arguments for the quantization and saving process.
Returns:
- The scaled loss.
+ model: A qdq model or packed model based on the configurations
+ folders: The folder paths where the quantized models are saved.
+
+ Raises:
+ ValueError: If an unsupported format is specified.
"""
- scale_loss = loss * 1000
- scale_loss.backward()
- if is_hpex_available():
- htcore.mark_step()
- return scale_loss
+ # Validate and process the specified formats
+ self.output_dir = output_dir
+ self.compress_context.output_dir = output_dir
- def _step(self, scaler: Any, optimizer: Any, lr_schedule: Any):
- """Performs a step in the optimization process.
+ # check and update the format based on the current configuration
+ if format and self.formats is None:
+ logger.warning(
+ f"quantize_and_save with format is deprecated and will be deleted in auto_round version 1.0."
+ f" Please use AutoRound(format='{format}' instead)."
+ )
+ self.formats = format
+ if self.formats is None:
+ logger.info("format is not set, using default auto_round format.")
+ self.formats = "auto_round"
- Args:
- scaler: The scaler to be used.
- optimizer: The optimizer for the step.
- lr_schedule: The learning rate schedule.
+ # If multiple formats are specified, enforce inplace=False
+ if len(self.formats.split(",")) > 1:
+ inplace = False
+ self.inplace = kwargs.get("inplace", inplace)
+ kwargs.pop("inplace", None)
- Returns:
- None
- """
- optimizer.step()
- # for hpu
- if is_hpex_available():
- htcore.mark_step()
- optimizer.zero_grad()
- lr_schedule.step()
-
- @classmethod
- @torch.no_grad()
- def _sampling_inputs(
- cls,
- input_ids: Union[list[torch.Tensor], dict],
- input_others: dict,
- indices: list[int] | torch.Tensor,
- seqlen: int,
- batch_dim: int = 0,
- share_cache_keys: tuple = (),
- ):
- """Samples inputs based on the given indices and sequence length.
+ # Perform model quantization
+ # IMPORTANT: post_init() must run outside any @torch.inference_mode() context
+ # because AutoScheme's delta-loss selection requires gradient tracking.
+ self.post_init()
+ # If post_init() was called manually before quantize_and_save() (e.g. ar.post_init()
+ # in tests), _resolve_formats saw formats=None and was a no-op. Now that we have set
+ # self.formats to a default string above, resolve it into OutputFormat objects so that
+ # quantize() and save_quantized() receive proper objects, not a raw string.
+ if isinstance(self.formats, str):
+ self.formats = get_formats(self.formats, self)
+ self.compress_context.formats = self.formats
+ # Derive descriptive export dir after post_init so scheme-resolved attrs are available.
+ _fmt_str = format or (self.formats if isinstance(self.formats, str) else "")
+ output_dir = self._get_export_dir(output_dir, _fmt_str)
+ self.output_dir = output_dir
+ self.compress_context.output_dir = output_dir
+ if self.static_attention_dtype is not None:
+ from auto_round.experimental.attention import attention_quant_ctx
- Args:
- input_ids: The list of input tensor containing input_ids.
- input_others: A dictionary containing other input data.
- indices: The indices to sample from the input.
- seqlen: The sequence length.
+ with attention_quant_ctx(self.model_context.model, static_attention_dtype=self.static_attention_dtype):
+ self.quantize()
+ self.model_context.quantized = True
+ elif self.static_kv_dtype is not None:
+ from auto_round.experimental.kv_cache import kvcache_quant_context
- Returns:
- current_input_ids: The sampled input IDs.
- current_input_others: The sampled other input data.
- """
- if isinstance(input_ids, list):
- current_input_ids = [input_ids[i] for i in indices]
- current_input_ids = torch.cat(current_input_ids, dim=batch_dim)
- elif isinstance(input_ids, dict):
- current_input_ids = defaultdict(list)
- for k in input_ids.keys():
- current_input_ids[k].extend([input_ids[k][i] for i in indices])
- current_input_ids[k] = torch.cat(current_input_ids[k], dim=batch_dim)
-
- current_input_others = {"positional_inputs": input_others["positional_inputs"]}
- for key in input_others.keys():
- if "positional_inputs" in key:
- continue
- # Shared cache keys (e.g. position_embeddings, position_ids, cache_position) are stored
- # directly as-is (not wrapped in a per-sample list) when batch_size > 1. Indexing such
- # values by sample index would incorrectly decompose them (e.g. (cos, sin)[0] == cos).
- # Always pass them through unchanged.
- if key in share_cache_keys or isinstance(input_others[key], (str, bool, type(None))):
- current_input_others[key] = input_others[key]
- elif input_others[key] is not None:
- current_input_others[key] = [input_others[key][i] for i in indices]
- if len(indices) == 1:
- current_input_others[key] = current_input_others[key][0]
- else:
- try:
- current_input_others[key] = torch.cat(current_input_others[key], dim=0)
- except TypeError as err:
- logger.warning_once("Please check the model cache inputs or try setting batch_size to 1.")
- else:
- current_input_others[key] = None
+ with kvcache_quant_context(self.model_context.model, static_kv_dtype=self.static_kv_dtype):
+ self.quantize()
+ self.model_context.quantized = True
+ else:
+ self.quantize()
+ self.model_context.quantized = True
- return current_input_ids, current_input_others
+ # Ensure ShardWriter is ready before saving (deferred from post_init).
+ self._ensure_shard_writer()
+ # Save the quantized model in the specified format_list
+ model, folders = self.save_quantized(output_dir, inplace=inplace, return_folders=True, **kwargs)
+ memory_monitor.log_summary()
-class LLMCompressor(BaseCompressor):
- pass
+ return model, folders
diff --git a/auto_round/compressors_new/calib.py b/auto_round/compressors/data_driven.py
similarity index 54%
rename from auto_round/compressors_new/calib.py
rename to auto_round/compressors/data_driven.py
index 4041f9379..829408e80 100644
--- a/auto_round/compressors_new/calib.py
+++ b/auto_round/compressors/data_driven.py
@@ -30,8 +30,8 @@ from auto_round.calibration.utils import (
_infer_last_cache_name,
_update_inputs,
)
-from auto_round.compressors_new.base import BaseCompressor
-from auto_round.compressors_new.utils import (
+from auto_round.compressors.base import BaseCompressor
+from auto_round.compressors.utils import (
_get_quantized_layer_names_outside_blocks,
check_skippable_keywords,
immediate_pack,
@@ -70,7 +70,7 @@ from auto_round.utils.device import (
from auto_round.wrapper import WrapperLinear, WrapperMultiblock
-class CalibCompressor(BaseCompressor):
+class DataDrivenCompressor(BaseCompressor):
need_calib: bool = True
def __init__(
@@ -89,7 +89,6 @@ class CalibCompressor(BaseCompressor):
low_cpu_mem_usage: bool = True,
**kwargs,
):
- self.dataset = dataset
self.iters = iters
super().__init__(
config=config,
@@ -104,623 +103,125 @@ class CalibCompressor(BaseCompressor):
low_cpu_mem_usage=low_cpu_mem_usage,
**kwargs,
)
+ # Routed to ``self._calibration_state.dataset`` via @property.
+ # Set after ``super().__init__()`` because the state object is created there.
+ self.dataset = dataset
if iters == 0:
self.lr = 5e-3
- @torch.no_grad()
- def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):
- """Attempts to cache intermediate data on GPU, if failed, then using CPU.
-
- Args:
- block_names (list): List of block names to cache data for.
- nsamples (int): Number of samples to use for caching.
- layer_names (list, optional): List of layer names to cache data for. Defaults to [].
- last_cache_name (str, optional): Name of the last cache. Defaults to None.
+ def post_init(self) -> None:
+ """Run base post-init then attach the registered calibrator strategy.
- Returns:
- all_inputs: Cached intermediate data.
-
- Raises:
- Exception: If caching on GPU fails, switches to CPU and caches there.
+ Subclasses (MLLM/Diffusion) override ``calib`` directly on the
+ Compressor; the calibrator owns ``try_cache_inter_data_gpucpu`` /
+ ``cache_inter_data`` orchestration plus the LLM ``calib`` body.
"""
- if is_quantized_input_module(self.model_context.model):
- layer_names = []
- if layer_names is None:
- layer_names = []
-
- block_names = flatten_list(block_names)
- self.blocks_requiring_input_ids = [data if isinstance(data, str) else data[0] for data in block_names]
-
- calibrate_on_cpu = False
- cannot_calibrate_on_cpu = False
- if self.compress_context.low_gpu_mem_usage or (
- len(block_names) == 1
- and len(layer_names) == 0
- and not self.quantizer.has_qlayer_outside_block
- and (last_cache_name is None or last_cache_name in block_names)
- and not getattr(self, "mllm", False)
- ):
- # low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU
- calibrate_on_cpu = True
- try:
- all_inputs = self.cache_inter_data(
- block_names, nsamples, layer_names=[], last_cache_name=last_cache_name
- )
- except NotImplementedError as error:
- error_msg = str(error)
- if "flash_attn::" in error_msg and "CPU" in error_msg:
- cannot_calibrate_on_cpu = True # fallback to GPU when flash attention is not supported on CPU
- else:
- raise error
+ if self._post_init_done:
+ return
+ super().post_init()
+ if self.calibration is None:
+ from auto_round.calibration import get_calibrator
- if not calibrate_on_cpu or cannot_calibrate_on_cpu:
- try:
- if any(p.device.type == "meta" for p in self.model_context.model.parameters()):
- materialize_model_(self.model_context.model)
+ kind = self._get_calibrator_kind()
+ self.calibration = get_calibrator(kind)(self)
- if (
- hasattr(self.model_context.model, "hf_device_map")
- and len(self.model_context.model.hf_device_map) > 1
- ):
- self.model_context.model = dispatch_model(
- self.model_context.model, device_map=self.model_context.model.hf_device_map
- )
- else:
- # Change this if new device is supported
- if str(self.model_context.model.device) == "cpu" and (
- not self.compress_context.device.startswith("hpu")
- ):
- # type(self.model_context.model._no_split_modules) changes from list to set
- # when transformers > 5.0
- no_split_modules = list(getattr(self.model_context.model, "_no_split_modules", []))
- devices = parse_available_devices(self.compress_context.device_map)
-
- max_memory = get_max_memory()
- new_max_memory = {}
- if "cpu" not in devices:
- devices.append("cpu")
- for device in devices:
- if ":" in device:
- device = int(device.split(":")[-1])
- elif device == "cpu":
- device = "cpu"
- elif isinstance(device, str):
- device = 0
- else:
- raise ValueError(
- f"Unsupported device {device} in device_map: {self.compress_context.device_map}"
- )
- if device not in max_memory:
- # Skip devices that aee not reported by accelerate's max_memory.
- # This is expected when a device is unavailable or cannot provide memory info.
- continue
- # Use 90% of the reported max memory to leave headroom for activations,
- # temporary tensors, other processes, and allocator fragmentation, reducing
- # the chance of runtime OOM while still utilizing most available memory.
- new_max_memory[device] = max_memory[device] * 0.9
-
- # If non-CPU devices were requested but none survived, fall back to CPU caching
- # via the OOM handler below, avoiding unnecessary dispatch overhead.
- requested_non_cpu = any((d != "cpu") for d in devices)
- has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)
- if requested_non_cpu and not has_non_cpu_memory:
- raise torch.OutOfMemoryError(
- "No non-CPU device available in accelerate's reported memory. "
- "Falling back to CPU caching."
- )
-
- # Keep ngram_embeddings on CPU
- has_ngram_embeddings, raw_ngram_embeddings = hook_ngram_embeddings_on_cpu(
- self.model_context.model
- )
- new_max_memory = get_balanced_memory(
- self.model_context.model,
- max_memory=new_max_memory,
- no_split_module_classes=no_split_modules,
- )
- if hasattr(self.model_context.model, "tie_weights"):
- self.model_context.model.tie_weights()
- device_map = infer_auto_device_map(
- self.model_context.model,
- max_memory=new_max_memory,
- no_split_module_classes=no_split_modules,
- )
- if len(devices) > 1 and "cpu" in device_map.values():
- logger.warning(
- "Some layers are offloaded to cpu, which may severely impact calibration speed."
- " Please consider using more cards."
- )
-
- try:
-
- self.model_context.model = dispatch_model(self.model_context.model, device_map=device_map)
- if has_ngram_embeddings:
- self.model_context.model.model.ngram_embeddings = raw_ngram_embeddings
- except ValueError as e:
- if "offload_dir" in e.__str__():
- logger.warning(
- f"Due to insufficient resources, disk is used to store the model."
- f" `offload_dir={envs.AR_WORK_SPACE}`"
- )
- self.model_context.model = dispatch_model(
- self.model_context.model, device_map=device_map, offload_dir=envs.AR_WORK_SPACE
- )
- else:
- raise
- else:
-
- self.model_context.model = self.model_context.model.to(self.compress_context.device)
+ def _get_calibrator_kind(self) -> str:
+ """Return the registry name of the calibrator to use.
- all_inputs = self.cache_inter_data(
- block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
- )
- if (
- hasattr(self.model_context.model, "hf_device_map")
- and len(self.model_context.model.hf_device_map) > 1
- ):
- accelerate.hooks.remove_hook_from_submodules(self.model_context.model)
-
- except torch.OutOfMemoryError as e:
- if cannot_calibrate_on_cpu:
- raise e
- cuda_error_msg = traceback.format_exc()
- try:
- logger.info("switch to cpu to cache block inputs")
- self.compress_context.cache_device = torch.device("cpu")
- if self.quantizer.has_qlayer_outside_block or self.__class__.__name__ == "AutoRoundMLLM":
- logger.warning(
- "we recommend using more GPUs in calibration."
- " Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy."
- )
- accelerate.hooks.remove_hook_from_submodules(self.model_context.model)
- self.model_context.model = mv_module_from_gpu(self.model_context.model)
- clear_memory(device_list=self.compress_context.device_list)
- # Important change after v0.51, on cpu, we use rtn mode for layers in layer_names
- all_inputs = self.cache_inter_data(
- block_names, nsamples, layer_names=[], last_cache_name=last_cache_name
- )
- except Exception as e:
- logger.error(cuda_error_msg)
- raise
- return all_inputs
+ Default ``"llm"``. ``MLLMMixin`` / ``DiffusionMixin`` override this
+ to select ``"mllm"`` / ``"diffusion"``.
+ """
+ return "llm"
@torch.no_grad()
- def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):
- """Save the inputs of block_name for calibration.
-
- This method temporarily replaces the forward method of the model to capture
- the inputs passing through the specified block. It then calibrates the model
- using a specified number of samples. Finally, it restores the original forward
- method and returns the inputs for the specified block.
- Args:
- block_names (list): The names of the blocks for which inputs are to be saved.
- layer_names (list):The names of the layers for which inputs are to be saved.
- nsamples (int): The number of samples to use for calibration.
- last_cache_name (str, optional): The name of the last layer to be cached,
- we could break the forward in this layer to save time
+ def try_cache_inter_data_gpucpu(self, block_names, nsamples, layer_names=None, last_cache_name=None):
+ """Thin wrapper around ``self.calibration.collect``.
- Returns:
- dict: A dictionary containing the inputs for the specified block.
+ Public API kept for backward compatibility (entry.py and
+ LLM-Compressor integration).
"""
- if layer_names is None:
- layer_names = []
-
- if not self._post_init_done:
+ if self.calibration is None:
self.post_init()
+ return self.calibration.collect(block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name)
- if hasattr(self, "quantizer") and hasattr(self.quantizer, "attention_mask"):
- self.quantizer.attention_mask = []
+ @torch.no_grad()
+ def cache_inter_data(self, block_names, nsamples, layer_names=None, last_cache_name=None):
+ """Thin wrapper around ``self.calibration.cache_inter_data``.
- self.inputs = {}
- block_names = flatten_list(block_names)
- self.to_cached_layers = block_names + layer_names
-
- tmp_dtype = None # TODO delete this as most model is not fp32 now
- ## have bug if block name is not the first block
- if (len(block_names) > 1 or len(layer_names) > 0) and self.compress_context.low_gpu_mem_usage:
- tmp_dtype = self.model_context.model.dtype
- if self.model_context.amp:
- if self.model_context.model.dtype != self.model_context.model.dtype:
- self.model_context.model = self.model_context.model.to(torch.bfloat16)
- else:
- self.model_context.model = self.model_context.model.to(torch.float32) ##model on cpu
-
- self.last_cache_name = _infer_last_cache_name(block_names, layer_names, last_cache_name)
- self._cache_target_set = set(self.to_cached_layers)
- self._cache_seen_targets = set()
- calib_bs = self.quantizer.batch_size
- self.hook_handles = []
- self._replace_forward()
- try:
- self.calib(nsamples, calib_bs)
- finally:
- # Use finally to recover_forward and delattr in case of that
- # self.calib raises NotImplementedError, such as: flash_attn on CPU.
- self.model_context.recover_forward()
- for attr in ("last_cache_name", "_cache_target_set", "_cache_seen_targets", "to_cached_layers"):
- if hasattr(self, attr):
- delattr(self, attr)
- # Release calibration dataloader to free tokenized sample tensors
- if hasattr(self, "dataloader"):
- del self.dataloader
- res = self.inputs
- if tmp_dtype is not None:
- self.model_context.model = self.model_context.model.to(tmp_dtype)
-
- return res
+ Public API kept for backward compatibility.
+ """
+ if self.calibration is None:
+ self.post_init()
+ return self.calibration.cache_inter_data(
+ block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
+ )
@torch.no_grad()
def calib(self, nsamples, bs):
- """Perform calibration for quantization.
+ """Thin wrapper around ``self.calibration.calib``.
- This method calibrates the model for quantization by processing a specified
- number of samples from the calibration dataset. It ensures that the data is
- properly formatted and feeds it to the model. If the number of samples processed
- is less than the specified number, it logs a warning. If no samples are processed,
- it logs an error and exits.
- Args:
- nsamples (int): The number of samples to use for calibration.
- bs (int): The number of samples to use for calibration
+ ``MLLMMixin`` and ``DiffusionMixin`` override this method directly via
+ Python MRO; for plain LLM models this routes into ``LLMCalibrator.calib``.
"""
- from auto_round.calib_dataset import get_dataloader
-
- need_attention_mask = True
- if isinstance(self.dataset, str):
- need_attention_mask = False # all supported datasets does not use pad
- dataset = self.dataset.replace(" ", "") ##remove all whitespaces
-
- # slow here
- self.dataloader = get_dataloader(
- self.model_context.tokenizer,
- self.seqlen,
- dataset,
- self.seed,
- bs,
- self.nsamples,
- )
- else:
- self.dataloader = self.dataset
- total_cnt = 0
- if self.dataloader.__class__.__name__ == "BatchEncoding":
- self.dataloader = [self.dataloader.data]
-
- for data in self.dataloader:
- if data.__class__.__name__ == "BatchEncoding":
- data = data.data
- if data is None:
- continue
- if isinstance(data, torch.Tensor):
- input_ids = data.to(self.model.device)
- data_new = input_ids
- elif isinstance(data, str):
- if self.model_context.tokenizer is None:
- logger.error("please provide tokenizer for string input")
- exit(-1)
- data = self.model_context.tokenizer(
- data, truncation=True, max_length=self.seqlen, return_tensors="pt"
- ).data
- data_new = {}
- for key in data.keys():
- data_new[key] = data[key].to(self.model.device)
- input_ids = data_new["input_ids"]
- elif isinstance(data, tuple) or isinstance(data, list):
- data_new = to_device(data, self.model.device)
- input_ids = data_new[0]
- else:
- data_new = {}
- for key in data.keys():
- data_new[key] = to_device(data[key], self.model.device)
- if key == "images":
- data_new[key] = to_dtype(data_new[key], self.model.dtype)
- input_ids = data_new["input_ids"]
- if input_ids.shape[-1] < self.seqlen:
- continue
- if need_attention_mask:
- if (
- isinstance(data_new, dict)
- and "attention_mask" in data_new
- and data_new["attention_mask"] is not None
- ):
- new_attention_mask = data_new["attention_mask"]
- elif (
- self.model_context.tokenizer is not None
- and hasattr(self.model_context.tokenizer, "pad_token")
- and self.model_context.tokenizer.pad_token is not None
- ):
- new_attention_mask = (input_ids != self.model_context.tokenizer.pad_token_id).to(torch.long)
- else:
- # Default all ones
- new_attention_mask = torch.ones_like(input_ids, dtype=torch.long)
-
- # For each sample, check if there are trailing repeated tokens
- # If so, set the mask of the last token to 0
- batch_size, seq_len = input_ids.shape
- for i in range(batch_size):
- last_token = input_ids[i, -1]
- # Check for trailing repeats
- j = seq_len - 2
- repeated = False
- while j >= 0 and input_ids[i, j] == last_token:
- repeated = True
- new_attention_mask[i, j] = 0
- j -= 1
- # If there was at least one repeat, set last token mask to 0
- if repeated:
- new_attention_mask[i, -1] = 0
-
- # Workaround: some models treat an all-1 attention mask as equivalent to None and
- # will internally replace it with None for block inputs, which can cause tensor
- # concatenation / shape-mismatch issues in downstream code. To avoid providing an
- # all-1 mask, we force the last token in each sequence to be masked out (set to 0)
- # so that the mask is never "all ones". This means the model will not attend to the
- # last position, so the impact on accuracy is minimal as basically equivalent to dropping a single token
- new_attention_mask[:, -1] = 0
-
- if not hasattr(self.quantizer, "attention_mask"):
- self.quantizer.attention_mask = []
- self.quantizer.attention_mask.extend(list(torch.split(new_attention_mask, 1, dim=0)))
- else:
- new_attention_mask = None
- try:
- kwargs = {"use_cache": False}
- if new_attention_mask is not None and not (isinstance(data_new, dict) and "attention_mask" in data_new):
- kwargs["attention_mask"] = new_attention_mask
-
- if isinstance(data_new, torch.Tensor):
- self.model(data_new, **kwargs)
- elif isinstance(data_new, tuple) or isinstance(data_new, list):
- self.model(*data_new, **kwargs)
- else:
- self.model(**data_new, **kwargs)
- except NotImplementedError as error:
- error_msg = str(error)
- # Raise NotImplementedError to fallback to CUDA device
- if "flash_attn::" in error_msg and "CPU" in error_msg:
- raise NotImplementedError(
- "Could not run 'flash_attn::_flash_attn_varlen_forward'"
- " with arguments from the 'CPU' backend."
- )
- else:
- pass
- except RuntimeError as error:
- error_msg = str(error)
- if "The expanded size of the tensor" in str(error_msg) and "must match the existing size" in error_msg:
- check_seqlen_compatible(self.seqlen, self.model_context.tokenizer, self.model)
- logger.warning(
- "When quantization encounters tensor shape mismatch error, "
- "you can try to avoid it with batch_size=1"
- )
- raise error
- except Exception as error:
- raise error
-
- total_cnt += input_ids.shape[0] if len(input_ids.shape) > 1 else 1
- if total_cnt >= nsamples:
- break
- if total_cnt == 0:
- logger.error(
- f"no data has been cached, please provide more data with sequence length "
- f">={self.seqlen} in the dataset or decease the sequence length"
- )
- exit(-1)
- elif total_cnt < nsamples:
- logger.warning_once(
- f"An insufficient number of samples likely reduces the accuracy of the quantized model. "
- f"Target samples count is {nsamples}, while valid samples count is {total_cnt}"
- )
+ if self.calibration is None:
+ self.post_init()
+ return self.calibration.calib(nsamples, bs)
@torch.no_grad()
def _get_block_forward_func(self, name: str) -> Callable:
- """Gets the forward function.
+ """Build the block-forward replacement, then let the calibrator wrap it.
- Args:
- name (str): The name of the function.
- Returns:
- function: The forward function.
+ ``Calibrator.wrap_block_forward`` defaults to passthrough; the
+ Diffusion calibrator overrides it to convert positional → kwargs.
"""
+ from auto_round.calibration.hooks import make_block_forward_func
- def post_process_cache_data(batch_size, data, data_name):
- """
- Processes store data for batch handling, reshaping if necessary.
-
- Args:
- batch_size (int): The size of the batch.
- data: The data value to store, potentially for caching.
- data_name (str): Name of the data.
-
- Returns:
- Processed data or None
- """
- new_data = data
- if data_name in self.model_context.shared_cache_keys:
- return None
- if batch_size <= 1:
- return new_data
- if "alibi" in data_name:
- if isinstance(data, torch.Tensor):
- alibi = data
- alibi = alibi.reshape(batch_size, -1, alibi.shape[1], alibi.shape[2])
- new_data = alibi
- return new_data
-
- def forward(m, hidden_states=None, *positional_inputs, **kwargs):
- """Rewrite forward function, process and collect input data.
-
- Args:
- hidden_states (torch.Tensor): The hidden states tensor.
- *positional_inputs: Variable number of positional arguments.
- **kwargs: Variable number of keyword arguments.
-
- Returns:
- NotImplementedError: Getting the first layer inputs and then raise the error to save runtime.
- """
- if name not in self.inputs:
- self.inputs[name] = {}
- init_cache(positional_inputs, self.inputs[name])
-
- if self.quantizer.batch_dim is None:
- self.quantizer.batch_dim = 0
- if hidden_states is not None and self.quantizer.batch_size > 1:
- if hidden_states.shape[0] > self.quantizer.batch_size:
- self.quantizer.batch_dim = 1
- if len(hidden_states.shape) > 1 and hidden_states.shape[1] > self.quantizer.batch_size:
- logger.error(
- "this model has not been supported, "
- "please raise an issue in https://github.com/intel/auto-round/issues"
- " or try to set the `batch_size` to 1 and "
- "`gradient_accumulate_steps` to your current batch size."
- )
- exit(-1)
-
- if hidden_states is not None:
- kwargs["hidden_states"] = hidden_states
-
- for key in kwargs.keys():
- if (
- isinstance(kwargs[key], torch.Tensor)
- or isinstance(kwargs[key], list)
- or isinstance(kwargs[key], tuple)
- ):
- if (
- self.has_variable_block_shape
- and name not in self.blocks_requiring_input_ids
- and key == "hidden_states"
- ):
- continue
- if key not in self.inputs[name].keys(): # initialization
- data = to_device(kwargs[key], device=torch.device("cpu"))
- if data is None or key in self.model_context.shared_cache_keys:
- self.inputs[name][key] = data
- continue
- if self.quantizer.batch_size <= 1:
- self.inputs[name][key] = [data]
- else:
- data = post_process_cache_data(self.quantizer.batch_size, data, key)
- if isinstance(data, torch.Tensor):
- self.inputs[name][key] = list(torch.split(data, 1, dim=self.quantizer.batch_dim))
- else:
- self.inputs[name][key] = [data]
- else: # append cache inputs
- new_data = post_process_cache_data(self.quantizer.batch_size, kwargs[key], key)
- if new_data is None: # shareable args or NoneType
- continue
- new_data = to_device(new_data, device=torch.device("cpu"))
- if self.quantizer.batch_size <= 1:
- self.inputs[name][key].append(new_data)
- else:
- if isinstance(new_data, torch.Tensor):
- self.inputs[name][key].extend(
- list(torch.split(new_data, 1, dim=self.quantizer.batch_dim))
- )
- else:
- self.inputs[name][key].append(new_data)
- elif isinstance(kwargs[key], (str, bool, type(None))):
- if key not in self.inputs[name].keys():
- self.inputs[name][key] = kwargs[key]
- else:
- # Parameters not to be cached
- if check_skippable_keywords(key):
- logger.warning_once(
- f"Please note that '{key}' key" " is not currently used in quantization fine-tuning."
- )
- reset_params(self.inputs[name])
-
- if self._should_stop_cache_forward(name):
- raise NotImplementedError
- else:
- if hidden_states is not None:
- kwargs.pop("hidden_states")
- return m.orig_forward(hidden_states, *positional_inputs, **kwargs)
- else:
- # Currently only for Llama-3.2-Vision-Instruct Series
- return m.orig_forward(*positional_inputs, **kwargs)
-
- return forward
+ fn = make_block_forward_func(self, name)
+ if self.calibration is not None:
+ fn = self.calibration.wrap_block_forward(fn)
+ return fn
@torch.no_grad()
def _get_cache_data_hook_for_layer(self, name):
- """A forward hook to save input max of a module
- :param name: the module name
- :return: A hook function."""
-
- def cache_input_hook(module, inputs, outputs):
- input = inputs
- if isinstance(inputs, tuple) or isinstance(input, list):
- input = inputs[0]
- if name in self.inputs:
- self.inputs[name].extend(list(torch.split(input.to("cpu"), 1, dim=0)))
- else:
- self.inputs[name] = list(torch.split(input.to("cpu"), 1, dim=0))
-
- if self._should_stop_cache_forward(name):
- raise NotImplementedError
+ """Thin wrapper around ``auto_round.calibration.hooks.make_layer_cache_hook``."""
+ from auto_round.calibration.hooks import make_layer_cache_hook
- return cache_input_hook
+ return make_layer_cache_hook(self, name)
def _replace_forward(self):
- """Replaces the forward function."""
-
- def register_hook(n, m, hook_handles):
- if n in self.to_cached_layers and type(m) not in SUPPORTED_LAYER_TYPES: ##block
- m.orig_forward = m.forward
- m.forward = partial(self._get_block_forward_func(n), m)
- elif n in self.to_cached_layers: ##linear layer or conv1d layer
- hook_func = self._get_cache_data_hook_for_layer(n)
- hook_handle = m.register_forward_hook(hook_func)
- hook_handles.append(hook_handle)
+ """Thin wrapper around ``auto_round.calibration.hooks.replace_forward_with_hooks``."""
+ from auto_round.calibration.hooks import replace_forward_with_hooks
- self.model_context.replace_forward(register_hook)
+ replace_forward_with_hooks(self)
def _should_stop_cache_forward(self, name: str) -> bool:
- """Determine whether current forward pass can stop after caching `name`."""
- if name == self.last_cache_name:
- return True
-
- if self.last_cache_name is not None:
- return False
-
- if not hasattr(self, "_cache_target_set") or not hasattr(self, "_cache_seen_targets"):
- return False
+ """Delegate the early-stop policy to the active calibrator.
- if name in self._cache_target_set:
- self._cache_seen_targets.add(name)
-
- if not self._cache_target_set.issubset(self._cache_seen_targets):
- return False
+ Falls back to the default helper when the calibrator has not been
+ constructed yet (very early init code paths).
+ """
+ if self.calibration is not None:
+ return self.calibration.should_stop(name)
+ from auto_round.calibration.hooks import should_stop_cache_forward
- # Lock the last cache name after the first full forward pass.
- self.last_cache_name = name
- return True
+ return should_stop_cache_forward(self, name)
def _preprocess_block_inputs(self, inputs, first_input_name="input_ids"):
- input_ids, input_others = self._split_inputs(inputs, first_input_name)
- clear_memory(device_list=self.compress_context.device_list)
- tmp_dtype = self.model_context.amp_dtype if self.model_context.amp else torch.float32
- if input_ids is not None:
- input_ids = to_device(input_ids, self.compress_context.cache_device)
- input_ids = to_dtype(input_ids, tmp_dtype)
- input_others = to_device(input_others, self.compress_context.cache_device)
-
- for key in input_others.keys():
- if isinstance(input_others[key], torch.Tensor) and (
- input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16
- ):
- input_others[key] = input_others[key].to(tmp_dtype)
- elif isinstance(input_others[key], list):
- for i in range(len(input_others[key])):
- to_dtype(input_others[key][i], tmp_dtype)
- return input_ids, input_others
+ # Thin wrapper around auto_round.calibration.inputs.preprocess_block_inputs.
+ from auto_round.calibration.inputs import preprocess_block_inputs
+
+ return preprocess_block_inputs(
+ inputs,
+ model_context=self.model_context,
+ compress_context=self.compress_context,
+ first_input_name=first_input_name,
+ )
def _split_inputs(self, inputs: dict, first_input_name: str) -> tuple[torch.Tensor, dict]:
- if self.model_context.is_diffusion:
- input_id_str = [key for key in inputs.keys() if "hidden_state" in key]
- input_ids = {k: inputs.pop(k, None) for k in input_id_str}
- input_others = inputs
- return input_ids, input_others
- input_ids = inputs.get(first_input_name, None)
- inputs.pop(first_input_name, None)
- input_others = inputs
- return input_ids, input_others
+ # Thin wrapper around auto_round.calibration.inputs.split_inputs.
+ from auto_round.calibration.inputs import split_inputs
+
+ return split_inputs(inputs, first_input_name, is_diffusion=self.model_context.is_diffusion)
def normalize_decoding_layer_inputs_(self, decoding_layer_inputs: list[tuple[tuple[Any, dict[str, Any]]]]) -> None:
"""Replay captured decoding-layer calls to populate ``self.inputs``.
@@ -755,7 +256,7 @@ class CalibCompressor(BaseCompressor):
def quantize_block(
self,
block: torch.nn.Module,
- inputs: tuple,
+ inputs,
q_input: Union[torch.Tensor, dict, None] = None,
device: Union[str, torch.device] = "cpu",
auto_offload: bool = True,
@@ -774,8 +275,15 @@ class CalibCompressor(BaseCompressor):
Args:
block: The transformer block (decoder layer) to quantize.
- inputs: Raw decoding-layer inputs captured by LLM-Compressor's hook.
- Format: list of ``((args, kwargs),)`` tuples as produced by the hook.
+ inputs: Either:
+
+ - the raw decoding-layer inputs captured by
+ LLM-Compressor's hook (list of ``((args, kwargs),)`` tuples),
+ in which case they are normalized via
+ :meth:`normalize_decoding_layer_inputs_`; **or**
+ - a :class:`~auto_round.calibration.state.CalibrationState`
+ instance produced by a :class:`~auto_round.calibration.base.Calibrator`,
+ which is bound directly without re-normalization.
q_input: Optional quantized input from the previous block. ``None`` on
the first block.
device: Target device for quantization (e.g. ``"cuda:0"``).
@@ -788,9 +296,11 @@ class CalibCompressor(BaseCompressor):
``enable_quanted_input`` is ``False``), and *reference_output* is the
full-precision reference output collected before optimization.
"""
- assert (
- not self.diffusion
- ), f"Currently, {self.__class__.__name__} does not support quantize_block for diffusion models."
+ from auto_round.calibration.state import CalibrationState
+
+ assert not self.mllm and not self.diffusion, (
+ f"Currently, {self.__class__.__name__} does not support quantize_block " "for MLLM / diffusion models."
+ )
# Ensure post_init has been called (sets up model_context, compress_context,
# quantizer, layer_config, etc.).
@@ -804,92 +314,100 @@ class CalibCompressor(BaseCompressor):
orig_is_mllm = self.model_context.is_mllm
self.model_context.is_mllm = False
- self.normalize_decoding_layer_inputs_(inputs)
- block_inputs = self.inputs[self.quant_block_list[0][0]]
- input_ids, input_others = self._preprocess_block_inputs(block_inputs, "hidden_states")
+ try:
+ if isinstance(inputs, CalibrationState):
+ # Caller already produced a CalibrationState (typically via
+ # ``Calibrator.collect``). Bind it as the authoritative store so
+ # the quantizer reads the same ``inputs`` / ``attention_mask`` /
+ # ``batch_dim``.
+ self.calibration_state = inputs
+ else:
+ self.normalize_decoding_layer_inputs_(inputs)
+ block_inputs = self.inputs[self.quant_block_list[0][0]]
+ input_ids, input_others = self._preprocess_block_inputs(block_inputs, "hidden_states")
- # ── Infrastructure: materialize, dtype convert, device placement ──────
- materialize_model_(block)
- convert_module_to_hp_if_necessary(block, self.model_context.amp_dtype, device)
+ # ── Infrastructure: materialize, dtype convert, device placement ──────
+ materialize_model_(block)
+ convert_module_to_hp_if_necessary(block, self.model_context.amp_dtype, device)
- if auto_offload:
- if (
- is_auto_device_mapping(self.compress_context.device_map)
- and len(self.compress_context.device_list) > 1
- and not self.model_context.is_diffusion
- ):
- from auto_round.utils.device import set_auto_device_map_for_block_with_tuning
+ if auto_offload:
+ if (
+ is_auto_device_mapping(self.compress_context.device_map)
+ and len(self.compress_context.device_list) > 1
+ and not self.model_context.is_diffusion
+ ):
+ from auto_round.utils.device import set_auto_device_map_for_block_with_tuning
- card_0_in_high_risk, loss_device = set_auto_device_map_for_block_with_tuning(
- block,
- self.compress_context.device_map,
- input_ids,
- self.compress_context.low_gpu_mem_usage,
- self.quantizer.batch_size,
- device,
- )
+ card_0_in_high_risk, loss_device = set_auto_device_map_for_block_with_tuning(
+ block,
+ self.compress_context.device_map,
+ input_ids,
+ self.compress_context.low_gpu_mem_usage,
+ self.quantizer.batch_size,
+ device,
+ )
+ else:
+ block = block.to(device)
+ card_0_in_high_risk, loss_device = False, device
else:
- block = block.to(device)
card_0_in_high_risk, loss_device = False, device
- else:
- card_0_in_high_risk, loss_device = False, device
-
- if len(self.compress_context.device_list) > 1 and auto_offload:
- from accelerate.hooks import AlignDevicesHook, add_hook_to_module
-
- for n, m in block.named_modules():
- if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
- continue
- add_hook_to_module(m, AlignDevicesHook(m.tuning_device, io_same_device=True), True)
-
- # ── Infrastructure: collect reference output and act_max ──────────────
- bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff
- if q_input is None:
- hook_handles = self.quantizer._register_act_max_hook(block)
- reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)
- for h in hook_handles:
- h.remove()
- else:
- reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)
- hook_handles = self.quantizer._register_act_max_hook(block)
- if hook_handles:
- self.quantizer._get_block_outputs(block, q_input, input_others, bs, save_output=False)
- for h in hook_handles:
- h.remove()
- if input_ids is not q_input:
- clear_memory(input_ids, device_list=self.compress_context.device_list)
+
+ if len(self.compress_context.device_list) > 1 and auto_offload:
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+
+ for n, m in block.named_modules():
+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):
+ continue
+ add_hook_to_module(m, AlignDevicesHook(m.tuning_device, io_same_device=True), True)
+
+ # ── Infrastructure: collect reference output and act_max ──────────────
+ bs = self.quantizer.batch_size * self.quantizer.infer_bs_coeff
+ if q_input is None:
+ hook_handles = self.quantizer._register_act_max_hook(block)
+ reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)
+ for h in hook_handles:
+ h.remove()
else:
- clear_memory(device_list=self.compress_context.device_list)
- input_ids = q_input
-
- # ── Pure algorithm: delegates to quantizer ────────────────────────────
- mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk
- self.quantizer.quantize_block(
- block,
- input_ids,
- input_others,
- reference_output,
- loss_device=loss_device,
- mid_iter_mem_check=mid_iter_mem_check,
- )
+ reference_output = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)
+ hook_handles = self.quantizer._register_act_max_hook(block)
+ if hook_handles:
+ self.quantizer._get_block_outputs(block, q_input, input_others, bs, save_output=False)
+ for h in hook_handles:
+ h.remove()
+ if input_ids is not q_input:
+ clear_memory(input_ids, device_list=self.compress_context.device_list)
+ else:
+ clear_memory(device_list=self.compress_context.device_list)
+ input_ids = q_input
- # ── MoE scale alignment for FP8 dispatch efficiency ────────────────
- if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):
- set_amax_for_all_moe_layers(block, attr_name="act_max")
+ # ── Pure algorithm: delegates to quantizer ────────────────────────────
+ mid_iter_mem_check = self.compress_context.low_gpu_mem_usage and card_0_in_high_risk
+ self.quantizer.quantize_block(
+ block,
+ input_ids,
+ input_others,
+ reference_output,
+ loss_device=loss_device,
+ mid_iter_mem_check=mid_iter_mem_check,
+ )
- # ── Collect quantized-block outputs ───────────────────────────────────
- if self.quantizer.enable_quanted_input:
- q_outputs = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)
- else:
- q_outputs = None
+ # ── MoE scale alignment for FP8 dispatch efficiency ────────────────
+ if is_nv_fp(self.quantizer.act_data_type) or is_static_wfp8afp8(self.quantizer):
+ set_amax_for_all_moe_layers(block, attr_name="act_max")
- # ── Cleanup ───────────────────────────────────────────────────────────
- if len(self.compress_context.device_list) > 1:
- accelerate.hooks.remove_hook_from_submodules(block)
- mv_module_from_gpu(block)
+ # ── Collect quantized-block outputs ───────────────────────────────────
+ if self.quantizer.enable_quanted_input:
+ q_outputs = self.quantizer._get_block_outputs(block, input_ids, input_others, bs)
+ else:
+ q_outputs = None
- self.model_context.is_mllm = orig_is_mllm
- return q_outputs, reference_output
+ # ── Cleanup ───────────────────────────────────────────────────────────
+ if len(self.compress_context.device_list) > 1:
+ accelerate.hooks.remove_hook_from_submodules(block)
+ mv_module_from_gpu(block)
+ return q_outputs, reference_output
+ finally:
+ self.model_context.is_mllm = orig_is_mllm
def _quantize_blocks(
self,
@@ -1043,7 +561,7 @@ class CalibCompressor(BaseCompressor):
if self.compress_context.is_immediate_packing:
for _n, _mod in m.named_modules():
if hasattr(_mod, "bits") and check_to_quantized(_mod):
- from auto_round.compressors_new.utils import immediate_pack as _immediate_pack
+ from auto_round.compressors.utils import immediate_pack as _immediate_pack
_immediate_pack(_mod.global_name, self.quantizer.layer_config)
@@ -1319,26 +837,8 @@ class CalibCompressor(BaseCompressor):
def _check_compatibility(self) -> None:
"""Checks compatibility of the configurations and model."""
- if (
- self.seqlen is not None
- and hasattr(self.model_context.model, "config")
- and hasattr(self.model_context.model.config, "max_position_embeddings")
- ):
- if self.model_context.model.config.max_position_embeddings < self.seqlen:
- logger.warning(
- f"Change sequence length to {self.model_context.model.config.max_position_embeddings} "
- "due to the limitation of max_position_embeddings"
- )
- self.seqlen = min(self.seqlen, self.model_context.model.config.max_position_embeddings)
-
- if self.seqlen is not None and hasattr(self.model_context.tokenizer, "model_max_length"):
- if self.model_context.tokenizer.model_max_length < self.seqlen:
- logger.warning(
- f"Change sequence length to {self.model_context.tokenizer.model_max_length} "
- "due to the limitation of model_max_length. "
- "You can also try to increase the model_max_length to avoid this issue."
- )
- self.seqlen = min(self.seqlen, self.model_context.tokenizer.model_max_length)
+ # ``seqlen`` clamping is owned by ``CalibrationState``.
+ self._calibration_state.clamp_seqlen(self.model_context)
if self.group_size == 0 and "fp8" not in self.data_type:
logger.warning("`group_size==0` is not supported for data_type other than fp8 ")
@@ -1353,8 +853,8 @@ class CalibCompressor(BaseCompressor):
)
-class CalibratedRTNCompressor(CalibCompressor):
- """CalibCompressor variant for iters=0 RTN that needs calibration data.
+class CalibratedRTNCompressor(DataDrivenCompressor):
+ """DataDrivenCompressor variant for iters=0 RTN that needs calibration data.
Handles two cases that require forward passes through the model:
- Weight quantization with imatrix (importance-matrix statistics for
@@ -1432,8 +932,8 @@ class CalibratedRTNCompressor(CalibCompressor):
clear_memory(self.inputs, device_list=self.compress_context.device_list)
total_samples = len(inputs["input_ids"])
- if total_samples < self.quantize_config.batch_size:
- self.quantize_config.batch_size = total_samples
+ if total_samples < self.batch_size:
+ self.batch_size = total_samples
logger.warning(f"Forcing batch size to {total_samples}")
tmp_dtype = self.model_context.amp_dtype if self.model_context.amp else torch.float32
@@ -1558,23 +1058,8 @@ class CalibratedRTNCompressor(CalibCompressor):
"""
logger.info("start to compute imatrix")
- # Load dataset
- from auto_round.calib_dataset import get_dataloader
-
- if isinstance(self.dataset, str):
- if self.model_context.tokenizer is None:
- raise ValueError("A tokenizer must be set for the model when using a dataset string.")
- dataset_name = self.dataset.replace(" ", "")
- self.dataloader = get_dataloader(
- self.model_context.tokenizer,
- self.seqlen,
- dataset_name,
- self.seed,
- self.quantize_config.batch_size,
- self.nsamples,
- )
- else:
- self.dataloader = self.dataset
+ # Dataloader resolution is owned by ``CalibrationState``.
+ self._calibration_state.ensure_dataloader(self.model_context, self.seed)
model = self.model_context.model
diff --git a/auto_round/compressors/diffusion/README.md b/auto_round/compressors/diffusion/README.md
index e87dc5dea..03125e7da 100644
--- a/auto_round/compressors/diffusion/README.md
+++ b/auto_round/compressors/diffusion/README.md
@@ -2,18 +2,18 @@
This feature is experimental and may be subject to changes, including potential bug fixes, API modifications, or adjustments to default parameters.
+AutoRound uses the new compressor and calibration architecture for diffusion quantization. Diffusion models are routed by `auto_round/compressors/entry.py`, diffusion-specific compressor behavior lives in `auto_round/compressors/diffusion_mixin.py`, and calibration is handled by `auto_round/calibration/diffusion.py`.
+
## Quantization
Quantization for diffusion models is limited:
-1. Only transformer module of diffusion models will be quantized..
-2. Loading quantized model is not supported yet, so please use `fake` format for quantization.
-3. Calibration dataset only supports `coco2014` and user customized `.tsv` file.
-
+1. Only the transformer module of diffusion models is quantized.
+2. Loading quantized diffusion models is not supported yet, so use `fake` format for quantization.
+3. Calibration dataset currently supports `coco2014` and user customized `.tsv` files.
### API Usage (CPU/GPU) Recommended
-
```python
import torch
from auto_round import AutoRound
@@ -31,24 +31,20 @@ autoround = AutoRound(
# Save the quantized model
output_dir = "./tmp_autoround"
-# Currently loading the quantized diffusion model is not supported, so use fake format
+# Loading quantized diffusion models is not supported yet, so use fake format.
autoround.quantize_and_save(output_dir, format="fake", inplace=True)
```
-- `dataset`: the dataset for quantization training. Currently only support coco2014 and user customized .tsv file.
-
-- `num_inference_steps`: The reference number of denoising steps.
-
-- `guidance_scale`: Control how much the image generation process follows the text prompt. The more it is, the more closely it follows the prompt.
+- `dataset`: the dataset for quantization training. Currently supports `coco2014` and user customized `.tsv` files.
+- `num_inference_steps`: the reference number of denoising steps.
+- `guidance_scale`: controls how much the image generation process follows the text prompt.
+- `generator_seed`: a seed that controls the initial noise from which an image is generated.
-- `generator_seed`: A seed that controls the initial noise from which an image is generated.
-
-for more hyperparameters introduction, please refer [Homepage Detailed Hyperparameters](../../../README.md#quantization-scheme--configuration)
+For more hyperparameters, refer to [Homepage Detailed Hyperparameters](../../../README.md#quantization-scheme--configuration).
### CLI Usage
-A user guide detailing the full list of supported arguments is provided by calling ```auto-round -h``` on the
-terminal.
+A user guide detailing the full list of supported arguments is provided by calling `auto-round -h` on the terminal.
```bash
auto-round \
@@ -64,29 +60,29 @@ auto-round \
For diffusion models, currently we validate quantization on the following models, which involves quantizing the transformer component of the pipeline.
-| Model | calibration dataset | Model Link |
-|---------------|---------------------|--------------|
-| black-forest-labs/FLUX.1-dev | COCO2014 | - |
-| Tongyi-MAI/Z-Image | COCO2014 | - |
-| Tongyi-MAI/Z-Image-Turb | COCO2014 | - |
-| stepfun-ai/NextStep-1.1 | COCO2014 | - |
-| AIDC-AI/Ovis-Image-7B | COCO2014 | - |
-
+| Model | calibration dataset | Model Link |
+| --- | --- | --- |
+| black-forest-labs/FLUX.1-dev | COCO2014 | - |
+| Tongyi-MAI/Z-Image | COCO2014 | - |
+| Tongyi-MAI/Z-Image-Turb | COCO2014 | - |
+| stepfun-ai/NextStep-1.1 | COCO2014 | - |
+| AIDC-AI/Ovis-Image-7B | COCO2014 | - |
<details>
<summary style="font-size:17px;">Calibration Dataset</summary>
-For diffusion models, we used [**coco2014**](https://github.com/mlcommons/inference/raw/refs/heads/master/text_to_image/coco2014/captions/captions_source.tsv) calibration dataset as our default.
+For diffusion models, we use [**coco2014**](https://github.com/mlcommons/inference/raw/refs/heads/master/text_to_image/coco2014/captions/captions_source.tsv) calibration dataset as the default.
-If users want to use their own dataset, please build the dataset file in ".tsv" format following below structure and use it through argument --dataset (tsv file):
-```
+To use a custom dataset, build a `.tsv` file with the following structure and pass it through `--dataset`:
+
+```text
id caption
0 YOUR_PROMPT
1 YOUR_PROMPT
... ...
```
-- `id`: The id used to map generated images and prompts.
-- `caption`: The text prompt used to generate the images.
+- `id`: the id used to map generated images and prompts.
+- `caption`: the text prompt used to generate the images.
-</details>
+</details>
\ No newline at end of file
diff --git a/auto_round/compressors/diffusion/__init__.py b/auto_round/compressors/diffusion/__init__.py
index b084e94f2..9d9ec6483 100644
--- a/auto_round/compressors/diffusion/__init__.py
+++ b/auto_round/compressors/diffusion/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025 Intel Corporation
+# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.compressors.diffusion.dataset import get_diffusion_dataloader
-from auto_round.compressors.diffusion.compressor import DiffusionCompressor
+from auto_round.compressors.diffusion.dataset import DIFFUSION_DATASET, get_diffusion_dataloader
from auto_round.compressors.diffusion.eval import diffusion_eval
+
+__all__ = ["DIFFUSION_DATASET", "diffusion_eval", "get_diffusion_dataloader"]
diff --git a/auto_round/compressors/diffusion/compressor.py b/auto_round/compressors/diffusion/compressor.py
deleted file mode 100644
index 7330f5018..000000000
--- a/auto_round/compressors/diffusion/compressor.py
+++ /dev/null
@@ -1,552 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-from collections import defaultdict
-from copy import deepcopy
-from functools import partial
-from typing import Union
-
-import torch
-from tqdm import tqdm
-
-from auto_round.compressors.base import BaseCompressor
-from auto_round.compressors.diffusion.dataset import get_diffusion_dataloader
-from auto_round.compressors.utils import block_forward
-from auto_round.formats import OutputFormat
-from auto_round.logger import logger
-from auto_round.schemes import QuantizationScheme
-from auto_round.utils import (
- LazyImport,
- clear_memory,
- config_save_pretrained,
- copy_python_files_from_model_cache,
- diffusion_load_model,
- dispatch_model_by_all_available_devices,
- extract_block_names_to_str,
- find_matching_blocks,
- get_block_names,
- merge_block_output_keys,
- rename_weights_files,
- wrap_block_forward_positional_to_kwargs,
-)
-
-pipeline_utils = LazyImport("diffusers.pipelines.pipeline_utils")
-
-output_configs = {
- "FluxTransformerBlock": ["encoder_hidden_states", "hidden_states"],
- "FluxSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],
- "OvisImageTransformerBlock": ["encoder_hidden_states", "hidden_states"],
- "OvisImageSingleTransformerBlock": ["encoder_hidden_states", "hidden_states"],
-}
-
-
-class DiffusionCompressor(BaseCompressor):
- """Class for automatic rounding-based quantization with Diffusion models.
-
- Args:
- model: The PyTorch model to be quantized.
- tokenizer: An optional tokenizer for processing input data, is not used for diffusion models.
- platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
- guidance_scale (float): Control how much the image generation process follows the text prompt.
- The more it is, the more closely it follows the prompt (default is 7.5).
- num_inference_steps (int): The reference number of denoising steps (default is 50).
- generator_seed (int): A sees that controls the initial noise from which an image is generated (default is None).
- pipeline_fn (callable, optional): Custom callable to run the pipeline during calibration.
- Signature: ``fn(pipe, prompts, *, guidance_scale, num_inference_steps, generator, **kwargs)``.
- Use this to support models whose inference API differs from the standard convention
- (e.g. NextStep). If ``None``, the standard ``pipe(prompts, ...)`` call is used unless
- the loaded pipeline already exposes an ``_autoround_pipeline_fn`` attribute.
- scheme: (str| dict | QuantizationScheme ): A preset scheme that defines the quantization configurations.
- layer_config (dict): Configuration for weight quantization (default is None).
- dataset: The path or name of the calib dataset.
- iters (int): Number of iterations (default is 200).
- seqlen (int): Length of the sequence.
- nsamples (int): Number of samples (default is 128).
- batch_size (int): Batch size for training (default is 8).
- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
- device_map (str | dict | int | torch.device, optional): Device placement map. Defaults to 0.
- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer
- **kwargs: Additional keyword arguments.
- """
-
- bits: int | None
- group_size: int | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
- is_diffusion: bool = True
-
- def __init__(
- self,
- model: Union[object, str],
- tokenizer=None,
- platform: str = "hf",
- guidance_scale: float = 7.5,
- num_inference_steps: int = 50,
- generator_seed: int = None,
- scheme: Union[str, dict, QuantizationScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "coco2014",
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- pipeline_fn: callable = None,
- **kwargs,
- ):
- logger.warning("Diffusion model quantization is experimental and is only validated on a few models.")
- if dataset == "NeelNanda/pile-10k":
- dataset = "coco2014"
- logger.warning(
- "Dataset 'NeelNanda/pile-10k' is not suitable for diffusion "
- + "model quantization, use coco2014 dataset instead."
- )
- model_dtype = kwargs.pop("model_dtype", None)
-
- self.guidance_scale = guidance_scale
- self.num_inference_steps = num_inference_steps
- self.generator_seed = generator_seed
-
- to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)
- if device_map is None:
- device_map = 0
- self._set_device(device_map)
-
- pipe, model = diffusion_load_model(model, platform=platform, device=self.device, model_dtype=model_dtype)
-
- self.model = model
- self.pipe = pipe
- # Use explicit pipeline_fn; fall back to whatever diffusion_load_model attached to the pipe
- self.pipeline_fn = pipeline_fn or getattr(pipe, "_autoround_pipeline_fn", None)
-
- all_blocks = get_block_names(model)
- self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
- if to_quant_block_names is None:
- to_quant_block_names = extract_block_names_to_str(self.quant_block_list)
-
- if iters > 0 and batch_size != 1:
- logger.warning(
- f"reset batch_size({batch_size}) to 1 and "
- f"gradient_accumulate_steps({gradient_accumulate_steps}) "
- f"to {batch_size * gradient_accumulate_steps}, "
- f"because batch_size={batch_size} cannot be used for calibrating non-text modules."
- )
- gradient_accumulate_steps = batch_size * gradient_accumulate_steps
- batch_size = 1
-
- seqlen = 2048 if seqlen is None else seqlen
-
- if nsamples % batch_size != 0:
- nsamples = (nsamples // batch_size + 1) * batch_size
- logger.warning(f"'nsamples' is not divisible by 'batch_size', will adjusted to {nsamples}")
-
- kwargs["diffusion"] = True
- super(DiffusionCompressor, self).__init__(
- model=model,
- tokenizer=None,
- platform=platform,
- scheme=scheme,
- layer_config=layer_config,
- dataset=dataset,
- iters=iters,
- seqlen=seqlen,
- nsamples=nsamples,
- batch_size=batch_size,
- gradient_accumulate_steps=gradient_accumulate_steps,
- low_gpu_mem_usage=low_gpu_mem_usage,
- device_map=device_map,
- enable_torch_compile=enable_torch_compile,
- seed=seed,
- to_quant_block_names=to_quant_block_names,
- **kwargs,
- )
- self._align_device_and_dtype()
-
- def _update_inputs(self, inputs: dict, q_inputs: dict) -> tuple[dict, dict]:
- # flux transformer model's blocks will update hidden_states and encoder_hidden_states
- input_id_str = [key for key in inputs.keys() if "hidden_state" in key]
- if q_inputs is not None:
- q_inputs = {k: q_inputs.pop(k, None) for k in input_id_str}
- return inputs, q_inputs
-
- def _get_block_forward_func(self, name):
- return wrap_block_forward_positional_to_kwargs(super()._get_block_forward_func(name))
-
- def _split_inputs(self, inputs: dict, first_input_name: str) -> tuple[dict, dict]:
- input_id_str = [key for key in inputs.keys() if "hidden_state" in key]
- input_ids = {k: inputs.pop(k, None) for k in input_id_str}
- input_others = inputs
- return input_ids, input_others
-
- def _get_current_output(self, output: dict, indices: list[int]) -> torch.Tensor:
- assert "hidden_states" in output
- current_output = [output["hidden_states"][x] for x in indices]
- current_output = torch.cat(current_output, dim=self.batch_dim)
- return current_output
-
- def _get_current_q_output(
- self,
- block: torch.nn.Module,
- input_ids: dict,
- input_others: dict,
- indices: list[int],
- device: str,
- cache_device: str = "cpu",
- ) -> torch.Tensor:
- output_config = output_configs.get(block.__class__.__name__, ["hidden_states"])
- idx = None if "hidden_states" not in output_config else output_config.index("hidden_states")
- current_input_ids, current_input_others = self._sampling_inputs(
- input_ids,
- input_others,
- indices,
- seqlen=self.seqlen,
- batch_dim=self.batch_dim,
- share_cache_keys=self.shared_cache_keys,
- )
- if isinstance(current_input_ids, dict):
- hidden_states = current_input_ids.pop("hidden_states")
- merge_block_output_keys(block, current_input_others, current_input_ids)
- current_input_ids = hidden_states
- output_q = block_forward(block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device, idx)
- return output_q.to(cache_device)
-
- @torch.no_grad()
- def _get_block_outputs(
- self,
- block: torch.nn.Module,
- input_ids: Union[torch.Tensor, dict],
- input_others: torch.Tensor,
- bs: int,
- device: Union[str, torch.device],
- cache_device: Union[str, torch.device],
- save_output: bool = True,
- ):
- """Compute the output of a given block of the model for a given input.
-
- Args:
- block: The block of the model.
- input_ids: The input tensor containing tokenized input ids.
- input_others: A dictionary containing additional input data.
- bs: The batch size for computing the output.
- device: The device for computation.
- cache_device: The device for storing the output.
- batch_dim: The batch dimension of the output tensor.
-
- Returns:
- The output tensor of the block.
- """
-
- output = defaultdict(list)
- output_config = output_configs.get(block.__class__.__name__, ["hidden_states"])
- if isinstance(input_ids, dict):
- nsamples = len(input_ids["hidden_states"])
- else:
- nsamples = len(input_ids)
-
- for i in range(0, nsamples, bs):
- end_index = min(nsamples, i + bs)
- indices = torch.arange(i, end_index).to(torch.long)
- tmp_input_ids, tmp_input_others = self._sampling_inputs(
- input_ids, input_others, indices, self.seqlen, self.batch_dim, share_cache_keys=self.shared_cache_keys
- )
- if isinstance(tmp_input_ids, dict):
- hidden_states = tmp_input_ids.pop("hidden_states")
- merge_block_output_keys(block, tmp_input_others, tmp_input_ids)
- tmp_input_ids = hidden_states
-
- tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device, None)
- if isinstance(tmp_output, torch.Tensor):
- tmp_output = [tmp_output]
- assert len(output_config) == len(tmp_output)
- tmp_output = dict(zip(output_config, tmp_output))
-
- if save_output:
- for name, out in tmp_output.items():
- if self.batch_size == 1:
- output[name].append(out.to(cache_device))
- else:
- output[name].extend(list(torch.split(out.to(cache_device), 1, dim=self.batch_dim)))
- if self.low_gpu_mem_usage:
- clear_memory()
-
- return output
-
- def _get_current_num_elm(
- self,
- input_ids: list[torch.Tensor],
- indices: list[int],
- ) -> int:
- current_input_ids = [input_ids["hidden_states"][i] for i in indices]
- return sum(id.numel() for id in current_input_ids)
-
- def _run_pipeline(self, prompts: list) -> None:
- """Execute one full diffusion pipeline forward pass for calibration input capture.
-
- This drives all transformer blocks so that their intermediate inputs are recorded
- by the hooks installed during calibration.
-
- **Extending for custom models** – choose whichever approach is simpler:
-
- * Pass a ``pipeline_fn`` to the constructor (no subclassing required). The
- callable receives ``(pipe, prompts, *, guidance_scale, num_inference_steps,
- generator, **kwargs)`` and must trigger a full forward pass.
- * Subclass :class:`DiffusionCompressor` and override this method directly for
- full control over the inference logic.
-
- Example – NextStep model::
-
- def nextstep_fn(pipe, prompts, guidance_scale=7.5,
- num_inference_steps=28, generator=None,
- hw=(1024, 1024), **kwargs):
- for prompt in (prompts if isinstance(prompts, list) else [prompts]):
- pipe.generate_image(
- prompt,
- cfg=guidance_scale,
- num_sampling_steps=num_inference_steps,
- hw=hw,
- **kwargs,
- )
-
- compressor = DiffusionCompressor(
- model="path/to/nextstep",
- pipeline_fn=nextstep_fn,
- pipeline_fn_kwargs={"hw": (512, 512)},
- )
-
- Args:
- prompts (list[str]): Text prompts for the current calibration batch.
- """
- generator = (
- None
- if self.generator_seed is None
- else torch.Generator(device=self.pipe.device).manual_seed(self.generator_seed)
- )
- if self.pipeline_fn is not None:
- self.pipeline_fn(
- self.pipe,
- prompts,
- guidance_scale=self.guidance_scale,
- num_inference_steps=self.num_inference_steps,
- generator=generator,
- )
- else:
- self.pipe(
- prompts,
- guidance_scale=self.guidance_scale,
- num_inference_steps=self.num_inference_steps,
- generator=generator,
- )
-
- def calib(self, nsamples, bs):
- """Perform calibration for quantization.
-
- This method calibrates the model for quantization by processing a specified
- number of samples from the calibration dataset. It ensures that the data is
- properly formatted and feeds it to the model. If the number of samples processed
- is less than the specified number, it logs a warning. If no samples are processed,
- it logs an error and exits.
- Args:
- nsamples (int): The number of samples to use for calibration.
- bs (int): The number of samples to use for calibration
- """
- logger.warning(
- "Diffusion model will catch nsamples * num_inference_steps inputs, "
- "you can reduce nsamples or num_inference_steps if OOM or take too much time."
- )
- if isinstance(self.dataset, str):
- dataset = self.dataset.replace(" ", "")
- self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_diffusion_dataloader(
- dataset=dataset,
- bs=self.batch_size,
- seed=self.seed,
- nsamples=self.nsamples,
- gradient_accumulate_steps=self.gradient_accumulate_steps,
- )
- else:
- self.dataloader = self.dataset
- total_cnt = 0
-
- total = nsamples if not hasattr(self.dataloader, "len") else min(nsamples, len(self.dataloader))
-
- with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:
- for ids, prompts in self.dataloader:
- if isinstance(prompts, tuple):
- prompts = list(prompts)
- try:
- self._run_pipeline(prompts)
- except NotImplementedError:
- pass
- except Exception as error:
- raise error
- step = len(prompts)
- total_cnt += step
- pbar.update(step)
- if total_cnt >= nsamples:
- break
- if total_cnt == 0:
- logger.error(
- f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
- f"dataset or decease the sequence length"
- )
- exit(-1)
- elif total_cnt < nsamples:
- logger.warning(
- f"Insufficient number of samples collected may affect the quantization. "
- f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
- )
- if total_cnt < self.batch_size:
- raise ValueError(
- f"valid samples is less than batch_size({self.batch_size}),"
- " please adjust self.batch_size or seqlen."
- )
- max_len = (total_cnt // self.batch_size) * self.batch_size
- for k, v in self.inputs.items():
- for key in v:
- if isinstance(v[key], list) and len(v[key]) == total_cnt:
- self.inputs[k][key] = v[key][:max_len]
-
- # torch.cuda.empty_cache()
- logger.info(f"Restore num_inference_steps to {self.num_inference_steps} after calibration")
-
- def _should_stop_cache_forward(self, name: str) -> bool:
- """Determine whether current forward pass can stop after caching `name`."""
- # diffusion model needs to run all steps to collect input
- return False
-
- def _get_save_folder_name(self, format: OutputFormat) -> str:
- """Generates the save folder name based on the provided format string.
-
- If there are multiple formats to handle, the function creates a subfolder
- named after the format string with special characters replaced. If there's
- only one format, it returns the original output directory directly.
-
- Args:
- format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
-
- Returns:
- str: The path to the folder where results should be saved.
- """
- # Replace special characters to make the folder name filesystem-safe
- sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")
- if hasattr(self.model, "config") and getattr(self.model.config, "model_type", None) == "nextstep":
- # Use a subfolder only if there are multiple formats
- if len(self.formats) > 1:
- return os.path.join(self.orig_output_dir, sanitized_format)
-
- return self.orig_output_dir
-
- # Use a subfolder only if there are multiple formats
- if len(self.formats) > 1:
- return (
- os.path.join(self.orig_output_dir, sanitized_format, "transformer")
- if self.is_immediate_saving
- else os.path.join(self.orig_output_dir, sanitized_format, "transformer")
- )
-
- # if use is_immediate_saving, we need to save model in self.orig_output_dir/transformer folder
- return os.path.join(self.orig_output_dir, "transformer") if self.is_immediate_saving else self.orig_output_dir
-
- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
- """Save the quantized model to the specified output directory in the specified format.
-
- Args:
- output_dir (str, optional): The directory to save the quantized model. Defaults to None.
- format (str, optional): The format in which to save the model. Defaults to "auto_round".
- inplace (bool, optional): Whether to modify the model in place. Defaults to True.
- **kwargs: Additional keyword arguments specific to the export format.
-
- Returns:
- object: The compressed model object.
- """
- if output_dir is None:
- return super().save_quantized(output_dir, format=format, inplace=inplace, **kwargs)
-
- compressed_model = None
- if hasattr(self.model, "config") and getattr(self.model.config, "model_type", None) == "nextstep":
- compressed_model = super().save_quantized(
- output_dir=output_dir,
- format=format,
- inplace=inplace,
- **kwargs,
- )
- self.pipe.tokenizer.save_pretrained(output_dir)
- copy_python_files_from_model_cache(self.model, output_dir, copy_folders=["models", "vae", "utils"])
- return compressed_model
-
- for name in self.pipe.components.keys():
- val = getattr(self.pipe, name)
- sub_module_path = (
- os.path.join(output_dir, name) if os.path.basename(os.path.normpath(output_dir)) != name else output_dir
- )
- if (
- hasattr(val, "config")
- and hasattr(val.config, "_name_or_path")
- and val.config._name_or_path == self.model.config._name_or_path
- ):
- compressed_model = super().save_quantized(
- output_dir=sub_module_path if not self.is_immediate_saving else output_dir,
- format=format,
- inplace=inplace,
- **kwargs,
- )
- elif val is not None and hasattr(val, "save_pretrained"):
- val.save_pretrained(sub_module_path)
-
- if name == "transformer":
- # To suit "diffusion_pytorch_model.safetensors"
- for single_format in format:
- rename_weights_files(self._get_save_folder_name(single_format))
- if not hasattr(self.pipe.config, "save_pretrained"):
- # For Z-Image, tuning will cause this attribute missing, we need to add it back before saving config
- setattr(
- self.pipe.config,
- "save_pretrained",
- partial(config_save_pretrained, self.pipe.config, "model_index.json"),
- )
- self.pipe.config.save_pretrained(output_dir)
- return compressed_model
-
- def _align_device_and_dtype(self):
- if hasattr(self.model, "config") and getattr(self.model.config, "model_type", None) == "nextstep":
- return
- if (
- hasattr(self.model, "hf_device_map")
- and len(self.model.hf_device_map) > 0
- and type(self.pipe.device) != type(self.model.device)
- and self.pipe.device != self.model.device
- and torch.device(self.model.device).type in ["cuda", "xpu"]
- ):
- logger.error(
- "Diffusion model is activated sequential model offloading, it will crash during moving to GPU/XPU. "
- "Please use model path for quantization or "
- "move the pipeline object to GPU/XPU before passing them into API."
- )
- exit(-1)
-
- self.pipe.to(self.model.dtype)
-
- dispatch_model_by_all_available_devices(self.pipe, self.device_map)
diff --git a/auto_round/compressors/diffusion/hybrid.py b/auto_round/compressors/diffusion/hybrid.py
deleted file mode 100644
index 81c80d700..000000000
--- a/auto_round/compressors/diffusion/hybrid.py
+++ /dev/null
@@ -1,719 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""HybridCompressor for models with both AR and diffusion components.
-
-This compressor handles models that have a hybrid architecture consisting of:
- - An autoregressive (AR) language model component
- - A diffusion transformer (DiT) component
-
-It quantizes both components in a single workflow:
- Phase 1: Quantize the AR model using MLLM-style text calibration
- Phase 2: Quantize the DiT model using diffusion-style pipeline calibration
-
-Supported hybrid pipelines are registered in ``HYBRID_AR_COMPONENTS``.
-To add a new model, register its AR component attribute name and (optionally)
-its DiT block output config in ``output_configs``.
-"""
-
-from __future__ import annotations
-
-import copy
-import os
-import shutil
-import time
-from typing import Any, Union
-
-import torch
-
-from auto_round.compressors.diffusion.compressor import DiffusionCompressor, output_configs
-from auto_round.logger import logger
-from auto_round.schemes import QuantizationScheme
-from auto_round.utils import (
- LazyImport,
- clear_memory,
- extract_block_names_to_str,
- find_matching_blocks,
- get_block_names,
-)
-
-pipeline_utils = LazyImport("diffusers.pipelines.pipeline_utils")
-
-# ---------------------------------------------------------------------------
-# Registry: known AR component attribute names in hybrid diffusion pipelines.
-# Each entry maps a pipeline attribute name to the component role.
-# When a pipeline has *both* "transformer" and one of these attributes,
-# it is recognised as a hybrid model.
-# To support a new hybrid architecture, simply add its AR attribute name here.
-# ---------------------------------------------------------------------------
-HYBRID_AR_COMPONENTS = [
- "vision_language_encoder", # GLM-Image
- # Add new AR component names here, e.g.:
- # "language_model",
- # "text_decoder",
-]
-
-# ---------------------------------------------------------------------------
-# Register DiT block output configs for hybrid models.
-# Maps block class name -> ordered list of output tensor names.
-# Pure-diffusion blocks (Flux*) are already registered in DiffusionCompressor.
-# ---------------------------------------------------------------------------
-output_configs["GlmImageTransformerBlock"] = ["hidden_states", "encoder_hidden_states"]
-
-
-# ---------------------------------------------------------------------------
-# Detection
-# ---------------------------------------------------------------------------
-
-
-def _find_ar_component_name(model_or_path):
- """Return the AR component attribute name if model_or_path is a hybrid pipeline, else None."""
- if isinstance(model_or_path, str):
- index_path = os.path.join(model_or_path, "model_index.json")
- if not os.path.exists(index_path):
- from huggingface_hub import hf_hub_download
-
- try:
- index_path = hf_hub_download(model_or_path, "model_index.json")
- except Exception:
- return None
-
- import json
-
- with open(index_path) as f:
- data = json.load(f)
- if "transformer" not in data:
- return None
- for name in HYBRID_AR_COMPONENTS:
- if name in data:
- return name
- return None
-
- # Runtime pipeline object
- if hasattr(model_or_path, "transformer"):
- for name in HYBRID_AR_COMPONENTS:
- if hasattr(model_or_path, name) and getattr(model_or_path, name) is not None:
- return name
- return None
-
-
-def is_hybrid_diffusion_model(model_or_path):
- """Return True if *model_or_path* represents a hybrid AR+Diffusion pipeline."""
- return _find_ar_component_name(model_or_path) is not None
-
-
-class HybridCompressor(DiffusionCompressor):
- """Compressor for hybrid AR + diffusion models.
-
- Quantizes both the autoregressive component and the diffusion transformer
- component in a single workflow. The AR component is discovered automatically
- from ``HYBRID_AR_COMPONENTS``.
-
- Args:
- model: Model name/path or DiffusionPipeline object.
- tokenizer: Tokenizer (auto-loaded from pipeline if None).
- guidance_scale: Guidance scale for diffusion calibration.
- num_inference_steps: Denoising steps for diffusion calibration.
- generator_seed: Seed for noise generator.
- scheme: Quantization scheme.
- dataset: Calibration dataset for DiT (default: "coco2014").
- ar_dataset: Calibration dataset for AR model (default: "NeelNanda/pile-10k").
- quant_nontext_module: Whether to also quantize vision encoder in AR model.
- iters: Optimization iterations.
- seqlen: Calibration sequence length for AR model.
- nsamples: Number of calibration samples.
- batch_size: Calibration batch size.
- quant_ar: Whether to quantize the AR component.
- quant_dit: Whether to quantize the DiT component.
- height: Image height passed to the pipeline during DiT calibration (required by some pipelines
- such as GLM-Image; ignored if the pipeline does not accept it).
- width: Image width passed to the pipeline during DiT calibration.
- **kwargs: Additional keyword arguments passed to base compressor.
- """
-
- def __init__(
- self,
- model: Union[object, str],
- tokenizer=None,
- platform: str = "hf",
- guidance_scale: float = 1.5,
- num_inference_steps: int = 10,
- generator_seed: int = None,
- scheme: Union[str, dict, QuantizationScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "coco2014",
- ar_dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- quant_nontext_module: bool = False,
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = True,
- device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- quant_ar: bool = True,
- quant_dit: bool = True,
- height: int = None,
- width: int = None,
- **kwargs,
- ):
- logger.warning("Hybrid AR+Diffusion model quantization is experimental.")
- model_dtype = kwargs.pop("model_dtype", None)
-
- self.guidance_scale = guidance_scale
- self.num_inference_steps = num_inference_steps
- self.generator_seed = generator_seed
- self.quant_ar = quant_ar
- self.quant_dit = quant_dit
- self.quant_nontext_module = quant_nontext_module
- self.ar_dataset = ar_dataset
- self.height = height
- self.width = width
-
- to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)
- if device_map is None:
- device_map = 0
- self._set_device(device_map)
-
- # --- Load the pipeline ---
- if isinstance(model, str):
- from auto_round.utils.model import diffusion_load_model
-
- pipe, dit_model = diffusion_load_model(
- model, platform=platform, device=self.device, model_dtype=model_dtype
- )
- elif isinstance(model, pipeline_utils.DiffusionPipeline):
- pipe = model
- dit_model = pipe.transformer
- else:
- raise ValueError(f"HybridCompressor requires a model path or DiffusionPipeline, got {type(model)}")
-
- # --- Discover the AR component dynamically ---
- self.ar_component_name = _find_ar_component_name(pipe)
- if self.ar_component_name is None and self.quant_ar:
- logger.warning(
- f"No AR component found in pipeline (checked: {HYBRID_AR_COMPONENTS}), " "skipping AR quantization."
- )
- self.quant_ar = False
-
- self.pipe = pipe
- self.dit_model = dit_model
- self.ar_model = getattr(pipe, self.ar_component_name, None) if self.ar_component_name else None
-
- if not self.quant_ar and not self.quant_dit:
- raise ValueError("At least one of quant_ar and quant_dit must be True.")
-
- model = dit_model
-
- # --- Detect DiT blocks ---
- all_blocks = get_block_names(model)
- dit_blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)
-
- # Filter to only blocks whose class has a registered output_config.
- # get_block_names may discover non-transformer ModuleLists (e.g. MLP projectors)
- # that don't match the expected output format.
- if to_quant_block_names is None:
- filtered = []
- for group in dit_blocks:
- if not group:
- continue
- parts = group[0].split(".")
- m = model
- for p in parts:
- m = getattr(m, p)
- if m.__class__.__name__ in output_configs:
- filtered.append(group)
- if filtered:
- dit_blocks = filtered
- self.dit_quant_block_list = dit_blocks
-
- # --- Detect AR blocks ---
- if self.quant_ar and self.ar_model is not None:
- from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
-
- model_type = getattr(getattr(self.ar_model, "config", None), "model_type", None)
- if model_type and model_type in SPECIAL_MULTIMODAL_BLOCK:
- self.ar_quant_block_list = SPECIAL_MULTIMODAL_BLOCK[model_type](
- self.ar_model, quant_vision=quant_nontext_module
- )
- else:
- self.ar_quant_block_list = [get_block_names(self.ar_model)]
- else:
- self.ar_quant_block_list = []
-
- self.quant_block_list = self.dit_quant_block_list
- if to_quant_block_names is None:
- to_quant_block_names = extract_block_names_to_str(self.quant_block_list)
-
- # Force batch_size to 1 for diffusion calibration
- if iters > 0 and batch_size != 1:
- logger.warning(
- f"reset batch_size({batch_size}) to 1 and "
- f"gradient_accumulate_steps({gradient_accumulate_steps}) "
- f"to {batch_size * gradient_accumulate_steps}, "
- f"because batch_size > 1 cannot be used for diffusion calibration."
- )
- gradient_accumulate_steps = batch_size * gradient_accumulate_steps
- batch_size = 1
-
- seqlen = 2048 if seqlen is None else seqlen
-
- if nsamples % batch_size != 0:
- nsamples = (nsamples // batch_size + 1) * batch_size
- logger.warning(f"'nsamples' is not divisible by 'batch_size', adjusted to {nsamples}")
-
- kwargs["diffusion"] = True
- self._saved_pipe = pipe
- self._saved_dit_model = dit_model
- self._saved_ar_model = self.ar_model
-
- from auto_round.compressors.base import BaseCompressor
-
- BaseCompressor.__init__(
- self,
- model=model,
- tokenizer=None,
- platform=platform,
- scheme=scheme,
- layer_config=layer_config,
- dataset=dataset,
- iters=iters,
- seqlen=seqlen,
- nsamples=nsamples,
- batch_size=batch_size,
- gradient_accumulate_steps=gradient_accumulate_steps,
- low_gpu_mem_usage=low_gpu_mem_usage,
- device_map=device_map,
- enable_torch_compile=enable_torch_compile,
- seed=seed,
- to_quant_block_names=to_quant_block_names,
- **kwargs,
- )
-
- # Restore references that BaseCompressor.__init__ may have overwritten
- self.pipe = self._saved_pipe
- self.dit_model = self._saved_dit_model
- self.ar_model = self._saved_ar_model
-
- # ------------------------------------------------------------------
- # Quantization
- # ------------------------------------------------------------------
-
- def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:
- """Quantize both AR and DiT components.
-
- Phase 1: AR model via MLLM-style text calibration.
- Phase 2: DiT model via diffusion pipeline calibration.
- """
- start_time = time.time()
- combined_layer_config = {}
-
- # =================== Phase 1: AR Model ===================
- if self.quant_ar and self.ar_model is not None:
- logger.info("=" * 60)
- logger.info(f"Phase 1: Quantizing AR model ({self.ar_component_name})")
- logger.info("=" * 60)
-
- ar_compressor = self._create_ar_compressor()
- ar_model, ar_layer_config = ar_compressor.quantize()
-
- self.ar_model = ar_model
- setattr(self.pipe, self.ar_component_name, ar_model)
- combined_layer_config.update({f"ar.{k}": v for k, v in ar_layer_config.items()})
- self.ar_layer_config = ar_layer_config
-
- # Preserve serialization-relevant attributes from the AR compressor
- # so save_quantized can build the correct serialization_dict.
- from auto_round.compressors.base import SERIALIZATION_KEYS
-
- self._ar_serialization = {k: getattr(ar_compressor, k, None) for k in SERIALIZATION_KEYS}
-
- # Move AR model to CPU to free GPU for Phase 2
- self.ar_model.to("cpu")
- clear_memory(device_list=self.device_list)
- logger.info(f"Phase 1 complete: AR model ({self.ar_component_name}) quantized")
-
- # =================== Phase 2: DiT Model ===================
- if self.quant_dit:
- logger.info("=" * 60)
- logger.info("Phase 2: Quantizing DiT model (transformer)")
- logger.info("=" * 60)
-
- # Move DiT to target device for calibration
- self.dit_model = self.dit_model.to(self.device)
- self.model = self.dit_model
- self.quant_block_list = self.dit_quant_block_list
- self.quantized = False
- self.batch_dim = None
-
- for n, m in self.model.named_modules():
- m.global_name = n
-
- dit_model, dit_layer_config = self._quantize_dit()
-
- self.dit_model = dit_model
- self.pipe.transformer = dit_model
- combined_layer_config.update({f"dit.{k}": v for k, v in dit_layer_config.items()})
- self.dit_layer_config = dit_layer_config
-
- logger.info("Phase 2 complete: DiT model quantized")
-
- end_time = time.time()
- logger.info(f"Total hybrid quantization time: {end_time - start_time:.1f}s")
-
- self.quantized = True
- self.layer_config = combined_layer_config
- self.model = self.dit_model
- return self.model, self.layer_config
-
- def _create_ar_compressor(self):
- """Create an MLLM compressor for the AR component."""
- from auto_round.compressors.mllm.compressor import MLLMCompressor
-
- processor = getattr(self.pipe, "processor", None)
- tokenizer = getattr(self.pipe, "tokenizer", None)
-
- ar = MLLMCompressor(
- model=self.ar_model,
- tokenizer=tokenizer,
- processor=processor,
- image_processor=None,
- platform=self.platform,
- scheme=copy.deepcopy(self.orig_scheme) if hasattr(self, "orig_scheme") else self.scheme,
- dataset=self.ar_dataset,
- quant_nontext_module=self.quant_nontext_module,
- iters=self.iters,
- seqlen=self.seqlen,
- nsamples=self.nsamples,
- batch_size=1,
- gradient_accumulate_steps=self.gradient_accumulate_steps,
- low_gpu_mem_usage=self.low_gpu_mem_usage,
- device_map=self.device_map,
- enable_torch_compile=self.enable_torch_compile,
- seed=self.seed,
- )
- if hasattr(self, "formats"):
- ar.formats = self.formats
- # Required by base.quantize() → _adjust_immediate_packing_and_saving();
- # None disables immediate packing (correct since we call quantize() directly).
- ar.orig_output_dir = None
- ar.inplace = True
- return ar
-
- def _quantize_dit(self):
- """Quantize the DiT model using the parent DiffusionCompressor's quantize flow."""
- return DiffusionCompressor.quantize(self)
-
- def calib(self, nsamples, bs):
- """Override calib to pass extra pipeline kwargs (e.g. height/width) if set.
-
- Pipelines like GLM-Image require explicit image dimensions; standard diffusion
- pipelines (FLUX etc.) accept but ignore them.
- """
- import inspect
-
- pipe_sig = inspect.signature(self.pipe.__call__)
- extra = {}
- if "height" in pipe_sig.parameters and self.height is not None:
- extra["height"] = self.height
- if "width" in pipe_sig.parameters and self.width is not None:
- extra["width"] = self.width
-
- if not extra:
- # No extra kwargs needed — delegate to parent as-is
- return DiffusionCompressor.calib(self, nsamples, bs)
-
- # Replicate parent calib() with extra kwargs injected into the pipe call
- from tqdm import tqdm
-
- from auto_round.compressors.diffusion.dataset import get_diffusion_dataloader
- from auto_round.utils import clear_memory
-
- logger.warning(
- "Diffusion model will catch nsamples * num_inference_steps inputs, "
- "you can reduce nsamples or num_inference_steps if OOM or take too much time."
- )
- if isinstance(self.dataset, str):
- dataset = self.dataset.replace(" ", "")
- self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_diffusion_dataloader(
- dataset=dataset,
- bs=self.batch_size,
- seed=self.seed,
- nsamples=self.nsamples,
- gradient_accumulate_steps=self.gradient_accumulate_steps,
- )
- else:
- self.dataloader = self.dataset
- total_cnt = 0
-
- total = nsamples if not hasattr(self.dataloader, "len") else min(nsamples, len(self.dataloader))
- if self.pipe.dtype != self.model.dtype:
- self.pipe.to(self.model.dtype)
- if self.pipe.device != self.model.device:
- self.pipe.to(self.model.device)
-
- with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:
- for ids, prompts in self.dataloader:
- if isinstance(prompts, tuple):
- prompts = list(prompts)
- try:
- self.pipe(
- prompt=prompts,
- guidance_scale=self.guidance_scale,
- num_inference_steps=self.num_inference_steps,
- generator=(
- None
- if self.generator_seed is None
- else torch.Generator(device=self.pipe.device).manual_seed(self.generator_seed)
- ),
- **extra,
- )
- except NotImplementedError:
- pass
- except Exception as error:
- raise error
- step = len(prompts)
- total_cnt += step
- pbar.update(step)
- if total_cnt >= nsamples:
- break
-
- if total_cnt == 0:
- raise RuntimeError(
- f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
- f"dataset or decrease the sequence length"
- )
- elif total_cnt < nsamples:
- logger.warning(
- f"Insufficient number of samples collected may affect the quantization. "
- f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
- )
- if total_cnt < self.batch_size:
- raise ValueError(
- f"valid samples is less than batch_size({self.batch_size}),"
- " please adjust self.batch_size or seqlen."
- )
- max_len = (total_cnt // self.batch_size) * self.batch_size
- for k, v in self.inputs.items():
- for key in v:
- if isinstance(v[key], list) and len(v[key]) == total_cnt:
- self.inputs[k][key] = v[key][:max_len]
-
- # ------------------------------------------------------------------
- # Saving
- # ------------------------------------------------------------------
-
- @staticmethod
- def _flatten_nested_component_dir(component_output_dir: str, component_name: str) -> None:
- """Fix accidental nested save layouts (e.g. transformer/transformer/config.json).
-
- Some model config savers may create a same-name nested directory under the
- component output path. Flatten it so pipeline loaders find config files in
- the expected component root.
- """
- model_markers = (
- "config.json",
- "generation_config.json",
- "model.safetensors",
- "model.safetensors.index.json",
- )
-
- nested_dir = os.path.join(component_output_dir, component_name)
-
- # Some exporters write into a single nested model folder (for example,
- # vision_language_encoder/transformer). If same-name nesting does not
- # exist, try to detect this pattern and flatten it as well.
- if not os.path.isdir(nested_dir):
- child_dirs = [
- os.path.join(component_output_dir, name)
- for name in os.listdir(component_output_dir)
- if os.path.isdir(os.path.join(component_output_dir, name))
- ]
- has_model_files_at_root = any(
- os.path.exists(os.path.join(component_output_dir, marker)) for marker in model_markers
- )
- if len(child_dirs) == 1 and not has_model_files_at_root:
- candidate = child_dirs[0]
- has_model_files_in_child = any(
- os.path.exists(os.path.join(candidate, marker)) for marker in model_markers
- )
- if has_model_files_in_child:
- nested_dir = candidate
- else:
- return
- else:
- return
-
- moved = 0
- for entry in os.listdir(nested_dir):
- src = os.path.join(nested_dir, entry)
- dst = os.path.join(component_output_dir, entry)
- if os.path.exists(dst):
- continue
- shutil.move(src, dst)
- moved += 1
-
- if moved > 0:
- logger.warning(
- "Flattened nested component directory %s -> %s",
- nested_dir,
- component_output_dir,
- )
-
- # Remove the nested directory when empty.
- if os.path.isdir(nested_dir) and len(os.listdir(nested_dir)) == 0:
- os.rmdir(nested_dir)
-
- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
- """Save both quantized AR and DiT models into a pipeline directory structure.
-
- The output directory mirrors the original pipeline layout::
-
- output_dir/
- model_index.json
- <ar_component>/ (quantized AR model)
- transformer/ (quantized DiT model)
- ... (unchanged auxiliary components)
-
- Args:
- format: Export format for both the AR and DiT components.
- """
- if output_dir is None:
- logger.warning("output_dir is None, skipping save")
- return
-
- from auto_round.compressors.base import BaseCompressor
- from auto_round.formats import get_formats
-
- saved_formats = self.formats # preserve original
-
- # Save DiT
- if self.quant_dit:
- dit_subdir = "transformer"
- logger.info(f"Saving quantized DiT model ({dit_subdir}) [format={format}]")
- dit_output_dir = os.path.join(output_dir, dit_subdir)
- os.makedirs(dit_output_dir, exist_ok=True)
-
- self.model = self.dit_model
- if hasattr(self, "dit_layer_config"):
- self.layer_config = self.dit_layer_config
-
- self.formats = get_formats(format, self)
- BaseCompressor.save_quantized(self, output_dir=dit_output_dir, format=format, inplace=inplace, **kwargs)
- self._flatten_nested_component_dir(dit_output_dir, dit_subdir)
-
- # Save AR
- if self.quant_ar and self.ar_model is not None:
- ar_subdir = self.ar_component_name # e.g. "vision_language_encoder"
- logger.info(f"Saving quantized AR model ({ar_subdir}) [format={format}]")
- ar_output_dir = os.path.join(output_dir, ar_subdir)
- os.makedirs(ar_output_dir, exist_ok=True)
-
- self.model = self.ar_model
- if hasattr(self, "ar_layer_config"):
- self.layer_config = self.ar_layer_config
-
- # Swap serialization attributes from the AR compressor so that
- # BaseCompressor.save_quantized builds the correct config.
- ar_set = getattr(self, "_ar_serialization", {})
- saved_attrs = {}
- for k, v in ar_set.items():
- saved_attrs[k] = getattr(self, k, None)
- setattr(self, k, v)
-
- self.formats = get_formats(format, self)
- BaseCompressor.save_quantized(self, output_dir=ar_output_dir, format=format, inplace=inplace, **kwargs)
- self._flatten_nested_component_dir(ar_output_dir, ar_subdir)
-
- # Restore DiT serialization attributes
- for k, v in saved_attrs.items():
- setattr(self, k, v)
-
- self.formats = saved_formats
- self._save_pipeline_metadata(output_dir)
- self.model = self.dit_model
- logger.info(f"Full hybrid quantized model saved to {output_dir}")
-
- def _save_pipeline_metadata(self, output_dir):
- """Save model_index.json and auxiliary pipeline components."""
- src_path = getattr(getattr(self.pipe, "config", None), "_name_or_path", None) or getattr(
- self.pipe, "name_or_path", None
- )
- if src_path and os.path.exists(os.path.join(src_path, "model_index.json")):
- import shutil
-
- dst_index = os.path.join(output_dir, "model_index.json")
- if not os.path.exists(dst_index):
- shutil.copy2(os.path.join(src_path, "model_index.json"), dst_index)
-
- # Save non-quantized pipeline components so the exported directory remains
- # loadable as a complete diffusers pipeline even when only one branch is quantized.
- component_names = [
- "scheduler",
- "tokenizer",
- "processor",
- "vae",
- "text_encoder",
- ]
- if not self.quant_ar and self.ar_component_name is not None:
- component_names.append(self.ar_component_name)
- if not self.quant_dit:
- component_names.append("transformer")
-
- for component_name in component_names:
- component = getattr(self.pipe, component_name, None)
- if component is None:
- continue
- component_dir = os.path.join(output_dir, component_name)
- if os.path.exists(component_dir):
- continue
- try:
- if hasattr(component, "save_pretrained"):
- component.save_pretrained(component_dir)
- except Exception as e:
- logger.warning(f"Failed to save {component_name}: {e}")
-
- def quantize_and_save(
- self,
- output_dir: str = "tmp_autoround",
- format: str = "auto_round",
- inplace: bool = True,
- **kwargs,
- ):
- """Quantize both components and save the complete pipeline.
-
- Args:
- format: Export format for both the AR and DiT components.
- """
- from auto_round.formats import get_formats
-
- format_list = get_formats(format, self)
- self.formats = format_list
- self.inplace = inplace
- # Keep orig_output_dir as None so _adjust_immediate_packing_and_saving()
- # disables immediate saving — diffusers models must go through
- # model.save_pretrained() to get correct weight file names.
- self.orig_output_dir = None
-
- self.quantize()
- self.save_quantized(
- output_dir,
- format=format,
- inplace=inplace,
- **kwargs,
- )
- logger.info(f"Hybrid quantized model saved to {output_dir}")
- return self.model, [output_dir]
diff --git a/auto_round/compressors_new/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.py
similarity index 52%
rename from auto_round/compressors_new/diffusion_mixin.py
rename to auto_round/compressors/diffusion_mixin.py
index 515839c1f..359b3e0f3 100644
--- a/auto_round/compressors_new/diffusion_mixin.py
+++ b/auto_round/compressors/diffusion_mixin.py
@@ -15,22 +15,20 @@ import os
from typing import Union
import torch
-from tqdm import tqdm
from auto_round.logger import logger
-from auto_round.utils.model import wrap_block_forward_positional_to_kwargs
class DiffusionMixin:
"""Diffusion-specific functionality mixin.
This mixin adds diffusion model-specific functionality to any compressor
- (CalibCompressor, ZeroShotCompressor, ImatrixCompressor, etc). It handles
+ (DataDrivenCompressor, ZeroShotCompressor, ImatrixCompressor, etc). It handles
diffusion models (like Stable Diffusion, FLUX) that require special pipeline
handling and data generation logic.
Can be combined with:
- - CalibCompressor (for AutoRound with calibration)
+ - DataDrivenCompressor (for AutoRound with calibration)
- ImatrixCompressor (for RTN with importance matrix)
- ZeroShotCompressor (for basic RTN)
@@ -69,14 +67,13 @@ class DiffusionMixin:
iters = 200
if iters > 0:
- batch_size = kwargs.get("batch_size", None)
- if batch_size is None and _alg_cfg is not None:
- cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
- for cfg in cfgs:
- if hasattr(cfg, "batch_size") and cfg.batch_size is not None:
- batch_size = cfg.batch_size
- break
- if batch_size is not None and batch_size != 1:
+ # ``batch_size`` is owned by the compressor / shared
+ # CalibrationState; it only comes from kwargs now (entry.py forwards
+ # it explicitly). AlgConfig no longer carries it. Treat a missing
+ # value as ``BaseCompressor``'s default (8) so the reset path always
+ # triggers when the user didn't explicitly opt out.
+ batch_size = kwargs.get("batch_size", 8)
+ if batch_size != 1:
grad_acc = kwargs.get("gradient_accumulate_steps", 1)
if _alg_cfg is not None:
cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
@@ -90,8 +87,6 @@ class DiffusionMixin:
if _alg_cfg is not None:
cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
for cfg in cfgs:
- if hasattr(cfg, "batch_size"):
- cfg.batch_size = 1
if hasattr(cfg, "gradient_accumulate_steps"):
cfg.gradient_accumulate_steps = new_grad_acc
logger.warning(
@@ -100,7 +95,7 @@ class DiffusionMixin:
f"because batch_size={batch_size} cannot be used for calibrating non-text modules."
)
- # Call parent class __init__ (will be CalibCompressor, ImatrixCompressor, etc)
+ # Call parent class __init__ (will be DataDrivenCompressor, ImatrixCompressor, etc)
super().__init__(*args, **kwargs)
# Mirror old-arch DiffusionCompressor._align_device_and_dtype: unconditionally
@@ -116,121 +111,15 @@ class DiffusionMixin:
if not is_nextstep:
pipe.to(model.dtype)
- def _get_block_forward_func(self, name: str):
- """Diffusion models pass positional args; wrap the base forward func accordingly.
+ def _get_calibrator_kind(self) -> str:
+ """Select the diffusion calibration strategy.
- The MRO guarantees that super() resolves to CalibCompressor._get_block_forward_func,
- mirroring the old-arch pattern in compressors/diffusion/compressor.py.
+ ``DiffusionCalibrator`` lives at
+ :mod:`auto_round.calibration.diffusion` and owns what used to be
+ ``DiffusionMixin.calib`` / ``_get_block_forward_func`` /
+ ``_should_stop_cache_forward``.
"""
- return wrap_block_forward_positional_to_kwargs(super()._get_block_forward_func(name))
-
- def _should_stop_cache_forward(self, name: str) -> bool:
- """Diffusion models must run all denoising steps to collect enough inputs.
-
- Mirrors old-arch DiffusionCompressor._should_stop_cache_forward which always
- returns False so the pipeline never exits early after the first block hit.
- Without this, CalibCompressor._should_stop_cache_forward would stop after the
- first inference step, yielding only nsamples inputs instead of
- nsamples * num_inference_steps.
- """
- return False
-
- @torch.no_grad()
- def calib(self, nsamples, bs):
- """Perform diffusion-specific calibration for quantization.
-
- Override parent's calib method to use diffusion dataset loading logic.
- The diffusion pipeline is read from ``self.model_context.pipe``.
- """
- from auto_round.compressors.diffusion.dataset import get_diffusion_dataloader
-
- pipe = self.model_context.pipe
- if pipe is None:
- raise ValueError(
- "Diffusion pipeline not found in model_context. " "Ensure the model was loaded as a diffusion model."
- )
-
- logger.warning(
- "Diffusion model will catch nsamples * num_inference_steps inputs, "
- "you can reduce nsamples or num_inference_steps if OOM or take too much time."
- )
- if isinstance(self.dataset, str):
- dataset = self.dataset.replace(" ", "")
- self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_diffusion_dataloader(
- dataset=dataset,
- bs=self.batch_size,
- seed=self.seed,
- nsamples=self.nsamples,
- gradient_accumulate_steps=self.gradient_accumulate_steps,
- )
- else:
- self.dataloader = self.dataset
- total_cnt = 0
-
- total = nsamples if not hasattr(self.dataloader, "len") else min(nsamples, len(self.dataloader))
-
- if (
- hasattr(self.model, "hf_device_map")
- and len(self.model.hf_device_map) > 1
- and pipe.device != self.model.device
- and torch.device(self.model.device).type in ["cuda", "xpu"]
- ):
- logger.error(
- "Diffusion model is activated sequential model offloading, it will crash during moving to GPU/XPU. "
- "Please use model path for quantization or "
- "move the pipeline object to GPU/XPU before passing them into API."
- )
- exit(-1)
-
- if pipe.device != self.model.device:
- pipe.to(self.model.device)
- with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:
- for ids, prompts in self.dataloader:
- if isinstance(prompts, tuple):
- prompts = list(prompts)
- try:
- pipe(
- prompts,
- guidance_scale=self.guidance_scale,
- num_inference_steps=self.num_inference_steps,
- generator=(
- None
- if self.generator_seed is None
- else torch.Generator(device=pipe.device).manual_seed(self.generator_seed)
- ),
- )
- except NotImplementedError:
- pass
- except Exception as error:
- raise error
- step = len(prompts)
- total_cnt += step
- pbar.update(step)
- if total_cnt >= nsamples:
- break
- if total_cnt == 0:
- logger.error(
- f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
- f"dataset or decease the sequence length"
- )
- exit(-1)
- elif total_cnt < nsamples:
- logger.warning(
- f"Insufficient number of samples collected may affect the quantization. "
- f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
- )
- if total_cnt < self.batch_size:
- raise ValueError(
- f"valid samples is less than batch_size({self.batch_size}),"
- " please adjust self.batch_size or seqlen."
- )
- max_len = (total_cnt // self.batch_size) * self.batch_size
- for k, v in self.inputs.items():
- for key in v:
- if isinstance(v[key], list) and len(v[key]) == total_cnt:
- self.inputs[k][key] = v[key][:max_len]
-
- # torch.cuda.empty_cache()
+ return "diffusion"
def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
"""Save the quantized model to the specified output directory in the specified format.
diff --git a/auto_round/compressors_new/entry.py b/auto_round/compressors/entry.py
similarity index 92%
rename from auto_round/compressors_new/entry.py
rename to auto_round/compressors/entry.py
index c99f0977b..23b76cf55 100644
--- a/auto_round/compressors_new/entry.py
+++ b/auto_round/compressors/entry.py
@@ -11,9 +11,9 @@ from auto_round.algorithms.quantization.rtn.config import RTNConfig
from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
from auto_round.algorithms.transforms.rotation.config import RotationConfig as _NewArchRotationConfig
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
-from auto_round.compressors_new.calib import CalibCompressor, CalibratedRTNCompressor
-from auto_round.compressors_new.utils import check_need_act_calibration
-from auto_round.compressors_new.zero_shot import ZeroShotCompressor
+from auto_round.compressors.data_driven import CalibratedRTNCompressor, DataDrivenCompressor
+from auto_round.compressors.utils import check_need_act_calibration
+from auto_round.compressors.zero_shot import ZeroShotCompressor
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme, _parse_scheme
@@ -94,11 +94,11 @@ def _get_compressor_class(model_type: str, base_cls: type) -> type:
if key in _COMPRESSOR_REGISTRY:
return _COMPRESSOR_REGISTRY[key]
if model_type == "mllm":
- from auto_round.compressors_new.mllm_mixin import MLLMMixin
+ from auto_round.compressors.mllm_mixin import MLLMMixin
mixin = MLLMMixin
elif model_type == "diffusion":
- from auto_round.compressors_new.diffusion_mixin import DiffusionMixin
+ from auto_round.compressors.diffusion_mixin import DiffusionMixin
mixin = DiffusionMixin
else:
@@ -159,8 +159,6 @@ def detect_model_type(model):
class AutoRound(object):
- SKIP_ARGS = ("local_args", "kwargs", "cls", "alg_configs", "quant_config", "quant_configs")
-
# Mapping from string alias to config class (and optional defaults override).
_CONFIG_ALIASES: dict[str, type] = {
"sign_round": SignRoundConfig,
@@ -223,8 +221,24 @@ class AutoRound(object):
# callers get ValueError/NotImplementedError on construction, not deferred.
_eager_validate_scheme(quant_config, scheme)
- # using different compressor base on AlgConfigs
- local_args = {k: v for k, v in locals().items() if k not in cls.SKIP_ARGS}
+ # Explicitly build the dict of constructor args to forward to the
+ # compressor. This avoids the fragile locals()-based approach that
+ # required a growing SKIP_ARGS blocklist.
+ local_args = dict(
+ model=model,
+ tokenizer=tokenizer,
+ platform=platform,
+ format=format,
+ scheme=scheme,
+ low_gpu_mem_usage=low_gpu_mem_usage,
+ device_map=device_map,
+ enable_torch_compile=enable_torch_compile,
+ seed=seed,
+ low_cpu_mem_usage=low_cpu_mem_usage,
+ layer_config=layer_config,
+ nsamples=nsamples,
+ seqlen=seqlen,
+ )
# Detect model type to determine if we need special compressor
model_type = detect_model_type(model)
@@ -235,8 +249,18 @@ class AutoRound(object):
if has_multimodal_assets and model_type != "mllm":
model_type = "mllm"
+ # Pop kwargs that are only consumed by specific Mixins so they don't
+ # leak through to BaseCompressor as unrecognized keys.
+ if model_type != "diffusion":
+ for _k in ("guidance_scale", "num_inference_steps", "generator_seed"):
+ kwargs.pop(_k, None)
+ if model_type != "mllm":
+ for _k in ("processor", "image_processor", "template", "extra_data_dir", "quant_nontext_module"):
+ kwargs.pop(_k, None)
+ kwargs.pop("disable_opt_rtn", None) # consumed by RTN routing above, not a compressor param
+
if isinstance(quant_config, SignRoundConfig):
- return _get_compressor_class(model_type, CalibCompressor)(alg_configs, **local_args, **kwargs)
+ return _get_compressor_class(model_type, DataDrivenCompressor)(alg_configs, **local_args, **kwargs)
elif isinstance(quant_config, RTNConfig):
enable_imatrix = False
@@ -323,7 +347,7 @@ class AutoRoundCompatible:
Example:
>>> # Old API - still works
- >>> from auto_round.compressors_new.entry import AutoRoundCompatible
+ >>> from auto_round.compressors.entry import AutoRoundCompatible
>>> autoround = AutoRoundCompatible(
... model="/models/opt-125m",
... bits=4,
@@ -407,7 +431,7 @@ class AutoRoundCompatible:
# ---- Model-free fast-path detection --------------------------------
if is_model_free_route(model, scheme, iters, kwargs.get("disable_opt_rtn"), kwargs):
- from auto_round.compressors_new.model_free import ModelFreeCompressor
+ from auto_round.compressors.model_free import ModelFreeCompressor
if not isinstance(model, str):
raise ValueError("model_free=True requires `model` to be a HuggingFace ID or local path string.")
@@ -455,8 +479,6 @@ class AutoRoundCompatible:
act_data_type=act_data_type,
act_dynamic=act_dynamic,
disable_opt_rtn=disable_opt_rtn,
- # for optRTN
- batch_size=batch_size,
**common_config_kwargs,
)
else:
@@ -469,7 +491,6 @@ class AutoRoundCompatible:
config = SignRoundConfig(
iters=iters,
- batch_size=batch_size,
gradient_accumulate_steps=gradient_accumulate_steps,
bits=bits,
group_size=group_size,
@@ -546,6 +567,7 @@ class AutoRoundCompatible:
layer_config=layer_config,
nsamples=nsamples,
seqlen=seqlen,
+ batch_size=batch_size,
# MLLM parameters
processor=processor,
image_processor=image_processor,
diff --git a/auto_round/compressors/mllm/README.md b/auto_round/compressors/mllm/README.md
index 343e5d52a..ee44675db 100644
--- a/auto_round/compressors/mllm/README.md
+++ b/auto_round/compressors/mllm/README.md
@@ -96,7 +96,8 @@ liuhaotian/llava_instruct_80k", "liuhaotian/llava_instruct_150k" or a file path
For autoround MLLMs, using Template to customize different operations for different models. User can use template to support new model which not in support list.
```python
-from auto_round.mllm.template import _register_template
+from auto_round.compressors.mllm.processor import PROCESSORS
+from auto_round.compressors.mllm.template import _register_template
model_type = model.config.model_type
_register_template(model_type=model_type, default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])
@@ -106,7 +107,7 @@ _register_template(model_type=model_type, default_dataset="NeelNanda/pile-10k",
Processor is callback interface for calling different processors, such as texts or images processors, for MLLMs. User
can define own processor and use registration function to declare. For more information, please refer to the relevant
-code in ```auto_round/mllm/processor.py```.
+code in ```auto_round/compressors/mllm/processor.py```.
</details>
diff --git a/auto_round/compressors/mllm/__init__.py b/auto_round/compressors/mllm/__init__.py
index e92f378d4..1abc19f37 100644
--- a/auto_round/compressors/mllm/__init__.py
+++ b/auto_round/compressors/mllm/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2024 Intel Corporation
+# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,7 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from auto_round.compressors.mllm.dataset import get_mllm_dataloader
-from auto_round.compressors.mllm.template import Template, get_template, TEMPLATES
-from auto_round.compressors.mllm.compressor import MLLMCompressor
-from auto_round.compressors.mllm.eval import mllm_eval, lmms_eval
+from auto_round.compressors.mllm.dataset import MLLM_DATASET, get_mllm_dataloader
+from auto_round.compressors.mllm.processor import PROCESSORS, BasicProcessor, register_processor
+from auto_round.compressors.mllm.template import TEMPLATES, Template, get_template
+
+__all__ = [
+ "BasicProcessor",
+ "MLLM_DATASET",
+ "PROCESSORS",
+ "TEMPLATES",
+ "Template",
+ "get_mllm_dataloader",
+ "get_template",
+ "register_processor",
+]
diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.py
deleted file mode 100644
index 5b9f64ca8..000000000
--- a/auto_round/compressors/mllm/compressor.py
+++ /dev/null
@@ -1,539 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import importlib.util
-import traceback
-from copy import deepcopy
-from typing import Union
-
-import accelerate
-import torch
-from accelerate.big_modeling import dispatch_model, infer_auto_device_map
-from tqdm import tqdm
-
-from auto_round.compressors.base import BaseCompressor
-from auto_round.compressors.mllm.dataset import get_mllm_dataloader
-from auto_round.compressors.mllm.template import TEMPLATES, Template, get_template
-from auto_round.logger import logger
-from auto_round.schemes import QuantizationScheme
-from auto_round.special_model_handler import (
- MISTRAL_3_2_MODELS,
- NOT_SUPPORT_ONLY_TEXT_MODELS,
- SUPPORT_ONLY_TEXT_MODELS,
- _handle_special_model,
-)
-from auto_round.utils import (
- check_to_quantized,
- clear_memory,
- detect_device,
- extract_block_names_to_str,
- find_matching_blocks,
- get_block_names,
- get_max_vram,
- mllm_load_model,
- mv_module_from_gpu,
- to_device,
- to_dtype,
-)
-
-
-def _only_text_test(model, tokenizer, device, model_type):
- """Test if the model whether can use text-only datasets."""
-
- if model_type in SUPPORT_ONLY_TEXT_MODELS: # save time
- return True
-
- if model_type in NOT_SUPPORT_ONLY_TEXT_MODELS:
- return False
-
- new_tokenizer = deepcopy(tokenizer)
- device = detect_device(device)
- text = ["only text", "test"]
- new_tokenizer.padding_side = "left"
- if hasattr(new_tokenizer, "pad_token") and new_tokenizer.pad_token is None:
- new_tokenizer.pad_token = new_tokenizer.eos_token
- inputs = new_tokenizer(text, return_tensors="pt", padding=True, truncation=True)
-
- # Estimate model size and check if it fits in GPU memory.
- # When the model is too large, we skip the GPU transfer and test on CPU only.
- use_gpu = True if device != "cpu" else False
- if device != "cpu" and torch.cuda.is_available():
- model_bytes = sum(p.numel() * p.element_size() for p in model.parameters())
- dev_idx_str = device.split(":")[-1] if ":" in device else "0"
- if dev_idx_str.isdigit():
- dev_idx = int(dev_idx_str)
- if 0 <= dev_idx < torch.cuda.device_count():
- free_bytes, _ = torch.cuda.mem_get_info(dev_idx)
- if model_bytes > free_bytes * 0.9:
- use_gpu = False
-
- if use_gpu:
- try:
- inputs = inputs.to(device)
- model = model.to(device)
- model(**inputs)
- return True
- except RuntimeError as e:
- model = model.to("cpu")
- inputs = inputs.to("cpu")
- if torch.cuda.is_available():
- torch.cuda.empty_cache()
- try:
- model(**inputs)
- except Exception:
- return False
- return True
- except Exception as e:
- if torch.cuda.is_available():
- torch.cuda.empty_cache()
- return False
-
- # Model too large for GPU — run the test on CPU directly
- try:
- inputs = inputs.to("cpu")
- model = model.to("cpu")
- model(**inputs)
- return True
- except Exception:
- return False
-
-
-class MLLMCompressor(BaseCompressor):
- """Class for automatic rounding-based quantization with MLLMs.
-
- Args:
- model: The PyTorch model to be quantized.
- tokenizer: An optional tokenizer for processing input data.
- platform (str): The platform to load pretrained moded, options: ["hf", "model_scope"]
- processor: Any multi-modal model will require an object to encode or
- decode the data that groups several modalities (among text, vision and audio).
- image_processor: Image processor for special model like llava.
- bits (int): Number of bits for quantization (default is 4).
- group_size (int): Size of the quantization group (default is 128).
- sym (bool): Whether sym to be used (default is True).
- layer_config (dict): Configuration for weight quantization (default is None).
- batch_size (int): Batch size for training (default is 8).
- amp (bool): Whether to use automatic mixed precision (default is True).
- device: The device to be used for training (default is "auto").
- lr_scheduler: The learning rate scheduler to be used.
- dataset: The path or name of the calib dataset.
- extra_data_dir: The path of extra data such as images, audio and videos.
- template: The path or name of template used to specify process for different MLLMs.
- quant_nontext_module: Whether to quantize nontext module.
- enable_quanted_input (bool): Whether to use quantized input data (default is True).
- enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
- lr (float): The learning rate (default is 0.005).
- minmax_lr (float): The learning rate for min-max tuning (default is None).
- low_gpu_mem_usage (bool): Whether to use low GPU memory (default is False).
- iters (int): Number of iterations (default is 200).
- seqlen (int): Length of the sequence.
- nsamples (int): Number of samples (default is 128).
- sampler (str): The sampling method (default is "rand").
- seed (int): The random seed (default is 42).s
- nblocks (int): Number of blocks (default is 1).
- gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
- not_use_best_mse (bool): Whether to use mean squared error (default is False).
- dynamic_max_gap (int): The dynamic maximum gap (default is -1).
- data_type (str): The data type to be used (default is "int").
- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
- have different choices.
- act_bits (int): Number of bits for activation quantization. Default is 32.
- act_group_size (int): Group size for activation quantization. Default is None.
- act_sym (bool): Whether to use symmetric activation quantization. Default is None.
- act_dynamic (bool): Whether to use dynamic activation quantization. Default is True.
- to_quant_block_names (str|list): A string or list whose elements are list of
- block's layer names to be quantized.
- enable_torch_compile (bool): Whether to enable torch compile to optimize quant_block/layer
- **kwargs: Additional keyword arguments.
- """
-
- bits: int | None
- group_size: int | None
- sym: bool | None
- data_type: str | None
- act_bits: int | None
- act_group_size: int | None
- act_sym: bool | None
- act_data_type: str | None
- act_dynamic: bool | None
- super_bits: int | None
- super_group_size: int | None
-
- def __init__(
- self,
- model: Union[torch.nn.Module, str],
- tokenizer=None,
- platform: str = "hf",
- processor=None,
- image_processor=None,
- scheme: Union[str, dict, QuantizationScheme] = "W4A16",
- layer_config: dict[str, Union[str, dict, QuantizationScheme]] = None,
- dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = "NeelNanda/pile-10k",
- quant_nontext_module: bool = False,
- iters: int = 200,
- seqlen: int = 2048,
- nsamples: int = 128,
- batch_size: int = 8,
- gradient_accumulate_steps: int = 1,
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- **kwargs,
- ):
-
- extra_data_dir = kwargs.pop("extra_data_dir", None)
- template = kwargs.pop("template", None)
- model_dtype = kwargs.pop("model_dtype", None)
-
- to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)
- if device_map is None:
- device_map = 0
- self._set_device(device_map)
-
- if isinstance(model, str):
- model, processor, tokenizer, image_processor = mllm_load_model(
- model, platform=platform, device=self.device, model_dtype=model_dtype
- )
-
- self.model = model
- quant_nontext_module = self._check_quant_nontext(layer_config, quant_nontext_module)
- if quant_nontext_module and iters > 0:
- logger.warning(
- "quant_nontext_module is experimental feature and not work well for all models,"
- "please remove --quant_nontext_module and retry if you meet some errors."
- )
- import importlib.util
-
- missing_libs = []
- required_libs = {
- "pillow": "PIL",
- "torchvision": "torchvision",
- }
- for pip_name, module_name in required_libs.items():
- if importlib.util.find_spec(module_name) is None:
- missing_libs.append(pip_name)
- if len(missing_libs) > 0:
- logger.error(
- f"{', '.join(missing_libs)} are required for quantizing non-text modules,"
- f" please install them with `pip install {' '.join(missing_libs)}`",
- )
- all_blocks = get_block_names(model, quant_nontext_module)
- self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
- if to_quant_block_names is None:
- to_quant_block_names = extract_block_names_to_str(self.quant_block_list)
- self.extra_data_dir = extra_data_dir
- self.quant_nontext_module = quant_nontext_module
- self.processor = processor
- self.image_processor = image_processor
- from transformers import PreTrainedModel
-
- # if model is not the object of transformers PreTrainedModel, there maybe no config attribute
- if isinstance(model, PreTrainedModel) and model.config.model_type == "llava":
- template = "default"
- if hasattr(model, "name_or_path") and any([name in model.name_or_path for name in MISTRAL_3_2_MODELS]):
- template = "mistral3_2"
- if iters > 0:
- # TODO: Remove after fixing https://github.com/huggingface/transformers/issues/43005
- model.config.model_type = model.config.to_dict()["model_type"]
-
- # try default template if model_type not in preset templates
- if template is None and model.config.model_type not in TEMPLATES:
- self.template = "default"
- else:
- self.template = template if template is not None else model.config.model_type
- if not isinstance(dataset, torch.utils.data.DataLoader):
- self.template = get_template(
- self.template,
- model=model,
- tokenizer=tokenizer,
- processor=processor,
- image_processor=image_processor,
- use_rtn=iters == 0,
- quiet=not self.quant_nontext_module,
- )
- dataset = self.template.default_dataset if dataset is None else dataset
- else:
- self.template = None
-
- model = _handle_special_model(model)
-
- from auto_round.calib_dataset import CALIB_DATASETS
-
- from .dataset import MLLM_DATASET
-
- if iters > 0 and isinstance(dataset, str) and dataset in CALIB_DATASETS.keys():
- if quant_nontext_module:
- logger.warning(
- "Text only dataset cannot be used for calibrating non-text modules,"
- " switching to liuhaotian/llava_conv_58k"
- )
- dataset = "liuhaotian/llava_conv_58k"
- elif self.template is not None and not _only_text_test(
- model,
- tokenizer,
- self.device,
- getattr(getattr(model, "config", None), "model_type", None) or self.template.model_type,
- ):
- logger.warning(
- f"{model.config.model_type} does not support for {dataset},"
- " will use liuhaotian/llava_conv_58k with default config as an alternative."
- )
- dataset = "liuhaotian/llava_conv_58k"
-
- if dataset in MLLM_DATASET.keys():
- truncation = False
- seqlen = 512 if seqlen is None else seqlen
- if batch_size != 1:
- logger.warning(
- f"reset batch_size({batch_size}) to 1 and "
- f"gradient_accumulate_steps({gradient_accumulate_steps}) "
- f"to {batch_size * gradient_accumulate_steps}, "
- f"because batch_size={batch_size} cannot be used for {dataset}"
- )
- gradient_accumulate_steps = batch_size * gradient_accumulate_steps
- batch_size = 1
- if quant_nontext_module and batch_size != 1:
- logger.warning(
- f"reset batch_size({batch_size}) to 1 and "
- f"gradient_accumulate_steps({gradient_accumulate_steps}) "
- f"to {batch_size * gradient_accumulate_steps}, "
- f"because batch_size={batch_size} cannot be used for calibrating non-text modules."
- )
- gradient_accumulate_steps = batch_size * gradient_accumulate_steps
- batch_size = 1
- seqlen = 2048 if seqlen is None else seqlen
- truncation = True
- self.truncation = truncation
-
- if nsamples % batch_size != 0:
- nsamples = (nsamples // batch_size + 1) * batch_size
- logger.warning(f"'nsamples' is not divisible by 'batch_size', will adjusted to {nsamples}")
-
- kwargs["mllm"] = True
- super(MLLMCompressor, self).__init__(
- model=model,
- tokenizer=tokenizer,
- platform=platform,
- scheme=scheme,
- layer_config=layer_config,
- dataset=dataset,
- iters=iters,
- seqlen=seqlen,
- nsamples=nsamples,
- batch_size=batch_size,
- gradient_accumulate_steps=gradient_accumulate_steps,
- low_gpu_mem_usage=low_gpu_mem_usage,
- device_map=device_map,
- enable_torch_compile=enable_torch_compile,
- seed=seed,
- to_quant_block_names=to_quant_block_names,
- **kwargs,
- )
-
- def calib(self, nsamples, bs):
- """Perform calibration for quantization.
-
- This method calibrates the model for quantization by processing a specified
- number of samples from the calibration dataset. It ensures that the data is
- properly formatted and feeds it to the model. If the number of samples processed
- is less than the specified number, it logs a warning. If no samples are processed,
- it logs an error and exits.
- Args:
- nsamples (int): The number of samples to use for calibration.
- bs (int): The number of samples to use for calibration
- """
- if isinstance(self.dataset, str):
- dataset = self.dataset.replace(" ", "")
- (
- self.dataloader,
- self.batch_size,
- self.seqlen,
- self.gradient_accumulate_steps,
- ) = get_mllm_dataloader(
- template=self.template,
- model=self.model,
- tokenizer=self.tokenizer,
- processor=self.processor,
- image_processor=self.image_processor,
- dataset=dataset,
- extra_data_dir=self.extra_data_dir,
- seqlen=self.seqlen,
- bs=self.batch_size,
- seed=self.seed,
- truncation=self.truncation,
- nsamples=self.nsamples,
- gradient_accumulate_steps=self.gradient_accumulate_steps,
- quant_nontext_module=self.quant_nontext_module,
- )
- else:
- self.dataloader = self.dataset
-
- total_cnt = 0
- calib_kwargs = {"use_cache": False}
- total = nsamples if not hasattr(self.dataloader, "len") else min(nsamples, len(self.dataloader))
- with tqdm(range(1, total + 1), desc="cache block inputs") as pbar:
- for data in self.dataloader:
- if data is None:
- pbar.update(1)
- continue
-
- if isinstance(data, torch.Tensor):
- input_ids = data.to(self.model.device)
- data_new = input_ids
- elif isinstance(data, str):
- if self.tokenizer is None:
- logger.error("please provide tokenizer for string input")
- exit()
- data = self.template.processor.get_input(
- text=data,
- images=None,
- max_length=self.seqlen,
- squeeze=False,
- )
- data_new = {k: v.to(self.model.device) for k, v in data.items()}
- input_ids = data_new["input_ids"]
- elif isinstance(data, dict) and "text" in data.keys():
- text = data["text"]
- if isinstance(text, dict):
- text = [text]
- input_text = self.template._encode(text)
- data = self.template.processor.get_input(
- text=input_text,
- images=data["image"],
- max_length=self.seqlen,
- squeeze=False,
- )
- data_new = {}
- for key, value in data.items():
- if isinstance(value, torch.Tensor):
- tensor_value = value
- else:
- tensor_value = torch.as_tensor(value)
- tensor_value = to_device(tensor_value, self.model.device)
- if key == "images":
- tensor_value = to_dtype(tensor_value, self.model.dtype)
- data_new[key] = tensor_value
- input_ids = data_new["input_ids"]
- elif isinstance(data, (tuple, list)):
- data_new = to_device(data, self.model.device)
- input_ids = data_new[0]
- else:
- data_new = {}
- for key, value in data.items():
- tensor_value = to_device(value, self.model.device)
- if key in ["images", "pixel_values"]:
- tensor_value = to_dtype(tensor_value, self.model.dtype)
- data_new[key] = tensor_value
- input_ids = data_new["input_ids"] if "input_ids" in data_new else data_new["inputs_embeds"]
-
- if input_ids.shape[-1] < self.seqlen:
- pbar.update(1)
- continue
-
- try:
- if isinstance(data_new, torch.Tensor):
- self.model(data_new, **calib_kwargs)
- elif isinstance(data_new, (tuple, list)):
- self.model(*data_new, **calib_kwargs)
- else:
- self.model(**data_new, **calib_kwargs)
- except NotImplementedError:
- pass
- except Exception as error:
- raise error
-
- step = input_ids.shape[0] if len(input_ids.shape) > 1 else 1
- total_cnt += step
- pbar.update(step)
- if total_cnt >= nsamples:
- break
- if total_cnt == 0:
- logger.error(
- f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
- f"dataset or decease the sequence length"
- )
- exit(-1)
- elif total_cnt < nsamples:
- logger.warning(
- f"Insufficient number of samples collected may affect the quantization. "
- f"target samples count is {nsamples}, while valid samples count is {total_cnt}"
- )
- if total_cnt < self.batch_size:
- raise ValueError(
- f"valid samples is less than batch_size({self.batch_size}),"
- " please adjust self.batch_size or seqlen."
- )
- max_len = (total_cnt // self.batch_size) * self.batch_size
- for k, v in self.inputs.items():
- for key in v:
- if isinstance(v[key], list) and len(v[key]) == total_cnt:
- self.inputs[k][key] = v[key][:max_len]
-
- if hasattr(self.dataloader, "dataset") and hasattr(self.dataloader.dataset, "clear_runtime_cache"):
- self.dataloader.dataset.clear_runtime_cache()
-
- # torch.cuda.empty_cache()
-
- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
- """Save the quantized model to the specified output directory in the specified format.
-
- Args:
- output_dir (str, optional): The directory to save the quantized model. Defaults to None.
- format (str, optional): The format in which to save the model. Defaults to "auto_round".
- inplace (bool, optional): Whether to modify the model in place. Defaults to True.
- **kwargs: Additional keyword arguments specific to the export format.
-
- Returns:
- object: The compressed model object.
- """
- if self.processor is not None and not hasattr(self.processor, "chat_template"):
- self.processor.chat_template = None
- compressed_model = super().save_quantized(
- output_dir=output_dir,
- format=format,
- inplace=inplace,
- processor=self.processor,
- image_processor=self.image_processor,
- quant_nontext_module=self.quant_nontext_module if hasattr(self, "quant_nontext_module") else False,
- **kwargs,
- )
- return compressed_model
-
- def _check_quant_nontext(self, layer_config, quant_nontext_module):
- if not layer_config:
- return quant_nontext_module
- from auto_round.compressors.mllm.utils import VISUAL_KEYS
-
- for layer_name in layer_config.keys():
- for vlm_key in VISUAL_KEYS:
- if vlm_key in layer_name and check_to_quantized(layer_config[layer_name]):
- return True
- return quant_nontext_module
-
- def _immediate_pack(self, name: str):
- if not self.is_immediate_packing: # pylint: disable=E1101
- return
- self.formats[0].immediate_pack(
- name=name,
- model=self.model,
- device=self.device,
- output_dir=self._get_save_folder_name(self.formats[0]),
- mllm=self.mllm,
- layer_config=self.layer_config,
- tokenizer=self.tokenizer,
- processor=self.processor if hasattr(self, "processor") else None,
- image_processor=self.image_processor if hasattr(self, "image_processor") else None,
- quant_nontext_module=self.quant_nontext_module if hasattr(self, "quant_nontext_module") else False,
- )
diff --git a/auto_round/compressors/mllm/eval.py b/auto_round/compressors/mllm/eval.py
deleted file mode 100644
index 527ea2be4..000000000
--- a/auto_round/compressors/mllm/eval.py
+++ /dev/null
@@ -1,433 +0,0 @@
-# Copyright (c) 2024 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Copyright 2023 VLMEvalKit Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Copyright (c) 2024 LMMs-Lab
-
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-import json
-import os
-import time
-from functools import partial
-
-import numpy as np
-
-from auto_round.logger import logger
-from auto_round.utils import LazyImport
-
-vlmeval = LazyImport("vlmeval")
-
-MODEL_TYPE_TO_VLMEVAL_MODEL = {
- # model_name
- "Qwen-VL": dict(cls="QwenVL"),
- "Qwen-VL-Chat": dict(cls="QwenVLChat"),
- "Qwen2-VL": dict(cls="Qwen2VLChat", min_pixels=1280 * 28 * 28, max_pixels=16384 * 28 * 28, verbose=False),
- "Llama-3.2": dict(cls="llama_vision"),
- "Phi-3-vision": dict(cls="Phi3Vision"),
- "Phi-3.5-vision": dict(cls="Phi3_5Vision"),
- "llava_v1.5": dict(cls="LLaVA"),
- "llava_v1.6": dict(cls="LLaVA_Next"),
- "llava-onevision-qwen2": dict(cls="LLaVA_OneVision"),
- "cogvlm2": dict(cls="CogVlm"),
- "SliME": dict(cls="SliME"),
- "Eagle": dict(cls="Eagle"),
- "Molmo": dict(cls="molmo"),
- # config.model_type
- "qwen2_vl": dict(cls="Qwen2VLChat", min_pixels=1280 * 28 * 28, max_pixels=16384 * 28 * 28),
- "qwen": dict(cls="QwenVL"),
- "qwen_chat": dict(cls="QwenVLChat"),
- "llava": dict(cls="LLaVA"),
- "llava_next": dict(cls="LLaVA_Next"),
- "phi3_v": dict(cls="Phi3Vision"),
- "mllama": dict(cls="llama_vision"),
- "glm-4v-9b": dict(cls="GLM4v"),
-}
-
-
-def mllm_eval(
- pretrained_model_name_or_path: str,
- work_dir: str,
- dataset: list,
- data_store_dir: None,
- pack: bool = False,
- use_subtitle: bool = False,
- fps: float = -1,
- nframe: int = 8,
- rerun: bool = False,
- judge: bool = False,
- verbose: bool = False,
- mode: str = "all",
- ignore: bool = False,
-):
- model = None
- if data_store_dir is not None:
- if not os.path.exists(data_store_dir):
- oldmask = os.umask(000)
- os.makedirs(data_store_dir, mode=0o777)
- os.umask(oldmask)
- os.environ["LMUData"] = data_store_dir
-
- model_name = pretrained_model_name_or_path
- if "/" in model_name:
- model_name = model_name[:-1] if model_name[-1] == "/" else model_name
- model_name = model_name.split("/")[-1]
-
- if model_name in MODEL_TYPE_TO_VLMEVAL_MODEL:
- model_type = model_name
- else:
- model_type = None
- split_name = model_name.split("-")
- for i in range(len(split_name), 0, -1):
- tmp = "-".join(split_name[0:i])
- if tmp in MODEL_TYPE_TO_VLMEVAL_MODEL:
- model_type = tmp
- break
- if model_type is None:
- from transformers import AutoConfig
-
- config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=True)
- model_type = config.model_type
- if "chat" in model_name.lower():
- model_type += "_chat"
-
- kwargs = MODEL_TYPE_TO_VLMEVAL_MODEL[model_type]
- kwargs["model_path"] = pretrained_model_name_or_path
- model_cls = kwargs.pop("cls")
- model_cls = getattr(vlmeval.vlm, model_cls)
- if "verbose" in kwargs:
- kwargs["verbose"] = verbose
- vlmeval.config.supported_VLM[model_name] = partial(model_cls, **kwargs)
-
- pred_root = os.path.join(work_dir, model_name)
- os.makedirs(pred_root, exist_ok=True)
-
- st = time.time()
- rt_file = open(f"{pred_root}/{model_name}_eval_cost.txt", "w")
- for dataset_name in dataset:
- task_st = time.time()
- try:
- dataset_kwargs = {}
- if dataset_name in ["MMLongBench_DOC", "DUDE", "DUDE_MINI", "SLIDEVQA", "SLIDEVQA_MINI"]:
- dataset_kwargs["model"] = model_name
- if dataset_name == "MMBench-Video":
- dataset_kwargs["pack"] = pack
- if dataset_name == "Video-MME":
- dataset_kwargs["use_subtitle"] = use_subtitle
-
- dataset = vlmeval.dataset.build_dataset(dataset_name, **dataset_kwargs)
- if dataset is None:
- logger.error(f"Dataset {dataset_name} is not valid, will be skipped. ")
- continue
-
- result_file = f"{pred_root}/{model_name}_{dataset_name}.xlsx"
- if fps > 0: # For Video Dataset, set the fps for priority
- if dataset_name == "MVBench":
- raise ValueError("MVBench does not support fps setting, please transfer to MVBench_MP4!")
- nframe = 0
- if dataset_name in ["MMBench-Video"]:
- packstr = "pack" if pack else "nopack"
- if nframe > 0:
- result_file = f"{pred_root}/{model_name}_{dataset_name}_{nframe}frame_{packstr}.xlsx"
- else:
- result_file = f"{pred_root}/{model_name}_{dataset_name}_{fps}fps_{packstr}.xlsx"
- elif dataset.MODALITY == "VIDEO":
- if pack:
- logger.info(f"{dataset_name} not support Pack Mode, directly change to unpack")
- pack = False
- packstr = "pack" if pack else "nopack"
- if nframe > 0:
- result_file = f"{pred_root}/{model_name}_{dataset_name}_{nframe}frame_{packstr}.xlsx"
- else:
- result_file = f"{pred_root}/{model_name}_{dataset_name}_{fps}fps_{packstr}.xlsx"
- if dataset_name in ["Video-MME"]:
- subtitlestr = "subs" if use_subtitle else "nosubs"
- result_file = result_file.replace(".xlsx", f"_{subtitlestr}.xlsx")
-
- if dataset.TYPE == "MT":
- result_file = result_file.replace(".xlsx", ".tsv")
-
- if os.path.exists(result_file) and rerun:
- import re
-
- pattern = re.compile(f"{model_name}_{dataset_name}_[(openai)|(gpt)|(auxmatch)].*")
- for file_name in os.listdir(pred_root):
- if pattern.match(file_name):
- os.remove(os.path.join(pred_root, file_name))
-
- if model is None:
- model = model_name # which is only a name
-
- # Perform the Inference
- if dataset.MODALITY == "VIDEO":
- model = vlmeval.inference_video.infer_data_job_video(
- model,
- work_dir=pred_root,
- model_name=model_name,
- dataset=dataset,
- nframe=nframe,
- pack=pack,
- verbose=verbose,
- subtitle=use_subtitle,
- fps=fps,
- )
- elif dataset.TYPE == "MT":
- model = vlmeval.inference_mt.infer_data_job_mt(
- model,
- work_dir=pred_root,
- model_name=model_name,
- dataset=dataset,
- verbose=verbose,
- ignore_failed=ignore,
- )
- else:
- model = vlmeval.inference.infer_data_job(
- model,
- work_dir=pred_root,
- model_name=model_name,
- dataset=dataset,
- verbose=verbose,
- ignore_failed=ignore,
- )
-
- # Set the judge kwargs first before evaluation or dumping
- judge_kwargs = {
- "verbose": verbose,
- }
- if judge is not None:
- judge_kwargs["model"] = judge
- else:
- if dataset.TYPE in ["MCQ", "Y/N"] or vlmeval.smp.listinstr(["MathVerse"], dataset_name):
- judge_kwargs["model"] = "chatgpt-0125"
- elif vlmeval.smp.listinstr(
- ["MMVet", "MathVista", "LLaVABench", "MMBench-Video", "MathVision"], dataset_name
- ):
- judge_kwargs["model"] = "gpt-4-turbo"
- elif vlmeval.smp.listinstr(
- ["MMLongBench", "MMDU", "DUDE", "DUDE_MINI", "SLIDEVQA", "SLIDEVQA_MINI"], dataset_name
- ):
- judge_kwargs["model"] = "gpt-4o"
- if "OPENAI_API_KEY_JUDGE" in os.environ and len(os.environ["OPENAI_API_KEY_JUDGE"]):
- judge_kwargs["key"] = os.environ["OPENAI_API_KEY_JUDGE"]
- if "OPENAI_API_BASE_JUDGE" in os.environ and len(os.environ["OPENAI_API_BASE_JUDGE"]):
- judge_kwargs["api_base"] = os.environ["OPENAI_API_BASE_JUDGE"]
-
- if dataset_name in ["MMMU_TEST"]:
- result_json = vlmeval.utils.result_transfer.MMMU_result_transfer(result_file)
- logger.info(
- f"Transfer MMMU_TEST result to json for official evaluation, " f"json file saved in {result_json}"
- ) # noqa: E501
- continue
- elif "MMT-Bench_ALL" in dataset_name:
- submission_file = vlmeval.utils.result_transfer.MMTBench_result_transfer(result_file, **judge_kwargs)
- logger.info(
- f"Extract options from prediction of MMT-Bench FULL split for official evaluation "
- f"(https://eval.ai/web/challenges/challenge-page/2328/overview), "
- f"submission file saved in {submission_file}"
- ) # noqa: E501
- continue
- elif "MLLMGuard_DS" in dataset_name:
- logger.info("The evaluation of MLLMGuard_DS is not supported yet. ") # noqa: E501
- continue
- elif "AesBench_TEST" == dataset_name:
- logger.info(
- f"The results are saved in {result_file}. "
- f"Please send it to the AesBench Team via [email protected]."
- ) # noqa: E501
- continue
- elif dataset_name in ["DocVQA_TEST", "InfoVQA_TEST", "Q-Bench1_TEST", "A-Bench_TEST"]:
- logger.info(
- f"{dataset_name} is a test split without ground-truth. "
- "Thus only the inference part is supported for those datasets. "
- ) # noqa: E501
-
- if dataset_name in [
- "MMBench_TEST_CN",
- "MMBench_TEST_EN",
- "MMBench",
- "MMBench_CN",
- "MMBench_TEST_CN_V11",
- "MMBench_TEST_EN_V11",
- "MMBench_V11",
- "MMBench_CN_V11",
- ]:
- if not vlmeval.smp.MMBenchOfficialServer(dataset_name):
- logger.error(
- f"Can not evaluate {dataset_name} on non-official servers, " "will skip the evaluation. "
- )
- continue
-
- if mode == "all":
- eval_results = dataset.evaluate(result_file, **judge_kwargs)
- import pandas as pd
-
- if eval_results is not None:
- if not isinstance(eval_results, dict) and not isinstance(eval_results, pd.DataFrame):
- raise TypeError("Unsupported eval result type")
- logger.info(f"The evaluation of model {model_name} x dataset {dataset_name} has finished! ")
- logger.info("Evaluation Results:")
- if isinstance(eval_results, dict):
- logger.info("\n" + json.dumps(eval_results, indent=4))
- elif isinstance(eval_results, pd.DataFrame):
- if len(eval_results) < len(eval_results.columns):
- eval_results = eval_results.T
- try:
- import tabulate # pylint: disable=E0401
-
- logger.info("\n" + tabulate.tabulate(eval_results))
- except:
- logger.info(eval_results.to_string())
- rt_file.write("%s cost: %.4fs\n" % (dataset_name, time.time() - task_st))
- except Exception as e:
- logger.exception(
- f"Model {model_name} x Dataset {dataset_name} combination failed: {e}, " "skipping this combination."
- )
- continue
- rt_file.write("%d tasks cost: %.4fs\n" % (len(dataset), time.time() - st))
- rt_file.close()
-
-
-MODEL_TYPE_TO_LMMS_MODEL = {
- # model_name
- "Qwen-VL": "qwen_vl",
- "Qwen2-VL": "qwen2_vl",
- "cogvlm2": "cogvlm2",
- "llava_v1.5": "llava",
- "Llama-3.2": "llama_vision",
- "Phi-3-vision": "phi3v",
- "Phi-3.5-vision": "phi3v",
- # model_type
- "qwen2_vl": "qwen2_vl",
- "qwen": "qwen_vl",
- "llava": "llava",
- "phi3_v": "phi3v",
- "mllama": "llama_vision",
-}
-
-_lmms_eval = LazyImport("lmms_eval")
-
-
-def _handle_non_serializable(o):
- if isinstance(o, np.int64) or isinstance(o, np.int32):
- return int(o)
- elif isinstance(o, set):
- return list(o)
- else:
- return str(o)
-
-
-def lmms_eval(
- model,
- tasks,
- output_dir=None,
- num_fewshot=None,
- limit=None,
- batch_size=1,
- max_batch_size=None,
- device="cpu",
- use_cache=None,
- apply_chat_template=False,
-):
- from transformers.utils.versions import require_version
-
- require_version("lmms_eval", "lmms_eval need to be installed, `pip install lmms_eval`")
- if isinstance(tasks, str):
- tasks = tasks.replace(" ", "").split(",")
-
- model_name = model
- if model_name[-1] == "/":
- model_name = model_name[:-1]
- model_name = model_name.split("/")[-1]
-
- model_type = None
- split_name = model_name.split("-")
- for i in range(len(split_name), 0, -1):
- tmp = "-".join(split_name[0:i])
- if tmp in MODEL_TYPE_TO_LMMS_MODEL:
- model_type = tmp
- break
- if model_type is None:
- from transformers import AutoConfig
-
- config = AutoConfig.from_pretrained(model, trust_remote_code=True)
- model_type = config.model_type
-
- if model_type not in MODEL_TYPE_TO_LMMS_MODEL:
- raise ValueError(f"{model_type} is not support by lmms.")
-
- if MODEL_TYPE_TO_LMMS_MODEL[model_type] == "phi3v":
- model_args = f"model_id_name={model}"
- else:
- model_args = f"pretrained={model}"
- if MODEL_TYPE_TO_LMMS_MODEL[model_type] == "llama_vision":
- model_args += f",device_map={device}"
-
- class CliArgs:
- output_path = output_dir
-
- results = _lmms_eval.evaluator.simple_evaluate(
- model=MODEL_TYPE_TO_LMMS_MODEL[model_type],
- model_args=model_args,
- tasks=tasks,
- num_fewshot=num_fewshot,
- limit=limit,
- batch_size=batch_size,
- max_batch_size=max_batch_size,
- device=device,
- use_cache=use_cache,
- apply_chat_template=apply_chat_template,
- cli_args=CliArgs(),
- )
-
- # print and save result
- print(_lmms_eval.utils.make_table(results))
- if output_dir:
- os.makedirs(output_dir, exist_ok=True)
-
- from datetime import datetime
-
- now = datetime.now().strftime("%Y%m%d_%H%M%S")
- output_file = os.path.join(output_dir, f"{model_name}_{now}_result.json")
- json.dump(results, open(output_file, "w"), indent=4, default=_handle_non_serializable)
-
- return results
diff --git a/auto_round/compressors/mllm_mixin.py b/auto_round/compressors/mllm_mixin.py
new file mode 100644
index 000000000..26bf4d06a
--- /dev/null
+++ b/auto_round/compressors/mllm_mixin.py
@@ -0,0 +1,131 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from auto_round.logger import logger
+
+
+class MLLMMixin:
+ """MLLM-specific functionality mixin.
+
+ This mixin adds MLLM-specific functionality to any compressor (DataDrivenCompressor,
+ ZeroShotCompressor, ImatrixCompressor, etc). It handles multi-modal models
+ (vision-language models) that require special data loading and processing logic.
+
+ Can be combined with:
+ - DataDrivenCompressor (for AutoRound with calibration)
+ - ImatrixCompressor (for RTN with importance matrix)
+ - ZeroShotCompressor (for basic RTN)
+
+ MLLM-specific parameters:
+ processor: Multi-modal processor override (normally loaded by ModelContext)
+ image_processor: Image processor override (e.g. for LLaVA)
+ template: Template name for processing different MLLMs
+ extra_data_dir: Path to extra data (images, audio, videos)
+ quant_nontext_module: Whether to quantize non-text modules
+
+ Design note:
+ ``ModelContext._load_model()`` is responsible for loading the model and its
+ associated artifacts (processor, tokenizer, image_processor). This mixin
+ reads those artifacts from ``self.model_context`` during calibration.
+ If the caller passes explicit ``processor`` / ``image_processor`` overrides,
+ they are written into ``model_context`` after ``super().__init__()`` so that
+ ``model_context`` remains the single source of truth.
+ """
+
+ def __init__(
+ self,
+ *args,
+ processor=None,
+ image_processor=None,
+ template=None,
+ extra_data_dir=None,
+ quant_nontext_module=False,
+ **kwargs,
+ ):
+ self.template = template
+ self.extra_data_dir = extra_data_dir
+ self.quant_nontext_module = quant_nontext_module
+ self.template_obj = None
+
+ # Pass quant_nontext_module to ModelContext so get_block_names can include vision blocks
+ kwargs.setdefault("quant_nontext_module", quant_nontext_module)
+
+ if quant_nontext_module:
+ batch_size = kwargs.get("batch_size", 8)
+ if batch_size != 1:
+ grad_acc = kwargs.get("gradient_accumulate_steps", 1)
+ new_grad_acc = batch_size * grad_acc
+ kwargs["gradient_accumulate_steps"] = new_grad_acc
+ kwargs["batch_size"] = 1
+ # Also patch ``gradient_accumulate_steps`` on AlgConfig (still
+ # owned there) so behaviour matches the old arch.
+ _alg_cfg = args[0] if args else None
+ if _alg_cfg is not None:
+ cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
+ for cfg in cfgs:
+ if hasattr(cfg, "gradient_accumulate_steps"):
+ cfg.gradient_accumulate_steps = new_grad_acc
+ logger.warning(
+ f"reset batch_size({batch_size}) to 1 and "
+ f"gradient_accumulate_steps to {new_grad_acc} "
+ f"because batch_size={batch_size} cannot be used for calibrating non-text modules."
+ )
+
+ # super().__init__() creates model_context, which eagerly loads the model and
+ # populates model_context.processor / image_processor / tokenizer.
+ super().__init__(*args, **kwargs)
+
+ # Apply user-provided overrides into model_context (single source of truth).
+ if processor is not None:
+ self.model_context.processor = processor
+ if image_processor is not None:
+ self.model_context.image_processor = image_processor
+
+ def _get_calibrator_kind(self) -> str:
+ """Select the MLLM calibration strategy.
+
+ ``MLLMCalibrator`` lives at :mod:`auto_round.calibration.mllm`
+ and owns what used to be ``MLLMMixin.calib``.
+ """
+ return "mllm"
+
+ def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
+ """Save the quantized model to the specified output directory in the specified format.
+
+ Args:
+ output_dir (str, optional): The directory to save the quantized model. Defaults to None.
+ format (str, optional): The format in which to save the model. Defaults to "auto_round".
+ inplace (bool, optional): Whether to modify the model in place. Defaults to True.
+ **kwargs: Additional keyword arguments specific to the export format.
+
+ Returns:
+ object: The compressed model object.
+ """
+ mc = self.model_context
+ processor = mc.processor
+ image_processor = mc.image_processor
+ tokenizer = mc.tokenizer
+
+ if processor is not None and not hasattr(processor, "chat_template"):
+ processor.chat_template = None
+ compressed_model = super().save_quantized(
+ output_dir=output_dir,
+ format=format,
+ inplace=inplace,
+ processor=processor,
+ image_processor=image_processor,
+ quant_nontext_module=self.quant_nontext_module if hasattr(self, "quant_nontext_module") else False,
+ **kwargs,
+ )
+ return compressed_model
diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.py
index 77b3b7ea4..b25c26739 100644
--- a/auto_round/compressors/shard_writer.py
+++ b/auto_round/compressors/shard_writer.py
@@ -17,8 +17,10 @@ import os
from collections import OrderedDict
import torch
-from torch.nn import Parameter
+from auto_round.compressors.utils import _get_save_folder_name
+from auto_round.context.compress import CompressContext
+from auto_round.context.model import ModelContext
from auto_round.logger import logger
from auto_round.utils import (
get_lm_head_name,
@@ -33,8 +35,28 @@ class ShardWriter:
Handles shard-saving of model parameters to disk with memory management.
"""
- def __init__(self, rounder):
- self.model = rounder.model
+ _instance = None
+ _initialized = False
+
+ model = None
+ lm_head_name = None
+
+ def __new__(cls, *args, **kwargs):
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ cls._instance._data = {}
+ return cls._instance
+
+ def __init__(
+ self,
+ model,
+ bits,
+ max_shard_size=None,
+ safe_serialization=True,
+ ):
+ if ShardWriter._initialized:
+ return
+ self.model = model
self.lm_head_name = get_lm_head_name(self.model)
total_params = sum(p.numel() for p in self.model.parameters())
# Heuristic estimate of model size in GB used to choose a default max_shard_size:
@@ -45,12 +67,13 @@ class ShardWriter:
# smaller than the full model; this intentionally
# underestimates size before clamping below.
max_split_num = 10
- model_size = int(total_params * rounder.bits // 1e9 // 8 + max_split_num - 1) / max_split_num
+ model_size = int(total_params * bits // 1e9 // 8 + max_split_num - 1) / max_split_num
model_size = max(1, min(int(model_size), 5))
# Configuration
- self.max_shard_size = self._parse_size(getattr(rounder, "max_shard_size", f"{model_size}GB"))
- self.safe_serialization = getattr(rounder, "safe_serialization", True)
+ max_shard_size = max_shard_size or f"{model_size}GB"
+ self.max_shard_size = self._parse_size(max_shard_size)
+ self.safe_serialization = safe_serialization
# Internal State
self.use_safetensors = self._check_safetensors()
@@ -71,13 +94,39 @@ class ShardWriter:
self.total_param_size_bytes = 0
self.skipped_meta_tensors = []
- # Directory Setup
- base_dir = rounder._get_save_folder_name(rounder.formats[0])
+ ShardWriter._initialized = True
+
+ @property
+ def output_dir(self) -> str:
+ """Derive the output directory from the current CompressContext at access time.
+
+ Reading from context rather than caching the path at construction time ensures
+ the ShardWriter always uses the final export directory even if
+ ``CompressContext.output_dir`` is updated after the ShardWriter was created
+ (e.g. by ``_get_export_dir()`` in ``quantize_and_save()``).
+ """
+ compress_context = CompressContext.get_context()
+ formats = compress_context.formats
+ base_dir = _get_save_folder_name(formats[0])
subfolder = getattr(self.model, "_autoround_pipeline_subfolder", None)
if subfolder:
base_dir = os.path.join(base_dir, subfolder)
- self.output_dir = os.path.join(base_dir, "")
- os.makedirs(self.output_dir, exist_ok=True)
+ return os.path.join(base_dir, "")
+
+ @classmethod
+ def reset(cls):
+ """Reset the singleton state so the next instantiation creates a fresh ShardWriter."""
+ cls._initialized = False
+ cls._instance = None
+
+ @classmethod
+ def get_shard_writer(cls, *args, **kwargs):
+ """Return the current singleton instance, or None if not yet initialized.
+
+ Callers that require a valid writer should guard the result with
+ ``if self.compress_context.is_immediate_saving`` before use.
+ """
+ return cls._instance
def _parse_size(self, size_str: str) -> int:
if isinstance(size_str, int):
@@ -147,8 +196,6 @@ class ShardWriter:
Detects tied weights in the current shard and ensures they are only saved once.
This is done by tracking storage pointers of tensors and skipping duplicates.
"""
- from collections import defaultdict
-
storage_map = set()
filtered_tensors = {}
@@ -161,7 +208,6 @@ class ShardWriter:
if ptr not in storage_map:
storage_map.add(ptr)
filtered_tensors[name] = tensor
-
self.current_shard_tensors = filtered_tensors
def _flush_shard(self):
@@ -169,8 +215,10 @@ class ShardWriter:
return
self.shard_counter += 1
+ output_dir = self.output_dir
+ os.makedirs(output_dir, exist_ok=True)
tmp_name = f"model-shard-{self.shard_counter:05d}.{self.shard_suffix}"
- tmp_path = os.path.join(self.output_dir, tmp_name)
+ tmp_path = os.path.join(output_dir, tmp_name)
self._handle_tied_weights()
if self.use_safetensors:
@@ -186,7 +234,7 @@ class ShardWriter:
torch.save(self.current_shard_tensors, tmp_path)
saved_params = list(self.current_shard_tensors.keys())
- self.shard_meta.append({"tmp_file": tmp_name, "params": saved_params})
+ self.shard_meta.append({"tmp_file": tmp_name, "params": saved_params, "dir": output_dir})
self._all_saved.update(saved_params)
# Offload logic: move modules to meta device once all params are saved
@@ -207,44 +255,15 @@ class ShardWriter:
and isinstance(module, torch.nn.Module)
and all(f"{module_path}.{k}" in self._all_saved for k in module.state_dict().keys())
):
- self._move_module_to_meta(module)
-
- def _move_module_to_meta(self, module: torch.nn.Module):
- for child in module.children():
- self._move_module_to_meta(child)
-
- for name, param in list(module._parameters.items()):
- if param is None:
- continue
- if isinstance(param, Parameter):
- meta_param = Parameter(param.detach().to(device="meta"), requires_grad=param.requires_grad)
- elif isinstance(param, torch.Tensor):
- meta_param = Parameter(param.detach().to(device="meta"), requires_grad=param.requires_grad)
- else:
- continue
- module._parameters[name] = meta_param
-
- for name, buffer in list(module._buffers.items()):
- if isinstance(buffer, torch.Tensor):
- module._buffers[name] = buffer.detach().to(device="meta")
+ module.to("meta")
def finalize(self):
"""Saves remaining weights, renames files, and writes the index JSON."""
# 1. Capture remaining weights not yet saved
full_sd = self.model.state_dict()
tie_word_embeddings = False
- config = getattr(self.model, "config", None)
if hasattr(self.model, "config") and hasattr(self.model.config, "tie_word_embeddings"):
tie_word_embeddings = self.model.config.tie_word_embeddings
- if tie_word_embeddings is None:
- # For multimodal models, check nested text/thinker configs
- for sub_attr in ("text_config", "thinker_config", "language_config", "llm_config"):
- sub_config = getattr(config, sub_attr, None)
- if sub_config is not None:
- val = getattr(sub_config, "tie_word_embeddings", None)
- if val is not None:
- tie_word_embeddings = val
- break
finalize_skipped_meta_tensors = []
for pname, tensor in full_sd.items():
@@ -255,7 +274,7 @@ class ShardWriter:
layer_name = ".".join(pname.split(".")[:-1])
if self.lm_head_name is not None and layer_name == self.lm_head_name and tie_word_embeddings:
lm_head_module = get_module(self.model, self.lm_head_name)
- self._move_module_to_meta(lm_head_module) # Must to meta, otherwise model's saver will dump it again
+ lm_head_module.to("meta") # Must to meta, otherwise model's saver will dump it again
continue
self._add_tensor(pname, tensor.detach().to("cpu"))
@@ -270,21 +289,23 @@ class ShardWriter:
logger.warning("No tensors saved.")
return
+ output_dir = self.output_dir
for idx, meta in enumerate(self.shard_meta, start=1):
- old_path = os.path.join(self.output_dir, meta["tmp_file"])
+ shard_dir = meta.get("dir", output_dir)
+ old_path = os.path.join(shard_dir, meta["tmp_file"])
new_name = (
f"model.{self.shard_suffix}"
if self.shard_counter == 1
else f"model-{idx:05d}-of-{self.shard_counter:05d}.{self.shard_suffix}"
)
-
- os.rename(old_path, os.path.join(self.output_dir, new_name))
+ new_path = os.path.join(shard_dir, new_name)
+ os.rename(old_path, new_path)
for p in meta["params"]:
self.global_weight_map[p] = new_name
# 3. Write Index JSON
index_ext = "safetensors.index.json" if self.use_safetensors else "bin.index.json"
- index_path = os.path.join(self.output_dir, f"model.{index_ext}")
+ index_path = os.path.join(output_dir, f"model.{index_ext}")
index_data = {
"metadata": {
@@ -302,21 +323,17 @@ class ShardWriter:
logger.info(f"model has been saved to {self.output_dir}")
-
-def shard_writer(rounder: object, m: torch.nn.Module = None, name: str = None, is_finalize: bool = False):
- if m is None and name is None and not is_finalize and not is_finalize:
- raise ValueError("Must specify either name or m")
- if not hasattr(rounder, "_shard_writer"):
- rounder._shard_writer = ShardWriter(rounder)
-
- if m is None and name is not None:
- m = get_module(rounder.model, name)
- # Perform the save
- if m is not None:
- rounder._shard_writer.save_module(m, name)
-
- if is_finalize:
- rounder._shard_writer.finalize()
- # Optional: cleanup the saver object from rounder
- del rounder._shard_writer
+ @torch.no_grad()
+ def write(self, m: torch.nn.Module = None, name: str = None, is_finalize: bool = False):
+ if m is None and name is None and not is_finalize and not is_finalize:
+ raise ValueError("Must specify either name or m")
+ if m is None and name is not None:
+ m = get_module(self.model, name)
+ # Perform the save
+ if m is not None:
+ self.save_module(m, name)
+
+ if is_finalize:
+ self.finalize()
+ ShardWriter._initialized = False
+ ShardWriter._instance = None
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 6b11de9d3..59c3e5264 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -19,7 +19,7 @@ import re
import sys
from dataclasses import asdict, fields
from enum import Enum
-from typing import Callable, Union
+from typing import TYPE_CHECKING, Callable, Union
import torch
import transformers
@@ -27,8 +27,10 @@ from torch.amp import autocast
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
from auto_round.logger import logger
-from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme
-from auto_round.utils import check_to_quantized, infer_bits_by_data_type, to_standard_regex
+from auto_round.utils import check_to_quantized, get_layer_names_in_block, get_module, to_standard_regex
+
+if TYPE_CHECKING:
+ from auto_round.schemes import QuantizationScheme
class BackendDataType(str, Enum):
@@ -58,6 +60,16 @@ def is_nv_fp(backend):
return BackendDataType.NV_FP in backend
+def is_wint_woq(ar):
+ """Returns True for integer weight-only quantization with non-quantized activations (`act_bits >= 16`)."""
+ return "int" in ar.data_type and ar.act_bits >= 16 and ar.super_group_size is None
+
+
+def is_wint_a16(ar):
+ """Backward-compatible alias for `is_wint_woq()`."""
+ return is_wint_woq(ar)
+
+
def _is_weight_fp8_activation_static_fp8(
bit: int, group_size: int, sym: bool, data_type: str, act_dynamic: bool
) -> bool:
@@ -95,19 +107,9 @@ def is_static_wfp8afp8(ar_or_format: Union[str, Callable]) -> bool:
return False
-def is_wint_woq(ar: Callable) -> bool:
- """Returns True for integer weight-only quantization with non-quantized activations (`act_bits >= 16`)."""
- return "int" in ar.data_type and ar.act_bits >= 16 and ar.super_group_size is None
-
-
-def is_wint_a16(ar: Callable) -> bool:
- """Backward-compatible alias for `is_wint_woq()`."""
- return is_wint_woq(ar)
-
-
def is_dynamic_wint8aint8(ar_or_format: Union[str, Callable]) -> bool:
if isinstance(ar_or_format, str):
- return "int8_w8a8" in ar_or_format.lower() or "int8" in ar_or_format.lower()
+ return "int8_w8a8" in ar_or_format.lower()
if not ar_or_format.act_dynamic:
return False
if is_wint8aint8(ar_or_format):
@@ -225,6 +227,30 @@ def collect_best_params(block, cache_device="cpu"):
return params
+def infer_bits_by_data_type(data_type: str):
+ """Infer bits by data_type
+
+ Args:
+ data_type (str): data_type
+
+ Returns:
+ int: bits inferred by data_type, None means cannot infer correct bits by data_type
+ """
+ from auto_round.utils import SUPPORTED_DTYPES
+
+ if data_type is None:
+ return 16
+ for supported_dtype in SUPPORTED_DTYPES:
+ if data_type.startswith(supported_dtype) and len(data_type) > len(supported_dtype):
+ ##first check the following two bits
+ suc_2str = data_type[len(supported_dtype) : len(supported_dtype) + 2]
+ if str.isdigit(suc_2str):
+ return int(suc_2str)
+ if str.isdigit(data_type[len(supported_dtype)]):
+ return int(data_type[len(supported_dtype)])
+ return None
+
+
def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:
"""Collect layer names from safetensor files that are not loaded into the model.
@@ -295,13 +321,14 @@ def set_layer_config(
enable_gguf_official_mixed: bool = True,
is_mllm: bool = False,
fill_default_value=True,
+ gguf_format_name: str = None,
) -> tuple[dict, bool, dict]:
"""
Normalize, validate, and expand layer-specific quantization configs.
Returns (final_layer_config, has_quant_layer_outside_block)
"""
- from auto_round.schemes import get_gguf_scheme
+ from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme
from auto_round.utils.model import get_layer_names_in_block, get_lm_head_name, get_module, is_separate_lm_head
# ---- helpers -------------------------------------------------
@@ -323,18 +350,17 @@ def set_layer_config(
elif isinstance(item, QuantizationScheme):
config = asdict(item)
elif isinstance(item, dict):
- # Support "scheme" key inside dict: resolve the preset and merge overrides
- if "scheme" in item:
- scheme_name = item.pop("scheme")
- base = asdict(preset_name_to_scheme(scheme_name.upper()))
- base.update(item)
- item = base
+ # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;
+ # silently drop it before validation.
+ item = {k: v for k, v in item.items() if k != "in_blocks"}
+ scheme_name = item.pop("scheme", None)
+ config = asdict(preset_name_to_scheme(scheme_name.upper())) if scheme_name is not None else {}
invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
if invalid:
raise ValueError(
f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"
)
- config = dict(item)
+ config.update(item)
else:
raise TypeError(
f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
@@ -398,7 +424,7 @@ def set_layer_config(
# 5. collect supported modules
embedding_types = (torch.nn.Embedding,)
- gguf_name = get_gguf_scheme(default_scheme)
+ gguf_name = gguf_format_name or get_gguf_scheme(default_scheme)
if gguf_name:
if torch.nn.Embedding not in supported_types:
supported_types = (*supported_types, torch.nn.Embedding)
@@ -650,6 +676,7 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model
import gguf # pylint: disable=E0401
+ from auto_round.schemes import QuantizationScheme, get_gguf_scheme
from auto_round.utils.common import MM_KEYS, LazyImport
from auto_round.utils.model import get_lm_head_name, get_module
@@ -1116,3 +1143,120 @@ class IndexSampler:
batch = self.indices[self.index : self.index + self.batch_size]
self.index += self.batch_size
return batch
+
+
+def _get_quantized_layer_names_outside_blocks(model, layer_config, supported_types, quant_block_list) -> list:
+ """Gets the names of quantized layers outside blocks in the model.
+
+ Returns:
+ list: List of layer names outside blocks.
+ """
+ if layer_config is None or len(layer_config) == 0:
+ return []
+
+ layer_names = []
+ all_layers_in_block = get_layer_names_in_block(model, supported_types, quant_block_list)
+
+ for key in layer_config.keys():
+ if key in all_layers_in_block:
+ continue
+ layer = get_module(model, key)
+ if layer is None:
+ logger.error(f"could not find layer {key} in the model, exit...")
+ exit(-1)
+ if type(layer) in supported_types and check_to_quantized(layer_config[key]):
+ layer_names.append(key)
+
+ return layer_names
+
+
+def _get_diffusion_save_folder_name(format) -> str:
+ """Generates the save folder name based on the provided format string.
+
+ If there are multiple formats to handle, the function creates a subfolder
+ named after the format string with special characters replaced. If there's
+ only one format, it returns the original output directory directly.
+
+ Args:
+ format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
+
+ Returns:
+ str: The path to the folder where results should be saved.
+ """
+ from auto_round.context.compress import CompressContext
+ from auto_round.context.model import ModelContext
+
+ compress_context = CompressContext.get_context()
+ model_context = ModelContext.get_context()
+
+ # Replace special characters to make the folder name filesystem-safe
+ sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")
+
+ formats = compress_context.formats
+ # Use a subfolder only if there are multiple formats
+ if len(formats) > 1:
+ return (
+ os.path.join(compress_context.output_dir, sanitized_format, "transformer")
+ if compress_context.is_immediate_saving
+ else os.path.join(compress_context.output_dir, sanitized_format, "transformer")
+ )
+
+ # if use is_immediate_saving, we need to save model in self.output_dir/transformer folder
+ return (
+ os.path.join(compress_context.output_dir, "transformer")
+ if compress_context.is_immediate_saving
+ else compress_context.output_dir
+ )
+
+
+def _get_save_folder_name(format, *args, **kwargs) -> str:
+ """Generates the save folder name based on the provided format string.
+
+ If there are multiple formats to handle, the function creates a subfolder
+ named after the format string with special characters replaced. If there's
+ only one format, it returns the original output directory directly.
+
+ Args:
+ format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
+
+ Returns:
+ str: The path to the folder where results should be saved.
+ """
+ from auto_round.context.compress import CompressContext
+ from auto_round.context.model import ModelContext
+
+ compress_context = CompressContext.get_context()
+ model_context = ModelContext.get_context()
+ if model_context.is_diffusion:
+ return _get_diffusion_save_folder_name(format)
+ # Replace special characters to make the folder name filesystem-safe
+ sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")
+
+ # Use a subfolder only if there are multiple formats
+ if len(compress_context.formats) > 1:
+ return os.path.join(compress_context.output_dir, sanitized_format)
+
+ return compress_context.output_dir
+
+
+def immediate_pack(name: str, layer_config: dict):
+ from auto_round.context.compress import CompressContext
+ from auto_round.context.model import ModelContext
+
+ compress_context = CompressContext.get_context()
+ model_context = ModelContext.get_context()
+
+ if not compress_context.is_immediate_packing:
+ return
+ compress_context.formats[0].immediate_pack(
+ name=name,
+ model=model_context.model,
+ device=compress_context.device,
+ output_dir=_get_save_folder_name(compress_context.formats[0]),
+ layer_config=layer_config,
+ tokenizer=model_context.tokenizer,
+ mllm=model_context.is_mllm,
+ processor=getattr(model_context, "processor", None),
+ image_processor=getattr(model_context, "image_processor", None),
+ quant_nontext_module=getattr(model_context, "quant_nontext_module", False),
+ )
diff --git a/auto_round/compressors_new/zero_shot.py b/auto_round/compressors/zero_shot.py
similarity index 98%
rename from auto_round/compressors_new/zero_shot.py
rename to auto_round/compressors/zero_shot.py
index 31ccc2494..1cd2ced4c 100644
--- a/auto_round/compressors_new/zero_shot.py
+++ b/auto_round/compressors/zero_shot.py
@@ -18,8 +18,8 @@ import torch
from tqdm import tqdm
from auto_round.algorithms.alg_config import AlgConfig
-from auto_round.compressors_new.base import BaseCompressor
-from auto_round.compressors_new.utils import is_nv_fp, is_static_wfp8afp8
+from auto_round.compressors.base import BaseCompressor
+from auto_round.compressors.utils import is_nv_fp, is_static_wfp8afp8
from auto_round.logger import logger
from auto_round.modeling.fused_moe.replace_modules import materialize_model_
from auto_round.utils import (
diff --git a/auto_round/compressors_new/__init__.py b/auto_round/compressors_new/__init__.py
deleted file mode 100644
index 66dbe5de9..000000000
--- a/auto_round/compressors_new/__init__.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Lazy imports to avoid circular dependencies
-# Users should import from specific modules instead of this __init__.py
-
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
- from auto_round.compressors_new.calib import CalibCompressor, CalibratedRTNCompressor
- from auto_round.compressors_new.entry import AutoRoundCompatible, AutoRound
- from auto_round.compressors_new.zero_shot import ZeroShotCompressor
-
-__all__ = [
- "AutoRound",
- "CalibCompressor",
- "CalibratedRTNCompressor",
- "ZeroShotCompressor",
- "AutoRoundCompatible",
-]
-
-
-def __getattr__(name):
- """Lazy import to avoid circular dependencies."""
- if name == "AutoRound" or name == "AutoRoundCompatible":
- from auto_round.compressors_new.entry import AutoRound, AutoRoundCompatible
-
- if name == "AutoRound":
- return AutoRound
- return AutoRoundCompatible
- elif name in ("CalibCompressor", "CalibratedRTNCompressor"):
- from auto_round.compressors_new.calib import CalibCompressor, CalibratedRTNCompressor
-
- return {
- "CalibCompressor": CalibCompressor,
- "CalibratedRTNCompressor": CalibratedRTNCompressor,
- }[name]
- elif name == "ZeroShotCompressor":
- from auto_round.compressors_new.zero_shot import ZeroShotCompressor
-
- return ZeroShotCompressor
- raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
diff --git a/auto_round/compressors_new/architecture_visualization.py b/auto_round/compressors_new/architecture_visualization.py
deleted file mode 100644
index 7ff020733..000000000
--- a/auto_round/compressors_new/architecture_visualization.py
+++ /dev/null
@@ -1,332 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""
-New Architecture Visualization - Mixin Pattern Combination Table
-
-Demonstrates all possible combinations of model types and compression algorithms.
-"""
-
-
-def print_architecture_table():
- """Print architecture combination table"""
-
- print("\n" + "=" * 110)
- print("AutoRound New Architecture - Mixin Pattern Combination Table")
- print("=" * 110 + "\n")
-
- print(f"{'Model Type':<15} {'Config Type':<20} {'AutoRound (dynamic class)':<40} {'Base classes':<35}")
- print("-" * 110)
-
- # LLM combinations
- print(f"{'LLM':<15} {'SignRoundConfig':<20} {'CalibCompressor':<40} {'CalibCompressor':<35}")
- print(f"{'LLM':<15} {'RTNConfig':<20} {'CalibratedRTNCompressor':<40} {'CalibratedRTNCompressor':<35}")
- print(f"{'LLM':<15} {'RTNConfig':<20} {'ZeroShotCompressor':<40} {'ZeroShotCompressor':<35}")
-
- print()
-
- # MLLM combinations (dynamic classes created in entry.py)
- print(f"{'MLLM':<15} {'SignRoundConfig':<20} {'MLLMCalibCompressor':<40} {'MLLMMixin + CalibCompressor':<35}")
- print(
- f"{'MLLM':<15} {'RTNConfig':<20} {'MLLMCalibratedRTNCompressor':<40} "
- f"{'MLLMMixin + CalibratedRTNCompressor':<35}"
- )
- print(f"{'MLLM':<15} {'RTNConfig':<20} {'MLLMZeroShotCompressor':<40} {'MLLMMixin + ZeroShotCompressor':<35}")
-
- print()
-
- # Diffusion combinations (dynamic classes created in entry.py)
- print(
- f"{'Diffusion':<15} {'SignRoundConfig':<20} {'DiffusionCalibCompressor':<40} "
- f"{'DiffusionMixin + CalibCompressor':<35}"
- )
- print(
- f"{'Diffusion':<15} {'RTNConfig':<20} {'DiffusionCalibratedRTNCompressor':<40} "
- f"{'DiffusionMixin + CalibratedRTNCompressor':<35}"
- )
- print(
- f"{'Diffusion':<15} {'RTNConfig':<20} {'DiffusionZeroShotCompressor':<40} "
- f"{'DiffusionMixin + ZeroShotCompressor':<35}"
- )
-
- print("\n" + "=" * 110 + "\n")
-
-
-def print_mixin_explanation():
- """Print Mixin pattern explanation"""
-
- print("=" * 110)
- print("Mixin Pattern Explanation")
- print("=" * 110 + "\n")
-
- print("✨ Core Components:")
- print("-" * 110)
- print(" 1. MLLMMixin - MLLM features (processor, template, quant_nontext_module, etc.)")
- print(" 2. DiffusionMixin - Diffusion features (pipeline loading, guidance_scale, etc.)")
- print(" 3. CalibCompressor - AutoRoundCompatible: gradient-based calibration quantization")
- print(" 4. CalibratedRTNCompressor - RTN with importance-matrix (imatrix) or act calibration")
- print(" 5. ZeroShotCompressor - Zero-shot RTN (no calibration data needed)")
-
- print("\n🎯 Combination Approach:")
- print("-" * 110)
- print(" Dynamic classes created on-the-fly inside AutoRound.__new__():")
- print(" class MLLMCalibCompressor(MLLMMixin, CalibCompressor): pass")
- print(" class MLLMCalibratedRTNCompressor(MLLMMixin, CalibratedRTNCompressor): pass")
- print(" class MLLMZeroShotCompressor(MLLMMixin, ZeroShotCompressor): pass")
-
- print("\n💡 Advantages:")
- print("-" * 110)
- print(" ✓ Flexible Combination: Any model type can be combined with any compression algorithm")
- print(" ✓ Code Reuse: Mixin code is written once and reused across all compression algorithms")
- print(" ✓ Clear Separation: Model-specific logic (Mixin) and compression algorithm are independent")
- print(" ✓ Easy Extension: Add new model types without touching existing compressor code")
-
- print("\n" + "=" * 110 + "\n")
-
-
-def print_post_init_flow():
- """Print the post_init execution flow"""
-
- print("=" * 110)
- print("BaseCompressor.post_init() Execution Flow")
- print("=" * 110 + "\n")
-
- print("""
-BaseCompressor.post_init()
-│
-├─ Step 1: Resolve formats (str → list[OutputFormat])
-│ └─ get_formats(self.formats, self)
-│
-├─ Step 2: Apply format-specific model patches
-│ └─ model_context.apply_patches(formats)
-│ ├─ _patch_custom_moe_modules() # e.g. Qwen3VL MoE top_k fix
-│ ├─ update_module(model, formats) # add gguf_pack_linear etc.
-│ └─ assign global_name to all modules
-│
-├─ Step 3: Setup quantizer on the patched model
-│ └─ quantizer = BaseQuantizers.from_config(config)
-│ └─ quantizer.post_init()
-│ ├─ get ModelContext / CompressContext singletons
-│ ├─ _parse_scheme() → resolve final quant attrs
-│ ├─ get_block_names(quant_vision=quant_nontext_module)
-│ ├─ find_matching_blocks() → quant_block_list
-│ └─ back-fill to_quant_block_names if it was None
-│
-└─ Step 4: Setup device map, torch compile, offloader
- """)
-
- print("=" * 110 + "\n")
-
-
-def print_usage_examples():
- """Print usage examples"""
-
- print("=" * 110)
- print("Usage Examples")
- print("=" * 110 + "\n")
-
- print("Example 1: MLLM + AutoRoundCompatible (gradient-based)")
- print("-" * 110)
- print("""
-from auto_round.compressors_new.entry import AutoRound
-from auto_round.algorithms.quantization.sign_round.config import SignRoundConfig
-
-config = SignRoundConfig(scheme="W4A16", iters=200, nsamples=128)
-compressor = AutoRound(
- config=config,
- model="/models/Qwen2-VL-2B-Instruct",
- processor=processor,
- template="qwen2_vl",
- quant_nontext_module=False, # set True to also quantize vision encoder
-)
-# Dynamically creates: class MLLMCalibCompressor(MLLMMixin, CalibCompressor)
- """)
-
- print("\nExample 2: MLLM + RTN with imatrix")
- print("-" * 110)
- print("""
-from auto_round.algorithms.quantization.rtn.config import RTNConfig
-
-config = RTNConfig(scheme="W4A16")
-compressor = AutoRound(
- config=config,
- model="/models/Qwen2-VL-2B-Instruct",
- format="gguf_k", # gguf_k triggers CalibratedRTNCompressor
- processor=processor,
-)
-# Dynamically creates: class MLLMCalibratedRTNCompressor(MLLMMixin, CalibratedRTNCompressor)
- """)
-
- print("\nExample 3: Diffusion + AutoRoundCompatible")
- print("-" * 110)
- print("""
-config = SignRoundConfig(scheme="W4A16", iters=200)
-compressor = AutoRound(
- config=config,
- model="/models/stable-diffusion-2-1",
- guidance_scale=7.5,
-)
-# Dynamically creates: class DiffusionCalibCompressor(DiffusionMixin, CalibCompressor)
- """)
-
- print("\n" + "=" * 110 + "\n")
-
-
-def print_mro_example():
- """Print MRO (Method Resolution Order) example"""
-
- print("=" * 110)
- print("Method Resolution Order (MRO) Example")
- print("=" * 110 + "\n")
-
- print("For class MLLMCalibCompressor(MLLMMixin, CalibCompressor):")
- print("-" * 110)
- print("""
-MLLMCalibCompressor (dynamic, created in AutoRound.__new__)
- └─> MLLMMixin
- └─> CalibCompressor
- └─> BaseCompressor
- └─> object
-
-Execution order when calling __init__():
- 1. MLLMCalibCompressor.__init__() → not defined, falls through
- 2. MLLMMixin.__init__()
- - Save MLLM-specific attrs: processor, template, quant_nontext_module, …
- - kwargs.setdefault("quant_nontext_module", quant_nontext_module)
- - Call super().__init__() → enters CalibCompressor
- 3. CalibCompressor.__init__() → BaseCompressor.__init__()
- - pops quant_nontext_module from kwargs
- - Creates ModelContext(…, quant_nontext_module=quant_nontext_module)
- - ModelContext.__init__ eagerly loads the model
- - Creates CompressContext singleton
-
-MLLMCalibCompressor instance has:
- ✓ MLLM features from MLLMMixin (processor, template, calib() override)
- ✓ Calibration compression from CalibCompressor
- ✓ Model/context management from BaseCompressor
- """)
-
- print("=" * 110 + "\n")
-
-
-def print_decision_tree():
- """Print decision tree"""
-
- print("=" * 110)
- print("AutoRound Creation Decision Tree")
- print("=" * 110 + "\n")
-
- print("""
-AutoRound.__new__(config, model, format, **kwargs)
-│
-├─ Step 1: Detect model type
-│ model_type = detect_model_type(model)
-│ ├─ is_diffusion_model() → "diffusion"
-│ ├─ is_mllm_model() → "mllm"
-│ └─ else → "llm"
-│
-├─ isinstance(config, SignRoundConfig)
-│ ├─ model_type == "mllm"
-│ │ └─> class MLLMCalibCompressor(MLLMMixin, CalibCompressor)
-│ ├─ model_type == "diffusion"
-│ │ └─> class DiffusionCalibCompressor(DiffusionMixin, CalibCompressor)
-│ └─ model_type == "llm"
-│ └─> CalibCompressor
-│
-└─ isinstance(config, RTNConfig)
- │
- ├─ enable_imatrix OR needs_act_calib → CalibratedRTNCompressor path
- │ ├─ gguf_k format → enable_imatrix = True
- │ ├─ symmetric int RTN → enable_imatrix = True
- │ ├─ static activation quantization → needs_act_calib = True
- │ │
- │ ├─ model_type == "mllm"
- │ │ └─> class MLLMCalibratedRTNCompressor(MLLMMixin, CalibratedRTNCompressor)
- │ ├─ model_type == "diffusion"
- │ │ └─> class DiffusionCalibratedRTNCompressor(DiffusionMixin, CalibratedRTNCompressor)
- │ └─ model_type == "llm"
- │ └─> CalibratedRTNCompressor
- │
- └─ else (zero-shot) → ZeroShotCompressor path
- ├─ model_type == "mllm"
- │ └─> class MLLMZeroShotCompressor(MLLMMixin, ZeroShotCompressor)
- ├─ model_type == "diffusion"
- │ └─> class DiffusionZeroShotCompressor(DiffusionMixin, ZeroShotCompressor)
- └─ model_type == "llm"
- └─> ZeroShotCompressor
- """)
-
- print("=" * 110 + "\n")
-
-
-def print_quantizer_interface():
- """Print the BaseQuantizers interface contract"""
-
- print("=" * 110)
- print("BaseQuantizers Interface - Name-based quantize_block / quantize_layer")
- print("=" * 110 + "\n")
-
- print("""
-All quantizers use module *names* (str) instead of module objects.
-The module is retrieved internally via get_module(model, name).
-
- BaseQuantizers (abstract)
- ├─ quantize_block(block_name: Union[str, list[str]], input_ids, input_others, **kwargs)
- │ str → get_module(model, block_name)
- │ list[str] → WrapperMultiblock([get_module(model, n) for n in block_name])
- │ (used when nblocks > 1 in CalibCompressor)
- │
- └─ quantize_layer(layer_name: str, **kwargs)
- → get_module(model, layer_name)
-
- Implementations:
- ├─ RTNQuantizer.quantize_block(block_name: str)
- ├─ OptimizedRTNQuantizer.quantize_block(block_name: str, input_ids, input_others)
- └─ SignRoundQuantizer.quantize_block(block_name: Union[str, list[str]], input_ids, input_others)
- """)
-
- print("=" * 110 + "\n")
-
-
-def main():
- """Run all visualizations"""
-
- print_architecture_table()
- print_mixin_explanation()
- print_post_init_flow()
- print_usage_examples()
- print_mro_example()
- print_decision_tree()
- print_quantizer_interface()
-
- print("=" * 110)
- print("🎉 New architecture supports 9 combinations (3 model types × 3 compression algorithms)")
- print(" CalibratedRTNCompressor (was ImatrixCompressor) lives in calib.py")
- print("=" * 110)
-
-
-if __name__ == "__main__":
- main()
-
- print(f"{'LLM':<15} {'RTNConfig':<20} {'RTN (zero-shot)':<20} {'ZeroShotCompressor':<35}")
-
- print()
-
- # MLLM combinations
- print(f"{'MLLM':<15} {'SignRoundConfig':<20} {'AutoRoundCompatible':<20} {'MLLMCalibCompressor':<35}")
- print(f"{'':<15} {'':<20} {'':<20} {' = MLLMMixin + CalibCompressor':<35}")
- print(f"{'MLLM':<15} {'RTNConfig':<20} {'RTN + imatrix':<20} {'MLLMImatrixCompressor':<35}")
- print(f"{'':<15} {'':<20} {'':<20} {' = MLLMMixin + ImatrixCompressor':<35}")
- print(f"{'MLLM':<15} {'RTNConfig':<20} {'RTN (zero-shot)':<20} {'MLLMZeroShotCompressor':<35}")
- print(f"{'':<15} {'':<20} {'':<20} {' = MLLMMixin + ZeroShotCompressor':<35}")
-
- print()
-
- # Diffusion combinations
- print(f"{'Diffusion':<15} {'SignRoundConfig':<20} {'AutoRoundCompatible':<20} {'DiffusionCalibCompressor':<35}")
- print(f"{'':<15} {'':<20} {'':<20} {' = DiffusionMixin + CalibCompressor':<35}")
- print(f"{'Diffusion':<15} {'RTNConfig':<20} {'RTN + imatrix':<20} {'DiffusionImatrixCompressor':<35}")
- print(f"{'':<15} {'':<20} {'':<20} {' = DiffusionMixin + ImatrixCompressor':<35}")
- print(f"{'Diffusion':<15} {'RTNConfig':<20} {'RTN (zero-shot)':<20} {'DiffusionZeroShotCompressor':<35}")
- print(f"{'':<15} {'':<20} {'':<20} {' = DiffusionMixin + ZeroShotCompressor':<35}")
-
- print("\n" + "=" * 100 + "\n")
diff --git a/auto_round/compressors_new/base.py b/auto_round/compressors_new/base.py
deleted file mode 100644
index 09c01ed3f..000000000
--- a/auto_round/compressors_new/base.py
+++ /dev/null
@@ -1,1352 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import copy
-import gc
-import os
-import sys
-from dataclasses import asdict, dataclass, fields
-from typing import Any, Optional, Union
-
-import torch
-from transformers import set_seed
-
-from auto_round.algorithms.alg_config import AlgConfig
-from auto_round.algorithms.quantization import BaseQuantizers, QuantizationConfig
-from auto_round.algorithms.transforms import (
- BaseRotationConfig,
- apply_rotation,
-)
-from auto_round.compressors.utils import is_mx_fp, is_nv_fp
-from auto_round.compressors_new.shard_writer import ShardWriter
-from auto_round.compressors_new.utils import _get_save_folder_name, set_layer_config
-from auto_round.context.compress import CompressContext
-from auto_round.context.model import ModelContext
-from auto_round.formats import OutputFormat, get_formats
-from auto_round.logger import logger
-from auto_round.schemes import (
- QuantizationScheme,
- _handle_special_schemes,
- _parse_scheme,
- get_gguf_scheme,
- preset_name_to_scheme,
-)
-from auto_round.special_model_handler import get_predefined_fixed_attr, get_predefined_ignore_layers
-from auto_round.utils import (
- INNER_SUPPORTED_LAYER_TYPES,
- SUPPORTED_LAYER_TYPES,
- TORCH_VERSION_AT_LEAST_2_6,
- compress_layer_names,
- convert_dtype_str2torch,
- extract_block_names_to_str,
- find_matching_blocks,
- get_block_names,
- get_reverse_checkpoint_conversion_mapping,
- is_debug_mode,
- is_hpex_available,
- is_quantized_input_module,
- memory_monitor,
- revert_checkpoint_conversion_mapping,
-)
-from auto_round.utils.device import (
- _force_trim_malloc,
- get_major_device,
- patch_xpu_sdpa_drop_causal_mask,
- set_non_auto_device_map,
-)
-from auto_round.utils.offload import OffloadManager
-from auto_round.wrapper import wrapper_block
-
-
-@dataclass
-class SerializedCompressorConfig:
- bits: Optional[int] = None
- act_bits: Optional[int] = None
- data_type: Optional[str] = None
- act_data_type: Optional[str] = None
- group_size: Optional[int] = None
- act_group_size: Optional[int] = None
- sym: Optional[bool] = None
- act_sym: Optional[bool] = None
- act_dynamic: Optional[bool] = None
- amp: Optional[bool] = None
- batch_size: Optional[int] = None
- enable_minmax_tuning: Optional[bool] = True
- enable_norm_bias_tuning: Optional[bool] = False
- enable_quanted_input: Optional[bool] = True
- gradient_accumulate_steps: Optional[int] = None
- iters: Optional[int] = None
- lr: Optional[float] = None
- low_gpu_mem_usage: Optional[bool] = None
- minmax_lr: Optional[float] = None
- nsamples: Optional[int] = None
- quant_block_list: Optional[list[str]] = None
- regex_config: Optional[dict[str, Any]] = None
- scale_dtype: Optional[str] = None
- seqlen: Optional[int] = None
- supported_types: Optional[list[str]] = SUPPORTED_LAYER_TYPES
- static_attention_dtype: Optional[str] = None
- static_kv_dtype: Optional[str] = None
- super_bits: Optional[int] = None
- super_group_size: Optional[int] = None
- to_quant_block_names: Optional[list[str]] = None
- rotation_configs: Optional[list[dict[str, Any]]] = None
-
-
-class BaseCompressor(object):
- need_calib: bool = True
- compress_context: CompressContext = None
- model_context: ModelContext = None
- shard_writer: ShardWriter = None
- supported_types = SUPPORTED_LAYER_TYPES
- inner_supported_types = INNER_SUPPORTED_LAYER_TYPES
-
- # ── Scheme state (populated during resolve_scheme / _scheme_post_init) ──
- is_auto_scheme: bool = False
- orig_scheme = None
- scheme = None
- scale_dtype = None
- layer_config = None
- has_qlayer_outside_block: bool = False
- regex_config: dict = None
- quant_block_list: list = None
- to_quant_block_names = None
- ignore_layers: str = ""
- quant_lm_head: bool = False
- _scheme_resolved: bool = False
- scheme_generator = None
-
- def __init__(
- self,
- config: Union[AlgConfig, list[AlgConfig]],
- model: Union[torch.nn.Module, str],
- tokenizer=None,
- platform="hf",
- format=None,
- scheme="W4A16",
- low_gpu_mem_usage: bool = False,
- device_map: Union[str, torch.device, int, dict] = 0,
- enable_torch_compile: bool = False,
- seed: int = 42,
- low_cpu_mem_usage: bool = True,
- layer_config=None,
- nsamples: int = None,
- seqlen: int = None,
- **kwargs,
- ):
- self.quantize_config = None
- self.rotation_configs: list[BaseRotationConfig] = []
- _config_list = config if isinstance(config, list) else [config]
- for _cfg in _config_list:
- if isinstance(_cfg, QuantizationConfig):
- self.quantize_config = _cfg
- elif isinstance(_cfg, BaseRotationConfig):
- self.rotation_configs.append(_cfg)
- assert self.quantize_config is not None, "QuantizationConfig is required for Compressor"
-
- # Compressor-level calibration/layer params (do not live in QuantizationConfig).
- self.layer_config = layer_config
- self.nsamples = nsamples if nsamples is not None else 128
- self.seqlen = seqlen if seqlen is not None else 2048
-
- # Scheme is passed directly to the compressor, not stored in QuantizationConfig.
- self.scheme = scheme
-
- # TODO: refactor calibration
- self.calibration = None
-
- self.formats = format
-
- # Extra/legacy kwargs for backward compatibility
- # Major version releases may pack them with extra configuration options
- amp = kwargs.pop("amp", True)
- nblocks = kwargs.pop("nblocks", 1)
- disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)
- enable_deterministic_algorithms = kwargs.pop("enable_deterministic_algorithms", False)
-
- self._offloader = OffloadManager(enabled=low_cpu_mem_usage, mode="offload", offload_dir_prefix="compressor")
-
- # Model related
- model_dtype = kwargs.pop("model_dtype", None)
- trust_remote_code = kwargs.pop("trust_remote_code") if "trust_remote_code" in kwargs else True
- quant_nontext_module = kwargs.pop("quant_nontext_module", False)
-
- self.static_attention_dtype = kwargs.pop("static_attention_dtype", None)
- # Attention static dtype
- if self.static_attention_dtype is not None:
- logger.warning("The static attention dtype is experimental and currently has limited support.")
- # KV cache, this one does not affect tuning but will collect some infos during tuning
- self.static_kv_dtype = kwargs.pop("static_kv_dtype", None)
- if self.static_kv_dtype is not None:
- logger.warning("The static kv is experimental and currently has limited support.")
-
- if kwargs:
- logger.warning_once(
- f"unrecognized keys {list(kwargs.keys())} were passed. "
- "Please check them. If you use old api, just ignore this warning."
- )
- if "CUBLAS_WORKSPACE_CONFIG" not in os.environ:
- os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
- # Deprecated, default not to use torch.use_deterministic_algorithms
- if not disable_deterministic_algorithms or enable_deterministic_algorithms:
- if not disable_deterministic_algorithms:
- logger.warning(
- "default not use deterministic_algorithms. disable_deterministic_algorithms is deprecated,"
- " please use enable_deterministic_algorithms instead. "
- )
-
- torch.use_deterministic_algorithms(True, warn_only=False)
- else:
- torch.use_deterministic_algorithms(True, warn_only=True)
-
- # XPU SDPA workaround: drop pure causal masks so FLASH backend is used,
- # and set torch.use_deterministic_algorithms(False)
- # instead of MATH (avoids ~10x peak-VRAM blow-up during block tuning).
- patch_xpu_sdpa_drop_causal_mask()
-
- device = kwargs.pop("device", None)
- if device is not None:
- logger.warning("`device` is deprecated, please use `device_map` instead")
-
- # Tuning hyperparameters
- self.seed = seed
- set_seed(self.seed)
-
- self.nblocks = nblocks
-
- self.enable_torch_compile = enable_torch_compile
-
- # Whether to pack the layer immediately after tuning
- # Managed via self.compress_context.is_immediate_packing / is_immediate_saving
-
- torch.set_printoptions(precision=3, sci_mode=True)
-
- if is_hpex_available():
- logger.info("habana_frameworks is available, import htcore explicitly.")
- import habana_frameworks.torch.core as htcore # pylint: disable=E0401
-
- # Reset both context singletons before creating fresh instances so that
- # consecutive AutoRound creations don't inherit stale config from earlier ones.
- CompressContext.reset_context()
- ModelContext.reset_context()
-
- # Resolve the device eagerly so ModelContext can be created before
- # CompressContext. Creating ModelContext first places the large model
- # allocation early in the heap, matching the OLD arch allocation order
- # and reducing C-heap fragmentation (which is amplified on HPU).
- _device = get_major_device(device_map if device_map is not None else 0)
-
- self.model_context = ModelContext(
- model,
- tokenizer=tokenizer,
- platform=platform,
- model_dtype=model_dtype,
- trust_remote_code=trust_remote_code,
- amp=amp,
- need_calib=self.need_calib,
- device=_device,
- formats=self.formats,
- is_act_quantize=self.quantize_config.is_act_quantize,
- quant_nontext_module=quant_nontext_module,
- )
- # Alternatively, you can use CompressContext.create_context
- self.compress_context = CompressContext(
- low_cpu_mem_usage,
- low_gpu_mem_usage,
- device_map,
- enable_torch_compile,
- formats=self.formats,
- static_kv_dtype=self.static_kv_dtype,
- static_attention_dtype=self.static_attention_dtype,
- )
- self.shard_writer = None
-
- # scale_dtype is resolved in quantizer.resolve_scheme() after scheme resolution,
- # so it is not initialized here to avoid premature evaluation with an unresolved scheme.
-
- # Flag for post_init idempotency. Set to False here so post_init() can be called
- # either via quantize_and_save() (preferred, outside inference_mode) or directly
- # from quantize() as a fallback for non-AutoScheme cases.
- self._post_init_done = False
-
- # Apply eager state-only adjustments so ar.enable_torch_compile reflects
- # unsupported combinations immediately after construction.
- self._apply_eager_torch_compile_guards(enable_torch_compile)
- self.compress_context.enable_torch_compile = self.enable_torch_compile
-
- self.blocks_requiring_input_ids = []
- self.has_variable_block_shape = False
- fixed_attr = get_predefined_fixed_attr(self.model_context.model) or {}
- for key, value in fixed_attr.items():
- setattr(self, key, value)
-
- # ── Scheme resolution ─────────────────────────────────────────────────────
-
- def resolve_scheme(self, model_context=None, compress_context=None, dataset: str = None) -> None:
- """Phase-1 init: resolve scheme and bind config attrs (no model structure needed).
-
- Must be called BEFORE ``get_formats()`` and BEFORE ``_scheme_post_init()``.
- Idempotent: safe to call multiple times.
- """
- if self._scheme_resolved:
- return
-
- if model_context is not None:
- self.model_context = model_context
- if compress_context is not None:
- self.compress_context = compress_context
- if dataset is not None:
- self.dataset = dataset
-
- scheme_fields = {f.name for f in fields(QuantizationScheme)}
- user_scheme_overrides = {
- k: getattr(self.quantize_config, k)
- for k in scheme_fields
- if getattr(self.quantize_config, k, None) is not None
- }
- default_scheme, self.is_auto_scheme, final_attrs = _parse_scheme(self.scheme, user_scheme_overrides)
-
- for key, value in final_attrs.items():
- setattr(self.quantize_config, key, value)
- if hasattr(self, key):
- setattr(self, key, value)
- self.quantize_config.check_config()
-
- self.orig_scheme = copy.deepcopy(self.scheme)
- self.scheme = default_scheme
-
- gguf_scheme_name = get_gguf_scheme(self.scheme)
- if self.scale_dtype is None:
- self.scale_dtype = "fp32" if gguf_scheme_name else "fp16"
- self.scale_dtype = convert_dtype_str2torch(self.scale_dtype)
-
- self._scheme_resolved = True
-
- def _scheme_post_init(self) -> None:
- """Phase-4 init: build layer config on the patched model.
-
- Requires ``resolve_scheme()`` to have been called first.
- Must be called AFTER ``model_context.apply_patches()``.
- """
- assert self._scheme_resolved, (
- "resolve_scheme() must be called before _scheme_post_init(). "
- "BaseCompressor.post_init() does this automatically."
- )
-
- enable_gguf_official_mixed = not self.is_auto_scheme
-
- if self.quant_block_list is None:
- quant_nontext_module = getattr(self.model_context, "quant_nontext_module", False)
- all_blocks = get_block_names(self.model_context.model, quant_vision=quant_nontext_module)
- self.quant_block_list = find_matching_blocks(
- self.model_context.model, all_blocks, self.to_quant_block_names
- )
- if self.to_quant_block_names is None and self.quant_block_list:
- self.to_quant_block_names = extract_block_names_to_str(self.quant_block_list)
- self.quantize_config.to_quant_block_names = self.to_quant_block_names
-
- self.configure_layer_config(enable_gguf_official_mixed=enable_gguf_official_mixed)
-
- def _gen_auto_scheme(self) -> dict[str, dict]:
- """Generate per-layer config via AutoScheme delta-loss selection."""
- if self.model_context.is_mllm:
- # AutoScheme on a VLM only scores the language tower (the block
- # walker in delta_loss already skips vision/audio sub-trees) and
- # uses a pure-text calibration dataset by default, falling back to
- # the multimodal dataloader if the VLM rejects text-only forward.
- logger.info(
- "AutoScheme on multimodal LLM: scoring the language tower only "
- "with text-only calibration (multimodal dataloader will be used "
- "as a fallback if needed)."
- )
-
- if is_quantized_input_module(self.model_context.model):
- raise NotImplementedError("AutoScheme does not currently support quantized input models (e.g., FP8).")
-
- all_dtypes = []
- all_gguf = True
- for option in self.orig_scheme.options:
- dtype = "int"
- if isinstance(option, str):
- if not option.lower().startswith("gguf"):
- all_gguf = False
- option = preset_name_to_scheme(option)
- else:
- all_gguf = False
-
- if isinstance(option, QuantizationScheme):
- dtype = option.data_type
- elif isinstance(option, dict):
- dtype = option.get("data_type", "int")
-
- all_dtypes.append(dtype)
-
- unique_dtypes = set(all_dtypes)
- if len(unique_dtypes) > 1 and not all_gguf:
- logger.warning(
- "Models with mixed data_types "
- "cannot yet be exported to real formats except GGUF. "
- "Please save the model using the `fake` format for now."
- )
-
- layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
- self.model_context.model,
- self.layer_config,
- self.scheme,
- self.scale_dtype,
- self.supported_types,
- self.inner_supported_types,
- self.quant_block_list,
- self.ignore_layers,
- self.quant_lm_head,
- enable_gguf_official_mixed=False,
- is_mllm=self.model_context.is_mllm,
- )
- quant_layer_names = layer_config.keys()
-
- # ---- VLM: peel non-text sub-trees AutoScheme should not score ---- #
- nontext_skipped_layers: dict[str, dict] = {}
- if self.model_context.is_mllm:
- from auto_round.utils import get_block_names
-
- quant_nontext = getattr(self, "quant_nontext_module", False)
- scoreable_blocks = get_block_names(self.model_context.model, quant_vision=quant_nontext)
- scoreable_block_prefixes = tuple(blk for group in scoreable_blocks for blk in group)
- if quant_nontext:
- peel_markers = ("audio", "speech", "wav", "waveform")
- tower_label = "language+vision"
- peel_label = "audio/speech"
- else:
- peel_markers = (
- "vision",
- "visual",
- "image",
- "img",
- "audio",
- "speech",
- "wav",
- "waveform",
- )
- tower_label = "language"
- peel_label = "vision/audio"
-
- def _is_scoreable_layer(name: str) -> bool:
- if any(name == p or name.startswith(p + ".") for p in scoreable_block_prefixes):
- return True
- lname = name.lower()
- return not any(marker in lname for marker in peel_markers)
-
- scoreable_layer_config = {}
- for name, cfg in layer_config.items():
- if _is_scoreable_layer(name):
- scoreable_layer_config[name] = cfg
- else:
- nontext_skipped_layers[name] = cfg
-
- if nontext_skipped_layers:
- logger.info(
- "AutoScheme (VLM): scoring %d %s-tower layers; "
- "%d %s-tower layers kept at their original 16-bit configuration.",
- len(scoreable_layer_config),
- tower_label,
- len(nontext_skipped_layers),
- peel_label,
- )
- layer_config = scoreable_layer_config
- quant_layer_names = layer_config.keys()
-
- scheme_keys = {f.name for f in fields(QuantizationScheme)}
- fixed_layer_scheme_new = {
- k: {key: v[key] for key in scheme_keys & v.keys()}
- for k, v in layer_config.items()
- if v.get("fixed_by_user", False)
- }
-
- from auto_round.auto_scheme.gen_auto_scheme import GenScheme
-
- if (
- not self.compress_context.enable_torch_compile
- and self.quantize_config.super_bits is None
- and not self.orig_scheme.low_gpu_mem_usage
- ):
- logger.warning("we strongly recommend to set `enable_torch_compile` to True for AutoScheme to save VRAM")
- self.scheme_generator = GenScheme(
- self.orig_scheme,
- self.model_context.model,
- quant_layer_names,
- fixed_layer_scheme_new,
- self.dataset,
- device_map=self.compress_context.device_map,
- tokenizer=self.model_context.tokenizer,
- enable_torch_compile=self.compress_context.enable_torch_compile,
- processor=self.model_context.processor,
- )
- layer_config = self.scheme_generator.get_layer_config()
- # Re-attach vision/audio-tower layers we peeled off earlier so the
- # downstream quantization pipeline sees the complete layer map.
- if nontext_skipped_layers:
- allowed_keys = {f.name for f in fields(QuantizationScheme)} | {
- "fixed_by_user",
- "scale_dtype",
- "scheme",
- }
- for name, cfg in nontext_skipped_layers.items():
- clean_cfg = {k: v for k, v in cfg.items() if k in allowed_keys} if isinstance(cfg, dict) else cfg
- layer_config.setdefault(name, clean_cfg)
- return layer_config
-
- def configure_layer_config(self, enable_gguf_official_mixed: bool | None = True) -> None:
- """Build ``self.layer_config`` from the resolved scheme on the patched model."""
- is_gguf_format = (f := getattr(self.compress_context, "formats", None)) is not None and "gguf" in f
- predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)
- compressed_predefined_ignore_layers = compress_layer_names(predefined_ignore_layers)
- if not is_gguf_format:
- predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)
- if predefined_ignore_layers:
- logger.info(f"Using predefined ignore_layers: {compressed_predefined_ignore_layers}")
- tmp_str = ",".join(predefined_ignore_layers)
- if self.ignore_layers == "":
- self.ignore_layers = tmp_str
- else:
- self.ignore_layers += "," + tmp_str
-
- if self.is_auto_scheme:
- self.layer_config = self._gen_auto_scheme()
- else:
- self.layer_config = _handle_special_schemes(
- self.orig_scheme,
- self.layer_config,
- self.model_context.model,
- supported_types=SUPPORTED_LAYER_TYPES,
- inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,
- quant_lm_head=self.quant_lm_head,
- mllm=self.model_context.is_mllm,
- )
- _gguf_orig_fmt = getattr(self, "_gguf_original_format_name", None)
- if _gguf_orig_fmt and "_MIXED" in _gguf_orig_fmt.upper():
- self.layer_config = _handle_special_schemes(
- _gguf_orig_fmt.lower(),
- self.layer_config,
- self.model_context.model,
- supported_types=SUPPORTED_LAYER_TYPES,
- inner_supported_types=INNER_SUPPORTED_LAYER_TYPES,
- quant_lm_head=self.quant_lm_head,
- mllm=self.model_context.is_mllm,
- )
-
- fill_default_value = not self.is_auto_scheme
- self.layer_config, self.has_qlayer_outside_block, self.regex_config = set_layer_config(
- self.model_context.model,
- self.layer_config,
- self.scheme,
- self.scale_dtype,
- SUPPORTED_LAYER_TYPES,
- INNER_SUPPORTED_LAYER_TYPES,
- self.quant_block_list,
- self.ignore_layers,
- self.quant_lm_head,
- enable_gguf_official_mixed=enable_gguf_official_mixed,
- is_mllm=self.model_context.is_mllm,
- fill_default_value=fill_default_value,
- gguf_format_name=getattr(self, "_gguf_format_name", None),
- )
-
- # ─────────────────────────────────────────────────────────────────────────
-
- @property
- def mllm(self):
- return self.model_context.is_mllm
-
- @property
- def diffusion(self):
- return self.model_context.is_diffusion
-
- def _get_torch_compile_state(self) -> dict[str, Any]:
- """Collect raw torch.compile decision inputs from the current config state."""
- cfg = self.quantize_config
- raw_scheme = self.scheme if isinstance(self.scheme, str) else ""
- raw_dt = (cfg.data_type or "").lower()
- raw_adt = (cfg.act_data_type or "").lower()
- raw_scheme_upper = raw_scheme.upper()
-
- is_raw_nv_fp = "nv_fp" in raw_dt or "nv_fp" in raw_adt or "NVFP" in raw_scheme_upper
- is_raw_fp8 = (
- "fp8" in raw_dt
- or "fp8" in raw_adt
- or "FP8" in raw_scheme_upper
- or ("fp" in raw_dt and getattr(cfg, "bits", 16) == 8)
- or ("fp" in raw_adt and getattr(cfg, "act_bits", 16) == 8)
- )
-
- act_bits = getattr(cfg, "act_bits", 16) or 16
- super_group_size = getattr(cfg, "super_group_size", None)
- enable_alg_ext = getattr(cfg, "enable_alg_ext", False)
- return {
- "is_raw_nv_fp": is_raw_nv_fp,
- "is_raw_fp8": is_raw_fp8,
- "act_bits": act_bits,
- "super_group_size": super_group_size,
- "enable_alg_ext": enable_alg_ext,
- }
-
- def _apply_eager_torch_compile_guards(self, enable_torch_compile: bool) -> None:
- """Apply torch.compile disable rules that should take effect at construction time."""
- self.enable_torch_compile = enable_torch_compile
- state = self._get_torch_compile_state()
-
- # On HPU, we rely on torch.compile to speed up the model execution.
- if self.enable_torch_compile and state["is_raw_fp8"] and not is_hpex_available():
- self.enable_torch_compile = False
- logger.warning_once("reset enable_torch_compile to `False` as fp8 is enabled")
- # TODO: fix https://github.com/intel/auto-round/issues/1109
- if self.enable_torch_compile and state["is_raw_nv_fp"]:
- self.enable_torch_compile = False
- logger.warning_once("reset enable_torch_compile to `False` as nvfp4 is enabled")
- if self.enable_torch_compile and state["super_group_size"] is not None and state["enable_alg_ext"]:
- self.enable_torch_compile = False
- logger.warning_once(
- "reset enable_torch_compile to `False` as super_group_size is set for algorithm extension"
- )
-
- def _adjust_torch_compile(self, enable_torch_compile: bool) -> None:
- """Finalize torch compile configuration for tuning after init state is resolved."""
- state = self._get_torch_compile_state()
- self._apply_eager_torch_compile_guards(enable_torch_compile)
-
- if (
- not enable_torch_compile
- and TORCH_VERSION_AT_LEAST_2_6
- and state["act_bits"] > 8
- and not is_debug_mode()
- and not state["is_raw_fp8"]
- and self.need_calib
- ):
- logger.info(
- "%s",
- "'enable_torch_compile' is set to `False` by default. "
- "Enabling it can reduce tuning cost by 20%, but it might throw an exception.",
- )
-
- def _get_calibration_dataset(self) -> str:
- """Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""
- dataset = self.__dict__.get("dataset", None)
- if dataset is not None:
- return dataset
- from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
-
- scheme = self.scheme
- if isinstance(scheme, AutoScheme) and scheme.dataset:
- return scheme.dataset
- return "NeelNanda/pile-10k"
-
- def post_init(self) -> None:
- """One-time initialization that requires a loaded model.
-
- Call this OUTSIDE any ``@torch.inference_mode()`` context when using
- AutoScheme – delta-loss selection needs autograd (backward pass).
- ``quantize_and_save()`` does this automatically before entering the
- inference-mode quantize loop.
-
- Delegates to six ordered pipeline phases; see each ``_resolve_scheme``,
- ``_resolve_formats``, ``_create_quantizer``, ``_patch_model``,
- ``_build_layer_config``, and ``_hardware_setup`` for the precise
- preconditions and postconditions.
- """
- if self._post_init_done:
- return
-
- self._resolve_scheme()
-
- # After scheme resolution, is_act_quantize is known. When activation
- # quantization is enabled and the model is in float16, convert to
- # bfloat16 to match the old arch. This also detaches any parameter
- # tensors that are still backed by safetensors' mmap, preventing
- # per-block RSS growth (~14 MB/block) when .to(device) page-faults
- # the underlying file pages into physical memory.
- if self.quantize_config.is_act_quantize and self.model_context.amp_dtype == torch.float16:
- logger.warning("force to use bf16 for quantization tuning when enabling activation quantization")
- self.model_context.amp_dtype = torch.bfloat16
- if self.model_context.model.dtype != torch.bfloat16:
- self.model_context.model = self.model_context.model.to(torch.bfloat16)
-
- self._resolve_formats()
- self._create_quantizer()
- self._patch_model()
- self._build_layer_config()
- self._apply_rotations()
-
- # Reclaim temporaries from Phases 1-4 (scheme resolution, format
- # parsing, model patching, layer-config walk) before Phase 5
- # allocates hardware/compile objects. This compacts the heap so that
- # the fragmentation gap between live and freed blocks is minimised.
- gc.collect()
- _force_trim_malloc()
-
- self._hardware_setup()
-
- # Final trim after all init phases.
- gc.collect()
- _force_trim_malloc()
-
- self._post_init_done = True
-
- # ── Pipeline phase methods ────────────────────────────────────────────────
-
- def _resolve_scheme(self) -> None:
- """Phase 1 – Scheme resolution.
-
- Preconditions:
- - ``self.quantize_config`` is a valid :class:`QuantizationConfig`.
-
- Work performed:
- - Seeds scheme-related attrs (``scale_dtype``, ``ignore_layers``,
- ``quant_lm_head``, ``to_quant_block_names``) from ``quantize_config``.
- - Calls :meth:`resolve_scheme` to derive ``data_type``, ``bits``,
- ``sym``, ``scale_dtype`` etc. and write them back to both ``self``
- and ``self.quantize_config``.
-
- Postconditions:
- - ``self.scheme`` and ``self.quantize_config`` carry resolved scheme attrs.
- - ``self.quantizer`` is NOT yet created; that happens at the end of
- :meth:`_resolve_formats` once format-driven overrides (e.g. GGUF)
- have been applied to ``self.quantize_config``.
- """
- cfg = self.quantize_config
- self.scale_dtype = cfg.scale_dtype
- # self.layer_config is already set from __init__ (direct compressor param).
- self.ignore_layers = cfg.ignore_layers
- self.quant_lm_head = cfg.quant_lm_head
- self.to_quant_block_names = cfg.to_quant_block_names
-
- # Resolve the scheme (pure config work: sets data_type / bits / sym /
- # scale_dtype etc. on both self and self.quantize_config).
- self.resolve_scheme(
- model_context=self.model_context,
- compress_context=self.compress_context,
- dataset=self._get_calibration_dataset(),
- )
-
- def _resolve_formats(self) -> None:
- """Phase 2 – Format resolution and quantizer construction.
-
- Preconditions:
- - Phase 1 complete: the scheme is resolved (``data_type``, ``bits``,
- ``sym`` etc. are set on both ``self`` and ``self.quantize_config``).
-
- Work performed:
- - Converts a string ``self.formats`` to a list of
- :class:`~auto_round.formats.OutputFormat` objects via
- :func:`~auto_round.formats.get_formats`.
- - Initialises :class:`~auto_round.compressors_new.shard_writer.ShardWriter`
- when formats are present.
- - Detects format-driven attribute overrides (``bits``, ``sym``,
- ``data_type``, ``group_size``, etc.) that ``gguf_args_check`` may
- have written onto ``self`` inside ``get_formats``, syncs them back
- to ``self.quantize_config``, and rebuilds ``self.scheme``.
- - Merges any format-injected entries into ``self.layer_config``.
- Postconditions:
- - ``self.formats`` is a list (or ``None``).
- - ``self.compress_context.formats`` mirrors ``self.formats``.
- - ``self.quantize_config`` and ``self.scheme`` reflect the final attrs.
- - ``self.quantizer`` is NOT yet created; that happens in the next
- phase :meth:`_create_quantizer`.
- """
- # get_formats() inspects data_type / bits etc. that were just resolved.
- if isinstance(self.formats, str):
- self.formats = get_formats(self.formats, self)
- if self.formats is not None:
- self.compress_context.formats = self.formats
- ShardWriter.reset()
- # Defer ShardWriter construction to _ensure_shard_writer() to avoid
- # heap fragmentation during post_init (parameter iteration).
-
- # Snapshot the user-specified layer_config before format processing may
- # inject extra per-layer entries (e.g. GGUF embedding / lm_head).
- _pre_gguf_layer_config = copy.copy(self.layer_config) or {}
-
- # ── Propagate format-adjusted attrs back to quantize_config ──────────
- # gguf_args_check (called inside get_formats) may have overridden
- # bits / sym / data_type / super_bits / super_group_size / group_size
- # on *this* BaseCompressor object via setattr(self, ...). We need to
- # sync those changes to self.quantize_config before creating the
- # quantizer, so it is constructed with the correct final values.
- _gguf_forwarded_attrs = (
- "bits",
- "sym",
- "data_type",
- "super_bits",
- "super_group_size",
- "group_size",
- "act_bits",
- "scale_dtype",
- )
- _any_gguf_attr_changed = False
- for _attr in _gguf_forwarded_attrs:
- if _attr not in self.__dict__:
- continue
- config_val = getattr(self.quantize_config, _attr, None)
- self_val = self.__dict__[_attr]
- if _attr not in ("scale_dtype", "act_bits") and config_val != self_val:
- _any_gguf_attr_changed = True
- if config_val != self_val:
- setattr(self.quantize_config, _attr, self_val)
- # If format resolution changed scheme attrs, rebuild self.scheme so that
- # configure_layer_config() / set_layer_config() see the correct values.
- if _any_gguf_attr_changed:
- from auto_round.schemes import PRESET_SCHEMES
- from auto_round.schemes import QuantizationScheme as _QS
-
- # Prefer to derive the scheme directly from the gguf format name to
- # avoid ambiguity (e.g. Q4_K_S and Q4_K_M share identical weight attrs).
- _gguf_preset_scheme = None
- _gguf_fmt_name = None
- _gguf_original_fmt_name = None
- for _fmt in self.formats or []:
- # GGUFFormat (outer) has output_format="gguf" but backend.output_format="gguf:q4_k_m"
- # GGUFFormat (inner/standalone) has output_format="gguf:q4_k_m"
- _of = getattr(_fmt, "output_format", "")
- if "gguf" in str(_of):
- if str(_of) == "gguf":
- # outer GGUFFormat: full format in _original_format (e.g. "gguf:q2_k_mixed")
- # or backend.output_format (e.g. "gguf:q2_k_s" after _mixed → _s conversion)
- _orig = getattr(_fmt, "_original_format", None)
- if _orig:
- _gguf_original_fmt_name = str(_orig).upper()
- _backend = getattr(_fmt, "backend", None)
- _of = getattr(_backend, "output_format", _of) if _backend is not None else _of
- _preset_key = str(_of).upper()
- if _preset_key in PRESET_SCHEMES:
- _gguf_preset_scheme = PRESET_SCHEMES[_preset_key]
- _gguf_fmt_name = _preset_key
- break
- if _gguf_preset_scheme is not None:
- # Update scheme on both compressor and quantizer.
- self.scheme = _gguf_preset_scheme
- # Store the exact gguf format name so configure_layer_config /
- # set_layer_config can use it directly, avoiding Q4_K_S / Q4_K_M ambiguity.
- self._gguf_format_name = _gguf_fmt_name
- # Store original format name (may include _mixed) for _handle_special_schemes
- if _gguf_original_fmt_name:
- self._gguf_original_format_name = _gguf_original_fmt_name
- else:
- _new_scheme_dict = {f.name: getattr(self, f.name, None) for f in fields(_QS)}
- _new_scheme = _QS.from_dict({k: v for k, v in _new_scheme_dict.items() if v is not None})
- self.scheme = _new_scheme
-
- _gguf_layer_cfg = {
- k: v for k, v in (self.__dict__.get("layer_config") or {}).items() if k not in (_pre_gguf_layer_config)
- }
- if _gguf_layer_cfg:
- if self.layer_config is None:
- self.layer_config = {}
- for _lname, _lval in _gguf_layer_cfg.items():
- self.layer_config.setdefault(_lname, _lval)
-
- def _create_quantizer(self) -> None:
- """Phase 2.5 – Quantizer construction.
-
- Preconditions:
- - Phase 1 complete: ``self.quantize_config`` has the resolved scheme
- attrs (``data_type``, ``bits``, ``sym``, etc.).
- - Phase 2 complete: ``_resolve_formats`` has already synced any
- format-driven overrides (e.g. GGUF ``data_type=int_asym_dq``) back
- into ``self.quantize_config``. The quantizer is therefore created
- with the definitive, fully-finalised config in one shot.
-
- Postconditions:
- - ``self.quantizer`` is constructed and wired to model/context.
- - ``self.wrapper_block`` is set for use in the quantizer.
- """
- self.quantizer = BaseQuantizers.from_config(self.quantize_config)
- self.quantizer.model_context = self.model_context
- self.quantizer.compress_context = self.compress_context
- self.quantizer.model = self.model_context.model
- self.quantizer.scale_dtype = self.scale_dtype
- self.quantizer.seqlen = self.seqlen
- self.quantizer.nsamples = self.nsamples
- self.wrapper_block = wrapper_block
-
- def _apply_rotations(self) -> None:
- """Phase 4.5 – Apply Hadamard / rotation transforms to the model.
-
- Preconditions:
- - Phase 3 complete: model topology is final (``apply_patches`` has
- replaced / merged layers, e.g. MoE experts), so rotation operates
- on the same modules that quantization will later see.
- - Phase 4 complete: ``self.layer_config`` is built; rotation only
- transforms weights and does not change layer names, so this
- ordering matches the old arch where rotation ran after
- ``configure_layer_config``.
- - ``self.quantize_config.data_type`` is final (rotation backend
- dispatch depends on it).
-
- Work performed:
- - Iterates ``self.rotation_configs`` and calls
- :func:`~auto_round.algorithms.transforms.apply_rotation` on the
- model for each config.
-
- Postconditions:
- - ``self.model_context.model`` carries the rotated weights and any
- inserted online-Hadamard hooks.
- """
- if not self.rotation_configs:
- return
- logger.info("Applying Hadamard transform to the model.")
- for rotation_cfg in self.rotation_configs:
- self.model_context.model = apply_rotation(
- self.model_context.model,
- rotation_cfg,
- data_type=self.quantize_config.data_type,
- )
-
- def _patch_model(self) -> None:
- """Phase 3 – Model structure patching.
-
- Preconditions:
- - Phase 2 complete: ``self.formats`` is resolved so that
- ``apply_patches`` can inspect format-specific requirements.
-
- Work performed:
- - Delegates to :meth:`~auto_round.context.model.ModelContext.apply_patches`
- which may replace or merge layers (e.g. MoE expert merging, adding
- static-kv wrappers) to produce the final model topology.
-
- Postconditions:
- - ``self.model_context.model`` reflects the definitive topology that
- :meth:`_build_layer_config` will walk.
- """
- # apply_patches() may replace layers (e.g. MoE expert merging); must
- # happen before configure_layer_config() so it sees the final topology.
- self.model_context.apply_patches(self.formats)
-
- def _build_layer_config(self) -> None:
- """Phase 4 – Layer-config construction and quantizer sync.
-
- Preconditions:
- - Phase 3 complete: model topology is final.
- - ``self.scheme`` and all scheme-resolved attrs are consistent with
- the (possibly GGUF-adjusted) values set in Phase 2.
-
- Work performed:
- - Calls :meth:`_scheme_post_init` which walks the patched model to
- build ``self.layer_config``, ``self.quant_block_list``, etc.
- On the AutoScheme path this also runs delta-loss forward/backward
- passes to select per-layer schemes.
- - Syncs the fully-resolved ``layer_config`` and related attrs to
- ``self.quantizer`` so quantization methods have the complete view.
-
- Postconditions:
- - ``self.layer_config`` is fully populated.
- - ``self.quantizer`` mirrors ``layer_config``, ``has_qlayer_outside_block``,
- ``regex_config``, ``quant_block_list``, ``to_quant_block_names``,
- ``scale_dtype``, and ``ignore_layers``.
- """
- # configure_layer_config() walks the patched model; _gen_auto_scheme()
- # (AutoScheme path) runs delta-loss forward+backward passes.
- self._scheme_post_init()
-
- # Sync the fully-resolved scheme state to the quantizer so that
- # quantization methods (quantize_block, quantize_layer, etc.) have
- # access to layer_config, scale_dtype, quant_block_list, etc.
- self.quantizer.layer_config = self.layer_config
- self.quantizer.has_qlayer_outside_block = self.has_qlayer_outside_block
- self.quantizer.regex_config = self.regex_config
- self.quantizer.quant_block_list = self.quant_block_list
- self.quantizer.to_quant_block_names = self.to_quant_block_names
- self.quantizer.scale_dtype = self.scale_dtype
- self.quantizer.ignore_layers = self.ignore_layers
-
- def _hardware_setup(self) -> None:
- """Phase 5 – Hardware and compile configuration.
-
- Runs after layer_config is built and format/scheme attrs are final.
- Applies device placement, re-checks torch.compile eligibility, resets
- offload state, and finalizes inplace/immediate packing flags.
- """
- set_non_auto_device_map(self.model_context.model, self.compress_context.device_map)
- # Re-evaluate torch.compile eligibility now that data_type is resolved.
- self._adjust_torch_compile(self.enable_torch_compile)
- self.compress_context.enable_torch_compile = self.enable_torch_compile
- if self.compress_context.low_cpu_mem_usage:
- self._offloader.reset()
-
- # Disable inplace when quantized layers live outside transformer blocks.
- # Default to True when not already initialised (e.g. post_init() called
- # directly rather than via quantize_and_save()).
- if not hasattr(self, "inplace"):
- self.inplace = True
- if self.has_qlayer_outside_block and self.need_calib:
- self.inplace = False
-
- if not hasattr(self, "formats"):
- logger.warning("this API is deprecated, please use `quantize_and_save` instead")
- else:
- self._adjust_immediate_packing_and_saving()
-
- # backward compatible with the legacy API
- def __getattr__(self, name: str) -> Any:
- if name in self.__dict__:
- return self.__dict__[name]
-
- for obj in ["quantizer", "quantize_config", "model_context", "compress_context"]:
- if obj not in self.__dict__:
- continue
- obj = object.__getattribute__(self, obj)
- try:
- return object.__getattribute__(obj, name)
- except AttributeError:
- continue
-
- raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
-
- @property
- def optimizer(self):
- """Return the actual optimizer class, converting string to class for backward compat.
-
- Old API stored ``self.optimizer = torch.optim.AdamW`` (the class itself).
- New arch stores the optimizer name as a string in ``quantize_config.optimizer``.
- This property converts it so that ``ar.optimizer == torch.optim.AdamW`` works.
- """
- if self.quantize_config is None:
- return None
- opt = getattr(self.quantize_config, "optimizer", None)
- if opt is None:
- # Default to AdamW when enable_adam=True and no explicit optimizer was set
- if getattr(self.quantize_config, "enable_adam", False):
- return torch.optim.AdamW
- return None
- if isinstance(opt, str):
- return getattr(torch.optim, opt, None)
- return opt
-
- def _adjust_immediate_packing_and_saving(self):
- from auto_round.algorithms.quantization.rtn.config import RTNConfig
-
- if self.formats is None:
- return
-
- formats = getattr(self, "formats", [])
- if len(formats) == 1 and not formats[0].is_fake() and self.inplace:
- self.compress_context.is_immediate_packing = True
-
- if self.has_qlayer_outside_block and self.need_calib:
- self.compress_context.is_immediate_packing = False
-
- if not ("causallm" in self.model_context.model.__class__.__name__.lower() and not self.model_context.is_mllm):
- # TODO For tied keys, there may some issues, we haven't not verified this
- tied_weight_keys = getattr(self.model_context.model, "_tied_weight_keys", {})
- if len(tied_weight_keys) > 1:
- self.compress_context.is_immediate_saving = False
- if self.compress_context.low_cpu_mem_usage:
- logger.warning("reset low_cpu_mem_usage to False due to tied weights")
- return
- if len(tied_weight_keys) == 1:
- key = list(tied_weight_keys.keys())[0]
- if "lm_head" not in key:
- self.compress_context.is_immediate_saving = False
- if self.compress_context.low_cpu_mem_usage:
- logger.warning("reset low_cpu_mem_usage to False due to tied weights")
- return
-
- if self.compress_context.low_cpu_mem_usage and self.compress_context.is_immediate_packing:
- self.compress_context.is_immediate_saving = True
-
- if self.compress_context.low_cpu_mem_usage and self.compress_context.is_immediate_packing:
- if formats[0].is_gguf():
- logger.warning(
- "`low_cpu_mem_usage` is not fully supported for gguf format. "
- "Setting `low_cpu_mem_usage` to False."
- )
- self.compress_context.low_cpu_mem_usage = False
- self.compress_context.is_immediate_saving = False
- elif (
- self.has_qlayer_outside_block
- and getattr(self, "disable_opt_rtn", None)
- and isinstance(self.quantize_config, RTNConfig)
- ):
- logger.info(
- "Keeping `low_cpu_mem_usage` enabled in RTN mode (iters=0): "
- "RTN path uses blockwise quantization and supports per-block offloading."
- )
- elif self.has_qlayer_outside_block and not isinstance(self.quantize_config, RTNConfig):
- logger.warning(
- "`low_cpu_mem_usage` is not fully supported "
- "when there are quantized layers outside blocks and optimized RTN is disabled. "
- "Setting low_cpu_mem_usage to False."
- )
- self.compress_context.low_cpu_mem_usage = False
- self.compress_context.is_immediate_saving = False
-
- if self.compress_context.is_immediate_saving and not (
- "int" in self.quantize_config.data_type
- or is_nv_fp(self.quantize_config.data_type)
- or is_mx_fp(self.quantize_config.data_type)
- ):
- logger.warning("immediate_saving is only supported for int/nv_fp/mx_fp quantization, set to False")
- self.compress_context.is_immediate_saving = False
-
- if self.output_dir is None:
- self.compress_context.is_immediate_saving = False
-
- # Create ShardWriter eagerly only when immediate saving is active
- # (it interleaves with the quantize loop). Otherwise keep it deferred
- # until save_quantized() to avoid heap fragmentation during init.
- if self.compress_context.is_immediate_saving:
- self._ensure_shard_writer()
-
- def _ensure_shard_writer(self):
- """Lazily create ShardWriter if it hasn't been created yet."""
- if self.shard_writer is None and self.formats is not None:
- self.shard_writer = ShardWriter(self.model_context.model, bits=8)
-
- def quantize(self) -> tuple[torch.nn.Module, dict[str, Any]]:
- """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.
- Returns:
- The quantized model and layer configurations.
- """
- raise NotImplementedError("quantize method must be implemented in subclass")
-
- def save_quantized(
- self,
- output_dir: str = None,
- format: Union[str, list[OutputFormat]] = None,
- inplace: bool = True,
- return_folders=False,
- **kwargs,
- ) -> torch.nn.Module:
- """Save the quantized model to the specified output directory in the specified format.
-
- Args:
- output_dir (str, optional): The directory to save the quantized model. Defaults to None.
- format (str, optional): The format in which to save the model. Defaults to "auto_round".
- inplace (bool, optional): Whether to modify the model in place. Defaults to True.
- **kwargs: Additional keyword arguments specific to the export format.
-
- Returns:
- object: The compressed model object.
- """
- self.output_dir = output_dir
- if output_dir is not None:
- self.compress_context.output_dir = output_dir
- if format is not None:
- if isinstance(format, str) and getattr(self, "formats", None) is None:
- logger.warning(
- f"save_quantized with format is deprecated and will be deleted in auto_round version 1.0."
- f" Please use Compressor(format='{format}' instead)."
- )
- self.formats = get_formats(format, self)
- self.compress_context.formats = self.formats
-
- if not self.model_context.quantized:
- logger.warning("please run autoround.quantize first")
- return
- folders = []
- if self.formats is None:
- logger.info("format is not set, using default auto_round format.")
- self.formats = "auto_round"
- if isinstance(self.formats, str):
- self.formats = get_formats(self.formats, self)
- self.compress_context.formats = self.formats
- for format in self.formats:
- save_folder = _get_save_folder_name(format)
- if self.act_bits <= 8 and format.is_fake():
- logger.warning(
- "Support for exporting activation quantization is limited. "
- "Please ensure that your configuration is supported."
- )
-
- serialization_dict = asdict(SerializedCompressorConfig())
- for key in serialization_dict:
- serialization_dict[key] = getattr(self, key, serialization_dict[key])
- from auto_round.version import __version__
-
- serialization_dict["autoround_version"] = __version__
- if serialization_dict.get("to_quant_block_names") is None and self.quantizer.quant_block_list:
- serialization_dict["to_quant_block_names"] = extract_block_names_to_str(self.quantizer.quant_block_list)
- if "scale_dtype" in serialization_dict.keys():
- serialization_dict["scale_dtype"] = str(serialization_dict["scale_dtype"])
-
- # to match the original name
- reverse_checkpoint_conversion_mapping = get_reverse_checkpoint_conversion_mapping(self.model)
-
- if isinstance(serialization_dict["to_quant_block_names"], str):
- serialization_dict["to_quant_block_names"] = revert_checkpoint_conversion_mapping(
- serialization_dict["to_quant_block_names"], reverse_checkpoint_conversion_mapping
- )
-
- elif isinstance(serialization_dict["to_quant_block_names"], list):
- for idx in range(len(serialization_dict["to_quant_block_names"])):
- serialization_dict["to_quant_block_names"][idx] = revert_checkpoint_conversion_mapping(
- serialization_dict["to_quant_block_names"][idx], reverse_checkpoint_conversion_mapping
- )
-
- compressed_model = format.save_quantized(
- save_folder,
- model=self.model_context.model,
- layer_config=self.quantizer.layer_config,
- inplace=inplace,
- tokenizer=self.model_context.tokenizer,
- device=self.compress_context.device,
- serialization_dict=serialization_dict,
- **kwargs,
- )
- folders.append(save_folder)
-
- if return_folders:
- if len(folders) == 1:
- folders = folders[0]
- return compressed_model, folders
- else:
- return compressed_model
-
- def _get_export_dir(self, output_dir: str, format_str: str) -> str:
- """Derive a descriptive export directory from model name and quantization config.
-
- Must be called after ``post_init()`` so that scheme-resolved attrs
- (bits, group_size, data_type, etc.) are available on ``self.quantize_config``.
-
- Mirrors the logic previously in ``__main__.py`` so callers only need to
- pass the base ``output_dir`` and the format string.
- """
- # Diffusion models use save_quantized from DiffusionMixin which manages its own
- # directory layout (model_index.json + per-component subdirs). Appending a
- # scheme-derived suffix here would place files one level too deep.
- if getattr(self, "diffusion", False):
- return output_dir
-
- model_name = (getattr(self.model_context.model, "name_or_path", "") or "").rstrip("/")
- cfg = self.quantize_config
- group_size = cfg.group_size
- bits = cfg.bits
- data_type = cfg.data_type or "int"
- act_bits = cfg.act_bits or 16
- act_data_type = cfg.act_data_type or "float"
-
- is_gguf = "gguf" in (format_str or "")
- last = model_name.split("/")[-1].strip(".")
-
- if last == "" and not is_gguf:
- # model path is just '.' or './' – put inside output_dir with suffix
- if group_size <= 0:
- suffix = f"afp{act_bits}" if "fp" in act_data_type else f"a{act_bits}"
- else:
- suffix = f"g{group_size}"
- return os.path.join(output_dir, f"w{bits}{suffix}")
-
- if last == "" and is_gguf:
- return output_dir
-
- if is_gguf:
- return os.path.join(output_dir, model_name.split("/")[-1] + "-gguf")
-
- # Normal case: derive suffix from group_size / act config
- if isinstance(group_size, tuple):
- assert len(group_size) == 2, f"Only support 2D group_size, but got {group_size}"
- suffix = f"g{group_size[0]}x{group_size[1]}"
- elif group_size <= 0:
- suffix = f"afp{act_bits}" if "fp" in act_data_type else f"a{act_bits}"
- else:
- suffix = f"g{group_size}"
-
- prefix = data_type.lower().replace("_", "") if "int" not in data_type or "mx" in data_type else ""
- return os.path.join(
- output_dir,
- model_name.split("/")[-1] + (f"-{prefix}" if prefix else "") + f"-w{bits}{suffix}",
- )
-
- def quantize_and_save(
- self, output_dir: str = "tmp_autoround", format: str = None, inplace: bool = True, **kwargs
- ) -> tuple[torch.nn.Module, dict[str, Any]]:
- """Quantizes the model and saves it in the specified format(s).
-
- This function checks the validity of the requested format(s), quantizes
- the model accordingly, and saves it to the specified output directory.
- If multiple formats are provided, the model is saved separately for each format.
-
- Args:
- output_dir (str, optional): The directory where the quantized model
- will be saved. Defaults to "tmp_autoround".
- format (str, optional): The quantization format(s) to use, separated
- by commas if multiple. Defaults to "auto_round".
- inplace (bool, optional): Whether to modify the model in place if only
- one format is used. Defaults to True.
- **kwargs: Additional arguments for the quantization and saving process.
-
- Returns:
- model: A qdq model or packed model based on the configurations
- folders: The folder paths where the quantized models are saved.
-
- Raises:
- ValueError: If an unsupported format is specified.
- """
- # Validate and process the specified formats
- self.output_dir = output_dir
- self.compress_context.output_dir = output_dir
-
- # check and update the format based on the current configuration
- if format and self.formats is None:
- logger.warning(
- f"quantize_and_save with format is deprecated and will be deleted in auto_round version 1.0."
- f" Please use Compressor(format='{format}' instead)."
- )
- self.formats = format
- if self.formats is None:
- logger.info("format is not set, using default auto_round format.")
- self.formats = "auto_round"
-
- # If multiple formats are specified, enforce inplace=False
- if len(self.formats.split(",")) > 1:
- inplace = False
- self.inplace = kwargs.get("inplace", inplace)
- kwargs.pop("inplace", None)
-
- # Perform model quantization
- # IMPORTANT: post_init() must run outside any @torch.inference_mode() context
- # because AutoScheme's delta-loss selection requires gradient tracking.
- self.post_init()
- # If post_init() was called manually before quantize_and_save() (e.g. ar.post_init()
- # in tests), _resolve_formats saw formats=None and was a no-op. Now that we have set
- # self.formats to a default string above, resolve it into OutputFormat objects so that
- # quantize() and save_quantized() receive proper objects, not a raw string.
- if isinstance(self.formats, str):
- self.formats = get_formats(self.formats, self)
- self.compress_context.formats = self.formats
- # Derive descriptive export dir after post_init so scheme-resolved attrs are available.
- _fmt_str = format or (self.formats if isinstance(self.formats, str) else "")
- output_dir = self._get_export_dir(output_dir, _fmt_str)
- self.output_dir = output_dir
- self.compress_context.output_dir = output_dir
- if self.static_attention_dtype is not None:
- from auto_round.experimental.attention import attention_quant_ctx
-
- with attention_quant_ctx(self.model_context.model, static_attention_dtype=self.static_attention_dtype):
- self.quantize()
- self.model_context.quantized = True
- elif self.static_kv_dtype is not None:
- from auto_round.experimental.kv_cache import kvcache_quant_context
-
- with kvcache_quant_context(self.model_context.model, static_kv_dtype=self.static_kv_dtype):
- self.quantize()
- self.model_context.quantized = True
- else:
- self.quantize()
- self.model_context.quantized = True
-
- # Ensure ShardWriter is ready before saving (deferred from post_init).
- self._ensure_shard_writer()
-
- # Save the quantized model in the specified format_list
- model, folders = self.save_quantized(output_dir, inplace=inplace, return_folders=True, **kwargs)
- memory_monitor.log_summary()
-
- return model, folders
diff --git a/auto_round/compressors_new/config.py b/auto_round/compressors_new/config.py
deleted file mode 100644
index dd028b6cd..000000000
--- a/auto_round/compressors_new/config.py
+++ /dev/null
@@ -1,296 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from __future__ import annotations
-
-from dataclasses import dataclass, fields
-from typing import Any, Callable, Optional, Union
-
-import torch
-
-
-class ExtraConfig:
- """Class for extra or legacy configs."""
-
- _model_config = None
- _scheme_config = None
- _tuning_config = None
- _mllm_config = None
- _diffusion_config = None
-
- def __init__(
- self,
- # tuning
- amp: bool = True,
- disable_opt_rtn: bool | None = None,
- enable_alg_ext: bool = False,
- enable_minmax_tuning: bool = True,
- enable_norm_bias_tuning: bool = False,
- enable_quanted_input: bool = True,
- enable_deterministic_algorithms: bool = False,
- lr: float = None,
- lr_scheduler: Callable = None,
- minmax_lr: float = None,
- nblocks: int = 1,
- to_quant_block_names: Union[str, list, None] = None,
- scale_dtype: str = "fp16",
- # scheme
- bits: int = None,
- group_size: int = None,
- sym: bool = None,
- data_type: str = None,
- act_bits: int = None,
- act_group_size: int = None,
- act_sym: bool = None,
- act_data_type: str = None,
- act_dynamic: bool = None,
- super_bits: int = None,
- super_group_size: int = None,
- static_kv_dtype: Union[str, torch.dtype] = None,
- quant_lm_head: bool = False,
- ignore_layers: str = None,
- # mllm
- processor: Callable = None,
- image_processor: Callable = None,
- quant_nontext_module: bool = False,
- extra_data_dir: str = None,
- template: str = None,
- # diffusion
- guidance_scale: float = 7.5,
- num_inference_steps: int = 50,
- generator_seed: int = None,
- ):
- """Initialize
-
- Args:
- amp (bool): Whether to use automatic mixed precision (default is True).
- disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to True.
- enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.
- enable_minmax_tuning (bool, optional): Enable weight min-max tuning. Defaults to True.
- enable_norm_bias_tuning (bool): Whether to enable fast norm/layer_bias tuning.
- enable_quanted_input (bool): Whether to use quantized input data (default is True).
- enable_deterministic_algorithms (bool): Whether to use deterministic_algorithms.
- lr (float): The learning rate (default is 0.005).
- lr_scheduler: The learning rate scheduler to be used.
- minmax_lr (float): The learning rate for min-max tuning (default is None).
- nblocks (int): Number of blocks (default is 1).
- quant_lm_head (bool): Whether to quant lm_head.
- to_quant_block_names (str|list): Names of quantitative blocks, please use commas to separate them.
- scale_dtype (str): The data type of quantization scale to be used (default is "float16"), different kernels
- bits (int, optional): Weight quantization bits. Defaults to 4.
- group_size (int, optional): Weight quantization group size. Defaults to 128.
- sym (bool, optional): Symmetric weight quantization. Defaults to True.
- data_type (str, optional): Weight data type string, e.g., "int". Defaults to "int".
- act_bits (int, optional): Activation quantization bits. Defaults to 16.
- act_group_size (int, optional): Activation group size. Defaults to None.
- act_sym (bool, optional): Symmetric activation quantization. Defaults to None.
- act_data_type (str, optional): Activation data type; inherits weight dtype if None and act_bits < 16.
- act_dynamic (bool, optional): Dynamic activation quantization. Defaults to True.
- super_bits (int): number of scale and mins quant bits for double quant.
- super_group_size (int): the number of super group size when use double quant.
- static_kv_dtype (str): The data type of kv-cache to be used.
- processor: Any multi-modal model will require an object to encode or
- decode the data that groups several modalities (among text, vision and audio).
- image_processor: Image processor for special model like llava.
- quant_nontext_module: Whether to quantize nontext module.
- extra_data_dir: The path of extra data such as images, audio and videos.
- template: The path or name of template used to specify process for different MLLMs.
- guidance_scale (float): Control how much the image generation process follows the text prompt.
- The more it is, the more closely it follows the prompt (default is 7.5).
- num_inference_steps (int): The reference number of denoising steps (default is 50).
- generator_seed (int): A seed that controls the initial noise for image generation (default is None).
- """
- self.tuning_config = TuningExtraConfig(
- amp=amp,
- disable_opt_rtn=disable_opt_rtn,
- enable_alg_ext=enable_alg_ext,
- enable_minmax_tuning=enable_minmax_tuning,
- enable_norm_bias_tuning=enable_norm_bias_tuning,
- enable_quanted_input=enable_quanted_input,
- enable_deterministic_algorithms=enable_deterministic_algorithms,
- lr=lr,
- lr_scheduler=lr_scheduler,
- minmax_lr=minmax_lr,
- nblocks=nblocks,
- to_quant_block_names=to_quant_block_names,
- scale_dtype=scale_dtype,
- )
- self.scheme_config = SchemeExtraConfig(
- bits=bits,
- group_size=group_size,
- sym=sym,
- data_type=data_type,
- act_bits=act_bits,
- act_group_size=act_group_size,
- act_sym=act_sym,
- act_data_type=act_data_type,
- act_dynamic=act_dynamic,
- super_bits=super_bits,
- super_group_size=super_group_size,
- static_kv_dtype=static_kv_dtype,
- quant_lm_head=quant_lm_head,
- ignore_layers=ignore_layers,
- )
- self.mllm_config = MLLMExtraConfig(
- processor=processor,
- image_processor=image_processor,
- quant_nontext_module=quant_nontext_module,
- extra_data_dir=extra_data_dir,
- template=template,
- )
- self.diffusion_config = DiffusionExtraConfig(
- guidance_scale=guidance_scale,
- num_inference_steps=num_inference_steps,
- generator_seed=generator_seed,
- )
-
- @property
- def tuning_config(self):
- return self._tuning_config
-
- @tuning_config.setter
- def tuning_config(self, config: TuningExtraConfig):
- assert isinstance(
- config, TuningExtraConfig
- ), f"tuning_config should be ModelExtraConfig, but got {config.__class__.__name__}"
- self._tuning_config = config
-
- @property
- def scheme_config(self):
- return self._scheme_config
-
- @scheme_config.setter
- def scheme_config(self, config: SchemeExtraConfig):
- assert isinstance(
- config, SchemeExtraConfig
- ), f"scheme_config should be SchemeExtraConfig, but got {config.__class__.__name__}"
- self._scheme_config = config
-
- @property
- def mllm_config(self):
- return self._mllm_config
-
- @mllm_config.setter
- def mllm_config(self, config: MLLMExtraConfig):
- if config is None:
- self._mllm_config = None
- else:
- assert isinstance(
- config, MLLMExtraConfig
- ), f"mllm_config should be MLLMExtraConfig, but got {config.__class__.__name__}"
- self._mllm_config = config
-
- @property
- def diffusion_config(self):
- return self._diffusion_config
-
- @diffusion_config.setter
- def diffusion_config(self, config: DiffusionExtraConfig):
- if config is None:
- self._diffusion_config = None
- else:
- assert isinstance(
- config, DiffusionExtraConfig
- ), f"diffusion_config should be DiffusionExtraConfig, but got {config.__class__.__name__}"
- self._diffusion_config = config
-
- def to_dict(self):
- output_dict = {}
- for config in self.__dict__.values():
- if config:
- output_dict.update(config.to_dict())
- return output_dict
-
-
-@dataclass
-class BaseExtraConfig:
-
- @classmethod
- def get_attributes(cls: "BaseExtraConfig") -> list[str]:
- return [field.name for field in fields(cls)]
-
- def __getitem__(self, key: str):
- if key not in self.get_attributes():
- raise KeyError(f"{key} is not a valid attribute")
- return getattr(self, key)
-
- def __setitem__(self, key: str, value: None | int | str):
- if key not in self.get_attributes():
- raise KeyError(f"{key} is not a valid attribute")
- setattr(self, key, value)
-
- def __contains__(self, item):
- return item in self.get_attributes()
-
- def to_dict(self):
- return self.__dict__
-
- def is_default(self):
- for field in fields(self):
- default_value = field.default
- current_value = getattr(self, field.name)
- if current_value != default_value:
- return False
- return True
-
-
-@dataclass
-class TuningExtraConfig(BaseExtraConfig):
- amp: bool = True
- disable_opt_rtn: bool | None = None
- enable_alg_ext: bool = False
- enable_minmax_tuning: bool = True
- enable_norm_bias_tuning: bool = False
- enable_quanted_input: bool = True
- enable_deterministic_algorithms: bool = False
- lr: float = None
- lr_scheduler: Callable = None
- minmax_lr: float = None
- nblocks: int = 1
- to_quant_block_names: Union[str, list, None] = None
- scale_dtype: str = "fp16"
-
-
-@dataclass
-class SchemeExtraConfig(BaseExtraConfig):
- bits: int = None
- group_size: int = None
- sym: bool = None
- data_type: str = None
- act_bits: int = None
- act_group_size: int = None
- act_sym: bool = None
- act_data_type: str = None
- act_dynamic: bool = None
- super_bits: int = None
- super_group_size: int = None
- static_kv_dtype: Union[str, torch.dtype] = None
- static_attention_dtype: Union[str, torch.dtype] = None
- quant_lm_head: bool = False
- ignore_layers: str = None
-
-
-@dataclass
-class MLLMExtraConfig(BaseExtraConfig):
- processor: Callable = None
- image_processor: Callable = None
- quant_nontext_module: bool = False
- extra_data_dir: str = None
- template: str = None
-
-
-@dataclass
-class DiffusionExtraConfig(BaseExtraConfig):
- guidance_scale: float = 7.5
- num_inference_steps: int = 50
- generator_seed: int = None
diff --git a/auto_round/compressors_new/mllm_mixin.py b/auto_round/compressors_new/mllm_mixin.py
deleted file mode 100644
index ada1eacf5..000000000
--- a/auto_round/compressors_new/mllm_mixin.py
+++ /dev/null
@@ -1,281 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import torch
-
-from auto_round.logger import logger
-from auto_round.utils import to_device
-
-
-class MLLMMixin:
- """MLLM-specific functionality mixin.
-
- This mixin adds MLLM-specific functionality to any compressor (CalibCompressor,
- ZeroShotCompressor, ImatrixCompressor, etc). It handles multi-modal models
- (vision-language models) that require special data loading and processing logic.
-
- Can be combined with:
- - CalibCompressor (for AutoRound with calibration)
- - ImatrixCompressor (for RTN with importance matrix)
- - ZeroShotCompressor (for basic RTN)
-
- MLLM-specific parameters:
- processor: Multi-modal processor override (normally loaded by ModelContext)
- image_processor: Image processor override (e.g. for LLaVA)
- template: Template name for processing different MLLMs
- extra_data_dir: Path to extra data (images, audio, videos)
- quant_nontext_module: Whether to quantize non-text modules
-
- Design note:
- ``ModelContext._load_model()`` is responsible for loading the model and its
- associated artifacts (processor, tokenizer, image_processor). This mixin
- reads those artifacts from ``self.model_context`` during calibration.
- If the caller passes explicit ``processor`` / ``image_processor`` overrides,
- they are written into ``model_context`` after ``super().__init__()`` so that
- ``model_context`` remains the single source of truth.
- """
-
- def __init__(
- self,
- *args,
- processor=None,
- image_processor=None,
- template=None,
- extra_data_dir=None,
- quant_nontext_module=False,
- **kwargs,
- ):
- self.template = template
- self.extra_data_dir = extra_data_dir
- self.quant_nontext_module = quant_nontext_module
- self.template_obj = None
-
- # Pass quant_nontext_module to ModelContext so get_block_names can include vision blocks
- kwargs.setdefault("quant_nontext_module", quant_nontext_module)
-
- # Mirror old arch: reset batch_size to 1 when quantizing non-text modules,
- # because vision encoder blocks have non-standard hidden_states shapes that
- # break batch_dim detection, and image collation fails with batch_size > 1.
- if quant_nontext_module:
- # batch_size may come from kwargs (placed there by AutoRoundCompatible local_args)
- # or from the AlgConfig object in args[0] (the authoritative source for quantizer.batch_size).
- # We must update both so that quantizer.batch_size is also reset to 1.
- batch_size = kwargs.get("batch_size", None)
- _alg_cfg = args[0] if args else None
- if batch_size is None and _alg_cfg is not None:
- cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
- for cfg in cfgs:
- if hasattr(cfg, "batch_size") and cfg.batch_size is not None:
- batch_size = cfg.batch_size
- break
- if batch_size is not None and batch_size != 1:
- grad_acc = kwargs.get("gradient_accumulate_steps", 1)
- new_grad_acc = batch_size * grad_acc
- kwargs["gradient_accumulate_steps"] = new_grad_acc
- kwargs["batch_size"] = 1
- # Also patch the AlgConfig object so that BaseCompressor.quantize_config.batch_size == 1
- if _alg_cfg is not None:
- cfgs = _alg_cfg if isinstance(_alg_cfg, list) else [_alg_cfg]
- for cfg in cfgs:
- if hasattr(cfg, "batch_size"):
- cfg.batch_size = 1
- if hasattr(cfg, "gradient_accumulate_steps"):
- cfg.gradient_accumulate_steps = new_grad_acc
- logger.warning(
- f"reset batch_size({batch_size}) to 1 and "
- f"gradient_accumulate_steps to {new_grad_acc} "
- f"because batch_size={batch_size} cannot be used for calibrating non-text modules."
- )
-
- # super().__init__() creates model_context, which eagerly loads the model and
- # populates model_context.processor / image_processor / tokenizer.
- super().__init__(*args, **kwargs)
-
- # Apply user-provided overrides into model_context (single source of truth).
- if processor is not None:
- self.model_context.processor = processor
- if image_processor is not None:
- self.model_context.image_processor = image_processor
-
- @torch.no_grad()
- def calib(self, nsamples, bs):
- """Perform MLLM-specific calibration for quantization.
-
- Override parent's calib method to use MLLM dataset loading logic.
- All multimodal artifacts are read from ``self.model_context``.
- """
- from transformers import PreTrainedModel
-
- from auto_round.compressors.mllm.dataset import get_mllm_dataloader
- from auto_round.compressors.mllm.template import get_template
- from auto_round.special_model_handler import MISTRAL_3_2_MODELS
-
- mc = self.model_context
- processor = mc.processor
- image_processor = mc.image_processor
- tokenizer = mc.tokenizer
-
- # Handle template selection
- if isinstance(mc.model, PreTrainedModel):
- model_type = getattr(mc.model.config, "model_type", None)
- if model_type == "llava" and self.template is None:
- self.template = "default"
-
- if hasattr(mc.model, "name_or_path"):
- name = mc.model.name_or_path
- if any([m in name for m in MISTRAL_3_2_MODELS]):
- self.template = "mistral3_2"
-
- template_name = self.template
- if template_name is None and hasattr(mc.model.config, "model_type"):
- template_name = mc.model.config.model_type
- if template_name is None:
- template_name = "default"
-
- self.template_obj = get_template(
- template_name,
- model=mc.model,
- tokenizer=tokenizer,
- processor=processor,
- image_processor=image_processor,
- use_rtn=getattr(self.quantize_config, "iters", None) == 0,
- quiet=not self.quant_nontext_module,
- )
-
- logger.info(f"Using MLLM template: {template_name}")
-
- dataset = self.dataset.replace(" ", "") if isinstance(self.dataset, str) else self.dataset
- if dataset is None:
- dataset = self.template_obj.default_dataset
-
- if isinstance(self.dataset, str):
- dataset = self.dataset.replace(" ", "")
- # Mirror old arch __init__: switch text-only dataset to MLLM dataset when
- # quant_nontext_module=True, as text datasets cannot calibrate vision modules.
- from auto_round.calib_dataset import CALIB_DATASETS
-
- if self.quant_nontext_module and dataset in CALIB_DATASETS:
- logger.warning(
- "Text only dataset cannot be used for calibrating non-text modules,"
- " switching to liuhaotian/llava_conv_58k"
- )
- dataset = "liuhaotian/llava_conv_58k"
- (
- self.dataloader,
- self.batch_size,
- self.seqlen,
- self.gradient_accumulate_steps,
- ) = get_mllm_dataloader(
- template=self.template_obj,
- model=mc.model,
- tokenizer=tokenizer,
- processor=processor,
- image_processor=image_processor,
- dataset=dataset,
- extra_data_dir=self.extra_data_dir,
- seqlen=self.seqlen,
- bs=bs,
- seed=self.seed,
- nsamples=nsamples,
- quant_nontext_module=self.quant_nontext_module,
- )
- else:
- self.dataloader = self.dataset
-
- # Process data through the model for calibration
- total_cnt = 0
- for data in self.dataloader:
- if data is None:
- continue
-
- try:
- if isinstance(data, str):
- # List-of-strings dataset: process through template → model inputs
- processed = self.template_obj.processor.get_input(
- text=data, images=None, max_length=self.seqlen, squeeze=False
- )
- data_new = {k: to_device(v, mc.model.device) for k, v in processed.items()}
- elif isinstance(data, dict) and "text" in data:
- # FakeDataLoader-style {"text": ..., "image": ...}: process through template
- text = data["text"]
- if isinstance(text, dict):
- text = [text]
- input_text = self.template_obj._encode(text)
- processed = self.template_obj.processor.get_input(
- text=input_text,
- images=data.get("image", None),
- max_length=self.seqlen,
- squeeze=False,
- )
- data_new = {}
- for key, value in processed.items():
- tensor_val = value if isinstance(value, torch.Tensor) else torch.as_tensor(value)
- data_new[key] = to_device(tensor_val, mc.model.device)
- elif isinstance(data, dict):
- data_new = {
- key: value.to(mc.model.device) if isinstance(value, torch.Tensor) else value
- for key, value in data.items()
- }
- else:
- data_new = data
-
- if isinstance(data_new, dict):
- mc.model(**data_new)
- else:
- mc.model(data_new)
- except NotImplementedError:
- pass
- except Exception as e:
- logger.warning(f"Calibration forward pass failed: {e}")
- continue
-
- total_cnt += bs
- if total_cnt >= nsamples:
- break
-
- if total_cnt == 0:
- logger.error("no data has been cached, please provide more data")
- exit(-1)
- elif total_cnt < nsamples:
- logger.warning(f"Insufficient number of samples: required {nsamples}, but only {total_cnt} were processed.")
-
- def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):
- """Save the quantized model to the specified output directory in the specified format.
-
- Args:
- output_dir (str, optional): The directory to save the quantized model. Defaults to None.
- format (str, optional): The format in which to save the model. Defaults to "auto_round".
- inplace (bool, optional): Whether to modify the model in place. Defaults to True.
- **kwargs: Additional keyword arguments specific to the export format.
-
- Returns:
- object: The compressed model object.
- """
- mc = self.model_context
- processor = mc.processor
- image_processor = mc.image_processor
- tokenizer = mc.tokenizer
-
- if processor is not None and not hasattr(processor, "chat_template"):
- processor.chat_template = None
- compressed_model = super().save_quantized(
- output_dir=output_dir,
- format=format,
- inplace=inplace,
- processor=processor,
- image_processor=image_processor,
- quant_nontext_module=self.quant_nontext_module if hasattr(self, "quant_nontext_module") else False,
- **kwargs,
- )
- return compressed_model
diff --git a/auto_round/compressors_new/model_free.py b/auto_round/compressors_new/model_free.py
deleted file mode 120000
index de605bdd6..000000000
--- a/auto_round/compressors_new/model_free.py
+++ /dev/null
@@ -1 +0,0 @@
-../compressors/model_free.py
\ No newline at end of file
diff --git a/auto_round/compressors_new/shard_writer.py b/auto_round/compressors_new/shard_writer.py
deleted file mode 100644
index 4a446ad07..000000000
--- a/auto_round/compressors_new/shard_writer.py
+++ /dev/null
@@ -1,339 +0,0 @@
-# Copyright (c) 2026 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import os
-from collections import OrderedDict
-
-import torch
-
-from auto_round.compressors_new.utils import _get_save_folder_name
-from auto_round.context.compress import CompressContext
-from auto_round.context.model import ModelContext
-from auto_round.logger import logger
-from auto_round.utils import (
- get_lm_head_name,
- get_module,
- get_reverse_checkpoint_conversion_mapping,
- revert_checkpoint_conversion_mapping,
-)
-
-
-class ShardWriter:
- """
- Handles shard-saving of model parameters to disk with memory management.
- """
-
- _instance = None
- _initialized = False
-
- model = None
- lm_head_name = None
-
- def __new__(cls, *args, **kwargs):
- if cls._instance is None:
- cls._instance = super().__new__(cls)
- cls._instance._data = {}
- return cls._instance
-
- def __init__(
- self,
- model,
- bits,
- max_shard_size=None,
- safe_serialization=True,
- ):
- if ShardWriter._initialized:
- return
- self.model = model
- self.lm_head_name = get_lm_head_name(self.model)
- total_params = sum(p.numel() for p in self.model.parameters())
- # Heuristic estimate of model size in GB used to choose a default max_shard_size:
- # - total_params * rounder.bits -> total number of bits in all parameters
- # - // 8 -> convert bits to bytes
- # - // 1e9 -> approx convert bytes to GB (1e9 bytes ~= 1 GB)
- # - final // 10 -> apply a safety margin so default shards are
- # smaller than the full model; this intentionally
- # underestimates size before clamping below.
- max_split_num = 10
- model_size = int(total_params * bits // 1e9 // 8 + max_split_num - 1) / max_split_num
- model_size = max(1, min(int(model_size), 5))
-
- # Configuration
- max_shard_size = max_shard_size or f"{model_size}GB"
- self.max_shard_size = self._parse_size(max_shard_size)
- self.safe_serialization = safe_serialization
-
- # Internal State
- self.use_safetensors = self._check_safetensors()
- self.shard_suffix = "safetensors" if self.use_safetensors else "bin"
- self.current_shard_tensors = OrderedDict()
- self.current_shard_size = 0
- self.shard_meta = [] # List of {tmp_file: str, params: list}
- self.global_weight_map = {}
- self.shard_counter = 0
- self.reverse_checkpoint_conversion_mapping = get_reverse_checkpoint_conversion_mapping(self.model)
-
- # Persistent set of all parameter names already flushed to a shard file.
- # Maintained incrementally in _flush_shard to avoid O(N^2) rebuilds in _add_tensor.
- self._all_saved = set()
-
- # Stats
- self.total_param_elems = 0
- self.total_param_size_bytes = 0
- self.skipped_meta_tensors = []
-
- ShardWriter._initialized = True
-
- @property
- def output_dir(self) -> str:
- """Derive the output directory from the current CompressContext at access time.
-
- Reading from context rather than caching the path at construction time ensures
- the ShardWriter always uses the final export directory even if
- ``CompressContext.output_dir`` is updated after the ShardWriter was created
- (e.g. by ``_get_export_dir()`` in ``quantize_and_save()``).
- """
- compress_context = CompressContext.get_context()
- formats = compress_context.formats
- base_dir = _get_save_folder_name(formats[0])
- subfolder = getattr(self.model, "_autoround_pipeline_subfolder", None)
- if subfolder:
- base_dir = os.path.join(base_dir, subfolder)
- return os.path.join(base_dir, "")
-
- @classmethod
- def reset(cls):
- """Reset the singleton state so the next instantiation creates a fresh ShardWriter."""
- cls._initialized = False
- cls._instance = None
-
- @classmethod
- def get_shard_writer(cls, *args, **kwargs):
- """Return the current singleton instance, or None if not yet initialized.
-
- Callers that require a valid writer should guard the result with
- ``if self.compress_context.is_immediate_saving`` before use.
- """
- return cls._instance
-
- def _parse_size(self, size_str: str) -> int:
- if isinstance(size_str, int):
- return size_str
- s = size_str.strip().upper()
- units = {"GB": 1024**3, "MB": 1024**2, "KB": 1024, "B": 1}
- for unit, mult in units.items():
- if s.endswith(unit):
- return int(float(s[: -len(unit)]) * mult)
- return int(s)
-
- def _check_safetensors(self) -> bool:
- if self.safe_serialization:
- try:
- import safetensors.torch
-
- return True
- except ImportError:
- logger.warning("safetensors not installed; falling back to torch.save.")
- return False
-
- def save_module(self, m: torch.nn.Module, name: str = None):
- """Extracts and accumulates tensors from a module."""
- prefix = name if name is not None else getattr(m, "global_name", "model")
- sd = m.state_dict()
-
- for k, v in sd.items():
- if not isinstance(v, torch.Tensor):
- continue
- param_name = f"{prefix}.{k}"
- self._add_tensor(param_name, v)
-
- def _add_tensor(self, name: str, tensor: torch.Tensor):
-
- # transformers will handle _checkpoint_conversion_mapping automatically if is_immediate_saving=False
- name = revert_checkpoint_conversion_mapping(name, self.reverse_checkpoint_conversion_mapping)
-
- if isinstance(tensor, torch.Tensor) and tensor.device.type == "meta":
- self.skipped_meta_tensors.append(name)
- return
-
- # Guard against duplicate saving of the same parameter
- if name in self._all_saved or name in self.current_shard_tensors:
- return
-
- t_size = tensor.nbytes
- self.total_param_elems += tensor.numel()
- self.total_param_size_bytes += t_size
- tensor = tensor.detach().cpu()
- # If single tensor exceeds limit, flush current, save it solo, then continue
- if t_size > self.max_shard_size:
- self._flush_shard()
- self.current_shard_tensors[name] = tensor
- self.current_shard_size = t_size
- self._flush_shard()
- # If adding exceeds limit, flush first
- elif self.current_shard_size + t_size > self.max_shard_size and self.current_shard_size > 0:
- self._flush_shard()
- self.current_shard_tensors[name] = tensor
- self.current_shard_size = t_size
- else:
- self.current_shard_tensors[name] = tensor
- self.current_shard_size += t_size
-
- def _handle_tied_weights(self):
- """
- Detects tied weights in the current shard and ensures they are only saved once.
- This is done by tracking storage pointers of tensors and skipping duplicates.
- """
- storage_map = set()
- filtered_tensors = {}
-
- for name, tensor in self.current_shard_tensors.items():
- if not isinstance(tensor, torch.Tensor):
- filtered_tensors[name] = tensor
- continue
-
- ptr = tensor.untyped_storage().data_ptr() + tensor.storage_offset() * tensor.element_size()
- if ptr not in storage_map:
- storage_map.add(ptr)
- filtered_tensors[name] = tensor
- self.current_shard_tensors = filtered_tensors
-
- def _flush_shard(self):
- if not self.current_shard_tensors:
- return
-
- self.shard_counter += 1
- output_dir = self.output_dir
- os.makedirs(output_dir, exist_ok=True)
- tmp_name = f"model-shard-{self.shard_counter:05d}.{self.shard_suffix}"
- tmp_path = os.path.join(output_dir, tmp_name)
- self._handle_tied_weights()
-
- if self.use_safetensors:
- from safetensors.torch import save_file
-
- # Ensure tensors are contiguous in-place to avoid duplicating them in a separate dict,
- # which can increase peak RAM usage during saving.
- for k, v in list(self.current_shard_tensors.items()):
- if isinstance(v, torch.Tensor) and not v.is_contiguous():
- self.current_shard_tensors[k] = v.contiguous()
- save_file(self.current_shard_tensors, tmp_path)
- else:
- torch.save(self.current_shard_tensors, tmp_path)
-
- saved_params = list(self.current_shard_tensors.keys())
- self.shard_meta.append({"tmp_file": tmp_name, "params": saved_params, "dir": output_dir})
- self._all_saved.update(saved_params)
-
- # Offload logic: move modules to meta device once all params are saved
- self._offload_to_meta(saved_params)
-
- self.current_shard_tensors = OrderedDict()
- self.current_shard_size = 0
-
- def _offload_to_meta(self, saved_params):
- """Attempts to move fully saved modules to the 'meta' device to free RAM."""
- for param_full_name in saved_params:
- module_path = param_full_name.rsplit(".", 1)[0]
-
- module = get_module(self.model, module_path)
- # Check if all parameters of this module are now in '_all_saved'
- if (
- module is not None
- and isinstance(module, torch.nn.Module)
- and all(f"{module_path}.{k}" in self._all_saved for k in module.state_dict().keys())
- ):
- module.to("meta")
-
- def finalize(self):
- """Saves remaining weights, renames files, and writes the index JSON."""
- # 1. Capture remaining weights not yet saved
- full_sd = self.model.state_dict()
- tie_word_embeddings = False
- if hasattr(self.model, "config") and hasattr(self.model.config, "tie_word_embeddings"):
- tie_word_embeddings = self.model.config.tie_word_embeddings
-
- finalize_skipped_meta_tensors = []
- for pname, tensor in full_sd.items():
- if pname in self._all_saved:
- continue
- if tensor.device.type == "meta":
- continue
- layer_name = ".".join(pname.split(".")[:-1])
- if self.lm_head_name is not None and layer_name == self.lm_head_name and tie_word_embeddings:
- lm_head_module = get_module(self.model, self.lm_head_name)
- lm_head_module.to("meta") # Must to meta, otherwise model's saver will dump it again
- continue
- self._add_tensor(pname, tensor.detach().to("cpu"))
-
- self._flush_shard()
-
- total_skipped = len(self.skipped_meta_tensors) + len(finalize_skipped_meta_tensors)
- if total_skipped > 0:
- examples = (self.skipped_meta_tensors + finalize_skipped_meta_tensors)[:5]
-
- # 2. Rename temp files to HF standard and map weights
- if self.shard_counter == 0:
- logger.warning("No tensors saved.")
- return
-
- output_dir = self.output_dir
- for idx, meta in enumerate(self.shard_meta, start=1):
- shard_dir = meta.get("dir", output_dir)
- old_path = os.path.join(shard_dir, meta["tmp_file"])
- new_name = (
- f"model.{self.shard_suffix}"
- if self.shard_counter == 1
- else f"model-{idx:05d}-of-{self.shard_counter:05d}.{self.shard_suffix}"
- )
- new_path = os.path.join(shard_dir, new_name)
- os.rename(old_path, new_path)
- for p in meta["params"]:
- self.global_weight_map[p] = new_name
-
- # 3. Write Index JSON
- index_ext = "safetensors.index.json" if self.use_safetensors else "bin.index.json"
- index_path = os.path.join(output_dir, f"model.{index_ext}")
-
- index_data = {
- "metadata": {
- "format": "safetensors" if self.use_safetensors else "pytorch",
- "total_shards": self.shard_counter,
- "total_parameters": int(self.total_param_elems),
- "total_size": int(self.total_param_size_bytes),
- },
- "weight_map": self.global_weight_map,
- }
-
- if self.shard_counter > 1:
- with open(index_path, "w", encoding="utf-8") as f:
- json.dump(index_data, f, indent=2)
-
- logger.info(f"model has been saved to {self.output_dir}")
-
- @torch.no_grad()
- def write(self, m: torch.nn.Module = None, name: str = None, is_finalize: bool = False):
- if m is None and name is None and not is_finalize and not is_finalize:
- raise ValueError("Must specify either name or m")
- if m is None and name is not None:
- m = get_module(self.model, name)
- # Perform the save
- if m is not None:
- self.save_module(m, name)
-
- if is_finalize:
- self.finalize()
- ShardWriter._initialized = False
- ShardWriter._instance = None
diff --git a/auto_round/compressors_new/utils.py b/auto_round/compressors_new/utils.py
deleted file mode 100644
index 864168ff3..000000000
--- a/auto_round/compressors_new/utils.py
+++ /dev/null
@@ -1,1252 +0,0 @@
-# Copyright (c) 2025 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import copy
-import json
-import os
-import random
-import re
-import sys
-from dataclasses import asdict, fields
-from enum import Enum
-from typing import TYPE_CHECKING, Callable, Union
-
-import torch
-import transformers
-from torch.amp import autocast
-
-from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
-from auto_round.logger import logger
-from auto_round.utils import check_to_quantized, get_layer_names_in_block, get_module, to_standard_regex
-
-if TYPE_CHECKING:
- from auto_round.schemes import QuantizationScheme
-
-
-class BackendDataType(str, Enum):
- STANDARD_FP = "fp"
- MX_FP = "mx_fp"
- NV_FP = "nv_fp"
- MX_INT = "mx_int"
-
-
-def is_standard_fp(backend):
- backend = backend.lower()
- return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)
-
-
-def is_mx_fp(backend):
- backend = backend.lower()
- return BackendDataType.MX_FP in backend
-
-
-def is_mx_int(backend):
- backend = backend.lower()
- return BackendDataType.MX_INT in backend
-
-
-def is_nv_fp(backend):
- backend = backend.lower()
- return BackendDataType.NV_FP in backend
-
-
-def is_wint_woq(ar):
- """Returns True for integer weight-only quantization with non-quantized activations (`act_bits >= 16`)."""
- return "int" in ar.data_type and ar.act_bits >= 16 and ar.super_group_size is None
-
-
-def is_wint_a16(ar):
- """Backward-compatible alias for `is_wint_woq()`."""
- return is_wint_woq(ar)
-
-
-def _is_weight_fp8_activation_static_fp8(
- bit: int, group_size: int, sym: bool, data_type: str, act_dynamic: bool
-) -> bool:
- return bit == 8 and group_size == -1 and sym and data_type == "fp" and not act_dynamic
-
-
-def is_wfp8afp8(ar):
- if (
- ("fp8" in ar.act_data_type or ("fp" in ar.act_data_type and ar.act_bits == 8))
- and ("fp8" in ar.data_type or ("fp" in ar.data_type and ar.bits == 8))
- and is_standard_fp(ar.act_data_type)
- and is_standard_fp(ar.data_type)
- ):
- return True
- else:
- return False
-
-
-def is_wint8aint8(ar):
- if ("int8" in ar.act_data_type or ("int" in ar.act_data_type and ar.act_bits == 8)) and (
- "int8" in ar.data_type or ("int" in ar.data_type and ar.bits == 8)
- ):
- return True
- else:
- return False
-
-
-def is_static_wfp8afp8(ar_or_format: Union[str, Callable]) -> bool:
- if isinstance(ar_or_format, str):
- return "fp8_static" in ar_or_format.lower()
- if ar_or_format.act_dynamic:
- return False
- if is_wfp8afp8(ar_or_format):
- return True
- return False
-
-
-def is_dynamic_wint8aint8(ar_or_format: Union[str, Callable]) -> bool:
- if isinstance(ar_or_format, str):
- return "int8_w8a8" in ar_or_format.lower()
- if not ar_or_format.act_dynamic:
- return False
- if is_wint8aint8(ar_or_format):
- return True
- return False
-
-
-def is_dynamic_afp8(ar_or_format: Callable) -> bool:
- return ar_or_format.act_dynamic and ar_or_format.act_data_type.startswith("fp") and ar_or_format.act_bits == 8
-
-
-def is_block_wfp8(ar_or_format: Callable) -> bool:
- return (
- isinstance(ar_or_format.group_size, tuple)
- and len(ar_or_format.group_size) == 2
- and ar_or_format.data_type.startswith("fp")
- and ar_or_format.bits == 8
- )
-
-
-def block_forward(
- block: torch.nn.Module,
- input_ids: torch.Tensor,
- input_others: dict,
- amp: bool = False,
- amp_dtype: torch.dtype = torch.float16,
- device: torch.device = torch.device("cpu"),
- output_return_id: int = 0,
-) -> Union[torch.Tensor, dict]:
- """Performs a forward pass through a block with the given inputs.
-
- Args:
- block: The block to perform the forward pass on.
- input_ids: The input IDs.
- input_others: A dictionary containing other input data.
- amp: A boolean indicating whether to use automatic mixed precision.
- amp_dtype: The data type for automatic mixed precision.
- device: The target device.
- output_return_id: if the output has more than one tenor, return the specified idx tensor.
-
- Returns:
- output: The output of the forward pass.
- """
- from auto_round.utils.model import to_device
-
- if input_ids.device != device:
- input_ids = to_device(input_ids, device)
- input_others = to_device(input_others, device)
- input_tuple = input_others.pop("positional_inputs", None)
- if "alibi" in input_others.keys() and input_others["alibi"] is not None:
- alibi = input_others["alibi"]
- input_others["alibi"] = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
- if amp:
- with autocast(device_type=str(device).split(":")[0], dtype=amp_dtype): # pragma: no cover
- output = block(input_ids, *input_tuple, **input_others)
- else:
- output = block(input_ids, *input_tuple, **input_others)
- if isinstance(output_return_id, int) and (isinstance(output, list) or isinstance(output, tuple)):
- output = output[output_return_id]
- return output
-
-
-def check_skippable_keywords(key):
- """
- Prints a reminder if a key is not stored during quantization fine-tuning.
- """
- skippable_cache_keys = ("past_key_value",)
- for cache_key in skippable_cache_keys:
- if cache_key not in key:
- return True
- return False
-
-
-def check_need_act_calibration(
- is_act_dynamic: Union[bool, None],
- act_data_type: Union[str, None] = None,
- act_bits: Union[int, None] = 16,
- static_kv_dtype: Union[str, None] = None,
- static_attention_dtype: Union[str, None] = None,
-) -> bool:
- if static_kv_dtype is not None or static_attention_dtype is not None:
- return True
- if act_bits is None or act_bits > 8:
- return False
- # None is dynamic
- if is_act_dynamic is not None and not is_act_dynamic:
- return True
- if act_data_type is not None and "static" in act_data_type:
- return True
- return False
-
-
-def collect_best_params(block, cache_device="cpu"):
- """Collect the best parameters from the block to the specified device."""
- params = {}
- if hasattr(block, "orig_layer"):
- for key in block.params.keys():
- params[key] = block.params[key].data.to(cache_device, copy=True)
- else:
- for n, m in block.named_modules():
- if hasattr(m, "orig_layer"):
- params[n] = {}
- for key in m.params.keys():
- params[n][key] = m.params[key].data.to(cache_device, copy=True)
- return params
-
-
-def infer_bits_by_data_type(data_type: str):
- """Infer bits by data_type
-
- Args:
- data_type (str): data_type
-
- Returns:
- int: bits inferred by data_type, None means cannot infer correct bits by data_type
- """
- from auto_round.utils import SUPPORTED_DTYPES
-
- if data_type is None:
- return 16
- for supported_dtype in SUPPORTED_DTYPES:
- if data_type.startswith(supported_dtype) and len(data_type) > len(supported_dtype):
- ##first check the following two bits
- suc_2str = data_type[len(supported_dtype) : len(supported_dtype) + 2]
- if str.isdigit(suc_2str):
- return int(suc_2str)
- if str.isdigit(data_type[len(supported_dtype)]):
- return int(data_type[len(supported_dtype)])
- return None
-
-
-def _get_safetensor_layer_names_not_in_model(model, all_module_names: list) -> list:
- """Collect layer names from safetensor files that are not loaded into the model.
-
- Some tensors (e.g. MTP layers) exist in the original checkpoint but are not
- instantiated by ``transformers``. This function discovers them so that regex
- patterns in ``layer_config`` can still match them.
-
- Returns:
- List of layer names (the path without the ``.weight`` suffix) for weight
- tensors present in the safetensor files but absent from *all_module_names*.
- """
- name_or_path = None
- if hasattr(model, "config") and hasattr(model.config, "name_or_path"):
- name_or_path = model.config.name_or_path
- if not name_or_path:
- return []
-
- if not os.path.isdir(name_or_path):
- try:
- from auto_round.utils.model import download_hf_model
-
- name_or_path = download_hf_model(name_or_path)
- except Exception as e:
- logger.debug(f"Could not resolve source model path to check for missing tensors: {e}")
- return []
-
- try:
- from safetensors import safe_open
- except ImportError:
- return []
-
- # Build tensor-name list from the safetensors index or single file
- source_index_file = os.path.join(name_or_path, "model.safetensors.index.json")
- source_single_file = os.path.join(name_or_path, "model.safetensors")
-
- tensor_names: list = []
- if os.path.exists(source_index_file):
- with open(source_index_file) as f:
- src_index = json.load(f)
- tensor_names = list(src_index["weight_map"].keys())
- elif os.path.exists(source_single_file):
- with safe_open(source_single_file, framework="pt", device="cpu") as f:
- tensor_names = list(f.keys())
- else:
- return []
-
- module_name_set = set(all_module_names)
- extra_layer_names = []
- for tensor_name in tensor_names:
- if not tensor_name.endswith(".weight"):
- continue
- layer_name = tensor_name[: -len(".weight")]
- if layer_name not in module_name_set:
- extra_layer_names.append(layer_name)
- return extra_layer_names
-
-
-def set_layer_config(
- model: torch.nn.Module,
- layer_config: dict[str, Union[str, dict, "QuantizationScheme"]],
- default_scheme: Union[str, "QuantizationScheme"],
- default_scale_dtype: torch.dtype | str,
- supported_types: tuple,
- inner_supported_types: tuple,
- quant_block_list=None,
- ignore_layers: str = "",
- quant_lm_head: bool = False,
- enable_gguf_official_mixed: bool = True,
- is_mllm: bool = False,
- fill_default_value=True,
- gguf_format_name: str = None,
-) -> tuple[dict, bool, dict]:
- """
- Normalize, validate, and expand layer-specific quantization configs.
- Returns (final_layer_config, has_quant_layer_outside_block)
- """
-
- from auto_round.schemes import QuantizationScheme, get_gguf_scheme, preset_name_to_scheme
- from auto_round.utils.model import get_layer_names_in_block, get_lm_head_name, get_module, is_separate_lm_head
-
- # ---- helpers -------------------------------------------------
- def dispatch_layer_config(layer_config: dict[str, dict]) -> None:
- """Assign scheme values as attributes to matched modules."""
- for layer_name, scheme in layer_config.items():
- module = get_module(model, layer_name)
- if module is None:
- # Layer exists in safetensor files but is not loaded into the model
- # (e.g. MTP layers that transformers does not instantiate). Skip.
- continue
- for attr, value in scheme.items():
- setattr(module, attr, value)
-
- def normalize_item(item: Union[str, dict, "QuantizationScheme"], layer_name: str) -> dict:
- """Convert config entry into dict and validate keys."""
- if isinstance(item, str):
- config = asdict(preset_name_to_scheme(item.upper()))
- elif isinstance(item, QuantizationScheme):
- config = asdict(item)
- elif isinstance(item, dict):
- # "in_blocks" is an internal bookkeeping key injected by LLM-Compressor;
- # silently drop it before validation.
- item = {k: v for k, v in item.items() if k != "in_blocks"}
- # Support "scheme" key inside dict: resolve the preset and merge overrides.
- if "scheme" in item:
- scheme_name = item.pop("scheme")
- base = asdict(preset_name_to_scheme(scheme_name.upper()))
- base.update(item)
- item = base
- invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
- if invalid:
- raise ValueError(
- f"Invalid keys {invalid} in layer_config for '{layer_name}'. " f"Allowed keys: {scheme_keys}"
- )
- config = dict(item)
- else:
- raise TypeError(
- f"Unsupported type for layer_config[{layer_name}]: {type(item)}. "
- f"Expected str, dict, or QuantizationScheme."
- )
- # Clean up
- config = {k: v for k, v in config.items() if v is not None}
- config["fixed_by_user"] = True
- return config
-
- # ---- main logic ----------------------------------------------
- extra_scheme_keys = ("scale_dtype",)
- scheme_keys = tuple(f.name for f in fields(QuantizationScheme)) + ("scale_dtype",)
- layer_config = copy.deepcopy(layer_config) or {}
- ignore_layer_patterns = set()
- if ignore_layers:
- ignore_layers = ignore_layers.replace(" ", "").split(",")
- ignore_layers = [name + "." if name[-1].isdigit() else name for name in ignore_layers]
- ignore_layer_patterns = set(ignore_layers)
-
- # 1. ignore_layers -> force 16
- for name in get_fp_layer_names(model, ignore_layers):
- layer_config[name] = {
- "bits": 16,
- "act_bits": 16,
- "data_type": "float",
- "act_data_type": "float",
- "fixed_by_user": True,
- }
-
- # 2. normalize
- layer_config = {k: normalize_item(v, k) for k, v in layer_config.items()}
-
- # 3. infer missing bits
- for cfg in layer_config.values():
- if "data_type" in cfg and "bits" not in cfg:
- if (b := infer_bits_by_data_type(cfg["data_type"])) is not None:
- cfg["bits"] = b
- if "act_data_type" in cfg and "act_bits" not in cfg:
- if (b := infer_bits_by_data_type(cfg["act_data_type"])) is not None:
- cfg["act_bits"] = b
-
- # 4. fill defaults
- if isinstance(default_scheme, str):
- default_dict = asdict(preset_name_to_scheme(default_scheme.upper()))
- else:
- default_dict = asdict(default_scheme)
- default_dict["scale_dtype"] = default_scale_dtype
-
- # In AutoScheme with mixed gguf:q4_k_m, the super_group_size of gguf:q8_0 layer is None,
- # which should not be filled by default q4km again
- for cfg in layer_config.values():
- for key in scheme_keys:
- if fill_default_value:
- cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
- else:
- if key in extra_scheme_keys:
- cfg.setdefault(key, copy.deepcopy(default_dict.get(key)))
- else:
- cfg.setdefault(key, None)
-
- # 5. collect supported modules
- embedding_types = (torch.nn.Embedding,)
- gguf_name = gguf_format_name or get_gguf_scheme(default_scheme)
- if gguf_name:
- if torch.nn.Embedding not in supported_types:
- supported_types = (*supported_types, torch.nn.Embedding)
-
- # for some Embedding which type() is not torch.nn.Embedding
- # for example: transformers.models.gemma3.modeling_gemma3.Gemma3TextScaledWordEmbedding
- model_module_name = model.__class__.__module__
- module_cls = sys.modules[model_module_name]
- for name in module_cls.__dict__:
- if name.endswith("Embedding") and not name.endswith("RotaryEmbedding"):
- embedding_types = (*embedding_types, getattr(module_cls, name))
- supported_types = (*supported_types, *embedding_types)
-
- all_supported_layer_names, embedding_layer_names = [], []
- all_module_names = []
- for n, m in model.named_modules():
- all_module_names.append(n)
- # cleanup stale attributes
- for key in scheme_keys:
- if hasattr(m, key):
- delattr(m, key)
- if type(m) not in supported_types and m.__class__.__name__ not in inner_supported_types:
- continue
- all_supported_layer_names.append(n)
- if isinstance(m, embedding_types) or m.__class__.__name__.endswith("Embedding"):
- embedding_layer_names.append(n)
-
- # Also include layer names from safetensor files not loaded into the model
- # (e.g. MTP layers that transformers does not instantiate).
- safetensor_only_names = _get_safetensor_layer_names_not_in_model(model, all_module_names)
-
- # 6. expand regex configs
- regex_config = {}
- for name in list(layer_config.keys()):
- if name in all_supported_layer_names:
- continue
- if name in all_module_names:
- m = get_module(model, name)
- if len(list(m.children())) == 0 and type(m) not in supported_types:
- val = layer_config.pop(name)
- if name in ignore_layer_patterns:
- # Keep unsupported ignore_layers entries so export can serialize
- # them into regex-based extra_config for loaders like vLLM INC.
- regex_config[name] = val
- else:
- logger.warning(
- f"'{name}' exists in the model but is not a supported quantization target "
- f"in the current scheme, ignoring its setting in `layer_config`"
- )
- continue
-
- regex = re.compile(to_standard_regex(name))
- matched = [ln for ln in all_supported_layer_names if regex.search(ln)]
- safetensor_only_matched = [ln for ln in safetensor_only_names if regex.search(ln)]
- # skip it for mtp layers not loaded in transformers
- if not matched and not safetensor_only_matched:
- # type(mlp.gate) is Qwen3VLMoeTextTopKRouter instead of Linear
- logger.warning_once(
- f"Layer name or regex '{name}' in layer_config does not match any supported layers. "
- + "Please check for typos or update the regex pattern, ignore it for now"
- )
- val = layer_config.pop(name)
- regex_config[name] = val # keep regex config
- for match in matched:
- layer_config[match] = val
-
- # 7. lm_head
- lm_head_name = get_lm_head_name(model)
- tie_word_embeddings = False
- if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):
- tie_word_embeddings = model.config.tie_word_embeddings
-
- if lm_head_name in layer_config:
- quant_lm_head = True
-
- if quant_lm_head and tie_word_embeddings and not gguf_name:
- quant_lm_head = False
- logger.warning(
- "reset `quant_lm_head` to false as quantizing " "lm_head with tied weights has not been supported currently"
- )
-
- if lm_head_name not in layer_config and quant_lm_head:
- layer_config[lm_head_name] = copy.deepcopy(default_dict)
-
- if not quant_lm_head and not gguf_name:
- layer_config.pop(lm_head_name, None)
-
- # 8. enforce shape divisibility for int weight-only
- if default_dict["data_type"] == "int" and default_dict["act_bits"] >= 16 and not gguf_name:
- for n, m in model.named_modules():
- if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
- if m.weight.shape[0] % 32 or m.weight.shape[1] % 32:
- layer_config.setdefault(n, copy.deepcopy(default_dict))
- layer_config[n].update({"bits": 16, "data_type": "fp", "fixed_by_user": True})
- # logger.warning_once(f"{n} skipped quantization (shape not divisible by 32).")
- # enforce shape divisibility for mxfp/nvfp
- if (is_nv_fp(default_dict["data_type"]) or is_mx_fp(default_dict["data_type"])) and not gguf_name:
- for n, m in model.named_modules():
- if type(m) in supported_types or m.__class__.__name__ in inner_supported_types:
- if m.weight.shape[1] % default_dict["group_size"]:
- layer_config.setdefault(n, copy.deepcopy(default_dict))
- layer_config[n].update(
- {"bits": 16, "data_type": "fp", "act_bits": 16, "act_data_type": "fp", "fixed_by_user": True}
- )
- logger.warning_once(
- f"{n} skipped quantization (shape not divisible by {default_dict['group_size']})."
- )
-
- # 9. block layers: mark as in_blocks=True
- for name in get_layer_names_in_block(model, supported_types, quant_block_list, inner_supported_types):
- if name not in layer_config:
- layer_config[name] = copy.deepcopy(default_dict)
- layer_config[name]["fixed_by_user"] = False
- layer_config[name]["in_blocks"] = True
-
- # ---- restore: ensure missing in_blocks are set to False and compute flag ----
- has_qlayer_outside_block = False
- for cfg in layer_config.values():
- if "in_blocks" not in cfg:
- cfg["in_blocks"] = False
- # mark layer outside block
- if not cfg["in_blocks"] and check_to_quantized(cfg):
- has_qlayer_outside_block = True
-
- # 10. GGUF handling
- if not gguf_name:
- dispatch_layer_config(layer_config)
- return layer_config, has_qlayer_outside_block, regex_config
-
- # embed + lm_head defaults for gguf
- tie_word_embeddings &= not is_separate_lm_head(model)
- if lm_head_name not in layer_config and not tie_word_embeddings:
- cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]
- cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}
- layer_config[lm_head_name] = cfg
- has_qlayer_outside_block = True
- for emd_name in embedding_layer_names:
- if emd_name in layer_config:
- continue
- if not tie_word_embeddings:
- cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["embedding"]]
- else:
- cfg = GGUF_INNER_CONFIG[GGUF_CONFIG[gguf_name.lower()]["lm_head"]]
- cfg = {**cfg, "fixed_by_user": False, "scale_dtype": default_scale_dtype}
- layer_config[emd_name] = cfg
-
- if enable_gguf_official_mixed:
- model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT
- layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)
-
- dispatch_layer_config(layer_config)
- return layer_config, has_qlayer_outside_block, regex_config
-
-
-def _use_more_bits(i_layer: int, n_layer: int):
- return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer // 8) % 3 == 2)
-
-
-def _search_gguf_type(gguf_type):
- if gguf_type in GGUF_INNER_CONFIG:
- return gguf_type
- pattern = re.compile("gguf:q([0-9]{1,})_[01k]")
- bits = re.search(pattern, gguf_type)
- if not bits:
- raise KeyError(f"{gguf_type} is not a correct gguf type, please check")
-
- for suffix in ["_k", "_0", "_1"]:
- if gguf_type.endswith(suffix):
- continue
- if (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:
- return tmp_type
- return None
-
-
-def gguf_type_fallback(gguf_type: str) -> str:
- gguf_type = gguf_type.lower()
- if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):
- gguf_type = "gguf:q5_0"
- elif gguf_type == "gguf:q5_k":
- gguf_type = "gguf:q5_0"
- elif gguf_type == "gguf:q6_k":
- gguf_type = "gguf:q8_0"
- return gguf_type
-
-
-def get_gguf_qtype_by_layer_config(layer_config):
- import gguf # pylint: disable=E0401
-
- if layer_config["bits"] >= 16:
- return None
- bits = layer_config["bits"]
- super_bits = layer_config.get("super_bits", None)
- sym = layer_config["sym"]
- group_size = layer_config.get("group_size", None)
- super_group_size = layer_config.get("super_group_size", None)
- if bits == 2 and super_bits == 4 and not sym and group_size == 16 and super_group_size == 16:
- return gguf.GGMLQuantizationType.Q2_K
- if bits == 3 and super_bits == 6 and sym and group_size == 16 and super_group_size == 16:
- return gguf.GGMLQuantizationType.Q3_K
- if bits == 4:
- if super_bits is not None and super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:
- return gguf.GGMLQuantizationType.Q4_K
- if super_bits is None and sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q4_0
- if super_bits is None and not sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q4_1
- if bits == 5:
- if super_bits == 6 and not sym and group_size == 32 and super_group_size == 8:
- return gguf.GGMLQuantizationType.Q5_K
- if super_bits is None and sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q5_0
- if super_bits is None and not sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q5_1
- if bits == 6 and super_bits == 8 and group_size == 16 and super_group_size == 16:
- return gguf.GGMLQuantizationType.Q6_K
- if bits == 8 and sym and group_size == 32:
- return gguf.GGMLQuantizationType.Q8_0
- raise ValueError("Unknown layer config")
-
-
-def _get_digital_in_layer_name(layer_name):
- pattern = re.compile(r"([a-zA-Z]+\.){1,}(\d+)")
- res = re.search(pattern, layer_name)
- if res:
- return int(res[2])
- else:
- return None
-
-
-def _gguf_type_fallback(gguf_type: str) -> str:
- gguf_type = gguf_type.lower()
- if gguf_type in ("gguf:q2_k", "gguf:q3_k", "gguf:q4_k"):
- gguf_type = "gguf:q5_0"
- elif gguf_type == "gguf:q5_k":
- gguf_type = "gguf:q5_0"
- elif gguf_type == "gguf:q6_k":
- gguf_type = "gguf:q8_0"
- return gguf_type
-
-
-##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129
-def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT):
- # # TODO: support for other format later
- # target_gguf_format = next((fmt for fmt in gguf_format if fmt != "fake"), None)
-
- import gguf # pylint: disable=E0401
-
- from auto_round.schemes import QuantizationScheme, get_gguf_scheme
- from auto_round.utils.common import MM_KEYS, LazyImport
- from auto_round.utils.model import get_lm_head_name, get_module
-
- # from auto_round.export.export_to_gguf.convert import ModelBase, get_model_architecture
- convert_hf_to_gguf = LazyImport("auto_round.export.export_to_gguf.convert_hf_to_gguf")
-
- try:
- model_architecture = convert_hf_to_gguf.get_model_architecture(
- hparams=model.config.to_dict(), model_type=model_type
- )
- except AttributeError as e:
- raise ImportError(
- "Please use the latest gguf-py, you can use the following command to install it:\n"
- "git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py"
- " && pip install . sentencepiece"
- )
- try:
- if model_type != ModelType.TEXT:
- model_class_vision = convert_hf_to_gguf.ModelBase.from_model_architecture(
- model_architecture, model_type=model_type
- )
- model_class = convert_hf_to_gguf.ModelBase.from_model_architecture(
- model_architecture, model_type=ModelType.TEXT
- )
-
- except NotImplementedError:
- return layer_config, {}
-
- n_layer = None
- if model_type != ModelType.TEXT:
- n_layer_vision = None
- for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:
- if hasattr(model.config, name):
- n_layer = getattr(model.config, name)
- if model_type != ModelType.TEXT:
- if n_layer is not None and hasattr(model.config, "text_config"):
- if hasattr(getattr(model.config, "text_config"), name):
- n_layer = getattr(getattr(model.config, "text_config"), name)
- for config_name in ["vision_config", "vision_encoder"]:
- if hasattr(model.config, config_name):
- if hasattr(getattr(model.config, config_name), name):
- n_layer_vision = getattr(getattr(model.config, config_name), name)
- break
- if n_layer and n_layer_vision:
- break
-
- if n_layer is None:
- return layer_config, {}
-
- tensor_map = gguf.get_tensor_name_map(model_class.model_arch, n_layer)
- if model_type != ModelType.TEXT:
- tensor_map_vision = gguf.get_tensor_name_map(model_class_vision.model_arch, n_layer_vision)
-
- def _set_config(config, target_config):
- for k, v in target_config.items():
- if isinstance(config, dict):
- config[k] = v
- else:
- setattr(config, k, v)
- return config
-
- gguf_format_config = {}
- lm_head_name = get_lm_head_name(model)
- inner_gguf_format = GGUF_CONFIG[target_gguf_format]["mostly"]
- # ggml_type = getattr(gguf.GGMLQuantizationType,inner_gguf_format.split(":")[-1].upper())
- block_size = GGML_QUANT_SIZES[inner_gguf_format.split(":")[-1].lower()][0]
- tie_word_embeddings = True
- if hasattr(model, "config") and hasattr(model.config, "tie_word_embeddings"):
- tie_word_embeddings = model.config.tie_word_embeddings
-
- n_gqa = 1
- if (
- hasattr(model, "config")
- and hasattr(model.config, "num_attention_heads")
- and hasattr(model.config, "num_key_value_heads")
- ):
- n_gqa = model.config.num_attention_heads // model.config.num_key_value_heads
- n_expert = 0
- for name in ["num_experts", "num_local_experts", "n_routed_experts"]:
- if hasattr(model.config, name):
- n_expert = getattr(model.config, name)
-
- i_attention_wv = 0
- i_ffn_down = 0
- layer_config_copy = copy.deepcopy(layer_config)
- base_target_bits = None
- if inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():
- base_target_bits = int(inner_gguf_format[6])
-
- for layer_name, config in layer_config_copy.items():
- if not check_to_quantized(config):
- continue
- # Reset target_bits each iteration to prevent lm_head/embedding settings
- # from bleeding into subsequent block layers and bypassing their special logic.
- target_bits = base_target_bits
- new_type = GGUF_CONFIG[target_gguf_format]["mostly"]
- layer = get_module(model, layer_name)
- if type(layer) == transformers.pytorch_utils.Conv1D:
- input_features = layer.weight.shape[0]
- else:
- input_features = layer.weight.shape[-1]
- i_layer = _get_digital_in_layer_name(layer_name)
-
- if lm_head_name is not None and layer_name == lm_head_name:
- target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["lm_head"]).group(1))
- if isinstance(layer, torch.nn.Embedding):
- target_bits = int(
- re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]["embedding"]).group(1)
- )
-
- if model_type != ModelType.TEXT and any([key in layer_name for key in MM_KEYS]):
- gguf_name = tensor_map_vision.get_name(layer_name)
- if gguf_name is None:
- for key in MM_KEYS:
- gguf_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))
- if gguf_name is not None:
- break
- else:
- gguf_name = tensor_map.get_name(layer_name)
- if gguf_name is None:
- gguf_name = tensor_map.get_name(layer_name.replace(".language_model", ""))
- bits_index = 6
- if config.get("fixed_by_user", False):
- if "bits" not in config:
- logger.warning(
- f"Setting layer_config requires providing bits, {layer_name} has not bits,"
- f" using bits={target_bits} instead."
- )
- new_type = new_type[:bits_index] + target_bits + new_type[bits_index + 1 :]
- else:
- config_tmp = config.copy()
- scheme_keys = [f.name for f in fields(QuantizationScheme)]
- for key in config.keys():
- if key not in scheme_keys:
- config_tmp.pop(key, None)
- matched_scheme = get_gguf_scheme(QuantizationScheme.from_dict(config_tmp)) # check matched
- if not matched_scheme:
- if config.get("super_group_size", None) is not None or config.get("super_bits", None) is not None:
- new_type = new_type[:bits_index] + str(config["bits"]) + "_k"
- if new_type not in GGUF_INNER_CONFIG:
- prefix_idx = 0 if config.get("sym", True) else 1
- new_type = new_type[:bits_index] + str(config["bits"]) + f"_{prefix_idx}"
- if new_type not in GGUF_INNER_CONFIG:
- new_type = new_type[:bits_index] + str(config["bits"]) + f"_{1-prefix_idx}"
- if new_type not in GGUF_INNER_CONFIG:
- raise ValueError(
- f"the setting in layer_config {layer_name} "
- f"could not match any supported gguf format, please have a check."
- )
-
- new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]
- new_type = _search_gguf_type(new_type)
- if new_type is None:
- raise ValueError(f"invalid bit setting for {layer_name}")
- elif target_bits is not None and "bits" in config and config["bits"] != target_bits:
- new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1 :]
- new_type = _search_gguf_type(new_type)
- if new_type is None:
- raise ValueError(f"invalid bit setting for {layer_name}")
- elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:
- if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:
- new_type = "gguf:q8_0"
- elif "lm_head" in GGUF_CONFIG[target_gguf_format]:
- new_type = GGUF_CONFIG[target_gguf_format]["lm_head"]
- elif new_type != "gguf:q8_0":
- new_type = "gguf:q6_k"
- elif lm_head_name is not None and layer_name == lm_head_name and tie_word_embeddings:
- # new_type = GGUF_CONFIG[target_gguf_format]["lm_head"]
- continue
- elif isinstance(layer, torch.nn.Embedding):
- if "embedding" in GGUF_CONFIG[target_gguf_format]:
- new_type = GGUF_CONFIG[target_gguf_format]["embedding"]
- elif gguf_name is None:
- pass
- # attn_v
- elif "attn_v" in gguf_name:
- if target_gguf_format == "gguf:q2_k":
- new_type = "gguf:q4_k" if n_gqa >= 4 else "gguf:q3_k"
- elif target_gguf_format == "gguf:q2_k_s" and n_gqa >= 4:
- new_type = "gguf:q4_k"
- elif target_gguf_format == "gguf:q3_k_m":
- new_type = "gguf:q5_k" if i_attention_wv < 2 else "gguf:q4_k"
- elif target_gguf_format == "gguf:q3_k_l":
- new_type = "gguf:q5_k"
- elif (target_gguf_format == "gguf:q4_k_m" or target_gguf_format == "gguf:q5_k_m") and _use_more_bits(
- i_layer, n_layer
- ):
- new_type = "gguf:q6_k"
- elif target_gguf_format == "gguf:q4_k_s" and i_attention_wv < 4:
- new_type = "gguf:q5_k"
- ##TODO check which models are be grouped into to LLM_TYPE_70B
- # if (qs.model.type == LLM_TYPE_70B) {
- # // In the 70B model we have 8 heads sharing the same attn_v weights.
- # As a result, the attn_v.weight tensor is
- # // 8x smaller compared to attn_q.weight.Hence, we can get a nice boost in quantization accuracy with
- # // nearly negligible increase in model size by quantizing this tensor with more bits:
- # if
- # (new_type == GGML_TYPE_Q3_K | | new_type == GGML_TYPE_Q4_K)
- # new_type = GGML_TYPE_Q5_K;
- # }
- if n_expert == 8:
- new_type = "gguf:q8_k"
- i_attention_wv += 1
-
- elif "attn_k" in gguf_name:
- if n_expert == 8:
- new_type = "gguf:q8_0"
- # ffn_down
- elif "ffn_down" in gguf_name:
- if target_gguf_format == "gguf:q2_k":
- new_type = "gguf:q3_k"
- elif target_gguf_format == "gguf:q2_k_s":
- if i_layer < n_layer / 8:
- new_type = "gguf:q4_k"
- elif target_gguf_format == "gguf:q3_k_m":
- if i_layer < n_layer / 16:
- new_type = "gguf:q5_k"
- elif gguf.MODEL_ARCH.FALCON == model_class.model_arch or _use_more_bits(i_layer, n_layer):
- new_type = "gguf:q4_k"
- else:
- new_type = "gguf:q3_k"
- elif target_gguf_format == "gguf:q3_k_l":
- if gguf.MODEL_ARCH.FALCON == model_class.model_arch:
- new_type = "gguf:q4_k"
- else:
- new_type = "gguf:q5_k"
- elif target_gguf_format == "gguf:q4_k_m":
- if gguf.MODEL_ARCH.FALCON == model_class.model_arch:
- if i_layer < n_layer // 16:
- new_type = "gguf:q6_k"
- elif _use_more_bits(i_layer, n_layer):
- new_type = "gguf:q5_k"
- else:
- new_type = "gguf:q4_k"
- else:
- if _use_more_bits(i_layer, n_layer):
- new_type = "gguf:q6_k"
- elif target_gguf_format == "gguf:q5_k_m" and _use_more_bits(i_layer, n_layer):
- new_type = "gguf:q6_k"
- elif (
- target_gguf_format == "gguf:q4_k_s"
- and model_class.model_arch != gguf.MODEL_ARCH.FALCON
- and i_layer < n_layer / 8
- ):
- new_type = "gguf:q5_k"
- elif (target_gguf_format == "gguf:q4_0" or target_gguf_format == "gguf:q5_0") and i_layer < n_layer / 8:
- if target_gguf_format == "gguf:q4_0":
- new_type = "gguf:q4_1"
- else:
- new_type = "gguf:q5_1"
- i_ffn_down += 1
-
- # attn_output
- elif "attn_output" in gguf_name:
- if gguf.MODEL_ARCH.FALCON != model_class.model_arch:
- if n_expert == 8:
- if target_gguf_format in (
- "gguf:q2_k",
- "gguf:q3_k_s",
- "gguf:q3_k_m",
- "gguf:q4_k_s",
- "gguf:q4_k_m",
- "gguf:q5_k",
- ):
- new_type = "gguf:q5_k"
- elif target_gguf_format == "gguf:q2_k":
- new_type = "gguf:q3_k"
- elif target_gguf_format == "gguf:q3_k_m":
- new_type = "gguf:q4_k"
- elif target_gguf_format == "gguf:q3_k_l":
- new_type = "gguf:q5_k"
- else:
- if target_gguf_format == "gguf:q3_k_l":
- new_type = "gguf:q4_k"
- # attn_qkv
- elif "attn_qkv" in gguf_name:
- if target_gguf_format in ("gguf:q3_k_m", "gguf:q3_k_l"):
- new_type = "gguf:q4_k"
- elif target_gguf_format == "gguf:q4_k_m":
- new_type = "gguf:q5_k"
- elif target_gguf_format == "gguf:q5_k_m":
- new_type = "gguf:q5_k"
- new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
- if input_features % new_block_size != 0:
- new_type = _gguf_type_fallback(new_type)
- new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
- if input_features % new_block_size != 0:
- new_type = "gguf:bf16"
- logger.warning(
- f"fallback {layer_name} to {new_type}, "
- f"because input_features({input_features}) % block_size({block_size}) != 0"
- )
- # for deepseek v2
- if layer_name.endswith("kv_b_proj") and new_type.endswith("_k") and "Deepseek" in model.config.architectures[0]:
- fallback = False
-
- # calc if need fallback
- qk_nope_head_dim = model.config.qk_nope_head_dim
- kv_b_shape = get_module(model, layer_name).weight.shape
-
- if (
- qk_nope_head_dim < QK_K
- or qk_nope_head_dim % QK_K != 0
- or kv_b_shape[-1] < QK_K
- or kv_b_shape[-1] % QK_K != 0
- ):
- fallback = True
- if fallback:
- tmp_type = _gguf_type_fallback(new_type)
- logger.warning_once(
- f"self_attn.kv_b_proj does not support the use of {new_type}, replace it with {tmp_type}"
- )
- new_type = tmp_type
-
- target_config = GGUF_INNER_CONFIG[new_type]
-
- _set_config(layer_config[layer_name], target_config)
- _set_config(layer, target_config)
- gguf_format_config[layer_name] = new_type
-
- return layer_config, gguf_format_config
-
-
-def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):
- """Identifies and returns layers in the model to exclude from quantization.
-
- This function processes a comma-separated list of fully precision (FP) layers,
- matches them to the names of layers in the model, and returns a list of such
- layers to exclude from quantization.
-
- Args:
- model (torch.nn.Module): The model whose layers will be inspected.
- ignore_layers (str): A comma-separated string of layer names to be excluded
- from quantization. Whitespace is ignored in this string.
-
- Returns:
- list: A list of layer names that match the specified FP layers or are
- subcomponents of those layers.
- """
- from auto_round.utils import SUPPORTED_LAYER_TYPES
-
- if not ignore_layers:
- return []
-
- all_layer_names = []
- for n, m in model.named_modules():
- if type(m) in SUPPORTED_LAYER_TYPES:
- all_layer_names.append(n)
- not_to_quantized_layers = []
-
- for fp_layer in ignore_layers:
- if fp_layer == "":
- continue
- if fp_layer in all_layer_names:
- not_to_quantized_layers.append(fp_layer)
- continue
- for name in all_layer_names:
- if fp_layer in name:
- not_to_quantized_layers.append(name)
- not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use
- logger.trace(f"not_to_quantized_layers: {not_to_quantized_layers}")
- return not_to_quantized_layers
-
-
-def get_shared_keys(model):
- """
- Retrieves shared keys from the model's state dictionary.
-
- Args:
- model (torch.nn.Module): The model to retrieve shared keys from.
-
- Returns:
- tuple: tuple of shared keys.
- """
- from auto_round.special_model_handler import SPECIAL_SHARED_CACHE_KEYS
- from auto_round.utils import SHARED_CACHE_KEYS
-
- shared_keys = SHARED_CACHE_KEYS
- shared_keys += SPECIAL_SHARED_CACHE_KEYS.get(model.__class__.__name__, ())
- return shared_keys
-
-
-def init_cache(positional_inputs, inputs):
- """
- Initializes special model inputs by adding positional inputs if missing.
-
- Args:
- positional_inputs (list): List of positional inputs to add to inputs.
- inputs (dict): Dictionary of model inputs.
-
- Modifies:
- inputs (dict): Adds "positional_inputs" key if not present.
- """
- from auto_round.utils.model import to_device
-
- if "positional_inputs" not in inputs: # for chatglm Series
- inputs["positional_inputs"] = []
- for idx, item in enumerate(positional_inputs):
- inputs["positional_inputs"] = to_device(positional_inputs)
-
-
-def reset_params(inputs):
- """
- Resets specific input parameters to avoid saving the key-value cache during fine-tuning.
-
- Args:
- inputs (dict): Dictionary of model inputs.
-
- Modifies:
- inputs (dict): Sets "use_cache" to False if the key is present.
- """
- if "use_cache" in inputs.keys(): # Not storing kv cache
- inputs["use_cache"] = False
-
-
-class IndexSampler:
- """A cyclic sampler that returns shuffled index batches.
-
- This sampler maintains internal state so that each call to `next_batch()`
- continues from where it left off. When the remaining number of samples is
- less than `batch_size`, the sampler reshuffles all indices and starts from
- the beginning, discarding the last incomplete batch.
-
- Attributes:
- nsamples (int): Total number of samples.
- batch_size (int): Number of indices to return in each batch.
- index (int): Current position in the index list.
- indices (List[int]): Shuffled list of indices.
- """
-
- def __init__(self, nsamples: int, batch_size: int) -> None:
- """Initializes the sampler.
-
- Args:
- nsamples (int): Total number of samples (must be >= batch_size).
- batch_size (int): Number of indices per batch.
-
- Raises:
- ValueError: If batch_size is not in the range (0, nsamples].
- """
- if batch_size <= 0 or batch_size > nsamples:
- raise ValueError("batch_size must be > 0 and <= nsamples")
-
- self.nsamples: int = nsamples
- self.batch_size: int = batch_size
- self.index: int = 0
-
- self.indices: list[int] = list(range(nsamples))
- random.shuffle(self.indices)
-
- def next_batch(self) -> list[int]:
- """Returns the next batch of shuffled indices.
-
- If the remaining indices are fewer than `batch_size`, the sampler
- reshuffles the entire list and starts from the beginning.
-
- Returns:
- list[int]: A list of size `batch_size` containing sample indices.
- """
- if self.index + self.batch_size > self.nsamples:
- random.shuffle(self.indices)
- self.index = 0
-
- batch = self.indices[self.index : self.index + self.batch_size]
- self.index += self.batch_size
- return batch
-
-
-def _get_quantized_layer_names_outside_blocks(model, layer_config, supported_types, quant_block_list) -> list:
- """Gets the names of quantized layers outside blocks in the model.
-
- Returns:
- list: List of layer names outside blocks.
- """
- if layer_config is None or len(layer_config) == 0:
- return []
-
- layer_names = []
- all_layers_in_block = get_layer_names_in_block(model, supported_types, quant_block_list)
-
- for key in layer_config.keys():
- if key in all_layers_in_block:
- continue
- layer = get_module(model, key)
- if layer is None:
- logger.error(f"could not find layer {key} in the model, exit...")
- exit(-1)
- if type(layer) in supported_types and check_to_quantized(layer_config[key]):
- layer_names.append(key)
-
- return layer_names
-
-
-def _get_diffusion_save_folder_name(format) -> str:
- """Generates the save folder name based on the provided format string.
-
- If there are multiple formats to handle, the function creates a subfolder
- named after the format string with special characters replaced. If there's
- only one format, it returns the original output directory directly.
-
- Args:
- format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
-
- Returns:
- str: The path to the folder where results should be saved.
- """
- from auto_round.context.compress import CompressContext
- from auto_round.context.model import ModelContext
-
- compress_context = CompressContext.get_context()
- model_context = ModelContext.get_context()
-
- # Replace special characters to make the folder name filesystem-safe
- sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")
-
- formats = compress_context.formats
- # Use a subfolder only if there are multiple formats
- if len(formats) > 1:
- return (
- os.path.join(compress_context.output_dir, sanitized_format, "transformer")
- if compress_context.is_immediate_saving
- else os.path.join(compress_context.output_dir, sanitized_format, "transformer")
- )
-
- # if use is_immediate_saving, we need to save model in self.output_dir/transformer folder
- return (
- os.path.join(compress_context.output_dir, "transformer")
- if compress_context.is_immediate_saving
- else compress_context.output_dir
- )
-
-
-def _get_save_folder_name(format, *args, **kwargs) -> str:
- """Generates the save folder name based on the provided format string.
-
- If there are multiple formats to handle, the function creates a subfolder
- named after the format string with special characters replaced. If there's
- only one format, it returns the original output directory directly.
-
- Args:
- format_str (str): The format identifier (e.g., 'gguf:q2_k_s').
-
- Returns:
- str: The path to the folder where results should be saved.
- """
- from auto_round.context.compress import CompressContext
- from auto_round.context.model import ModelContext
-
- compress_context = CompressContext.get_context()
- model_context = ModelContext.get_context()
- if model_context.is_diffusion:
- return _get_diffusion_save_folder_name(format)
- # Replace special characters to make the folder name filesystem-safe
- sanitized_format = format.get_backend_name().replace(":", "-").replace("_", "-")
-
- # Use a subfolder only if there are multiple formats
- if len(compress_context.formats) > 1:
- return os.path.join(compress_context.output_dir, sanitized_format)
-
- return compress_context.output_dir
-
-
-def immediate_pack(name: str, layer_config: dict):
- from auto_round.context.compress import CompressContext
- from auto_round.context.model import ModelContext
-
- compress_context = CompressContext.get_context()
- model_context = ModelContext.get_context()
-
- if not compress_context.is_immediate_packing:
- return
- compress_context.formats[0].immediate_pack(
- name=name,
- model=model_context.model,
- device=compress_context.device,
- output_dir=_get_save_folder_name(compress_context.formats[0]),
- layer_config=layer_config,
- tokenizer=model_context.tokenizer,
- mllm=model_context.is_mllm,
- processor=getattr(model_context, "processor", None),
- image_processor=getattr(model_context, "image_processor", None),
- quant_nontext_module=getattr(model_context, "quant_nontext_module", False),
- )
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 1a419d880..6541ad0de 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -450,12 +450,18 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatri
def quant_tensor_gguf_asym_dq(
tensor: torch.Tensor,
bits: int = 4,
+ v=0,
scale_dtype=torch.float16,
imatrix=None,
scale=None,
wmin=None,
d_scale=None,
d_wmin=None,
+ prev_scale=None,
+ prev_wmin=None,
+ prev_d_scale=None,
+ prev_d_wmin=None,
+ iter=None,
**kwargs,
):
"""Quantizes and dequantizes a tensor using asymmetric integer quantization for formats like Q2_K, Q4_K, and Q5_K.
@@ -481,14 +487,26 @@ def quant_tensor_gguf_asym_dq(
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
tensor = tensor.to(torch.float32)
+ if iter is None:
+ iter = 0
+ use_cached_params = prev_scale is not None and iter % 10 != 0 and iter != -1
+ if use_cached_params and scale is None:
+ scale = prev_scale.detach()
+ wmin = prev_wmin.detach()
+ d_scale = prev_d_scale.detach()
+ d_wmin = prev_d_wmin.detach()
if scale is None:
scale, wmin, d_scale, d_wmin = search_gguf_scale_min_asym(
tensor, bits, scale_dtype, imatrix, split_num=split_num
)
+ scale = scale.clone()
+ wmin = wmin.clone()
+ d_scale = d_scale.clone()
+ d_wmin = d_wmin.clone()
inverse_scale = get_reciprocal(scale)
tensor = tensor + wmin
- tensor = (tensor.mul_(inverse_scale)).round_().clamp_(0, maxq)
+ tensor = round_ste(tensor.mul_(inverse_scale) + v).clamp_(0, maxq)
tensor = tensor.mul_(scale)
tensor = tensor.sub_(wmin).to(orig_dtype)
tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
@@ -667,9 +685,13 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num):
def quant_tensor_gguf_sym_dq(
tensor,
bits=3,
+ v=0,
imatrix=None,
scale=None,
d_scale=None,
+ prev_scale=None,
+ prev_d_scale=None,
+ iter=None,
scale_dtype=torch.float16,
split_num=1,
**kwargs,
@@ -708,20 +730,25 @@ def quant_tensor_gguf_sym_dq(
n_blocks = tensor.nelement() // block_size
# (nb, 16, 16)
tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
- if scale is None and d_scale is None:
+ if iter is None:
+ iter = 0
+ use_cached_params = prev_scale is not None and iter % 10 != 0 and iter != -1
+ if use_cached_params and scale is None:
+ scale = prev_scale.detach()
+ d_scale = prev_d_scale.detach()
+ if scale is None or d_scale is None:
scale = search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num=split_num)
-
- scale = scale.to(scale_dtype)
- scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)
- # conduct double quant
- scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)
-
- scale = scale.unsqueeze(-1)
+ scale = scale.to(scale_dtype)
+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)
+ scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)
+ scale = scale.clone()
+ d_scale = d_scale.clone()
+ scale = scale.unsqueeze(-1)
# zp = torch.full_like(scale, maxq) # pylint: disable=E1130
inverse_scale = get_reciprocal(scale)
# int_w = round_ste(tensor * inverse_scale).clip(-maxq, maxq - 1) + maxq
# qdq_result = (scale * (int_w - zp)).to(orig_dtype)
- tensor = tensor.mul_(inverse_scale).round_().clamp_(-maxq, maxq - 1)
+ tensor = round_ste(tensor.mul_(inverse_scale) + v).clamp_(-maxq, maxq - 1)
tensor = tensor.mul_(scale).to(orig_dtype)
tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.py
index 73645dd60..897dbb17a 100644
--- a/auto_round/data_type/int.py
+++ b/auto_round/data_type/int.py
@@ -174,6 +174,7 @@ def quant_tensor_sym(
tensor_min=None,
tensor_max=None,
q_scale_thresh=1e-5,
+ init_scale=None,
**kwargs
):
"""Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community
@@ -196,6 +197,16 @@ def quant_tensor_sym(
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
maxq = 2 ** (bits - 1)
+ if init_scale is not None:
+ scale = init_scale * max_scale.unsqueeze(dim=-1)
+ scale = scale.to(scale_dtype)
+ scale = torch.where(scale < 0, torch.clamp(scale, max=-q_scale_thresh), torch.clamp(scale, min=q_scale_thresh))
+ int_w = round_ste(tensor / scale + v)
+ q = torch.clamp(int_w, -maxq, maxq - 1)
+ qdq_result = (scale * q).to(tensor.dtype)
+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)
+ return qdq_result, scale, maxq
+
if tensor_min is None or tensor_max is None:
wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)
wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index 9a7315a11..0b8b435cf 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -85,7 +85,51 @@ def quant_element(tensor, ebits, mbits, max_norm, mantissa_rounding="even"):
return tensor
-def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_rounding="even", data_type="mx_fp", **kwargs):
+def qdq_mxfp(tensor, max_val, max_norm, emax, ebits, mbits):
+ shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), torch.log2(max_val))
+ shared_exp = torch.floor(shared_exp)
+ scale_emax = 2 ** (8 - 1) - 1
+ shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)
+
+ scale = torch.pow(2.0, shared_exp)
+ tensor = tensor / scale
+ tensor = torch.clamp(tensor, min=-max_norm, max=max_norm)
+ tensor = quant_element(tensor, ebits, mbits, max_norm)
+ return tensor * scale
+
+
+def search_mx_scale(tensor, bits, qw=None):
+ data_type = "mx_fp" + str(bits)
+ ebits, mbits, emax, max_norm, _ = MXFP_FORMAT_CACHE[data_type]
+ tensor = tensor.to(torch.float32)
+ max_val, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)
+ qdq_t = qdq_mxfp(tensor, max_val, max_norm, emax, ebits, mbits)
+ best_loss = torch.sum((qdq_t - tensor) ** 2 * qw, dim=-1)
+ scales = torch.ones_like(max_val)
+ for scale_value in range(50, 152):
+ tmp_scale = scale_value / 100.0
+ if tmp_scale == 1.0:
+ continue
+ tmp_qdq_t = qdq_mxfp(tensor, max_val * tmp_scale, max_norm, emax, ebits, mbits)
+ loss = torch.sum((tmp_qdq_t - tensor) ** 2 * qw, dim=-1)
+ replace_id = loss < best_loss
+ scales[replace_id] = tmp_scale
+ best_loss[replace_id] = loss[replace_id]
+ return scales
+
+
+def quant_mx(
+ tensor,
+ bits=4,
+ group_size=-1,
+ v=0,
+ max_scale=1.0,
+ init_scale=1.0,
+ mantissa_rounding="even",
+ data_type="mx_fp",
+ **kwargs,
+):
"""Quantize the given tensor using the specified parameters.
This function performs quantization on the `tensor` tensor according to the
@@ -109,15 +153,16 @@ def quant_mx(tensor, bits=4, group_size=-1, v=0, max_scale=1.0, mantissa_roundin
KeyError: If `data_type` is not found in `MXFP_FORMAT_CACHE`.
"""
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
+ init_scale = 1.0 if init_scale is None else init_scale
data_type = data_type if data_type in MXFP_FORMAT_CACHE else "mx_fp" + str(bits)
ebits, mbits, emax, max_norm, min_norm = MXFP_FORMAT_CACHE[data_type]
orig_dtype = tensor.dtype
tensor = tensor.to(torch.float32)
max_val, _ = torch.max(torch.abs(tensor), dim=-1, keepdim=True)
if isinstance(max_scale, torch.Tensor):
- max_val *= (max_scale.unsqueeze(dim=-1)).to(tensor.device)
+ max_val *= init_scale * (max_scale.unsqueeze(dim=-1)).to(tensor.device)
else:
- max_val *= max_scale
+ max_val *= init_scale * max_scale
# shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))
shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), torch.log2(max_val))
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index 60cee640a..7a3a2df52 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -79,15 +79,38 @@ def ref_nvfp4_quant(x, global_scale, block_size=16, v=0, scale_coeff=1.0):
return (cast_to_fp4(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale
+def search_nvfp4_scale(tensor, bits=4, qw=None):
+ tensor = tensor.to(torch.float32)
+ qdq_t, dummy_scale, _ = nv_fp4(tensor, bits=bits, group_size=16, v=0, max_scale=1.0)
+ best_loss = torch.sum((qdq_t - tensor) ** 2 * qw, dim=-1)
+ scales = torch.ones_like(dummy_scale)
+ for scale_value in range(50, 152):
+ tmp_scale = scale_value / 100.0
+ if tmp_scale == 1.0:
+ continue
+ scales_new = torch.ones_like(dummy_scale) * tmp_scale
+ tmp_qdq_t, _, _ = nv_fp4(tensor, bits=bits, group_size=16, v=0, max_scale=scales_new)
+ loss = torch.sum((tmp_qdq_t - tensor) ** 2 * qw, dim=-1)
+ replace_id = loss < best_loss
+ scales[replace_id] = scales_new[replace_id]
+ best_loss[replace_id] = loss[replace_id]
+ return scales
+
+
@register_dtype("nv_fp4")
-def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, **kwargs):
+def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, max_scale=1.0, init_scale=1.0, **kwargs):
orig_dtype = tensor.dtype
+ init_scale = 1.0 if init_scale is None else init_scale
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
if global_scale is None:
tensor_max = tensor.abs().max().to(torch.float32)
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)
global_scale = global_scale.to(device=tensor.device, dtype=torch.float32)
- qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v, scale_coeff=max_scale)
+ if isinstance(max_scale, torch.Tensor):
+ max_scale = max_scale.view(-1).to(tensor.device)
+ if isinstance(init_scale, torch.Tensor):
+ init_scale = init_scale.view(-1).to(tensor.device)
+ qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v, scale_coeff=max_scale * init_scale)
qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)
return qdq_res.to(orig_dtype), scale, None
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 471739599..6a1d7948c 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -34,7 +34,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"AR_DISABLE_DATASET_SUBPROCESS": lambda: os.getenv("AR_DISABLE_DATASET_SUBPROCESS", "0").lower() in ("1", "true"),
"AR_DISABLE_COPY_MTP_WEIGHTS": lambda: os.getenv("AR_DISABLE_COPY_MTP_WEIGHTS", "0").lower()
in ("1", "true", "yes"),
- "AR_DISABLE_NEW_ARCH": lambda: os.getenv("AR_DISABLE_NEW_ARCH", "0").lower() in ("1", "true", "yes"),
"AR_ACT_SCALE": lambda: float(os.getenv("AR_ACT_SCALE", "1.0")),
"AR_ENABLE_ACT_MINMAX_TUNING": lambda: os.getenv("AR_ENABLE_ACT_MINMAX_TUNING", "0").lower()
in ("1", "true", "yes"),
@@ -87,7 +86,7 @@ def set_config(**kwargs):
for key, value in kwargs.items():
if key in environment_variables:
# Convert value to appropriate string format
- if key in ("AR_USE_MODELSCOPE", "AR_DISABLE_NEW_ARCH"):
+ if key == "AR_USE_MODELSCOPE":
# Handle boolean values for boolean env flags
str_value = "true" if value in [True, "True", "true", "1", 1] else "false"
else:
diff --git a/auto_round/experimental/rotation_inplace/__init__.py b/auto_round/experimental/rotation_inplace/__init__.py
deleted file mode 100644
index 07b3d40c8..000000000
--- a/auto_round/experimental/rotation_inplace/__init__.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.inplace`.
-"""
-
-from auto_round.algorithms.transforms.rotation.inplace.apply import ( # noqa: F401
- apply_rotation_transform,
-)
-from auto_round.algorithms.transforms.rotation.inplace.hooks import ( # noqa: F401
- clear_random_hadamard_cache,
-)
diff --git a/auto_round/experimental/rotation_inplace/apply_rotation_transform.py b/auto_round/experimental/rotation_inplace/apply_rotation_transform.py
deleted file mode 100644
index 86e429c10..000000000
--- a/auto_round/experimental/rotation_inplace/apply_rotation_transform.py
+++ /dev/null
@@ -1,12 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.inplace.apply`.
-"""
-
-from auto_round.algorithms.transforms.rotation.inplace.apply import * # noqa: F401, F403
-from auto_round.algorithms.transforms.rotation.inplace.apply import ( # noqa: F401
- apply_rotation_transform,
-)
diff --git a/auto_round/experimental/rotation_inplace/model_config.py b/auto_round/experimental/rotation_inplace/model_config.py
deleted file mode 100644
index 35078cd28..000000000
--- a/auto_round/experimental/rotation_inplace/model_config.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.inplace.model_config`.
-"""
-
-from auto_round.algorithms.transforms.rotation.inplace.model_config import * # noqa: F401, F403
diff --git a/auto_round/experimental/rotation_inplace/utils.py b/auto_round/experimental/rotation_inplace/utils.py
deleted file mode 100644
index 4ddd80d48..000000000
--- a/auto_round/experimental/rotation_inplace/utils.py
+++ /dev/null
@@ -1,9 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.inplace.hooks`.
-"""
-
-from auto_round.algorithms.transforms.rotation.inplace.hooks import * # noqa: F401, F403
diff --git a/auto_round/experimental/transform/__init__.py b/auto_round/experimental/transform/__init__.py
deleted file mode 100644
index 0b3bf2eae..000000000
--- a/auto_round/experimental/transform/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
diff --git a/auto_round/experimental/transform/apply.py b/auto_round/experimental/transform/apply.py
deleted file mode 100644
index e0aa77e13..000000000
--- a/auto_round/experimental/transform/apply.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.apply`.
-"""
-
-from auto_round.algorithms.transforms.rotation.apply import ( # noqa: F401
- apply_rotation_transform,
-)
-
-__all__ = ["apply_rotation_transform"]
diff --git a/auto_round/experimental/transform/hadamards.py b/auto_round/experimental/transform/hadamards.py
deleted file mode 100644
index 83efa91b6..000000000
--- a/auto_round/experimental/transform/hadamards.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.transforms`.
-"""
-
-from auto_round.algorithms.transforms.rotation.transforms import (
- HADAMARDS,
- HadamardTransform,
- RandomHadamardTransform,
-)
-from auto_round.algorithms.transforms.rotation.transforms import _filter_kwargs as filter_kwarg_dict # noqa: F401
-from auto_round.algorithms.transforms.rotation.transforms import (
- build_hadamard_transform,
-)
-
-__all__ = [
- "HADAMARDS",
- "HadamardTransform",
- "RandomHadamardTransform",
- "build_hadamard_transform",
-]
diff --git a/auto_round/experimental/transform/patch_modules.py b/auto_round/experimental/transform/patch_modules.py
deleted file mode 100644
index 980c28588..000000000
--- a/auto_round/experimental/transform/patch_modules.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.patch`.
-"""
-
-from auto_round.algorithms.transforms.rotation.patch import ( # noqa: F401
- patch_quantlinear,
- patch_wrapperlinear_to_apply_transform,
- patch_wrapperwalayer_forward_to_apply_transform,
-)
-
-__all__ = [
- "patch_quantlinear",
- "patch_wrapperlinear_to_apply_transform",
- "patch_wrapperwalayer_forward_to_apply_transform",
-]
diff --git a/auto_round/experimental/transform/rotation_config.py b/auto_round/experimental/transform/rotation_config.py
deleted file mode 100644
index fad2ede62..000000000
--- a/auto_round/experimental/transform/rotation_config.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical ``RotationConfig`` schema now lives in
-:mod:`auto_round.algorithms.transforms.rotation.config`.
-"""
-
-from auto_round.algorithms.transforms.rotation.config import RotationConfig # noqa: F401
-
-__all__ = ["RotationConfig"]
diff --git a/auto_round/experimental/transform/triton/__init__.py b/auto_round/experimental/transform/triton/__init__.py
deleted file mode 100644
index 0b3bf2eae..000000000
--- a/auto_round/experimental/transform/triton/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
diff --git a/auto_round/experimental/transform/triton/mxfp4.py b/auto_round/experimental/transform/triton/mxfp4.py
deleted file mode 100644
index 3cc26a7d0..000000000
--- a/auto_round/experimental/transform/triton/mxfp4.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.utils.triton.mxfp4`.
-"""
-
-from auto_round.algorithms.transforms.rotation.utils.triton.mxfp4 import ( # noqa: F401
- mxfp4_forward_kernel,
- mxfp4_forward_kernel_wrapper,
-)
-
-__all__ = ["mxfp4_forward_kernel", "mxfp4_forward_kernel_wrapper"]
diff --git a/auto_round/experimental/transform/utils/__init__.py b/auto_round/experimental/transform/utils/__init__.py
deleted file mode 100644
index 0b3bf2eae..000000000
--- a/auto_round/experimental/transform/utils/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
diff --git a/auto_round/experimental/transform/utils/hadamard.py b/auto_round/experimental/transform/utils/hadamard.py
deleted file mode 100644
index 8bed8a7c1..000000000
--- a/auto_round/experimental/transform/utils/hadamard.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.utils.math`.
-"""
-
-from auto_round.algorithms.transforms.rotation.utils.math import _HADAMARD_MATRICES_PATH as REPO_PATH # noqa: F401
-from auto_round.algorithms.transforms.rotation.utils.math import (
- _fetch_hadamard_divisor,
- _matmul_hadU,
- deterministic_hadamard_matrix,
- is_pow2,
- random_hadamard_matrix,
-)
-
-__all__ = ["random_hadamard_matrix", "deterministic_hadamard_matrix", "is_pow2"]
diff --git a/auto_round/experimental/transform/utils/matrix.py b/auto_round/experimental/transform/utils/matrix.py
deleted file mode 100644
index 184d91055..000000000
--- a/auto_round/experimental/transform/utils/matrix.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# # Copyright (C) 2026 Intel Corporation
-# # SPDX-License-Identifier: Apache-2.0
-"""Backward-compat re-export shim.
-
-The canonical implementation now lives in
-:mod:`auto_round.algorithms.transforms.rotation.utils.matrix`.
-"""
-
-from auto_round.algorithms.transforms.rotation.utils.matrix import ( # noqa: F401
- apply_transform_weight,
- multihead_matmul,
-)
-
-# Old private name kept for backward compatibility.
-_multihead_matmul = multihead_matmul
-
-__all__ = ["apply_transform_weight"]
diff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.py
index 80e830bed..d27c73abb 100644
--- a/auto_round/experimental/utils.py
+++ b/auto_round/experimental/utils.py
@@ -16,9 +16,9 @@ from typing import Any
import torch
+from auto_round.algorithms.transforms.rotation.config import RotationConfig
+from auto_round.algorithms.transforms.rotation.transforms import HADAMARDS
from auto_round.compressors.utils import is_mx_fp, is_nv_fp
-from auto_round.experimental.transform.hadamards import HADAMARDS
-from auto_round.experimental.transform.rotation_config import RotationConfig
from auto_round.utils import logger
SUPPORTED_QUANTIZATION_SCHEMES = ["MXFP8", "MXFP4", "NVFP4"]
@@ -130,7 +130,9 @@ def is_triton_kernel_available(data_type: str) -> bool:
return False
try:
- from auto_round.experimental.transform.triton.mxfp4 import mxfp4_forward_kernel_wrapper # pylint: disable=E0401
+ from auto_round.algorithms.transforms.rotation.utils.triton.mxfp4 import ( # pylint: disable=E0401
+ mxfp4_forward_kernel_wrapper,
+ )
except Exception:
return False
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 370baf29a..25f6225bf 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -830,8 +830,8 @@ def convert_hf_model(model: nn.Module, target_device: str = "cpu") -> tuple[nn.M
rotation_config = getattr(quantization_config, "rotation_config", None)
if rotation_config is not None and rotation_config:
- from auto_round.experimental.transform.apply import apply_rotation_transform
- from auto_round.experimental.transform.rotation_config import RotationConfig
+ from auto_round.algorithms.transforms.rotation.apply import apply_rotation_transform
+ from auto_round.algorithms.transforms.rotation.config import RotationConfig
# apply forward hook
act_rotation_config = RotationConfig(
diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.py
index b58a030c7..77109d02a 100755
--- a/auto_round/utils/weight_handler.py
+++ b/auto_round/utils/weight_handler.py
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
"""
Extensible Module Weight Type Conversion Framework.
diff --git a/auto_round/wrapper.py b/auto_round/wrapper.py
index 93324334b..bd6a78570 100644
--- a/auto_round/wrapper.py
+++ b/auto_round/wrapper.py
@@ -73,6 +73,8 @@ class WrapperLinear(torch.nn.Module):
device (str): Device on which to run computations (e.g., 'cpu' or 'cuda').
"""
+ minmax_scale_bound = (0.0, 1.0)
+
def __init__(
self,
orig_layer,
@@ -236,8 +238,9 @@ class WrapperLinear(torch.nn.Module):
"""
if self.orig_layer.bits >= 16:
return self.orig_layer.weight, None, None
- min_scale.data.clamp_(0.0, 1.0)
- max_scale.data.clamp_(0.0, 1.0)
+ min_bound, max_bound = self.minmax_scale_bound
+ min_scale.data.clamp_(min_bound, max_bound)
+ max_scale.data.clamp_(min_bound, max_bound)
weight = self.orig_layer.weight
if weight.device.type == "meta":
weight = self.orig_layer.get_weight().to(self.device)
@@ -248,6 +251,8 @@ class WrapperLinear(torch.nn.Module):
if hasattr(self.orig_layer, "super_bits"):
quant_kwargs["super_bits"] = self.orig_layer.super_bits
quant_kwargs["super_group_size"] = self.orig_layer.super_group_size
+ if hasattr(self, "_extra_quant_kwargs"):
+ quant_kwargs.update(self._extra_quant_kwargs())
weight_q, scale, zp = self.weight_quant_func(
weight.to(self.device),
@@ -263,6 +268,7 @@ class WrapperLinear(torch.nn.Module):
q_scale_thresh=self.q_scale_thresh,
imatrix=self.orig_layer.imatrix.to(weight.device) if hasattr(self.orig_layer, "imatrix") else None,
global_scale=getattr(self, "weight_global_scale", None),
+ init_scale=getattr(self, "init_scale", None),
**quant_kwargs,
)
weight_q = weight_q.to(weight.dtype)
@@ -749,6 +755,7 @@ def wrapper_block(
enable_norm_bias_tuning,
enable_torch_compile=False,
device="cpu",
+ wrapper_cls=WrapperLinear,
**kwargs,
):
"""Wraps the layers in the given block with a custom Wrapper module.
@@ -770,7 +777,7 @@ def wrapper_block(
if not check_to_quantized(m):
unquantized_layers.append(n)
continue
- new_m = WrapperLinear(
+ new_m = wrapper_cls(
m,
enable_minmax_tuning=enable_minmax_tuning,
enable_norm_bias_tuning=enable_norm_bias_tuning,
diff --git a/test/test_cpu/utils/test_calibration_inputs.py b/test/test_cpu/utils/test_calibration_inputs.py
new file mode 100644
index 000000000..30e84b2fa
--- /dev/null
+++ b/test/test_cpu/utils/test_calibration_inputs.py
@@ -0,0 +1,97 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from types import SimpleNamespace
+
+import torch
+
+from auto_round.calibration.inputs import preprocess_block_inputs, split_inputs
+
+
+def test_split_inputs_llm():
+ inputs = {
+ "input_ids": torch.tensor([[1, 2, 3]]),
+ "attention_mask": torch.tensor([[1, 1, 1]]),
+ }
+
+ input_ids, input_others = split_inputs(inputs, "input_ids", is_diffusion=False)
+
+ assert torch.equal(input_ids, torch.tensor([[1, 2, 3]]))
+ assert "input_ids" not in inputs
+ assert input_others is inputs
+ assert set(input_others) == {"attention_mask"}
+
+
+def test_split_inputs_diffusion():
+ inputs = {
+ "hidden_states": torch.randn(1, 4),
+ "encoder_hidden_states": torch.randn(1, 4),
+ "timestep": torch.tensor([1]),
+ }
+
+ input_ids, input_others = split_inputs(inputs, "ignored", is_diffusion=True)
+
+ assert set(input_ids) == {"hidden_states", "encoder_hidden_states"}
+ assert "hidden_states" not in input_others
+ assert "encoder_hidden_states" not in input_others
+ assert set(input_others) == {"timestep"}
+
+
+def test_preprocess_block_inputs_casts_tensors(monkeypatch):
+ calls = []
+ monkeypatch.setattr(
+ "auto_round.calibration.inputs.clear_memory",
+ lambda device_list: calls.append(tuple(device_list)),
+ )
+
+ model_context = SimpleNamespace(is_diffusion=False, amp=True, amp_dtype=torch.bfloat16)
+ compress_context = SimpleNamespace(cache_device=torch.device("cpu"), device_list=["cpu"])
+ inputs = {
+ "input_ids": torch.ones((1, 3), dtype=torch.float16),
+ "attention_mask": torch.ones((1, 3), dtype=torch.float16),
+ "position_ids": torch.ones((1, 3), dtype=torch.int64),
+ }
+
+ input_ids, input_others = preprocess_block_inputs(
+ inputs,
+ model_context=model_context,
+ compress_context=compress_context,
+ )
+
+ assert calls == [("cpu",)]
+ assert input_ids.dtype == torch.bfloat16
+ assert input_others["attention_mask"].dtype == torch.bfloat16
+ assert input_others["position_ids"].dtype == torch.int64
+ assert "input_ids" not in inputs
+
+
+def test_preprocess_block_inputs_keeps_list_dtype():
+ model_context = SimpleNamespace(is_diffusion=False, amp=True, amp_dtype=torch.bfloat16)
+ compress_context = SimpleNamespace(cache_device=torch.device("cpu"), device_list=["cpu"])
+ cached_states = [torch.ones((1, 3), dtype=torch.float16)]
+ inputs = {
+ "input_ids": torch.ones((1, 3), dtype=torch.float16),
+ "past_key_values": cached_states,
+ }
+
+ _, input_others = preprocess_block_inputs(
+ inputs,
+ model_context=model_context,
+ compress_context=compress_context,
+ )
+
+ # The extracted helper intentionally preserves the legacy implementation,
+ # which iterates list entries through to_dtype() without writing the result
+ # back. Lock this down so the refactor does not change runtime behaviour.
+ assert input_others["past_key_values"][0].dtype == torch.float16
diff --git a/test/test_cpu/utils/test_set_layer_config.py b/test/test_cpu/utils/test_set_layer_config.py
index 91a2222e7..d4e0e7844 100644
--- a/test/test_cpu/utils/test_set_layer_config.py
+++ b/test/test_cpu/utils/test_set_layer_config.py
@@ -65,6 +65,7 @@ class _Router(nn.Module):
class _MoeBlock(nn.Module):
+
def __init__(self, size: int = 32):
super().__init__()
self.gate = _Router(size)
@@ -72,6 +73,7 @@ class _MoeBlock(nn.Module):
class _UnsupportedGateModel(nn.Module):
+
def __init__(self, size: int = 32):
super().__init__()
self.mlp = _MoeBlock(size)
@@ -172,3 +174,22 @@ class TestIgnoreLayersRegexFallback:
assert "mlp.gate" in regex_config
assert regex_config["mlp.gate"]["bits"] == 16
assert regex_config["mlp.gate"]["data_type"] == "float"
+
+
+class TestSchemeShorthand:
+ """layer_config dict entries may use a legacy scheme shorthand plus overrides."""
+
+ def test_scheme_key_resolves_before_validation(self, model):
+ layer_config, _, _ = _call_set_layer_config(
+ model,
+ layer_config={
+ "layers.0.fc1": {"scheme": "W8A16"},
+ "layers.0.fc2": {"scheme": "W2A16", "group_size": 32},
+ },
+ )
+
+ assert layer_config["layers.0.fc1"]["bits"] == 8
+ assert layer_config["layers.0.fc1"]["act_bits"] == 16
+ assert "scheme" not in layer_config["layers.0.fc1"]
+ assert layer_config["layers.0.fc2"]["bits"] == 2
+ assert layer_config["layers.0.fc2"]["group_size"] == 32
diff --git a/test/test_cpu/utils/test_shard_writer.py b/test/test_cpu/utils/test_shard_writer.py
index 61d00ba92..15e78a4ae 100644
--- a/test/test_cpu/utils/test_shard_writer.py
+++ b/test/test_cpu/utils/test_shard_writer.py
@@ -18,15 +18,19 @@ from types import SimpleNamespace
import torch
from auto_round.compressors.shard_writer import ShardWriter
+from auto_round.context.compress import CompressContext
+from auto_round.context.model import ModelContext
class _ToyBlock(torch.nn.Module):
+
def __init__(self):
super().__init__()
self.linear = torch.nn.Linear(4, 4)
class _DiffusionStyleModel(torch.nn.Module):
+
def __init__(self):
super().__init__()
self.transformer_blocks = torch.nn.ModuleList([_ToyBlock()])
@@ -34,23 +38,24 @@ class _DiffusionStyleModel(torch.nn.Module):
self.config = SimpleNamespace(model_type="toy-diffusion")
-class _RounderStub:
- def __init__(self, model, output_dir):
- self.model = model
- self.bits = 4
- self.formats = [object()]
- self.max_shard_size = "1MB"
- self.safe_serialization = False
- self._output_dir = output_dir
+class _FormatStub:
+
+ def get_backend_name(self):
+ return "auto_round"
+
- def _get_save_folder_name(self, _format):
- return self._output_dir
+def _make_writer(model, output_dir, monkeypatch):
+ ShardWriter.reset()
+ compress_context = SimpleNamespace(formats=[_FormatStub()], output_dir=output_dir)
+ model_context = SimpleNamespace(is_diffusion=False)
+ monkeypatch.setattr(CompressContext, "get_context", classmethod(lambda cls: compress_context))
+ monkeypatch.setattr(ModelContext, "get_context", classmethod(lambda cls: model_context))
+ return ShardWriter(model, bits=4, max_shard_size="1MB", safe_serialization=False)
-def test_finalize_saves_tail_layer_when_tie_word_embeddings_missing(tmp_path):
+def test_finalize_saves_tail_layer_when_tie_word_embeddings_missing(tmp_path, monkeypatch):
model = _DiffusionStyleModel()
- rounder = _RounderStub(model, str(tmp_path))
- writer = ShardWriter(rounder)
+ writer = _make_writer(model, str(tmp_path), monkeypatch)
assert writer.lm_head_name == "proj_out"
assert not hasattr(model.config, "tie_word_embeddings")
@@ -76,12 +81,11 @@ class _LMStyleModel(torch.nn.Module):
self.config = SimpleNamespace(model_type="toy-lm", tie_word_embeddings=True)
-def test_finalize_skips_lm_head_when_tie_word_embeddings_true(tmp_path):
+def test_finalize_skips_lm_head_when_tie_word_embeddings_true(tmp_path, monkeypatch):
"""Complementary test: when tie_word_embeddings=True the lm_head should be
skipped (not written to disk) and offloaded to meta."""
model = _LMStyleModel()
- rounder = _RounderStub(model, str(tmp_path))
- writer = ShardWriter(rounder)
+ writer = _make_writer(model, str(tmp_path), monkeypatch)
assert writer.lm_head_name == "lm_head"
@@ -96,11 +100,10 @@ def test_finalize_skips_lm_head_when_tie_word_embeddings_true(tmp_path):
assert model.lm_head.weight.device.type == "meta"
-def test_finalize_offloads_module_with_tensor_in_parameters(tmp_path):
+def test_finalize_offloads_module_with_tensor_in_parameters(tmp_path, monkeypatch):
model = _DiffusionStyleModel()
model.transformer_blocks[0].linear._parameters["weight"] = model.transformer_blocks[0].linear.weight.to("cpu")
- rounder = _RounderStub(model, str(tmp_path))
- writer = ShardWriter(rounder)
+ writer = _make_writer(model, str(tmp_path), monkeypatch)
writer.save_module(model.transformer_blocks[0], "transformer_blocks.0")
writer.finalize()
diff --git a/test/test_cuda/quantization/test_torch_compile.py b/test/test_cuda/quantization/test_torch_compile.py
index 21e9bbc97..a578e8821 100644
--- a/test/test_cuda/quantization/test_torch_compile.py
+++ b/test/test_cuda/quantization/test_torch_compile.py
@@ -8,7 +8,7 @@ from transformers import AutoTokenizer
from auto_round import AutoRound
from auto_round.algorithms.quantization.base import BaseQuantizers
from auto_round.algorithms.quantization.rtn.config import RTNConfig
-from auto_round.compressors_new.utils import block_forward
+from auto_round.compressors.utils import block_forward
from ...envs import require_gguf
from ...helpers import get_model_path, get_tiny_model