Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enable AutoScheme parallel and disk stream by default (#2105)

Xin He8a9a1d138aad
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index 33e39be9b..7a1584937 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -132,15 +132,15 @@ class AutoSchemeWrapperLinear(WrapperLinear):
qdq_x, scale, zp = self.act_qdq_func(x, act_min_scale, act_max_scale, act_max)
if self.grad_mode:
with torch.no_grad():
- self.max_act_value = torch.abs(x).max()
- if torch.abs(x).max() != 0:
+ max_act_value = torch.abs(x).max()
+ self.max_act_value = max_act_value
+ if max_act_value != 0:
self.act_cnt += 1
- x_diff = x - qdq_x
- self.x_diff = x_diff.to("cpu")
+ x_diff = (x - qdq_x).to("cpu")
def save_grad(grad):
"""Backward hook: accumulate activation score from grad * (x - qdq_x)."""
- if self.max_act_value == 0:
+ if max_act_value == 0:
if torch.abs(grad).max() != 0:
raise ValueError
"""
@@ -149,13 +149,12 @@ class AutoSchemeWrapperLinear(WrapperLinear):
def test_multi_card(self):
model_name = "/models/Qwen3-8B"
"""
- if torch.isnan(grad).any() or torch.isnan(self.x_diff).any():
+ if torch.isnan(grad).any() or torch.isnan(x_diff).any():
self.act_cnt -= 1
return None
- self.act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()
+ self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()
self.mix_score = self.weight_score + self.act_score
- self.x_diff = None
return None
if qdq_x.requires_grad:
@@ -286,15 +285,15 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
qdq_x, scale, zp = self.act_qdq_func(x, act_min_scale, act_max_scale, act_max)
if self.grad_mode:
with torch.no_grad():
- self.max_act_value = torch.abs(x).max()
- if torch.abs(x).max() != 0:
+ max_act_value = torch.abs(x).max()
+ self.max_act_value = max_act_value
+ if max_act_value != 0:
self.act_cnt += 1
- x_diff = x - qdq_x
- self.x_diff = x_diff.to("cpu")
+ x_diff = (x - qdq_x).to("cpu")
def save_grad(grad):
"""Backward hook: accumulate activation score from grad * (x - qdq_x)."""
- if self.max_act_value == 0:
+ if max_act_value == 0:
if torch.abs(grad).max() != 0:
raise ValueError
"""
@@ -303,13 +302,12 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):
def test_multi_card(self):
model_name = "/models/Qwen3-8B"
"""
- if torch.isnan(grad).any() or torch.isnan(self.x_diff).any():
+ if torch.isnan(grad).any() or torch.isnan(x_diff).any():
self.act_cnt -= 1
return None
- self.act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()
+ self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()
self.mix_score = self.weight_score + self.act_score
- self.x_diff = None
return None
if qdq_x.requires_grad:
@@ -560,9 +558,6 @@ class MyCustomError(Exception):
super().__init__(message)
-last_grad_input = None
-
-
def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_device="cpu", disk_index=None):
"""Wrap every block's forward so that, for one calibration batch, it (1) moves itself to
``major_device`` on demand, (2) records its own inputs into ``block_inputs`` (on CPU) so
@@ -700,6 +695,20 @@ def model_forward(model, data, **forward_kwargs):
return model(**prepared, **forward_kwargs), prepared
+def _prepare_replay_input(block_input_args, block_input_kwargs, block_name):
+ """Find the floating hidden-state tensor whose gradient feeds the preceding block."""
+ candidates = []
+ if "hidden_states" in block_input_kwargs:
+ candidates.append(block_input_kwargs["hidden_states"])
+ candidates.extend(block_input_args)
+ candidates.extend(value for key, value in block_input_kwargs.items() if key != "hidden_states")
+ for value in candidates:
+ if isinstance(value, torch.Tensor) and value.is_floating_point():
+ value.requires_grad_(True)
+ return value
+ raise RuntimeError(f"No floating replay input found for block {block_name}")
+
+
def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, scheme_tag=None, disk_index=None):
"""Run one full scoring pass (all calibration batches) in low-GPU-memory mode.
@@ -722,130 +731,126 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, sch
module = get_module(model, name)
module.orig_forward = module.forward
+ captured_grad = None
+
def backward_pre_hook(module, grad_input):
"""Hook executed before backward propagation."""
- global last_grad_input
- last_grad_input = grad_input
+ nonlocal captured_grad
+ captured_grad = grad_input
get_current_device_manager().synchronize()
raise MyCustomError("Interrupt backward pass")
for batch_idx, data in enumerate(dataloader, start=1):
- prepare_model_low_gpu(model, block_inputs, major_device=major_device, pbar=pbar, disk_index=disk_index)
-
- # lm_head sits outside every decoder block, so it never gets `grad_mode=True`
- # in the manual block-by-block backward below. Scope the fix narrowly to
- # just lm_head (rather than every non-block module) to avoid enabling grad
- # tracking / scoring hooks on unrelated out-of-block layers, which would
- # add extra autograd-graph memory for no benefit. The backward flow is:
- # loss → lm_head (hook fires here) → norm → last_block (hook raises error)
- head_name = get_lm_head_name(model)
- if head_name is not None:
- # Once lm_head has been wrapped for scoring, `get_lm_head_name` resolves
- # to the inner original Linear (e.g. "lm_head.orig_layer") rather than
- # the wrapper itself ("lm_head") -- strip the suffix to reach the wrapper.
- head_name = head_name.removesuffix(".orig_layer")
- head_module = get_module(model, head_name)
- if hasattr(head_module, "grad_mode"):
- head_module.grad_mode = True
-
- # Register backward hook on the last block
- last_block = get_module(model, block_names[-1])
- last_block_backward_hook = last_block.register_full_backward_pre_hook(backward_pre_hook)
-
- data = to_device(data, model.device)
- # VLM datasets often already include ``labels``; LLM ones don't. Strip
- # any pre-existing ``labels`` from kwargs so we don't pass it twice.
- labels = data["labels"] if isinstance(data, dict) and "labels" in data else data["input_ids"]
- if isinstance(data, dict):
- data_for_forward = {k: v for k, v in data.items() if k != "labels"}
- else:
- data_for_forward = data
- # Route through the unified mllm forward so ``pixel_values`` /
- # ``images`` get cast to ``model.dtype`` (otherwise the vision tower
- # is silently bypassed on dtype mismatch and vision grad stays 0).
- output, _prepared = model_forward(model, data_for_forward, labels=labels, use_cache=False)
- clear_memory(device_list=major_device)
- memory_monitor.log_summary()
-
+ captured_grad = None
+ interrupted = False
+ last_block_backward_hook = None
try:
- # Backward pass (will be interrupted by the hook)
- output.loss.to(torch.float32).backward()
- except MyCustomError:
- pass
+ prepare_model_low_gpu(model, block_inputs, major_device=major_device, pbar=pbar, disk_index=disk_index)
+
+ # lm_head sits outside every decoder block, so it never gets `grad_mode=True`
+ # in the manual block-by-block backward below. Scope the fix narrowly to
+ # just lm_head (rather than every non-block module) to avoid enabling grad
+ # tracking / scoring hooks on unrelated out-of-block layers, which would
+ # add extra autograd-graph memory for no benefit. The backward flow is:
+ # loss → lm_head (hook fires here) → norm → last_block (hook raises error)
+ head_name = get_lm_head_name(model)
+ if head_name is not None:
+ # Once lm_head has been wrapped for scoring, `get_lm_head_name` resolves
+ # to the inner original Linear (e.g. "lm_head.orig_layer") rather than
+ # the wrapper itself ("lm_head") -- strip the suffix to reach the wrapper.
+ head_name = head_name.removesuffix(".orig_layer")
+ head_module = get_module(model, head_name)
+ if hasattr(head_module, "grad_mode"):
+ head_module.grad_mode = True
+
+ last_block = get_module(model, block_names[-1])
+ last_block_backward_hook = last_block.register_full_backward_pre_hook(backward_pre_hook)
+
+ data = to_device(data, model.device)
+ # VLM datasets often already include ``labels``; LLM ones don't. Strip
+ # any pre-existing ``labels`` from kwargs so we don't pass it twice.
+ labels = data["labels"] if isinstance(data, dict) and "labels" in data else data["input_ids"]
+ if isinstance(data, dict):
+ data_for_forward = {k: v for k, v in data.items() if k != "labels"}
+ else:
+ data_for_forward = data
+ # Route through the unified mllm forward so ``pixel_values`` /
+ # ``images`` get cast to ``model.dtype`` (otherwise the vision tower
+ # is silently bypassed on dtype mismatch and vision grad stays 0).
+ output, _prepared = model_forward(model, data_for_forward, labels=labels, use_cache=False)
+ clear_memory(device_list=major_device)
+ memory_monitor.log_summary()
+
+ try:
+ output.loss.to(torch.float32).backward()
+ except MyCustomError:
+ interrupted = True
+ if not interrupted or captured_grad is None:
+ raise RuntimeError("AutoScheme failed to capture the last block gradient for replay")
+ current_grad = captured_grad
+ finally:
+ if last_block_backward_hook is not None:
+ last_block_backward_hook.remove()
+ for name in block_names:
+ module = get_module(model, name)
+ module.forward = module.orig_forward
- current_grad = last_grad_input
del output, data
# Manually compute gradients block by block
- last_block_backward_hook.remove()
-
- for name in block_names:
- module = get_module(model, name)
- module.forward = module.orig_forward
- index = 0
for block_name in reversed(block_names):
- index += 1
# Retrieve stored inputs for the block
block_input_info = block_inputs.get(block_name, {})
block_input_args = to_device(block_input_info.get("args", []), major_device)
block_input_kwargs = to_device(block_input_info.get("kwargs", {}), major_device)
- block_input_args[0].requires_grad_(True)
+ replay_input = _prepare_replay_input(block_input_args, block_input_kwargs, block_name)
# Move the block module to GPU
block_module = get_module(model, block_name)
for n, m in block_module.named_modules():
if hasattr(m, "grad_mode"):
m.grad_mode = True
- if disk_index is not None:
- from auto_round.utils.disk_stream_util import materialize_module
-
- materialize_module(block_module, block_name, disk_index, device=major_device)
- move_module_to_tuning_device(block_module, major_device=major_device)
-
- # Set the block to eval mode while enabling gradient computation
- block_module.eval()
-
- # Recompute the block output
- block_output = block_module(*block_input_args, **block_input_kwargs)
+ materialized = False
+ try:
+ if disk_index is not None:
+ from auto_round.utils.disk_stream_util import materialize_module
+
+ materialize_module(block_module, block_name, disk_index, device=major_device)
+ materialized = True
+ move_module_to_tuning_device(block_module, major_device=major_device)
+
+ block_module.eval()
+ block_output = block_module(*block_input_args, **block_input_kwargs)
+
+ if isinstance(block_output, tuple):
+ main_output = block_output[0]
+ if isinstance(main_output, torch.Tensor) and main_output.is_floating_point():
+ main_output = main_output.requires_grad_(True)
+ elif isinstance(block_output, torch.Tensor) and block_output.is_floating_point():
+ main_output = block_output.requires_grad_(True)
+ else:
+ main_output = block_output
- # Ensure the output requires gradients
- if isinstance(block_output, tuple):
- # For tuple outputs, we usually care about the first element (hidden states)
- main_output = block_output[0]
- if isinstance(main_output, torch.Tensor) and main_output.is_floating_point():
- main_output = main_output.requires_grad_(True)
- elif isinstance(block_output, torch.Tensor) and block_output.is_floating_point():
- main_output = block_output.requires_grad_(True)
- else:
- main_output = block_output
-
- # Backward pass for the current block
- torch.autograd.backward(
- tensors=main_output,
- # inputs=block_input_args,
- grad_tensors=current_grad,
- retain_graph=True, # False may lead to zero gradients for some cases (e.g., MXFP4)
- )
+ torch.autograd.backward(
+ tensors=main_output,
+ grad_tensors=current_grad,
+ retain_graph=True, # False may lead to zero gradients for some cases (e.g., MXFP4)
+ )
- # Extract gradients w.r.t. the block input
- if block_input_args and isinstance(block_input_args[0], torch.Tensor):
- if block_input_args[0].grad is not None:
- current_grad = block_input_args[0].grad.detach().clone()
- else:
+ if replay_input.grad is None:
logger.warning(f"No gradient found for input of {block_name}, stopping backward replay")
break
- else:
- logger.warning(f"No suitable input gradient found for {block_name}")
- break
-
- del block_output, main_output, block_input_args, block_input_kwargs
- if disk_index is not None:
- from auto_round.utils.disk_stream_util import free_module
+ current_grad = replay_input.grad.detach().clone()
+ finally:
+ for parameter in block_module.parameters():
+ parameter.grad = None
+ if disk_index is not None and materialized:
+ from auto_round.utils.disk_stream_util import free_module
- free_module(block_module)
- else:
- block_module.to("cpu")
+ free_module(block_module)
+ elif disk_index is None:
+ block_module.to("cpu")
# clear_memory(device_list=major_device) # this one is very slow and seems does not affect max ram usage
memory_monitor.update()
@@ -1422,6 +1427,14 @@ def _get_scheme_bits(scheme):
return scheme.get("bits", 16)
+def _get_next_scheme_bits(schemes, indices, floor_bits):
+ """Return the smallest candidate bit width strictly above ``floor_bits``."""
+ higher_bits = {
+ _get_scheme_bits(schemes[index]) for index in indices if _get_scheme_bits(schemes[index]) > floor_bits
+ }
+ return min(higher_bits, default=None)
+
+
# Delta loss does not handle lm-head well, it is prone to assign low bit to lm-head which is not optimal
def _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores):
@@ -1516,17 +1529,28 @@ def _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_pa
def _scheme_repr(s):
- """Normalize a scheme (str/QuantizationScheme/dict) to a hashable repr."""
+ """Normalize a scheme to a stable representation independent of preset aliases."""
if isinstance(s, str):
- return s
+ try:
+ s = preset_name_to_scheme(s)
+ except KeyError:
+ return s.upper()
if isinstance(s, QuantizationScheme):
- return json.dumps(asdict(s), sort_keys=True, default=str)
+ s = asdict(s)
if isinstance(s, dict):
- return json.dumps(s, sort_keys=True, default=str)
+ return {key: value for key, value in sorted(s.items()) if value is not None}
return str(s)
-def _autoscheme_cache_key(
+def _stable_model_id(model_name):
+ """Return a portable model identifier for local paths and Hub model IDs."""
+ if not isinstance(model_name, str):
+ return model_name
+ normalized = model_name.rstrip("/\\")
+ return os.path.basename(normalized) or normalized
+
+
+def _autoscheme_cache_config(
model_name,
dataset,
nsamples,
@@ -1537,27 +1561,58 @@ def _autoscheme_cache_key(
scheme,
force_mllm,
low_gpu_mem_usage,
+ need_weight_grad=False,
):
- """Return a 16-char hex digest that uniquely identifies a **single-scheme** scoring run.
-
- The key covers every parameter that directly affects per-layer loss values
- **except** ``avg_bits`` / ``target_bits`` (only drive the DP step).
- Unlike the old version, this key is generated **per-scheme**, not per-run,
- so caching is granular: adding/removing schemes doesn't invalidate cached
- scores for unchanged schemes.
- """
- key_data = {
- "model_name": model_name,
+ """Build the portable, implementation-independent identity of a scoring run."""
+ return {
+ "model_id": _stable_model_id(model_name),
"dataset": dataset,
"nsamples": nsamples,
"seqlen": seqlen,
"batch_size": batch_size,
"quant_layer_names": sorted(quant_layer_names),
- "fixed_layer_scheme": {k: v for k, v in sorted(fixed_layer_scheme.items())},
+ "fixed_layer_scheme": {key: _scheme_repr(value) for key, value in sorted(fixed_layer_scheme.items())},
"scheme": _scheme_repr(scheme),
"force_mllm": force_mllm,
"low_gpu_mem_usage": low_gpu_mem_usage,
+ "need_weight_grad": need_weight_grad,
}
+
+
+def _autoscheme_cache_key(
+ model_name,
+ dataset,
+ nsamples,
+ seqlen,
+ batch_size,
+ quant_layer_names,
+ fixed_layer_scheme,
+ scheme,
+ force_mllm,
+ low_gpu_mem_usage,
+ need_weight_grad=False,
+):
+ """Return a 16-char hex digest that uniquely identifies a **single-scheme** scoring run.
+
+ The key covers every parameter that directly affects per-layer loss values
+ **except** ``avg_bits`` / ``target_bits`` (only drive the DP step).
+ Unlike the old version, this key is generated **per-scheme**, not per-run,
+ so caching is granular: adding/removing schemes doesn't invalidate cached
+ scores for unchanged schemes.
+ """
+ key_data = _autoscheme_cache_config(
+ model_name,
+ dataset,
+ nsamples,
+ seqlen,
+ batch_size,
+ quant_layer_names,
+ fixed_layer_scheme,
+ scheme,
+ force_mllm,
+ low_gpu_mem_usage,
+ need_weight_grad,
+ )
key_str = json.dumps(key_data, sort_keys=True, default=str)
return hashlib.sha256(key_str.encode()).hexdigest()[:16]
@@ -1565,7 +1620,8 @@ def _autoscheme_cache_key(
def _autoscheme_cache_path(cache_key, scheme_index):
"""Return the full path to the JSON cache file for a **single scheme**.
- Each scheme gets its own cache file under ``{AR_WORK_SPACE}/auto_scheme_cache/``
+ Each scheme gets its own cache file under ``AR_AUTO_SCHEME_CACHE`` or the
+ default ``~/.cache/auto_round`` directory
to enable granular reuse: adding/removing schemes or changing non-scoring
parameters (e.g., target_bits) doesn't invalidate caches for unmodified schemes.
@@ -1575,7 +1631,7 @@ def _autoscheme_cache_path(cache_key, scheme_index):
"""
from auto_round import envs as _envs
- cache_dir = os.path.join(_envs.AR_WORK_SPACE, "auto_scheme_cache")
+ cache_dir = os.path.expanduser(_envs.AR_AUTO_SCHEME_CACHE or "~/.cache/auto_round")
os.makedirs(cache_dir, exist_ok=True)
return os.path.join(cache_dir, f"scheme_{scheme_index:02d}_{cache_key}.json")
@@ -1588,6 +1644,7 @@ def _save_autoscheme_scores(
layer_scores,
total_loss_for_scheme,
total_params,
+ cache_config,
):
"""Persist scoring results for **a single scheme** to *cache_path* as JSON.
@@ -1595,12 +1652,13 @@ def _save_autoscheme_scores(
or changing unrelated parameters (e.g., target_bits) doesn't invalidate cached
scores for unchanged schemes.
- Schema version 3 (per-scheme, per-op cache)::
+ Schema version 1 (portable per-scheme, per-op cache)::
{
- "version": 3,
+ "version": 1,
"score_granularity": "per_op",
"cache_key": "<hex>",
+ "cache_config": { ... scoring inputs ... },
"scheme_index": 0,
"scheme": { ... scheme dict ... },
"created_at": "<ISO datetime>",
@@ -1616,9 +1674,10 @@ def _save_autoscheme_scores(
# re-apply grouping when loading a cache so the on-disk format stays
# per-op and backward/forward compatible.
data = {
- "version": 3,
+ "version": 1,
"score_granularity": "per_op",
"cache_key": cache_key,
+ "cache_config": cache_config,
"scheme_index": scheme_index,
"scheme": scheme_dict,
"created_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
@@ -1635,20 +1694,20 @@ def _save_autoscheme_scores(
def _load_autoscheme_scores(cache_path):
- """Load and validate a **single-scheme** per-op scoring cache file (version 3).
+ """Load and validate a **single-scheme** per-op scoring cache file (version 1).
Returns the parsed dict with keys ``layer_scores``, ``total_loss_for_scheme``,
and ``total_params`` on success, or ``None`` if the file is missing, malformed,
or fails the version sanity check.
"""
- _required = ("layer_scores", "total_loss_for_scheme", "total_params")
+ _required = ("cache_config", "layer_scores", "total_loss_for_scheme", "total_params")
try:
with open(cache_path, encoding="utf-8") as _f:
data = json.load(_f)
- if data.get("version") != 3 or data.get("score_granularity") != "per_op":
+ if data.get("version") != 1 or data.get("score_granularity") != "per_op":
logger.warning(
"AutoScheme: per-scheme cache schema mismatch "
- "(expected version=3, score_granularity=per_op; got version=%s, score_granularity=%s)",
+ "(expected version=1, score_granularity=per_op; got version=%s, score_granularity=%s)",
data.get("version"),
data.get("score_granularity"),
)
@@ -1669,6 +1728,42 @@ def _is_per_op_cache_compatible(cached_data, quant_layer_names, fixed_layer_sche
return set(cached_data["layer_scores"]) == expected_layers
+def _find_compatible_autoscheme_cache(
+ expected_path,
+ cache_config,
+ quant_layer_names,
+ fixed_layer_scheme,
+ total_params,
+):
+ """Find a compatible cache even when a downloaded JSON has a different filename."""
+ candidates = [expected_path]
+ cache_dir = os.path.dirname(expected_path)
+ try:
+ candidates.extend(
+ os.path.join(cache_dir, filename)
+ for filename in sorted(os.listdir(cache_dir))
+ if filename.endswith(".json") and os.path.join(cache_dir, filename) != expected_path
+ )
+ except OSError:
+ pass
+
+ for candidate in candidates:
+ if not os.path.isfile(candidate):
+ continue
+ cached_data = _load_autoscheme_scores(candidate)
+ if cached_data is None or not _is_per_op_cache_compatible(cached_data, quant_layer_names, fixed_layer_scheme):
+ continue
+ if cached_data["cache_config"] != cache_config:
+ continue
+ if cached_data.get("total_params") != total_params:
+ continue
+ cached_data["_cache_path"] = candidate
+ if candidate != expected_path:
+ logger.info("AutoScheme: using compatible downloaded cache %s", candidate)
+ return cached_data
+ return None
+
+
def _refresh_cached_layer_bits(
model,
quant_layer_names,
@@ -1762,6 +1857,20 @@ def _get_scheme_worker_count(num_schemes, num_gpus):
return num_schemes
+def _can_parallel_scheme_scoring(
+ parallel_enabled, model_id, num_gpus, uncached_count, need_imatrix, disk_stream_model, is_vlm
+):
+ """Return whether candidate schemes can be scored in separate workers."""
+ return (
+ parallel_enabled
+ and model_id is not None
+ and num_gpus >= 1
+ and uncached_count >= 2
+ and not need_imatrix
+ and (not disk_stream_model or not is_vlm)
+ )
+
+
def _load_scheme_worker_model(model_name, use_model_replacements, low_cpu_mem_usage):
"""Load an isolated worker model without an extra full-size CPU initialization copy."""
return load_model(
@@ -1773,6 +1882,24 @@ def _load_scheme_worker_model(model_name, use_model_replacements, low_cpu_mem_us
)
+def _load_disk_stream_scheme_worker_model(model_name, use_model_replacements=False):
+ """Build an isolated meta model and checkpoint index for a scoring worker."""
+ from auto_round.utils.disk_stream_util import build_meta_model
+
+ model, tokenizer, disk_index = build_meta_model(model_name)
+ if use_model_replacements:
+ from auto_round.special_model_handler import _handle_special_model, update_module
+
+ model = update_module(model, formats=None, cleanup_original=False)
+ model = _handle_special_model(model)
+ return model, tokenizer, disk_index
+
+
+def _prefer_disk_stream_scheme_worker(model_id, is_vlm, low_gpu_mem_usage):
+ """Prefer block-wise disk streaming whenever the worker scoring path supports it."""
+ return model_id is not None and not is_vlm and low_gpu_mem_usage
+
+
def _score_scheme_worker(args):
"""Score one scheme and return its index, scores, and worker VRAM peak."""
import traceback as _tb
@@ -1798,6 +1925,7 @@ def _score_scheme_worker(args):
worker_device,
total_schemes,
progress_queue,
+ disk_stream_model,
) = args
from auto_round.auto_scheme.utils import _scheme_short_name as _short_name
@@ -1806,19 +1934,40 @@ def _score_scheme_worker(args):
from auto_round.utils import get_block_names as _get_block_names
from auto_round.utils import get_module as _get_module
- try:
- model, tokenizer, processor, _, _, is_vlm, _ = _load_scheme_worker_model(
- model_name,
- use_model_replacements,
- low_cpu_mem_usage,
- )
- except Exception as exc:
- raise RuntimeError(
- f"_score_scheme_worker[{index}]: failed to load model {model_name!r}\n{_tb.format_exc()}"
- ) from exc
+ disk_index = None
+ if disk_stream_model:
+ try:
+ model, tokenizer, disk_index = _load_disk_stream_scheme_worker_model(
+ model_name, use_model_replacements=use_model_replacements
+ )
+ processor = None
+ except Exception as exc: # noqa: BLE001
+ logger.warning(
+ "_score_scheme_worker[%d]: disk-stream load failed for %r (%s); falling back to regular loading.",
+ index,
+ model_name,
+ exc,
+ )
+ disk_index = None
+
+ if disk_index is None:
+ try:
+ model, tokenizer, processor, _, _, is_vlm, _ = _load_scheme_worker_model(
+ model_name,
+ use_model_replacements,
+ low_cpu_mem_usage,
+ )
+ except Exception as exc:
+ raise RuntimeError(
+ f"_score_scheme_worker[{index}]: failed to load model {model_name!r}\n{_tb.format_exc()}"
+ ) from exc
safe_to_cpu_(model)
block_names = _get_block_names(model, quant_vision=force_mllm)[0]
+ if disk_index is not None:
+ from auto_round.utils.disk_stream_util import materialize_non_block_params
+
+ materialize_non_block_params(model, block_names, disk_index, device="cpu")
for block_name in block_names:
block = _get_module(model, block_name)
block.in_block = True
@@ -1833,7 +1982,8 @@ def _score_scheme_worker(args):
from auto_round.modeling.fused_moe.replace_modules import materialize_model_
- materialize_model_(model)
+ if disk_index is None:
+ materialize_model_(model)
# MoE materialization can replace fused expert modules with newly-created
# Linear layers. Assign tuning metadata only after the final module tree exists.
for layer_name in quant_layer_names:
@@ -1888,6 +2038,7 @@ def _score_scheme_worker(args):
force_mllm=force_mllm,
model_name=model_name,
scheme_tag=f"{index + 1}/{total_schemes} {_short_name(scheme)}",
+ disk_index=disk_index,
)
return index, scores, _get_worker_memory_report(worker_device)
@@ -2172,6 +2323,16 @@ def _gen_layer_config(
pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")
scored_layer_names = set(quant_layer_names + embedding_layers_names)
+ cache_total_params = sum(
+ (
+ module.weight.numel()
+ if hasattr(module, "weight") and module.weight is not None
+ else getattr(module, "_cached_weight_numel", 0)
+ )
+ for name, module in model.named_modules()
+ if name in scored_layer_names
+ )
+ scheme_cache_configs = []
def _group_per_op_scores(index, per_op_scores):
"""Apply the current shared-layer grouping without mutating cached per-op scores."""
@@ -2217,21 +2378,15 @@ def _gen_layer_config(
scheme_dict=scheme_dict,
layer_scores={name: list(score) for name, score in per_op_scores.items()},
total_loss_for_scheme=sum(score[1] for score in per_op_scores.values()),
- total_params=sum(
- (
- module.weight.numel()
- if hasattr(module, "weight") and module.weight is not None
- else getattr(module, "_cached_weight_numel", 0)
- )
- for name, module in model.named_modules()
- if name in scored_layer_names
- ),
+ total_params=cache_total_params,
+ cache_config=scheme_cache_configs[index],
)
scheme_cache_meta = []
for index, scheme in enumerate(schemes):
if check_bf16_scheme(scheme) or _model_id_for_cache is None:
scheme_cache_meta.append((None, None, None))
+ scheme_cache_configs.append(None)
if check_bf16_scheme(scheme):
logger.info(
"AutoScheme: scheme %d/%d (%s) is a BF16 baseline; skipping scoring and cache lookup.",
@@ -2240,7 +2395,7 @@ def _gen_layer_config(
_scheme_short_name(scheme),
)
continue
- cache_key = _autoscheme_cache_key(
+ cache_config = _autoscheme_cache_config(
model_name=_model_id_for_cache,
dataset=dataset,
nsamples=nsamples,
@@ -2251,19 +2406,19 @@ def _gen_layer_config(
scheme=scheme,
force_mllm=force_mllm,
low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,
+ need_weight_grad=need_weight_grad,
)
+ cache_key = hashlib.sha256(json.dumps(cache_config, sort_keys=True, default=str).encode()).hexdigest()[:16]
cache_path = _autoscheme_cache_path(cache_key, index)
- cached_data = _load_autoscheme_scores(cache_path) if os.path.exists(cache_path) else None
- if cached_data is not None and not _is_per_op_cache_compatible(
- cached_data, quant_layer_names, fixed_layer_scheme
- ):
- logger.warning(
- "AutoScheme: cache %s does not contain complete per-op scores; rescoring scheme %d",
- cache_path,
- index,
- )
- cached_data = None
+ cached_data = _find_compatible_autoscheme_cache(
+ cache_path,
+ cache_config,
+ quant_layer_names,
+ fixed_layer_scheme,
+ cache_total_params,
+ )
scheme_cache_meta.append((cache_key, cache_path, cached_data))
+ scheme_cache_configs.append(cache_config)
uncached_indices = [
index
@@ -2273,23 +2428,29 @@ def _gen_layer_config(
worker_device_pool = [device for device in device_list if str(device).startswith("cuda:")]
num_gpus = len(worker_device_pool)
parallel_enabled = _envs.AR_ENABLE_AUTO_SCHEME_PARALLEL
- can_parallel = (
- parallel_enabled
- and _model_id_for_cache is not None
- and num_gpus >= 1
- and len(uncached_indices) >= 2
- and not need_imatrix
- # Each parallel worker fully loads its own copy of the model
- # (_load_scheme_worker_model) in a separate process -- incompatible
- # with a meta-device streaming skeleton, whose entire point is to
- # avoid ever materializing a full copy. Force serial scoring
- # (which honors disk_index via materialize_module/free_module)
- # instead when streaming is active.
- and disk_index is None
+ worker_disk_stream_model = _prefer_disk_stream_scheme_worker(
+ _model_id_for_cache, is_vlm, auto_scheme.low_gpu_mem_usage
+ )
+ # Vision scoring requires a full-model backward and therefore cannot use
+ # the block-wise materialize/free path used by disk streaming.
+ can_parallel = _can_parallel_scheme_scoring(
+ parallel_enabled,
+ _model_id_for_cache,
+ num_gpus,
+ len(uncached_indices),
+ need_imatrix,
+ worker_disk_stream_model,
+ is_vlm,
+ )
+ logger.info(
+ "AutoScheme scoring mode: parallel_configured=%s, parallel_enabled=%s, disk_stream_enabled=%s",
+ parallel_enabled,
+ can_parallel,
+ worker_disk_stream_model and can_parallel,
)
if not parallel_enabled and len(uncached_indices) >= 2:
logger.info(
- "AutoScheme: parallel scoring is disabled; set AR_ENABLE_AUTO_SCHEME_PARALLEL=1 to enable it. "
+ "AutoScheme: parallel scoring was disabled by AR_ENABLE_AUTO_SCHEME_PARALLEL=0; "
"scoring %d uncached non-BF16 schemes serially.",
len(uncached_indices),
)
@@ -2344,6 +2505,7 @@ def _gen_layer_config(
worker_devices[slot],
len(schemes),
progress_queue,
+ worker_disk_stream_model,
)
for slot, index in enumerate(uncached_indices)
]
@@ -2380,11 +2542,12 @@ def _gen_layer_config(
bits, _ = compute_layer_bits(get_module(model, name), auto_scheme.ignore_scale_zp_bits)
per_op_scores[name] = [bits, 0.0]
elif cached_data is not None:
+ loaded_cache_path = cached_data.get("_cache_path", cache_path)
logger.info(
"AutoScheme: loading per-scheme cache for scheme %d from %s."
" Delete this file to disable reuse and rescore.",
index,
- cache_path,
+ loaded_cache_path,
)
per_op_scores = _refresh_cached_layer_bits(
model,
@@ -2437,11 +2600,12 @@ def _gen_layer_config(
cache_key, cache_path, cached_data = scheme_cache_meta[index]
if cached_data is not None:
+ loaded_cache_path = cached_data.get("_cache_path", cache_path)
logger.info(
"AutoScheme: loading per-scheme cache for scheme %d from %s."
" Delete this file to disable reuse and rescore.",
index,
- cache_path,
+ loaded_cache_path,
)
per_op_scores = _refresh_cached_layer_bits(
model,
@@ -2604,15 +2768,14 @@ def _gen_layer_config(
if not candidates:
candidates = list(sorted_indices)
else:
- # Not shared lm_head: prefer options with bits < floor(target_bits)
+ # Not shared lm_head: prefer the nearest available bit width at or
+ # above floor(target_bits), then choose the lowest-loss option at
+ # that width.
floor_bits = math.floor(target_bits)
candidates = [idx for idx in sorted_indices if _get_scheme_bits(schemes[idx]) == floor_bits]
if not candidates:
- # find the first bits that greater than floor bits
- embedding_bits = [bits for idx in sorted_indices if _get_scheme_bits(schemes[idx]) > floor_bits]
- if len(embedding_bits) > 0:
- sorted(embedding_bits)
- embedding_bits = embedding_bits[0]
+ embedding_bits = _get_next_scheme_bits(schemes, sorted_indices, floor_bits)
+ if embedding_bits is not None:
candidates = [idx for idx in sorted_indices if _get_scheme_bits(schemes[idx]) == embedding_bits]
candidates.extend(sorted_indices) # to make sure if the above candidate exceed the budget
@@ -2720,8 +2883,6 @@ def _gen_layer_config(
for n, m in model.named_parameters():
if hasattr(m, "grad"):
m.grad = None
- global last_grad_input
- last_grad_input = None
clear_memory(device_list=device_list)
# # Log AutoScheme memory usage
diff --git a/auto_round/envs.py b/auto_round/envs.py
index 17444e71c..b72b14ae3 100644
--- a/auto_round/envs.py
+++ b/auto_round/envs.py
@@ -26,7 +26,8 @@ if TYPE_CHECKING:
LLAMA_CPP_ROOT: Optional[str] = None
AR_AUTO_SCHEME_NSAMPLES: Optional[int] = None
AR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None
- AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = False
+ AR_AUTO_SCHEME_CACHE: Optional[str] = None
+ AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = True
AR_DISK_STREAM_MODEL: bool = False
AR_RESUME_DIR: Optional[str] = None
@@ -93,9 +94,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
# when ``AutoScheme.batch_size`` is not explicitly set.
# When unset, AutoScheme uses its built-in heuristic (8 for low GPU memory mode, 1 for normal mode).
"AR_AUTO_SCHEME_BATCH_SIZE": lambda: _get_optional_positive_int_env("AR_AUTO_SCHEME_BATCH_SIZE"),
- # Enables AutoScheme to score schemes in parallel. Disabled by default to
- # avoid multiple model-loading workers exhausting host RAM or device memory.
- "AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "0").lower()
+ # Stores persistent AutoScheme scoring results independently from AR_WORK_SPACE,
+ # whose contents are temporary working data and may be cleaned after a run.
+ "AR_AUTO_SCHEME_CACHE": lambda: os.getenv("AR_AUTO_SCHEME_CACHE", None),
+ # Enables AutoScheme to score schemes in parallel. Enabled by default;
+ # set it to 0 when workers could exhaust host RAM or device memory.
+ "AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1").lower()
in ("1", "true", "yes"),
# When set, the model is built as a meta-device skeleton and streamed
# block-by-block from disk during quantization instead of being fully
diff --git a/docs/environments.md b/docs/environments.md
index 5934cef78..a61492314 100644
--- a/docs/environments.md
+++ b/docs/environments.md
@@ -141,18 +141,28 @@ export AR_AUTO_SCHEME_NSAMPLES=1 # set 1 for quick execution
export AR_AUTO_SCHEME_BATCH_SIZE=1
```
+### AR_AUTO_SCHEME_CACHE
+- **Description**: Stores persistent per-scheme AutoScheme scoring JSON files. This directory is independent of `AR_WORK_SPACE`, which is reserved for temporary working data.
+- **Default**: `~/.cache/auto_round`
+- **Valid Values**: any writable directory path
+- **Usage**: Set this to place reusable AutoScheme scores in a different cache directory
+
+```bash
+export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cache
+```
+
### AR_ENABLE_AUTO_SCHEME_PARALLEL
-- **Description**: Enables multiprocessing across AutoScheme candidates. Keep it disabled when parallel model-loading workers could exhaust host RAM or device memory.
-- **Default**: `False` (schemes are scored serially)
-- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) enable parallel scoring; any other value keeps parallel scoring disabled
-- **Usage**: Set this before running AutoScheme to enable parallel candidate scoring
+- **Description**: Enables multiprocessing across AutoScheme candidates. It can be combined with `AR_DISK_STREAM_MODEL=1`; each worker then builds its own meta-model skeleton and streams blocks independently. Disable it when concurrent workers could exhaust host RAM or device memory.
+- **Default**: `"1"` (schemes are scored in parallel when multiprocessing requirements are met)
+- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) enable parallel scoring; any other value disables parallel scoring
+- **Usage**: Set this to `0` before running AutoScheme to force serial candidate scoring
```bash
-export AR_ENABLE_AUTO_SCHEME_PARALLEL=1
+export AR_ENABLE_AUTO_SCHEME_PARALLEL=0
```
### AR_DISK_STREAM_MODEL
-- **Description**: When enabled, `AutoRound(model=<path>, ...)` builds the model as a meta-device skeleton instead of fully materializing the checkpoint on CPU RAM up front, and streams each decoder block's real weights from the checkpoint's safetensors shards on demand -- materializing right before a block is used (calibration, tuning, or `AutoScheme` sensitivity scoring) and freeing it back to meta right after. This keeps peak CPU RAM roughly flat regardless of checkpoint size, instead of proportional to it. Non-block parameters (embeddings, `lm_head`, final norm) are still loaded up front, since they are typically small.
+- **Description**: When enabled, `AutoRound(model=<path>, ...)` builds the model as a meta-device skeleton instead of fully materializing the checkpoint on CPU RAM up front, and streams each decoder block's real weights from the checkpoint's safetensors shards on demand -- materializing right before a block is used (calibration, tuning, or `AutoScheme` sensitivity scoring) and freeing it back to meta right after. This keeps peak CPU RAM roughly flat regardless of checkpoint size, instead of proportional to it. Non-block parameters (embeddings, `lm_head`, final norm) are still loaded up front, since they are typically small. Text-model AutoScheme scoring also supports combining this with parallel scoring, which is enabled by default; each worker streams its own block copy.
- **Default**: `False`
- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) for enabling; any other value for disabling
- **Usage**: Enable this to quantize checkpoints larger than available CPU RAM + GPU VRAM combined. Only applies when `model` is a string (local directory) path; has no effect on already-loaded model objects.
diff --git a/docs/environments_CN.md b/docs/environments_CN.md
index 9dd02d251..0272dbd1f 100644
--- a/docs/environments_CN.md
+++ b/docs/environments_CN.md
@@ -141,18 +141,28 @@ export AR_AUTO_SCHEME_NSAMPLES=1
export AR_AUTO_SCHEME_BATCH_SIZE=1
```
+### AR_AUTO_SCHEME_CACHE
+- **描述**:存放可持久复用的 AutoScheme 单方案评分 JSON 文件。该目录独立于用于临时工作数据的 `AR_WORK_SPACE`。
+- **默认值**:`~/.cache/auto_round`
+- **有效值**:任意可写目录路径
+- **用途**:将可复用的 AutoScheme 评分结果保存到其他缓存目录
+
+```bash
+export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cache
+```
+
### AR_ENABLE_AUTO_SCHEME_PARALLEL
-- **描述**:启用 AutoScheme 候选方案之间的多进程并行。当并行加载多个模型可能耗尽主机内存或显存时,请保持关闭。
-- **默认值**:`False`(串行评分各方案)
-- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用并行评分;其他值保持关闭
-- **用途**:运行 AutoScheme 前设置,以启用候选方案并行评分
+- **描述**:启用 AutoScheme 候选方案之间的多进程并行。可与 `AR_DISK_STREAM_MODEL=1` 同时使用;此时每个 worker 会构建独立的 meta 模型骨架并分别流式加载 block。当并发 worker 可能耗尽主机内存或显存时,请将其关闭。
+- **默认值**:`"1"`(满足多进程要求时并行评分各方案)
+- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用并行评分;其他值表示关闭
+- **用途**:运行 AutoScheme 前将其设为 `0`,以强制串行评分候选方案
```bash
-export AR_ENABLE_AUTO_SCHEME_PARALLEL=1
+export AR_ENABLE_AUTO_SCHEME_PARALLEL=0
```
### AR_DISK_STREAM_MODEL
-- **描述**:启用后,`AutoRound(model=<path>, ...)` 会将模型构建为 meta 设备骨架,而不是先把整个 checkpoint 完全加载到 CPU 内存;随后按需从 checkpoint 的 safetensors 分片中流式加载每个解码器块的真实权重——在该块被使用前(校准、调优或 `AutoScheme` 敏感度评分)才实体化,用完后立即释放回 meta。这样峰值 CPU 内存基本保持平稳,而不会随 checkpoint 大小成比例增长。非块参数(embedding、`lm_head`、最终归一化层)体积通常较小,仍会一次性加载。
+- **描述**:启用后,`AutoRound(model=<path>, ...)` 会将模型构建为 meta 设备骨架,而不是先把整个 checkpoint 完全加载到 CPU 内存;随后按需从 checkpoint 的 safetensors 分片中流式加载每个解码器块的真实权重——在该块被使用前(校准、调优或 `AutoScheme` 敏感度评分)才实体化,用完后立即释放回 meta。这样峰值 CPU 内存基本保持平稳,而不会随 checkpoint 大小成比例增长。非块参数(embedding、`lm_head`、最终归一化层)体积通常较小,仍会一次性加载。文本模型的 AutoScheme 评分也支持与默认启用的并行评分组合使用;每个 worker 会流式加载自己的 block 副本。
- **默认值**:`False`
- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用;其他任何值表示禁用
- **用途**:用于量化体积超过可用 CPU 内存 + GPU 显存总和的 checkpoint。仅在 `model` 为字符串(本地目录)路径时生效,对已加载的模型对象无效。
diff --git a/test/test_cpu/schemes/test_auto_scheme.py b/test/test_cpu/schemes/test_auto_scheme.py
index d6a2ce168..d71ac0e01 100644
--- a/test/test_cpu/schemes/test_auto_scheme.py
+++ b/test/test_cpu/schemes/test_auto_scheme.py
@@ -60,13 +60,22 @@ def test_env_ar_auto_scheme_batch_size_zero_raises(monkeypatch):
_ = envs.AR_AUTO_SCHEME_BATCH_SIZE
+def test_env_ar_auto_scheme_cache(monkeypatch, tmp_path):
+ """AR_AUTO_SCHEME_CACHE should expose an independent cache directory."""
+ import auto_round.envs as envs
+
+ cache_dir = str(tmp_path / "auto_scheme_cache")
+ monkeypatch.setenv("AR_AUTO_SCHEME_CACHE", cache_dir)
+ assert envs.AR_AUTO_SCHEME_CACHE == cache_dir
+
+
def test_env_ar_enable_auto_scheme_parallel(monkeypatch):
import auto_round.envs as envs
monkeypatch.delenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", raising=False)
- assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is False
- monkeypatch.setenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1")
assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is True
+ monkeypatch.setenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "0")
+ assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is False
def test_build_layer_config_header_rows_merges_adjacent_prefixes():
@@ -135,6 +144,60 @@ def test_choose_bits_per_layer_reconstructs_optimal_path():
assert path == [(["layer.0"], 1), (["layer.1"], 0)]
+def test_activation_scoring_handles_reused_wrapper():
+ """Each forward call should keep its own activation error until backward."""
+ import types
+
+ import torch
+
+ from auto_round.auto_scheme.delta_loss import AutoSchemeWrapperLinear
+
+ wrapper = AutoSchemeWrapperLinear.__new__(AutoSchemeWrapperLinear)
+ torch.nn.Module.__init__(wrapper)
+ wrapper.orig_layer = types.SimpleNamespace(act_bits=8)
+ wrapper.act_qdq_func = lambda x, *_args, **_kwargs: (x * 0.5, 1.0, None)
+ wrapper.grad_mode = True
+ wrapper.act_cnt = 0
+ wrapper.act_score = 0.0
+ wrapper.weight_score = 0.0
+ wrapper.mix_score = 0.0
+ wrapper.max_act_value = 0
+
+ first = torch.tensor([1.0, -2.0], requires_grad=True)
+ second = torch.tensor([3.0, -4.0], requires_grad=True)
+ qdq_first, _, _ = wrapper._qdq_act(first)
+ qdq_second, _, _ = wrapper._qdq_act(second)
+
+ (qdq_first.sum() + qdq_second.sum()).backward()
+
+ assert wrapper.act_cnt == 2
+ assert wrapper.act_score == pytest.approx(5.0)
+ assert wrapper.mix_score == pytest.approx(5.0)
+
+
+def test_prepare_replay_input_supports_keyword_hidden_states():
+ import torch
+
+ from auto_round.auto_scheme.delta_loss import _prepare_replay_input
+
+ hidden_states = torch.randn(2, 4)
+ attention_mask = torch.ones(2, 4, dtype=torch.int64)
+
+ replay_input = _prepare_replay_input([], {"attention_mask": attention_mask, "hidden_states": hidden_states}, "0")
+
+ assert replay_input is hidden_states
+ assert replay_input.requires_grad
+
+
+def test_prepare_replay_input_rejects_missing_floating_tensor():
+ import torch
+
+ from auto_round.auto_scheme.delta_loss import _prepare_replay_input
+
+ with pytest.raises(RuntimeError, match="No floating replay input found for block 0"):
+ _prepare_replay_input([], {"attention_mask": torch.ones(2, dtype=torch.int64)}, "0")
+
+
def test_build_expert_groups_groups_experts_per_block():
"""Expert layers in the same block should be grouped together."""
import torch
@@ -350,8 +413,8 @@ class TestAutoScheme:
from auto_round.auto_scheme.delta_loss import _load_autoscheme_scores
- cache_dir = str(tmp_path / "ar_work_space")
- monkeypatch.setenv("AR_WORK_SPACE", cache_dir)
+ cache_dir = str(tmp_path / "auto_scheme_cache")
+ monkeypatch.setenv("AR_AUTO_SCHEME_CACHE", cache_dir)
scheme = AutoScheme(
avg_bits=3,
@@ -363,7 +426,7 @@ class TestAutoScheme:
_, layer_config = ar.quantize()
# Cache files must exist — one per scheme (2 schemes here)
- cache_files = glob.glob(f"{cache_dir}/auto_scheme_cache/scheme_*.json")
+ cache_files = glob.glob(f"{cache_dir}/scheme_*.json")
assert (
len(cache_files) == 2
), f"Expected 2 cache files (one per scheme), found {len(cache_files)}: {cache_files}"
@@ -371,7 +434,7 @@ class TestAutoScheme:
for path in cache_files:
data = _load_autoscheme_scores(path)
assert data is not None, f"Cache file {path} could not be loaded"
- assert data["version"] == 3
+ assert data["version"] == 1
assert data["score_granularity"] == "per_op"
assert "layer_scores" in data
assert "total_loss_for_scheme" in data
@@ -528,6 +591,49 @@ def test_autoscheme_cache_key_insensitive_to_layer_order():
assert key1 == key2 # Should match after internal sorting
+def test_autoscheme_cache_key_is_portable_across_model_paths():
+ """A locally downloaded model should keep the same key when its parent directory changes."""
+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key
+
+ kwargs = {
+ "dataset": "pile-10k",
+ "nsamples": 16,
+ "seqlen": 256,
+ "batch_size": 8,
+ "quant_layer_names": ["layer.0"],
+ "fixed_layer_scheme": {},
+ "scheme": "W4A16",
+ "force_mllm": False,
+ "low_gpu_mem_usage": True,
+ }
+
+ assert _autoscheme_cache_key(model_name="/models/org/test-model", **kwargs) == _autoscheme_cache_key(
+ model_name="/tmp/downloads/test-model", **kwargs
+ )
+
+
+def test_autoscheme_cache_key_normalizes_preset_scheme():
+ """A preset name and its resolved scheme should identify the same scoring run."""
+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key
+ from auto_round.schemes import preset_name_to_scheme
+
+ kwargs = {
+ "model_name": "test-model",
+ "dataset": "pile-10k",
+ "nsamples": 16,
+ "seqlen": 256,
+ "batch_size": 8,
+ "quant_layer_names": ["layer.0"],
+ "fixed_layer_scheme": {},
+ "force_mllm": False,
+ "low_gpu_mem_usage": True,
+ }
+
+ assert _autoscheme_cache_key(scheme="W4A16", **kwargs) == _autoscheme_cache_key(
+ scheme=preset_name_to_scheme("W4A16"), **kwargs
+ )
+
+
def test_autoscheme_cache_key_changes_only_with_scoring_config():
"""Execution settings that affect scoring invalidate the cache; bit accounting does not."""
from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key
@@ -547,6 +653,17 @@ def test_autoscheme_cache_key_changes_only_with_scoring_config():
baseline = _autoscheme_cache_key(**kwargs)
assert baseline != _autoscheme_cache_key(**{**kwargs, "low_gpu_mem_usage": False})
+ assert baseline != _autoscheme_cache_key(**{**kwargs, "need_weight_grad": True})
+
+
+def test_get_next_scheme_bits_is_order_independent():
+ from auto_round.auto_scheme.delta_loss import _get_next_scheme_bits
+
+ schemes = [{"bits": 8}, {"bits": 4}, {"bits": 6}]
+
+ assert _get_next_scheme_bits(schemes, [0, 1, 2], 5) == 6
+ assert _get_next_scheme_bits(schemes, [2, 1, 0], 5) == 6
+ assert _get_next_scheme_bits(schemes, [0, 1, 2], 8) is None
def test_refresh_cached_layer_bits_preserves_loss():
@@ -585,6 +702,7 @@ def test_autoscheme_cache_save_and_load(tmp_path):
}
total_loss = 2.1
total_params = 1000000
+ cache_config = {"model_id": "test-model", "scheme": scheme_dict}
_save_autoscheme_scores(
cache_path,
@@ -594,6 +712,7 @@ def test_autoscheme_cache_save_and_load(tmp_path):
layer_scores,
total_loss,
total_params,
+ cache_config,
)
loaded = _load_autoscheme_scores(cache_path)
@@ -603,6 +722,89 @@ def test_autoscheme_cache_save_and_load(tmp_path):
assert loaded["total_params"] == total_params
+def test_find_compatible_downloaded_autoscheme_cache(tmp_path):
+ """A compatible downloaded cache should work without renaming it to the locally computed key."""
+ from auto_round.auto_scheme.delta_loss import (
+ _autoscheme_cache_config,
+ _find_compatible_autoscheme_cache,
+ _save_autoscheme_scores,
+ )
+ from auto_round.schemes import preset_name_to_scheme
+
+ quant_layer_names = ["layer.0", "layer.1"]
+ cache_config = _autoscheme_cache_config(
+ model_name="/models/test-model",
+ dataset="pile-10k",
+ nsamples=16,
+ seqlen=256,
+ batch_size=8,
+ quant_layer_names=quant_layer_names,
+ fixed_layer_scheme={},
+ scheme="W4A16",
+ force_mllm=False,
+ low_gpu_mem_usage=True,
+ )
+ downloaded_path = tmp_path / "downloaded-from-release.json"
+ layer_scores = {"layer.0": [4, 1.2], "layer.1": [4, 0.9]}
+ _save_autoscheme_scores(
+ downloaded_path,
+ "key-from-another-commit",
+ 0,
+ preset_name_to_scheme("W4A16").to_dict(),
+ layer_scores,
+ 2.1,
+ 1000,
+ cache_config=cache_config,
+ )
+
+ loaded = _find_compatible_autoscheme_cache(
+ str(tmp_path / "scheme_00_current-key.json"),
+ cache_config,
+ quant_layer_names,
+ {},
+ 1000,
+ )
+
+ assert loaded is not None
+ assert loaded["layer_scores"] == layer_scores
+ assert loaded["_cache_path"] == str(downloaded_path)
+ assert (
+ _find_compatible_autoscheme_cache(
+ str(tmp_path / "scheme_00_current-key.json"),
+ cache_config,
+ quant_layer_names,
+ {},
+ 999,
+ )
+ is None
+ )
+
+
+def test_autoscheme_cache_path_is_independent_from_workspace(tmp_path, monkeypatch):
+ """AutoScheme caches default to the user cache and ignore AR_WORK_SPACE."""
+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_path
+
+ monkeypatch.setenv("HOME", str(tmp_path / "home"))
+ monkeypatch.setenv("AR_WORK_SPACE", str(tmp_path / "workspace"))
+ monkeypatch.delenv("AR_AUTO_SCHEME_CACHE", raising=False)
+
+ cache_path = _autoscheme_cache_path("cache_key", 1)
+
+ assert cache_path == str(tmp_path / "home" / ".cache" / "auto_round" / "scheme_01_cache_key.json")
+
+
+def test_autoscheme_cache_path_can_be_overridden(tmp_path, monkeypatch):
+ """AR_AUTO_SCHEME_CACHE overrides the default user cache directory."""
+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_path
+
+ cache_dir = tmp_path / "custom_cache"
+ monkeypatch.setenv("AR_AUTO_SCHEME_CACHE", str(cache_dir))
+
+ cache_path = _autoscheme_cache_path("cache_key", 2)
+
+ assert cache_path == str(cache_dir / "scheme_02_cache_key.json")
+
+
def test_parallel_progress_events_are_applied_in_parent():
"""Worker progress events should update only the parent-owned progress bar."""
import queue
@@ -674,6 +876,33 @@ def test_scheme_worker_count_rejects_no_gpu():
_get_scheme_worker_count(2, 0)
+def test_parallel_scheme_scoring_supports_disk_stream_model():
+ from auto_round.auto_scheme.delta_loss import _can_parallel_scheme_scoring
+
+ assert _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, False)
+
+
+def test_parallel_scheme_scoring_rejects_disk_stream_vlm():
+ from auto_round.auto_scheme.delta_loss import _can_parallel_scheme_scoring
+
+ assert not _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, True)
+
+
+ "model_id,is_vlm,low_gpu_mem_usage,expected",
+ [
+ ("local-model", False, True, True),
+ ("local-model", True, True, False),
+ ("local-model", False, False, False),
+ (None, False, True, False),
+ ],
+)
+def test_prefer_disk_stream_scheme_worker(model_id, is_vlm, low_gpu_mem_usage, expected):
+ from auto_round.auto_scheme.delta_loss import _prefer_disk_stream_scheme_worker
+
+ assert _prefer_disk_stream_scheme_worker(model_id, is_vlm, low_gpu_mem_usage) is expected
+
+
def test_opt_scheme_worker_uses_low_cpu_memory_loading(monkeypatch):
from test.helpers import opt_name_or_path
@@ -700,6 +929,50 @@ def test_opt_scheme_worker_uses_low_cpu_memory_loading(monkeypatch):
}
+def test_disk_stream_scheme_worker_builds_meta_model(monkeypatch):
+ from auto_round.auto_scheme import delta_loss
+ from auto_round.utils import disk_stream_util
+
+ expected = (object(), object(), object())
+ monkeypatch.setattr(disk_stream_util, "build_meta_model", lambda model_name: expected)
+
+ assert delta_loss._load_disk_stream_scheme_worker_model("local-model") == expected
+
+
+def test_disk_stream_scheme_worker_applies_model_replacements(monkeypatch):
+ from auto_round import special_model_handler
+ from auto_round.auto_scheme import delta_loss
+ from auto_round.utils import disk_stream_util
+
+ model = object()
+ updated_model = object()
+ handled_model = object()
+ tokenizer = object()
+ disk_index = object()
+ calls = []
+
+ monkeypatch.setattr(disk_stream_util, "build_meta_model", lambda model_name: (model, tokenizer, disk_index))
+
+ def fake_update_module(input_model, formats, cleanup_original):
+ calls.append(("update", input_model, formats, cleanup_original))
+ return updated_model
+
+ def fake_handle_special_model(input_model):
+ calls.append(("handle", input_model))
+ return handled_model
+
+ monkeypatch.setattr(special_model_handler, "update_module", fake_update_module)
+ monkeypatch.setattr(special_model_handler, "_handle_special_model", fake_handle_special_model)
+
+ result = delta_loss._load_disk_stream_scheme_worker_model("local-model", use_model_replacements=True)
+
+ assert result == (handled_model, tokenizer, disk_index)
+ assert calls == [
+ ("update", model, None, False),
+ ("handle", updated_model),
+ ]
+
+
def test_per_op_cache_compatibility_rejects_grouped_scores():
from auto_round.auto_scheme.delta_loss import _is_per_op_cache_compatible
@@ -718,7 +991,7 @@ def test_per_op_cache_compatibility_rejects_grouped_scores():
)
-def test_version_two_cache_is_rejected(tmp_path):
+def test_non_version_one_cache_is_rejected(tmp_path):
import json
from auto_round.auto_scheme.delta_loss import _load_autoscheme_scores
@@ -727,7 +1000,7 @@ def test_version_two_cache_is_rejected(tmp_path):
cache_path.write_text(
json.dumps(
{
- "version": 2,
+ "version": 3,
"layer_scores": {"layer.0": [4, 1.0]},
"total_loss_for_scheme": 1.0,
"total_params": 4,