PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enable AutoScheme parallel and disk stream by default (#2105)
Xin He8a9a1d138aad
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex 33e39be9b..7a1584937 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -132,15 +132,15 @@ class AutoSchemeWrapperLinear(WrapperLinear):qdq_x, scale, zp = self.act_qdq_func(x, act_min_scale, act_max_scale, act_max)if self.grad_mode:with torch.no_grad():- self.max_act_value = torch.abs(x).max()- if torch.abs(x).max() != 0:+ max_act_value = torch.abs(x).max()+ self.max_act_value = max_act_value+ if max_act_value != 0:self.act_cnt += 1- x_diff = x - qdq_x- self.x_diff = x_diff.to("cpu")+ x_diff = (x - qdq_x).to("cpu")def save_grad(grad):"""Backward hook: accumulate activation score from grad * (x - qdq_x)."""- if self.max_act_value == 0:+ if max_act_value == 0:if torch.abs(grad).max() != 0:raise ValueError"""@@ -149,13 +149,12 @@ class AutoSchemeWrapperLinear(WrapperLinear):def test_multi_card(self):model_name = "/models/Qwen3-8B""""- if torch.isnan(grad).any() or torch.isnan(self.x_diff).any():+ if torch.isnan(grad).any() or torch.isnan(x_diff).any():self.act_cnt -= 1return None- self.act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()+ self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()self.mix_score = self.weight_score + self.act_score- self.x_diff = Nonereturn Noneif qdq_x.requires_grad:@@ -286,15 +285,15 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):qdq_x, scale, zp = self.act_qdq_func(x, act_min_scale, act_max_scale, act_max)if self.grad_mode:with torch.no_grad():- self.max_act_value = torch.abs(x).max()- if torch.abs(x).max() != 0:+ max_act_value = torch.abs(x).max()+ self.max_act_value = max_act_value+ if max_act_value != 0:self.act_cnt += 1- x_diff = x - qdq_x- self.x_diff = x_diff.to("cpu")+ x_diff = (x - qdq_x).to("cpu")def save_grad(grad):"""Backward hook: accumulate activation score from grad * (x - qdq_x)."""- if self.max_act_value == 0:+ if max_act_value == 0:if torch.abs(grad).max() != 0:raise ValueError"""@@ -303,13 +302,12 @@ class AutoSchemeWrapperLinearIMatrix(WrapperLinear):def test_multi_card(self):model_name = "/models/Qwen3-8B""""- if torch.isnan(grad).any() or torch.isnan(self.x_diff).any():+ if torch.isnan(grad).any() or torch.isnan(x_diff).any():self.act_cnt -= 1return None- self.act_score += torch.abs((grad * self.x_diff.to(grad.device))).sum().item()+ self.act_score += torch.abs((grad * x_diff.to(grad.device))).sum().item()self.mix_score = self.weight_score + self.act_score- self.x_diff = Nonereturn Noneif qdq_x.requires_grad:@@ -560,9 +558,6 @@ class MyCustomError(Exception):super().__init__(message)-last_grad_input = None--def prepare_model_low_gpu(model, block_inputs: dict = None, pbar=None, major_device="cpu", disk_index=None):"""Wrap every block's forward so that, for one calibration batch, it (1) moves itself to``major_device`` on demand, (2) records its own inputs into ``block_inputs`` (on CPU) so@@ -700,6 +695,20 @@ def model_forward(model, data, **forward_kwargs):return model(**prepared, **forward_kwargs), prepared+def _prepare_replay_input(block_input_args, block_input_kwargs, block_name):+ """Find the floating hidden-state tensor whose gradient feeds the preceding block."""+ candidates = []+ if "hidden_states" in block_input_kwargs:+ candidates.append(block_input_kwargs["hidden_states"])+ candidates.extend(block_input_args)+ candidates.extend(value for key, value in block_input_kwargs.items() if key != "hidden_states")+ for value in candidates:+ if isinstance(value, torch.Tensor) and value.is_floating_point():+ value.requires_grad_(True)+ return value+ raise RuntimeError(f"No floating replay input found for block {block_name}")++def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, scheme_tag=None, disk_index=None):"""Run one full scoring pass (all calibration batches) in low-GPU-memory mode.@@ -722,130 +731,126 @@ def model_forward_low_gpu(model, dataloader, major_device="cuda", pbar=None, schmodule = get_module(model, name)module.orig_forward = module.forward+ captured_grad = None+def backward_pre_hook(module, grad_input):"""Hook executed before backward propagation."""- global last_grad_input- last_grad_input = grad_input+ nonlocal captured_grad+ captured_grad = grad_inputget_current_device_manager().synchronize()raise MyCustomError("Interrupt backward pass")for batch_idx, data in enumerate(dataloader, start=1):- prepare_model_low_gpu(model, block_inputs, major_device=major_device, pbar=pbar, disk_index=disk_index)-- # lm_head sits outside every decoder block, so it never gets `grad_mode=True`- # in the manual block-by-block backward below. Scope the fix narrowly to- # just lm_head (rather than every non-block module) to avoid enabling grad- # tracking / scoring hooks on unrelated out-of-block layers, which would- # add extra autograd-graph memory for no benefit. The backward flow is:- # loss → lm_head (hook fires here) → norm → last_block (hook raises error)- head_name = get_lm_head_name(model)- if head_name is not None:- # Once lm_head has been wrapped for scoring, `get_lm_head_name` resolves- # to the inner original Linear (e.g. "lm_head.orig_layer") rather than- # the wrapper itself ("lm_head") -- strip the suffix to reach the wrapper.- head_name = head_name.removesuffix(".orig_layer")- head_module = get_module(model, head_name)- if hasattr(head_module, "grad_mode"):- head_module.grad_mode = True-- # Register backward hook on the last block- last_block = get_module(model, block_names[-1])- last_block_backward_hook = last_block.register_full_backward_pre_hook(backward_pre_hook)-- data = to_device(data, model.device)- # VLM datasets often already include ``labels``; LLM ones don't. Strip- # any pre-existing ``labels`` from kwargs so we don't pass it twice.- labels = data["labels"] if isinstance(data, dict) and "labels" in data else data["input_ids"]- if isinstance(data, dict):- data_for_forward = {k: v for k, v in data.items() if k != "labels"}- else:- data_for_forward = data- # Route through the unified mllm forward so ``pixel_values`` /- # ``images`` get cast to ``model.dtype`` (otherwise the vision tower- # is silently bypassed on dtype mismatch and vision grad stays 0).- output, _prepared = model_forward(model, data_for_forward, labels=labels, use_cache=False)- clear_memory(device_list=major_device)- memory_monitor.log_summary()-+ captured_grad = None+ interrupted = False+ last_block_backward_hook = Nonetry:- # Backward pass (will be interrupted by the hook)- output.loss.to(torch.float32).backward()- except MyCustomError:- pass+ prepare_model_low_gpu(model, block_inputs, major_device=major_device, pbar=pbar, disk_index=disk_index)++ # lm_head sits outside every decoder block, so it never gets `grad_mode=True`+ # in the manual block-by-block backward below. Scope the fix narrowly to+ # just lm_head (rather than every non-block module) to avoid enabling grad+ # tracking / scoring hooks on unrelated out-of-block layers, which would+ # add extra autograd-graph memory for no benefit. The backward flow is:+ # loss → lm_head (hook fires here) → norm → last_block (hook raises error)+ head_name = get_lm_head_name(model)+ if head_name is not None:+ # Once lm_head has been wrapped for scoring, `get_lm_head_name` resolves+ # to the inner original Linear (e.g. "lm_head.orig_layer") rather than+ # the wrapper itself ("lm_head") -- strip the suffix to reach the wrapper.+ head_name = head_name.removesuffix(".orig_layer")+ head_module = get_module(model, head_name)+ if hasattr(head_module, "grad_mode"):+ head_module.grad_mode = True++ last_block = get_module(model, block_names[-1])+ last_block_backward_hook = last_block.register_full_backward_pre_hook(backward_pre_hook)++ data = to_device(data, model.device)+ # VLM datasets often already include ``labels``; LLM ones don't. Strip+ # any pre-existing ``labels`` from kwargs so we don't pass it twice.+ labels = data["labels"] if isinstance(data, dict) and "labels" in data else data["input_ids"]+ if isinstance(data, dict):+ data_for_forward = {k: v for k, v in data.items() if k != "labels"}+ else:+ data_for_forward = data+ # Route through the unified mllm forward so ``pixel_values`` /+ # ``images`` get cast to ``model.dtype`` (otherwise the vision tower+ # is silently bypassed on dtype mismatch and vision grad stays 0).+ output, _prepared = model_forward(model, data_for_forward, labels=labels, use_cache=False)+ clear_memory(device_list=major_device)+ memory_monitor.log_summary()++ try:+ output.loss.to(torch.float32).backward()+ except MyCustomError:+ interrupted = True+ if not interrupted or captured_grad is None:+ raise RuntimeError("AutoScheme failed to capture the last block gradient for replay")+ current_grad = captured_grad+ finally:+ if last_block_backward_hook is not None:+ last_block_backward_hook.remove()+ for name in block_names:+ module = get_module(model, name)+ module.forward = module.orig_forward- current_grad = last_grad_inputdel output, data# Manually compute gradients block by block- last_block_backward_hook.remove()-- for name in block_names:- module = get_module(model, name)- module.forward = module.orig_forward- index = 0for block_name in reversed(block_names):- index += 1# Retrieve stored inputs for the blockblock_input_info = block_inputs.get(block_name, {})block_input_args = to_device(block_input_info.get("args", []), major_device)block_input_kwargs = to_device(block_input_info.get("kwargs", {}), major_device)- block_input_args[0].requires_grad_(True)+ replay_input = _prepare_replay_input(block_input_args, block_input_kwargs, block_name)# Move the block module to GPUblock_module = get_module(model, block_name)for n, m in block_module.named_modules():if hasattr(m, "grad_mode"):m.grad_mode = True- if disk_index is not None:- from auto_round.utils.disk_stream_util import materialize_module-- materialize_module(block_module, block_name, disk_index, device=major_device)- move_module_to_tuning_device(block_module, major_device=major_device)-- # Set the block to eval mode while enabling gradient computation- block_module.eval()-- # Recompute the block output- block_output = block_module(*block_input_args, **block_input_kwargs)+ materialized = False+ try:+ if disk_index is not None:+ from auto_round.utils.disk_stream_util import materialize_module++ materialize_module(block_module, block_name, disk_index, device=major_device)+ materialized = True+ move_module_to_tuning_device(block_module, major_device=major_device)++ block_module.eval()+ block_output = block_module(*block_input_args, **block_input_kwargs)++ if isinstance(block_output, tuple):+ main_output = block_output[0]+ if isinstance(main_output, torch.Tensor) and main_output.is_floating_point():+ main_output = main_output.requires_grad_(True)+ elif isinstance(block_output, torch.Tensor) and block_output.is_floating_point():+ main_output = block_output.requires_grad_(True)+ else:+ main_output = block_output- # Ensure the output requires gradients- if isinstance(block_output, tuple):- # For tuple outputs, we usually care about the first element (hidden states)- main_output = block_output[0]- if isinstance(main_output, torch.Tensor) and main_output.is_floating_point():- main_output = main_output.requires_grad_(True)- elif isinstance(block_output, torch.Tensor) and block_output.is_floating_point():- main_output = block_output.requires_grad_(True)- else:- main_output = block_output-- # Backward pass for the current block- torch.autograd.backward(- tensors=main_output,- # inputs=block_input_args,- grad_tensors=current_grad,- retain_graph=True, # False may lead to zero gradients for some cases (e.g., MXFP4)- )+ torch.autograd.backward(+ tensors=main_output,+ grad_tensors=current_grad,+ retain_graph=True, # False may lead to zero gradients for some cases (e.g., MXFP4)+ )- # Extract gradients w.r.t. the block input- if block_input_args and isinstance(block_input_args[0], torch.Tensor):- if block_input_args[0].grad is not None:- current_grad = block_input_args[0].grad.detach().clone()- else:+ if replay_input.grad is None:logger.warning(f"No gradient found for input of {block_name}, stopping backward replay")break- else:- logger.warning(f"No suitable input gradient found for {block_name}")- break-- del block_output, main_output, block_input_args, block_input_kwargs- if disk_index is not None:- from auto_round.utils.disk_stream_util import free_module+ current_grad = replay_input.grad.detach().clone()+ finally:+ for parameter in block_module.parameters():+ parameter.grad = None+ if disk_index is not None and materialized:+ from auto_round.utils.disk_stream_util import free_module- free_module(block_module)- else:- block_module.to("cpu")+ free_module(block_module)+ elif disk_index is None:+ block_module.to("cpu")# clear_memory(device_list=major_device) # this one is very slow and seems does not affect max ram usagememory_monitor.update()@@ -1422,6 +1427,14 @@ def _get_scheme_bits(scheme):return scheme.get("bits", 16)+def _get_next_scheme_bits(schemes, indices, floor_bits):+ """Return the smallest candidate bit width strictly above ``floor_bits``."""+ higher_bits = {+ _get_scheme_bits(schemes[index]) for index in indices if _get_scheme_bits(schemes[index]) > floor_bits+ }+ return min(higher_bits, default=None)++# Delta loss does not handle lm-head well, it is prone to assign low bit to lm-head which is not optimaldef _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_params_cnt, total_scores):@@ -1516,17 +1529,28 @@ def _apply_head_trick(head_name, schemes, sorted_indices, target_bits, target_padef _scheme_repr(s):- """Normalize a scheme (str/QuantizationScheme/dict) to a hashable repr."""+ """Normalize a scheme to a stable representation independent of preset aliases."""if isinstance(s, str):- return s+ try:+ s = preset_name_to_scheme(s)+ except KeyError:+ return s.upper()if isinstance(s, QuantizationScheme):- return json.dumps(asdict(s), sort_keys=True, default=str)+ s = asdict(s)if isinstance(s, dict):- return json.dumps(s, sort_keys=True, default=str)+ return {key: value for key, value in sorted(s.items()) if value is not None}return str(s)-def _autoscheme_cache_key(+def _stable_model_id(model_name):+ """Return a portable model identifier for local paths and Hub model IDs."""+ if not isinstance(model_name, str):+ return model_name+ normalized = model_name.rstrip("/\\")+ return os.path.basename(normalized) or normalized+++def _autoscheme_cache_config(model_name,dataset,nsamples,@@ -1537,27 +1561,58 @@ def _autoscheme_cache_key(scheme,force_mllm,low_gpu_mem_usage,+ need_weight_grad=False,):- """Return a 16-char hex digest that uniquely identifies a **single-scheme** scoring run.-- The key covers every parameter that directly affects per-layer loss values- **except** ``avg_bits`` / ``target_bits`` (only drive the DP step).- Unlike the old version, this key is generated **per-scheme**, not per-run,- so caching is granular: adding/removing schemes doesn't invalidate cached- scores for unchanged schemes.- """- key_data = {- "model_name": model_name,+ """Build the portable, implementation-independent identity of a scoring run."""+ return {+ "model_id": _stable_model_id(model_name),"dataset": dataset,"nsamples": nsamples,"seqlen": seqlen,"batch_size": batch_size,"quant_layer_names": sorted(quant_layer_names),- "fixed_layer_scheme": {k: v for k, v in sorted(fixed_layer_scheme.items())},+ "fixed_layer_scheme": {key: _scheme_repr(value) for key, value in sorted(fixed_layer_scheme.items())},"scheme": _scheme_repr(scheme),"force_mllm": force_mllm,"low_gpu_mem_usage": low_gpu_mem_usage,+ "need_weight_grad": need_weight_grad,}+++def _autoscheme_cache_key(+ model_name,+ dataset,+ nsamples,+ seqlen,+ batch_size,+ quant_layer_names,+ fixed_layer_scheme,+ scheme,+ force_mllm,+ low_gpu_mem_usage,+ need_weight_grad=False,+):+ """Return a 16-char hex digest that uniquely identifies a **single-scheme** scoring run.++ The key covers every parameter that directly affects per-layer loss values+ **except** ``avg_bits`` / ``target_bits`` (only drive the DP step).+ Unlike the old version, this key is generated **per-scheme**, not per-run,+ so caching is granular: adding/removing schemes doesn't invalidate cached+ scores for unchanged schemes.+ """+ key_data = _autoscheme_cache_config(+ model_name,+ dataset,+ nsamples,+ seqlen,+ batch_size,+ quant_layer_names,+ fixed_layer_scheme,+ scheme,+ force_mllm,+ low_gpu_mem_usage,+ need_weight_grad,+ )key_str = json.dumps(key_data, sort_keys=True, default=str)return hashlib.sha256(key_str.encode()).hexdigest()[:16]@@ -1565,7 +1620,8 @@ def _autoscheme_cache_key(def _autoscheme_cache_path(cache_key, scheme_index):"""Return the full path to the JSON cache file for a **single scheme**.- Each scheme gets its own cache file under ``{AR_WORK_SPACE}/auto_scheme_cache/``+ Each scheme gets its own cache file under ``AR_AUTO_SCHEME_CACHE`` or the+ default ``~/.cache/auto_round`` directoryto enable granular reuse: adding/removing schemes or changing non-scoringparameters (e.g., target_bits) doesn't invalidate caches for unmodified schemes.@@ -1575,7 +1631,7 @@ def _autoscheme_cache_path(cache_key, scheme_index):"""from auto_round import envs as _envs- cache_dir = os.path.join(_envs.AR_WORK_SPACE, "auto_scheme_cache")+ cache_dir = os.path.expanduser(_envs.AR_AUTO_SCHEME_CACHE or "~/.cache/auto_round")os.makedirs(cache_dir, exist_ok=True)return os.path.join(cache_dir, f"scheme_{scheme_index:02d}_{cache_key}.json")@@ -1588,6 +1644,7 @@ def _save_autoscheme_scores(layer_scores,total_loss_for_scheme,total_params,+ cache_config,):"""Persist scoring results for **a single scheme** to *cache_path* as JSON.@@ -1595,12 +1652,13 @@ def _save_autoscheme_scores(or changing unrelated parameters (e.g., target_bits) doesn't invalidate cachedscores for unchanged schemes.- Schema version 3 (per-scheme, per-op cache)::+ Schema version 1 (portable per-scheme, per-op cache)::{- "version": 3,+ "version": 1,"score_granularity": "per_op","cache_key": "<hex>",+ "cache_config": { ... scoring inputs ... },"scheme_index": 0,"scheme": { ... scheme dict ... },"created_at": "<ISO datetime>",@@ -1616,9 +1674,10 @@ def _save_autoscheme_scores(# re-apply grouping when loading a cache so the on-disk format stays# per-op and backward/forward compatible.data = {- "version": 3,+ "version": 1,"score_granularity": "per_op","cache_key": cache_key,+ "cache_config": cache_config,"scheme_index": scheme_index,"scheme": scheme_dict,"created_at": time.strftime("%Y-%m-%dT%H:%M:%S"),@@ -1635,20 +1694,20 @@ def _save_autoscheme_scores(def _load_autoscheme_scores(cache_path):- """Load and validate a **single-scheme** per-op scoring cache file (version 3).+ """Load and validate a **single-scheme** per-op scoring cache file (version 1).Returns the parsed dict with keys ``layer_scores``, ``total_loss_for_scheme``,and ``total_params`` on success, or ``None`` if the file is missing, malformed,or fails the version sanity check."""- _required = ("layer_scores", "total_loss_for_scheme", "total_params")+ _required = ("cache_config", "layer_scores", "total_loss_for_scheme", "total_params")try:with open(cache_path, encoding="utf-8") as _f:data = json.load(_f)- if data.get("version") != 3 or data.get("score_granularity") != "per_op":+ if data.get("version") != 1 or data.get("score_granularity") != "per_op":logger.warning("AutoScheme: per-scheme cache schema mismatch "- "(expected version=3, score_granularity=per_op; got version=%s, score_granularity=%s)",+ "(expected version=1, score_granularity=per_op; got version=%s, score_granularity=%s)",data.get("version"),data.get("score_granularity"),)@@ -1669,6 +1728,42 @@ def _is_per_op_cache_compatible(cached_data, quant_layer_names, fixed_layer_schereturn set(cached_data["layer_scores"]) == expected_layers+def _find_compatible_autoscheme_cache(+ expected_path,+ cache_config,+ quant_layer_names,+ fixed_layer_scheme,+ total_params,+):+ """Find a compatible cache even when a downloaded JSON has a different filename."""+ candidates = [expected_path]+ cache_dir = os.path.dirname(expected_path)+ try:+ candidates.extend(+ os.path.join(cache_dir, filename)+ for filename in sorted(os.listdir(cache_dir))+ if filename.endswith(".json") and os.path.join(cache_dir, filename) != expected_path+ )+ except OSError:+ pass++ for candidate in candidates:+ if not os.path.isfile(candidate):+ continue+ cached_data = _load_autoscheme_scores(candidate)+ if cached_data is None or not _is_per_op_cache_compatible(cached_data, quant_layer_names, fixed_layer_scheme):+ continue+ if cached_data["cache_config"] != cache_config:+ continue+ if cached_data.get("total_params") != total_params:+ continue+ cached_data["_cache_path"] = candidate+ if candidate != expected_path:+ logger.info("AutoScheme: using compatible downloaded cache %s", candidate)+ return cached_data+ return None++def _refresh_cached_layer_bits(model,quant_layer_names,@@ -1762,6 +1857,20 @@ def _get_scheme_worker_count(num_schemes, num_gpus):return num_schemes+def _can_parallel_scheme_scoring(+ parallel_enabled, model_id, num_gpus, uncached_count, need_imatrix, disk_stream_model, is_vlm+):+ """Return whether candidate schemes can be scored in separate workers."""+ return (+ parallel_enabled+ and model_id is not None+ and num_gpus >= 1+ and uncached_count >= 2+ and not need_imatrix+ and (not disk_stream_model or not is_vlm)+ )++def _load_scheme_worker_model(model_name, use_model_replacements, low_cpu_mem_usage):"""Load an isolated worker model without an extra full-size CPU initialization copy."""return load_model(@@ -1773,6 +1882,24 @@ def _load_scheme_worker_model(model_name, use_model_replacements, low_cpu_mem_us)+def _load_disk_stream_scheme_worker_model(model_name, use_model_replacements=False):+ """Build an isolated meta model and checkpoint index for a scoring worker."""+ from auto_round.utils.disk_stream_util import build_meta_model++ model, tokenizer, disk_index = build_meta_model(model_name)+ if use_model_replacements:+ from auto_round.special_model_handler import _handle_special_model, update_module++ model = update_module(model, formats=None, cleanup_original=False)+ model = _handle_special_model(model)+ return model, tokenizer, disk_index+++def _prefer_disk_stream_scheme_worker(model_id, is_vlm, low_gpu_mem_usage):+ """Prefer block-wise disk streaming whenever the worker scoring path supports it."""+ return model_id is not None and not is_vlm and low_gpu_mem_usage++def _score_scheme_worker(args):"""Score one scheme and return its index, scores, and worker VRAM peak."""import traceback as _tb@@ -1798,6 +1925,7 @@ def _score_scheme_worker(args):worker_device,total_schemes,progress_queue,+ disk_stream_model,) = argsfrom auto_round.auto_scheme.utils import _scheme_short_name as _short_name@@ -1806,19 +1934,40 @@ def _score_scheme_worker(args):from auto_round.utils import get_block_names as _get_block_namesfrom auto_round.utils import get_module as _get_module- try:- model, tokenizer, processor, _, _, is_vlm, _ = _load_scheme_worker_model(- model_name,- use_model_replacements,- low_cpu_mem_usage,- )- except Exception as exc:- raise RuntimeError(- f"_score_scheme_worker[{index}]: failed to load model {model_name!r}\n{_tb.format_exc()}"- ) from exc+ disk_index = None+ if disk_stream_model:+ try:+ model, tokenizer, disk_index = _load_disk_stream_scheme_worker_model(+ model_name, use_model_replacements=use_model_replacements+ )+ processor = None+ except Exception as exc: # noqa: BLE001+ logger.warning(+ "_score_scheme_worker[%d]: disk-stream load failed for %r (%s); falling back to regular loading.",+ index,+ model_name,+ exc,+ )+ disk_index = None++ if disk_index is None:+ try:+ model, tokenizer, processor, _, _, is_vlm, _ = _load_scheme_worker_model(+ model_name,+ use_model_replacements,+ low_cpu_mem_usage,+ )+ except Exception as exc:+ raise RuntimeError(+ f"_score_scheme_worker[{index}]: failed to load model {model_name!r}\n{_tb.format_exc()}"+ ) from excsafe_to_cpu_(model)block_names = _get_block_names(model, quant_vision=force_mllm)[0]+ if disk_index is not None:+ from auto_round.utils.disk_stream_util import materialize_non_block_params++ materialize_non_block_params(model, block_names, disk_index, device="cpu")for block_name in block_names:block = _get_module(model, block_name)block.in_block = True@@ -1833,7 +1982,8 @@ def _score_scheme_worker(args):from auto_round.modeling.fused_moe.replace_modules import materialize_model_- materialize_model_(model)+ if disk_index is None:+ materialize_model_(model)# MoE materialization can replace fused expert modules with newly-created# Linear layers. Assign tuning metadata only after the final module tree exists.for layer_name in quant_layer_names:@@ -1888,6 +2038,7 @@ def _score_scheme_worker(args):force_mllm=force_mllm,model_name=model_name,scheme_tag=f"{index + 1}/{total_schemes} {_short_name(scheme)}",+ disk_index=disk_index,)return index, scores, _get_worker_memory_report(worker_device)@@ -2172,6 +2323,16 @@ def _gen_layer_config(pbar = tqdm(total=pbar_cnt, desc="Generating AutoScheme")scored_layer_names = set(quant_layer_names + embedding_layers_names)+ cache_total_params = sum(+ (+ module.weight.numel()+ if hasattr(module, "weight") and module.weight is not None+ else getattr(module, "_cached_weight_numel", 0)+ )+ for name, module in model.named_modules()+ if name in scored_layer_names+ )+ scheme_cache_configs = []def _group_per_op_scores(index, per_op_scores):"""Apply the current shared-layer grouping without mutating cached per-op scores."""@@ -2217,21 +2378,15 @@ def _gen_layer_config(scheme_dict=scheme_dict,layer_scores={name: list(score) for name, score in per_op_scores.items()},total_loss_for_scheme=sum(score[1] for score in per_op_scores.values()),- total_params=sum(- (- module.weight.numel()- if hasattr(module, "weight") and module.weight is not None- else getattr(module, "_cached_weight_numel", 0)- )- for name, module in model.named_modules()- if name in scored_layer_names- ),+ total_params=cache_total_params,+ cache_config=scheme_cache_configs[index],)scheme_cache_meta = []for index, scheme in enumerate(schemes):if check_bf16_scheme(scheme) or _model_id_for_cache is None:scheme_cache_meta.append((None, None, None))+ scheme_cache_configs.append(None)if check_bf16_scheme(scheme):logger.info("AutoScheme: scheme %d/%d (%s) is a BF16 baseline; skipping scoring and cache lookup.",@@ -2240,7 +2395,7 @@ def _gen_layer_config(_scheme_short_name(scheme),)continue- cache_key = _autoscheme_cache_key(+ cache_config = _autoscheme_cache_config(model_name=_model_id_for_cache,dataset=dataset,nsamples=nsamples,@@ -2251,19 +2406,19 @@ def _gen_layer_config(scheme=scheme,force_mllm=force_mllm,low_gpu_mem_usage=auto_scheme.low_gpu_mem_usage,+ need_weight_grad=need_weight_grad,)+ cache_key = hashlib.sha256(json.dumps(cache_config, sort_keys=True, default=str).encode()).hexdigest()[:16]cache_path = _autoscheme_cache_path(cache_key, index)- cached_data = _load_autoscheme_scores(cache_path) if os.path.exists(cache_path) else None- if cached_data is not None and not _is_per_op_cache_compatible(- cached_data, quant_layer_names, fixed_layer_scheme- ):- logger.warning(- "AutoScheme: cache %s does not contain complete per-op scores; rescoring scheme %d",- cache_path,- index,- )- cached_data = None+ cached_data = _find_compatible_autoscheme_cache(+ cache_path,+ cache_config,+ quant_layer_names,+ fixed_layer_scheme,+ cache_total_params,+ )scheme_cache_meta.append((cache_key, cache_path, cached_data))+ scheme_cache_configs.append(cache_config)uncached_indices = [index@@ -2273,23 +2428,29 @@ def _gen_layer_config(worker_device_pool = [device for device in device_list if str(device).startswith("cuda:")]num_gpus = len(worker_device_pool)parallel_enabled = _envs.AR_ENABLE_AUTO_SCHEME_PARALLEL- can_parallel = (- parallel_enabled- and _model_id_for_cache is not None- and num_gpus >= 1- and len(uncached_indices) >= 2- and not need_imatrix- # Each parallel worker fully loads its own copy of the model- # (_load_scheme_worker_model) in a separate process -- incompatible- # with a meta-device streaming skeleton, whose entire point is to- # avoid ever materializing a full copy. Force serial scoring- # (which honors disk_index via materialize_module/free_module)- # instead when streaming is active.- and disk_index is None+ worker_disk_stream_model = _prefer_disk_stream_scheme_worker(+ _model_id_for_cache, is_vlm, auto_scheme.low_gpu_mem_usage+ )+ # Vision scoring requires a full-model backward and therefore cannot use+ # the block-wise materialize/free path used by disk streaming.+ can_parallel = _can_parallel_scheme_scoring(+ parallel_enabled,+ _model_id_for_cache,+ num_gpus,+ len(uncached_indices),+ need_imatrix,+ worker_disk_stream_model,+ is_vlm,+ )+ logger.info(+ "AutoScheme scoring mode: parallel_configured=%s, parallel_enabled=%s, disk_stream_enabled=%s",+ parallel_enabled,+ can_parallel,+ worker_disk_stream_model and can_parallel,)if not parallel_enabled and len(uncached_indices) >= 2:logger.info(- "AutoScheme: parallel scoring is disabled; set AR_ENABLE_AUTO_SCHEME_PARALLEL=1 to enable it. "+ "AutoScheme: parallel scoring was disabled by AR_ENABLE_AUTO_SCHEME_PARALLEL=0; ""scoring %d uncached non-BF16 schemes serially.",len(uncached_indices),)@@ -2344,6 +2505,7 @@ def _gen_layer_config(worker_devices[slot],len(schemes),progress_queue,+ worker_disk_stream_model,)for slot, index in enumerate(uncached_indices)]@@ -2380,11 +2542,12 @@ def _gen_layer_config(bits, _ = compute_layer_bits(get_module(model, name), auto_scheme.ignore_scale_zp_bits)per_op_scores[name] = [bits, 0.0]elif cached_data is not None:+ loaded_cache_path = cached_data.get("_cache_path", cache_path)logger.info("AutoScheme: loading per-scheme cache for scheme %d from %s."" Delete this file to disable reuse and rescore.",index,- cache_path,+ loaded_cache_path,)per_op_scores = _refresh_cached_layer_bits(model,@@ -2437,11 +2600,12 @@ def _gen_layer_config(cache_key, cache_path, cached_data = scheme_cache_meta[index]if cached_data is not None:+ loaded_cache_path = cached_data.get("_cache_path", cache_path)logger.info("AutoScheme: loading per-scheme cache for scheme %d from %s."" Delete this file to disable reuse and rescore.",index,- cache_path,+ loaded_cache_path,)per_op_scores = _refresh_cached_layer_bits(model,@@ -2604,15 +2768,14 @@ def _gen_layer_config(if not candidates:candidates = list(sorted_indices)else:- # Not shared lm_head: prefer options with bits < floor(target_bits)+ # Not shared lm_head: prefer the nearest available bit width at or+ # above floor(target_bits), then choose the lowest-loss option at+ # that width.floor_bits = math.floor(target_bits)candidates = [idx for idx in sorted_indices if _get_scheme_bits(schemes[idx]) == floor_bits]if not candidates:- # find the first bits that greater than floor bits- embedding_bits = [bits for idx in sorted_indices if _get_scheme_bits(schemes[idx]) > floor_bits]- if len(embedding_bits) > 0:- sorted(embedding_bits)- embedding_bits = embedding_bits[0]+ embedding_bits = _get_next_scheme_bits(schemes, sorted_indices, floor_bits)+ if embedding_bits is not None:candidates = [idx for idx in sorted_indices if _get_scheme_bits(schemes[idx]) == embedding_bits]candidates.extend(sorted_indices) # to make sure if the above candidate exceed the budget@@ -2720,8 +2883,6 @@ def _gen_layer_config(for n, m in model.named_parameters():if hasattr(m, "grad"):m.grad = None- global last_grad_input- last_grad_input = Noneclear_memory(device_list=device_list)# # Log AutoScheme memory usagediff --git a/auto_round/envs.py b/auto_round/envs.pyindex 17444e71c..b72b14ae3 100644--- a/auto_round/envs.py+++ b/auto_round/envs.py@@ -26,7 +26,8 @@ if TYPE_CHECKING:LLAMA_CPP_ROOT: Optional[str] = NoneAR_AUTO_SCHEME_NSAMPLES: Optional[int] = NoneAR_AUTO_SCHEME_BATCH_SIZE: Optional[int] = None- AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = False+ AR_AUTO_SCHEME_CACHE: Optional[str] = None+ AR_ENABLE_AUTO_SCHEME_PARALLEL: bool = TrueAR_DISK_STREAM_MODEL: bool = FalseAR_RESUME_DIR: Optional[str] = None@@ -93,9 +94,12 @@ environment_variables: dict[str, Callable[[], Any]] = {# when ``AutoScheme.batch_size`` is not explicitly set.# When unset, AutoScheme uses its built-in heuristic (8 for low GPU memory mode, 1 for normal mode)."AR_AUTO_SCHEME_BATCH_SIZE": lambda: _get_optional_positive_int_env("AR_AUTO_SCHEME_BATCH_SIZE"),- # Enables AutoScheme to score schemes in parallel. Disabled by default to- # avoid multiple model-loading workers exhausting host RAM or device memory.- "AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "0").lower()+ # Stores persistent AutoScheme scoring results independently from AR_WORK_SPACE,+ # whose contents are temporary working data and may be cleaned after a run.+ "AR_AUTO_SCHEME_CACHE": lambda: os.getenv("AR_AUTO_SCHEME_CACHE", None),+ # Enables AutoScheme to score schemes in parallel. Enabled by default;+ # set it to 0 when workers could exhaust host RAM or device memory.+ "AR_ENABLE_AUTO_SCHEME_PARALLEL": lambda: os.getenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1").lower()in ("1", "true", "yes"),# When set, the model is built as a meta-device skeleton and streamed# block-by-block from disk during quantization instead of being fullydiff --git a/docs/environments.md b/docs/environments.mdindex 5934cef78..a61492314 100644--- a/docs/environments.md+++ b/docs/environments.md@@ -141,18 +141,28 @@ export AR_AUTO_SCHEME_NSAMPLES=1 # set 1 for quick executionexport AR_AUTO_SCHEME_BATCH_SIZE=1```+### AR_AUTO_SCHEME_CACHE+- **Description**: Stores persistent per-scheme AutoScheme scoring JSON files. This directory is independent of `AR_WORK_SPACE`, which is reserved for temporary working data.+- **Default**: `~/.cache/auto_round`+- **Valid Values**: any writable directory path+- **Usage**: Set this to place reusable AutoScheme scores in a different cache directory++```bash+export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cache+```+### AR_ENABLE_AUTO_SCHEME_PARALLEL-- **Description**: Enables multiprocessing across AutoScheme candidates. Keep it disabled when parallel model-loading workers could exhaust host RAM or device memory.-- **Default**: `False` (schemes are scored serially)-- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) enable parallel scoring; any other value keeps parallel scoring disabled-- **Usage**: Set this before running AutoScheme to enable parallel candidate scoring+- **Description**: Enables multiprocessing across AutoScheme candidates. It can be combined with `AR_DISK_STREAM_MODEL=1`; each worker then builds its own meta-model skeleton and streams blocks independently. Disable it when concurrent workers could exhaust host RAM or device memory.+- **Default**: `"1"` (schemes are scored in parallel when multiprocessing requirements are met)+- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) enable parallel scoring; any other value disables parallel scoring+- **Usage**: Set this to `0` before running AutoScheme to force serial candidate scoring```bash-export AR_ENABLE_AUTO_SCHEME_PARALLEL=1+export AR_ENABLE_AUTO_SCHEME_PARALLEL=0```### AR_DISK_STREAM_MODEL-- **Description**: When enabled, `AutoRound(model=<path>, ...)` builds the model as a meta-device skeleton instead of fully materializing the checkpoint on CPU RAM up front, and streams each decoder block's real weights from the checkpoint's safetensors shards on demand -- materializing right before a block is used (calibration, tuning, or `AutoScheme` sensitivity scoring) and freeing it back to meta right after. This keeps peak CPU RAM roughly flat regardless of checkpoint size, instead of proportional to it. Non-block parameters (embeddings, `lm_head`, final norm) are still loaded up front, since they are typically small.+- **Description**: When enabled, `AutoRound(model=<path>, ...)` builds the model as a meta-device skeleton instead of fully materializing the checkpoint on CPU RAM up front, and streams each decoder block's real weights from the checkpoint's safetensors shards on demand -- materializing right before a block is used (calibration, tuning, or `AutoScheme` sensitivity scoring) and freeing it back to meta right after. This keeps peak CPU RAM roughly flat regardless of checkpoint size, instead of proportional to it. Non-block parameters (embeddings, `lm_head`, final norm) are still loaded up front, since they are typically small. Text-model AutoScheme scoring also supports combining this with parallel scoring, which is enabled by default; each worker streams its own block copy.- **Default**: `False`- **Valid Values**: `"1"`, `"true"`, `"yes"` (case-insensitive) for enabling; any other value for disabling- **Usage**: Enable this to quantize checkpoints larger than available CPU RAM + GPU VRAM combined. Only applies when `model` is a string (local directory) path; has no effect on already-loaded model objects.diff --git a/docs/environments_CN.md b/docs/environments_CN.mdindex 9dd02d251..0272dbd1f 100644--- a/docs/environments_CN.md+++ b/docs/environments_CN.md@@ -141,18 +141,28 @@ export AR_AUTO_SCHEME_NSAMPLES=1export AR_AUTO_SCHEME_BATCH_SIZE=1```+### AR_AUTO_SCHEME_CACHE+- **描述**:存放可持久复用的 AutoScheme 单方案评分 JSON 文件。该目录独立于用于临时工作数据的 `AR_WORK_SPACE`。+- **默认值**:`~/.cache/auto_round`+- **有效值**:任意可写目录路径+- **用途**:将可复用的 AutoScheme 评分结果保存到其他缓存目录++```bash+export AR_AUTO_SCHEME_CACHE=/path/to/auto_scheme_cache+```+### AR_ENABLE_AUTO_SCHEME_PARALLEL-- **描述**:启用 AutoScheme 候选方案之间的多进程并行。当并行加载多个模型可能耗尽主机内存或显存时,请保持关闭。-- **默认值**:`False`(串行评分各方案)-- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用并行评分;其他值保持关闭-- **用途**:运行 AutoScheme 前设置,以启用候选方案并行评分+- **描述**:启用 AutoScheme 候选方案之间的多进程并行。可与 `AR_DISK_STREAM_MODEL=1` 同时使用;此时每个 worker 会构建独立的 meta 模型骨架并分别流式加载 block。当并发 worker 可能耗尽主机内存或显存时,请将其关闭。+- **默认值**:`"1"`(满足多进程要求时并行评分各方案)+- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用并行评分;其他值表示关闭+- **用途**:运行 AutoScheme 前将其设为 `0`,以强制串行评分候选方案```bash-export AR_ENABLE_AUTO_SCHEME_PARALLEL=1+export AR_ENABLE_AUTO_SCHEME_PARALLEL=0```### AR_DISK_STREAM_MODEL-- **描述**:启用后,`AutoRound(model=<path>, ...)` 会将模型构建为 meta 设备骨架,而不是先把整个 checkpoint 完全加载到 CPU 内存;随后按需从 checkpoint 的 safetensors 分片中流式加载每个解码器块的真实权重——在该块被使用前(校准、调优或 `AutoScheme` 敏感度评分)才实体化,用完后立即释放回 meta。这样峰值 CPU 内存基本保持平稳,而不会随 checkpoint 大小成比例增长。非块参数(embedding、`lm_head`、最终归一化层)体积通常较小,仍会一次性加载。+- **描述**:启用后,`AutoRound(model=<path>, ...)` 会将模型构建为 meta 设备骨架,而不是先把整个 checkpoint 完全加载到 CPU 内存;随后按需从 checkpoint 的 safetensors 分片中流式加载每个解码器块的真实权重——在该块被使用前(校准、调优或 `AutoScheme` 敏感度评分)才实体化,用完后立即释放回 meta。这样峰值 CPU 内存基本保持平稳,而不会随 checkpoint 大小成比例增长。非块参数(embedding、`lm_head`、最终归一化层)体积通常较小,仍会一次性加载。文本模型的 AutoScheme 评分也支持与默认启用的并行评分组合使用;每个 worker 会流式加载自己的 block 副本。- **默认值**:`False`- **有效值**:`"1"`、`"true"`、`"yes"`(不区分大小写)表示启用;其他任何值表示禁用- **用途**:用于量化体积超过可用 CPU 内存 + GPU 显存总和的 checkpoint。仅在 `model` 为字符串(本地目录)路径时生效,对已加载的模型对象无效。diff --git a/test/test_cpu/schemes/test_auto_scheme.py b/test/test_cpu/schemes/test_auto_scheme.pyindex d6a2ce168..d71ac0e01 100644--- a/test/test_cpu/schemes/test_auto_scheme.py+++ b/test/test_cpu/schemes/test_auto_scheme.py@@ -60,13 +60,22 @@ def test_env_ar_auto_scheme_batch_size_zero_raises(monkeypatch):_ = envs.AR_AUTO_SCHEME_BATCH_SIZE+def test_env_ar_auto_scheme_cache(monkeypatch, tmp_path):+ """AR_AUTO_SCHEME_CACHE should expose an independent cache directory."""+ import auto_round.envs as envs++ cache_dir = str(tmp_path / "auto_scheme_cache")+ monkeypatch.setenv("AR_AUTO_SCHEME_CACHE", cache_dir)+ assert envs.AR_AUTO_SCHEME_CACHE == cache_dir++def test_env_ar_enable_auto_scheme_parallel(monkeypatch):import auto_round.envs as envsmonkeypatch.delenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", raising=False)- assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is False- monkeypatch.setenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "1")assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is True+ monkeypatch.setenv("AR_ENABLE_AUTO_SCHEME_PARALLEL", "0")+ assert envs.AR_ENABLE_AUTO_SCHEME_PARALLEL is Falsedef test_build_layer_config_header_rows_merges_adjacent_prefixes():@@ -135,6 +144,60 @@ def test_choose_bits_per_layer_reconstructs_optimal_path():assert path == [(["layer.0"], 1), (["layer.1"], 0)]+def test_activation_scoring_handles_reused_wrapper():+ """Each forward call should keep its own activation error until backward."""+ import types++ import torch++ from auto_round.auto_scheme.delta_loss import AutoSchemeWrapperLinear++ wrapper = AutoSchemeWrapperLinear.__new__(AutoSchemeWrapperLinear)+ torch.nn.Module.__init__(wrapper)+ wrapper.orig_layer = types.SimpleNamespace(act_bits=8)+ wrapper.act_qdq_func = lambda x, *_args, **_kwargs: (x * 0.5, 1.0, None)+ wrapper.grad_mode = True+ wrapper.act_cnt = 0+ wrapper.act_score = 0.0+ wrapper.weight_score = 0.0+ wrapper.mix_score = 0.0+ wrapper.max_act_value = 0++ first = torch.tensor([1.0, -2.0], requires_grad=True)+ second = torch.tensor([3.0, -4.0], requires_grad=True)+ qdq_first, _, _ = wrapper._qdq_act(first)+ qdq_second, _, _ = wrapper._qdq_act(second)++ (qdq_first.sum() + qdq_second.sum()).backward()++ assert wrapper.act_cnt == 2+ assert wrapper.act_score == pytest.approx(5.0)+ assert wrapper.mix_score == pytest.approx(5.0)+++def test_prepare_replay_input_supports_keyword_hidden_states():+ import torch++ from auto_round.auto_scheme.delta_loss import _prepare_replay_input++ hidden_states = torch.randn(2, 4)+ attention_mask = torch.ones(2, 4, dtype=torch.int64)++ replay_input = _prepare_replay_input([], {"attention_mask": attention_mask, "hidden_states": hidden_states}, "0")++ assert replay_input is hidden_states+ assert replay_input.requires_grad+++def test_prepare_replay_input_rejects_missing_floating_tensor():+ import torch++ from auto_round.auto_scheme.delta_loss import _prepare_replay_input++ with pytest.raises(RuntimeError, match="No floating replay input found for block 0"):+ _prepare_replay_input([], {"attention_mask": torch.ones(2, dtype=torch.int64)}, "0")++def test_build_expert_groups_groups_experts_per_block():"""Expert layers in the same block should be grouped together."""import torch@@ -350,8 +413,8 @@ class TestAutoScheme:from auto_round.auto_scheme.delta_loss import _load_autoscheme_scores- cache_dir = str(tmp_path / "ar_work_space")- monkeypatch.setenv("AR_WORK_SPACE", cache_dir)+ cache_dir = str(tmp_path / "auto_scheme_cache")+ monkeypatch.setenv("AR_AUTO_SCHEME_CACHE", cache_dir)scheme = AutoScheme(avg_bits=3,@@ -363,7 +426,7 @@ class TestAutoScheme:_, layer_config = ar.quantize()# Cache files must exist — one per scheme (2 schemes here)- cache_files = glob.glob(f"{cache_dir}/auto_scheme_cache/scheme_*.json")+ cache_files = glob.glob(f"{cache_dir}/scheme_*.json")assert (len(cache_files) == 2), f"Expected 2 cache files (one per scheme), found {len(cache_files)}: {cache_files}"@@ -371,7 +434,7 @@ class TestAutoScheme:for path in cache_files:data = _load_autoscheme_scores(path)assert data is not None, f"Cache file {path} could not be loaded"- assert data["version"] == 3+ assert data["version"] == 1assert data["score_granularity"] == "per_op"assert "layer_scores" in dataassert "total_loss_for_scheme" in data@@ -528,6 +591,49 @@ def test_autoscheme_cache_key_insensitive_to_layer_order():assert key1 == key2 # Should match after internal sorting+def test_autoscheme_cache_key_is_portable_across_model_paths():+ """A locally downloaded model should keep the same key when its parent directory changes."""+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key++ kwargs = {+ "dataset": "pile-10k",+ "nsamples": 16,+ "seqlen": 256,+ "batch_size": 8,+ "quant_layer_names": ["layer.0"],+ "fixed_layer_scheme": {},+ "scheme": "W4A16",+ "force_mllm": False,+ "low_gpu_mem_usage": True,+ }++ assert _autoscheme_cache_key(model_name="/models/org/test-model", **kwargs) == _autoscheme_cache_key(+ model_name="/tmp/downloads/test-model", **kwargs+ )+++def test_autoscheme_cache_key_normalizes_preset_scheme():+ """A preset name and its resolved scheme should identify the same scoring run."""+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key+ from auto_round.schemes import preset_name_to_scheme++ kwargs = {+ "model_name": "test-model",+ "dataset": "pile-10k",+ "nsamples": 16,+ "seqlen": 256,+ "batch_size": 8,+ "quant_layer_names": ["layer.0"],+ "fixed_layer_scheme": {},+ "force_mllm": False,+ "low_gpu_mem_usage": True,+ }++ assert _autoscheme_cache_key(scheme="W4A16", **kwargs) == _autoscheme_cache_key(+ scheme=preset_name_to_scheme("W4A16"), **kwargs+ )++def test_autoscheme_cache_key_changes_only_with_scoring_config():"""Execution settings that affect scoring invalidate the cache; bit accounting does not."""from auto_round.auto_scheme.delta_loss import _autoscheme_cache_key@@ -547,6 +653,17 @@ def test_autoscheme_cache_key_changes_only_with_scoring_config():baseline = _autoscheme_cache_key(**kwargs)assert baseline != _autoscheme_cache_key(**{**kwargs, "low_gpu_mem_usage": False})+ assert baseline != _autoscheme_cache_key(**{**kwargs, "need_weight_grad": True})+++def test_get_next_scheme_bits_is_order_independent():+ from auto_round.auto_scheme.delta_loss import _get_next_scheme_bits++ schemes = [{"bits": 8}, {"bits": 4}, {"bits": 6}]++ assert _get_next_scheme_bits(schemes, [0, 1, 2], 5) == 6+ assert _get_next_scheme_bits(schemes, [2, 1, 0], 5) == 6+ assert _get_next_scheme_bits(schemes, [0, 1, 2], 8) is Nonedef test_refresh_cached_layer_bits_preserves_loss():@@ -585,6 +702,7 @@ def test_autoscheme_cache_save_and_load(tmp_path):}total_loss = 2.1total_params = 1000000+ cache_config = {"model_id": "test-model", "scheme": scheme_dict}_save_autoscheme_scores(cache_path,@@ -594,6 +712,7 @@ def test_autoscheme_cache_save_and_load(tmp_path):layer_scores,total_loss,total_params,+ cache_config,)loaded = _load_autoscheme_scores(cache_path)@@ -603,6 +722,89 @@ def test_autoscheme_cache_save_and_load(tmp_path):assert loaded["total_params"] == total_params+def test_find_compatible_downloaded_autoscheme_cache(tmp_path):+ """A compatible downloaded cache should work without renaming it to the locally computed key."""+ from auto_round.auto_scheme.delta_loss import (+ _autoscheme_cache_config,+ _find_compatible_autoscheme_cache,+ _save_autoscheme_scores,+ )+ from auto_round.schemes import preset_name_to_scheme++ quant_layer_names = ["layer.0", "layer.1"]+ cache_config = _autoscheme_cache_config(+ model_name="/models/test-model",+ dataset="pile-10k",+ nsamples=16,+ seqlen=256,+ batch_size=8,+ quant_layer_names=quant_layer_names,+ fixed_layer_scheme={},+ scheme="W4A16",+ force_mllm=False,+ low_gpu_mem_usage=True,+ )+ downloaded_path = tmp_path / "downloaded-from-release.json"+ layer_scores = {"layer.0": [4, 1.2], "layer.1": [4, 0.9]}+ _save_autoscheme_scores(+ downloaded_path,+ "key-from-another-commit",+ 0,+ preset_name_to_scheme("W4A16").to_dict(),+ layer_scores,+ 2.1,+ 1000,+ cache_config=cache_config,+ )++ loaded = _find_compatible_autoscheme_cache(+ str(tmp_path / "scheme_00_current-key.json"),+ cache_config,+ quant_layer_names,+ {},+ 1000,+ )++ assert loaded is not None+ assert loaded["layer_scores"] == layer_scores+ assert loaded["_cache_path"] == str(downloaded_path)+ assert (+ _find_compatible_autoscheme_cache(+ str(tmp_path / "scheme_00_current-key.json"),+ cache_config,+ quant_layer_names,+ {},+ 999,+ )+ is None+ )+++def test_autoscheme_cache_path_is_independent_from_workspace(tmp_path, monkeypatch):+ """AutoScheme caches default to the user cache and ignore AR_WORK_SPACE."""+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_path++ monkeypatch.setenv("HOME", str(tmp_path / "home"))+ monkeypatch.setenv("AR_WORK_SPACE", str(tmp_path / "workspace"))+ monkeypatch.delenv("AR_AUTO_SCHEME_CACHE", raising=False)++ cache_path = _autoscheme_cache_path("cache_key", 1)++ assert cache_path == str(tmp_path / "home" / ".cache" / "auto_round" / "scheme_01_cache_key.json")+++def test_autoscheme_cache_path_can_be_overridden(tmp_path, monkeypatch):+ """AR_AUTO_SCHEME_CACHE overrides the default user cache directory."""+ from auto_round.auto_scheme.delta_loss import _autoscheme_cache_path++ cache_dir = tmp_path / "custom_cache"+ monkeypatch.setenv("AR_AUTO_SCHEME_CACHE", str(cache_dir))++ cache_path = _autoscheme_cache_path("cache_key", 2)++ assert cache_path == str(cache_dir / "scheme_02_cache_key.json")++def test_parallel_progress_events_are_applied_in_parent():"""Worker progress events should update only the parent-owned progress bar."""import queue@@ -674,6 +876,33 @@ def test_scheme_worker_count_rejects_no_gpu():_get_scheme_worker_count(2, 0)+def test_parallel_scheme_scoring_supports_disk_stream_model():+ from auto_round.auto_scheme.delta_loss import _can_parallel_scheme_scoring++ assert _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, False)+++def test_parallel_scheme_scoring_rejects_disk_stream_vlm():+ from auto_round.auto_scheme.delta_loss import _can_parallel_scheme_scoring++ assert not _can_parallel_scheme_scoring(True, "local-model", 1, 2, False, True, True)+++ "model_id,is_vlm,low_gpu_mem_usage,expected",+ [+ ("local-model", False, True, True),+ ("local-model", True, True, False),+ ("local-model", False, False, False),+ (None, False, True, False),+ ],+)+def test_prefer_disk_stream_scheme_worker(model_id, is_vlm, low_gpu_mem_usage, expected):+ from auto_round.auto_scheme.delta_loss import _prefer_disk_stream_scheme_worker++ assert _prefer_disk_stream_scheme_worker(model_id, is_vlm, low_gpu_mem_usage) is expected++def test_opt_scheme_worker_uses_low_cpu_memory_loading(monkeypatch):from test.helpers import opt_name_or_path@@ -700,6 +929,50 @@ def test_opt_scheme_worker_uses_low_cpu_memory_loading(monkeypatch):}+def test_disk_stream_scheme_worker_builds_meta_model(monkeypatch):+ from auto_round.auto_scheme import delta_loss+ from auto_round.utils import disk_stream_util++ expected = (object(), object(), object())+ monkeypatch.setattr(disk_stream_util, "build_meta_model", lambda model_name: expected)++ assert delta_loss._load_disk_stream_scheme_worker_model("local-model") == expected+++def test_disk_stream_scheme_worker_applies_model_replacements(monkeypatch):+ from auto_round import special_model_handler+ from auto_round.auto_scheme import delta_loss+ from auto_round.utils import disk_stream_util++ model = object()+ updated_model = object()+ handled_model = object()+ tokenizer = object()+ disk_index = object()+ calls = []++ monkeypatch.setattr(disk_stream_util, "build_meta_model", lambda model_name: (model, tokenizer, disk_index))++ def fake_update_module(input_model, formats, cleanup_original):+ calls.append(("update", input_model, formats, cleanup_original))+ return updated_model++ def fake_handle_special_model(input_model):+ calls.append(("handle", input_model))+ return handled_model++ monkeypatch.setattr(special_model_handler, "update_module", fake_update_module)+ monkeypatch.setattr(special_model_handler, "_handle_special_model", fake_handle_special_model)++ result = delta_loss._load_disk_stream_scheme_worker_model("local-model", use_model_replacements=True)++ assert result == (handled_model, tokenizer, disk_index)+ assert calls == [+ ("update", model, None, False),+ ("handle", updated_model),+ ]++def test_per_op_cache_compatibility_rejects_grouped_scores():from auto_round.auto_scheme.delta_loss import _is_per_op_cache_compatible@@ -718,7 +991,7 @@ def test_per_op_cache_compatibility_rejects_grouped_scores():)-def test_version_two_cache_is_rejected(tmp_path):+def test_non_version_one_cache_is_rejected(tmp_path):import jsonfrom auto_round.auto_scheme.delta_loss import _load_autoscheme_scores@@ -727,7 +1000,7 @@ def test_version_two_cache_is_rejected(tmp_path):cache_path.write_text(json.dumps({- "version": 2,+ "version": 3,"layer_scores": {"layer.0": [4, 1.0]},"total_loss_for_scheme": 1.0,"total_params": 4,