PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import ctypes
import functools
import gc
import os
import re
import shutil
import sys
import tempfile
from collections.abc import Callable
from contextlib import ContextDecorator, contextmanager
from functools import lru_cache
from threading import Lock
from typing import Any
import cpuinfo
import psutil
import torch
from accelerate import dispatch_model, infer_auto_device_map
from accelerate.utils import get_balanced_memory, get_max_memory
from auto_round.logger import logger
from auto_round.utils.device_manager import (
clear_memory,
detect_device_count,
get_ar_device,
get_available_device_types,
get_current_device_manager,
get_device_memory,
get_major_device,
)
from auto_round.utils.model import check_to_quantized, get_block_names, get_layer_features, get_module
DEVICE_ENVIRON_VARIABLE_MAPPING = {
"cuda": "CUDA_VISIBLE_DEVICES",
"xpu": "ZE_AFFINITY_MASK",
"hpu": "HABANA_VISIBLE_MODULES",
}
# Note on HPU usage:
# There are two modes available for enabling auto-round on HPU:
# 1. Compile Mode
# 1) Use PyTorch version ≥ 2.4 (Intel® Gaudi® v1.18 or later)
# 2) Set `PT_HPU_LAZY_MODE=0` and `PT_ENABLE_INT64_SUPPORT=1`
# The compile mode can speed up quantization process but still in experimental stage.
# 2. Lazy Mode (By default)
################ Check available sys.module to decide behavior #################
def probe_usable_bytes(device_key: str) -> int | None:
"""Corrected free bytes on an accelerator device (raw free + reserved-but-unallocated).
Routes through the device manager so every backend it supports (cuda,
xpu, hpu, mps) probes through its own API instead of silently reporting
nothing on non-cuda fleets.
"""
try:
dev = torch.device(str(device_key))
if dev.type == "cpu" or dev.index is None:
return None
from auto_round.utils.device_manager import get_ar_device
ar_dev = get_ar_device(dev.type)
free, total = ar_dev.mem_get_info(dev.index)
if total <= 0:
return None
free += ar_dev.memory_reserved(dev.index) - ar_dev.memory_allocated(dev.index)
return max(free, 0)
except (ValueError, RuntimeError, AttributeError) as e:
logger.debug("[device] free-memory probe failed for %s (%s)", device_key, e)
return None
def is_package_available(package_name: str) -> bool:
"""Check if the package exists in the environment without importing.
Args:
package_name (str): package name
"""
from importlib.util import find_spec
package_spec = find_spec(package_name)
return package_spec is not None
def is_hpu_lazy_mode():
return os.getenv("PT_HPU_LAZY_MODE") != "0"
def _use_hpu_compile_mode():
from auto_round.utils.common import TORCH_VERSION_AT_LEAST_2_4
return TORCH_VERSION_AT_LEAST_2_4 and not is_hpu_lazy_mode()
def _bump_dynamo_cache_limit(min_size: int | None = None):
"""Raise torch._dynamo cache/recompile limits.
The same quant function (e.g. ``quant_tensor_sym``) is reused across
every linear layer in a transformer block (q/k/v/o_proj, gate/up/
down_proj, ...), each with a different weight shape. Because dynamo's
compile cache is keyed by the function's code object (shared across
all WrapperLinear instances), per-layer static recompiles quickly
exceed the default ``recompile_limit`` (8) and trigger a fallback to
eager with a noisy warning. We keep static-shape compilation (best
perf) and just allow more cache entries.
The threshold can be overridden via the ``AR_DYNAMO_CACHE_SIZE_LIMIT``
environment variable (default: 16).
"""
try:
if min_size is None:
from auto_round import envs
min_size = envs.AR_DYNAMO_CACHE_SIZE_LIMIT
from torch._dynamo import config as _dynamo_config
for attr in (
"cache_size_limit",
"accumulated_cache_size_limit",
"recompile_limit",
"accumulated_recompile_limit", # newer torch: process-wide cap (binds on MoE shape zoos)
):
if hasattr(_dynamo_config, attr) and getattr(_dynamo_config, attr) < min_size:
setattr(_dynamo_config, attr, min_size)
except Exception: # pragma: no cover - best effort
pass
def compile_func(fun: torch.nn.Module | Callable, device: str | torch.device | int) -> torch.nn.Module | Callable:
"""Compile a function on the specified device.
The shared dynamo cache-limit bump lives in :func:`_bump_dynamo_cache_limit`;
the per-device ``torch.compile`` customization (whether to compile and which
backend to use) is delegated to the corresponding :class:`Device`, keeping
this entry point device-agnostic.
"""
return get_ar_device(device).compile_func(fun)
def is_numba_available(): # pragma: no cover
"""Check if Numba is available."""
try:
import numba
return True
except ImportError:
return False
def _is_tbb_installed(): # pragma: no cover
import importlib.metadata
try:
importlib.metadata.version("tbb")
return True
except importlib.metadata.PackageNotFoundError:
return False
def _is_tbb_configured(): # pragma: no cover
try:
from numba.np.ufunc.parallel import _check_tbb_version_compatible
# check if TBB is present and compatible
_check_tbb_version_compatible()
return True
except ImportError as e:
logger.warning_once(f"TBB not available: {e}")
return False
def is_tbb_available(): # pragma: no cover
"""Check if TBB is available."""
if not _is_tbb_installed():
logger.warning_once("TBB is not installed, please install it with `pip install tbb`.")
return False
if not _is_tbb_configured():
logger.warning_once(
"TBB is installed but not configured correctly. \n"
"Please add the TBB library path to `LD_LIBRARY_PATH`, "
"for example: `export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/`."
)
return False
return True
def can_pack_with_numba(): # pragma: no cover
"""Check if Numba and TBB are available for packing.
To pack tensor with Numba, both Numba and TBB are required, and TBB should be configured correctly.
"""
if not is_numba_available():
logger.warning_once("Numba is not installed, please install it with `pip install numba`.")
return False
return is_tbb_available()
## check hpex
if is_package_available("habana_frameworks"):
_hpex_available = True
import habana_frameworks.torch.hpex # pylint: disable=E0401
else:
_hpex_available = False
@torch._dynamo.disable()
@lru_cache(None)
def is_hpex_available():
return _hpex_available
_xpu_sdpa_patched = False
# TODO: This is a workaround for the XPU SDPA memory blow-up issue.
# We should remove this patch after the issue is fixed in XPU side.
# https://github.com/intel/auto-round/issues/990
def patch_xpu_sdpa_drop_causal_mask():
"""Workaround for XPU peak-VRAM blow-up in SDPA.
On Intel XPU, ``torch.nn.functional.scaled_dot_product_attention`` falls back
to the MATH backend whenever ``attn_mask`` is non-None (FLASH on XPU does
not support explicit attn_mask, EFFICIENT/CUDNN are not implemented).
The MATH backend materializes the full ``(B, H, S, S)`` probability matrix
in both forward and backward, costing several GB at typical
``batch_size=8, seqlen=2048``.
HuggingFace transformers happily passes a *pure causal* 4D mask, even
though the same effect is achievable via ``is_causal=True`` (which the
FLASH backend supports and which uses ~12x less memory).
This monkey-patch detects a pure causal mask, drops it, and forwards
``is_causal=True`` instead -- only on XPU and only when no real mask info
would be lost. CPU/CUDA/HPU paths are left untouched.
Idempotent. Safe to call multiple times.
"""
global _xpu_sdpa_patched
if _xpu_sdpa_patched:
return
if not (hasattr(torch, "xpu") and torch.xpu.is_available()):
return
torch.use_deterministic_algorithms(False)
_orig_sdpa = torch.nn.functional.scaled_dot_product_attention
def _is_pure_causal_mask(mask: torch.Tensor, s: int) -> bool:
# Cheap shape check first
if mask.shape[-1] != s or mask.shape[-2] != s:
return False
if mask.dtype not in (torch.float16, torch.bfloat16, torch.float32):
return False
# Pick the first (B,H) slice; HF mask is broadcast across batch/heads.
m2d = mask.reshape(-1, s, s)[0]
tri_up = torch.triu(torch.ones(s, s, dtype=torch.bool, device=mask.device), 1)
return bool(torch.isinf(m2d[tri_up]).all().item()) and bool((m2d[~tri_up] == 0).all().item())
def _patched_sdpa(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None, **kwargs):
if (
query.device.type == "xpu"
and attn_mask is not None
and not is_causal
and query.shape[-2] == key.shape[-2] # square QK length (no kv-cache)
and _is_pure_causal_mask(attn_mask, query.shape[-2])
):
attn_mask = None
is_causal = True
return _orig_sdpa(
query,
key,
value,
attn_mask=attn_mask,
dropout_p=dropout_p,
is_causal=is_causal,
scale=scale,
**kwargs,
)
torch.nn.functional.scaled_dot_product_attention = _patched_sdpa
_xpu_sdpa_patched = True
logger.info("torch.use_deterministic_algorithms(False) is set for XPU.")
logger.info(
"Patched torch SDPA on XPU to use is_causal=True for pure causal masks "
"(avoids ~10x peak-VRAM blow-up from MATH backend)."
)
def check_is_cpu(device):
"""Check if the device is a CPU.
Args:
device: The device to be checked.
Returns:
bool: True if the device is a CPU, False otherwise.
"""
return device == torch.device("cpu") or device == "cpu"
def is_pipeline_parallel_supported(device_type: str) -> bool:
"""Whether multi-card (naive pipeline) parallel tuning is enabled.
Split out of ``get_device_and_parallelism`` so the parallelism policy stays a
standalone concern instead of living on the device manager. Currently only
CUDA supports multi-card pipeline parallel tuning.
"""
return device_type == "cuda"
def set_cuda_visible_devices(device: str):
if device == "cuda":
devices = ["0"]
elif device == "auto":
return
else:
devices = device.replace(" ", "").split(",")
devices = [device.split(":")[-1] for device in devices]
if all(s.isdigit() for s in devices):
if "CUDA_VISIBLE_DEVICES" in os.environ:
current_visible_devices = os.environ["CUDA_VISIBLE_DEVICES"]
current_visible_devices = current_visible_devices.split(",")
indices = [int(device) for device in devices]
try:
pick_device = [current_visible_devices[i] for i in indices]
except Exception:
raise ValueError(
"Invalid '--device' value: It must be smaller than the number of available devices."
" For example, with CUDA_VISIBLE_DEVICES=4,5, "
"--device 0,1 is valid, but --device 4,5 is not supported."
)
visible_devices = ",".join(pick_device)
os.environ["CUDA_VISIBLE_DEVICES"] = visible_devices
else:
# Use the cleaned/normalized device indices (no spaces, no type
# prefixes) when initially setting the environment variable.
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(devices)
class override_cuda_device_capability(ContextDecorator):
"""Context manager/decorator to temporarily override CUDA capability checks."""
def __init__(self, major: int = 100, minor: int = 1) -> None:
self.major = major
self.minor = minor
self._orig_func = None
def __enter__(self):
self._orig_func = torch.cuda.get_device_capability
def _override_capability(*_args, **_kwargs):
return self.major, self.minor
torch.cuda.get_device_capability = _override_capability
return self
def __exit__(self, exc_type, exc, exc_tb):
if self._orig_func is not None:
torch.cuda.get_device_capability = self._orig_func
self._orig_func = None
return False
class fake_cuda_for_hpu(ContextDecorator):
"""Context manager/decorator to fake CUDA availability for HPU devices."""
def __init__(self):
self._orig_is_available = None
def __enter__(self):
if is_hpex_available():
self._orig_is_available = torch.cuda.is_available
torch.cuda.is_available = lambda: True
return self
def __exit__(self, exc_type, exc, exc_tb):
if is_hpex_available() and hasattr(self, "_orig_is_available"):
torch.cuda.is_available = self._orig_is_available
del self._orig_is_available
return False
class fake_triton_for_hpu(ContextDecorator):
"""Context manager/decorator to fake triton availability for HPU devices."""
def __init__(self):
self._orig_triton = None
self._orig_triton_language = None
self._had_triton = False
self._had_triton_language = False
def __enter__(self):
if is_hpex_available():
# Save original state
self._had_triton = "triton" in sys.modules
self._had_triton_language = "triton.language" in sys.modules
if self._had_triton:
self._orig_triton = sys.modules["triton"]
if self._had_triton_language:
self._orig_triton_language = sys.modules["triton.language"]
# Create and inject fake triton module
class FakeTriton:
def __getattr__(self, name):
return None
fake_triton = FakeTriton()
fake_triton.jit = lambda func: func # Make triton.jit a no-op decorator
sys.modules["triton"] = fake_triton
sys.modules["triton.language"] = FakeTriton()
return self
def __exit__(self, exc_type, exc, exc_tb):
if is_hpex_available():
# Restore original state
if self._had_triton and self._orig_triton is not None:
sys.modules["triton"] = self._orig_triton
elif not self._had_triton and "triton" in sys.modules:
del sys.modules["triton"]
if self._had_triton_language and self._orig_triton_language is not None:
sys.modules["triton.language"] = self._orig_triton_language
elif not self._had_triton_language and "triton.language" in sys.modules:
del sys.modules["triton.language"]
return False
class CpuInfo:
"""Get CPU Info."""
def __init__(self):
"""Get whether the cpu numerical format is bf16, the number of sockets, cores and cores per socket."""
self._bf16 = False
info = cpuinfo.get_cpu_info()
if "arch" in info and "X86" in info["arch"]:
cpuid = cpuinfo.CPUID()
max_extension_support = cpuid.get_max_extension_support()
if max_extension_support >= 7:
eax = cpuid._run_asm(
b"\xb9\x01\x00\x00\x00", # mov ecx, 1
b"\xb8\x07\x00\x00\x00" b"\x0f\xa2" b"\xc3", # mov eax, 7 # cpuid # ret
)
self._bf16 = bool(eax & (1 << 5))
@property
def bf16(self):
"""Get whether it is bf16."""
return self._bf16
def bytes_to_gigabytes(bytes) -> int:
"""
Converts bytes to gigabytes.
Args:
bytes (int): The number of bytes.
Returns:
int: The equivalent number of gigabytes.
"""
return bytes / 1024 / 1024 / 1024
_malloc_trim_counter = 0
def _force_trim_malloc() -> None:
"""Unconditionally release glibc heap pages back to the OS on Linux.
Unlike :func:`_maybe_trim_malloc`, this ignores the call-count throttle and
always invokes ``malloc_trim(0)``. Use at critical lifecycle boundaries
(end of model loading, end of post_init, start of quantize loop) where a
one-time trim has a meaningful impact on peak RSS.
"""
if os.name != "posix":
return
if os.environ.get("AR_ENABLE_MALLOC_TRIM", "1") != "1":
return
try:
libc = ctypes.CDLL("libc.so.6")
libc.malloc_trim(0)
except Exception:
pass
def _maybe_trim_malloc() -> None:
"""Optionally release glibc heap pages back to OS on Linux.
Controlled by environment variables:
- AR_ENABLE_MALLOC_TRIM: default "1" (enabled)
- AR_MALLOC_TRIM_EVERY: default "10" (trigger every N clear_memory calls)
"""
global _malloc_trim_counter
if os.name != "posix":
return
if os.environ.get("AR_ENABLE_MALLOC_TRIM", "1") != "1":
return
try:
every = int(os.environ.get("AR_MALLOC_TRIM_EVERY", "10"))
except ValueError:
every = 10
every = max(1, every)
_malloc_trim_counter += 1
if _malloc_trim_counter % every != 0:
return
try:
libc = ctypes.CDLL("libc.so.6")
libc.malloc_trim(0)
except Exception:
pass
def clear_memory_if_reached_threshold(threshold=0.85, device_list=None):
"""Check all available devices and clear memory if any device is using close to the threshold.
Args:
threshold (float): Memory usage threshold (default: 0.85 for 85%).
If any device exceeds this percentage, clear_memory() will be called.
Returns:
bool: True if memory was cleared, False otherwise.
"""
# Detect the active device (CUDA/XPU/HPU/...)
dev_mgr = get_current_device_manager()
if not dev_mgr.is_available() or dev_mgr.type == "cpu":
return False
name = dev_mgr.type
num_devices = dev_mgr.device_count()
for i in range(num_devices):
try:
total_memory = dev_mgr.total_memory(i)
reserved_memory = dev_mgr.memory_reserved(i)
memory_usage_ratio = reserved_memory / total_memory
if memory_usage_ratio >= threshold:
logger.warning_once(
f"Major device ({name}:{i}) has reached memory threshold. "
+ "Memory clearing operation will be called during each iteration, which "
+ "will result in more time consumption."
)
logger.warning_once(
"To alleviate high memory usage on the major device, consider reducing the `batch_size` "
+ "(and correspondingly increasing `gradient_accumulation_steps) or shortening the seqlen."
)
clear_memory(device_list=device_list)
return True
except Exception as e:
logger.warning_once(f"Failed to check memory for {name}:{i}: {e}")
return False
def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):
"""Checks the availability of memory on the specified device for processing inputs using a given weight tensor.
Args:
device (str): The device type ('cuda' for GPU or 'hpu' for HPU).
inputs (torch.Tensor): Input tensor.
weight (torch.Tensor): Weight tensor.
org_seqlen (int): Original sequence length.
org_bs (int): Original batch size.
Returns:
tuple: A tuple containing availability status (bool), modified sequence length (int),
and modified batch size (int).
"""
weight_memory = weight.numel() * weight.element_size()
device_type = str(device).split(":")[0]
if device_type in ("cpu", "") or not get_ar_device(device_type).is_available():
return True, org_seqlen, org_bs
dev_mgr = get_ar_device(device_type)
current_index = dev_mgr.current_device()
free_space, _ = dev_mgr.mem_get_info(current_index)
free_space = free_space - weight_memory * 10 # for min_max_scale & grad usage
seqlen = org_seqlen
bs = org_bs
in_feature = weight.shape[1]
out_feature = weight.shape[0]
while seqlen >= 128:
input_size = bs * seqlen * in_feature
output_size = bs * seqlen * out_feature
input_output_memory = 2 * (input_size * inputs.element_size() + output_size * inputs.element_size())
if input_output_memory < free_space:
return True, seqlen, bs
seqlen = seqlen // 2
bs = 1
return False, seqlen, bs
def set_tuning_device_for_layer(model, name: str, device: str) -> None:
"""Sets the device for a module if it matches the given name."""
module = get_module(model, name)
if hasattr(module, "tuning_device") and module.tuning_device != device:
logger.warning(
f"multiple devices have been set for layer {name}, keeping original device {module.tuning_device}"
)
else:
module.tuning_device = device
def set_non_auto_device_map(
model: torch.nn.Module, device_map: str | int | dict, quant_layer_names: None | list | tuple = None
) -> None:
if not device_map or device_map == "auto" or isinstance(device_map, int):
return
if isinstance(device_map, str):
if "," in device_map: # auto device map
return
device_map = device_map.replace(" ", "")
infos = device_map.split(",")
device_map_dict = {}
for info in infos:
if ":" not in info:
continue
index = info.find(":")
key = info[:index]
value = info[index + 1 :]
device_map_dict[key] = value
device_map = device_map_dict
if quant_layer_names is not None:
names = quant_layer_names
else:
names = [
n for n, m in model.named_modules() if len(list(m.children())) == 0
] # if it's a block, it will be incorrect
for key, device in device_map.items():
if isinstance(device, str) and device.isdigit():
device = int(device)
device = get_major_device(device)
if key in names:
module = get_module(model, key)
module.tuning_device = device
else:
matching_names = [name for name in names if re.match(key, name)]
for name in matching_names:
set_tuning_device_for_layer(model, name, device)
if not matching_names:
logger.warning(f"{key} in `device_map` dose not match any modules, please have a check")
def _resolve_moe_chunk() -> int:
"""The configured experts-per-chunk (``AR_MOE_CHUNK``), for device co-allocation.
``auto`` / invalid -> the default fixed chunk (16); ``<=0`` means "whole container"
(fuse everything -> a container is only ever placed as a single device-local unit).
"""
from auto_round import envs
try:
return int(str(envs.AR_MOE_CHUNK))
except (TypeError, ValueError):
return 16
def _preassign_moe_experts(
layer_memory_dict: dict, device_memory: dict, gpu_devices: list, mem_per_param: float
) -> dict:
"""Place a MoE experts container's experts chunk-locally so the grouped path stays on.
The grouped experts forward only needs the experts *of one fused chunk* to sit on the same
device (else that chunk falls back to the slow per-expert loop, which also churns
``torch.compile`` recompiles). This pre-pass keeps each ``AR_MOE_CHUNK``-aligned group whole
on one device while spreading the groups across cards, so memory stays balanced across GPUs
instead of piling a whole (potentially hundreds of experts) container onto a single card.
* Per chunk -- place each ``AR_MOE_CHUNK``-aligned expert group whole on the device with
the most free memory that fits it, so grouped GEMM stays single-device within a chunk.
* Overflow -- groups that do not fit on any device are left unassigned here and fall
through to the general load balancer (grouped will loop-fallback for those layers).
Relies on the per-card memory budgets (``device_memory``) being set correctly by the caller
(symmetric transient reservation); otherwise best-fit repeatedly picks the same card and
piles experts there. Only placement is affected -- correctness is unchanged (the forward's
device checks and AlignDevicesHook handle whatever layout results). ``device_memory`` is
decremented in place for the layers this function assigns.
Returns a ``{layer_name: device}`` map for the experts it placed.
"""
# Group expert projections by their container -> {expert_idx: [(name, mem_info), ...]}.
containers: dict[str, dict[int, list]] = {}
for name, info in layer_memory_dict.items():
if not info.get("is_moe_expert"):
continue
head, _, _slot = name.rpartition(".") # ".../<container>.<idx>.<slot>"
container, _, idx_str = head.rpartition(".")
try:
idx = int(idx_str)
except ValueError:
continue # unexpected naming; leave it to the general balancer
containers.setdefault(container, {}).setdefault(idx, []).append((name, info))
assigned: dict[str, str] = {}
if not containers:
return assigned
def mem_of(items) -> float:
return sum(mem_info["param_memory"] * mem_per_param for _, mem_info in items)
def best_fit(required: float):
candidates = [d for d in gpu_devices if device_memory[d] >= required]
return max(candidates, key=lambda d: device_memory[d]) if candidates else None
def place(items, device) -> None:
for layer_name, _ in items:
assigned[layer_name] = device
chunk = _resolve_moe_chunk()
for container, by_idx in containers.items():
idxs = sorted(by_idx)
# Spread chunk-aligned groups across cards: each group stays whole on one device (so a
# chunk's grouped GEMM is single-device) but different groups balance across GPUs.
# ``chunk <= 0`` ("fuse everything") keeps the whole container as one group.
# On overflow the chunk SHRINKS (halving, floor 1 = a single expert whole) instead of
# dropping to the per-Linear balancer: dropped chunks are what scatter one expert's
# gate/up/down across devices and force the grouped path to fall back to the loop.
step = len(idxs) if chunk <= 0 else chunk
start = 0
while start < len(idxs):
take = min(step, len(idxs) - start)
group_items = [item for i in idxs[start : start + take] for item in by_idx[i]]
group_mem = mem_of(group_items)
group_device = best_fit(group_mem)
if group_device is None:
if take <= 1:
logger.debug(
"[moe-prepass] single expert of %s fits no device budget; leaving for the balancer", container
)
start += 1
continue
step = max(1, step // 2)
continue
place(group_items, group_device)
device_memory[group_device] -= group_mem
start += take
return assigned
def _allocate_layers_to_devices(
layer_memory_dict: dict, device_memory: dict, gpu_devices: list, mem_per_param: float
) -> tuple[dict, list]:
"""
Allocates layers to devices using a load-balancing strategy.
Strategy:
0. MoE pre-pass: keep each ``AR_MOE_CHUNK``-aligned expert group whole on one device while
spreading the groups across cards, so a chunk's grouped GEMM stays single-device without
piling a whole experts container onto one card.
1. Sort remaining layers by memory size (descending), preserve order for equal sizes
2. Assign largest N layers to higher-index devices (N = num_devices)
3. Remaining layers use memory availability + layer continuity scorings
Args:
layer_memory_dict (dict): Mapping of layer names to memory info (order preserved)
device_memory (dict): Available memory for each device (will be modified)
gpu_devices (list): List of device names (e.g., ["cuda:0", "cuda:1"])
mem_per_param (float): Memory multiplier per parameter GB
Returns:
tuple[dict, list]: (device_map, names)
"""
device_map = {}
names = []
layer_names_in_order = list(layer_memory_dict.keys())
layer_order = {name: idx for idx, name in enumerate(layer_names_in_order)}
# Phase 0: keep each MoE expert chunk device-local (grouped GEMM stays single-device, avoids
# the per-expert loop-fallback and its torch.compile recompiles) while spreading chunks across
# cards. Depends on the caller's symmetric per-card budgets so best-fit alternates cards
# instead of piling every chunk onto one card.
preassigned = _preassign_moe_experts(layer_memory_dict, device_memory, gpu_devices, mem_per_param)
device_map.update(preassigned)
names.extend(preassigned.keys())
# The general balancer handles everything the pre-pass did not place.
remaining = {name: layer_memory_dict[name] for name in layer_names_in_order if name not in preassigned}
sorted_layers = sorted(remaining.items(), key=lambda x: (-x[1]["param_memory"], -layer_order[x[0]]))
num_devices = len(gpu_devices)
def find_best_device(layer_name: str, estimated_memory: float, layer_idx: int, strict: bool = False) -> str:
"""Find the best device for a layer.
strict=True skips the Phase-1 shortcut (used by the atomic-group
allocator so every unit goes through budget-aware scoring). When no
device passes the score filter, the device with the most remaining
budget still takes the unit: estimates are balancing reservations and
real overflow is owned by the per-layer OOM handlers.
"""
# Phase 1: Direct assign largest layers to higher-index devices first.
# Skipped under strict: the atomic allocator's fail-loudly contract must
# hold for the LARGEST units too, and this shortcut assigns the first
# num_devices-1 units to fixed devices without any budget check.
if layer_idx < num_devices - 1 and not strict:
return gpu_devices[-(layer_idx + 1)]
# Phase 2: Choose device with best score (memory + continuity)
best_device = None
best_score = float("-inf")
current_layer_order = layer_order[layer_name]
for device in gpu_devices:
if device_memory[device] < estimated_memory:
continue
# Memory score (normalized)
memory_score = device_memory[device] / estimated_memory
# Continuity bonus for adjacent layers
continuity_bonus = 0
for offset in [-1, 1]: # Check previous and next neighbors
neighbor_idx = current_layer_order + offset
if 0 <= neighbor_idx < len(layer_names_in_order):
neighbor_name = layer_names_in_order[neighbor_idx]
if neighbor_name in device_map and device_map[neighbor_name] == device:
continuity_bonus += 1.0
total_score = memory_score + continuity_bonus
if total_score > best_score:
best_score = total_score
best_device = device
# Fallback: device with most available memory. Estimates are
# balancing reservations, never admission control -- when nothing
# fits by score, the max-remaining device gets the unit and the
# real OOM handlers own any actual overflow.
return best_device or max(gpu_devices, key=lambda d: device_memory[d])
# Allocate the remaining (non-preassigned) layers in ATOMIC GROUPS: leaves under the
# same indexed container (e.g. ``experts.7.gate/up/down``) are one indivisible unit —
# the streaming lane's ``_atomic_groups`` pattern — so the balancer can never split an
# expert across devices even when the MoE pre-pass left it behind.
import re as _re
_indexed = _re.compile(r"^(.*)\.(\d+)\.[^.]+$")
_groups: dict = {}
_group_order: list = []
for layer_name in remaining:
m = _indexed.match(layer_name)
key = f"{m.group(1)}.{m.group(2)}" if m else layer_name
if key not in _groups:
_groups[key] = []
_group_order.append(key)
_groups[key].append(layer_name)
grouped_units = [
(key, sum(remaining[n]["param_memory"] for n in _groups[key]), min(layer_order[n] for n in _groups[key]))
for key in _group_order
]
grouped_units.sort(key=lambda u: (-u[1], -u[2]))
for layer_idx, (key, unit_param_memory, _order) in enumerate(grouped_units):
members = _groups[key]
estimated_memory = unit_param_memory * mem_per_param
best_device = find_best_device(members[0], estimated_memory, layer_idx, strict=True)
if device_memory[best_device] < estimated_memory:
# The proportional share is a balancing reservation, not a
# footprint (sum of shares == the whole pool, so fat units can
# exhaust every remaining budget on dense blocks) -- placed
# anyway; the OOM ladder owns real overflows.
logger.debug(
"atomic module group '%s' share %.2f GB exceeds %s's remaining budget; placed for best fit",
key,
estimated_memory,
best_device,
)
for layer_name in members:
names.append(layer_name)
device_map[layer_name] = best_device
device_memory[best_device] -= estimated_memory
# Restore original order
ordered_device_map = {name: device_map[name] for name in layer_names_in_order if name in device_map}
return ordered_device_map, names
def get_first_available_attr(obj, attr_names: list[str], default=None):
"""
Get the first available attribute from a list of attribute names.
Args:
obj: The object to get the attribute from.
attr_names (list[str]): List of attribute names to try in order.
default: Default value to return if none of the attributes exist.
Returns:
The value of the first available attribute, or default if none exist.
"""
for attr_name in attr_names:
value = getattr(obj, attr_name, None)
if value is not None:
return value
return default
def get_moe_memory_ratio(block: torch.nn.Module, config: Any = None) -> float:
"""
Calculate the memory ratio for MoE (Mixture of Experts) models.
For MoE models, only num_experts_per_tok experts are activated per token,
not all experts. This function returns the ratio of active experts to total experts.
Args:
block (torch.nn.Module): The model block to analyze.
config: Model config to read when the block itself carries none
(block-wise tuning blocks have no ``.config``; without this the
ratio silently falls back to 1.0 and prices ALL experts' outputs
at full width -- an order-of-magnitude overcharge on models with
many sparsely-active experts).
Returns:
float: Memory ratio (num_experts_per_tok / num_experts).
Returns 1.0 for non-MoE models.
bool: True if the model is MoE, False otherwise.
Examples:
- Non-MoE model: returns 1.0
- Mixtral (2/8 experts): returns 0.25
- Qwen2MoE (4/60 experts): returns ~0.067
"""
from auto_round.utils.model import is_moe_layer
for name, module in block.named_modules():
if not is_moe_layer(module):
continue
config = config if config is not None else getattr(block, "config", None)
if config is None:
# container-attr fallback before giving up: expert count from the
# container itself, active count from its top_k-ish attrs
num_experts = getattr(module, "num_experts", None)
active = getattr(module, "num_experts_per_tok", None) or getattr(module, "top_k", None)
if isinstance(num_experts, int) and num_experts > 0 and isinstance(active, int) and active > 0:
return active / num_experts, True
break
# Try to get num_experts_per_tok (active experts count)
# Mixtral, Qwen2MoE, DeepSeek, GPT-OSS, Llama4, LLaDAMoE, SmallThinker
num_experts_per_tok = get_first_available_attr(
config, ["num_experts_per_tok", "moe_num_active_primary_experts"]
)
# HunYuan MoE uses moe_topk (array), get first element
if num_experts_per_tok is None:
moe_topk = getattr(config, "moe_topk", None) # HunYuan MoE V1
if moe_topk is not None and isinstance(moe_topk, (list, tuple)) and len(moe_topk) > 0:
num_experts_per_tok = moe_topk[0]
elif moe_topk is not None:
num_experts_per_tok = moe_topk
if num_experts_per_tok is None:
break
# Get total number of experts
# Mixtral, PhiMoE, Grok, Llama4, Qwen2MoE, Olmo, BailingMoE, GroveMoE, HunYuan, LLaDAMoE, SmallThinker, DeepSeek
num_experts = get_first_available_attr(
config, ["num_local_experts", "num_experts", "moe_num_primary_experts", "n_routed_experts"]
)
if num_experts is not None and num_experts > 0:
moe_ratio = num_experts_per_tok / num_experts
logger.debug(
f"Using MoE memory ratio: {moe_ratio:.4f} ({num_experts_per_tok}/{num_experts} experts active per token)"
)
return moe_ratio, True
break # Only check once per block
return 1.0, False # Default ratio for non-MoE models
def estimate_tuning_block_mem(
block: torch.nn.Module,
input_ids: Any,
batch_size: int,
config: Any = None,
) -> tuple[dict[str, Any], float, float, float, dict[str, float], dict[str, int]]:
"""
Calculates the memory consumption of a specific block in the model.
Args:
block (torch.nn.Module): The block of the model to analyze.
input_ids: Cached block inputs as a tensor, a list of tensors, or a dictionary containing
``hidden_states`` and auxiliary block inputs.
batch_size (int): Number of samples to consider for memory estimation.
Returns:
tuple: A 6-tuple containing:
- layer_memory_dict (dict): A dictionary mapping layer names to their memory consumption (in GB).
Format: {layer_name: {"param_memory": float, "output_memory": float}}
- layer_activation_memory (float): Sum of per-layer output memory (GB), expert outputs
ratio-scaled by active/total experts.
- input_output_memory (float): The memory consumption (in GB) for input and output
tensors of the block.
- additional_memory (float): Additional memory overhead (in GB) for operations like attention.
- per_device_activation (dict): {device: GB} of output+grad charges bucketed by weight home.
- per_device_experts (dict): {device: count} of routed-expert leaves homed per device.
"""
# Calculate all block parameters memory and build layer-wise memory dict
from auto_round.utils.model import get_layer_features, is_moe_layer
layer_memory_dict = {}
def iter_input_tensors(value):
if isinstance(value, torch.Tensor):
yield value
elif isinstance(value, dict):
for nested_value in value.values():
yield from iter_input_tensors(nested_value)
elif isinstance(value, (list, tuple)):
for nested_value in value:
yield from iter_input_tensors(nested_value)
input_tensors = list(iter_input_tensors(input_ids))
hidden_states = input_ids.get("hidden_states") if isinstance(input_ids, dict) else input_ids
reference_tensor = next(iter_input_tensors(hidden_states), input_tensors[0] if input_tensors else None)
# Calculate sequence length and dtype from hidden states for output memory estimation.
seq_len = reference_tensor.shape[1] if reference_tensor is not None and reference_tensor.ndim >= 2 else 1
element_size = reference_tensor.element_size() if reference_tensor is not None else 2
moe_ratio, has_moe = get_moe_memory_ratio(block, config) # ratio 1.0 for non-MoE models
for name, module in block.named_modules():
if check_to_quantized(module):
# V2 wrappers keep bits/act_bits on orig_layer (check_to_quantized
# reads them there); reading them off the wrapper itself raises
enable_act_quant = (
getattr(getattr(module, "orig_layer", module), "act_bits", getattr(module, "act_bits", 16)) <= 8
)
layer_name = name
param_size = module.weight.nbytes
param_memory_gb = param_size / 1024**3
param_memory_gb *= 2 # considering the v tensor for weight rounding
# Estimate output memory based on input_features and out_features
in_features, out_features = get_layer_features(module)
if in_features is not None and out_features is not None:
# Output tensor size: batch_size * seq_len * out_features * element_size
output_size = batch_size * seq_len * out_features * element_size
output_memory_gb = output_size / 1024**3
# If enable_act_quant, add input tensor memory to param_memory
if enable_act_quant:
input_size = batch_size * seq_len * in_features * element_size
input_memory_gb = input_size / 1024**3
param_memory_gb += input_memory_gb
else:
output_memory_gb = 0.0
if has_moe:
pparent_module = get_module(block, layer_name.rsplit(".", 2)[0]) if "." in layer_name else block
# Our unfuse builds the experts module as a plain nn.Module with numbered
# _ExpertContainer children (checkpoint-format keys), NOT an nn.ModuleList;
# the num_experts attribute identifies it just as well.
# The experts container itself may be a plain class with a
# generic name (neither ModuleList nor num_experts, and no
# "moe" in ITS class name) -- the MoE marker sits on an
# ANCESTOR. Walk the chain; a single-parent check prices
# every expert full-width (an order-of-magnitude overcharge
# on models with many sparsely-active experts).
is_experts_container = isinstance(pparent_module, torch.nn.ModuleList) or hasattr(
pparent_module, "num_experts"
)
if not is_experts_container:
_and = layer_name
while "." in _and and not is_experts_container:
_and = _and.rsplit(".", 1)[0]
_mod = get_module(block, _and) if _and else None
if _mod is not None and (
isinstance(_mod, torch.nn.ModuleList) or hasattr(_mod, "num_experts") or is_moe_layer(_mod)
):
is_experts_container = True
# shared experts are plain modules outside the dispatch (same
# exclusion the repo's is_moe_expert predicate applies)
is_moe_expert = (
"expert" in layer_name.lower() and "shared" not in layer_name.lower() and is_experts_container
)
else:
is_moe_expert = False
# memory * 2, because it contains grad tensor.
layer_memory_dict[layer_name] = {
"param_memory": param_memory_gb * 2,
"output_memory": output_memory_gb * 2,
"is_moe_expert": is_moe_expert,
# weight home: fwd/bwd executes where the weight lives, so the
# saved output + grad transient lands on this device
"device": str(module.weight.device),
}
# Assuming bfloat16 or float32, input and output
block_input_output_memory = 2 * sum(tensor.nbytes for tensor in input_tensors) / 1024**3
# Roughly estimate additional memory for attention and other operations
# For MoE expert layers, multiply activation memory by the ratio of active experts
# For non-MoE layers (attention, norm, etc.), use full activation memory
layer_activation_memory = 0.0
per_device_activation = {}
per_device_experts = {}
for layer_name, info in layer_memory_dict.items():
if info.get("is_moe_expert", False):
# MoE expert layer: only a fraction of experts are active
layer_activation_memory += info["output_memory"] * moe_ratio
per_device_activation[info["device"]] = (
per_device_activation.get(info["device"], 0.0) + info["output_memory"] * moe_ratio
)
per_device_experts[info["device"]] = per_device_experts.get(info["device"], 0) + 1
else:
# Non-MoE layer: use full activation memory
layer_activation_memory += info["output_memory"]
per_device_activation[info["device"]] = (
per_device_activation.get(info["device"], 0.0) + info["output_memory"]
)
# layer_activation_memory considers other ops activation memory
# 1GB considers norm weight, sdpa, reference_output, etc.
additional_memory = layer_activation_memory + 1 # GB
if has_moe:
# TODO: Cannot estimate the memory usage correctly for MoE models yet.
# For MoE models, additional memory usage can be higher due to routing, gating,
# and multiple expert activations. Here we use a conservative estimate.
moe_additional_memory = additional_memory * 6 # GB
additional_memory += moe_additional_memory
return (
layer_memory_dict,
layer_activation_memory,
block_input_output_memory,
additional_memory,
per_device_activation,
per_device_experts,
)
def set_auto_device_map_for_block_with_tuning(
block: torch.nn.Module,
device_list,
input_ids: list[torch.Tensor],
low_gpu_mem_usage: bool = False,
batch_size: int = 8,
output_device: str | torch.device = None,
card_0_threshold: float = 0.9,
):
"""
Automatically sets the device map for the block based on available GPUs and memory constraints.
Args:
block (torch.nn.Module): The model block whose device map is to be set.
device_list (str | int | dict): Specifies the device mapping.
input_ids (list[torch.Tensor]): List of input tensors used for estimating memory requirements.
low_gpu_mem_usage (bool, optional): If True, ignoring input/output memory. Defaults to False.
batch_size (int, optional): Number of samples to consider for memory estimation. Defaults to 8.
output_device (str | torch.device, optional): Device to move unassigned modules to. Defaults to None.
card_0_threshold (float, optional): Threshold ratio to determine if the first device is at high risk of
running out of memory. Defaults to 0.9 (90%).
Returns:
card_0_in_high_risk (bool): True if the first device is at risk of running out of memory, False otherwise.
card_0_in_high_risk = card_0_used_memory / device_0_memory > card_0_threshold
card_0_used_memory = card_0_left_memory + block_input_output_memory + additional_memory
We may need to clear card 0 memory more frequently during training/inference in that case.
Raises:
RuntimeError: If no CUDA or XPU devices are found.
Note:
This function is intended for internal use in device memory management and tuning.
"""
card_0_in_high_risk, loss_device = False, output_device
from auto_round.utils.model import (
_module_manages_own_device,
module_pinned_execution_device,
move_to_device_preserving_cpu_pinned,
place_ngram_embeddings_for_tuning_,
)
# Distribute huge, non-quantizable ngram embeddings before any block placement:
# row-shard them across the GPUs when >1 is available (on-device lookup, no card-0 OOM and
# far faster than a CPU-pinned table), otherwise keep them pinned on CPU. Must run first.
placed = place_ngram_embeddings_for_tuning_(block, list(device_list) if device_list else None)
if placed:
logger.debug(f"Placed {len(placed)} ngram embedding module(s) for tuning: {placed}")
dev_mgr = get_current_device_manager()
if dev_mgr.is_available() and dev_mgr.type != "cpu":
num_devices = dev_mgr.device_count()
device_name = dev_mgr.type
else:
return card_0_in_high_risk, loss_device
if len(device_list) <= 1: # Only 1 card is available or non-auto device map
move_to_device_preserving_cpu_pinned(block, output_device)
return card_0_in_high_risk, loss_device
if device_list:
gpu_devices = device_list
device_0 = gpu_devices[0]
device_1 = gpu_devices[1]
else:
gpu_devices = [f"{device_name}:{i}" for i in range(num_devices)]
device_0 = f"{device_name}:0"
device_1 = f"{device_name}:1"
device_0_memory = get_device_memory(device_list[0] if device_list else 0)
device_1_memory = get_device_memory(device_list[1] if device_list else 1)
layer_memory_dict, layer_activation_memory, block_input_output_memory, additional_memory, _, _ = (
estimate_tuning_block_mem(block, input_ids, batch_size)
)
loss_memory = block_input_output_memory / 2 # GB, rough estimate for loss tensor memory
if low_gpu_mem_usage:
block_input_output_memory = 0
total_block_param_memory = sum(info["param_memory"] for info in layer_memory_dict.values())
# Average dispatch strategy
# card_0_left_memory = card_0_mem - block_input_output_memory - additional_memory - layer_outputs_memory
card_0_used_memory = block_input_output_memory + layer_activation_memory + additional_memory
logger.debug(f"Card 0 used memory details [Estimated]: {card_0_used_memory} GB")
logger.debug(f" Block input output cache memory: {block_input_output_memory} GB")
logger.debug(f" Quantized layer outputs memory: {layer_activation_memory} GB")
logger.debug(f" Additional_memory from other ops: {additional_memory} GB")
card_0_left_memory = max(0, (device_0_memory - card_0_used_memory))
card_0_in_high_risk = card_0_used_memory / device_0_memory >= card_0_threshold
card_1_left_memory = max(0, device_1_memory - loss_memory) if card_0_in_high_risk else device_1_memory
loss_device = device_1 if card_0_in_high_risk else output_device
if not layer_memory_dict:
output_device = device_0 if output_device is None else output_device
move_to_device_preserving_cpu_pinned(block, output_device)
logger.debug(f"No layers require tuning; moved the block to {output_device}")
return card_0_in_high_risk, loss_device
# Calculate total available memory across all devices
total_available_memory = card_0_left_memory + card_1_left_memory
for i in range(2, len(gpu_devices)):
device_idx = device_list[i] if device_list else i
total_available_memory += get_device_memory(device_idx)
# Calculate total params (in GB, considering param_memory only for calculation)
total_params = total_block_param_memory
mem_per_param = total_available_memory / total_params
# Initialize device memory tracking
device_memory = {device_0: card_0_left_memory}
for i in range(1, len(gpu_devices)):
device_idx = device_list[i] if device_list else i
device_memory[gpu_devices[i]] = get_device_memory(device_idx)
# Allocate layers to devices using load-balancing strategy
device_map, names = _allocate_layers_to_devices(layer_memory_dict, device_memory, gpu_devices, mem_per_param)
logger.debug(f"Auto device map for block: {device_map}")
set_non_auto_device_map(block, device_map, names)
# Ensure all remaining modules with parameters/buffers are moved to expected device, by default device_0
output_device = device_0 if output_device is None else output_device
for name, module in block.named_modules():
if name in names: # This module was already assigned a device
continue
if module_pinned_execution_device(module) is not None:
# Intentionally pinned to a fixed execution device (CPU, or a specific card for
# huge non-quantizable ngram embeddings); never relocate it or it could OOM / break
# the chosen placement.
continue
if _module_manages_own_device(module):
# Places its own (possibly multi-device) storage, e.g. a row-sharded ngram
# embedding split across GPUs; moving its buffers here would collapse the shards.
continue
# Move only this module's OWN tensors (recurse=False). Using module.to()
# would recurse and drag a CPU-pinned child (e.g. a nested ngram
# embedding) back onto the accelerator. Because named_modules() visits
# every module, each tensor is still covered exactly once here.
for _, param in module.named_parameters(recurse=False):
if param.device.type != "meta":
param.data = param.data.to(output_device)
for _, buf in module.named_buffers(recurse=False):
if buf.device.type != "meta":
buf.data = buf.data.to(output_device)
return card_0_in_high_risk, loss_device
def partition_dict_numbers(number_dict, n):
"""
Partition a dictionary of numbers into N groups with approximately equal sums
"""
# Quick handling of edge cases
if n <= 0:
return []
items = list(number_dict.items())
m = len(items)
if n >= m:
# Put each item in its own bucket, then pad empty buckets
groups = [{k: v} for k, v in items]
groups.extend({} for _ in range(n - m))
return groups
# Greedy largest-first balancing (Longest Processing Time / LPT-like):
# - Sort items by value descending
# - Assign each item to the group with the current smallest sum
# Complexity: O(m log n) which scales well for large m (layers)
groups_sums = [0.0] * n
groups = [{} for _ in range(n)]
# Sort items descending by size
items_sorted = sorted(items, key=lambda kv: kv[1], reverse=True)
for key, val in items_sorted:
# choose the group with minimum current sum
idx = min(range(n), key=lambda i: groups_sums[i])
groups[idx][key] = val
groups_sums[idx] += val
return groups
def dispatch_model_block_wise(model: torch.nn.Module, device_map: str, max_mem_ratio=0.9):
if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:
import accelerate
accelerate.hooks.remove_hook_from_submodules(model)
no_split_modules = getattr(model, "_no_split_modules", [])
devices = parse_available_devices(device_map)
if len(devices) == 1:
model.to(devices[0])
return model
max_memory = get_max_memory()
new_max_memory = {}
if "cpu" not in devices:
devices.append("cpu")
for device in devices:
if ":" in device:
device = int(device.split(":")[-1])
elif device == "cpu":
device = "cpu"
elif isinstance(device, str):
device = 0
else:
raise ValueError(f"Unsupported device {device} in device_map: {device_map}")
# Use 90% of the reported max memory to leave headroom for activations,
# temporary tensors, other processes, and allocator fragmentation, reducing
# the chance of runtime OOM while still utilizing most available memory.
new_max_memory[device] = max_memory[device] * max_mem_ratio
new_max_memory = get_balanced_memory(
model,
max_memory=new_max_memory,
no_split_module_classes=no_split_modules,
)
if hasattr(model, "tie_weights"):
model.tie_weights()
device_map = infer_auto_device_map(model, max_memory=new_max_memory, no_split_module_classes=no_split_modules)
if len(devices) > 1 and "cpu" in device_map.values():
logger.warning(
"Some layers are offloaded to cpu, which may severely impact calibration speed."
" Please consider using more cards."
)
model = dispatch_model(model, device_map=device_map)
return model
def set_avg_auto_device_map(model: torch.nn.Module, device_map):
block_name_list = get_block_names(model)
device_list = parse_available_devices(device_map)
gpu_devices = []
for device in device_list:
if device.startswith("hpu") and len(device_list) > 1:
logger.warning_once("Auto-scheme does not support multiple HPUs.")
if device.startswith(("cpu", "hpu")):
continue
gpu_devices.append(device)
num_devices = len(gpu_devices)
if num_devices <= 1:
return
for block_names in block_name_list:
for block_name in block_names:
params_dict = {}
block_module = get_module(model, block_name)
for n, m in block_module.named_modules():
in_features, out_features = get_layer_features(m)
if in_features is None:
continue
params_dict[n] = in_features * out_features
res_list = partition_dict_numbers(params_dict, num_devices)
for device_index, res in enumerate(res_list):
for key in res.keys():
set_tuning_device_for_layer(block_module, key, gpu_devices[device_index])
if __name__ == "__main__":
# Example usage
number_dict = {"item1": 90, "item2": 20, "item3": 30, "item4": 40, "item5": 50, "item6": 60}
groups = partition_dict_numbers(number_dict, 10)
for i, group in enumerate(groups):
print(f"Group {i + 1}: {group}, Sum: {sum(group.values())}")
groups = partition_dict_numbers(number_dict, 6)
for i, group in enumerate(groups):
print(f"Group {i + 1}: {group}, Sum: {sum(group.values())}")
groups = partition_dict_numbers(number_dict, 4)
for i, group in enumerate(groups):
print(f"Group {i + 1}: {group}, Sum: {sum(group.values())}")
groups = partition_dict_numbers(number_dict, 3)
for i, group in enumerate(groups):
print(f"Group {i + 1}: {group}, Sum: {sum(group.values())}")
groups = partition_dict_numbers(number_dict, 2)
for i, group in enumerate(groups):
print(f"Group {i + 1}: {group}, Sum: {sum(group.values())}")
def parse_available_devices(device_map: str | torch.device | int | dict | None = None) -> list:
"""
Parse the device map and return a list of all available devices.
Supported input formats:
- None: Automatically detect all available devices
- int: A single device index (e.g., 0)
- str: Examples:
"cpu"
"cuda:0,cuda:1"
"0,1" (numeric device indices)
- dict: Extract all device values from the dictionary
- torch.device: e.g. torch.device("cuda:0")
Returns:
list[str]: Normalized device names, e.g., ["cuda:0", "cuda:1"] or ["cpu"]
"""
# === Step 1. Detect available device types ===
device_types = get_available_device_types()
# Always include CPU as a fallback
if not device_types:
device_types = ["cpu"]
# === Step 2. Parse different input formats ===
if device_map is None:
# Automatically detect one available device
if "cuda" in device_types:
return ["cuda:0"]
elif "xpu" in device_types:
return ["xpu:0"]
elif "hpu" in device_types:
return ["hpu:0"]
else:
return ["cpu"]
if isinstance(device_map, torch.device):
# Handle torch.device objects
dev_type = device_map.type
index = device_map.index
if dev_type == "cpu":
return ["cpu"]
if index is None:
index = 0
return [f"{dev_type}:{index}"]
if isinstance(device_map, int):
# Integer input → use primary available device type
device_type = device_types[0]
return [f"{device_type}:{device_map}"] if device_type != "cpu" else ["cpu"]
# ---- dict-like string ----
if isinstance(device_map, str) and ":" in device_map and "," in device_map:
pairs = [p.strip() for p in device_map.split(",") if ":" in p]
devices = []
for pair in pairs:
try:
key, *value_parts = pair.split(":")
value = ":".join(value_parts).strip()
if value.isdigit() and device_types[0] != "cpu":
value = device_types[0] + ":" + value
devices.append(value)
except ValueError:
continue
return devices
if isinstance(device_map, str):
# Remove whitespace
device_map = device_map.strip()
if device_map.lower() == "cpu":
return ["cpu"]
if device_map.lower() == "auto":
device_count = detect_device_count()
if "cuda" in device_types:
return [f"cuda:{i}" for i in range(device_count)]
elif "xpu" in device_types:
return [f"xpu:{i}" for i in range(device_count)]
elif "hpu" in device_types:
return [f"hpu:{i}" for i in range(device_count)]
else:
return ["cpu"]
# Split by commas
parts = [x.strip() for x in device_map.split(",") if x.strip()]
parsed = []
for p in parts:
if p.isdigit():
# Numeric → assign to first available device type
device_type = device_types[0]
parsed.append(f"{device_type}:{p}" if device_type != "cpu" else "cpu")
elif p in device_types and ":" not in p:
# Bare device type string (e.g., "cuda", "xpu") → normalize to "type:0".
# Special-case CPU: keep it as "cpu" without an index.
if p.lower() == "cpu":
parsed.append("cpu")
else:
parsed.append(f"{p}:0")
else:
parsed.append(p)
# Preserve user-specified device order while removing duplicates.
return list(dict.fromkeys(parsed))
if isinstance(device_map, dict):
# Extract all devices recursively from dict values
devices = set()
for v in device_map.values():
devices.update(parse_available_devices(v))
return sorted(devices)
raise TypeError(f"Unsupported device_map type: {type(device_map)}")
@functools.cache
def is_gaudi2():
try:
import habana_frameworks.torch.utils.experimental as htexp
is_hpu_gaudi2 = htexp._get_device_type() == htexp.synDeviceType.synDeviceGaudi2
return is_hpu_gaudi2
except ImportError:
return False
class MemoryMonitor:
"""Global memory monitor for tracking peak RAM and VRAM usage."""
_instance = None
_lock = Lock()
_initialized = False
def __new__(cls):
if cls._instance is None:
with cls._lock:
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance._initialized = False
return cls._instance
def __init__(self):
if self._initialized:
return
self._initialized = True
self.peak_ram = 0.0 # GB
self.peak_vram = {} # {device_id: peak_mb}
self.enabled = True
# devices registered for sampling BEYOND the caller's device_map list
# (e.g. DDP replica devices the parallel lane engaged outside the map)
self._extra_devices: set[str] = set()
@staticmethod
def _process_tree_rss() -> float:
"""Return the combined RSS of this process and all live descendants in GB."""
process = psutil.Process()
rss = process.memory_info().rss
for child in process.children(recursive=True):
try:
rss += child.memory_info().rss
except (psutil.NoSuchProcess, psutil.AccessDenied):
continue
return rss / 1024**3
def register_extra_devices(self, devices) -> None:
"""Add accelerator devices to the monitored set beyond the device_map.
The final ``'peak_vram': {...}`` summary only reports devices the
``update`` calls sample; callers pass the device-map list, so devices
engaged outside the map (single-process DDP replicas on all visible
GPUs while ``--device_map`` is single-device) would stay invisible.
Registered devices are merged into every subsequent ``update``/
``update_hpu`` sample; ``reset`` clears the peaks but keeps the
registration (engagement re-registers per block).
"""
if not hasattr(self, "_extra_devices"): # older singleton instance
self._extra_devices = set()
for d in devices or []:
key = str(d)
if key.split(":")[0] in ("cuda", "xpu", "hpu"):
self._extra_devices.add(key)
def _merged_device_list(self, device_list):
"""The caller's device list plus any registered extra devices (deduped)."""
if not getattr(self, "_extra_devices", None):
return device_list
seen = [str(d) for d in device_list]
return list(device_list) + [d for d in sorted(self._extra_devices) if d not in seen]
def update(self, device_list=None):
"""Update current memory usage and track peaks."""
if not self.enabled:
return
# Track RAM
current_ram = self._process_tree_rss()
self.peak_ram = max(self.peak_ram, current_ram)
if device_list is None: # TODO this has issue, wait for clean_memory all pass device_list
device_list = [0]
if device_list is not None:
if not isinstance(device_list, (list, tuple)):
device_list = [device_list]
else:
dev_mgr = get_current_device_manager()
if dev_mgr.is_available() and dev_mgr.type != "cpu":
device_list = list(range(dev_mgr.device_count()))
else:
device_list = [0]
device_list = self._merged_device_list(device_list)
dev_mgr = get_current_device_manager()
for device in device_list:
if str(device) == "cpu":
continue
if dev_mgr.is_available() and dev_mgr.type != "cpu":
if str(device) == dev_mgr.type:
device = "0"
try:
index = int(str(device).split(":")[-1])
except ValueError:
index = 0
try:
current_vram = dev_mgr.memory_reserved(index) / 1024**3 # GB
except Exception:
continue # Skip devices that are not initialized or out of range
else:
return
device = str(device).split(":")[-1]
if current_vram > 0:
if device not in self.peak_vram:
self.peak_vram[device] = 0.0
self.peak_vram[device] = max(self.peak_vram[device], current_vram)
def update_cpu(self):
if not self.enabled:
return
current_ram = self._process_tree_rss()
self.peak_ram = max(self.peak_ram, current_ram)
def update_hpu(self, device_list=None):
"""Update memory usage for HPU devices."""
if not self.enabled:
return
# Track RAM
self.update_cpu()
# Track HPU VRAM
if not is_hpex_available():
return
hpu = get_ar_device("hpu")
if device_list is None:
count = hpu.device_count()
device_list = list(range(count)) if count > 0 else [0]
elif not isinstance(device_list, (list, tuple)):
device_list = [device_list]
device_list = self._merged_device_list(device_list)
for device in device_list:
if str(device) == "cpu":
continue
try:
index = int(str(device).split(":")[-1]) if str(device) != "hpu" else 0
current_vram = hpu.memory_allocated(index) / 1024**3 # GB
except Exception:
continue
dev_key = str(device)
if dev_key == "hpu":
dev_key = "0"
dev_key = dev_key.split(":")[-1]
if current_vram > 0:
if dev_key not in self.peak_vram:
self.peak_vram[dev_key] = 0.0
self.peak_vram[dev_key] = max(self.peak_vram[dev_key], current_vram)
def reset(self):
"""Reset all statistics."""
self.peak_ram = 0.0
self.peak_vram = {}
def get_summary(self):
"""Get summary of peak memory usage."""
summary = f"'peak_ram': {round(self.peak_ram, 2)}GB"
if len(self.peak_vram) > 0:
sorted_items = sorted(self.peak_vram.items())
if len(self.peak_vram) == 1:
key, value = sorted_items[0]
summary += f", 'peak_vram': {round(value, 2)}GB"
else:
items_str = ", ".join([f"'{k}': {round(v, 2)}GB" for k, v in sorted_items])
summary += f", 'peak_vram': {{{items_str}}}"
return summary
def log_summary(self, msg: str = "", level: str = "info"):
"""Log memory usage summary."""
summary = self.get_summary()
logger_method = getattr(logger, level.lower(), logger.info)
if len(msg):
logger_method(f"{msg} {summary}")
else:
logger_method(f"{summary}")
return summary
# Global singleton instance
memory_monitor = MemoryMonitor()
@contextmanager
def dump_memory_usage_ctx(msg: str = "", log_level: str = "info"):
"""Context manager to dump memory usage before and after a code block."""
memory_monitor.update_cpu()
logger_method = getattr(logger, log_level.lower(), logger.info)
logger_method(f"[Memory Monitor] Before {msg}: {memory_monitor.get_summary()}")
try:
yield
finally:
memory_monitor.update_cpu()
logger_method(f"[Memory Monitor] After {msg}: {memory_monitor.get_summary()}")
def dump_mem_usage(msg: str = "", log_level: str = "info"):
"""Decorator to dump memory usage before and after a function call."""
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
memory_monitor.update_cpu()
logger_method = getattr(logger, log_level.lower(), logger.info)
logger_method(f"[Memory Monitor] Before {msg}: {memory_monitor.get_summary()}")
result = func(*args, **kwargs)
memory_monitor.update_cpu()
logger_method(f"[Memory Monitor] After {msg}: {memory_monitor.get_summary()}")
return result
return wrapper
return decorator
# This function is designed for Auto Scheme and Diffusion Pipeline,
# which requires dispatching the whole model on all available devices.
def dispatch_model_by_all_available_devices(
model: torch.nn.Module, device_map: str | int | dict | None
) -> torch.nn.Module:
# Important Notice: This dispatch does not follow dict device_map, just extract all available devices and use them
device_type = get_major_device()
if device_type in DEVICE_ENVIRON_VARIABLE_MAPPING:
existing_env = os.environ.get(DEVICE_ENVIRON_VARIABLE_MAPPING[device_type])
if existing_env is None:
logger.warning_once(
"`get_balanced_memory` is used here, but no environment variable "
+ "is set to specify device visibility. This may lead to OOM issue even the memory "
+ "is large enough."
)
# Handle DiffusionPipeline: dispatch only the main sub-model (transformer / unet)
# across devices and move the remaining pipeline components to the primary device.
is_diffusion_pipeline = False
try:
from diffusers.pipelines.pipeline_utils import DiffusionPipeline
if isinstance(model, DiffusionPipeline):
is_diffusion_pipeline = True
except ImportError:
pass
if is_diffusion_pipeline:
pipe = model
_device_map = 0 if device_map is None else device_map
devices = parse_available_devices(_device_map)
# Identify the main quantisable sub-model
main_attr = next(
(attr for attr in ("transformer", "unet") if isinstance(getattr(pipe, attr, None), torch.nn.Module)),
None,
)
if main_attr is None or len(devices) == 1:
# No identifiable main sub-model, or single target device:
# move the entire pipeline to the (first) device.
pipe.to(devices[0] if devices else "cuda:0")
return pipe
# Multi-device path: dispatch the main sub-model across devices,
# reserving memory on the primary device for non-target components
# (text encoder, VAE, etc.) to avoid OOM.
main_model = getattr(pipe, main_attr)
primary_device = devices[0]
# Place non-main components on the last device
comp_device = devices[-1]
for attr, component in pipe.components.items():
if attr == main_attr:
continue
if not isinstance(component, torch.nn.Module):
continue
# Align dtype on CPU first to avoid wasting GPU memory
if hasattr(component, "dtype") and component.dtype != main_model.dtype:
try:
component.to(dtype=main_model.dtype)
except Exception:
pass
try:
component.to(comp_device)
except (NotImplementedError, RuntimeError):
pass
from auto_round.utils.common import normalize_no_split_modules
no_split_modules = normalize_no_split_modules(getattr(main_model, "_no_split_modules", []))
# dispatch_model_block_wise queries free memory after non-main
# components are already placed, so the budget naturally accounts for
# them. Use the same approach here.
dispatched = dispatch_model_block_wise(main_model, device_map)
setattr(pipe, main_attr, dispatched)
# Install manual pre/post hooks to move tensors.
unique_devices = set()
if hasattr(dispatched, "hf_device_map"):
unique_devices = {v for v in dispatched.hf_device_map.values() if v not in ("cpu", "disk")}
if len(unique_devices) <= 1:
execution_device = primary_device
try:
execution_device = next(dispatched.parameters()).device
except StopIteration:
execution_device = torch.device(primary_device)
if hasattr(dispatched, "_hf_hook") and hasattr(dispatched._hf_hook, "execution_device"):
dispatched._hf_hook.execution_device = execution_device
if not getattr(dispatched, "_autoround_align_inputs_hook_installed", False):
_first_param_device = execution_device
_pipeline_device = torch.device(comp_device)
def _align_all_inputs_pre_hook(module, args, kwargs):
try:
target = next(module.parameters()).device
except StopIteration:
target = _first_param_device
new_args = tuple(a.to(target) if isinstance(a, torch.Tensor) else a for a in args)
new_kwargs = {k: v.to(target) if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()}
return new_args, new_kwargs
def _move_outputs_back_hook(module, input, output):
def _to_device(obj, device):
if isinstance(obj, torch.Tensor):
return obj.to(device) if obj.device != device else obj
if isinstance(obj, (tuple, list)):
converted = [_to_device(o, device) for o in obj]
return type(obj)(converted)
if isinstance(obj, dict):
return {k: _to_device(v, device) for k, v in obj.items()}
return obj
return _to_device(output, _pipeline_device)
dispatched.register_forward_pre_hook(_align_all_inputs_pre_hook, with_kwargs=True)
dispatched.register_forward_hook(_move_outputs_back_hook)
dispatched._autoround_align_inputs_hook_installed = True
return pipe
if device_map is None:
device_map = 0
from auto_round.utils.common import normalize_no_split_modules
no_split_modules = normalize_no_split_modules(getattr(model, "_no_split_modules", []))
if device_map == "auto":
max_memory = get_balanced_memory(
model,
max_memory=None,
no_split_module_classes=no_split_modules,
)
device_map = infer_auto_device_map(model, max_memory=max_memory, no_split_module_classes=no_split_modules)
model = dispatch_model(model, device_map=device_map)
return model
devices = parse_available_devices(device_map)
if len(devices) == 1:
model.to(devices[0])
return model
max_memory = get_balanced_memory(
model,
max_memory=None,
no_split_module_classes=no_split_modules,
)
# Filter max_memory with devices
# assume only one GPU model
new_max_memory = {}
for device in devices:
if ":" in device:
device = int(device.split(":")[-1])
elif device == "cpu":
device = "cpu"
elif isinstance(device, str):
device = 0
else:
raise ValueError(f"Unsupported device {device} in device_map: {device_map}")
new_max_memory[device] = max_memory[device]
if hasattr(model, "tie_weights") and callable(model.tie_weights):
model.tie_weights()
device_map = infer_auto_device_map(model, max_memory=new_max_memory, no_split_module_classes=no_split_modules)
model = dispatch_model(model, device_map=device_map)
return model