Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

46.2 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unified device backend abstraction for AutoRound.

The goal of this module is to make supporting a *new* hardware device as cheap
as possible.  Instead of sprinkling ``torch.cuda.* / torch.xpu.* / torch.hpu.*``
branches across the code base, all device-specific operations are funnelled
through a single :class:`DeviceManager` wrapper that delegates to PyTorch's
unified device APIs:

* ``torch.accelerator`` (PyTorch >= 2.6, expanded in 2.12) -- discovery and
  synchronization, see https://docs.pytorch.org/docs/2.12/accelerator.html
* ``torch.get_device_module(device)`` -- returns the backend runtime module
  (``torch.cuda``, ``torch.xpu``, ``torch.mps`` ...) exposing the *same* method
  names (``empty_cache``, ``synchronize``, ``memory_reserved`` ...).

Because every PyTorch device backend exposes the same method surface, a new
device that PyTorch already supports works here with **zero** extra code.  Only
out-of-tree backends that are not yet integrated into ``torch.accelerator``
(currently Intel Gaudi / ``hpu``) need a tiny shim, handled below.

Typical usage::

    from auto_round.utils.device_manager import get_current_device_manager, get_ar_device

    dev = get_current_device_manager()   # active device (cuda/xpu/hpu/...)
    if dev.is_available():
        dev.empty_cache()
        free, total = dev.mem_get_info(0)

    cuda = get_ar_device("cuda")    # a specific backend
"""

from __future__ import annotations

import contextlib
import functools
import gc
import re
import sys

import torch

from auto_round.logger import logger

__all__ = [
    "ARDevice",
    "ClearMemory",
    "DeviceManager",
    "clear_memory",
    "default_enable_torch_compile",
    "detect_device_count",
    "device_manager",
    "get_ar_device",
    "get_available_device_types",
    "get_current_device_manager",
    "get_current_device_type",
    "get_device_and_parallelism",
    "get_device_memory",
    "get_major_device",
    "get_packing_device",
    "is_auto_device_mapping",
    "is_device_available",
    "normalize_default_device_map",
]


# ---------------------------------------------------------------------------
# Backend discovery helpers
# ---------------------------------------------------------------------------
# Priority order used as a *fallback* hint when ``torch.accelerator`` is not
# available (PyTorch < 2.6).  ``hpu`` is kept explicit because Intel Gaudi is an
# out-of-tree backend that historically was not registered with
# ``torch.accelerator``.  Any backend that IS registered with
# ``torch.accelerator`` (cuda/xpu/mps/npu/...) is discovered automatically and
# does NOT need to appear in this list.
_PREFERRED_ORDER = ("cuda", "xpu", "hpu")  # add mps later


def normalize_default_device_map(device_map: None | str | int | torch.device | dict):
    """Normalize default device selection across entry points.

    On Apple Silicon, the default ``0`` / ``"0"`` / ``None`` / ``"auto"``
    selection would otherwise resolve to MPS.  That tends to OOM on larger
    models, so keep the historical behavior of defaulting to CPU unless the
    caller explicitly requests MPS.
    """
    if torch.mps.is_available() and device_map in (0, "0", None, "auto"):
        logger.warning(
            "MPS detected. Using CPU by default to avoid potential memory issues. "
            "Set --device_map=mps to force MPS usage."
        )
        return "cpu"
    return device_map


def _torch_accelerator_type() -> str | None:
    """Return the canonical accelerator type reported by ``torch.accelerator``.

    A PyTorch build exposes at most one accelerator backend; this returns its
    type string (e.g. ``"cuda"``, ``"xpu"``, ``"mps"``, ``"npu"`` ...) or
    ``None`` when the API is unavailable / no accelerator is present.
    """
    accelerator = getattr(torch, "accelerator", None)
    if accelerator is None:
        return None
    try:
        if not accelerator.is_available():
            return None
        current = accelerator.current_accelerator()
        return current.type if current is not None else None
    except Exception:
        return None


def _accelerator_api():
    """Return the ``torch.accelerator`` module when it is usable, else ``None``.

    "Usable" means the API exists *and* reports an available accelerator for the
    current build (so on CPU-only builds we fall back to the device module).
    """
    api = getattr(torch, "accelerator", None)
    if api is None:
        return None
    try:
        return api if api.is_available() else None
    except Exception:
        return None


def _module_call(api, names, *args):
    """Call the first existing attribute in ``names`` on ``api`` with ``args``.

    Tolerates PyTorch renames across versions (e.g. ``current_device_index`` in
    2.12 vs the deprecated ``current_device_idx`` in 2.6).  Returns
    ``(ok, result)``.
    """
    for name in names:
        fn = getattr(api, name, None)
        if callable(fn):
            return True, fn(*args)
    return False, None


@functools.lru_cache(None)
def _hpu_available() -> bool:
    """Whether the Intel Gaudi (hpu) backend is usable."""
    if hasattr(torch, "hpu") and torch.hpu.is_available():
        return True
    try:  # pragma: no cover - depends on Gaudi runtime
        import habana_frameworks.torch.hpu as _hthpu  # noqa: F401  pylint: disable=E0401

        return True
    except Exception:  # pragma: no cover
        return False


def _normalize_device_type(device: None | str | int | torch.device) -> str | None:
    """Reduce any device spec to a bare backend type string (``"cuda"`` ...)."""
    if device is None:
        return get_current_device_type()
    if isinstance(device, int):
        return get_current_device_type()
    if isinstance(device, torch.device):
        return device.type
    if isinstance(device, str):
        if device in ("auto", "tp"):
            return get_current_device_type()
        return device.split(":")[0]
    raise ValueError("Device type not recognized")


@functools.lru_cache(None)
def get_current_device_type() -> str:
    """Return the active device backend type, or "cpu" if CPU-only.

    Discovery order:
      1. Intel Gaudi ("hpu") -- out-of-tree, may not register with torch.accelerator.
      2. "torch.accelerator" -- the canonical API, covers cuda/xpu/mps/npu/...
      3. Manual probing of :data:`_PREFERRED_ORDER` for older PyTorch releases.
    """
    # "hpu" first: it may not be registered with torch.accelerator.
    if _hpu_available():
        return "hpu"

    accel_type = _torch_accelerator_type()
    if accel_type is not None:
        return accel_type

    # PyTorch < 2.6: torch.accelerator may not exist; probe common backends.
    for dtype in _PREFERRED_ORDER:
        if dtype == "hpu":
            continue
        mod = getattr(torch, dtype, None)
        is_avail = getattr(mod, "is_available", None)
        if callable(is_avail) and is_avail():
            return dtype

    return "cpu"


def is_device_available() -> bool:
    """Whether any (non-CPU) device is available."""
    return get_current_device_type() is not None


def get_available_device_types() -> list[str]:
    """Return all available (non-CPU) backend types, in preferred order.

    Uses ``torch.accelerator`` so backends registered with PyTorch -- including
    out-of-tree ones such as ``npu`` -- are discovered automatically, without
    callers ever probing ``torch.cuda`` / ``torch.xpu`` / ... by hand.
    """
    available: list[str] = []
    # Out-of-tree hpu first (may not be registered with torch.accelerator).
    if _hpu_available():
        available.append("hpu")
    # The canonical accelerator reported by torch.accelerator (cuda/xpu/mps/npu/...).
    accel_type = _torch_accelerator_type()
    if accel_type is not None and accel_type not in available:
        available.append(accel_type)
    return available


# ---------------------------------------------------------------------------
# Device handles -- a small inheritance hierarchy
# ---------------------------------------------------------------------------
class _DeviceIndexContext:
    """Fallback for ``torch.accelerator.device_index`` on older PyTorch/backends."""

    def __init__(self, device: ARDevice, index: int):
        self._device = device
        self._index = index
        self._prev = None

    def __enter__(self):
        try:
            self._prev = self._device.current_device()
        except Exception:
            self._prev = None
        self._device.set_device(self._index)
        return self

    def __exit__(self, *exc):
        if self._prev is not None:
            self._device.set_device(self._prev)
        return False


class ARDevice:
    """Base, backend-agnostic handle to a single PyTorch device *backend*.

    A :class:`Device` represents a backend *type* (``cuda``/``xpu``/...), not a
    single card -- every per-card operation takes an ``index`` so the same
    handle drives all cards of that backend (multi-card aware).

    The base implementation delegates to the backend runtime module obtained
    from :func:`get_device_module` and, when this is the build's active
    accelerator, to the unified ``torch.accelerator`` API.  Specialised
    backends subclass this and override only the methods that differ (e.g.
    :meth:`set_device` / :meth:`device_count`); subclasses self-register via the
    ``device_type`` class attribute so :class:`DeviceManager` can instantiate
    them by name.
    """

    #: Canonical backend type a subclass handles (e.g. ``"cuda"``).  Empty on
    #: the base class, which stays usable as a *generic* fallback for any
    #: PyTorch backend that lacks a dedicated subclass (e.g. a fresh ``npu``).
    device_type: str = ""

    _registry: dict[str, type[ARDevice]] = {}

    def __init_subclass__(cls, **kwargs):
        super().__init_subclass__(**kwargs)
        dtype = cls.__dict__.get("device_type", "")
        if dtype:
            ARDevice._registry[dtype] = cls

    @classmethod
    def create(cls, device_type: str) -> ARDevice:
        """Instantiate the most specific :class:`Device` for ``device_type``."""
        subclass = cls._registry.get(device_type)
        if subclass is not None:
            return subclass()
        return ARDevice(device_type)

    @staticmethod
    def get_device_module(device: None | str | int | torch.device = None):
        """Return the backend runtime module for ``device`` (e.g. ``torch.cuda``).

        This is a thin, version-tolerant wrapper around ``torch.get_device_module``
        that also understands ``hpu`` and plain device strings/indices.

        Args:
            device: ``"cuda"``, ``"xpu:0"``, ``torch.device(...)``, an int index
                (interpreted against the current device) or ``None`` (current
                device).

        Returns:
            The module exposing the device runtime API, or ``None`` for CPU / when
            no device is available.
        """
        device_type = _normalize_device_type(device)
        if hasattr(torch, "get_device_module"):
            try:
                return torch.get_device_module(device_type)
            except Exception:
                pass
        return getattr(torch, device_type, None)

    def __init__(self, device_type: str | None = None):
        self.type = device_type or self.device_type

        # Prefer the unified ``torch.accelerator`` API for runtime ops when this
        # handle represents the build's current accelerator (cuda/xpu/mps/npu/
        # ...).  Out-of-tree backends such as ``hpu`` are not exposed by
        # ``torch.accelerator`` and transparently fall back to ``self._module``.
        self._module = _accelerator_api() if self.type == _torch_accelerator_type() else None

        if self._module is None:
            self._module = self.get_device_module(self.type)

    # -- discovery ----------------------------------------------------------
    @property
    def module(self):
        """The backend runtime module (``torch.cuda`` ...) or ``None``."""
        return self._module

    def is_available(self) -> bool:
        """Whether this backend type is usable in the current build."""
        return True

    def device_count(self) -> int:
        fn = getattr(self._module, "device_count", None)
        return int(fn()) if callable(fn) else 0  # pylint: disable=E1102

    def current_device(self) -> int:
        ok, idx = _module_call(self._module, ("current_device_index", "current_device_idx", "current_device"))
        if ok:
            try:
                return int(idx)
            except Exception:
                pass
        return 0

    def set_device(self, index: int | str | torch.device) -> None:
        if self._module is None:
            return
        ok, _ = _module_call(self._module, ("set_device_index", "set_device_idx", "set_device"), index)
        if ok:
            return

    def device(self, index: int | str | torch.device | None = None) -> torch.device:
        """Build a ``torch.device`` for this backend / card ``index``."""
        if index is None:
            return torch.device(self.type)
        if isinstance(index, torch.device):
            return index
        if isinstance(index, str):
            return torch.device(index if ":" in index else f"{self.type}:{index}")
        return torch.device(f"{self.type}:{int(index)}")

    # def devices(self) -> list[torch.device]:
    #     """Enumerate ``torch.device`` for every card of this backend."""
    #     return [self.device(i) for i in range(self.device_count())]

    # -- runtime ------------------------------------------------------------
    def synchronize(self, index: int | None = None) -> None:
        if self._module is None:
            return
        fn = getattr(self._module, "synchronize", None)
        if not callable(fn):
            return
        try:
            fn(index) if index is not None else fn()  # pylint: disable=E1102
        except Exception:
            fn()  # pylint: disable=E1102

    def empty_cache(self) -> None:
        # ``torch.accelerator.empty_cache`` is broken on some backends (e.g. MPS
        # triggers a caching-allocator assertion).  Always use the per-device
        # runtime module (``torch.cuda`` / ``torch.mps`` / ...) instead.
        fn = getattr(self.module, "empty_cache", None)
        if callable(fn):
            try:
                fn()  # pylint: disable=E1102 # mps has issues
            except Exception:
                pass

    def device_index(self, index: int):
        """Context manager that sets the current device index for this backend.

        Uses ``torch.accelerator.device_index`` when available; otherwise falls
        back to a tiny save/restore around :meth:`set_device`.
        """
        if self._module is not None:
            ctx = getattr(self._module, "device_index", None)
            if callable(ctx):
                return ctx(index)
        return _DeviceIndexContext(self, index)

    def total_memory(self, index: int = 0) -> int:
        return self.mem_get_info(index)[1]

    def mem_get_info(self, index: int = 0) -> tuple[int, int]:
        """Return ``(free_bytes, total_bytes)`` for ``index``."""
        modules = (self._module, self.get_device_module(self.type))
        for module in dict.fromkeys(modules):
            if module is None:
                continue
            for name in ("get_memory_info", "mem_get_info"):
                fn = getattr(module, name, None)
                if not callable(fn):
                    continue
                try:
                    free, total = fn(index)  # pylint: disable=E1102
                    if int(total) > 0:
                        return int(free), int(total)
                except Exception:
                    continue

        module = self.get_device_module(self.type)
        get_properties = getattr(module, "get_device_properties", None)
        if callable(get_properties):
            try:
                total = int(get_properties(index).total_memory)
                if total > 0:
                    return max(0, total - self.memory_reserved(index)), total
            except Exception:
                pass
        return 0, 0

    def memory_reserved(self, index: int = 0) -> int:
        if self._module is None:
            return 0
        fn = getattr(self._module, "memory_reserved", None) or getattr(self._module, "memory_cached", None)
        try:
            return int(fn(index)) if callable(fn) else 0  # pylint: disable=E1102
        except Exception:
            return 0

    def memory_allocated(self, index: int = 0) -> int:
        if self._module is None:
            return 0
        fn = getattr(self._module, "memory_allocated", None)
        try:
            return int(fn(index)) if callable(fn) else 0  # pylint: disable=E1102
        except Exception:
            return 0

    # -- numeric format / mixed-precision policy ---------------------------
    def supports_bf16(self) -> bool:
        """Whether this backend can execute the ``bfloat16`` data type."""
        return True

    def prefers_bf16(self) -> bool:
        """Whether this backend prefers bf16 as the mixed-precision compute dtype.

        Defaults to ``True`` (bf16 is the preferred tuning dtype); backends that
        would rather honour the model's own non-fp32 dtype can override this.
        """
        return True

    def is_torch_compile_supported(self) -> bool:
        return True

    def compile_func(self, func):
        """Compile ``func`` using this backend's ``torch.compile`` customization.

        Generic compile machinery (the shared dynamo cache-limit bump) lives in
        :func:`auto_round.utils.device._bump_dynamo_cache_limit`; only the
        per-device knobs (whether to compile at all and which backend to use) are
        expressed here, so :func:`auto_round.utils.device.compile_func` stays
        device-agnostic.
        """
        # Lazy import: the helper lives in utils/device.py which imports this module.
        if not self.is_torch_compile_supported():
            return func
        from auto_round.utils.device import _bump_dynamo_cache_limit

        _bump_dynamo_cache_limit(_engaged_cache_limit())

        return torch.compile(func)


def __repr__(self) -> str:  # pragma: no cover - debug aid
    return f"{type(self).__name__}(type={self.type!r})"


class HpuARDevice(ARDevice):
    """Intel Gaudi (HPU) -- an out-of-tree backend.

    ``hpu`` is not exposed through ``torch.accelerator``, so it always drives
    ``torch.hpu`` directly.  ``set_device`` is overridden to guard against
    builds where the runtime omits it.
    """

    device_type = "hpu"

    @staticmethod
    def get_device_module(device: None | str | int | torch.device = None):
        """Return the backend runtime module for ``device`` (e.g. ``torch.cuda``).

        This is a thin, version-tolerant wrapper around ``torch.get_device_module``
        that also understands ``hpu`` and plain device strings/indices.

        Args:
            device: ``"cuda"``, ``"xpu:0"``, ``torch.device(...)``, an int index
                (interpreted against the current device) or ``None`` (current
                device).

        Returns:
            The module exposing the device runtime API, or ``None`` for CPU / when
            no device is available.
        """

        if hasattr(torch, "hpu"):
            return torch.hpu
        try:  # pragma: no cover - depends on Gaudi runtime
            import habana_frameworks.torch.hpu as hthpu  # pylint: disable=E0401

            return hthpu
        except Exception:  # pragma: no cover
            return None

    def set_device(self, index: int | str | torch.device) -> None:
        if self._module is None:
            return
        fn = getattr(self._module, "set_device", None)
        if callable(fn):
            try:
                fn(index)  # pylint: disable=E1102
            except Exception:
                pass

    def is_available(self) -> bool:
        return _hpu_available()

    def is_torch_compile_supported(self) -> bool:
        # HPU only compiles in compile mode (lazy mode keeps the eager function).
        from auto_round.utils.device import _use_hpu_compile_mode

        return _use_hpu_compile_mode()

    def compile_func(self, func):
        if self.is_torch_compile_supported():
            return torch.compile(func, backend="hpu_backend")
        return func

    def memory_allocated(self, index: int = 0) -> int:
        return torch.hpu.memory_allocated(index)

    def memory_reserved(self, index: int = 0) -> int:  # TODO have a check
        return torch.hpu.memory_allocated(index)

    def device_count(self) -> int:
        import habana_frameworks.torch.hpu as hthpu  # pylint: disable=E0401

        return hthpu.device_count()


class MpsARDevice(ARDevice):
    """Apple Silicon (MPS) backend.

    MPS's caching allocator is not yet compatible with ``torch.accelerator``'s
    generic ``empty_cache`` path (PyTorch asserts internally), so we bypass it
    and call ``torch.mps`` methods directly.
    """

    device_type = "mps"

    def __init__(self, device_type: str | None = None):
        # Always use torch.mps directly, never torch.accelerator.
        self.type = "mps"
        self._module = getattr(torch, "mps", None)

    @staticmethod
    def get_device_module(device: None | str | int | torch.device = None):
        """Return the backend runtime module for ``device`` (e.g. ``torch.cuda``).

        This is a thin, version-tolerant wrapper around ``torch.get_device_module``
        that also understands ``hpu`` and plain device strings/indices.

        Args:
            device: ``"cuda"``, ``"xpu:0"``, ``torch.device(...)``, an int index
                (interpreted against the current device) or ``None`` (current
                device).

        Returns:
            The module exposing the device runtime API, or ``None`` for CPU / when
            no device is available.
        """
        return torch.mps

    def is_available(self) -> bool:
        """Whether this backend type is usable in the current build."""
        return self._module.is_available()

    def current_device(self) -> int:
        return 0

    def set_device(self, index: int | str | torch.device) -> None:
        return None

    def device(self, index: int | str | torch.device | None = None) -> torch.device:
        """Build a ``torch.device`` for this backend / card ``index``."""
        return torch.device("mps")

    def device_index(self, index: int):
        """Context manager that sets the current device index for this backend.

        Uses ``torch.accelerator.device_index`` when available; otherwise falls
        back to a tiny save/restore around :meth:`set_device`.
        """
        if self._module is not None:
            ctx = getattr(self._module, "device_index", None)
            if callable(ctx):
                return ctx(index)
        return _DeviceIndexContext(self, index)

    def total_memory(self, index: int = 0) -> int:
        return torch.mps.recommended_max_memory()

    def memory_reserved(self, index: int = 0) -> int:
        return torch.mps.driver_allocated_memory()

    def memory_allocated(self, index: int = 0) -> int:
        return torch.mps.current_allocated_memory()

    # def mem_get_info(self, index: int = 0) -> tuple[int, int]:
    #     """Return ``(free_bytes, total_bytes)`` for ``index``.
    #
    #     Falls back to ``total - reserved`` when the backend lacks a native
    #     ``mem_get_info`` implementation.
    #     """
    #     module = self.get_device_module(self.type) if self._module is _accelerator_api() else self._module
    #     fn = getattr(module, "get_memory_info", None)
    #
    #     return fn(index) if callable(fn) else (0, 0)  # pylint: disable=E1102

    # -- numeric format / mixed-precision policy ---------------------------
    def supports_bf16(self) -> bool:
        """Whether this backend can execute the ``bfloat16`` data type."""
        return True

    def prefers_bf16(self) -> bool:
        """Whether this backend prefers bf16 as the mixed-precision compute dtype.

        Defaults to ``True`` (bf16 is the preferred tuning dtype); backends that
        would rather honour the model's own non-fp32 dtype can override this.
        """
        return True


class CpuARDevice(ARDevice):
    """First-class handle for the host CPU.

    CPU has no backend runtime module, so instead of letting every method fall
    through ``None`` checks we give it explicit, correct semantics:

    * :meth:`synchronize` / :meth:`empty_cache` are genuine no-ops (there is no
      async stream or caching allocator to flush on CPU).
    * memory introspection reports host RAM via ``psutil`` when available.
    """

    device_type = "cpu"

    @staticmethod
    def get_device_module(device: None | str | int | torch.device = None):
        return None

    # -- discovery ----------------------------------------------------------
    def is_available(self) -> bool:  # CPU is always present.
        return True

    def device_count(self) -> int:  # A single logical device from torch's view.
        return 1

    def current_device(self) -> int:
        return 0

    def set_device(self, index: int | str | torch.device) -> None:  # no-op
        return None

    def device(self, index: int | str | torch.device | None = None) -> torch.device:
        return torch.device("cpu")

    # -- runtime ------------------------------------------------------------
    def synchronize(self, index: int | None = None) -> None:  # no-op
        return None

    def empty_cache(self) -> None:  # no-op: CPU has no caching allocator.
        return gc.collect()

    def get_device_capability(self, index: int | None = None):
        return None

    def device_index(self, index: int):  # nothing to switch on CPU.
        return contextlib.nullcontext()

    # -- numeric format / mixed-precision policy ---------------------------
    def supports_bf16(self) -> bool:
        cached = getattr(self, "_bf16_supported", None)
        if cached is None:
            # Local import avoids a circular dependency (device.py imports this module).
            from auto_round.utils.device import CpuInfo

            cached = bool(CpuInfo().bf16)
            self._bf16_supported = cached
        return cached

    # -- memory introspection (host RAM) -----------------------------------
    def _virtual_memory(self):
        try:
            import psutil  # pylint: disable=C0415

            return psutil.virtual_memory()
        except Exception:
            return None

    def total_memory(self, index: int = 0) -> int:
        vm = self._virtual_memory()
        return int(vm.total) if vm is not None else 0

    def memory_reserved(self, index: int = 0) -> int:
        import psutil

        process = psutil.Process()
        current_ram = process.memory_info().rss
        return current_ram

    def memory_allocated(self, index: int = 0) -> int:
        return self.memory_reserved(index)

    # def mem_get_info(self, index: int = 0) -> tuple[int, int]:
    #     vm = self._virtual_memory()
    #     if vm is None:
    #         return 0, 0
    #     return int(vm.available), int(vm.total)

    def is_torch_compile_supported(self) -> bool:
        return True


# ---------------------------------------------------------------------------
# Device manager -- creates, caches and orchestrates Device handles
# ---------------------------------------------------------------------------
class DeviceManager:
    """Registry and orchestrator for :class:`Device` handles.

    Owns the mapping from backend type to a (cached) :class:`Device` instance,
    exposes the *current* device for the active backend, and enumerates every
    card across all available backends for multi-card scenarios.  Custom
    backends can be plugged in at runtime via :meth:`register` without touching
    this module.

    A manager can additionally be *configured* with a ``device_map`` so callers
    (e.g. the compressors) no longer keep their own ``device`` / ``device_list``
    state -- they ask the manager instead.

    The manager is a process-wide **singleton**: every ``DeviceManager(...)`` call
    returns the same instance.  Passing a ``device_map`` simply (re)configures that
    shared instance, so the active device / device_list is always single-sourced.
    """

    _instance: DeviceManager | None = None

    def __new__(cls, *args, **kwargs):
        if cls._instance is None:
            cls._instance = super().__new__(cls)
        return cls._instance

    def __init__(self, device_map: None | str | torch.device | int | dict = None):
        # Initialise backing state once; later constructions reuse the singleton.
        if not getattr(self, "_initialized", False):
            self._cache: dict[str, ARDevice] = {}
            self._device_map = None
            self._device_list: list | None = None
            self._major_device: str | None = None
            self._initialized = True
        if device_map is not None:
            self.configure(device_map)

    # -- device_map configuration ------------------------------------------
    def configure(self, device_map: None | str | torch.device | int | dict = 0) -> DeviceManager:
        """Resolve a ``device_map`` into a concrete device list and major device.

        Centralises the device-map parsing the compressors used to perform by
        hand, so they can rely on :attr:`device` / :attr:`device_list` instead of
        maintaining duplicate state.
        """
        if device_map is None:
            device_map = 0
        if isinstance(device_map, str):
            device_map = device_map.replace(" ", "")
        self._device_map = device_map
        # Lazy import: device.py imports this module, so a top-level import would
        # create a circular dependency.
        from auto_round.utils.device import parse_available_devices

        self._device_list = parse_available_devices(device_map)  # cuda:6
        self._major_device = get_major_device(device_map)  # cuda:4
        return self

    @property
    def device_map(self):
        """The raw ``device_map`` this manager was configured with."""
        return self._device_map

    @property
    def device_list(self) -> list:
        """All concrete devices selected by the configured ``device_map``."""
        scoped = self._thread_scope()
        if scoped is not None:
            return [scoped]
        if self._device_list is None:
            self.configure(self._device_map)
        return self._device_list

    @property
    def device(self) -> str:
        """The major (primary, non-CPU when possible) device string."""
        scoped = self._thread_scope()
        if scoped is not None:
            return scoped
        if self._major_device is None:
            self.configure(self._device_map)
        return self._major_device

    def _thread_scope(self) -> str | None:
        local = self.__dict__.get("_thread_local")
        return getattr(local, "device", None) if local is not None else None

    @contextlib.contextmanager
    def thread_device_scope(self, device: str | torch.device):
        """Pin :attr:`device` / :attr:`device_list` to ``device`` for the calling thread.

        Used by the data-parallel engine: each rank thread runs the unchanged
        per-block algorithm and must see its own replica device as the major
        device. Other threads keep the configured devices.
        """
        import threading

        local = self.__dict__.setdefault("_thread_local", threading.local())
        prev = getattr(local, "device", None)
        local.device = str(device)
        try:
            yield
        finally:
            local.device = prev

    @device.setter
    def device(self, value: str | torch.device) -> None:
        """Override the major device (e.g. an OOM fallback to ``"cpu"``)."""
        self._major_device = str(value) if isinstance(value, torch.device) else value

    def is_multi_device(self) -> bool:
        """Whether more than one concrete device is selected."""
        return len(self.device_list) > 1

    # -- registration -------------------------------------------------------
    def register(self, device_cls: type[ARDevice]) -> None:
        """Register a custom :class:`Device` subclass and drop any stale cache."""
        dtype = device_cls.device_type
        if not dtype:
            raise ValueError("Device subclass must define a non-empty 'device_type'")
        ARDevice._registry[dtype] = device_cls
        self._cache.pop(dtype, None)

    # -- lookup -------------------------------------------------------------
    def get_ar_device(self, device_type: None | str | int | torch.device = None) -> ARDevice:
        """Return the cached :class:`Device` for ``device_type`` (default: current)."""
        normalized = _normalize_device_type(device_type) or "cpu"
        device = self._cache.get(normalized)
        if device is None:
            device = ARDevice.create(normalized)
            self._cache[normalized] = device
        return device

    def current(self) -> ARDevice:
        """Return the :class:`Device` for the active backend (or CPU)."""
        return self.get_ar_device(get_current_device_type())

    def current_type(self) -> str:
        return get_current_device_type()

    # -- multi-card / multi-backend ----------------------------------------
    def available_types(self) -> list[str]:
        """All available (non-CPU) backend types, in preferred order."""
        return get_available_device_types()

    def available_devices(self) -> list[ARDevice]:
        """One :class:`Device` per available (non-CPU) backend type."""
        return [self.get_ar_device(dtype) for dtype in self.available_types()]

    def all_devices(self) -> list[torch.device]:
        """Enumerate every card across all available backends (multi-card)."""
        devices: list[torch.device] = []
        for device in self.available_devices():
            devices.extend(device.devices())
        return devices


# Process-wide singleton manager.
device_manager = DeviceManager()


def _engaged_cache_limit() -> int:
    """Dynamo cache-limit target scaled by the engaged device count.

    Multi-device runs multiply the static shape specializations of the shared
    compiled quant functions (per-layer shapes x per-device guard pins), so the
    cache limit must scale with the engaged devices or the lane silently
    degrades to eager mid-block once the limit is hit.  A single-device run
    keeps the plain env default.  The bump itself is monotone raise-only, so
    repeated calls are idempotent.
    """
    from auto_round import envs

    return int(envs.AR_DYNAMO_CACHE_SIZE_LIMIT) * max(1, len(device_manager.device_list))


def get_ar_device(device_type: None | str | int | torch.device = None) -> ARDevice:
    """Return the cached :class:`Device` handle for a specific backend type."""
    return device_manager.get_ar_device(device_type)


def default_enable_torch_compile(
    device: None | str | int | torch.device = None, platform_name: str | None = None
) -> bool:
    """Return the safe torch.compile default for a backend."""
    return (platform_name or sys.platform) != "win32"


def get_current_device_manager() -> ARDevice:
    """Return the :class:`Device` handle for the active backend (or CPU)."""
    return device_manager.current()


# ---------------------------------------------------------------------------
# Device resolution / parsing helpers (moved from utils/device.py)
# ---------------------------------------------------------------------------
def detect_device_count() -> int:
    """Detects the number of available computation devices."""
    return get_current_device_manager().device_count()


def get_device_and_parallelism(device: str | torch.device | int | dict) -> tuple[str, bool]:
    """Resolve a device spec into ``(device, parallelism)``.

    The multi-card *parallelism* policy itself is kept as a standalone function
    (:func:`auto_round.utils.device.is_pipeline_parallel_supported`) rather than
    living on the device manager.
    """
    if device is None:
        device = get_major_device(device)
        return device, False
    if isinstance(device, dict):
        unique_devices = set(device.values())
        if len(unique_devices) == 1:
            device = next(iter(unique_devices))
        else:
            device = "auto"
    if isinstance(device, torch.device):
        device = str(device)
    if isinstance(device, str):
        # A bare backend type (e.g. "cuda", "xpu", "hpu", "cpu", "mps") with no index
        if device not in ("auto", "tp") and ":" not in device and "," not in device and not device.isdigit():
            return get_major_device(device), False
        # Strip any "<type>:" prefixes (e.g. "cuda:0,1" -> "0,1") to obtain bare indices.
        device = re.sub(r"[a-zA-Z_]+:", "", device)
        devices = device.replace(" ", "").split(",")
    elif isinstance(device, int):
        devices = [str(device)]
    else:
        devices = [device]

    is_multi_card = all(s.isdigit() for s in devices) and len(devices) > 1
    if is_multi_card:
        # Pick the active backend generically rather than probing each one by hand.
        device_type = get_current_device_type() or "cpu"
        # Parallelism policy is intentionally not part of the device manager.
        from auto_round.utils.device import is_pipeline_parallel_supported

        return device_type, is_pipeline_parallel_supported(device_type)
    elif device == "auto":
        device = get_major_device(device)
        parallelism = True
    else:
        device = get_major_device(device)
        parallelism = False
    return device, parallelism


def get_packing_device(device: str | torch.device | None = "auto") -> torch.device:
    """Selects the packing device.

    - ``"auto"``: choose best available (active accelerator > CPU).
    - ``str``: parsed by ``torch.device`` (e.g., ``"cuda:2"``, ``"cpu"``).
    - ``torch.device``: returned as-is.
    - ``None``: treated as ``"auto"``.
    """
    if device is None or (isinstance(device, str) and device.lower() == "auto"):
        device_type = get_current_device_type()
        if device_type is not None and device_type != "cpu":
            return torch.device(f"{device_type}:0")
        return torch.device("cpu")

    if isinstance(device, torch.device):
        return device

    if isinstance(device, str):
        try:
            return torch.device(device)
        except Exception as e:
            raise ValueError(f"Invalid device string: {device}") from e

    raise TypeError(f"Unsupported device type: {type(device)} ({device})")


def is_auto_device_mapping(device_map: str | int | dict | None) -> bool:
    if device_map is None or isinstance(device_map, int):
        return False
    elif device_map == "auto" or (isinstance(device_map, str) and "," in device_map):
        return True
    elif isinstance(device_map, dict):
        return False
    else:
        return False


def get_major_device(device_map: None | str | torch.device | int | dict = None) -> str:
    if device_map is None or isinstance(device_map, (str, torch.device, int)):
        """Detects the appropriate computation device.

        Takes a specific device index/string or ``"auto"``/``None`` (auto-detect the
        active backend), and returns the resolved device as a string.
        "4,6"->cuda:4
        """

        def is_valid_digit(s):
            try:
                num = int(s)
                return 0 <= num
            except Exception:
                return False

        dev_idx = None
        device = device_map
        if is_valid_digit(device):
            dev_idx = int(device)
            device = "auto"
        if isinstance(device, str) and "," in device:  # device is "0,1,2"
            device_list = []
            for dev in device.split(","):
                if dev.isdigit():
                    device_list.append(int(dev))
                elif dev.split(":")[-1].isdigit():
                    device_list.append(int(dev.split(":")[-1]))
                elif 0 not in device_list:
                    device_list.append(0)
            dev_idx = device_list[0] if device_list else None
            device = "auto"
        if device is None or device == "auto":
            device_type = get_current_device_type()
            device = torch.device(device_type) if device_type is not None else torch.device("cpu")
            if dev_idx is not None and str(device) != "cpu":
                device = str(device) + f":{dev_idx}"
            return str(device)
        elif isinstance(device, torch.device):
            device = str(device)
        elif isinstance(device, str):  ## for cuda:0
            if device == "tp":  # pragma: no cover
                # should not specify card, e.g., cuda:0
                device = get_current_device_type() or "cpu"
        return device

    if isinstance(device_map, dict) and device_map:
        tmp_devices = []
        for val in device_map.values():
            if isinstance(val, (str, torch.device, int)):  # could optimize
                tmp_device = get_major_device(val)
                tmp_device = tmp_device.split(":")[0]
                tmp_devices.append(tmp_device)
        tmp_devices = list(set(tmp_devices))
        device = None
        for tmp_device in tmp_devices:
            if tmp_device != "cpu":
                device = tmp_device
                break
        if device is None:
            device = tmp_devices[0]
        if len(tmp_devices) > 1:
            logger.warning_once(
                f"there are multiple device types in the device_map, "
                f"please make sure they are correct,use the first none-cpu device {device} as the core device "
            )

        return device
    logger.warning_once(f"device_map should be [str, torch.device, int, dict], but got {type(device_map)}")
    return "cpu"


def get_device_memory(i: int = 0) -> int:
    """Gets the total memory on the specified device, in gigabytes."""
    dev_mgr = get_current_device_manager()
    if not dev_mgr.is_available() or dev_mgr.type == "cpu":
        raise RuntimeError("No supported device found (CUDA/XPU/HPU/...).")

    total_memory = dev_mgr.total_memory(i)
    if total_memory is None:
        raise RuntimeError(f"Failed to query total memory for device index {i} on backend {dev_mgr.type}.")
    if total_memory <= 0:
        raise RuntimeError(
            f"Detected non-positive total memory ({total_memory}) for device index {i} on backend {dev_mgr.type}."
        )
    return total_memory / 1024**3


def _clear_memory_for_cpu_and_cuda(
    tensor: torch.Tensor | list | None = None,
    device_list: tuple | list | str | torch.device | None = None,
):
    # ------------------------
    # Clear CPU-side references
    # ------------------------
    if isinstance(tensor, list):
        for i in range(len(tensor)):
            tensor[i] = None
    tensor = None
    gc.collect()

    # Lazy import: malloc-trim helpers live in utils/device.py.
    from auto_round.utils.device import _maybe_trim_malloc

    _maybe_trim_malloc()

    # ------------------------
    # Normalize device_list
    # ------------------------
    if isinstance(device_list, (str, torch.device)):
        device_list = [device_list]

    # -----------------------------------
    # Device-specific clearing
    # -----------------------------------
    # Group requested devices by backend type so we synchronize the exact
    # indices the caller asked for, then fall back to clearing the active
    # accelerator entirely when no list is provided.
    current_dev_type = get_current_device_type()
    if current_dev_type is None or current_dev_type == "cpu":
        return

    if not device_list:
        dev_mgr = get_current_device_manager()
        dev_mgr.synchronize()
        dev_mgr.empty_cache()
        return

    # Parse "<type>:<idx>" entries, grouping indices per backend.
    per_backend: dict[str, list[int]] = {}
    for dev in device_list:
        dev = str(dev)
        dev_type = dev.split(":")[0]
        if not dev_type or dev_type == "cpu" or dev_type.isdigit():
            # Bare indices (e.g. "0") are interpreted against the active device.
            dev_type = current_dev_type if dev_type.isdigit() else dev_type
            if not dev_type or dev_type == "cpu":
                continue
        devid = int(dev.split(":")[-1]) if ":" in dev else (int(dev) if dev.isdigit() else 0)
        per_backend.setdefault(dev_type, []).append(devid)

    for dev_type, ids in per_backend.items():
        dev_mgr = get_ar_device(dev_type)
        for devid in ids:
            dev_mgr.synchronize(devid)
        dev_mgr.empty_cache()


class ClearMemory:

    def __init__(self, device_list: list | tuple | None = None):
        self._device_list = device_list

    @property
    def device_list(self):
        if self._device_list is None:
            return device_manager.device_list
        return self._device_list

    @device_list.setter
    def device_list(self, value):
        self._device_list = value

    def __call__(
        self,
        tensor: torch.Tensor | None | list = None,
        device_list: list | tuple | None = None,
    ):
        # Lazy imports: these symbols live in utils/device.py.
        from auto_round.utils.device import _force_trim_malloc, is_hpex_available, memory_monitor

        if is_hpex_available():
            if device_list is not None:
                self.device_list = device_list
            final_device_list = self.device_list
            # Keep peak accounting consistent with non-HPU path: sample first,
            # then release memory so post-clear values do not hide true peaks.
            memory_monitor.update_hpu(final_device_list)
            # Clear CPU-side references so Python can reclaim them.
            if isinstance(tensor, list):
                for i in range(len(tensor)):
                    tensor[i] = None
            tensor = None
            gc.collect()
            _force_trim_malloc()
            return
        else:
            if device_list is not None:
                self.device_list = device_list
            final_device_list = self.device_list
            memory_monitor.update(final_device_list)
            _clear_memory_for_cpu_and_cuda(tensor, final_device_list)


clear_memory = torch._dynamo.disable()(ClearMemory(device_list=None))