Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Bug fix: Replace WAWrapperLinear with FakeActQuantLinear instead of Linear (#2245)

Xin He6b29e9c7602f
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index c01e5feb7..9145ce7d5 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -441,8 +441,11 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):
eval_folder = folders[-1] if folders else None
else:
eval_folder = folders
+
# set model_type for ModelFreeCompressor.
- model_type = detect_model_type(eval_folder)
+ # Fake format now always saves, so eval_folder is the canonical probe.
+ type_probe = eval_folder
+ model_type = detect_model_type(type_probe)
if hasattr(autoround, "model_context") and model_type in ("mllm", "diffusion"):
setattr(autoround.model_context, f"is_{model_type}", True)
else:
@@ -496,8 +499,12 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):
logger.warning("set add_bos_token=True for llama model.")
args.add_bos_token = True
- # Check if GGUF model
- eval_gguf_model = any(file.endswith("gguf") for file in os.listdir(eval_folder))
+ # Check if GGUF model in exported eval folder.
+ eval_gguf_model = (
+ eval_folder is not None
+ and os.path.isdir(eval_folder)
+ and any(file.endswith("gguf") for file in os.listdir(eval_folder))
+ )
# Determine if model instance evaluation is needed
need_model_instance = formats[-1] == "fake" or eval_gguf_model
diff --git a/auto_round/export/formats/backends/fake.py b/auto_round/export/formats/backends/fake.py
index 069d1aeae..d29e131f8 100644
--- a/auto_round/export/formats/backends/fake.py
+++ b/auto_round/export/formats/backends/fake.py
@@ -13,6 +13,8 @@
# limitations under the License.
import copy
+import glob
+import os
from typing import Any, Callable, Union
import torch
@@ -33,6 +35,51 @@ def _serialize_quantization_config_value(value):
return value
+def _normalize_state_dict_keys(state_dict: dict[str, torch.Tensor]) -> tuple[dict[str, torch.Tensor], bool]:
+ """Collapse wrapped-layer keys like ``*.orig_layer.weight`` to ``*.weight``."""
+ normalized = {}
+ changed = False
+ for key, value in state_dict.items():
+ new_key = key.replace(".orig_layer.", ".")
+ if new_key != key:
+ changed = True
+ # Prefer already-normalized keys if both happen to exist.
+ if new_key not in normalized:
+ normalized[new_key] = value
+ return normalized, changed
+
+
+def _rewrite_saved_weights_without_orig_layer(output_dir: str) -> None:
+ """Rewrite saved checkpoint shards in-place so no ``.orig_layer.`` keys remain."""
+ if not os.path.isdir(output_dir):
+ return
+
+ safetensor_files = sorted(glob.glob(os.path.join(output_dir, "*.safetensors")))
+ for file_path in safetensor_files:
+ try:
+ from safetensors.torch import load_file as safe_load_file
+ from safetensors.torch import save_file as safe_save_file
+
+ state = safe_load_file(file_path)
+ normalized, changed = _normalize_state_dict_keys(state)
+ if changed:
+ safe_save_file(normalized, file_path)
+ except Exception as exc:
+ logger.warning("Failed to normalize safetensors keys for %s: %s", file_path, exc)
+
+ bin_files = sorted(glob.glob(os.path.join(output_dir, "*.bin")))
+ for file_path in bin_files:
+ try:
+ state = torch.load(file_path, weights_only=True)
+ if not isinstance(state, dict):
+ continue
+ normalized, changed = _normalize_state_dict_keys(state)
+ if changed:
+ torch.save(normalized, file_path)
+ except Exception as exc:
+ logger.warning("Failed to normalize pytorch checkpoint keys for %s: %s", file_path, exc)
+
+
@OutputFormat.register("fake")
class FakeFormat(OutputFormat):
support_schemes = None
@@ -58,44 +105,30 @@ class FakeFormat(OutputFormat):
serialization_dict: dict = None,
**kwargs,
):
+ has_fake_act_quant = False
+ logger.warning(
+ "Saving fake-quantized model to disk. "
+ "Linear replacement is deferred to load-time (via auto_round:fake backend); "
+ "save-time now keeps the in-memory quantized model structure unchanged."
+ )
has_meta_device = unsupported_meta_device(model)
if not inplace and not has_meta_device:
model = copy.deepcopy(model.to("cpu"))
- from auto_round.utils.model import set_module
- from auto_round.wrapper import WrapperLinear, WrapperWALayer
+ config_act_bits = (serialization_dict or {}).get("act_bits")
+ if config_act_bits is not None and config_act_bits <= 8:
+ has_fake_act_quant = True
- if not has_meta_device:
- wrapped_modules = [
- (name, module)
- for name, module in model.named_modules()
- if name and isinstance(module, (WrapperLinear, WrapperWALayer))
- ]
- wrapped_names = {name for name, _ in wrapped_modules}
- for name, module in wrapped_modules:
- if any(name.startswith(f"{parent}.") for parent in wrapped_names if parent != name):
- continue
- orig_layer = module.orig_layer
- while isinstance(orig_layer, (WrapperLinear, WrapperWALayer)):
- orig_layer = orig_layer.orig_layer
- for attr_name in ("act_min_scale", "act_max_scale", "act_scale"):
- orig_layer._parameters.pop(attr_name, None)
- orig_layer._buffers.pop(attr_name, None)
- if hasattr(orig_layer, attr_name):
- delattr(orig_layer, attr_name)
- set_module(model, name, orig_layer.to("cpu"))
-
- is_nvfp4_v2 = (serialization_dict or {}).get("data_type") == "nvfp4_v2"
- if is_nvfp4_v2:
- quantization_config = _serialize_quantization_config_value(dict(serialization_dict or {}))
- quantization_config["quant_method"] = "auto-round"
+ quantization_config = _serialize_quantization_config_value(dict(serialization_dict or {}))
+ quantization_config["quant_method"] = "auto-round"
+ if has_fake_act_quant:
quantization_config["packing_format"] = "auto_round:fake"
- quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
- from auto_round.export.utils import filter_quantization_config
+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)
+ from auto_round.export.utils import filter_quantization_config
- filter_quantization_config(quantization_config)
- if hasattr(model, "config") and model.config is not None:
- model.config.quantization_config = quantization_config
+ filter_quantization_config(quantization_config)
+ if hasattr(model, "config") and model.config is not None:
+ model.config.quantization_config = quantization_config
if not has_meta_device:
model = model.to("cpu")
@@ -103,6 +136,11 @@ class FakeFormat(OutputFormat):
elif hasattr(model, "config") and model.config is not None:
model.config.save_pretrained(output_dir)
+ # Some save flows write wrapper keys first; normalize to plain Linear keys
+ # so HF loading does not report UNEXPECTED/MISSING pairs.
+ if has_fake_act_quant:
+ _rewrite_saved_weights_without_orig_layer(output_dir)
+
if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
tokenizer.save_pretrained(output_dir)
processor = kwargs.get("processor", None)
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index 289e163de..1ef8ba9b1 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -211,7 +211,7 @@ GPTQ_FORMAT = ["auto_round:auto_gptq"] # zp+-1
GPTQ_FORMAT_NO_ZP = ["auto_round", "auto_round:gptqmodel"]
AWQ_FORMAT = ["auto_round:auto_awq"]
LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor"]
-FAKE_FORMAT = ["auto_round:fake"]
+FAKE_FORMAT = ["auto_round:fake", "auto_round:auto_gptq", "auto_round", "auto_round:auto_awq"]
NVFP4_E5M3_LLM_COMPRESSOR_FORMAT = ["auto_round:llm_compressor_nvfp4_e5m3"]
WOQ_DEFAULT_ACT_BITS = [None, 16, 32]
@@ -351,17 +351,11 @@ BackendInfos["auto_round:torch_mxint4"] = BackendInfo(
BackendInfos["auto_round:fake"] = BackendInfo(
device=["xpu", "cuda", "cpu"],
packing_format=FAKE_FORMAT,
- sym=[True],
compute_dtype=["float32", "float16", "bfloat16"],
- data_type=["nvfp4_v2"],
- group_size=[16],
- bits=[4],
- act_bits=[4],
- act_group_size=[16],
- act_sym=[True],
- act_data_type=["nvfp4_v2"],
- priority=0,
- checkers=[mxfp_nvfp_feature_checker],
+ bits=[1, 2, 3, 4, 5, 6, 7, 8],
+ sym=[True, False],
+ # Keep fake backend as a fallback path; prefer real kernels when available.
+ priority=-1,
alias=["auto_round", "torch"],
requirements=["auto-round>0.12.0"],
)
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 6f738e2cd..1239fd81f 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1314,9 +1314,7 @@ def is_diffusion_model(model_or_path: Union[str, object], trust_remote_code: boo
if model_type == "nextstep":
return True
except:
- logger.warning(
- f"Failed to load config for {model_or_path}, trying to check model_index.json for diffusion pipeline."
- )
+ pass
index_file = None
if not os.path.isdir(model_or_path):
try:
diff --git a/test/unit/test_cpu/export/test_fake_format.py b/test/unit/test_cpu/export/test_fake_format.py
index 10b064e4a..b4d3a3842 100644
--- a/test/unit/test_cpu/export/test_fake_format.py
+++ b/test/unit/test_cpu/export/test_fake_format.py
@@ -20,12 +20,23 @@ import torch
from transformers import AutoModelForCausalLM, OPTConfig, OPTForCausalLM
from auto_round.experimental.qmodules.fake import FakeActQuantLinear
+from auto_round.export.formats.backends.fake import (
+ _normalize_state_dict_keys,
+ _rewrite_saved_weights_without_orig_layer,
+)
from auto_round.formats import FakeFormat
+from auto_round.inference.backend import get_layer_backend
from auto_round.inference.convert_model import convert_hf_model
from auto_round.schemes import PRESET_SCHEMES
from auto_round.wrapper import WrapperWALayer
+def _assert_has_act_hook(layer):
+ assert isinstance(layer, FakeActQuantLinear)
+ assert hasattr(layer, "qdq_input")
+ assert callable(layer.qdq_input)
+
+
class _WrappedLinear(WrapperWALayer):
def __init__(self, linear):
torch.nn.Module.__init__(self)
@@ -56,7 +67,7 @@ def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):
expected_weight = model.linear.orig_layer.weight.detach().clone()
output_dir = str(tmp_path / "fake_model")
- FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(
+ saved_model = FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(
output_dir=output_dir,
model=model,
inplace=False,
@@ -69,18 +80,30 @@ def test_fake_format_unwraps_quantized_layers_before_save(tmp_path):
"act_group_size": 16,
"act_sym": True,
"act_data_type": "nvfp4_v2",
+ "to_quant_block_names": ["block"],
"supported_types": [torch.nn.Linear],
},
)
state_dict = torch.load(os.path.join(output_dir, "pytorch_model.bin"), weights_only=True)
- assert set(state_dict) == {"linear.weight", "linear.bias"}
+ assert set(state_dict) == {"linear.weight", "linear.bias", "linear.act_max_scale"}
assert torch.equal(state_dict["linear.weight"], expected_weight)
- assert hasattr(model.linear, "orig_layer")
+ # Save-time should keep in-memory wrappers unchanged; replacement happens on load.
+ assert hasattr(saved_model.linear, "orig_layer")
with open(os.path.join(output_dir, "config.json")) as config_file:
quantization_config = json.load(config_file)["quantization_config"]
assert "supported_types" not in quantization_config
assert quantization_config["packing_format"] == "auto_round:fake"
+ assert quantization_config["quant_method"] == "auto-round"
+ assert quantization_config["act_bits"] == 4
+ assert quantization_config["block_name_to_quantize"] == ["block"]
+
+ loaded_model = _TinyLoadModel(SimpleNamespace(**quantization_config))
+ loaded_model, used_backends = convert_hf_model(loaded_model, target_device="cpu")
+ assert used_backends == ["auto_round:fake"]
+ _assert_has_act_hook(loaded_model.block.linear)
+ roundtrip_activation = torch.randn(2, 3, 16)
+ assert not torch.equal(loaded_model.block.linear.qdq_input(roundtrip_activation), roundtrip_activation)
class _TinyLoadModel(torch.nn.Module):
@@ -117,7 +140,7 @@ def test_fake_config_replaces_linear_and_qdq_activation_on_load():
model, used_backends = convert_hf_model(model, target_device="cpu")
assert used_backends == ["auto_round:fake"]
- assert isinstance(model.block.linear, FakeActQuantLinear)
+ _assert_has_act_hook(model.block.linear)
assert torch.equal(model.block.linear.weight, original_weight)
qdq_activation = model.block.linear.qdq_input(activation)
assert not torch.equal(qdq_activation, activation)
@@ -183,6 +206,218 @@ def test_transformers_load_replaces_fake_linear(tmp_path):
loaded_model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="cpu")
q_proj = loaded_model.model.decoder.layers[0].self_attn.q_proj
- assert isinstance(q_proj, FakeActQuantLinear)
+ _assert_has_act_hook(q_proj)
activation = torch.randn(1, 2, 16)
assert not torch.equal(q_proj.qdq_input(activation), activation)
+
+
+def test_fake_format_keeps_woq_packing_format(tmp_path):
+ model = _SaveableModel()
+ output_dir = str(tmp_path / "woq_model")
+
+ FakeFormat("fake", PRESET_SCHEMES["INT4"], SimpleNamespace(mllm=False)).save_quantized(
+ output_dir=output_dir,
+ model=model,
+ inplace=False,
+ serialization_dict={
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ "data_type": "int",
+ "quant_method": "auto-round",
+ "packing_format": "auto_round:auto_gptq",
+ "to_quant_block_names": ["block"],
+ "supported_types": [torch.nn.Linear],
+ },
+ )
+
+ with open(os.path.join(output_dir, "config.json")) as config_file:
+ quantization_config = json.load(config_file)["quantization_config"]
+
+ assert quantization_config["packing_format"] == "auto_round:auto_gptq"
+ assert "act_bits" not in quantization_config
+
+
+def test_fake_format_still_saves_when_env_disabled(tmp_path):
+ model = _SaveableModel()
+ output_dir = str(tmp_path / "save_model_even_when_env_disabled")
+
+ returned = FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(
+ output_dir=output_dir,
+ model=model,
+ inplace=False,
+ serialization_dict={
+ "bits": 4,
+ "group_size": 16,
+ "sym": True,
+ "data_type": "nvfp4_v2",
+ "act_bits": 4,
+ "act_group_size": 16,
+ "act_sym": True,
+ "act_data_type": "nvfp4_v2",
+ "to_quant_block_names": ["block"],
+ "supported_types": [torch.nn.Linear],
+ },
+ )
+ assert returned is not None
+ assert os.path.exists(output_dir)
+ assert os.path.exists(os.path.join(output_dir, "config.json"))
+
+
+def test_fake_backend_accepts_mxfp_roundtrip_config():
+ layer_backend = get_layer_backend(
+ "cpu",
+ "auto",
+ "auto_round:fake",
+ {
+ "bits": 4,
+ "group_size": 32,
+ "sym": True,
+ "data_type": "mx_fp",
+ "act_bits": 4,
+ "act_group_size": 32,
+ "act_sym": True,
+ "act_data_type": "mx_fp",
+ "act_dynamic": True,
+ },
+ 32,
+ 32,
+ )
+
+ assert layer_backend == "auto_round:fake"
+
+
+def test_fake_backend_accepts_mxfp8_roundtrip_config():
+ layer_backend = get_layer_backend(
+ "cpu",
+ "auto",
+ "auto_round:fake",
+ {
+ "bits": 8,
+ "group_size": 32,
+ "sym": True,
+ "data_type": "mx_fp",
+ "act_bits": 8,
+ "act_group_size": 32,
+ "act_sym": True,
+ "act_data_type": "mx_fp",
+ "act_dynamic": True,
+ },
+ 32,
+ 32,
+ )
+
+ assert layer_backend == "auto_round:fake"
+
+
+def test_fake_backend_accepts_nvfp_roundtrip_config():
+ layer_backend = get_layer_backend(
+ "cpu",
+ "auto",
+ "auto_round:fake",
+ {
+ "bits": 4,
+ "group_size": 16,
+ "sym": True,
+ "data_type": "nv_fp",
+ "act_bits": 4,
+ "act_group_size": 16,
+ "act_sym": True,
+ "act_data_type": "nv_fp",
+ "act_dynamic": True,
+ },
+ 16,
+ 16,
+ )
+
+ assert layer_backend == "auto_round:fake"
+
+
+def test_fake_backend_fallback_for_woq_w4a8_dynamic():
+ """WOQ packing with unsupported act scheme should gracefully fallback to fake backend."""
+ layer_backend = get_layer_backend(
+ "cpu",
+ "auto",
+ "auto_round:auto_gptq",
+ {
+ "bits": 4,
+ "group_size": 128,
+ "sym": True,
+ "data_type": "int",
+ "act_bits": 8,
+ "act_group_size": 32,
+ "act_sym": True,
+ "act_data_type": "int",
+ "act_dynamic": True,
+ },
+ 3072,
+ 768,
+ )
+
+ assert layer_backend == "auto_round:fake"
+
+
+def test_fake_format_keeps_in_memory_wrapper_structure_on_save(tmp_path):
+ model = _SaveableModel()
+ output_dir = str(tmp_path / "fake_keep_wrapper_model")
+
+ saved_model = FakeFormat("fake", PRESET_SCHEMES["NVFP4_E5M3"], SimpleNamespace(mllm=False)).save_quantized(
+ output_dir=output_dir,
+ model=model,
+ inplace=False,
+ serialization_dict={
+ "bits": 4,
+ "group_size": 16,
+ "sym": True,
+ "data_type": "nvfp4_v2",
+ "act_bits": 4,
+ "act_group_size": 16,
+ "act_sym": True,
+ "act_data_type": "nvfp4_v2",
+ "to_quant_block_names": ["block"],
+ },
+ )
+
+ assert hasattr(saved_model.linear, "orig_layer")
+ assert not isinstance(saved_model.linear, FakeActQuantLinear)
+
+
+def test_normalize_state_dict_keys_strips_orig_layer_segment():
+ state = {
+ "model.layers.0.fc1.orig_layer.weight": torch.ones(1),
+ "model.layers.0.fc1.orig_layer.bias": torch.zeros(1),
+ "model.layers.0.fc2.weight": torch.randn(1),
+ }
+
+ normalized, changed = _normalize_state_dict_keys(state)
+
+ assert changed is True
+ assert "model.layers.0.fc1.weight" in normalized
+ assert "model.layers.0.fc1.bias" in normalized
+ assert "model.layers.0.fc1.orig_layer.weight" not in normalized
+ assert "model.layers.0.fc2.weight" in normalized
+
+
+def test_rewrite_saved_weights_without_orig_layer_for_safetensors(tmp_path):
+ from safetensors.torch import load_file as safe_load_file
+ from safetensors.torch import save_file as safe_save_file
+
+ ckpt_dir = tmp_path / "fake_ckpt"
+ ckpt_dir.mkdir(parents=True)
+ ckpt_path = ckpt_dir / "model.safetensors"
+
+ safe_save_file(
+ {
+ "model.layers.0.fc1.orig_layer.weight": torch.randn(2, 2),
+ "model.layers.0.fc1.orig_layer.bias": torch.randn(2),
+ "model.layers.0.fc2.weight": torch.randn(2, 2),
+ },
+ str(ckpt_path),
+ )
+
+ _rewrite_saved_weights_without_orig_layer(str(ckpt_dir))
+
+ rewritten = safe_load_file(str(ckpt_path))
+ assert "model.layers.0.fc1.orig_layer.weight" not in rewritten
+ assert "model.layers.0.fc1.weight" in rewritten
+ assert "model.layers.0.fc1.bias" in rewritten
diff --git a/test/unit/test_cuda/quantization/test_act_quantization.py b/test/unit/test_cuda/quantization/test_act_quantization.py
index f100ae4c9..295510c32 100644
--- a/test/unit/test_cuda/quantization/test_act_quantization.py
+++ b/test/unit/test_cuda/quantization/test_act_quantization.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
import shutil
import pytest
@@ -19,6 +20,8 @@ import torch
from transformers import AutoModelForCausalLM
from auto_round import AutoRound
+from auto_round.experimental.qmodules.fake import FakeActQuantLinear
+from auto_round.wrapper import WrapperWALayer
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires CUDA")
@@ -132,3 +135,74 @@ class TestActQuantizationGpu:
)
with pytest.raises(ValueError, match="does not support exporting"):
autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
+
+ @pytest.mark.timeout(180)
+ def test_w4a8_reload_layer_type_diff_but_output_equal(self, tiny_opt_model_path):
+ """Save path keeps wrappers; reload path materializes FakeActQuantLinear, outputs should match."""
+ old_offload = os.environ.get("AR_DISABLE_OFFLOAD")
+ os.environ["AR_DISABLE_OFFLOAD"] = "1"
+ try:
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ bits=4,
+ act_bits=8,
+ group_size=128,
+ act_group_size=32,
+ act_dynamic=True,
+ sym=True,
+ iters=0,
+ disable_opt_rtn=True,
+ nsamples=2,
+ seqlen=16,
+ )
+ quantized_model, quantized_model_path = autoround.quantize_and_save(
+ output_dir=self.save_dir, format="auto_round"
+ )
+
+ quantized_model.eval()
+ reloaded_model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path, device_map="cuda:0", trust_remote_code=True
+ )
+ reloaded_model.eval()
+
+ src_layer = quantized_model.model.decoder.layers[0].fc2
+ dst_layer = reloaded_model.model.decoder.layers[0].fc2
+ assert isinstance(src_layer, WrapperWALayer)
+ assert isinstance(dst_layer, FakeActQuantLinear)
+ assert type(src_layer) is not type(dst_layer)
+
+ state_path = os.path.join(quantized_model_path, "model.safetensors")
+ if os.path.exists(state_path):
+ from safetensors.torch import load_file as load_state_dict
+
+ state_dict = load_state_dict(state_path)
+ else:
+ state_dict = torch.load(os.path.join(quantized_model_path, "pytorch_model.bin"), weights_only=True)
+
+ weight_key = "model.decoder.layers.0.fc2.weight"
+ bias_key = "model.decoder.layers.0.fc2.bias"
+ assert weight_key in state_dict
+ assert bias_key in state_dict
+ assert "model.decoder.layers.0.fc2.orig_layer.weight" not in state_dict
+
+ torch.manual_seed(7)
+ probe = torch.randn(2, 3, dst_layer.in_features, device="cuda:0", dtype=dst_layer.weight.dtype)
+ with torch.no_grad():
+ qdq_probe = dst_layer.qdq_input(probe)
+ src_out = (
+ torch.nn.functional.linear(
+ qdq_probe,
+ state_dict[weight_key].to(qdq_probe.device, qdq_probe.dtype),
+ state_dict[bias_key].to(qdq_probe.device, qdq_probe.dtype),
+ )
+ .float()
+ .cpu()
+ )
+ dst_out = dst_layer(probe).float().cpu()
+
+ torch.testing.assert_close(src_out, dst_out, rtol=1e-3, atol=1e-3)
+ finally:
+ if old_offload is None:
+ os.environ.pop("AR_DISABLE_OFFLOAD", None)
+ else:
+ os.environ["AR_DISABLE_OFFLOAD"] = old_offload