PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from types import SimpleNamespace
import torch
from auto_round.calibration.inputs import preprocess_block_inputs, split_inputs
def test_split_inputs_llm():
inputs = {
"input_ids": torch.tensor([[1, 2, 3]]),
"attention_mask": torch.tensor([[1, 1, 1]]),
}
input_ids, input_others = split_inputs(inputs, "input_ids", is_diffusion=False)
assert torch.equal(input_ids, torch.tensor([[1, 2, 3]]))
assert "input_ids" not in inputs
assert input_others is inputs
assert set(input_others) == {"attention_mask"}
def test_split_inputs_diffusion():
inputs = {
"hidden_states": torch.randn(1, 4),
"encoder_hidden_states": torch.randn(1, 4),
"timestep": torch.tensor([1]),
}
input_ids, input_others = split_inputs(inputs, "ignored", is_diffusion=True)
assert set(input_ids) == {"hidden_states"}
assert "hidden_states" not in input_others
assert "encoder_hidden_states" in input_others
assert set(input_others) == {"encoder_hidden_states", "timestep"}
def test_preprocess_block_inputs_casts_tensors(monkeypatch):
calls = []
monkeypatch.setattr(
"auto_round.calibration.inputs.clear_memory",
lambda device_list: calls.append(tuple(device_list)),
)
monkeypatch.setattr("auto_round.calibration.inputs.device_manager", SimpleNamespace(device_list=["cpu"]))
model_context = SimpleNamespace(is_diffusion=False, amp=True, amp_dtype=torch.bfloat16)
compress_context = SimpleNamespace(cache_device=torch.device("cpu"), device_list=["cpu"])
inputs = {
"input_ids": torch.ones((1, 3), dtype=torch.float16),
"attention_mask": torch.ones((1, 3), dtype=torch.float16),
"position_ids": torch.ones((1, 3), dtype=torch.int64),
}
input_ids, input_others = preprocess_block_inputs(
inputs,
model_context=model_context,
compress_context=compress_context,
)
assert calls == [("cpu",)]
assert input_ids.dtype == torch.bfloat16
assert input_others["attention_mask"].dtype == torch.bfloat16
assert input_others["position_ids"].dtype == torch.int64
assert "input_ids" not in inputs
def test_preprocess_block_inputs_keeps_list_dtype():
model_context = SimpleNamespace(is_diffusion=False, amp=True, amp_dtype=torch.bfloat16)
compress_context = SimpleNamespace(cache_device=torch.device("cpu"), device_list=["cpu"])
cached_states = [torch.ones((1, 3), dtype=torch.float16)]
inputs = {
"input_ids": torch.ones((1, 3), dtype=torch.float16),
"past_key_values": cached_states,
}
_, input_others = preprocess_block_inputs(
inputs,
model_context=model_context,
compress_context=compress_context,
)
# After Fix gpt-j-6b runtime error, list items are properly converted
# to the AMP dtype. Previously the result of to_dtype() was not written back.
assert input_others["past_key_values"][0].dtype == torch.bfloat16