PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
import torch
import torch.nn as nn
# ================= simple multimodal model =================
class TextEncoder(nn.Module):
def __init__(self, input_size, hidden_size):
super().__init__()
self.fc = nn.Linear(input_size, hidden_size)
def forward(self, x):
return self.fc(x)
class VisionEncoder(nn.Module):
def __init__(self, input_size, hidden_size):
super().__init__()
self.fc = nn.Linear(input_size, hidden_size)
def forward(self, x):
return self.fc(x)
class TextEncoderModuleList(nn.ModuleList):
pass
class VisionEncoderModuleList(nn.ModuleList):
pass
class SimpleMultimodalModel(nn.Module):
def __init__(self, text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders):
super().__init__()
self.text_encoders = TextEncoderModuleList(
[TextEncoder(text_input_size, hidden_size) for _ in range(num_text_encoders)]
)
self.vision_encoders = VisionEncoderModuleList(
[VisionEncoder(image_input_size, hidden_size) for _ in range(num_image_encoders)]
)
self.classifier = nn.Linear(hidden_size * 2, 1) # Binary classification task
def forward(self, text_input, image_input):
text_features = [encoder(text_input) for encoder in self.text_encoders]
image_features = [encoder(image_input) for encoder in self.vision_encoders]
# Summing the outputs of all text encoders and image encoders
combined_text_features = torch.stack(text_features, dim=1).sum(dim=1)
combined_image_features = torch.stack(image_features, dim=1).sum(dim=1)
combined_features = torch.cat((combined_text_features, combined_image_features), dim=1)
output = self.classifier(combined_features)
return output
# ================= simple MoE model =================
class Expert(nn.Module):
def __init__(self, input_size, hidden_size):
super().__init__()
self.fc = nn.Linear(input_size, hidden_size)
def forward(self, x):
return self.fc(x)
class ExpertGroupModuleList(nn.ModuleList):
pass
class ExpertModuleList(nn.ModuleList):
pass
class NestedMoEModel(nn.Module):
def __init__(self, input_size, hidden_size, num_groups, experts_per_group):
super().__init__()
self.expert_groups = ExpertModuleList(
[
ExpertGroupModuleList([Expert(input_size, hidden_size) for _ in range(experts_per_group)])
for _ in range(num_groups)
]
)
self.gate = nn.Linear(input_size, num_groups * experts_per_group)
def forward(self, x):
gate_outputs = torch.softmax(self.gate(x), dim=1)
expert_outputs = []
for group in self.expert_groups:
group_outputs = [expert(x) for expert in group]
group_output_sum = torch.stack(group_outputs, dim=1).sum(dim=1)
expert_outputs.append(group_output_sum)
expert_outputs = torch.stack(expert_outputs, dim=1).view(x.size(0), -1)
output = torch.sum(gate_outputs.unsqueeze(2) * expert_outputs.unsqueeze(1), dim=1)
return output
class TestQuantizationBlocks:
def test_moe_quant(self):
input_size = 10
hidden_size = 10
num_groups = 2
experts_per_group = 2
self.model = NestedMoEModel(input_size, hidden_size, num_groups, experts_per_group)
from auto_round.utils import get_block_names
llm_block_names = get_block_names(self.model)
all_block_names = []
all_block_names = get_block_names(self.model, quant_vision=True)
assert len(llm_block_names) == len(all_block_names)
def test_multimodal_quant(self):
num_text_encoders = 1
num_image_encoders = 1
image_input_size = 10
text_input_size = 10
hidden_size = 10
self.model = SimpleMultimodalModel(
text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders
)
from auto_round.utils import get_block_names
llm_block_names = get_block_names(self.model)
block_names_wo_vision = get_block_names(self.model, quant_vision=False)
block_names_with_vision = get_block_names(self.model, quant_vision=True)
assert block_names_wo_vision == llm_block_names
assert len(block_names_wo_vision) != (block_names_with_vision)