Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

4.3 KB
Raw Blame
import torch
import torch.nn as nn


# ================= simple multimodal model =================
class TextEncoder(nn.Module):
    def __init__(self, input_size, hidden_size):
        super().__init__()
        self.fc = nn.Linear(input_size, hidden_size)

    def forward(self, x):
        return self.fc(x)


class VisionEncoder(nn.Module):
    def __init__(self, input_size, hidden_size):
        super().__init__()
        self.fc = nn.Linear(input_size, hidden_size)

    def forward(self, x):
        return self.fc(x)


class TextEncoderModuleList(nn.ModuleList):
    pass


class VisionEncoderModuleList(nn.ModuleList):
    pass


class SimpleMultimodalModel(nn.Module):
    def __init__(self, text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders):
        super().__init__()
        self.text_encoders = TextEncoderModuleList(
            [TextEncoder(text_input_size, hidden_size) for _ in range(num_text_encoders)]
        )
        self.vision_encoders = VisionEncoderModuleList(
            [VisionEncoder(image_input_size, hidden_size) for _ in range(num_image_encoders)]
        )
        self.classifier = nn.Linear(hidden_size * 2, 1)  # Binary classification task

    def forward(self, text_input, image_input):
        text_features = [encoder(text_input) for encoder in self.text_encoders]
        image_features = [encoder(image_input) for encoder in self.vision_encoders]

        # Summing the outputs of all text encoders and image encoders
        combined_text_features = torch.stack(text_features, dim=1).sum(dim=1)
        combined_image_features = torch.stack(image_features, dim=1).sum(dim=1)

        combined_features = torch.cat((combined_text_features, combined_image_features), dim=1)
        output = self.classifier(combined_features)
        return output


# ================= simple MoE model =================
class Expert(nn.Module):
    def __init__(self, input_size, hidden_size):
        super().__init__()
        self.fc = nn.Linear(input_size, hidden_size)

    def forward(self, x):
        return self.fc(x)


class ExpertGroupModuleList(nn.ModuleList):
    pass


class ExpertModuleList(nn.ModuleList):
    pass


class NestedMoEModel(nn.Module):
    def __init__(self, input_size, hidden_size, num_groups, experts_per_group):
        super().__init__()
        self.expert_groups = ExpertModuleList(
            [
                ExpertGroupModuleList([Expert(input_size, hidden_size) for _ in range(experts_per_group)])
                for _ in range(num_groups)
            ]
        )
        self.gate = nn.Linear(input_size, num_groups * experts_per_group)

    def forward(self, x):
        gate_outputs = torch.softmax(self.gate(x), dim=1)
        expert_outputs = []

        for group in self.expert_groups:
            group_outputs = [expert(x) for expert in group]
            group_output_sum = torch.stack(group_outputs, dim=1).sum(dim=1)
            expert_outputs.append(group_output_sum)

        expert_outputs = torch.stack(expert_outputs, dim=1).view(x.size(0), -1)
        output = torch.sum(gate_outputs.unsqueeze(2) * expert_outputs.unsqueeze(1), dim=1)
        return output


class TestQuantizationBlocks:

    def test_moe_quant(self):
        input_size = 10
        hidden_size = 10
        num_groups = 2
        experts_per_group = 2
        self.model = NestedMoEModel(input_size, hidden_size, num_groups, experts_per_group)
        from auto_round.utils import get_block_names

        llm_block_names = get_block_names(self.model)
        all_block_names = []
        all_block_names = get_block_names(self.model, quant_vision=True)
        assert len(llm_block_names) == len(all_block_names)

    def test_multimodal_quant(self):
        num_text_encoders = 1
        num_image_encoders = 1
        image_input_size = 10
        text_input_size = 10
        hidden_size = 10
        self.model = SimpleMultimodalModel(
            text_input_size, image_input_size, hidden_size, num_text_encoders, num_image_encoders
        )
        from auto_round.utils import get_block_names

        llm_block_names = get_block_names(self.model)
        block_names_wo_vision = get_block_names(self.model, quant_vision=False)
        block_names_with_vision = get_block_names(self.model, quant_vision=True)
        assert block_names_wo_vision == llm_block_names
        assert len(block_names_wo_vision) != (block_names_with_vision)