PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support code scan in CI test (#19)
chen, suyue528252b918f5
diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.ymlnew file mode 100644index 000000000..9f5b46e27--- /dev/null+++ b/.azure-pipelines/code-scan.yml@@ -0,0 +1,47 @@+trigger: none++pr:+ autoCancel: true+ drafts: false+ branches:+ include:+ - master+ paths:+ include:+ - auto_round+ - setup.py+ - requirements.txt+ - .azure-pipelines/code-scan.yml+ - .azure-pipelines/scripts/codeScan++pool:+ vmImage: "ubuntu-latest"++variables:+ CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog"++stages:++ - stage: BanditCodeScan+ displayName: Bandit Code Scan+ dependsOn: []+ jobs:+ - job: Bandit+ displayName: Bandit+ steps:+ - template: template/code-scan-template.yml+ parameters:+ codeScanFileName: "bandit"+ uploadPath: "bandit.log"++ - stage: PylintCodeScan+ displayName: Pylint Code Scan+ dependsOn: []+ jobs:+ - job: Pylint+ displayName: Pylint+ steps:+ - template: template/code-scan-template.yml+ parameters:+ codeScanFileName: "pylint"+ uploadPath: "pylint.json"diff --git a/.azure-pipelines/docker/DockerfileCodeScan.devel b/.azure-pipelines/docker/DockerfileCodeScan.develnew file mode 100644index 000000000..110269eaf--- /dev/null+++ b/.azure-pipelines/docker/DockerfileCodeScan.devel@@ -0,0 +1,36 @@+#+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++ARG UBUNTU_VER=22.04+FROM ubuntu:${UBUNTU_VER} as devel++# See http://bugs.python.org/issue19846+ENV LANG C.UTF-8++RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \+ aspell \+ aspell-en \+ python3 \+ python3-pip \+ python3-dev \+ python3-distutils \+ wget++RUN ln -sf $(which python3) /usr/bin/python++RUN python -m pip install --no-cache-dir pylint==2.12.1\+ bandit++WORKDIR /diff --git a/.azure-pipelines/scripts/change_color.sh b/.azure-pipelines/scripts/change_color.shnew file mode 100644index 000000000..51342efec--- /dev/null+++ b/.azure-pipelines/scripts/change_color.sh@@ -0,0 +1,81 @@+#!/bin/bash++# -------------- general approach start----------------++# 1. import this file:+ # source path/change_color.sh+# 2. use COLOR/BG:+ # $VARIABLE_NAME && out_put_content && $RESET+# 3. COLOR + BG:+ # $COLOR/BG_VARIABLE_NAME && $BG/COLOR_VARIABLE_NAME && out_put_content && $RESET+# 4. custom+ # abbreviation(change number)+ # txt number range (30, 37)+ # bg number range (40, 47)+ # special effects number range (1, 7)+ # echo -en \\E[number1 + ; + number2 + ; + number3 + m"+ # e.g - BG_GRAY+LIGHT_RED = "echo -en \\E[47;31m"++# -------------- general approach end----------------==+++# general setting+# ------------- light_color start----------------+# black+LIGHT_BLACK="echo -en \\E[30m"+# red+LIGHT_RED="echo -en \\E[31m"+# green+LIGHT_GREEN="echo -en \\E[32m"+# yellow+LIGHT_YELLOW="echo -en \\E[33m"+# blue+LIGHT_BLUE="echo -en \\E[34m"+# purple+LIGHT_PURPLE="echo -en \\E[35m"+# cyan+LIGHT_CYAN="echo -en \\E[36m"+# gray+LIGHT_GRAY="echo -en \\E[37m"+# ------------- light_color end----------------++# ------------- bold_color start----------------+# black+BOLD_BLACK="echo -en \\E[1;30m"+# red+BOLD_RED="echo -en \\E[1;31m"+# green+BOLD_GREEN="echo -en \\E[1;32m"+# yellow+BOLD_YELLOW="echo -en \\E[1;33m"+# blue+BOLD_BLUE="echo -en \\E[1;34m"+# purple+BOLD_PURPLE="echo -en \\E[1;35m"+# cyan+BOLD_CYAN="echo -en \\E[1;36m"+# gray+BOLD_GRAY="echo -en \\E[1;37m"+# ------------- bold_color end----------------++# ------------- background_color start----------------+# black+BG_BLACK="echo -en \\E[40m"+# red+BG_RED="echo -en \\E[41m"+# green+BG_GREEN="echo -en \\E[42m"+# yellow+BG_YELLOW="echo -en \\E[43m"+# blue+BG_BLUE="echo -en \\E[44m"+# purple+BG_PURPLE="echo -en \\E[45m"+# cyan+BG_CYAN="echo -en \\E[46m"+# gray+BG_GRAY="echo -en \\E[47m"+# ------------- background_color end----------------++# close+RESET="echo -en \\E[0m"diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.shnew file mode 100644index 000000000..07c1efc92--- /dev/null+++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh@@ -0,0 +1,34 @@+#!/bin/bash++for var in "$@"+do+ case $var in+ --scan_module=*)+ scan_module=$(echo $var |cut -f2 -d=)+ ;;+ esac+done++source /auto-round/.azure-pipelines/scripts/change_color.sh+RESET="echo -en \\E[0m \\n" # close color++log_dir="/auto-round/.azure-pipelines/scripts/codeScan/scanLog"+mkdir -p $log_dir++python -m bandit -r -lll -iii "/auto-round/${scan_module}" >$log_dir/bandit.log+exit_code=$?++$BOLD_YELLOW && echo " ----------------- Current bandit cmd start --------------------------" && $RESET+echo "python -m bandit -r -lll -iii /auto-round/${scan_module} > $log_dir/bandit.log"+$BOLD_YELLOW && echo " ----------------- Current bandit cmd end --------------------------" && $RESET++$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------"+cat $log_dir/bandit.log+$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET++if [ ${exit_code} -ne 0 ]; then+ $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Bandit error details." && $RESET+ exit 1+fi+$BOLD_PURPLE && echo "Congratulations, Bandit check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET+exit 0diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.shnew file mode 100644index 000000000..d9e2bca37--- /dev/null+++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh@@ -0,0 +1,43 @@+#!/bin/bash++for var in "$@"+do+ case $var in+ --scan_module=*)+ scan_module=$(echo $var |cut -f2 -d=)+ ;;+ esac+done++source /auto-round/.azure-pipelines/scripts/change_color.sh+RESET="echo -en \\E[0m \\n" # close color++log_dir="/auto-round/.azure-pipelines/scripts/codeScan/scanLog"+mkdir -p $log_dir++apt-get install -y --no-install-recommends --fix-missing \+ autoconf \+ build-essential++pip install -r /auto-round/requirements.txt++echo "[DEBUG] list pipdeptree..."+pip install pipdeptree+pipdeptree++python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto \+--ignored-modules=tensorflow,keras,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch,intel_extension_for_tensorflow,torchinfo,horovod,transformers \+/auto-round/${scan_module} > $log_dir/pylint.json++exit_code=$?++$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" && $RESET+cat $log_dir/pylint.json+$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET++if [ ${exit_code} -ne 0 ]; then+ $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pylint error details." && $RESET+ exit 1+fi+$BOLD_PURPLE && echo "Congratulations, Pylint check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET+exit 0diff --git a/.azure-pipelines/template/code-scan-template.yml b/.azure-pipelines/template/code-scan-template.ymlnew file mode 100644index 000000000..724454df6--- /dev/null+++ b/.azure-pipelines/template/code-scan-template.yml@@ -0,0 +1,42 @@+parameters:+ - name: codeScanFileName+ type: string+ - name: uploadPath+ type: string++ - name: codeScanContainerName+ type: string+ default: "codeScan"+ - name: scanModule+ type: string+ default: "auto_round"++steps:+ - template: docker-template.yml+ parameters:+ dockerConfigName: "commonDockerConfig"+ repoName: "code-scan"+ repoTag: "1.0"+ dockerFileName: "DockerfileCodeScan"+ containerName: ${{ parameters.codeScanContainerName }}++ - script: |+ docker exec ${{ parameters.codeScanContainerName }} bash -c "bash /auto-round/.azure-pipelines/scripts/codeScan/${{ parameters.codeScanFileName }}/${{ parameters.codeScanFileName }}.sh \+ --scan_module=${{ parameters.scanModule }}"+ displayName: "${{ parameters.codeScanFileName }} Check"++ - task: PublishPipelineArtifact@1+ condition: succeededOrFailed()+ inputs:+ targetPath: .azure-pipelines/scripts/codeScan/scanLog/${{ parameters.uploadPath }}+ artifact: ${{ parameters.codeScanFileName }}+ publishLocation: "pipeline"+ displayName: "PublishPipelineArtifact"++ - task: Bash@3+ condition: always()+ inputs:+ targetType: "inline"+ script: |+ docker exec ${{ parameters.codeScanContainerName }} bash -c "rm -fr /auto-round/* && rm -fr /auto-round/.* || true"+ displayName: "Docker clean up"diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.ymlnew file mode 100644index 000000000..0d7a9d889--- /dev/null+++ b/.azure-pipelines/template/docker-template.yml@@ -0,0 +1,86 @@+parameters:+ - name: dockerConfigName+ type: string+ default: "commonDockerConfig"+ - name: repoName+ type: string+ default: "auto-round"+ - name: repoTag+ type: string+ default: "py310"+ - name: dockerFileName+ type: string+ default: "Dockerfile"+ - name: containerName+ type: string+ - name: repo+ type: string+ default: "https://github.com/intel/auto-round"++steps:+ - task: Bash@3+ inputs:+ targetType: "inline"+ script: |+ docker ps -a+ if [[ $(docker ps -a | grep -i '${{ parameters.containerName }}'$) ]]; then+ docker start $(docker ps -aq)+ echo "remove left files through container ..."+ docker exec ${{ parameters.containerName }} bash -c "ls -a /auto-round && rm -fr /auto-round/* && rm -fr /auto-round/.* && ls -a /auto-round || true"+ fi+ displayName: "Docker clean up"++ - ${{ if eq(parameters.dockerConfigName, 'commonDockerConfig') }}:+ - script: |+ rm -fr ${BUILD_SOURCESDIRECTORY} || sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true+ echo y | docker system prune+ displayName: "Clean workspace"++ - checkout: self+ clean: true+ displayName: "Checkout out Repo"++ - ${{ if eq(parameters.dockerConfigName, 'gitCloneDockerConfig') }}:+ - script: |+ rm -fr ${BUILD_SOURCESDIRECTORY} || sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true+ mkdir ${BUILD_SOURCESDIRECTORY}+ chmod 777 ${BUILD_SOURCESDIRECTORY}+ echo y | docker system prune+ displayName: "Clean workspace"++ - checkout: none++ - script: |+ git clone ${{ parameters.repo }} ${BUILD_SOURCESDIRECTORY}+ git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY}+ cd ${BUILD_SOURCESDIRECTORY}+ git checkout master+ displayName: "Checkout out master"++ - script: |+ if [[ ! $(docker images | grep -i ${{ parameters.repoName }}:${{ parameters.repoTag }}) ]]; then+ docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/${{parameters.dockerFileName}}.devel -t ${{ parameters.repoName }}:${{ parameters.repoTag }} .+ fi+ docker images | grep -i ${{ parameters.repoName }}+ if [[ $? -ne 0 ]]; then+ echo "NO Such Repo"+ exit 1+ fi+ displayName: "Build develop docker image"++ - script: |+ docker stop $(docker ps -aq)+ docker rm -vf ${{ parameters.containerName }} || true+ env | sort+ displayName: "Clean docker container"++ - ${{ if ne(parameters.containerName, '') }}:+ - task: Bash@3+ inputs:+ targetType: "inline"+ script: |+ docker run -dit --disable-content-trust --privileged --name=${{ parameters.containerName }} --shm-size="2g" \+ -v ${BUILD_SOURCESDIRECTORY}:/auto-round -v /tf_dataset:/tf_dataset -v /tf_dataset2:/tf_dataset2 ${{ parameters.repoName }}:${{ parameters.repoTag }}+ echo "Show the container list after docker run ... "+ docker ps -a+ displayName: "Docker run - ${{ parameters.containerName }} Container"diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlnew file mode 100644index 000000000..f969d01f2--- /dev/null+++ b/.pre-commit-config.yaml@@ -0,0 +1,115 @@+ci:+ autofix_prs: true+ autoupdate_schedule: quarterly++exclude: |+ (?x)^(+ neural_coder/test/.+|+ neural_coder/examples/.++ )$++repos:+ - repo: https://github.com/pre-commit/pre-commit-hooks+ rev: v4.5.0+ hooks:+ - id: end-of-file-fixer+ files: (.*\.(py|md|rst|yaml|yml))$+ exclude: |+ (?x)^(+ examples/.++ )$+ - id: check-json+ exclude: |+ (?x)^(+ .vscode/settings_recommended.json+ )$+ - id: check-yaml+ - id: debug-statements+ - id: file-contents-sorter+ args: [--unique]+ - id: requirements-txt-fixer+ exclude: |+ (?x)^(+ examples/.++ )$+ - id: trailing-whitespace+ files: (.*\.(py|rst|cmake|yaml|yml))$+ exclude: |+ (?x)^(+ examples/.++ )$++ - repo: https://github.com/asottile/yesqa+ rev: v1.5.0+ hooks:+ - id: yesqa+ name: Unused noqa++ - repo: https://github.com/pycqa/isort+ rev: 5.13.2+ hooks:+ - id: isort+ exclude: |+ (?x)^(+ examples/.++ )$++ - repo: https://github.com/PyCQA/docformatter+ rev: v1.7.5+ hooks:+ - id: docformatter+ args: [+ --in-place,+ --wrap-summaries=0, # 0 means disable wrap+ --wrap-descriptions=0, # 0 means disable wrap+ --black,+ --style=google,+ ]+ exclude: |+ (?x)^(+ examples/.++ )$++ - repo: https://github.com/psf/black.git+ rev: 24.1.1+ hooks:+ - id: black+ files: (.*\.py)$+ exclude: |+ (?x)^(+ examples/.++ )$++ - repo: https://github.com/asottile/blacken-docs+ rev: 1.16.0+ hooks:+ - id: blacken-docs+ args: [--line-length=120, --skip-errors]+ additional_dependencies:+ - black==24.1.1+ exclude: |+ (?x)^(+ examples/.+|+ docs/.++ )$++ - repo: https://github.com/codespell-project/codespell+ rev: v2.2.6+ hooks:+ - id: codespell+ additional_dependencies:+ - tomli+ exclude: |+ (?x)^(+ examples/.*(txt|patch)|+ )$++ - repo: https://github.com/astral-sh/ruff-pre-commit+ rev: v0.1.15+ hooks:+ - id: ruff+ args: [--fix, --exit-non-zero-on-fix, --no-cache]+ exclude: |+ (?x)^(+ examples/.++ )$diff --git a/README.md b/README.mdindex 93553a103..eb8516e7d 100644--- a/README.md+++ b/README.md@@ -32,16 +32,16 @@ model_name = "meta-llama/Llama-2-7b-hf"model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)bits, group_size, scheme = 4, 128, "asym"-tuning_device = "cuda:0" ## or "cpu"+tuning_device = "cuda:0" ## or "cpu"autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device=tuning_device)autoround.quantize()output_dir = "./tmp_autoround"-deployment_device = "cpu" ## or gpu+deployment_device = "cpu" ## or gpuif deployment_device == "cpu":- autoround.save_quantized(output_dir, format="itrex") ## export to itrex format+ autoround.save_quantized(output_dir, format="itrex") ## export to itrex formatelse:- autoround.save_quantized(output_dir, format="auto_gptq", use_triton=True) ##export to autogptq format+ autoround.save_quantized(output_dir, format="auto_gptq", use_triton=True) ##export to autogptq format```@@ -51,15 +51,14 @@ else:from transformers import AutoModelForCausalLM, AutoTokenizermodel_name = "meta-llama/Llama-2-7b-hf"-model = AutoModelForCausalLM.from_pretrained(- model_name, torch_dtype="auto", trust_remote_code=True)+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)bits, group_size, scheme = 4, 128, "asym"# need to load model first, then importfrom auto_round import AutoRound-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme,- device="hpu", amp=False)++autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device="hpu", amp=False)autoround.quantize()```@@ -125,25 +124,26 @@ Please run the tuning code first# currently please install neural-speed (https://github.com/intel/neural-speed) from sourcefrom intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfigfrom transformers import AutoTokenizer+quantized_model_path = "./tmp_autoround"-woq_config = WeightOnlyQuantConfig(group_size=group_size, scheme=scheme,- use_autoround=True) ##only supports 4 bits currently+woq_config = WeightOnlyQuantConfig(+ group_size=group_size, scheme=scheme, use_autoround=True+) ##only supports 4 bits currentlyprompt = "There is a girl who likes adventure,"tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, trust_remote_code=True)inputs = tokenizer(prompt, return_tensors="pt").input_ids-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, quantization_config=woq_config,- trust_remote_code=True, device="cpu")+model = AutoModelForCausalLM.from_pretrained(+ quantized_model_path, quantization_config=woq_config, trust_remote_code=True, device="cpu"+)outputs = model.generate(inputs, max_new_tokens=50)```### GPU```python# save_quantized to autogptq format first and then follow transformers or auto-gptq to load the model and inferencefrom transformers import AutoModelForCausalLM, AutoTokenizer+quantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map="auto",- trust_remote_code=True- )+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -197,6 +197,3 @@ If you find SignRound useful for your research, please cite our paper:year={2023}}```---diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex 01ad1c826..d26edcbf7 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -15,4 +15,3 @@ from .autoround import AutoRound, AutoAdamRound, AutoOPTRoundfrom .version import __version__from auto_round.export.export_to_itrex import compress_model, QuantConfigfrom auto_round.export.export_to_autogptq import save_quantized_to_autogptq-diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex eb3c2c1c5..5c5f53cd3 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -13,13 +13,30 @@# limitations under the License.-import torch-import copyimport time-from .utils import (quant_weight, set_module, get_module, get_block_names, block_forward, sampling_inputs,- get_scale_shape, move_input_to_device, check_is_cpu, collect_round_v,- collect_minmax_scale, get_batch_dim, htcore, is_hpu_available,check_to_quantized,logger)++import torch++from .utils import (+ block_forward,+ check_is_cpu,+ check_to_quantized,+ collect_minmax_scale,+ collect_round_v,+ get_batch_dim,+ get_block_names,+ get_module,+ get_scale_shape,+ htcore,+ is_hpu_available,+ logger,+ move_input_to_device,+ quant_weight,+ sampling_inputs,+ set_module,+)+class SaveInputs:"""Cache the inputs of the first block."""@@ -158,6 +175,7 @@ class SaveInputs:def _replace_forward(self):"""Replaces the forward function."""from functools import partial+for n, m in self.model.named_modules():if n == self.block_name:m.orig_forward = m.forward@@ -246,12 +264,19 @@ class WrapperLinear(torch.nn.Module):- torch.Tensor: The output tensor after applying the linear transformation with quantized weights."""from torch.functional import F+weight = self.orig_layer.weightself.min_scale.data.copy_(torch.clamp(self.min_scale.data, -1, 0))self.max_scale.data.copy_(torch.clamp(self.max_scale.data, -1, 0))weight_q, _, _ = quant_weight(- weight, self.num_bits, self.group_size, self.scheme, self.value, self.min_scale, self.max_scale,- self.scale_dtype+ weight,+ self.num_bits,+ self.group_size,+ self.scheme,+ self.value,+ self.min_scale,+ self.max_scale,+ self.scale_dtype,)weight_q = weight_q.to(weight.dtype)# pylint: disable=not-callable@@ -312,14 +337,7 @@ class WrapperTransformerConv1d(torch.nn.Module):min_scale.clamp_(-1, 0)max_scale.clamp_(-1, 0)weight_q, scale, zp = quant_weight(- self.weight_t,- self.num_bits,- self.group_size,- self.scheme,- v,- min_scale,- max_scale,- self.scale_dtype+ self.weight_t, self.num_bits, self.group_size, self.scheme, v, min_scale, max_scale, self.scale_dtype)self.orig_layer.weight.data.copy_(weight_q.t())self.orig_layer.weight.grad = None@@ -347,7 +365,7 @@ class WrapperTransformerConv1d(torch.nn.Module):self.value,self.min_scale,self.max_scale,- self.scale_dtype+ self.scale_dtype,)weight_q = weight_q.to(self.weight_t.dtype)size_out = x.size()[:-1] + (self.orig_layer.nf,)@@ -489,40 +507,41 @@ class AutoRound(object):"""def __init__(- self,- model,- tokenizer,- bits: int = 4,- group_size: int = 128,- scheme: str = "asym",- weight_config: dict = {},- enable_full_range: bool = False, ##for symmetric, TODO support later- bs: int = 8,- amp: bool = True,- device="cuda:0",- lr_scheduler=None,- dataloader=None, ## to support later- dataset_name: str = "NeelNanda/pile-10k",- dataset_split: str = "train",- use_quant_input: bool = True,- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = True,- iters: int = 200,- seqlen: int = 2048,- n_samples: int = 512,- sampler: str = "rand",- seed: int = 42,- n_blocks: int = 1,- gradient_accumulate_steps: int = 1,- not_use_best_mse: bool = False,- dynamic_max_gap: int = -1,- data_type: str = "int", ##only support data_type- scale_dtype="fp16",- **kwargs,+ self,+ model,+ tokenizer,+ bits: int = 4,+ group_size: int = 128,+ scheme: str = "asym",+ weight_config: dict = {},+ enable_full_range: bool = False, ##for symmetric, TODO support later+ bs: int = 8,+ amp: bool = True,+ device="cuda:0",+ lr_scheduler=None,+ dataloader=None, ## to support later+ dataset_name: str = "NeelNanda/pile-10k",+ dataset_split: str = "train",+ use_quant_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = None,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = True,+ iters: int = 200,+ seqlen: int = 2048,+ n_samples: int = 512,+ sampler: str = "rand",+ seed: int = 42,+ n_blocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_best_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int", ##only support data_type+ scale_dtype="fp16",+ **kwargs,):from .calib_dataset import CALIB_DATASETS+self.model = modelself.model_orig_dtype = model.dtypeself.model = self.model.to("cpu")@@ -552,9 +571,9 @@ class AutoRound(object):self.n_blocks = n_blocksself.device = device- if scale_dtype == 'fp16':+ if scale_dtype == "fp16":self.scale_dtype = torch.float16- elif scale_dtype == 'bf16':+ elif scale_dtype == "bf16":self.scale_dtype = torch.bfloat16else:self.scale_dtype = torch.float32@@ -573,16 +592,14 @@ class AutoRound(object):self.dataset_name = dataset_nameif dataloader is None:- get_dataloader = CALIB_DATASETS.get(self.dataset_name,- CALIB_DATASETS["NeelNanda/pile-10k"])+ get_dataloader = CALIB_DATASETS.get(self.dataset_name, CALIB_DATASETS["NeelNanda/pile-10k"])self.dataloader = get_dataloader(self.tokenizer,self.seqlen,seed=self.seed,bs=self.train_bs,split=self.dataset_split,- dataset_name=self.dataset_name-+ dataset_name=self.dataset_name,)else:self.dataloader = dataloader@@ -770,6 +787,7 @@ class AutoRound(object):Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output)"""from torch.amp import autocast+batch_dim = get_batch_dim(input_others)if not self.low_gpu_mem_usage and input_ids.device != device:input_ids = move_input_to_device(input_ids, device)@@ -881,12 +899,12 @@ class AutoRound(object):return None, outputdef qdq_weight_round(- self,- model: torch.nn.Module,- inputs,- block_names,- n_blocks=1,- device=torch.device("cpu"),+ self,+ model: torch.nn.Module,+ inputs,+ block_names,+ n_blocks=1,+ device=torch.device("cpu"),):"""Quantize and dequantize the weights of the specified blocks in the model.@@ -914,7 +932,7 @@ class AutoRound(object):logger.info(f"quantizing {i + 1}/{len(block_names)}, {n}")m = get_module(model, n)else:- names = block_names[i: i + n_blocks]+ names = block_names[i : i + n_blocks]logger.info(names)modules = [get_module(model, n) for n in names]m = WrapperMultiblock(modules)@@ -949,9 +967,7 @@ class AutoRound(object):return compress_modeldef save_quantized_as_autogptq(self, output_dir, use_triton=False, inplace=True):- """- Export the model to autogptq format to easily leverage cuda kernel- """+ """Export the model to autogptq format to easily leverage cuda kernel."""if not self.quantized:logger.warning("please run autoround.quantize first")return@@ -978,14 +994,15 @@ class AutoRound(object):modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptqif all_to_quantized:modules_in_block_to_quantize = None-+if inplace:model = self.model.to("cpu")else:model = copy.deepcopy(self.model.to("cpu"))from auto_gptq.modeling._utils import pack_model- sym = self.scheme == 'sym'++ sym = self.scheme == "sym"if self.bits == 3 or use_triton is False:if self.bits == 3 and use_triton is True:logger.warning("triton does not support 3 bits, reset it to False")@@ -994,35 +1011,70 @@ class AutoRound(object):info = self.weight_config[key]if not check_to_quantized(info):continue- quantizers[key] = (None, info['scale'], info['zp'], info['g_idx'])- pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,- force_layer_back_to_cpu=True, use_triton=False)+ quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])+ pack_model(+ model,+ quantizers,+ self.bits,+ self.group_size,+ use_cuda_fp16=True,+ desc_act=False,+ force_layer_back_to_cpu=True,+ use_triton=False,+ )else:quantizers = {}for key in self.weight_config:info = self.weight_config[key]if not check_to_quantized(info):continue- info['zp'] = info['zp'].to(torch.float32)- quantizers[key] = (None, info['scale'].to(torch.float32), info['zp'], info['g_idx'])- pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,- force_layer_back_to_cpu=True, use_triton=True)+ info["zp"] = info["zp"].to(torch.float32)+ quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"], info["g_idx"])+ pack_model(+ model,+ quantizers,+ self.bits,+ self.group_size,+ use_cuda_fp16=True,+ desc_act=False,+ force_layer_back_to_cpu=True,+ use_triton=True,+ )from auto_round import save_quantized_to_autogptq- save_quantized_to_autogptq(model, output_dir, bits=self.bits, group_size=self.group_size, sym=sym,- iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,- enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,- scale_dtype=self.scale_dtype,- use_safetensors=True, modules_in_block_to_quantize=modules_in_block_to_quantize)++ save_quantized_to_autogptq(+ model,+ output_dir,+ bits=self.bits,+ group_size=self.group_size,+ sym=sym,+ iters=self.iters,+ lr=self.lr,+ minmax_lr=self.minmax_lr,+ enable_minmax_tuning=self.enable_minmax_tuning,+ use_quant_input=self.use_quant_input,+ scale_dtype=self.scale_dtype,+ use_safetensors=True,+ modules_in_block_to_quantize=modules_in_block_to_quantize,+ )def save_quantized_as_itrex(self, output_dir, inplace=True):"""Save configure file and weights for CPU backend inference."""- from auto_round.export.export_to_itrex import compress_model, QuantConfig+ from auto_round.export.export_to_itrex import QuantConfig, compress_model+compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)sym = self.scheme == "sym"- quantize_config = QuantConfig(bits=self.bits, group_size=self.group_size, sym=sym,- iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,- enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,- scale_dtype=str(self.scale_dtype))+ quantize_config = QuantConfig(+ bits=self.bits,+ group_size=self.group_size,+ sym=sym,+ iters=self.iters,+ lr=self.lr,+ minmax_lr=self.minmax_lr,+ enable_minmax_tuning=self.enable_minmax_tuning,+ use_quant_input=self.use_quant_input,+ scale_dtype=str(self.scale_dtype),+ )if quantize_config is not None:config = compressed_model.configsetattr(config, "quantization_config", quantize_config.to_dict())@@ -1081,10 +1133,12 @@ class AutoRound(object):self.weight_config[n]["zp"] = m.zpif self.group_size <= 0:self.weight_config[n]["g_idx"] = torch.tensor(- [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu")+ [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"+ )else:self.weight_config[n]["g_idx"] = torch.tensor(- [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu")+ [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"+ )delattr(m, "scale")delattr(m, "zp")else:@@ -1144,38 +1198,38 @@ class AutoOPTRound(AutoRound):"""def __init__(- self,- model,- tokenizer=None,- bits: int = 4,- group_size: int = 128,- scheme: str = "asym",- weight_config: dict = {},- enable_full_range: bool = False,- bs: int = 8,- amp: bool = True,- device="cuda:0",- lr_scheduler=None,- dataloader=None,- dataset_name: str = "NeelNanda/pile-10k",- dataset_split: str = "train",- use_quant_input: bool = True,- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = True,- iters: int = 200,- seqlen: int = 2048,- n_samples: int = 512,- sampler: str = "rand",- seed: int = 42,- n_blocks: int = 1,- gradient_accumulate_steps: int = 1,- not_use_best_mse: bool = False,- dynamic_max_gap: int = -1,- data_type: str = "int",- optimizer="AdamW",- **kwargs,+ self,+ model,+ tokenizer=None,+ bits: int = 4,+ group_size: int = 128,+ scheme: str = "asym",+ weight_config: dict = {},+ enable_full_range: bool = False,+ bs: int = 8,+ amp: bool = True,+ device="cuda:0",+ lr_scheduler=None,+ dataloader=None,+ dataset_name: str = "NeelNanda/pile-10k",+ dataset_split: str = "train",+ use_quant_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = None,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = True,+ iters: int = 200,+ seqlen: int = 2048,+ n_samples: int = 512,+ sampler: str = "rand",+ seed: int = 42,+ n_blocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_best_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int",+ optimizer="AdamW",+ **kwargs,):super(AutoOPTRound, self).__init__(model,@@ -1294,38 +1348,38 @@ class AutoAdamRound(AutoOPTRound):"""def __init__(- self,- model,- tokenizer=None,- bits: int = 4,- group_size: int = 128,- scheme: str = "asym",- weight_config: dict = {},- enable_full_range: bool = False,- bs: int = 8,- amp: bool = True,- device="cuda:0",- lr_scheduler=None,- dataloader=None,- dataset_name: str = "NeelNanda/pile-10k",- dataset_split: str = "train",- use_quant_input: bool = True,- enable_minmax_tuning: bool = True,- lr: float = None,- minmax_lr: float = None,- low_gpu_mem_usage: bool = True,- iters: int = 200,- seqlen: int = 2048,- n_samples: int = 512,- sampler: str = "rand",- seed: int = 42,- n_blocks: int = 1,- gradient_accumulate_steps: int = 1,- not_use_best_mse: bool = False,- dynamic_max_gap: int = -1,- data_type: str = "int",- optimizer="AdamW",- **kwargs,+ self,+ model,+ tokenizer=None,+ bits: int = 4,+ group_size: int = 128,+ scheme: str = "asym",+ weight_config: dict = {},+ enable_full_range: bool = False,+ bs: int = 8,+ amp: bool = True,+ device="cuda:0",+ lr_scheduler=None,+ dataloader=None,+ dataset_name: str = "NeelNanda/pile-10k",+ dataset_split: str = "train",+ use_quant_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = None,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = True,+ iters: int = 200,+ seqlen: int = 2048,+ n_samples: int = 512,+ sampler: str = "rand",+ seed: int = 42,+ n_blocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_best_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int",+ optimizer="AdamW",+ **kwargs,):super(AutoAdamRound, self).__init__(model,@@ -1360,4 +1414,3 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)-diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.pyindex 7935112db..481b8b922 100644--- a/auto_round/calib_dataset.py+++ b/auto_round/calib_dataset.py@@ -1,6 +1,7 @@-import torchimport random+import torch+CALIB_DATASETS = {}@@ -89,7 +90,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split="@register_dataset("mbpp")-def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=['train', 'validation', 'test'], seed=42, bs=4):+def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=["train", "validation", "test"], seed=42, bs=4):"""Returns a dataloader for the specified dataset and split.Args:@@ -159,4 +160,3 @@ def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=['train',calib_dataloader = DataLoader(samples, batch_size=bs, shuffle=False, collate_fn=collate_batch)return calib_dataloader-diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.pyindex cc46ce3fb..10619175d 100644--- a/auto_round/export/export_to_autogptq.py+++ b/auto_round/export/export_to_autogptq.py@@ -13,6 +13,11 @@# limitations under the License.+import json+import os+from os.path import isdir, isfile, join+from typing import Dict, List, Optional, Union+# MIT License## Copyright (c) 2023 潘其威(William)@@ -35,25 +40,32 @@# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE# SOFTWARE.import torch-from auto_round.utils import logger-import os-from typing import Dict, List, Optional, Unionfrom safetensors.torch import save_file as safe_save-from os.path import join, isfile, isdir-import json++from auto_round.utils import logger-def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym=False, iters=200, lr=5e-3,- minmax_lr=5e-3,- enable_minmax_tuning=True, use_quant_input=True, use_safetensors: bool = True,- scale_dtype=torch.float32,- safetensors_metadata: Optional[Dict[str, str]] = None,- modules_in_block_to_quantize=None):- """save quantized model and configs to local disk for cuda """+def save_quantized_to_autogptq(+ model,+ save_dir: str,+ bits=4,+ group_size=128,+ sym=False,+ iters=200,+ lr=5e-3,+ minmax_lr=5e-3,+ enable_minmax_tuning=True,+ use_quant_input=True,+ use_safetensors: bool = True,+ scale_dtype=torch.float32,+ safetensors_metadata: Optional[Dict[str, str]] = None,+ modules_in_block_to_quantize=None,+):+ """Save quantized model and configs to local disk for cuda."""os.makedirs(save_dir, exist_ok=True)model.to("cpu")- model_base_name = f"model"+ model_base_name = "model"if use_safetensors:model_save_name = model_base_name + ".safetensors"state_dict = model.state_dict()@@ -74,34 +86,38 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, symnew_value = str(value)except Exception as e:raise TypeError(- f"safetensors_metadata: both keys and values must be strings" /- f" and an error occurred when trying to convert them: {e}")+ "safetensors_metadata: both keys and values must be strings"+ / f" and an error occurred when trying to convert them: {e}"+ )if new_key in new_safetensors_metadata:logger.warning(- f"After converting safetensors_metadata keys to strings, the key '{new_key}' " /- f"is duplicated. Ensure that all your metadata keys are strings to avoid overwriting.")+ f"After converting safetensors_metadata keys to strings, the key '{new_key}' "+ / "is duplicated. Ensure that all your metadata keys are strings to avoid overwriting."+ )new_safetensors_metadata[new_key] = new_valuesafetensors_metadata = new_safetensors_metadataif converted_keys:logger.debug(- f"One or more safetensors_metadata keys or values had to be converted to str()." /- f" Final safetensors_metadata: {safetensors_metadata}")+ "One or more safetensors_metadata keys or values had to be converted to str()."+ / f" Final safetensors_metadata: {safetensors_metadata}"+ )# Format is required to enable Accelerate to load the metadata# otherwise it raises an OSError- safetensors_metadata['format'] = "pt"+ safetensors_metadata["format"] = "pt"# Store the quantization configuration as safetensors metadatafrom auto_round import __version__- safetensors_metadata['autoround_version'] = str(__version__)- safetensors_metadata['bits'] = str(bits)- safetensors_metadata['group_size'] = str(group_size)- safetensors_metadata['iters'] = str(iters)- safetensors_metadata['lr'] = str(lr)- safetensors_metadata['minmax_lr'] = str(minmax_lr)- safetensors_metadata['enable_minmax_tuning'] = str(enable_minmax_tuning)- safetensors_metadata['use_quant_input'] = str(use_quant_input)- safetensors_metadata['scale_dtype'] = str(scale_dtype)++ safetensors_metadata["autoround_version"] = str(__version__)+ safetensors_metadata["bits"] = str(bits)+ safetensors_metadata["group_size"] = str(group_size)+ safetensors_metadata["iters"] = str(iters)+ safetensors_metadata["lr"] = str(lr)+ safetensors_metadata["minmax_lr"] = str(minmax_lr)+ safetensors_metadata["enable_minmax_tuning"] = str(enable_minmax_tuning)+ safetensors_metadata["use_quant_input"] = str(use_quant_input)+ safetensors_metadata["scale_dtype"] = str(scale_dtype)safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)else:model_save_name = model_base_name + ".bin"@@ -109,20 +125,26 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, symfrom auto_gptq.modeling._base import BaseQuantizeConfig- quantization_config = BaseQuantizeConfig(bits=bits, group_size=group_size, desc_act=False, sym=sym,- true_sequential=False, static_groups=False,- model_file_base_name=model_base_name)+ quantization_config = BaseQuantizeConfig(+ bits=bits,+ group_size=group_size,+ desc_act=False,+ sym=sym,+ true_sequential=False,+ static_groups=False,+ model_file_base_name=model_base_name,+ )quantization_config.model_file_base_name = model_base_nameconfig_dict = quantization_config.to_dict()config_dict["quant_method"] = "intel/auto-round"- config_dict['autoround_version'] = __version__- config_dict['iters'] = iters- config_dict['lr'] = lr- config_dict['minmax_lr'] = minmax_lr- config_dict['enable_minmax_tuning'] = enable_minmax_tuning- config_dict['use_quant_input'] = use_quant_input- config_dict['scale_dtype'] = str(scale_dtype)+ config_dict["autoround_version"] = __version__+ config_dict["iters"] = iters+ config_dict["lr"] = lr+ config_dict["minmax_lr"] = minmax_lr+ config_dict["enable_minmax_tuning"] = enable_minmax_tuning+ config_dict["use_quant_input"] = use_quant_input+ config_dict["scale_dtype"] = str(scale_dtype)if modules_in_block_to_quantize is not None:config_dict["modules_in_block_to_quantize"] = modules_in_block_to_quantize@@ -132,4 +154,3 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, symconfig_dict["quant_method"] = "gptq" ##hf transformers could only recognize this valuemodel.config.quantization_config = config_dictmodel.config.save_pretrained(save_dir)-diff --git a/auto_round/export/export_to_itrex/__init__.py b/auto_round/export/export_to_itrex/__init__.pyindex 0f9fc8525..d627bc3b6 100644--- a/auto_round/export/export_to_itrex/__init__.py+++ b/auto_round/export/export_to_itrex/__init__.py@@ -13,4 +13,3 @@# limitations under the License.from .export import compress_modelfrom .config import QuantConfig-diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.pyindex 4b2f861ba..f4eb3e2b8 100644--- a/auto_round/export/export_to_itrex/config.py+++ b/auto_round/export/export_to_itrex/config.py@@ -16,28 +16,32 @@# limitations under the License."""Configs for Autoround quantization."""-import torchimport copyimport jsonimport osfrom dataclasses import dataclass, fieldfrom typing import Any, Dict, Tuple, Union++import torchfrom transformers import PretrainedConfig+from auto_round.utils import loggerQUANT_CONFIG = "quantize_config.json"+class QuantConfig(PretrainedConfig):"""A brief quantization configuration for reference when performing model dequantization."""+def __init__(self,bits=4,- scale_dtype='torch.float32',+ scale_dtype="torch.float32",group_size=128,sym=False,quant_method="autoround",model_name_or_path=None,- model_file_base_name='model',+ model_file_base_name="model",enable_minmax_tuning=True,iters=1000,lr=0.001,@@ -52,42 +56,37 @@ class QuantConfig(PretrainedConfig):self.quant_method = quant_methodself.model_name_or_path = model_name_or_pathself.model_file_base_name = model_file_base_name- self.enable_minmax_tuning=enable_minmax_tuning- self.iters=iters- self.lr=lr- self.minmax_lr=minmax_lr- self.use_quant_input=use_quant_input-+ self.enable_minmax_tuning = enable_minmax_tuning+ self.iters = iters+ self.lr = lr+ self.minmax_lr = minmax_lr+ self.use_quant_input = use_quant_input+### Redundant parameters, will be removed later. ###self.damp_percent = 0.01self.desc_act = Falseself.true_sequential = Falseself.quant_method = "gptq"-def post_init(self):r"""Safety checker that arguments are correct - also replaces some NoneType arguments with their default values."""- if self.scale_dtype not in ['torch.float32', 'torch.float16', 'torch.bfloat16']:+ if self.scale_dtype not in ["torch.float32", "torch.float16", "torch.bfloat16"]:raise ValueError("scale_dtype must be 'fp32', 'fp16' or 'bf16'.")if self.group_size not in [-1, 32, 128]:raise ValueError("group_size must be an integer in [-1, 32, 128]")-def quantization_method(self):- r"""- This method returns the quantization method used for the model.- """+ r"""This method returns the quantization method used for the model."""return self.quant_method-@classmethoddef from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):- """- Instantiates a [`QuantConfig`] from a Python dictionary of parameters.+ """Instantiates a [`QuantConfig`] from a Python dictionary of parameters.+Args:config_dict (`Dict[str, Any]`):Dictionary that will be used to instantiate the configuration object.@@ -114,18 +113,16 @@ class QuantConfig(PretrainedConfig):return config, kwargselse:return config-@classmethoddef from_json_file(cls, json_file_path, return_unused_kwargs, **kwargs):with open(json_file_path, "r", encoding="utf-8") as f:config_dict = json.load(f)return cls.from_dict(config_dict, return_unused_kwargs, **kwargs)-def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool = True):- """- Save this instance to a JSON file.+ """Save this instance to a JSON file.+Args:json_file_path (`str` or `os.PathLike`):Path to the JSON file in which this configuration instance's parameters will be saved.@@ -134,9 +131,10 @@ class QuantConfig(PretrainedConfig):writer.write(self.to_json_string(use_diff=use_diff))def to_dict(self) -> Dict[str, Any]:- """- Serializes this instance to a Python dictionary. Returns:- `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.+ """Serializes this instance to a Python dictionary.++ Returns:+ `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance."""output = copy.deepcopy(self.__dict__)@@ -145,10 +143,9 @@ class QuantConfig(PretrainedConfig):def __repr__(self):return f"{self.__class__.__name__} {self.to_json_string()}"-def to_json_string(self, use_diff: bool = True) -> str:- """- Serializes this instance to a JSON string.+ """Serializes this instance to a JSON string.+Args:use_diff (`bool`, *optional*, defaults to `True`):If set to `True`, only the difference between the config instance and the default@@ -165,9 +162,9 @@ class QuantConfig(PretrainedConfig):return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"def to_diff_dict(self) -> Dict[str, Any]:- """- Removes all attributes from config which correspond to the default config attributes for better readability and+ """Removes all attributes from config which correspond to the default config attributes for better readability andserializes to a Python dictionary.+Returns:`Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,"""@@ -186,9 +183,9 @@ class QuantConfig(PretrainedConfig):return serializable_config_dictdef save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):- """- Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the+ """Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the[`~PretrainedConfig.from_pretrained`] class method.+Args:save_directory (`str` or `os.PathLike`):Directory where the configuration JSON file will be saved (will be created if it does not exist).diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex e1b3afce2..1026701da 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -12,28 +12,30 @@# See the License for the specific language governing permissions and# limitations under the License.-import torch-from auto_round.utils import logger+import copy+import jsonimport os+from os.path import isdir, isfile, joinfrom typing import Dict, List, Optional, Union++import torchfrom safetensors.torch import save_file as safe_save-from os.path import join, isfile, isdir-import copy-import json++from auto_round.utils import get_module, logger, quant_weight_w_scale, set_module+from .config import QuantConfigfrom .model_wrapper import WeightOnlyLinear-from auto_round.utils import quant_weight_w_scale, get_module, set_moduledef compress_model(- model,- weight_config: Union[str, dict],- enable_full_range=False,- compression_dtype=torch.int32,- compression_dim=1,- device="cpu",- use_optimum_format=True,- inplace=False+ model,+ weight_config: Union[str, dict],+ enable_full_range=False,+ compression_dtype=torch.int32,+ compression_dim=1,+ device="cpu",+ use_optimum_format=True,+ inplace=False,):"""Convert Linear to WeightOnlyLinear for low memory inference.@@ -103,4 +105,3 @@ def compress_model(set_module(compressed_model, k, new_module)return compressed_model-diff --git a/auto_round/export/export_to_itrex/model_wrapper.py b/auto_round/export/export_to_itrex/model_wrapper.pyindex 39270a84e..8e9f0aa85 100644--- a/auto_round/export/export_to_itrex/model_wrapper.py+++ b/auto_round/export/export_to_itrex/model_wrapper.py@@ -23,8 +23,8 @@ import torchfrom packaging.version import Versionfrom torch.autograd import Functionfrom torch.nn import functional as F-from auto_round.utils import logger+from auto_round.utils import loggerNF4 = [-1.0,@@ -57,6 +57,7 @@ FP4_E2M1_BIT = [-1, -2, -3, -4, -5, -6, -7, 0, 1, 2, 3, 4, 5, 6, 7]FLOAT_MAPPING = {"nf4": NF4, "fp4": FP4_BNB, "fp4_e2m1_bnb": FP4_BNB, "fp4_e2m1": FP4_E2M1}INT_MAPPING = {"nf4": NF4_BIT, "fp4": FP4_BNB_BIT, "fp4_e2m1_bnb": FP4_BNB_BIT, "fp4_e2m1": FP4_E2M1_BIT}+def get_torch_version():try:torch_version = torch.__version__.split("+")[0]@@ -68,6 +69,7 @@ def get_torch_version():PT_VERSION = get_torch_version().release+class WeightOnlyLinear(torch.nn.Module):def __init__(self,@@ -190,7 +192,7 @@ class WeightOnlyLinear(torch.nn.Module):self.scales = self.scales.t_().contiguous()self.qweight = self.qweight.t_().contiguous()self.qzeros = self.qzeros.t_().contiguous()-+int_weight = int_weight.to(self.device)if self.use_optimum_format and zp is None:# to avoid overflow@@ -349,4 +351,3 @@ class WeightOnlyLinear(torch.nn.Module):if self.use_optimum_format:tmp_str += ", use_optimum_format=True"return tmp_str-diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 614b5674a..64eb0d6b9 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1,4 +1,3 @@-# Copyright (c) 2023 Intel Corporation## Licensed under the Apache License, Version 2.0 (the "License");@@ -13,29 +12,33 @@# See the License for the specific language governing permissions and# limitations under the License.-import torchimport copy-from torch.amp import autocast-from collections import UserDictimport logging+from collections import UserDict++import torch+from torch.amp import autocast+logger = logging.getLogger("autoround")logger.setLevel(logging.INFO)fh = logging.StreamHandler()-fh_formatter = logging.Formatter('%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s',- "%Y-%m-%d %H:%M:%S")+fh_formatter = logging.Formatter("%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s", "%Y-%m-%d %H:%M:%S")fh.setFormatter(fh_formatter)logger.addHandler(fh)def is_optimum_habana_available():import importlib+from transformers.utils.import_utils import is_optimum_available- return is_optimum_available() and importlib.util.find_spec("optimum.habana") != None+ return is_optimum_available() and importlib.util.find_spec("optimum.habana") is not None+try:- import habana_frameworks.torch.hpu as hthpuimport habana_frameworks.torch.core as htcore+ import habana_frameworks.torch.hpu as hthpu+if is_optimum_habana_available():is_hpu_available = Trueelse:@@ -45,6 +48,7 @@ except ImportError:is_hpu_available = Falsehtcore = None+def round_ste(x: torch.Tensor):"""Straight-Through Estimator for rounding.This function is adapted from omniquant.@@ -71,7 +75,7 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dReturns:Quantized and dequantized weight, scale, zero-point"""- maxq = torch.tensor(2 ** num_bits - 1)+ maxq = torch.tensor(2**num_bits - 1)zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)# zeros = torch.zeros(weight.shape[0], device=weight.device)if isinstance(min_scale, torch.Tensor):@@ -84,7 +88,7 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_delse:wmin = torch.minimum(weight.min(1)[0], zeros)wmax = torch.maximum(weight.max(1)[0], zeros)-+tmp = (wmin == 0) & (wmax == 0)wmin[tmp] = -1wmax[tmp] = +1@@ -110,7 +114,7 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtReturns:Quantized and dequantized weight, scale, zero-point"""- maxq = torch.tensor(2 ** num_bits - 1)+ maxq = torch.tensor(2**num_bits - 1)zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)if isinstance(min_scale, torch.Tensor):wmin_tmp = torch.minimum(weight.min(1)[0], zeros)@@ -124,7 +128,7 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dtwmax = torch.maximum(weight.max(1)[0], zeros)wmax_new = torch.max(wmin.abs(), wmax)tmp = wmin < 0- wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur+ wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occurif torch.any(tmp):wmin_new[tmp] = -wmax_new[tmp]@@ -163,14 +167,8 @@ def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_def quant_weight(- weight,- num_bits=4,- group_size=-1,- scheme="asym",- v=0,- min_scale=0,- max_scale=0,- scale_dtype=torch.float16):+ weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16+):"""Quantizes and dequantizes weight, handing the group size issue .Args:@@ -186,8 +184,9 @@ def quant_weight(Quantized and dequantized weight, scale, zero-point"""if group_size == -1 or weight.shape[1] < group_size:- return quant_weight_actor(weight, num_bits, scheme=scheme, v=v,- min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype)+ return quant_weight_actor(+ weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype+ )orig_shape = weight.shapeif weight.shape[1] % group_size == 0:weight = weight.reshape(-1, group_size)@@ -220,8 +219,8 @@ def quant_weight(if zp is not None:zp = zp.reshape(weight_new.shape[0], -1)return weight_new, scale, zp--++def quant_weight_w_scale(weight, scale, zp, group_size=-1, device="cpu"):"""Quant and dequant tensor with group size.@@ -361,8 +360,6 @@ def get_block_names(model):return block_names--def collect_round_v(block):"""Collects the round values for wrapped linear modules in the given block.@@ -411,8 +408,8 @@ def get_batch_dim(input_others):"""dim = int(len(input_others["positional_inputs"]) > 0)return dim--++def sampling_inputs(input_ids, input_others, indices, seqlen):"""Samples inputs based on the given indices and sequence length.diff --git a/docs/Mistral-7B-v0.1-acc.md b/docs/Mistral-7B-v0.1-acc.mdindex 5bf71e989..d260fa41d 100644--- a/docs/Mistral-7B-v0.1-acc.md+++ b/docs/Mistral-7B-v0.1-acc.md@@ -20,4 +20,3 @@ lm_eval --model hf --model_args pretrained="Intel/Mistral-7B-v0.1-int4-inc",auto| rte | 0.6643 | 0.6968 || arc_easy | 0.8085 | 0.8060 || arc_challenge | 0.5009 | 0.4915 |-diff --git a/docs/Mixtral-8x7B-Instruct-v0.1-acc.md b/docs/Mixtral-8x7B-Instruct-v0.1-acc.mdindex 82a9e39dc..d7b10b775 100644--- a/docs/Mixtral-8x7B-Instruct-v0.1-acc.md+++ b/docs/Mixtral-8x7B-Instruct-v0.1-acc.md@@ -19,4 +19,4 @@ lm_eval --model hf --model_args pretrained="Intel/Mixtral-8x7B-Instruct-v0.1-int| boolq | 0.8850 | 0.8783 || rte | 0.7184 | 0.7004 || arc_easy | 0.8699 | 0.8712 |-| arc_challenge | 0.6220 | 0.6229 |\ No newline at end of file+| arc_challenge | 0.6220 | 0.6229 |diff --git a/docs/Mixtral-8x7B-v0.1-acc.md b/docs/Mixtral-8x7B-v0.1-acc.mdindex 5caf717e4..c3ea904b8 100644--- a/docs/Mixtral-8x7B-v0.1-acc.md+++ b/docs/Mixtral-8x7B-v0.1-acc.md@@ -21,4 +21,3 @@ lm_eval --model hf --model_args pretrained="Intel/Mixtral-8x7B-v0.1-int4-inc",au| rte | 0.7076 | 0.6931 | 0.7112 || arc_easy | 0.8430 | 0.8430 | 0.8333 || arc_challenge | 0.5666 | 0.5648 | 0.5469 |-diff --git a/docs/acc.md b/docs/acc.mdindex a7c3925b1..7a7bcd2bc 100644--- a/docs/acc.md+++ b/docs/acc.md@@ -2737,4 +2737,3 @@ For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequenc<td>11.35</td> <! lm-eval wikitext ppl --></tr></table>-diff --git a/docs/gemma-7b-acc.md b/docs/gemma-7b-acc.mdindex ec39e9d30..1ec091ce7 100644--- a/docs/gemma-7b-acc.md+++ b/docs/gemma-7b-acc.md@@ -17,4 +17,4 @@ lm_eval --model hf --model_args pretrained="Intel/gemma-7b-int4-inc",autogptq=Tr| boolq | 0.8278 | 0.8269 || rte | 0.6534 | 0.7076 || arc_easy | 0.8178 | 0.7959 |-| arc_challenge | 0.4991 | 0.4940 |\ No newline at end of file+| arc_challenge | 0.4991 | 0.4940 |diff --git a/docs/neural-chat-7b-v3-1-acc.md b/docs/neural-chat-7b-v3-1-acc.mdindex a55017d2d..f4df2353e 100644--- a/docs/neural-chat-7b-v3-1-acc.md+++ b/docs/neural-chat-7b-v3-1-acc.md@@ -17,4 +17,4 @@ lm_eval --model hf --model_args pretrained="Intel/neural-chat-v3-1-int4-inc",au| boolq | 0.8783 | 0.8743 || rte | 0.7690 | 0.7726 || arc_easy | 0.8413 | 0.8384 |-| arc_challenge | 0.5896 | 0.5785 |\ No newline at end of file+| arc_challenge | 0.5896 | 0.5785 |diff --git a/docs/neural-chat-7b-v3-3-acc.md b/docs/neural-chat-7b-v3-3-acc.mdindex abc220436..985e23ccc 100644--- a/docs/neural-chat-7b-v3-3-acc.md+++ b/docs/neural-chat-7b-v3-3-acc.md@@ -17,4 +17,4 @@ lm_eval --model hf --model_args pretrained="Intel/neural-chat-v3-3-int4-inc",au| boolq | 0.8694 | 0.8694 || rte | 0.7581 | 0.7509 || arc_easy | 0.8266 | 0.8249 |-| arc_challenge | 0.5734 | 0.5691 |\ No newline at end of file+| arc_challenge | 0.5734 | 0.5691 |diff --git a/docs/phi-2-acc.md b/docs/phi-2-acc.mdindex a2c612ae7..ecf765151 100644--- a/docs/phi-2-acc.md+++ b/docs/phi-2-acc.md@@ -19,4 +19,4 @@ Since we encountered an issue evaluating this model with lm-eval, we opted to ev| boolq | 0.8339 | 0.8327 || rte | 0.6245 | 0.6643 || arc_easy | 0.7997 | 0.7955 |-| arc_challenge | 0.5290 | 0.5196 |\ No newline at end of file+| arc_challenge | 0.5290 | 0.5196 |diff --git a/pyproject.toml b/pyproject.tomlnew file mode 100644index 000000000..dac86a1fc--- /dev/null+++ b/pyproject.toml@@ -0,0 +1,124 @@+[tool.isort]+profile = "black"+line_length = 120+known_first_party = ["auto_round"]+extend_skip_glob = ["**/__init__.py"]+++[tool.black]+line-length = 120+++[tool.codespell]+skip = '*.po,*.ts,*.js,*.map,*.js.map,*.css.map'+count = ''+quiet-level = 3+++[tool.ruff]+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.+select = ["E", "F"]+ignore = [+ "E402", # Module level import not at top of file+ "E501", # Line too long (121 > 120 characters)+ "E721", # Do not compare types, use isinstance()+ "E722", # Do not use bare except+ "E731", # Do not assign a lambda expression, use a def+ "E741", # Do not use variables named ‘l’, ‘O’, or ‘I’+ "F401", # {name} imported but unused+ "F403", # from {name} import * used; unable to detect undefined names+ "F405", # {name} may be undefined, or defined from star imports+ "F841", # Local variable is assigned to but never used{name}+]++ignore-init-module-imports = true++# Allow autofix for all enabled rules (when `--fix`) is provided.+fixable = [+ "A",+ "B",+ "C",+ "D",+ "E",+ "F",+ "G",+ "I",+ "N",+ "Q",+ "S",+ "T",+ "W",+ "ANN",+ "ARG",+ "BLE",+ "COM",+ "DJ",+ "DTZ",+ "EM",+ "ERA",+ "EXE",+ "FBT",+ "ICN",+ "INP",+ "ISC",+ "NPY",+ "PD",+ "PGH",+ "PIE",+ "PL",+ "PT",+ "PTH",+ "PYI",+ "RET",+ "RSE",+ "RUF",+ "SIM",+ "SLF",+ "TCH",+ "TID",+ "TRY",+ "UP",+ "YTT",+]+unfixable = []++# Exclude a variety of commonly ignored directories.+exclude = [+ ".bzr",+ ".direnv",+ ".eggs",+ ".git",+ ".git-rewrite",+ ".hg",+ ".mypy_cache",+ ".nox",+ ".pants.d",+ ".pytype",+ ".ruff_cache",+ ".svn",+ ".tox",+ ".venv",+ "__pypackages__",+ "_build",+ "buck-out",+ "build",+ "dist",+ "node_modules",+ "venv",+]++# Same as Black.+line-length = 120++# Allow unused variables when underscore-prefixed.+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"++# Assume Python 3.10.+target-version = "py310"++[tool.ruff.per-file-ignores]+"**/__init__.py" = ["F401"]++[tool.ruff.mccabe]+# Unlike Flake8, default to a complexity level of 10.+max-complexity = 10diff --git a/requirements.txt b/requirements.txtindex f7b9766e8..865bb9f39 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,6 +1,6 @@-transformers-torch-sentencepieceaccelerate+auto-gptqdatasets-auto-gptq\ No newline at end of file+sentencepiece+torch+transformersdiff --git a/setup.py b/setup.pyindex 4f4ca1613..4fe2b30d3 100644--- a/setup.py+++ b/setup.py@@ -1,5 +1,6 @@import refrom io import open+from setuptools import find_packages, setuptry:diff --git a/test/test_autoround.py b/test/test_autoround.pyindex 172ecf8f2..58992fd05 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -1,11 +1,13 @@import copyimport shutil-import unittestimport sys-sys.path.insert(0,".")+import unittest++sys.path.insert(0, ".")import torchimport transformers-from auto_round import (AutoRound)++from auto_round import AutoRoundclass SimpleDataLoader:@@ -37,8 +39,7 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):torchscript=True,)self.tokenizer = transformers.AutoTokenizer.from_pretrained(- "hf-internal-testing/tiny-random-GPTJForCausalLM",- trust_remote_code=True+ "hf-internal-testing/tiny-random-GPTJForCausalLM", trust_remote_code=True)self.gptj_no_jit = transformers.AutoModelForCausalLM.from_pretrained("hf-internal-testing/tiny-random-GPTJForCausalLM",@@ -50,19 +51,21 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):def tearDownClass(self):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)-def test_autoround_int_quant(self):model = copy.deepcopy(self.gptj)device = "cpu"- model=model+ model = modelout1 = model(self.lm_input)round = AutoRound- optq_1 = round(model, self.tokenizer, n_samples=20, device=device, amp=False, seqlen=10, iters=10, scale_dtype='fp32')+ optq_1 = round(+ model, self.tokenizer, n_samples=20, device=device, amp=False, seqlen=10, iters=10, scale_dtype="fp32"+ )q_model, weight_config1 = optq_1.quantize()q_model = q_modelfrom auto_round.export.export_to_itrex import compress_model- compressed_model,_ = compress_model(q_model, weight_config1)++ compressed_model, _ = compress_model(q_model, weight_config1)q_model = q_modelmodel = modelout2 = model(self.lm_input)@@ -73,30 +76,27 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):self.assertTrue(torch.all(out2[0] == out3[0]))self.assertTrue(torch.all(torch.isclose(out3[0], out4[0], atol=1e-5)))self.assertTrue("transformer.h.0.attn.k_proj.qzeros" in compressed_model.state_dict().keys())-+# model = copy.deepcopy(self.gptj)# out6 = model(self.lm_input)# optq_2 = round(model, self.tokenizer, n_samples=20, amp=False, seqlen=10)# q_model, weight_config2 = optq_2.quantize()# out4 = q_model(self.lm_input)# out5 = model(self.lm_input)-+# self.assertTrue(torch.all(out1[0] == out6[0]))# self.assertTrue(torch.all(out4[0] == out5[0]))# self.assertTrue(torch.all(torch.isclose(out6[0], out5[0], atol=1e-1)))--+def test_config(self):from auto_round import QuantConfig+config = QuantConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GPTQ")config.save_pretrained("quantization_config_dir")loaded_config = QuantConfig.from_pretrained("quantization_config_dir")self.assertEqual(config.group_size, loaded_config.group_size)self.assertEqual(config.true_sequential, loaded_config.true_sequential)----if __name__ == "__main__":- unittest.main()+if __name__ == "__main__":+ unittest.main()