Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Support code scan in CI test (#19)

chen, suyue528252b918f5
diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml
new file mode 100644
index 000000000..9f5b46e27
--- /dev/null
+++ b/.azure-pipelines/code-scan.yml
@@ -0,0 +1,47 @@
+trigger: none
+
+pr:
+ autoCancel: true
+ drafts: false
+ branches:
+ include:
+ - master
+ paths:
+ include:
+ - auto_round
+ - setup.py
+ - requirements.txt
+ - .azure-pipelines/code-scan.yml
+ - .azure-pipelines/scripts/codeScan
+
+pool:
+ vmImage: "ubuntu-latest"
+
+variables:
+ CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog"
+
+stages:
+
+ - stage: BanditCodeScan
+ displayName: Bandit Code Scan
+ dependsOn: []
+ jobs:
+ - job: Bandit
+ displayName: Bandit
+ steps:
+ - template: template/code-scan-template.yml
+ parameters:
+ codeScanFileName: "bandit"
+ uploadPath: "bandit.log"
+
+ - stage: PylintCodeScan
+ displayName: Pylint Code Scan
+ dependsOn: []
+ jobs:
+ - job: Pylint
+ displayName: Pylint
+ steps:
+ - template: template/code-scan-template.yml
+ parameters:
+ codeScanFileName: "pylint"
+ uploadPath: "pylint.json"
diff --git a/.azure-pipelines/docker/DockerfileCodeScan.devel b/.azure-pipelines/docker/DockerfileCodeScan.devel
new file mode 100644
index 000000000..110269eaf
--- /dev/null
+++ b/.azure-pipelines/docker/DockerfileCodeScan.devel
@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG UBUNTU_VER=22.04
+FROM ubuntu:${UBUNTU_VER} as devel
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+ aspell \
+ aspell-en \
+ python3 \
+ python3-pip \
+ python3-dev \
+ python3-distutils \
+ wget
+
+RUN ln -sf $(which python3) /usr/bin/python
+
+RUN python -m pip install --no-cache-dir pylint==2.12.1\
+ bandit
+
+WORKDIR /
diff --git a/.azure-pipelines/scripts/change_color.sh b/.azure-pipelines/scripts/change_color.sh
new file mode 100644
index 000000000..51342efec
--- /dev/null
+++ b/.azure-pipelines/scripts/change_color.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# -------------- general approach start----------------
+
+# 1. import this file:
+ # source path/change_color.sh
+# 2. use COLOR/BG:
+ # $VARIABLE_NAME && out_put_content && $RESET
+# 3. COLOR + BG:
+ # $COLOR/BG_VARIABLE_NAME && $BG/COLOR_VARIABLE_NAME && out_put_content && $RESET
+# 4. custom
+ # abbreviation(change number)
+ # txt number range (30, 37)
+ # bg number range (40, 47)
+ # special effects number range (1, 7)
+ # echo -en \\E[number1 + ; + number2 + ; + number3 + m"
+ # e.g - BG_GRAY+LIGHT_RED = "echo -en \\E[47;31m"
+
+# -------------- general approach end----------------==
+
+
+# general setting
+# ------------- light_color start----------------
+# black
+LIGHT_BLACK="echo -en \\E[30m"
+# red
+LIGHT_RED="echo -en \\E[31m"
+# green
+LIGHT_GREEN="echo -en \\E[32m"
+# yellow
+LIGHT_YELLOW="echo -en \\E[33m"
+# blue
+LIGHT_BLUE="echo -en \\E[34m"
+# purple
+LIGHT_PURPLE="echo -en \\E[35m"
+# cyan
+LIGHT_CYAN="echo -en \\E[36m"
+# gray
+LIGHT_GRAY="echo -en \\E[37m"
+# ------------- light_color end----------------
+
+# ------------- bold_color start----------------
+# black
+BOLD_BLACK="echo -en \\E[1;30m"
+# red
+BOLD_RED="echo -en \\E[1;31m"
+# green
+BOLD_GREEN="echo -en \\E[1;32m"
+# yellow
+BOLD_YELLOW="echo -en \\E[1;33m"
+# blue
+BOLD_BLUE="echo -en \\E[1;34m"
+# purple
+BOLD_PURPLE="echo -en \\E[1;35m"
+# cyan
+BOLD_CYAN="echo -en \\E[1;36m"
+# gray
+BOLD_GRAY="echo -en \\E[1;37m"
+# ------------- bold_color end----------------
+
+# ------------- background_color start----------------
+# black
+BG_BLACK="echo -en \\E[40m"
+# red
+BG_RED="echo -en \\E[41m"
+# green
+BG_GREEN="echo -en \\E[42m"
+# yellow
+BG_YELLOW="echo -en \\E[43m"
+# blue
+BG_BLUE="echo -en \\E[44m"
+# purple
+BG_PURPLE="echo -en \\E[45m"
+# cyan
+BG_CYAN="echo -en \\E[46m"
+# gray
+BG_GRAY="echo -en \\E[47m"
+# ------------- background_color end----------------
+
+# close
+RESET="echo -en \\E[0m"
diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh
new file mode 100644
index 000000000..07c1efc92
--- /dev/null
+++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+for var in "$@"
+do
+ case $var in
+ --scan_module=*)
+ scan_module=$(echo $var |cut -f2 -d=)
+ ;;
+ esac
+done
+
+source /auto-round/.azure-pipelines/scripts/change_color.sh
+RESET="echo -en \\E[0m \\n" # close color
+
+log_dir="/auto-round/.azure-pipelines/scripts/codeScan/scanLog"
+mkdir -p $log_dir
+
+python -m bandit -r -lll -iii "/auto-round/${scan_module}" >$log_dir/bandit.log
+exit_code=$?
+
+$BOLD_YELLOW && echo " ----------------- Current bandit cmd start --------------------------" && $RESET
+echo "python -m bandit -r -lll -iii /auto-round/${scan_module} > $log_dir/bandit.log"
+$BOLD_YELLOW && echo " ----------------- Current bandit cmd end --------------------------" && $RESET
+
+$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------"
+cat $log_dir/bandit.log
+$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET
+
+if [ ${exit_code} -ne 0 ]; then
+ $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Bandit error details." && $RESET
+ exit 1
+fi
+$BOLD_PURPLE && echo "Congratulations, Bandit check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET
+exit 0
diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh
new file mode 100644
index 000000000..d9e2bca37
--- /dev/null
+++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+for var in "$@"
+do
+ case $var in
+ --scan_module=*)
+ scan_module=$(echo $var |cut -f2 -d=)
+ ;;
+ esac
+done
+
+source /auto-round/.azure-pipelines/scripts/change_color.sh
+RESET="echo -en \\E[0m \\n" # close color
+
+log_dir="/auto-round/.azure-pipelines/scripts/codeScan/scanLog"
+mkdir -p $log_dir
+
+apt-get install -y --no-install-recommends --fix-missing \
+ autoconf \
+ build-essential
+
+pip install -r /auto-round/requirements.txt
+
+echo "[DEBUG] list pipdeptree..."
+pip install pipdeptree
+pipdeptree
+
+python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto \
+--ignored-modules=tensorflow,keras,torch,torch.quantization,torch.tensor,torchvision,fairseq,mxnet,onnx,onnxruntime,intel_extension_for_pytorch,intel_extension_for_tensorflow,torchinfo,horovod,transformers \
+/auto-round/${scan_module} > $log_dir/pylint.json
+
+exit_code=$?
+
+$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" && $RESET
+cat $log_dir/pylint.json
+$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET
+
+if [ ${exit_code} -ne 0 ]; then
+ $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pylint error details." && $RESET
+ exit 1
+fi
+$BOLD_PURPLE && echo "Congratulations, Pylint check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET
+exit 0
diff --git a/.azure-pipelines/template/code-scan-template.yml b/.azure-pipelines/template/code-scan-template.yml
new file mode 100644
index 000000000..724454df6
--- /dev/null
+++ b/.azure-pipelines/template/code-scan-template.yml
@@ -0,0 +1,42 @@
+parameters:
+ - name: codeScanFileName
+ type: string
+ - name: uploadPath
+ type: string
+
+ - name: codeScanContainerName
+ type: string
+ default: "codeScan"
+ - name: scanModule
+ type: string
+ default: "auto_round"
+
+steps:
+ - template: docker-template.yml
+ parameters:
+ dockerConfigName: "commonDockerConfig"
+ repoName: "code-scan"
+ repoTag: "1.0"
+ dockerFileName: "DockerfileCodeScan"
+ containerName: ${{ parameters.codeScanContainerName }}
+
+ - script: |
+ docker exec ${{ parameters.codeScanContainerName }} bash -c "bash /auto-round/.azure-pipelines/scripts/codeScan/${{ parameters.codeScanFileName }}/${{ parameters.codeScanFileName }}.sh \
+ --scan_module=${{ parameters.scanModule }}"
+ displayName: "${{ parameters.codeScanFileName }} Check"
+
+ - task: PublishPipelineArtifact@1
+ condition: succeededOrFailed()
+ inputs:
+ targetPath: .azure-pipelines/scripts/codeScan/scanLog/${{ parameters.uploadPath }}
+ artifact: ${{ parameters.codeScanFileName }}
+ publishLocation: "pipeline"
+ displayName: "PublishPipelineArtifact"
+
+ - task: Bash@3
+ condition: always()
+ inputs:
+ targetType: "inline"
+ script: |
+ docker exec ${{ parameters.codeScanContainerName }} bash -c "rm -fr /auto-round/* && rm -fr /auto-round/.* || true"
+ displayName: "Docker clean up"
diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.yml
new file mode 100644
index 000000000..0d7a9d889
--- /dev/null
+++ b/.azure-pipelines/template/docker-template.yml
@@ -0,0 +1,86 @@
+parameters:
+ - name: dockerConfigName
+ type: string
+ default: "commonDockerConfig"
+ - name: repoName
+ type: string
+ default: "auto-round"
+ - name: repoTag
+ type: string
+ default: "py310"
+ - name: dockerFileName
+ type: string
+ default: "Dockerfile"
+ - name: containerName
+ type: string
+ - name: repo
+ type: string
+ default: "https://github.com/intel/auto-round"
+
+steps:
+ - task: Bash@3
+ inputs:
+ targetType: "inline"
+ script: |
+ docker ps -a
+ if [[ $(docker ps -a | grep -i '${{ parameters.containerName }}'$) ]]; then
+ docker start $(docker ps -aq)
+ echo "remove left files through container ..."
+ docker exec ${{ parameters.containerName }} bash -c "ls -a /auto-round && rm -fr /auto-round/* && rm -fr /auto-round/.* && ls -a /auto-round || true"
+ fi
+ displayName: "Docker clean up"
+
+ - ${{ if eq(parameters.dockerConfigName, 'commonDockerConfig') }}:
+ - script: |
+ rm -fr ${BUILD_SOURCESDIRECTORY} || sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
+ echo y | docker system prune
+ displayName: "Clean workspace"
+
+ - checkout: self
+ clean: true
+ displayName: "Checkout out Repo"
+
+ - ${{ if eq(parameters.dockerConfigName, 'gitCloneDockerConfig') }}:
+ - script: |
+ rm -fr ${BUILD_SOURCESDIRECTORY} || sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
+ mkdir ${BUILD_SOURCESDIRECTORY}
+ chmod 777 ${BUILD_SOURCESDIRECTORY}
+ echo y | docker system prune
+ displayName: "Clean workspace"
+
+ - checkout: none
+
+ - script: |
+ git clone ${{ parameters.repo }} ${BUILD_SOURCESDIRECTORY}
+ git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY}
+ cd ${BUILD_SOURCESDIRECTORY}
+ git checkout master
+ displayName: "Checkout out master"
+
+ - script: |
+ if [[ ! $(docker images | grep -i ${{ parameters.repoName }}:${{ parameters.repoTag }}) ]]; then
+ docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/${{parameters.dockerFileName}}.devel -t ${{ parameters.repoName }}:${{ parameters.repoTag }} .
+ fi
+ docker images | grep -i ${{ parameters.repoName }}
+ if [[ $? -ne 0 ]]; then
+ echo "NO Such Repo"
+ exit 1
+ fi
+ displayName: "Build develop docker image"
+
+ - script: |
+ docker stop $(docker ps -aq)
+ docker rm -vf ${{ parameters.containerName }} || true
+ env | sort
+ displayName: "Clean docker container"
+
+ - ${{ if ne(parameters.containerName, '') }}:
+ - task: Bash@3
+ inputs:
+ targetType: "inline"
+ script: |
+ docker run -dit --disable-content-trust --privileged --name=${{ parameters.containerName }} --shm-size="2g" \
+ -v ${BUILD_SOURCESDIRECTORY}:/auto-round -v /tf_dataset:/tf_dataset -v /tf_dataset2:/tf_dataset2 ${{ parameters.repoName }}:${{ parameters.repoTag }}
+ echo "Show the container list after docker run ... "
+ docker ps -a
+ displayName: "Docker run - ${{ parameters.containerName }} Container"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 000000000..f969d01f2
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,115 @@
+ci:
+ autofix_prs: true
+ autoupdate_schedule: quarterly
+
+exclude: |
+ (?x)^(
+ neural_coder/test/.+|
+ neural_coder/examples/.+
+ )$
+
+repos:
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.5.0
+ hooks:
+ - id: end-of-file-fixer
+ files: (.*\.(py|md|rst|yaml|yml))$
+ exclude: |
+ (?x)^(
+ examples/.+
+ )$
+ - id: check-json
+ exclude: |
+ (?x)^(
+ .vscode/settings_recommended.json
+ )$
+ - id: check-yaml
+ - id: debug-statements
+ - id: file-contents-sorter
+ args: [--unique]
+ - id: requirements-txt-fixer
+ exclude: |
+ (?x)^(
+ examples/.+
+ )$
+ - id: trailing-whitespace
+ files: (.*\.(py|rst|cmake|yaml|yml))$
+ exclude: |
+ (?x)^(
+ examples/.+
+ )$
+
+ - repo: https://github.com/asottile/yesqa
+ rev: v1.5.0
+ hooks:
+ - id: yesqa
+ name: Unused noqa
+
+ - repo: https://github.com/pycqa/isort
+ rev: 5.13.2
+ hooks:
+ - id: isort
+ exclude: |
+ (?x)^(
+ examples/.+
+ )$
+
+ - repo: https://github.com/PyCQA/docformatter
+ rev: v1.7.5
+ hooks:
+ - id: docformatter
+ args: [
+ --in-place,
+ --wrap-summaries=0, # 0 means disable wrap
+ --wrap-descriptions=0, # 0 means disable wrap
+ --black,
+ --style=google,
+ ]
+ exclude: |
+ (?x)^(
+ examples/.+
+ )$
+
+ - repo: https://github.com/psf/black.git
+ rev: 24.1.1
+ hooks:
+ - id: black
+ files: (.*\.py)$
+ exclude: |
+ (?x)^(
+ examples/.+
+ )$
+
+ - repo: https://github.com/asottile/blacken-docs
+ rev: 1.16.0
+ hooks:
+ - id: blacken-docs
+ args: [--line-length=120, --skip-errors]
+ additional_dependencies:
+ - black==24.1.1
+ exclude: |
+ (?x)^(
+ examples/.+|
+ docs/.+
+ )$
+
+ - repo: https://github.com/codespell-project/codespell
+ rev: v2.2.6
+ hooks:
+ - id: codespell
+ additional_dependencies:
+ - tomli
+ exclude: |
+ (?x)^(
+ examples/.*(txt|patch)|
+ )$
+
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.1.15
+ hooks:
+ - id: ruff
+ args: [--fix, --exit-non-zero-on-fix, --no-cache]
+ exclude: |
+ (?x)^(
+ examples/.+
+ )$
diff --git a/README.md b/README.md
index 93553a103..eb8516e7d 100644
--- a/README.md
+++ b/README.md
@@ -32,16 +32,16 @@ model_name = "meta-llama/Llama-2-7b-hf"
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
bits, group_size, scheme = 4, 128, "asym"
-tuning_device = "cuda:0" ## or "cpu"
+tuning_device = "cuda:0" ## or "cpu"
autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device=tuning_device)
autoround.quantize()
output_dir = "./tmp_autoround"
-deployment_device = "cpu" ## or gpu
+deployment_device = "cpu" ## or gpu
if deployment_device == "cpu":
- autoround.save_quantized(output_dir, format="itrex") ## export to itrex format
+ autoround.save_quantized(output_dir, format="itrex") ## export to itrex format
else:
- autoround.save_quantized(output_dir, format="auto_gptq", use_triton=True) ##export to autogptq format
+ autoround.save_quantized(output_dir, format="auto_gptq", use_triton=True) ##export to autogptq format
```
@@ -51,15 +51,14 @@ else:
from transformers import AutoModelForCausalLM, AutoTokenizer
model_name = "meta-llama/Llama-2-7b-hf"
-model = AutoModelForCausalLM.from_pretrained(
- model_name, torch_dtype="auto", trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
bits, group_size, scheme = 4, 128, "asym"
# need to load model first, then import
from auto_round import AutoRound
-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme,
- device="hpu", amp=False)
+
+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme, device="hpu", amp=False)
autoround.quantize()
```
@@ -125,25 +124,26 @@ Please run the tuning code first
# currently please install neural-speed (https://github.com/intel/neural-speed) from source
from intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfig
from transformers import AutoTokenizer
+
quantized_model_path = "./tmp_autoround"
-woq_config = WeightOnlyQuantConfig(group_size=group_size, scheme=scheme,
- use_autoround=True) ##only supports 4 bits currently
+woq_config = WeightOnlyQuantConfig(
+ group_size=group_size, scheme=scheme, use_autoround=True
+) ##only supports 4 bits currently
prompt = "There is a girl who likes adventure,"
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, trust_remote_code=True)
inputs = tokenizer(prompt, return_tensors="pt").input_ids
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, quantization_config=woq_config,
- trust_remote_code=True, device="cpu")
+model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path, quantization_config=woq_config, trust_remote_code=True, device="cpu"
+)
outputs = model.generate(inputs, max_new_tokens=50)
```
### GPU
```python
# save_quantized to autogptq format first and then follow transformers or auto-gptq to load the model and inference
from transformers import AutoModelForCausalLM, AutoTokenizer
+
quantized_model_path = "./tmp_autoround"
-model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map="auto",
- trust_remote_code=True
- )
+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path, use_fast=True)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
@@ -197,6 +197,3 @@ If you find SignRound useful for your research, please cite our paper:
year={2023}
}
```
-
-
-
diff --git a/auto_round/__init__.py b/auto_round/__init__.py
index 01ad1c826..d26edcbf7 100644
--- a/auto_round/__init__.py
+++ b/auto_round/__init__.py
@@ -15,4 +15,3 @@ from .autoround import AutoRound, AutoAdamRound, AutoOPTRound
from .version import __version__
from auto_round.export.export_to_itrex import compress_model, QuantConfig
from auto_round.export.export_to_autogptq import save_quantized_to_autogptq
-
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index eb3c2c1c5..5c5f53cd3 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -13,13 +13,30 @@
# limitations under the License.
-import torch
-
import copy
import time
-from .utils import (quant_weight, set_module, get_module, get_block_names, block_forward, sampling_inputs,
- get_scale_shape, move_input_to_device, check_is_cpu, collect_round_v,
- collect_minmax_scale, get_batch_dim, htcore, is_hpu_available,check_to_quantized,logger)
+
+import torch
+
+from .utils import (
+ block_forward,
+ check_is_cpu,
+ check_to_quantized,
+ collect_minmax_scale,
+ collect_round_v,
+ get_batch_dim,
+ get_block_names,
+ get_module,
+ get_scale_shape,
+ htcore,
+ is_hpu_available,
+ logger,
+ move_input_to_device,
+ quant_weight,
+ sampling_inputs,
+ set_module,
+)
+
class SaveInputs:
"""Cache the inputs of the first block."""
@@ -158,6 +175,7 @@ class SaveInputs:
def _replace_forward(self):
"""Replaces the forward function."""
from functools import partial
+
for n, m in self.model.named_modules():
if n == self.block_name:
m.orig_forward = m.forward
@@ -246,12 +264,19 @@ class WrapperLinear(torch.nn.Module):
- torch.Tensor: The output tensor after applying the linear transformation with quantized weights.
"""
from torch.functional import F
+
weight = self.orig_layer.weight
self.min_scale.data.copy_(torch.clamp(self.min_scale.data, -1, 0))
self.max_scale.data.copy_(torch.clamp(self.max_scale.data, -1, 0))
weight_q, _, _ = quant_weight(
- weight, self.num_bits, self.group_size, self.scheme, self.value, self.min_scale, self.max_scale,
- self.scale_dtype
+ weight,
+ self.num_bits,
+ self.group_size,
+ self.scheme,
+ self.value,
+ self.min_scale,
+ self.max_scale,
+ self.scale_dtype,
)
weight_q = weight_q.to(weight.dtype)
# pylint: disable=not-callable
@@ -312,14 +337,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
min_scale.clamp_(-1, 0)
max_scale.clamp_(-1, 0)
weight_q, scale, zp = quant_weight(
- self.weight_t,
- self.num_bits,
- self.group_size,
- self.scheme,
- v,
- min_scale,
- max_scale,
- self.scale_dtype
+ self.weight_t, self.num_bits, self.group_size, self.scheme, v, min_scale, max_scale, self.scale_dtype
)
self.orig_layer.weight.data.copy_(weight_q.t())
self.orig_layer.weight.grad = None
@@ -347,7 +365,7 @@ class WrapperTransformerConv1d(torch.nn.Module):
self.value,
self.min_scale,
self.max_scale,
- self.scale_dtype
+ self.scale_dtype,
)
weight_q = weight_q.to(self.weight_t.dtype)
size_out = x.size()[:-1] + (self.orig_layer.nf,)
@@ -489,40 +507,41 @@ class AutoRound(object):
"""
def __init__(
- self,
- model,
- tokenizer,
- bits: int = 4,
- group_size: int = 128,
- scheme: str = "asym",
- weight_config: dict = {},
- enable_full_range: bool = False, ##for symmetric, TODO support later
- bs: int = 8,
- amp: bool = True,
- device="cuda:0",
- lr_scheduler=None,
- dataloader=None, ## to support later
- dataset_name: str = "NeelNanda/pile-10k",
- dataset_split: str = "train",
- use_quant_input: bool = True,
- enable_minmax_tuning: bool = True,
- lr: float = None,
- minmax_lr: float = None,
- low_gpu_mem_usage: bool = True,
- iters: int = 200,
- seqlen: int = 2048,
- n_samples: int = 512,
- sampler: str = "rand",
- seed: int = 42,
- n_blocks: int = 1,
- gradient_accumulate_steps: int = 1,
- not_use_best_mse: bool = False,
- dynamic_max_gap: int = -1,
- data_type: str = "int", ##only support data_type
- scale_dtype="fp16",
- **kwargs,
+ self,
+ model,
+ tokenizer,
+ bits: int = 4,
+ group_size: int = 128,
+ scheme: str = "asym",
+ weight_config: dict = {},
+ enable_full_range: bool = False, ##for symmetric, TODO support later
+ bs: int = 8,
+ amp: bool = True,
+ device="cuda:0",
+ lr_scheduler=None,
+ dataloader=None, ## to support later
+ dataset_name: str = "NeelNanda/pile-10k",
+ dataset_split: str = "train",
+ use_quant_input: bool = True,
+ enable_minmax_tuning: bool = True,
+ lr: float = None,
+ minmax_lr: float = None,
+ low_gpu_mem_usage: bool = True,
+ iters: int = 200,
+ seqlen: int = 2048,
+ n_samples: int = 512,
+ sampler: str = "rand",
+ seed: int = 42,
+ n_blocks: int = 1,
+ gradient_accumulate_steps: int = 1,
+ not_use_best_mse: bool = False,
+ dynamic_max_gap: int = -1,
+ data_type: str = "int", ##only support data_type
+ scale_dtype="fp16",
+ **kwargs,
):
from .calib_dataset import CALIB_DATASETS
+
self.model = model
self.model_orig_dtype = model.dtype
self.model = self.model.to("cpu")
@@ -552,9 +571,9 @@ class AutoRound(object):
self.n_blocks = n_blocks
self.device = device
- if scale_dtype == 'fp16':
+ if scale_dtype == "fp16":
self.scale_dtype = torch.float16
- elif scale_dtype == 'bf16':
+ elif scale_dtype == "bf16":
self.scale_dtype = torch.bfloat16
else:
self.scale_dtype = torch.float32
@@ -573,16 +592,14 @@ class AutoRound(object):
self.dataset_name = dataset_name
if dataloader is None:
- get_dataloader = CALIB_DATASETS.get(self.dataset_name,
- CALIB_DATASETS["NeelNanda/pile-10k"])
+ get_dataloader = CALIB_DATASETS.get(self.dataset_name, CALIB_DATASETS["NeelNanda/pile-10k"])
self.dataloader = get_dataloader(
self.tokenizer,
self.seqlen,
seed=self.seed,
bs=self.train_bs,
split=self.dataset_split,
- dataset_name=self.dataset_name
-
+ dataset_name=self.dataset_name,
)
else:
self.dataloader = dataloader
@@ -770,6 +787,7 @@ class AutoRound(object):
Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output)
"""
from torch.amp import autocast
+
batch_dim = get_batch_dim(input_others)
if not self.low_gpu_mem_usage and input_ids.device != device:
input_ids = move_input_to_device(input_ids, device)
@@ -881,12 +899,12 @@ class AutoRound(object):
return None, output
def qdq_weight_round(
- self,
- model: torch.nn.Module,
- inputs,
- block_names,
- n_blocks=1,
- device=torch.device("cpu"),
+ self,
+ model: torch.nn.Module,
+ inputs,
+ block_names,
+ n_blocks=1,
+ device=torch.device("cpu"),
):
"""Quantize and dequantize the weights of the specified blocks in the model.
@@ -914,7 +932,7 @@ class AutoRound(object):
logger.info(f"quantizing {i + 1}/{len(block_names)}, {n}")
m = get_module(model, n)
else:
- names = block_names[i: i + n_blocks]
+ names = block_names[i : i + n_blocks]
logger.info(names)
modules = [get_module(model, n) for n in names]
m = WrapperMultiblock(modules)
@@ -949,9 +967,7 @@ class AutoRound(object):
return compress_model
def save_quantized_as_autogptq(self, output_dir, use_triton=False, inplace=True):
- """
- Export the model to autogptq format to easily leverage cuda kernel
- """
+ """Export the model to autogptq format to easily leverage cuda kernel."""
if not self.quantized:
logger.warning("please run autoround.quantize first")
return
@@ -978,14 +994,15 @@ class AutoRound(object):
modules_in_block_to_quantize = [modules_in_block_to_quantize] ##align with autogptq
if all_to_quantized:
modules_in_block_to_quantize = None
-
+
if inplace:
model = self.model.to("cpu")
else:
model = copy.deepcopy(self.model.to("cpu"))
from auto_gptq.modeling._utils import pack_model
- sym = self.scheme == 'sym'
+
+ sym = self.scheme == "sym"
if self.bits == 3 or use_triton is False:
if self.bits == 3 and use_triton is True:
logger.warning("triton does not support 3 bits, reset it to False")
@@ -994,35 +1011,70 @@ class AutoRound(object):
info = self.weight_config[key]
if not check_to_quantized(info):
continue
- quantizers[key] = (None, info['scale'], info['zp'], info['g_idx'])
- pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,
- force_layer_back_to_cpu=True, use_triton=False)
+ quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])
+ pack_model(
+ model,
+ quantizers,
+ self.bits,
+ self.group_size,
+ use_cuda_fp16=True,
+ desc_act=False,
+ force_layer_back_to_cpu=True,
+ use_triton=False,
+ )
else:
quantizers = {}
for key in self.weight_config:
info = self.weight_config[key]
if not check_to_quantized(info):
continue
- info['zp'] = info['zp'].to(torch.float32)
- quantizers[key] = (None, info['scale'].to(torch.float32), info['zp'], info['g_idx'])
- pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,
- force_layer_back_to_cpu=True, use_triton=True)
+ info["zp"] = info["zp"].to(torch.float32)
+ quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"], info["g_idx"])
+ pack_model(
+ model,
+ quantizers,
+ self.bits,
+ self.group_size,
+ use_cuda_fp16=True,
+ desc_act=False,
+ force_layer_back_to_cpu=True,
+ use_triton=True,
+ )
from auto_round import save_quantized_to_autogptq
- save_quantized_to_autogptq(model, output_dir, bits=self.bits, group_size=self.group_size, sym=sym,
- iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,
- enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,
- scale_dtype=self.scale_dtype,
- use_safetensors=True, modules_in_block_to_quantize=modules_in_block_to_quantize)
+
+ save_quantized_to_autogptq(
+ model,
+ output_dir,
+ bits=self.bits,
+ group_size=self.group_size,
+ sym=sym,
+ iters=self.iters,
+ lr=self.lr,
+ minmax_lr=self.minmax_lr,
+ enable_minmax_tuning=self.enable_minmax_tuning,
+ use_quant_input=self.use_quant_input,
+ scale_dtype=self.scale_dtype,
+ use_safetensors=True,
+ modules_in_block_to_quantize=modules_in_block_to_quantize,
+ )
def save_quantized_as_itrex(self, output_dir, inplace=True):
"""Save configure file and weights for CPU backend inference."""
- from auto_round.export.export_to_itrex import compress_model, QuantConfig
+ from auto_round.export.export_to_itrex import QuantConfig, compress_model
+
compressed_model = compress_model(self.model, self.weight_config, inplace=inplace)
sym = self.scheme == "sym"
- quantize_config = QuantConfig(bits=self.bits, group_size=self.group_size, sym=sym,
- iters=self.iters, lr=self.lr, minmax_lr=self.minmax_lr,
- enable_minmax_tuning=self.enable_minmax_tuning, use_quant_input=self.use_quant_input,
- scale_dtype=str(self.scale_dtype))
+ quantize_config = QuantConfig(
+ bits=self.bits,
+ group_size=self.group_size,
+ sym=sym,
+ iters=self.iters,
+ lr=self.lr,
+ minmax_lr=self.minmax_lr,
+ enable_minmax_tuning=self.enable_minmax_tuning,
+ use_quant_input=self.use_quant_input,
+ scale_dtype=str(self.scale_dtype),
+ )
if quantize_config is not None:
config = compressed_model.config
setattr(config, "quantization_config", quantize_config.to_dict())
@@ -1081,10 +1133,12 @@ class AutoRound(object):
self.weight_config[n]["zp"] = m.zp
if self.group_size <= 0:
self.weight_config[n]["g_idx"] = torch.tensor(
- [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu")
+ [0 for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"
+ )
else:
self.weight_config[n]["g_idx"] = torch.tensor(
- [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu")
+ [i // self.group_size for i in range(m.weight.shape[1])], dtype=torch.int32, device="cpu"
+ )
delattr(m, "scale")
delattr(m, "zp")
else:
@@ -1144,38 +1198,38 @@ class AutoOPTRound(AutoRound):
"""
def __init__(
- self,
- model,
- tokenizer=None,
- bits: int = 4,
- group_size: int = 128,
- scheme: str = "asym",
- weight_config: dict = {},
- enable_full_range: bool = False,
- bs: int = 8,
- amp: bool = True,
- device="cuda:0",
- lr_scheduler=None,
- dataloader=None,
- dataset_name: str = "NeelNanda/pile-10k",
- dataset_split: str = "train",
- use_quant_input: bool = True,
- enable_minmax_tuning: bool = True,
- lr: float = None,
- minmax_lr: float = None,
- low_gpu_mem_usage: bool = True,
- iters: int = 200,
- seqlen: int = 2048,
- n_samples: int = 512,
- sampler: str = "rand",
- seed: int = 42,
- n_blocks: int = 1,
- gradient_accumulate_steps: int = 1,
- not_use_best_mse: bool = False,
- dynamic_max_gap: int = -1,
- data_type: str = "int",
- optimizer="AdamW",
- **kwargs,
+ self,
+ model,
+ tokenizer=None,
+ bits: int = 4,
+ group_size: int = 128,
+ scheme: str = "asym",
+ weight_config: dict = {},
+ enable_full_range: bool = False,
+ bs: int = 8,
+ amp: bool = True,
+ device="cuda:0",
+ lr_scheduler=None,
+ dataloader=None,
+ dataset_name: str = "NeelNanda/pile-10k",
+ dataset_split: str = "train",
+ use_quant_input: bool = True,
+ enable_minmax_tuning: bool = True,
+ lr: float = None,
+ minmax_lr: float = None,
+ low_gpu_mem_usage: bool = True,
+ iters: int = 200,
+ seqlen: int = 2048,
+ n_samples: int = 512,
+ sampler: str = "rand",
+ seed: int = 42,
+ n_blocks: int = 1,
+ gradient_accumulate_steps: int = 1,
+ not_use_best_mse: bool = False,
+ dynamic_max_gap: int = -1,
+ data_type: str = "int",
+ optimizer="AdamW",
+ **kwargs,
):
super(AutoOPTRound, self).__init__(
model,
@@ -1294,38 +1348,38 @@ class AutoAdamRound(AutoOPTRound):
"""
def __init__(
- self,
- model,
- tokenizer=None,
- bits: int = 4,
- group_size: int = 128,
- scheme: str = "asym",
- weight_config: dict = {},
- enable_full_range: bool = False,
- bs: int = 8,
- amp: bool = True,
- device="cuda:0",
- lr_scheduler=None,
- dataloader=None,
- dataset_name: str = "NeelNanda/pile-10k",
- dataset_split: str = "train",
- use_quant_input: bool = True,
- enable_minmax_tuning: bool = True,
- lr: float = None,
- minmax_lr: float = None,
- low_gpu_mem_usage: bool = True,
- iters: int = 200,
- seqlen: int = 2048,
- n_samples: int = 512,
- sampler: str = "rand",
- seed: int = 42,
- n_blocks: int = 1,
- gradient_accumulate_steps: int = 1,
- not_use_best_mse: bool = False,
- dynamic_max_gap: int = -1,
- data_type: str = "int",
- optimizer="AdamW",
- **kwargs,
+ self,
+ model,
+ tokenizer=None,
+ bits: int = 4,
+ group_size: int = 128,
+ scheme: str = "asym",
+ weight_config: dict = {},
+ enable_full_range: bool = False,
+ bs: int = 8,
+ amp: bool = True,
+ device="cuda:0",
+ lr_scheduler=None,
+ dataloader=None,
+ dataset_name: str = "NeelNanda/pile-10k",
+ dataset_split: str = "train",
+ use_quant_input: bool = True,
+ enable_minmax_tuning: bool = True,
+ lr: float = None,
+ minmax_lr: float = None,
+ low_gpu_mem_usage: bool = True,
+ iters: int = 200,
+ seqlen: int = 2048,
+ n_samples: int = 512,
+ sampler: str = "rand",
+ seed: int = 42,
+ n_blocks: int = 1,
+ gradient_accumulate_steps: int = 1,
+ not_use_best_mse: bool = False,
+ dynamic_max_gap: int = -1,
+ data_type: str = "int",
+ optimizer="AdamW",
+ **kwargs,
):
super(AutoAdamRound, self).__init__(
model,
@@ -1360,4 +1414,3 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
-
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 7935112db..481b8b922 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -1,6 +1,7 @@
-import torch
import random
+import torch
+
CALIB_DATASETS = {}
@@ -89,7 +90,7 @@ def get_dataloader(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", split="
@register_dataset("mbpp")
-def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=['train', 'validation', 'test'], seed=42, bs=4):
+def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=["train", "validation", "test"], seed=42, bs=4):
"""Returns a dataloader for the specified dataset and split.
Args:
@@ -159,4 +160,3 @@ def get_mbpp_dataloader(tokenizer, seqlen, dataset_name="mbpp", split=['train',
calib_dataloader = DataLoader(samples, batch_size=bs, shuffle=False, collate_fn=collate_batch)
return calib_dataloader
-
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py
index cc46ce3fb..10619175d 100644
--- a/auto_round/export/export_to_autogptq.py
+++ b/auto_round/export/export_to_autogptq.py
@@ -13,6 +13,11 @@
# limitations under the License.
+import json
+import os
+from os.path import isdir, isfile, join
+from typing import Dict, List, Optional, Union
+
# MIT License
#
# Copyright (c) 2023 潘其威(William)
@@ -35,25 +40,32 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import torch
-from auto_round.utils import logger
-import os
-from typing import Dict, List, Optional, Union
from safetensors.torch import save_file as safe_save
-from os.path import join, isfile, isdir
-import json
+
+from auto_round.utils import logger
-def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym=False, iters=200, lr=5e-3,
- minmax_lr=5e-3,
- enable_minmax_tuning=True, use_quant_input=True, use_safetensors: bool = True,
- scale_dtype=torch.float32,
- safetensors_metadata: Optional[Dict[str, str]] = None,
- modules_in_block_to_quantize=None):
- """save quantized model and configs to local disk for cuda """
+def save_quantized_to_autogptq(
+ model,
+ save_dir: str,
+ bits=4,
+ group_size=128,
+ sym=False,
+ iters=200,
+ lr=5e-3,
+ minmax_lr=5e-3,
+ enable_minmax_tuning=True,
+ use_quant_input=True,
+ use_safetensors: bool = True,
+ scale_dtype=torch.float32,
+ safetensors_metadata: Optional[Dict[str, str]] = None,
+ modules_in_block_to_quantize=None,
+):
+ """Save quantized model and configs to local disk for cuda."""
os.makedirs(save_dir, exist_ok=True)
model.to("cpu")
- model_base_name = f"model"
+ model_base_name = "model"
if use_safetensors:
model_save_name = model_base_name + ".safetensors"
state_dict = model.state_dict()
@@ -74,34 +86,38 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
new_value = str(value)
except Exception as e:
raise TypeError(
- f"safetensors_metadata: both keys and values must be strings" /
- f" and an error occurred when trying to convert them: {e}")
+ "safetensors_metadata: both keys and values must be strings"
+ / f" and an error occurred when trying to convert them: {e}"
+ )
if new_key in new_safetensors_metadata:
logger.warning(
- f"After converting safetensors_metadata keys to strings, the key '{new_key}' " /
- f"is duplicated. Ensure that all your metadata keys are strings to avoid overwriting.")
+ f"After converting safetensors_metadata keys to strings, the key '{new_key}' "
+ / "is duplicated. Ensure that all your metadata keys are strings to avoid overwriting."
+ )
new_safetensors_metadata[new_key] = new_value
safetensors_metadata = new_safetensors_metadata
if converted_keys:
logger.debug(
- f"One or more safetensors_metadata keys or values had to be converted to str()." /
- f" Final safetensors_metadata: {safetensors_metadata}")
+ "One or more safetensors_metadata keys or values had to be converted to str()."
+ / f" Final safetensors_metadata: {safetensors_metadata}"
+ )
# Format is required to enable Accelerate to load the metadata
# otherwise it raises an OSError
- safetensors_metadata['format'] = "pt"
+ safetensors_metadata["format"] = "pt"
# Store the quantization configuration as safetensors metadata
from auto_round import __version__
- safetensors_metadata['autoround_version'] = str(__version__)
- safetensors_metadata['bits'] = str(bits)
- safetensors_metadata['group_size'] = str(group_size)
- safetensors_metadata['iters'] = str(iters)
- safetensors_metadata['lr'] = str(lr)
- safetensors_metadata['minmax_lr'] = str(minmax_lr)
- safetensors_metadata['enable_minmax_tuning'] = str(enable_minmax_tuning)
- safetensors_metadata['use_quant_input'] = str(use_quant_input)
- safetensors_metadata['scale_dtype'] = str(scale_dtype)
+
+ safetensors_metadata["autoround_version"] = str(__version__)
+ safetensors_metadata["bits"] = str(bits)
+ safetensors_metadata["group_size"] = str(group_size)
+ safetensors_metadata["iters"] = str(iters)
+ safetensors_metadata["lr"] = str(lr)
+ safetensors_metadata["minmax_lr"] = str(minmax_lr)
+ safetensors_metadata["enable_minmax_tuning"] = str(enable_minmax_tuning)
+ safetensors_metadata["use_quant_input"] = str(use_quant_input)
+ safetensors_metadata["scale_dtype"] = str(scale_dtype)
safe_save(state_dict, join(save_dir, model_save_name), safetensors_metadata)
else:
model_save_name = model_base_name + ".bin"
@@ -109,20 +125,26 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
from auto_gptq.modeling._base import BaseQuantizeConfig
- quantization_config = BaseQuantizeConfig(bits=bits, group_size=group_size, desc_act=False, sym=sym,
- true_sequential=False, static_groups=False,
- model_file_base_name=model_base_name)
+ quantization_config = BaseQuantizeConfig(
+ bits=bits,
+ group_size=group_size,
+ desc_act=False,
+ sym=sym,
+ true_sequential=False,
+ static_groups=False,
+ model_file_base_name=model_base_name,
+ )
quantization_config.model_file_base_name = model_base_name
config_dict = quantization_config.to_dict()
config_dict["quant_method"] = "intel/auto-round"
- config_dict['autoround_version'] = __version__
- config_dict['iters'] = iters
- config_dict['lr'] = lr
- config_dict['minmax_lr'] = minmax_lr
- config_dict['enable_minmax_tuning'] = enable_minmax_tuning
- config_dict['use_quant_input'] = use_quant_input
- config_dict['scale_dtype'] = str(scale_dtype)
+ config_dict["autoround_version"] = __version__
+ config_dict["iters"] = iters
+ config_dict["lr"] = lr
+ config_dict["minmax_lr"] = minmax_lr
+ config_dict["enable_minmax_tuning"] = enable_minmax_tuning
+ config_dict["use_quant_input"] = use_quant_input
+ config_dict["scale_dtype"] = str(scale_dtype)
if modules_in_block_to_quantize is not None:
config_dict["modules_in_block_to_quantize"] = modules_in_block_to_quantize
@@ -132,4 +154,3 @@ def save_quantized_to_autogptq(model, save_dir: str, bits=4, group_size=128, sym
config_dict["quant_method"] = "gptq" ##hf transformers could only recognize this value
model.config.quantization_config = config_dict
model.config.save_pretrained(save_dir)
-
diff --git a/auto_round/export/export_to_itrex/__init__.py b/auto_round/export/export_to_itrex/__init__.py
index 0f9fc8525..d627bc3b6 100644
--- a/auto_round/export/export_to_itrex/__init__.py
+++ b/auto_round/export/export_to_itrex/__init__.py
@@ -13,4 +13,3 @@
# limitations under the License.
from .export import compress_model
from .config import QuantConfig
-
diff --git a/auto_round/export/export_to_itrex/config.py b/auto_round/export/export_to_itrex/config.py
index 4b2f861ba..f4eb3e2b8 100644
--- a/auto_round/export/export_to_itrex/config.py
+++ b/auto_round/export/export_to_itrex/config.py
@@ -16,28 +16,32 @@
# limitations under the License.
"""Configs for Autoround quantization."""
-import torch
import copy
import json
import os
from dataclasses import dataclass, field
from typing import Any, Dict, Tuple, Union
+
+import torch
from transformers import PretrainedConfig
+
from auto_round.utils import logger
QUANT_CONFIG = "quantize_config.json"
+
class QuantConfig(PretrainedConfig):
"""A brief quantization configuration for reference when performing model dequantization."""
+
def __init__(
self,
bits=4,
- scale_dtype='torch.float32',
+ scale_dtype="torch.float32",
group_size=128,
sym=False,
quant_method="autoround",
model_name_or_path=None,
- model_file_base_name='model',
+ model_file_base_name="model",
enable_minmax_tuning=True,
iters=1000,
lr=0.001,
@@ -52,42 +56,37 @@ class QuantConfig(PretrainedConfig):
self.quant_method = quant_method
self.model_name_or_path = model_name_or_path
self.model_file_base_name = model_file_base_name
- self.enable_minmax_tuning=enable_minmax_tuning
- self.iters=iters
- self.lr=lr
- self.minmax_lr=minmax_lr
- self.use_quant_input=use_quant_input
-
+ self.enable_minmax_tuning = enable_minmax_tuning
+ self.iters = iters
+ self.lr = lr
+ self.minmax_lr = minmax_lr
+ self.use_quant_input = use_quant_input
+
### Redundant parameters, will be removed later. ###
self.damp_percent = 0.01
self.desc_act = False
self.true_sequential = False
self.quant_method = "gptq"
-
def post_init(self):
r"""
Safety checker that arguments are correct - also replaces some NoneType arguments with their default values.
"""
- if self.scale_dtype not in ['torch.float32', 'torch.float16', 'torch.bfloat16']:
+ if self.scale_dtype not in ["torch.float32", "torch.float16", "torch.bfloat16"]:
raise ValueError("scale_dtype must be 'fp32', 'fp16' or 'bf16'.")
if self.group_size not in [-1, 32, 128]:
raise ValueError("group_size must be an integer in [-1, 32, 128]")
-
def quantization_method(self):
- r"""
- This method returns the quantization method used for the model.
- """
+ r"""This method returns the quantization method used for the model."""
return self.quant_method
-
@classmethod
def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs):
- """
- Instantiates a [`QuantConfig`] from a Python dictionary of parameters.
+ """Instantiates a [`QuantConfig`] from a Python dictionary of parameters.
+
Args:
config_dict (`Dict[str, Any]`):
Dictionary that will be used to instantiate the configuration object.
@@ -114,18 +113,16 @@ class QuantConfig(PretrainedConfig):
return config, kwargs
else:
return config
-
@classmethod
def from_json_file(cls, json_file_path, return_unused_kwargs, **kwargs):
with open(json_file_path, "r", encoding="utf-8") as f:
config_dict = json.load(f)
return cls.from_dict(config_dict, return_unused_kwargs, **kwargs)
-
def to_json_file(self, json_file_path: Union[str, os.PathLike], use_diff: bool = True):
- """
- Save this instance to a JSON file.
+ """Save this instance to a JSON file.
+
Args:
json_file_path (`str` or `os.PathLike`):
Path to the JSON file in which this configuration instance's parameters will be saved.
@@ -134,9 +131,10 @@ class QuantConfig(PretrainedConfig):
writer.write(self.to_json_string(use_diff=use_diff))
def to_dict(self) -> Dict[str, Any]:
- """
- Serializes this instance to a Python dictionary. Returns:
- `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
+ """Serializes this instance to a Python dictionary.
+
+ Returns:
+ `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance.
"""
output = copy.deepcopy(self.__dict__)
@@ -145,10 +143,9 @@ class QuantConfig(PretrainedConfig):
def __repr__(self):
return f"{self.__class__.__name__} {self.to_json_string()}"
-
def to_json_string(self, use_diff: bool = True) -> str:
- """
- Serializes this instance to a JSON string.
+ """Serializes this instance to a JSON string.
+
Args:
use_diff (`bool`, *optional*, defaults to `True`):
If set to `True`, only the difference between the config instance and the default
@@ -165,9 +162,9 @@ class QuantConfig(PretrainedConfig):
return json.dumps(config_dict, indent=2, sort_keys=True) + "\n"
def to_diff_dict(self) -> Dict[str, Any]:
- """
- Removes all attributes from config which correspond to the default config attributes for better readability and
+ """Removes all attributes from config which correspond to the default config attributes for better readability and
serializes to a Python dictionary.
+
Returns:
`Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance,
"""
@@ -186,9 +183,9 @@ class QuantConfig(PretrainedConfig):
return serializable_config_dict
def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):
- """
- Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the
+ """Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the
[`~PretrainedConfig.from_pretrained`] class method.
+
Args:
save_directory (`str` or `os.PathLike`):
Directory where the configuration JSON file will be saved (will be created if it does not exist).
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index e1b3afce2..1026701da 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -12,28 +12,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import torch
-from auto_round.utils import logger
+import copy
+import json
import os
+from os.path import isdir, isfile, join
from typing import Dict, List, Optional, Union
+
+import torch
from safetensors.torch import save_file as safe_save
-from os.path import join, isfile, isdir
-import copy
-import json
+
+from auto_round.utils import get_module, logger, quant_weight_w_scale, set_module
+
from .config import QuantConfig
from .model_wrapper import WeightOnlyLinear
-from auto_round.utils import quant_weight_w_scale, get_module, set_module
def compress_model(
- model,
- weight_config: Union[str, dict],
- enable_full_range=False,
- compression_dtype=torch.int32,
- compression_dim=1,
- device="cpu",
- use_optimum_format=True,
- inplace=False
+ model,
+ weight_config: Union[str, dict],
+ enable_full_range=False,
+ compression_dtype=torch.int32,
+ compression_dim=1,
+ device="cpu",
+ use_optimum_format=True,
+ inplace=False,
):
"""Convert Linear to WeightOnlyLinear for low memory inference.
@@ -103,4 +105,3 @@ def compress_model(
set_module(compressed_model, k, new_module)
return compressed_model
-
diff --git a/auto_round/export/export_to_itrex/model_wrapper.py b/auto_round/export/export_to_itrex/model_wrapper.py
index 39270a84e..8e9f0aa85 100644
--- a/auto_round/export/export_to_itrex/model_wrapper.py
+++ b/auto_round/export/export_to_itrex/model_wrapper.py
@@ -23,8 +23,8 @@ import torch
from packaging.version import Version
from torch.autograd import Function
from torch.nn import functional as F
-from auto_round.utils import logger
+from auto_round.utils import logger
NF4 = [
-1.0,
@@ -57,6 +57,7 @@ FP4_E2M1_BIT = [-1, -2, -3, -4, -5, -6, -7, 0, 1, 2, 3, 4, 5, 6, 7]
FLOAT_MAPPING = {"nf4": NF4, "fp4": FP4_BNB, "fp4_e2m1_bnb": FP4_BNB, "fp4_e2m1": FP4_E2M1}
INT_MAPPING = {"nf4": NF4_BIT, "fp4": FP4_BNB_BIT, "fp4_e2m1_bnb": FP4_BNB_BIT, "fp4_e2m1": FP4_E2M1_BIT}
+
def get_torch_version():
try:
torch_version = torch.__version__.split("+")[0]
@@ -68,6 +69,7 @@ def get_torch_version():
PT_VERSION = get_torch_version().release
+
class WeightOnlyLinear(torch.nn.Module):
def __init__(
self,
@@ -190,7 +192,7 @@ class WeightOnlyLinear(torch.nn.Module):
self.scales = self.scales.t_().contiguous()
self.qweight = self.qweight.t_().contiguous()
self.qzeros = self.qzeros.t_().contiguous()
-
+
int_weight = int_weight.to(self.device)
if self.use_optimum_format and zp is None:
# to avoid overflow
@@ -349,4 +351,3 @@ class WeightOnlyLinear(torch.nn.Module):
if self.use_optimum_format:
tmp_str += ", use_optimum_format=True"
return tmp_str
-
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 614b5674a..64eb0d6b9 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1,4 +1,3 @@
-
# Copyright (c) 2023 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,29 +12,33 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import torch
import copy
-from torch.amp import autocast
-from collections import UserDict
import logging
+from collections import UserDict
+
+import torch
+from torch.amp import autocast
+
logger = logging.getLogger("autoround")
logger.setLevel(logging.INFO)
fh = logging.StreamHandler()
-fh_formatter = logging.Formatter('%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s',
- "%Y-%m-%d %H:%M:%S")
+fh_formatter = logging.Formatter("%(asctime)s %(levelname)s %(filename)s L%(lineno)d: %(message)s", "%Y-%m-%d %H:%M:%S")
fh.setFormatter(fh_formatter)
logger.addHandler(fh)
def is_optimum_habana_available():
import importlib
+
from transformers.utils.import_utils import is_optimum_available
- return is_optimum_available() and importlib.util.find_spec("optimum.habana") != None
+ return is_optimum_available() and importlib.util.find_spec("optimum.habana") is not None
+
try:
- import habana_frameworks.torch.hpu as hthpu
import habana_frameworks.torch.core as htcore
+ import habana_frameworks.torch.hpu as hthpu
+
if is_optimum_habana_available():
is_hpu_available = True
else:
@@ -45,6 +48,7 @@ except ImportError:
is_hpu_available = False
htcore = None
+
def round_ste(x: torch.Tensor):
"""Straight-Through Estimator for rounding.
This function is adapted from omniquant.
@@ -71,7 +75,7 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_d
Returns:
Quantized and dequantized weight, scale, zero-point
"""
- maxq = torch.tensor(2 ** num_bits - 1)
+ maxq = torch.tensor(2**num_bits - 1)
zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)
# zeros = torch.zeros(weight.shape[0], device=weight.device)
if isinstance(min_scale, torch.Tensor):
@@ -84,7 +88,7 @@ def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_d
else:
wmin = torch.minimum(weight.min(1)[0], zeros)
wmax = torch.maximum(weight.max(1)[0], zeros)
-
+
tmp = (wmin == 0) & (wmax == 0)
wmin[tmp] = -1
wmax[tmp] = +1
@@ -110,7 +114,7 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dt
Returns:
Quantized and dequantized weight, scale, zero-point
"""
- maxq = torch.tensor(2 ** num_bits - 1)
+ maxq = torch.tensor(2**num_bits - 1)
zeros = torch.zeros(weight.shape[0], device=weight.device, dtype=scale_dtype)
if isinstance(min_scale, torch.Tensor):
wmin_tmp = torch.minimum(weight.min(1)[0], zeros)
@@ -124,7 +128,7 @@ def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0, scale_dt
wmax = torch.maximum(weight.max(1)[0], zeros)
wmax_new = torch.max(wmin.abs(), wmax)
tmp = wmin < 0
- wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur
+ wmin_new = wmin.clone() ##must clone, otherwise inplace backward will occur
if torch.any(tmp):
wmin_new[tmp] = -wmax_new[tmp]
@@ -163,14 +167,8 @@ def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale, scale_
def quant_weight(
- weight,
- num_bits=4,
- group_size=-1,
- scheme="asym",
- v=0,
- min_scale=0,
- max_scale=0,
- scale_dtype=torch.float16):
+ weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0, scale_dtype=torch.float16
+):
"""Quantizes and dequantizes weight, handing the group size issue .
Args:
@@ -186,8 +184,9 @@ def quant_weight(
Quantized and dequantized weight, scale, zero-point
"""
if group_size == -1 or weight.shape[1] < group_size:
- return quant_weight_actor(weight, num_bits, scheme=scheme, v=v,
- min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype)
+ return quant_weight_actor(
+ weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale, scale_dtype=scale_dtype
+ )
orig_shape = weight.shape
if weight.shape[1] % group_size == 0:
weight = weight.reshape(-1, group_size)
@@ -220,8 +219,8 @@ def quant_weight(
if zp is not None:
zp = zp.reshape(weight_new.shape[0], -1)
return weight_new, scale, zp
-
-
+
+
def quant_weight_w_scale(weight, scale, zp, group_size=-1, device="cpu"):
"""Quant and dequant tensor with group size.
@@ -361,8 +360,6 @@ def get_block_names(model):
return block_names
-
-
def collect_round_v(block):
"""Collects the round values for wrapped linear modules in the given block.
@@ -411,8 +408,8 @@ def get_batch_dim(input_others):
"""
dim = int(len(input_others["positional_inputs"]) > 0)
return dim
-
-
+
+
def sampling_inputs(input_ids, input_others, indices, seqlen):
"""Samples inputs based on the given indices and sequence length.
diff --git a/docs/Mistral-7B-v0.1-acc.md b/docs/Mistral-7B-v0.1-acc.md
index 5bf71e989..d260fa41d 100644
--- a/docs/Mistral-7B-v0.1-acc.md
+++ b/docs/Mistral-7B-v0.1-acc.md
@@ -20,4 +20,3 @@ lm_eval --model hf --model_args pretrained="Intel/Mistral-7B-v0.1-int4-inc",auto
| rte | 0.6643 | 0.6968 |
| arc_easy | 0.8085 | 0.8060 |
| arc_challenge | 0.5009 | 0.4915 |
-
diff --git a/docs/Mixtral-8x7B-Instruct-v0.1-acc.md b/docs/Mixtral-8x7B-Instruct-v0.1-acc.md
index 82a9e39dc..d7b10b775 100644
--- a/docs/Mixtral-8x7B-Instruct-v0.1-acc.md
+++ b/docs/Mixtral-8x7B-Instruct-v0.1-acc.md
@@ -19,4 +19,4 @@ lm_eval --model hf --model_args pretrained="Intel/Mixtral-8x7B-Instruct-v0.1-int
| boolq | 0.8850 | 0.8783 |
| rte | 0.7184 | 0.7004 |
| arc_easy | 0.8699 | 0.8712 |
-| arc_challenge | 0.6220 | 0.6229 |
\ No newline at end of file
+| arc_challenge | 0.6220 | 0.6229 |
diff --git a/docs/Mixtral-8x7B-v0.1-acc.md b/docs/Mixtral-8x7B-v0.1-acc.md
index 5caf717e4..c3ea904b8 100644
--- a/docs/Mixtral-8x7B-v0.1-acc.md
+++ b/docs/Mixtral-8x7B-v0.1-acc.md
@@ -21,4 +21,3 @@ lm_eval --model hf --model_args pretrained="Intel/Mixtral-8x7B-v0.1-int4-inc",au
| rte | 0.7076 | 0.6931 | 0.7112 |
| arc_easy | 0.8430 | 0.8430 | 0.8333 |
| arc_challenge | 0.5666 | 0.5648 | 0.5469 |
-
diff --git a/docs/acc.md b/docs/acc.md
index a7c3925b1..7a7bcd2bc 100644
--- a/docs/acc.md
+++ b/docs/acc.md
@@ -2737,4 +2737,3 @@ For wikitext2/ptb-new/c4-new ppl, we follow the code of gptq and set the sequenc
<td>11.35</td> <! lm-eval wikitext ppl -->
</tr>
</table>
-
diff --git a/docs/gemma-7b-acc.md b/docs/gemma-7b-acc.md
index ec39e9d30..1ec091ce7 100644
--- a/docs/gemma-7b-acc.md
+++ b/docs/gemma-7b-acc.md
@@ -17,4 +17,4 @@ lm_eval --model hf --model_args pretrained="Intel/gemma-7b-int4-inc",autogptq=Tr
| boolq | 0.8278 | 0.8269 |
| rte | 0.6534 | 0.7076 |
| arc_easy | 0.8178 | 0.7959 |
-| arc_challenge | 0.4991 | 0.4940 |
\ No newline at end of file
+| arc_challenge | 0.4991 | 0.4940 |
diff --git a/docs/neural-chat-7b-v3-1-acc.md b/docs/neural-chat-7b-v3-1-acc.md
index a55017d2d..f4df2353e 100644
--- a/docs/neural-chat-7b-v3-1-acc.md
+++ b/docs/neural-chat-7b-v3-1-acc.md
@@ -17,4 +17,4 @@ lm_eval --model hf --model_args pretrained="Intel/neural-chat-v3-1-int4-inc",au
| boolq | 0.8783 | 0.8743 |
| rte | 0.7690 | 0.7726 |
| arc_easy | 0.8413 | 0.8384 |
-| arc_challenge | 0.5896 | 0.5785 |
\ No newline at end of file
+| arc_challenge | 0.5896 | 0.5785 |
diff --git a/docs/neural-chat-7b-v3-3-acc.md b/docs/neural-chat-7b-v3-3-acc.md
index abc220436..985e23ccc 100644
--- a/docs/neural-chat-7b-v3-3-acc.md
+++ b/docs/neural-chat-7b-v3-3-acc.md
@@ -17,4 +17,4 @@ lm_eval --model hf --model_args pretrained="Intel/neural-chat-v3-3-int4-inc",au
| boolq | 0.8694 | 0.8694 |
| rte | 0.7581 | 0.7509 |
| arc_easy | 0.8266 | 0.8249 |
-| arc_challenge | 0.5734 | 0.5691 |
\ No newline at end of file
+| arc_challenge | 0.5734 | 0.5691 |
diff --git a/docs/phi-2-acc.md b/docs/phi-2-acc.md
index a2c612ae7..ecf765151 100644
--- a/docs/phi-2-acc.md
+++ b/docs/phi-2-acc.md
@@ -19,4 +19,4 @@ Since we encountered an issue evaluating this model with lm-eval, we opted to ev
| boolq | 0.8339 | 0.8327 |
| rte | 0.6245 | 0.6643 |
| arc_easy | 0.7997 | 0.7955 |
-| arc_challenge | 0.5290 | 0.5196 |
\ No newline at end of file
+| arc_challenge | 0.5290 | 0.5196 |
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..dac86a1fc
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,124 @@
+[tool.isort]
+profile = "black"
+line_length = 120
+known_first_party = ["auto_round"]
+extend_skip_glob = ["**/__init__.py"]
+
+
+[tool.black]
+line-length = 120
+
+
+[tool.codespell]
+skip = '*.po,*.ts,*.js,*.map,*.js.map,*.css.map'
+count = ''
+quiet-level = 3
+
+
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = [
+ "E402", # Module level import not at top of file
+ "E501", # Line too long (121 > 120 characters)
+ "E721", # Do not compare types, use isinstance()
+ "E722", # Do not use bare except
+ "E731", # Do not assign a lambda expression, use a def
+ "E741", # Do not use variables named ‘l’, ‘O’, or ‘I’
+ "F401", # {name} imported but unused
+ "F403", # from {name} import * used; unable to detect undefined names
+ "F405", # {name} may be undefined, or defined from star imports
+ "F841", # Local variable is assigned to but never used{name}
+]
+
+ignore-init-module-imports = true
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = [
+ "A",
+ "B",
+ "C",
+ "D",
+ "E",
+ "F",
+ "G",
+ "I",
+ "N",
+ "Q",
+ "S",
+ "T",
+ "W",
+ "ANN",
+ "ARG",
+ "BLE",
+ "COM",
+ "DJ",
+ "DTZ",
+ "EM",
+ "ERA",
+ "EXE",
+ "FBT",
+ "ICN",
+ "INP",
+ "ISC",
+ "NPY",
+ "PD",
+ "PGH",
+ "PIE",
+ "PL",
+ "PT",
+ "PTH",
+ "PYI",
+ "RET",
+ "RSE",
+ "RUF",
+ "SIM",
+ "SLF",
+ "TCH",
+ "TID",
+ "TRY",
+ "UP",
+ "YTT",
+]
+unfixable = []
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+ ".bzr",
+ ".direnv",
+ ".eggs",
+ ".git",
+ ".git-rewrite",
+ ".hg",
+ ".mypy_cache",
+ ".nox",
+ ".pants.d",
+ ".pytype",
+ ".ruff_cache",
+ ".svn",
+ ".tox",
+ ".venv",
+ "__pypackages__",
+ "_build",
+ "buck-out",
+ "build",
+ "dist",
+ "node_modules",
+ "venv",
+]
+
+# Same as Black.
+line-length = 120
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+# Assume Python 3.10.
+target-version = "py310"
+
+[tool.ruff.per-file-ignores]
+"**/__init__.py" = ["F401"]
+
+[tool.ruff.mccabe]
+# Unlike Flake8, default to a complexity level of 10.
+max-complexity = 10
diff --git a/requirements.txt b/requirements.txt
index f7b9766e8..865bb9f39 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-transformers
-torch
-sentencepiece
accelerate
+auto-gptq
datasets
-auto-gptq
\ No newline at end of file
+sentencepiece
+torch
+transformers
diff --git a/setup.py b/setup.py
index 4f4ca1613..4fe2b30d3 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,6 @@
import re
from io import open
+
from setuptools import find_packages, setup
try:
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 172ecf8f2..58992fd05 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -1,11 +1,13 @@
import copy
import shutil
-import unittest
import sys
-sys.path.insert(0,".")
+import unittest
+
+sys.path.insert(0, ".")
import torch
import transformers
-from auto_round import (AutoRound)
+
+from auto_round import AutoRound
class SimpleDataLoader:
@@ -37,8 +39,7 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):
torchscript=True,
)
self.tokenizer = transformers.AutoTokenizer.from_pretrained(
- "hf-internal-testing/tiny-random-GPTJForCausalLM",
- trust_remote_code=True
+ "hf-internal-testing/tiny-random-GPTJForCausalLM", trust_remote_code=True
)
self.gptj_no_jit = transformers.AutoModelForCausalLM.from_pretrained(
"hf-internal-testing/tiny-random-GPTJForCausalLM",
@@ -50,19 +51,21 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):
def tearDownClass(self):
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
-
def test_autoround_int_quant(self):
model = copy.deepcopy(self.gptj)
device = "cpu"
- model=model
+ model = model
out1 = model(self.lm_input)
round = AutoRound
- optq_1 = round(model, self.tokenizer, n_samples=20, device=device, amp=False, seqlen=10, iters=10, scale_dtype='fp32')
+ optq_1 = round(
+ model, self.tokenizer, n_samples=20, device=device, amp=False, seqlen=10, iters=10, scale_dtype="fp32"
+ )
q_model, weight_config1 = optq_1.quantize()
q_model = q_model
from auto_round.export.export_to_itrex import compress_model
- compressed_model,_ = compress_model(q_model, weight_config1)
+
+ compressed_model, _ = compress_model(q_model, weight_config1)
q_model = q_model
model = model
out2 = model(self.lm_input)
@@ -73,30 +76,27 @@ class TestPytorchWeightOnlyAdaptor(unittest.TestCase):
self.assertTrue(torch.all(out2[0] == out3[0]))
self.assertTrue(torch.all(torch.isclose(out3[0], out4[0], atol=1e-5)))
self.assertTrue("transformer.h.0.attn.k_proj.qzeros" in compressed_model.state_dict().keys())
-
+
# model = copy.deepcopy(self.gptj)
# out6 = model(self.lm_input)
# optq_2 = round(model, self.tokenizer, n_samples=20, amp=False, seqlen=10)
# q_model, weight_config2 = optq_2.quantize()
# out4 = q_model(self.lm_input)
# out5 = model(self.lm_input)
-
+
# self.assertTrue(torch.all(out1[0] == out6[0]))
# self.assertTrue(torch.all(out4[0] == out5[0]))
# self.assertTrue(torch.all(torch.isclose(out6[0], out5[0], atol=1e-1)))
-
-
+
def test_config(self):
from auto_round import QuantConfig
+
config = QuantConfig.from_pretrained("TheBloke/Llama-2-7B-Chat-GPTQ")
config.save_pretrained("quantization_config_dir")
loaded_config = QuantConfig.from_pretrained("quantization_config_dir")
self.assertEqual(config.group_size, loaded_config.group_size)
self.assertEqual(config.true_sequential, loaded_config.true_sequential)
-
-
-
-if __name__ == "__main__":
- unittest.main()
+if __name__ == "__main__":
+ unittest.main()