PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
update readme
wenhuach219f0db7eaa648
diff --git a/README.md b/README.mdindex d3a42a3bd..8a5a00a3f 100644--- a/README.md+++ b/README.md@@ -24,51 +24,64 @@ python setup.py install## Usage-### On Intel Gaudi2+++### On CPU/GPU```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer+from auto_round import AutoRound-model_name = "bigscience/bloom-560m"+model_name = "meta-llama/Llama-2-7b-hf"model = AutoModelForCausalLM.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)bits, group_size, scheme = 4, 128, "asym"--# need to load model first, then import-from auto_round import AutoRound-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme,- device="hpu", scale_dtype="bf16", amp=False)+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme)autoround.quantize()-# Intel CPU Inference, Currently, llama, bloom, and mistral are supported.-output_dir = "/path/to/quantized_model"-autoround.export(output_dir)+# Intel CPU Inference, For now only support llama, mistral and gpt-j.# then follow ITREX(https://github.com/intel/intel-extension-for-transformers/tree/main/intel_extension_for_transformers/llm/runtime/neural_speed) to load the model and do inference+# currently please install neural-speed (https://github.com/intel/neural-speed) from source+output_dir = "./tmp_autoround"+autoround.export(output_dir)+from intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfig++woq_config = WeightOnlyQuantConfig(group_size=group_size, scheme=scheme, use_autoround=True) ##only supports 4 bits currently+prompt = "Once upon a time, a little girl"++tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+inputs = tokenizer(prompt, return_tensors="pt").input_ids+model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=woq_config, trust_remote_code=True,device="cpu")+outputs = model.generate(inputs, max_new_tokens=30)```-### On CPU/GPU++### Tuning on Intel Gaudi2```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRound-model_name = "bigscience/bloom-560m"+model_name = "meta-llama/Llama-2-7b-hf"model = AutoModelForCausalLM.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)bits, group_size, scheme = 4, 128, "asym"-autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme)++# need to load model first, then import+from auto_round import AutoRound+autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, scheme=scheme,+ device="hpu", scale_dtype="bf16", amp=False)autoround.quantize()# Intel CPU Inference, Currently, llama, bloom, and mistral are supported.output_dir = "/path/to/quantized_model"autoround.export(output_dir)# then follow ITREX(https://github.com/intel/intel-extension-for-transformers/tree/main/intel_extension_for_transformers/llm/runtime/neural_speed) to load the model and do inference-```+<details><summary>Detailed Hyperparameters</summary>