PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add more files
wenhuach21b18cbcbb2e7c
diff --git a/autoround/eval/__init__.py b/autoround/eval/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/autoround/eval/eval.py b/autoround/eval/eval.pynew file mode 100644index 000000000..695f492f7--- /dev/null+++ b/autoround/eval/eval.py@@ -0,0 +1,369 @@+import os++os.environ["TOKENIZERS_PARALLELISM"] = "false"+import os.path+import torch+import torch.nn as nn+from parse_results import result_parser+import pprint+import json+import re+import shutil+import transformers+import time++EXT_TASKS = ['wikitext2', 'ptb', 'c4', 'ptb-new', 'c4-new']+fewshots_dict = {}+fewshots_dict['paper'] = {+ "lambada_openai": [0],+ "hellaswag": [0],+ "winogrande": [0],+ "piqa": [0],+ "hendrycksTest-*": [0],+ "wikitext": [0],+ "truthfulqa_mc": [0],+ "openbookqa": [0],+ "boolq": [0],+ "rte": [0],+ "arc_easy": [0],+ "arc_challenge": [0],+}+fewshots_dict['leadboard'] = {+ "hellaswag": [10],+ "winogrande": [5],+ "arc_easy": [25],+ "arc_challenge": [25],+ "hendrycksTest-*": [5],+ "drop": [3],+ "gsm8k": [5],+}+fewshots_dict['all'] = {+ "lambada_openai": [0],+ "hellaswag": [0, 10],+ "winogrande": [0, 5],+ "piqa": [0],+ "coqa": [], ## coqa is not enabled in llamav1 models+ "truthfulqa_mc": [0],+ "openbookqa": [0],+ "boolq": [0],+ "rte": [0],+ "arc_easy": [0, 25],+ "arc_challenge": [0, 25],+ "hendrycksTest-*": [0, 5],+ "wikitext": [0],+ "drop": [3],+ "gsm8k": [5]+}+++def simple_evaluate(+ model,+ model_args=None,+ tasks=[],+ num_fewshot=0,+ batch_size=None,+ max_batch_size=None,+ device=None,+ no_cache=True, ##changed by wenhua+ limit=None,+ bootstrap_iters=100000,+ description_dict=None,+ check_integrity=False,+ decontamination_ngrams_path=None,+ write_out=False,+ output_base_path=None,+ lm=None ##changed by wenhua+):+ """Instantiate and evaluate a model on a list of tasks.++ :param model: Union[str, LM]+ Name of model, transformers.PreTrainedModel object, or LM object, see lm_eval.models.get_model+ :param model_args: Optional[str]+ String arguments for each model class, see LM.create_from_arg_string.+ Ignored if `model` argument is a LM object.+ :param tasks: list[Union[str, Task]]+ List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.+ :param num_fewshot: int+ Number of examples in few-shot context+ :param batch_size: int or str, optional+ Batch size for model+ :param max_batch_size: int, optional+ Maximal batch size to try with automatic batch size detection+ :param device: str, optional+ PyTorch device (e.g. "cpu" or "cuda:0") for running models+ :param no_cache: bool+ Whether or not to cache+ :param limit: int or float, optional+ Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.+ :param bootstrap_iters:+ Number of iterations for bootstrap statistics+ :param description_dict: dict[str, str]+ Dictionary of custom task descriptions of the form: `task_name: description`+ :param check_integrity: bool+ Whether to run the relevant part of the test suite for the tasks+ :param write_out: bool+ If True, write details about prompts and logits to json for all tasks+ :param output_base_path: str, optional+ Directory to which detailed eval info will be written. Defaults to present working dir.+ :return+ Dictionary of results+ """+ import collections+ import itertools+ import random++ import lm_eval.metrics+ import lm_eval.models+ import lm_eval.tasks+ import lm_eval.base+ from lm_eval.utils import positional_deprecated, run_task_tests+ from lm_eval.models.gpt2 import HFLM++ import numpy as np+ import transformers+ from lm_eval.evaluator import evaluate+ random.seed(1234)+ np.random.seed(1234)++ assert tasks != [], "No tasks specified"+ if lm == None:+ if isinstance(model, str):+ if model_args is None:+ model_args = ""+ lm = lm_eval.models.get_model(model).create_from_arg_string(+ model_args,+ {+ "batch_size": batch_size,+ "max_batch_size": max_batch_size,+ "device": device,+ },+ )+ elif isinstance(model, transformers.PreTrainedModel):+ lm = lm_eval.models.get_model("hf-causal")(+ pretrained=model,+ batch_size=batch_size,+ max_batch_size=max_batch_size,+ )+ no_cache = True+ else:+ assert isinstance(model, lm_eval.base.LM)+ lm = model++ if not no_cache:+ lm = lm_eval.base.CachingLM(+ lm,+ "lm_cache/"+ + (model if isinstance(model, str) else model.model.config._name_or_path)+ + "_"+ + model_args.replace("=", "-").replace(",", "_").replace("/", "-")+ + ".db",+ )++ task_dict = lm_eval.tasks.get_task_dict(tasks)++ if check_integrity:+ run_task_tests(task_list=tasks)++ results = evaluate(+ lm=lm,+ task_dict=task_dict,+ num_fewshot=num_fewshot,+ limit=limit,+ bootstrap_iters=bootstrap_iters,+ description_dict=description_dict,+ decontamination_ngrams_path=decontamination_ngrams_path,+ write_out=write_out,+ output_base_path=output_base_path,+ )++ # add info about the model and few shot config+ model_name = None+ if isinstance(model, str):+ model_name = model+ elif isinstance(model, transformers.PreTrainedModel):+ model_name = "pretrained=" + model.config._name_or_path+ results["config"] = {+ "model": model_name,+ "model_args": model_args,+ "num_fewshot": num_fewshot,+ "batch_size": batch_size,+ "batch_sizes": list(lm.batch_sizes.values())+ if hasattr(lm, "batch_sizes")+ else [],+ "device": device,+ "no_cache": no_cache,+ "limit": limit,+ "bootstrap_iters": bootstrap_iters,+ "description_dict": description_dict,+ }++ return results, lm+++def eval_model(output_dir=None, model=None, tokenizer=None,+ tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],+ eval_bs=32, use_accelerate=True, dtype="float16", limit=None,+ device="cuda:0", seed=0, nsamples=128, eval_orig_float=False, mark="paper", excel_file="tmp.xlsx"):+ print("evaluation with official lm-eval", flush=True)+ try:+ import lm_eval+ from lm_eval import evaluator+ from lm_eval.tasks import ALL_TASKS, get_task_dict+ except:+ raise ImportError("""follow requirements to install dependencies.""")++ org_s = time.time()+ ##save model+ if output_dir is None:+ output_dir = "./tmp_signround"++ if os.path.exists(output_dir) and not eval_orig_float:+ shutil.rmtree(output_dir)+ if not eval_orig_float:+ model = model.to(torch.float16)+ model = model.to("cpu")+ model.save_pretrained(output_dir)+ tokenizer.save_pretrained(output_dir)++ if (hasattr(model, 'config') and model.config.torch_dtype is torch.bfloat16):+ dtype = 'bfloat16'++ external_tasks = []+ for each in EXT_TASKS:+ if each in tasks:+ external_tasks.append(each)+ tasks.remove(each)++ results = {}+ model = None+ lm = None+ for tmp_tasks in tasks:+ try:+ num_fewshot = fewshots_dict[mark][tmp_tasks]+ task_names = lm_eval.utils.pattern_match([tmp_tasks], ALL_TASKS)+ print(f'********* {tmp_tasks} evaluate ************')+ task_s = time.time()+ for shot in num_fewshot:+ if bool(re.search("chatglm", output_dir.lower())):+ model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'+ model_type = "hf-causal"+ else:+ model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'+ model_type = "hf-causal-experimental"++ if "wikitext" in task_names:+ tmp_eval_bs = 1+ else:+ tmp_eval_bs = eval_bs+ tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=task_names,+ num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,+ max_batch_size=tmp_eval_bs, lm=lm)+ sub_name = f'{tmp_tasks} {shot}-shot'+ print(f'{sub_name}: ')+ pprint.pprint(tmp_results["results"])+ print(f"\n{sub_name} cost time: {time.time() - task_s}\n")+ results[sub_name] = tmp_results+ except Exception as e:+ print(f'********* {tmp_tasks} ERROR ************')+ print(str(e))+ continue++ tokenizer = transformers.AutoTokenizer.from_pretrained(output_dir, use_fast=False, trust_remote_code=True)+ model = lm.model+ # for external tasks+ # maybe adjust for specific model+ # if hasattr(lm.model.config, "max_position_embeddings"):+ # lm.model.seqlen = lm.model.config.max_position_embeddings+ # else:+ # ## for llama-1, opt+ # lm.model.seqlen = 2048++ # if "opt" in model_name:+ # seqlen = model.config.max_position_embeddings+ # model.seqlen = model.config.max_position_embeddings+ # else:+ # seqlen = 2048+ # model.seqlen = seqlen++ model.seqlen = 2048+ from utils import get_loaders, eval_ppl_same_with_gptq+ for dataset in external_tasks:+ try:+ dataloader, testloader = get_loaders(+ dataset, nsamples=nsamples, seed=seed,+ tokenizer=tokenizer, seqlen=model.seqlen+ )+ ppl = eval_ppl_same_with_gptq(model, testloader, device)+ print(dataset, ppl)++ results.update({dataset: ppl})+ except Exception as e:+ print(str(e))+ continue++ print(results, flush=True)+ new_results = result_parser(results)+ print("cost time: ", time.time() - org_s)+ import pickle+ from collections import OrderedDict+ new_dict = OrderedDict()+ new_dict["model"] = "tmp"+ new_dict["paper-avg"] = 0+ new_dict["leaderboard-avg"] = 0+ new_dict["wikitext2"] = 0+ new_dict["ptb-new"] = 0+ new_dict["c4-new"] = 0+ new_dict["wikitext 0-shot_word_perplexity"] = 0+ new_dict["wikitext 0-shot_byte_perplexity"] = 0+ new_dict["wikitext 0-shot_bits_per_byte"] = 0+ for key in new_results.keys():+ if key == "model" or key == "paper-avg" or key == "leaderboard-avg":+ continue+ data = new_results[key]+ if not isinstance(data, dict):+ new_dict[key] = new_results[key]+ continue+ for sub_key in data.keys():+ for sub_sub_key in data[sub_key].keys():+ if "hendry" in key:+ new_key = key + "-" + sub_key + "-" + sub_sub_key+ else:+ new_key = key + "_" + sub_sub_key+ if "std" in new_key:+ continue+ # if "norm" in new_key:+ # continue+ new_dict[new_key] = data[sub_key][sub_sub_key]++ import pandas as pd+ df = pd.DataFrame(data=new_dict, index=[0])+ df.to_excel(excel_file)+++if __name__ == "__main__":+ import time+ import argparse++ parser = argparse.ArgumentParser()+ parser.add_argument(+ "--model_name", default="/models/opt-125m"+ )+ parser.add_argument(+ "--bs", default=1,+ )++ args = parser.parse_args()+ s = time.time()++ test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',+ "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte", "arc_easy",+ "arc_challenge"]+ test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',+ ]+ excel_name = (args.model_name).split('/')[-1] + ".xlsx"+ eval_model(output_dir=args.model_name,+ tasks=test_tasks,+ eval_bs=args.bs, eval_orig_float=True, limit=None, excel_file=excel_name)++ print("cost time: ", time.time() - s)+diff --git a/autoround/eval/parse_results.py b/autoround/eval/parse_results.pynew file mode 100644index 000000000..a1f237392--- /dev/null+++ b/autoround/eval/parse_results.py@@ -0,0 +1,140 @@++import sys+import json+import pprint++# input_file = sys.argv[1]++# with open(input_file) as f:+# data = json.load(f)++# data = data["results"]++task_domain = {+ "abstract_algebra": ["STEM"],+ "anatomy": ["Other"],+ "astronomy": ["STEM"],+ "business_ethics": ["Other"],+ "clinical_knowledge": ["Other"],+ "college_biology": ["STEM"],+ "college_chemistry": ["STEM"],+ "college_computer_science": ["STEM"],+ "college_mathematics": ["STEM"],+ "college_medicine": ["Other"],+ "college_physics": ["STEM"],+ "computer_security": ["STEM"],+ "conceptual_physics": ["STEM"],+ "econometrics": ["Social"],+ "electrical_engineering": ["STEM"],+ "elementary_mathematics": ["STEM"],+ "formal_logic": ["Hums"],+ "global_facts": ["Other"],+ "high_school_biology": ["STEM"],+ "high_school_chemistry": ["STEM"],+ "high_school_computer_science": ["STEM"],+ "high_school_european_history": ["Hums"],+ "high_school_geography": ["Social"],+ "high_school_government_and_politics": ["Social"],+ "high_school_macroeconomics": ["Social"],+ "high_school_mathematics": ["STEM"],+ "high_school_microeconomics": ["Social"],+ "high_school_physics": ["STEM"],+ "high_school_psychology": ["Social"],+ "high_school_statistics": ["STEM"],+ "high_school_us_history": ["Hums"],+ "high_school_world_history": ["Hums"],+ "human_aging": ["Other"],+ "human_sexuality": ["Social"],+ "international_law": ["Hums"],+ "jurisprudence": ["Hums"],+ "logical_fallacies": ["Hums"],+ "machine_learning": ["STEM"],+ "management": ["Other"],+ "marketing": ["Other"],+ "medical_genetics": ["Other"],+ "miscellaneous": ["Other"],+ "moral_disputes": ["Hums"],+ "moral_scenarios": ["Hums"],+ "nutrition": ["Other"],+ "philosophy": ["Hums"],+ "prehistory": ["Hums"],+ "professional_accounting": ["Other"],+ "professional_law": ["Hums"],+ "professional_medicine": ["Other"],+ "professional_psychology": ["Social"],+ "public_relations": ["Social"],+ "security_studies": ["Social"],+ "sociology": ["Social"],+ "us_foreign_policy": ["Social"],+ "virology": ["Other"],+ "world_religions": ["Hums"],+}+++def result_parser(data):+ mmlu_results = {}+ other_results = {}+ for task in data.keys():+ acc_dict = {"Hums": [0, 0., 0.], "STEM": [0, 0., 0.], "Social": [0, 0., 0.], "Other": [0, 0., 0.],+ "Avg.": [0, 0., 0.]}+ if isinstance(data[task], float):+ other_results[task] = data[task]+ else:+ results = data[task]['results']+ if "hendrycksTest" in task:+ for sub_task in results.keys():+ sub_name = sub_task.split("hendrycksTest-")[1]+ domain = task_domain[sub_name][0]+ domain_list = acc_dict[domain]+ domain_list[0] += 1+ domain_list[1] += results[sub_task]["acc"]+ domain_list[2] += results[sub_task]["acc_norm"]+ acc_dict['Avg.'][0] += 1+ acc_dict['Avg.'][1] += results[sub_task]["acc"]+ acc_dict['Avg.'][2] += results[sub_task]["acc_norm"]++ mmlu_results[task] = {}+ for key in acc_dict.keys():+ mmlu_results[task][key] = {}+ mmlu_results[task][key]['acc'] = acc_dict[key][1] / acc_dict[key][0]+ mmlu_results[task][key]['acc_norm'] = acc_dict[key][2] / acc_dict[key][0]+ else:+ other_results[task] = results++ # other_results = {}+ # for key in results.keys():+ # if 'hendrycksTest' in key:+ # sub_name = key.split("hendrycksTest-")[1]+ # domain=task_domain[sub_name][0]+ # domain_list = acc_dict[domain]+ # domain_list[0] += 1+ # domain_list[1] += results[key]["acc"]+ # domain_list[2] += results[key]["acc_norm"]+ # acc_dict['Avg.'][0] += 1+ # acc_dict['Avg.'][1] += results[key]["acc"]+ # acc_dict['Avg.'][2] += results[key]["acc_norm"]+ # else:+ # other_results[key] = results[key]++ # mmlu_results = {}+ # for key in acc_dict.keys():+ # mmlu_results[key] = {}+ # mmlu_results[key]['acc'] = acc_dict[key][1]/acc_dict[key][0]+ # mmlu_results[key]['acc_norm'] = acc_dict[key][2]/acc_dict[key][0]++ print("***** MMLU RESULTS *****")+ pprint.pprint(mmlu_results)+ print()++ print("***** OTHER RESULTS *****")+ pprint.pprint(other_results)+ print()+ new_results = mmlu_results+ new_results.update(other_results)+ return new_results++++++diff --git a/autoround/eval/utils.py b/autoround/eval/utils.pynew file mode 100644index 000000000..269b8e676--- /dev/null+++ b/autoround/eval/utils.py@@ -0,0 +1,195 @@++import numpy as np+import torch+import torch.nn as nn++def set_seed(seed):+ np.random.seed(seed)+ torch.random.manual_seed(seed)+++def get_wikitext2(nsamples, seed, seqlen, tokenizer):+ from datasets import load_dataset+ traindata = load_dataset('wikitext', 'wikitext-2-raw-v1', split='train')+ testdata = load_dataset('wikitext', 'wikitext-2-raw-v1', split='test')++ trainenc = tokenizer("\n\n".join(traindata['text']), return_tensors='pt')+ testenc = tokenizer("\n\n".join(testdata['text']), return_tensors='pt')++ import random+ random.seed(seed)+ trainloader = []+ for _ in range(nsamples):+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)+ j = i + seqlen+ inp = trainenc.input_ids[:, i:j]+ tar = inp.clone()+ tar[:, :-1] = -100+ trainloader.append((inp, tar))+ return trainloader, testenc++def get_ptb(nsamples, seed, seqlen, tokenizer):+ from datasets import load_dataset+ traindata = load_dataset('ptb_text_only', 'penn_treebank', split='train')+ valdata = load_dataset('ptb_text_only', 'penn_treebank', split='validation')++ trainenc = tokenizer("\n\n".join(traindata['sentence']), return_tensors='pt')+ testenc = tokenizer("\n\n".join(valdata['sentence']), return_tensors='pt')++ import random+ random.seed(seed)+ trainloader = []+ for _ in range(nsamples):+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)+ j = i + seqlen+ inp = trainenc.input_ids[:, i:j]+ tar = inp.clone()+ tar[:, :-1] = -100+ trainloader.append((inp, tar))+ return trainloader, testenc++def get_c4(nsamples, seed, seqlen, tokenizer):+ from datasets import load_dataset+ traindata = load_dataset(+ 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'+ )+ valdata = load_dataset(+ 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'+ )+++ import random+ random.seed(seed)+ trainloader = []+ for _ in range(nsamples):+ while True:+ i = random.randint(0, len(traindata) - 1)+ trainenc = tokenizer(traindata[i]['text'], return_tensors='pt')+ if trainenc.input_ids.shape[1] >= seqlen:+ break+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)+ j = i + seqlen+ inp = trainenc.input_ids[:, i:j]+ tar = inp.clone()+ tar[:, :-1] = -100+ trainloader.append((inp, tar))++ import random+ random.seed(0)+ valenc = []+ for _ in range(256):+ while True:+ i = random.randint(0, len(valdata) - 1)+ tmp = tokenizer(valdata[i]['text'], return_tensors='pt')+ if tmp.input_ids.shape[1] >= seqlen:+ break+ i = random.randint(0, tmp.input_ids.shape[1] - seqlen - 1)+ j = i + seqlen+ valenc.append(tmp.input_ids[:, i:j])+ valenc = torch.hstack(valenc)+ class TokenizerWrapper:+ def __init__(self, input_ids):+ self.input_ids = input_ids+ valenc = TokenizerWrapper(valenc)++ return trainloader, valenc++def get_ptb_new(nsamples, seed, seqlen, tokenizer):+ from datasets import load_dataset+ traindata = load_dataset('ptb_text_only', 'penn_treebank', split='train')+ testdata = load_dataset('ptb_text_only', 'penn_treebank', split='test')++ trainenc = tokenizer(" ".join(traindata['sentence']), return_tensors='pt')+ testenc = tokenizer(" ".join(testdata['sentence']), return_tensors='pt')++ import random+ random.seed(seed)+ trainloader = []+ for _ in range(nsamples):+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)+ j = i + seqlen+ inp = trainenc.input_ids[:, i:j]+ tar = inp.clone()+ tar[:, :-1] = -100+ trainloader.append((inp, tar))+ return trainloader, testenc++def get_c4_new(nsamples, seed, seqlen, tokenizer):+ from datasets import load_dataset+ traindata = load_dataset(+ 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'+ )+ valdata = load_dataset(+ 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'+ )++ import random+ random.seed(seed)+ trainloader = []+ for _ in range(nsamples):+ while True:+ i = random.randint(0, len(traindata) - 1)+ trainenc = tokenizer(traindata[i]['text'], return_tensors='pt')+ if trainenc.input_ids.shape[1] >= seqlen:+ break+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)+ j = i + seqlen+ inp = trainenc.input_ids[:, i:j]+ tar = inp.clone()+ tar[:, :-1] = -100+ trainloader.append((inp, tar))++ valenc = tokenizer(' '.join(valdata[:1100]['text']), return_tensors='pt')+ valenc = valenc.input_ids[:, :(256 * seqlen)]++ class TokenizerWrapper:+ def __init__(self, input_ids):+ self.input_ids = input_ids+ valenc = TokenizerWrapper(valenc)++ return trainloader, valenc+++def get_loaders(+ name, nsamples=128, seed=0, seqlen=2048, tokenizer=''+):+ if 'wikitext2' in name:+ return get_wikitext2(nsamples, seed, seqlen, tokenizer)+ if 'ptb' in name:+ if 'new' in name:+ return get_ptb_new(nsamples, seed, seqlen, tokenizer)+ return get_ptb(nsamples, seed, seqlen, tokenizer)+ if 'c4' in name:+ if 'new' in name:+ return get_c4_new(nsamples, seed, seqlen, tokenizer)+ return get_c4(nsamples, seed, seqlen, tokenizer)++[email protected]_grad()+def eval_ppl_same_with_gptq(model, testenc, dev):+ print('Evaluating ...', flush=True)++ testenc = testenc.input_ids+ nsamples = testenc.numel() // model.seqlen++ use_cache = model.config.use_cache+ model.config.use_cache = False++ testenc = testenc.to(dev)+ nlls = []+ for i in range(nsamples):+ batch = testenc[:, (i * model.seqlen):((i + 1) * model.seqlen)].to(dev)+ lm_logits = model(batch).logits+ # to main gpu for multi-gpus+ shift_logits = lm_logits[:, :-1, :].contiguous().to(dev)+ shift_labels = testenc[+ :, (i * model.seqlen):((i + 1) * model.seqlen)+ ][:, 1:].to(dev)+ loss_fct = nn.CrossEntropyLoss()+ loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))+ neg_log_likelihood = loss.float() * model.seqlen+ nlls.append(neg_log_likelihood)+ ppl = torch.exp(torch.stack(nlls).sum() / (nsamples * model.seqlen))++ model.config.use_cache = use_cache+ return ppl.item()diff --git a/autoround/main.py b/autoround/main.pynew file mode 100644index 000000000..4307e141e--- /dev/null+++ b/autoround/main.py@@ -0,0 +1,205 @@+import argparse+import sys+sys.path.insert(0, './')+from src.autoround import AutoRound, AutoAdamRound++parser = argparse.ArgumentParser()+import torch+import os++os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"+torch.use_deterministic_algorithms(True, warn_only=True)+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel++from transformers import set_seed++from eval.eval import eval_model++import re++os.environ["TOKENIZERS_PARALLELISM"] = "false"++++if __name__ == '__main__':++ parser.add_argument(+ "--model_name", default="/models/opt-125m"+ )++ parser.add_argument("--num_bits", default=4, type=int,+ help="number of bits")++ parser.add_argument("--group_size", default=128, type=int,+ help="group size")++ parser.add_argument("--train_bs", default=8, type=int,+ help="train batch size")++ parser.add_argument("--eval_bs", default=4, type=int,+ help="eval batch size")++ parser.add_argument("--device", default=0, type=str,+ help="device gpu int number, or 'cpu' ")+ #+ parser.add_argument("--sym", action='store_true',+ help=" sym quantization")++ parser.add_argument("--iters", default=400, type=int,+ help=" iters")++ parser.add_argument("--use_quant_input", action='store_true',+ help="whether to use the output of quantized block to tune the next block")++ parser.add_argument("--lr", default=0.05, type=float,+ help="step size")++ parser.add_argument("--minmax_lr", default=None, type=float,+ help="minmax learning rate, if None,it will set to be the same with lr")++ parser.add_argument("--seed", default=42, type=int,+ help="seed")++ parser.add_argument("--eval_fp16_baseline", action='store_true',+ help="whether to eval FP16 baseline")++ parser.add_argument("--amp", action='store_true',+ help="amp")++ parser.add_argument("--adam", action='store_true',+ help="adam")++ parser.add_argument("--seqlen", default=2048, type=int,+ help="sequence length")++ parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")++ parser.add_argument("--n_blocks", default=1, type=int, help="num of blocks to tune together")++ parser.add_argument("--n_samples", default=512, type=int,+ help="number of samples")++ parser.add_argument("--low_gpu_mem_usage", action='store_true',+ help="low_gpu_mem_usage")++ parser.add_argument("--enable_minmax_tuning", action='store_true',+ help="whether enable weight minmax tuning")++ # parser.add_argument("--tasks", default=["lambada_openai", "hellaswag", "winogrande", "piqa"],+ # help="lm-eval tasks")++ # parser.add_argument("--tasks", default=["lambada_openai"],+ # help="lm-eval tasks")++ # parser.add_argument("--tasks",+ # default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',+ # 'coqa', 'truthfulqa_mc', 'openbookqa', 'boolq', 'rte', 'arc_easy', 'arc_challenge',+ # 'hendrycksTest-*', 'wikitext', 'drop', 'gsm8k'],##all+ parser.add_argument("--tasks",+ default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',+ "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte",+ "arc_easy", "arc_challenge"],+ help="lm-eval tasks") # "truthfulqa_gen"++ # parser.add_argument("--tasks", default=["lambada_openai"],+ # help="lm-eval tasks")++ parser.add_argument("--output_dir", default="./tmp_signround", type=str,+ help="Where to store the final model.")++ args = parser.parse_args()+ set_seed(args.seed)++ model_name = args.model_name+ if model_name[-1] == "/":+ model_name = model_name[:-1]+ print(model_name, flush=True)++ tasks = args.tasks++ if args.device == "cpu":+ device_str = "cpu"+ else:+ device_str = f"cuda:{int(args.device)}"+ cuda_device = torch.device(device_str)+ is_glm = bool(re.search("chatglm", model_name.lower()))+ if is_glm:+ model = AutoModel.from_pretrained(model_name, trust_remote_code=True)+ else:+ model = AutoModelForCausalLM.from_pretrained(+ model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True+ )+ model = model.eval()+ # align wigh GPTQ to eval ppl+ if "opt" in model_name:+ seqlen = model.config.max_position_embeddings+ model.seqlen = model.config.max_position_embeddings+ else:+ seqlen = 2048+ model.seqlen = seqlen+ seqlen = args.seqlen++ if "llama" in model_name:+ from transformers import LlamaTokenizer++ tokenizer = LlamaTokenizer.from_pretrained(model_name)+ if tokenizer.pad_token is None:+ tokenizer.add_special_tokens({'pad_token': '[PAD]'})+ else:+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ if hasattr(tokenizer, "model_max_length"):+ if tokenizer.model_max_length < seqlen:+ print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",+ flush=True)+ seqlen = min(seqlen, tokenizer.model_max_length)+ args.seqlen = seqlen++ excel_name = f"{model_name}_{args.num_bits}_{args.group_size}"+ if args.eval_fp16_baseline:+ if not args.low_gpu_mem_usage:+ model = model.to(cuda_device)+ excel_name += "_fp16.xlsx"+ eval_model(output_dir=model_name, model=model, tokenizer=tokenizer, tasks=args.tasks, \+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device,+ eval_orig_float=True, excel_file=excel_name)+ exit()++ # if args.iters <= 0:+ # print("eval rtn", flush=True)+ # excel_name += "_rtn.xlsx"+ # q_dq_weight(model, num_bits=args.num_bits, group_size=args.group_size)+ # if not args.low_gpu_mem_usage:+ # model = model.to(cuda_device)+ # eval_model(output_dir=args.output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \+ # eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device,+ # excel_file=excel_name)+ # exit()++ if not args.low_gpu_mem_usage:+ model = model.to(cuda_device)++ scheme = "asym"+ if args.sym:+ scheme = "sym"+ round = AutoRound+ if args.adam:+ round = AutoAdamRound++ autoround = round(model, tokenizer, args.num_bits, args.group_size, scheme, bs=args.train_bs,+ seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,+ minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input,+ amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage, seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps) ##TODO args pass+ autoround.quantize()++ torch.cuda.empty_cache()+ model.eval()+ output_dir = args.output_dir + "_" + args.model_name.split('/')[-1] + f"_w{args.num_bits}_g{args.group_size}"++ # model.to(cuda_device)+ # eval_model(model, model_name, tokenizer, tasks=args.tasks, eval_bs=args.eval_bs)+ excel_name = f"{output_dir}_result.xlsx"+ output_dir += "/"+ print(excel_name, flush=True)+ eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device, excel_file=excel_name,+ limit=None)diff --git a/autoround/requirements.txt b/autoround/requirements.txtnew file mode 100644index 000000000..25c5f5238--- /dev/null+++ b/autoround/requirements.txt@@ -0,0 +1,10 @@+transformers==4.36.0+torch==2.0.1+git+https://github.com/EleutherAI/lm-evaluation-harness.git@008fc2a23245c40384f2312718433eeb1e0f87a9+fsspec==2023.9.2+tiktoken+transformers_stream_generator+peft+sentencepiece+einops+diff --git a/autoround/src/__init__.py b/autoround/src/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/autoround/src/autoround.py b/autoround/src/autoround.pynew file mode 100644index 000000000..fa66a25e0--- /dev/null+++ b/autoround/src/autoround.py@@ -0,0 +1,1667 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++try:+ from neural_compressor.utils.utility import LazyImport++ torch = LazyImport("torch")+ from neural_compressor.utils import logger+except: # pragma: no cover+ import logging++ import torch++ logger = logging.getLogger()++import copy+import time+from collections import UserDict+from functools import partial++from torch.amp import autocast+from torch.functional import F+++def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):+ """Quantizes and dequantizes weight asymmetrically.++ Args:+ weight: Tensor containing the weight to be quantized+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for weight+ max_scale: Maximum scale coefficient for weight++ Returns:+ Quantized and dequantized weight, scale, zero-point+ """+ maxq = torch.tensor(2**num_bits - 1)+ zeros = torch.zeros(weight.shape[0], device=weight.device)+ if isinstance(min_scale, torch.Tensor):+ wmin_tmp = torch.minimum(weight.min(1)[0], zeros)+ wmax_tmp = torch.maximum(weight.max(1)[0], zeros)+ wmin_tmp *= min_scale + 1.0+ wmax_tmp *= max_scale + 1.0+ wmax = torch.maximum(wmax_tmp, wmin_tmp)+ wmin = torch.minimum(wmax_tmp, wmin_tmp)+ else:+ wmin = torch.minimum(weight.min(1)[0], zeros)+ wmax = torch.maximum(weight.max(1)[0], zeros)+ tmp = (wmin == 0) & (wmax == 0)+ wmin[tmp] = -1+ wmax[tmp] = +1+ scale = (wmax - wmin) / maxq+ zp = round_ste(-wmin / scale)+ scale = scale.unsqueeze(dim=-1)+ zp = zp.unsqueeze(dim=-1)+ int_w = round_ste(weight / scale + v)+ q = torch.clamp(int_w + zp, 0, maxq)+ return scale * (q - zp), scale, zp+++def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):+ """Quantizes and dequantizes weight symmetrically.++ Args:+ weight: Tensor containing the weight to be quantized+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for weight+ max_scale: Maximum scale coefficient for weight++ Returns:+ Quantized and dequantized weight, scale, zero-point+ """+ maxq = torch.tensor(2 ** (num_bits - 1) - 1).to(weight.device)+ minq = torch.tensor(-(2 ** (num_bits - 1))).to(weight.device)+ if num_bits == 1:+ maxq = torch.tensor(2 ** (num_bits - 1))+ minq = torch.tensor(2 ** (num_bits - 1) - 1)++ wmax = torch.abs(weight).max(1)[0]+ wmax *= 1 + max_scale+ tmp = wmax == 0+ wmax[tmp] = +1+ scale = wmax / ((maxq - minq) / 2)+ scale.unsqueeze_(dim=-1)+ q = torch.clamp(round_ste(weight / scale + v), minq, maxq)+ return scale * q, scale, None+++def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale):+ """Quantizes and dequantizes weight symmetrically or asymmetrically .++ Args:+ weight: Tensor containing the weight to be quantized+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ scheme: Sym or asym+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for weight+ max_scale: Maximum scale coefficient for weight++ Returns:+ Quantized and dequantized weight, scale, zero-point+ """+ assert num_bits > 0, "num_bits should be larger than 0"+ if scheme == "sym":+ return quant_weight_sym(weight, num_bits, v, min_scale, max_scale)+ else:+ return quant_weight_asym(weight, num_bits, v, min_scale, max_scale)+++def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0):+ """Quantizes and dequantizes weight, handing the group size issue .++ Args:+ weight: Tensor containing the weight to be quantized+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: The number of elements shares scale and zero point+ scheme: Sym or asym+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for weight+ max_scale: Maximum scale coefficient for weight++ Returns:+ Quantized and dequantized weight, scale, zero-point+ """+ if group_size == -1 or weight.shape[1] < group_size:+ return quant_weight_actor(weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale)+ orig_shape = weight.shape+ if weight.shape[1] % group_size == 0:+ weight = weight.reshape(-1, group_size)+ if isinstance(v, torch.Tensor):+ v = v.reshape(-1, group_size)++ weight, scale, zp = quant_weight_actor(+ weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale+ )+ weight = weight.reshape(orig_shape)+ return weight, scale, zp++ else:+ pad_len = (weight.shape[1] + group_size - 1) // group_size * group_size - weight.shape[1]+ weight_new = torch.nn.functional.pad(weight, (0, pad_len))+ v = torch.nn.functional.pad(v, (0, pad_len))+ weight_new = weight_new.reshape(-1, group_size)+ if isinstance(v, torch.Tensor):+ v = v.reshape(-1, group_size)+ weight_new, scale, zp = quant_weight_actor(+ weight_new, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale+ )+ weight_new = weight_new.reshape(orig_shape[0], -1)++ weight_new = weight_new[:, :-pad_len]+ return weight_new, scale, zp+++def round_ste(x: torch.Tensor):+ """Straight-Through Estimator for rounding.+ This function is adapted from omniquant.++ Args:+ x: torch.Tensor++ Returns:+ torch.Tensor+ """+ return (x.round() - x).detach() + x+++class SaveInputs:+ """Cache the inputs of the first block."""++ def __init__(self, model, dataloader, seqlen=256, block_name=None):+ """Initializes the SaveInputs class.++ Args:+ model: The model to be used.+ dataloader: The dataloader for the input data.+ seqlen (int): The sequence length.+ block_name (str): The name of the block.+ """+ self.model = model.eval()+ self.dataloader = dataloader+ self.inputs = {}+ self.block_name = block_name+ self.seqlen = seqlen++ @torch.no_grad()+ def get_forward_func(self, name):+ """Gets the forward function.++ Args:+ name (str): The name of the function.++ Returns:+ function: The forward function.+ """++ def forward(_, hidden_states, *positional_args, **kwargs):+ dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type))+ if name in self.inputs:+ data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=dim)+ self.inputs[name]["input_ids"] = data+ else:+ self.inputs[name] = {}+ self.inputs[name]["input_ids"] = hidden_states.to("cpu")++ if "positional_inputs" not in self.inputs[name]:+ self.inputs[name]["positional_inputs"] = []+ for idx, item in enumerate(positional_args):+ self.inputs[name]["positional_inputs"] = move_input_to_device(positional_args)++ for key in kwargs.keys():+ if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) or (key == "alibi"):+ if "attention_mask" in key:+ if key not in self.inputs[name].keys():+ self.inputs[name][key] = None+ if kwargs[key] is not None:+ if self.inputs[name][key] is not None:+ self.inputs[name][key] = torch.cat(+ [self.inputs[name][key], kwargs[key].to("cpu")], dim=0+ )+ else:+ self.inputs[name][key] = kwargs[key].to("cpu")+ elif "alibi" in key:+ if key not in self.inputs[name].keys():+ self.inputs[name][key] = None+ if isinstance(kwargs[key], torch.Tensor):+ alibi = kwargs[key]+ batch = kwargs["attention_mask"].shape[0]+ alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2])+ if self.inputs[name][key] is not None:+ self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0)+ else:+ self.inputs[name][key] = alibi.to("cpu")+ elif key not in self.inputs[name].keys():+ self.inputs[name][key] = move_input_to_device(kwargs[key], device=torch.device("cpu"))+ raise NotImplementedError++ return forward++ @torch.no_grad()+ def get_inputs(self, n_samples=512):+ """Gets the inputs.++ Args:+ n_samples (int): The number of samples.++ Returns:+ dict: The inputs.+ """+ total_cnt = 0+ self._replace_forward()+ for data in self.dataloader:+ if data is None:+ continue+ if isinstance(data, torch.Tensor):+ input_ids = data.to(self.model.device)+ else:+ input_ids = data["input_ids"].to(self.model.device)+ if input_ids.shape[-1] < self.seqlen:+ continue+ if total_cnt + input_ids.shape[0] > n_samples:+ input_ids = input_ids[: n_samples - total_cnt, ...]+ try:+ self.model(input_ids)+ except NotImplementedError:+ pass+ except Exception as error:+ logger.error(error)+ total_cnt += input_ids.shape[0]+ if total_cnt >= n_samples:+ break+ self._recover_forward()+ if total_cnt == 0:+ logger.error(+ f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "+ f"dataloader or decease the sequence length"+ )+ exit()+ elif total_cnt < n_samples:+ logger.warning(+ f"Insufficient number of samples collected may affect the quantification. "+ f"Effective samples size:{total_cnt}, Target sample size:{n_samples}"+ )+ res = self.inputs[self.block_name]+ if "input_ids" in res.keys():+ total_samples = res["input_ids"].shape[0]+ if total_samples < n_samples:+ logger.warning("only cache {total_samples}")++ return res++ def _recover_forward(self):+ """Recovers the forward function."""+ for n, m in self.model.named_modules():+ if n == self.block_name:+ m.forward = m.orig_forward+ delattr(m, "orig_forward")+ break++ def _replace_forward(self):+ """Replaces the forward function."""+ for n, m in self.model.named_modules():+ if n == self.block_name:+ m.orig_forward = m.forward+ m.forward = partial(self.get_forward_func(n), m)+ break+++def get_module(model, key):+ """Get module from model by key name.++ Args:+ model (torch.nn.Module): original model+ key (str): module name to be replaced+ """+ module = model+ name_list = key.split(".")+ for name in name_list:+ if hasattr(module, name):+ module = getattr(module, name)+ module = module+ return module+++def set_module(model, key, new_module):+ """Set new module into model by key name.++ Args:+ model (torch.nn.Module): original model+ key (str): module name to be replaced+ new_module (torch.nn.Module): new module to be inserted+ """+ module = model+ name_list = key.split(".")+ for name in name_list[:-1]:+ if hasattr(module, name):+ module = getattr(module, name)+ else:+ module = module+ setattr(module, name_list[-1], new_module)+++def get_scale_shape(weight, group_size):+ """Computes the shape of the scale tensor for quantization based on the weight tensor and group size.++ Args:+ weight (torch.Tensor): The weight tensor of the layer.+ group_size (int): The size of the groups for quantization.++ Returns:+ The shape of the scale tensor to be used for quantization.+ """+ if group_size == -1 or weight.shape[1] < group_size:+ shape = weight.shape[0]+ else:+ shape = weight.shape[0] * ((weight.shape[1] + group_size - 1) // group_size)++ return shape+++class WrapperLinear(torch.nn.Module):+ def __init__(self, orig_layer, enable_minmax_tuning=True):+ """A wrapper module for linear layers that enables quantization and min-max tuning of weights.++ Args:+ - orig_layer (torch.nn.Module): The original linear layer to be wrapped.+ - enable_minmax_tuning (bool): Whether to enable min-max scaling tuning. Default is True.++ Attributes:+ - orig_layer (torch.nn.Module): The original linear layer being wrapped.+ - num_bits (int): The number of bits for quantization.+ - group_size (int): The size of the groups for quantization.+ - scheme (str): The quantization scheme to use.+ - value (torch.nn.Parameter): The learnable parameter for quantization.+ - enable_minmax_tuning (bool): Whether min-max scaling tuning is enabled.+ - min_scale (torch.nn.Parameter or torch.Tensor): The minimum scale for min-max tuning.+ - max_scale (torch.nn.Parameter or torch.Tensor): The maximum scale for min-max tuning.+ """+ super(WrapperLinear, self).__init__()+ self.orig_layer = orig_layer+ self.num_bits = self.orig_layer.bits+ self.group_size = self.orig_layer.group_size+ self.scheme = (self.orig_layer.scheme,)+ self.value = torch.nn.Parameter(+ torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device), requires_grad=True+ )+ self.enable_minmax_tuning = enable_minmax_tuning+ shape = get_scale_shape(self.orig_layer.weight, self.group_size)++ if self.enable_minmax_tuning:+ self.min_scale = torch.nn.Parameter(+ torch.zeros(shape, device=self.orig_layer.weight.device), requires_grad=True+ )+ self.max_scale = torch.nn.Parameter(+ torch.zeros(shape, device=self.orig_layer.weight.device), requires_grad=True+ )+ else:+ self.min_scale = torch.tensor(0, device=self.orig_layer.weight.device)+ self.max_scale = torch.tensor(0, device=self.orig_layer.weight.device)++ def unwrapper(self, v, min_scale, max_scale):+ """Unwrapper the layer to the original layer.++ Args:+ - v (torch.Tensor): The rounding v parameter for quantization.+ - min_scale (torch.nn.Parameter or torch.Tensor): The minimum scale for min-max tuning.+ - max_scale (torch.nn.Parameter or torch.Tensor): The maximum scale for min-max tuning.++ Returns:+ - torch.nn.Module: The original linear layer with updated weights after quantization and dequantization.+ """+ min_scale.clamp_(-1, 0)+ max_scale.clamp_(-1, 0)++ q_dq_weight, scale, zp = quant_weight(+ self.orig_layer.weight,+ self.num_bits,+ self.group_size,+ self.scheme,+ v,+ min_scale,+ max_scale,+ )+ self.orig_layer.weight.data.copy_(q_dq_weight)+ self.orig_layer.weight.grad = None ##clear grad+ self.orig_layer.scale = scale.to("cpu")+ self.orig_layer.zp = zp.to("cpu")+ return self.orig_layer++ def forward(self, x):+ """Performs forward pass through the wrapped linear layer with quantized weights.++ Args:+ - x (torch.Tensor): The input tensor.++ Returns:+ - torch.Tensor: The output tensor after applying the linear transformation with quantized weights.+ """+ weight = self.orig_layer.weight+ self.min_scale.data.copy_(torch.clamp(self.min_scale.data, -1, 0))+ self.max_scale.data.copy_(torch.clamp(self.max_scale.data, -1, 0))+ weight_q, _, _ = quant_weight(+ weight, self.num_bits, self.group_size, self.scheme, self.value, self.min_scale, self.max_scale+ )+ weight_q = weight_q.to(weight.dtype)+ return F.linear(x, weight_q, self.orig_layer.bias)+++class WrapperTransformerConv1d(torch.nn.Module):+ def __init__(self, orig_layer, enable_minmax_tuning=True):+ """A wrapper module for transformers 1D convolutional layers used in transformers,+ enabling quantization and min-max tuning of weights.++ Args:+ - orig_layer (torch.nn.Module): The original 1D convolutional layer to be wrapped.+ - num_bits (int): The number of bits for quantization.+ - group_size (int): The size of the groups for quantization.+ - scheme (str): The quantization scheme to use.+ - enable_minmax_tuning (bool): Whether to enable min-max scaling tuning. Default is True.++ Attributes:+ - orig_layer (torch.nn.Module): The original 1D convolutional layer being wrapped.+ - num_bits (int): The number of bits for quantization.+ - group_size (int): The size of the groups for quantization.+ - scheme (str): The quantization scheme to use.+ - weight_t (torch.Tensor): Transposed weight tensor of the original layer.+ - value (torch.nn.Parameter): The learnable parameter for quantization.+ - enable_minmax_tuning (bool): Whether min-max scaling tuning is enabled.+ - min_scale (torch.nn.Parameter or torch.Tensor): The minimum scale for min-max tuning.+ - max_scale (torch.nn.Parameter or torch.Tensor): The maximum scale for min-max tuning.+ """+ super(WrapperTransformerConv1d, self).__init__()+ self.orig_layer = orig_layer+ self.num_bits = self.orig_layer.bits+ self.group_size = self.orig_layer.group_size+ self.scheme = self.orig_layer.scheme+ device = self.orig_layer.weight.device+ self.weight_t = self.orig_layer.weight.t()+ self.value = torch.nn.Parameter(torch.zeros(self.weight_t.shape, device=device), requires_grad=True)+ shape = get_scale_shape(self.weight_t, self.group_size)++ if enable_minmax_tuning:+ self.min_scale = torch.nn.Parameter(torch.zeros(shape, device=device), requires_grad=True)+ self.max_scale = torch.nn.Parameter(torch.zeros(shape, device=device), requires_grad=True)+ else:+ self.min_scale = torch.tensor(0, device=device)+ self.max_scale = torch.tensor(0, device=device)++ def unwrapper(self, v, min_scale, max_scale):+ """Unwrapper the layer to the original conv1d layer..++ Args:+ - v (torch.Tensor): The scaling parameter for quantization.+ - min_scale (torch.nn.Parameter or torch.Tensor): The minimum scale for min-max tuning.+ - max_scale (torch.nn.Parameter or torch.Tensor): The maximum scale for min-max tuning.++ Returns:+ - torch.nn.Module: The original 1D convolutional layer with updated weights after inverse quantization.+ """+ min_scale.clamp_(-1, 0)+ max_scale.clamp_(-1, 0)+ weight_q, scale, zp = quant_weight(+ self.weight_t,+ self.num_bits,+ self.group_size,+ self.scheme,+ v,+ min_scale,+ max_scale,+ )+ self.orig_layer.weight.data.copy_(weight_q.t())+ self.orig_layer.weight.grad = None+ self.orig_layer.scale = scale.to("cpu")+ self.orig_layer.zp = zp.to("cpu")+ return self.orig_layer++ def forward(self, x):+ """Performs forward pass through the wrapped 1D convolutional layer with quantized weights.++ Args:+ x (torch.Tensor): The input tensor.++ Returns:+ torch.Tensor: The output tensor after applying the convolutional transformation with quantized weights.+ """+ with torch.no_grad():+ self.min_scale.clamp_(-1, 0)+ self.max_scale.clamp_(-1, 0)+ weight_q, _, _ = quant_weight(+ self.weight_t,+ self.num_bits,+ self.group_size,+ self.scheme,+ self.value,+ self.min_scale,+ self.max_scale,+ )+ weight_q = weight_q.to(self.weight_t.dtype)+ size_out = x.size()[:-1] + (self.orig_layer.nf,)+ x = torch.addmm(self.orig_layer.bias, x.view(-1, x.size(-1)), weight_q.t())+ x = x.view(*size_out)+ return x+++def wrapper_block(block, enable_minmax_tuning):+ """Wraps the layers in the given block with a custom Wrapper module.++ Args:+ block: The input block containing linear and conv1d layers to be wrapped.+ enable_minmax_tuning: A boolean indicating whether min-max tuning is enabled.++ Returns:+ list: A list of names of the wrapped layers.+ """+ names = []+ for n, m in block.named_modules():+ if isinstance(m, torch.nn.Linear):+ if "fp" in m.data_type or "float" in m.data_type:+ continue+ new_m = WrapperLinear(m, enable_minmax_tuning=enable_minmax_tuning)+ set_module(block, n, new_m)+ names.append(n)++ try:+ import transformers++ if isinstance(m, transformers.modeling_utils.Conv1D):+ if "fp" in m.data_type or "float" in m.data_type:+ continue+ new_m = WrapperTransformerConv1d(m, enable_minmax_tuning=enable_minmax_tuning)+ set_module(block, n, new_m)+ names.append(n)+ except:+ pass+ return names++[email protected]_grad()+def unwrapper_block(block, vs, min_scales, max_scales):+ """Unwraps the WrapperLinear and WrapperTransformerConv1d modules in the given block.++ Args:+ block: The input block containing wrapped modules to be unwrapped.+ vs: A dictionary of scaling parameters for the wrapped modules.+ min_scales: A dictionary of minimum scaling values for the wrapped modules.+ max_scales: A dictionary of maximum scaling values for the wrapped modules.+ """+ for n, m in block.named_modules():+ if hasattr(m, "orig_layer"):+ v = 0+ min_scale = 0+ max_scale = 0+ if isinstance(vs, dict):+ v = vs[n]+ if isinstance(min_scales, dict):+ min_scale = min_scales[n]+ min_scale = torch.clamp(min_scale, -1, 0)+ if isinstance(max_scales, dict):+ max_scale = max_scales[n]+ max_scale = torch.clamp(max_scale, -1, 0)+ orig_layer = m.unwrapper(v, min_scale, max_scale)+ set_module(block, n, orig_layer)+++def sampling_inputs(input_ids, input_others, indices, seqlen):+ """Samples inputs based on the given indices and sequence length.++ Args:+ input_ids: The input tensor containing IDs.+ input_others: A dictionary containing other input data.+ indices: The indices to sample from the input.+ seqlen: The sequence length.++ Returns:+ current_input_ids: The sampled input IDs.+ current_input_others: The sampled other input data.+ """+ if len(input_ids.shape) == 3:+ if int(len(input_others["positional_inputs"]) > 0):+ current_input_ids = input_ids[:, indices, :]+ else:+ current_input_ids = input_ids[indices, :, :]+ else:+ n_samples = input_ids.shape[0] // seqlen+ current_input_ids = input_ids.view(n_samples, seqlen, -1)+ current_input_ids = current_input_ids[indices, :, :]+ current_input_ids = current_input_ids.reshape(-1, input.shape[-1])++ current_input_others = {"positional_inputs": input_others["positional_inputs"]}+ for key in input_others.keys():+ if "attention_mask" in key or "alibi" in key:+ current_input_others[key] = None+ if input_others[key] is not None:+ current_input_others[key] = input_others[key][indices, ...]+ else:+ current_input_others[key] = input_others[key]++ return current_input_ids, current_input_others+++def move_input_to_device(input, device=torch.device("cpu")):+ """Moves input data to the specified device.++ Args:+ input: The input data to be moved.+ device: The target device.++ Returns:+ The input data on the specified device.+ """+ if isinstance(input, torch.Tensor):+ return input.to(device)+ if isinstance(input, dict) or isinstance(input, UserDict):+ for inp in input.keys():+ input[inp] = move_input_to_device(input[inp], device)+ elif isinstance(input, list) or isinstance(input, tuple):+ input_res = []+ for inp in input:+ input_res.append(move_input_to_device(inp, device))+ input = input_res+ return input+++def check_is_cpu(device):+ """Check if the device is a CPU.++ Args:+ device: The device to be checked.++ Returns:+ bool: True if the device is a CPU, False otherwise.+ """+ return device == torch.device("cpu") or device == "cpu"+++def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.float16, device=torch.device("cpu")):+ """Performs a forward pass through a block with the given inputs.++ Args:+ block: The block to perform the forward pass on.+ input_ids: The input IDs.+ input_others: A dictionary containing other input data.+ amp: A boolean indicating whether to use automatic mixed precision.+ amp_dtype: The data type for automatic mixed precision.+ device: The target device.++ Returns:+ output: The output of the forward pass.+ """+ if input_ids.device != device:+ # input_ids, input_others = move_to_device(input_ids, input_others, device)+ input_ids = move_input_to_device(input_ids, device)+ input_others = move_input_to_device(input_others, device)+ if "alibi" in input_others.keys():+ attention_mask = input_others["attention_mask"]+ alibi = input_others["alibi"]+ if alibi is not None:+ alibi = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])+ if amp and not check_is_cpu(device):+ with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover+ output = block(+ input_ids, attention_mask=attention_mask, alibi=alibi+ ) ##TODO is this correct for all models with alibi?+ elif amp and check_is_cpu(device):+ with torch.autocast(device_type="cpu", dtype=torch.bfloat16):+ output = block(input_ids, attention_mask=attention_mask, alibi=alibi)+ else:+ output = block(input_ids, attention_mask=attention_mask, alibi=alibi)+ else:+ input_tuple = input_others.pop("positional_inputs", None)+ if amp and not check_is_cpu(device):+ with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover+ output = block.forward(input_ids, *input_tuple, **input_others)+ elif amp and check_is_cpu(device):+ with torch.autocast(device_type="cpu", dtype=torch.bfloat16):+ output = block.forward(input_ids, *input_tuple, **input_others)+ else:+ output = block.forward(input_ids, *input_tuple, **input_others)+ if isinstance(output, list) or isinstance(output, tuple):+ output = output[0]+ return output+++def collect_round_v(block):+ """Collects the round values for wrapped linear modules in the given block.++ Args:+ block: The input block.++ Returns:+ vs: A dictionary of round values for the wrapped linear modules.+ """+ vs = {}+ for n, m in block.named_modules():+ if hasattr(m, "orig_layer"):+ v = m.value.data+ vs[n] = copy.deepcopy(v)+ return vs+++def collect_minmax_scale(block):+ """Collects the min-max scaling values for wrapped linear modules in the given block.++ Args:+ block: The input block.++ Returns:+ min_scales: A dictionary of minimum scaling values.+ max_scales: A dictionary of maximum scaling values.+ """+ min_scales = {}+ max_scales = {}+ for n, m in block.named_modules():+ if hasattr(m, "orig_layer"):+ min_scales[n] = copy.deepcopy(torch.clamp(m.min_scale.data, -1, 0))+ max_scales[n] = copy.deepcopy(torch.clamp(m.max_scale.data, -1, 0))+ return min_scales, max_scales++[email protected]_grad()+def get_batch_dim(input_others):+ """Gets the batch dimension based on the input positional inputs.++ Args:+ input_others: A dictionary containing input data.++ Returns:+ dim: The batch dimension.+ """+ dim = int(len(input_others["positional_inputs"]) > 0)+ return dim+++class WrapperMultiblock(torch.nn.Module):+ """A wrapper for a list of modules to be act as a single block.++ Args:+ module_list: The list of modules to wrap.+ """++ def __init__(self, module_list):+ super(WrapperMultiblock, self).__init__()+ self.layers = torch.nn.ModuleList(module_list)++ def forward(self, x, **kwargs):+ hidden_states = x+ for idx, decoder_layer in enumerate(self.layers):+ layer_outputs = decoder_layer(hidden_states, **kwargs)+ hidden_states = layer_outputs+ if isinstance(hidden_states, tuple) or isinstance(hidden_states, list):+ hidden_states = layer_outputs[0]+ return hidden_states+++def get_block_names(model):+ """Get the block names for transformers-like networks.++ Args:+ model: The model.++ Returns:+ block_names: A list of block names.+ """+ block_names = []+ target_m = None+ for n, m in model.named_modules():+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:+ target_m = (n, m)+ for n, m in target_m[1].named_children():+ block_names.append(target_m[0] + "." + n)+ return block_names+++def get_tokenizer_function(tokenizer, seqlen):+ """Returns a default tokenizer function.++ Args:+ tokenizer: The tokenizer to be used for tokenization.+ seqlen: The maximum sequence length.++ Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length of+ seqlen to the "text" field of examples.+ """++ def default_tokenizer_function(examples):+ example = tokenizer(examples["text"], truncation=True, max_length=seqlen)+ return example++ return default_tokenizer_function+++def get_dataloader(tokenizer, seqlen, data_name="NeelNanda/pile-10k", split="train", seed=42, bs=4):+ """Returns a dataloader for the specified dataset and split.++ Args:+ tokenizer: The tokenizer to be used for tokenization.+ seqlen: The maximum sequence length.+ data_name: The name of the dataset.+ split: The data split to be used (e.g., "train", "test").+ seed: The random seed for shuffling the dataset.+ bs: The batch size for the dataloader.++ Returns:+ A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.+ """+ from datasets import load_dataset+ from torch.utils.data import DataLoader++ tokenizer_function = get_tokenizer_function(tokenizer, seqlen)++ @torch.no_grad()+ def collate_batch(batch):+ input_ids_new = []+ for text in batch:+ input_ids = text["input_ids"]+ if input_ids.shape[0] < seqlen:+ continue+ input_ids = input_ids[:seqlen]+ input_ids_list = input_ids.tolist()+ if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:+ continue+ input_ids_new.append(input_ids)+ if len(input_ids_new) == 0:+ return None+ tmp = torch.vstack(input_ids_new)+ res = {"input_ids": tmp}+ return res++ calib_dataset = load_dataset(data_name, split=split)+ calib_dataset = calib_dataset.shuffle(seed=seed)+ calib_dataset = calib_dataset.map(tokenizer_function, batched=True)+ calib_dataset.set_format(type="torch", columns=["input_ids"])+ calib_dataloader = DataLoader(calib_dataset, batch_size=bs, shuffle=False, collate_fn=collate_batch)+ return calib_dataloader+++class AutoRound(object):+ """This is Signround+ which is an advanced version of Signround. For more information,+ please refer to Cheng, Wenhua, et al. "Optimize weight rounding via signed gradient descent+ for the quantization of llms." arXiv preprint arXiv:2309.05516 (2023).++ Args:+ model: The PyTorch model to be quantized.+ tokenizer: An optional tokenizer for processing input data.+ bits (int): Number of bits for quantization (default is 4).+ group_size (int): Size of the quantization group (default is 128).+ scheme (str): The quantization scheme to be used (default is "asym").+ weight_config (dict): Configuration for weight quantization (default is an empty dictionary).+ weight_config={+ 'layer1':##layer_name+ {+ 'data_type': 'int',+ 'bits': 4,+ 'group_size': 32,+ 'scheme': "asym", ## or sym+ }+ ...+ }+ enable_full_range (bool): Whether to enable full range quantization (default is False).+ bs (int): Batch size for training (default is 8).+ amp (bool): Whether to use automatic mixed precision (default is True).+ device: The device to be used for training (default is "cuda:0").+ lr_scheduler: The learning rate scheduler to be used.+ dataloader: The dataloader for input data (to be supported in future).+ dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").+ dataset_split (str): The split of the dataset to be used (default is "train").+ use_quant_input (bool): Whether to use quantized input data (default is True).+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).+ lr (float): The learning rate (default is 0.005).+ minmax_lr (float): The learning rate for min-max tuning (default is None).+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).+ iters (int): Number of iterations (default is 200).+ seqlen (int): Length of the sequence.+ n_samples (int): Number of samples (default is 512).+ sampler (str): The sampling method (default is "rand").+ seed (int): The random seed (default is 42).+ n_blocks (int): Number of blocks (default is 1).+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).+ not_use_mse (bool): Whether to use mean squared error (default is False).+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).+ data_type (str): The data type to be used (default is "int").+ **kwargs: Additional keyword arguments.++ Returns:+ The quantized model.+ """++ def __init__(+ self,+ model,+ tokenizer=None,+ bits: int = 4,+ group_size: int = 128,+ scheme: str = "asym",+ weight_config: dict = {},+ enable_full_range: bool = False, ##for symmetric, TODO support later+ bs: int = 8,+ amp: bool = True,+ device="cuda:0",+ lr_scheduler=None,+ dataloader=None, ## to support later+ dataset_name: str = "NeelNanda/pile-10k",+ dataset_split: str = "train",+ use_quant_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = 0.005,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = True,+ iters: int = 200,+ seqlen: int = 2048,+ n_samples: int = 512,+ sampler: str = "rand",+ seed: int = 42,+ n_blocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int", ##only support data_type+ **kwargs,+ ):+ self.model = model+ self.model = self.model.to("cpu")+ self.amp = amp+ self.use_quant_input = use_quant_input+ self.enable_minmax_tuning = enable_minmax_tuning+ self.n_samples = n_samples+ self.n_blocks = n_blocks+ self.bits = bits+ self.group_size = group_size+ self.scheme = scheme+ self.low_gpu_mem_usage = low_gpu_mem_usage+ self.data_type = data_type+ self.supported_types = [torch.nn.Linear]+ try:+ import transformers++ self.supported_types.append(transformers.modeling_utils.Conv1D)+ except:+ pass+ self.weight_config = weight_config+ assert dataloader is not None or tokenizer is not None+ self.dataset_split = dataset_split+ self.seed = seed+ self.tokenizer = tokenizer+ self.seqlen = seqlen+ self.train_bs = bs+ self.n_blocks = n_blocks+ self.device = device+ self.amp_dtype = torch.float16+ if self.model.dtype != torch.float32:+ self.amp_dtype = self.model.dtype+ if self.device == "cpu":+ self.amp_dtype = torch.bfloat16+ if self.amp:+ self.model = self.model.to(self.amp_dtype)+ logger.info(f"using {self.amp_dtype}")+ self.dataset_name = dataset_name++ if dataloader is None:+ self.dataloader = get_dataloader(+ self.tokenizer,+ self.seqlen,+ seed=self.seed,+ bs=self.train_bs,+ split=self.dataset_split,+ data_name=self.dataset_name,+ )+ else:+ self.dataloader = dataloader+ self.lr = lr+ self.minmax_lr = minmax_lr+ if self.minmax_lr is None:+ self.minmax_lr = self.lr+ self.iters = iters+ self.sampler = sampler+ self.gradient_accumulate_steps = gradient_accumulate_steps+ self.not_use_mse = not_use_mse+ self.dynamic_max_gap = dynamic_max_gap+ self.enable_full_range = enable_full_range+ assert self.enable_full_range is False, "only support enable_full_range=False currently"+ self.lr_scheduler = lr_scheduler+ self.set_layerwise_config(self.weight_config)+ self.optimizer = self.get_optimizer(None)+ self.check_configs()++ def get_optimizer(self, optimizer):+ """Returns the specified optimizer. In SignRound, we fix the optimizer.++ Args:+ optimizer: The optimizer to be used.++ Returns:+ The specified optimizer.+ """+ from .sign_sgd import SGD++ return SGD++ def get_scaler(self):+ """Returns scaler, in SignRound, no need to use scaler."""+ return None++ def scale_loss_and_backward(self, scaler, loss):+ """Scales the loss and performs backward pass.++ Args:+ scaler: The scaler to be used.+ loss: The loss to be scaled.++ Returns:+ The scaled loss.+ """+ scale_loss = loss * 1000+ scale_loss.backward()+ return scale_loss++ def step(self, scaler, optimizer, lr_schedule):+ """Performs a step in the optimization process.++ Args:+ scaler: The scaler to be used.+ optimizer: The optimizer for the step.+ lr_schedule: The learning rate schedule.++ Returns:+ None+ """+ optimizer.step()+ optimizer.zero_grad()+ lr_schedule.step()++ def check_configs(self):+ """Checks if the configurations are valid.++ Raises:+ AssertionError: If any of the configurations are invalid.+ """+ assert isinstance(self.model, torch.nn.Module)+ assert self.bits > 0, "bits must be positive"+ assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"+ assert self.train_bs > 0, "batch size must be positive"+ assert self.iters > 0, "iters must be positive"+ assert self.seqlen > 0, "seqlen must be positive"+ assert self.n_blocks > 0, "n_blocks must be positive"+ assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"++ def set_layerwise_config(self, weight_config):+ """Sets the layer-wise configuration based on the provided weight_config.++ Args:+ weight_config: The weight configuration.++ Returns:+ None+ """+ for n, m in self.model.named_modules():+ is_supported_type = False+ for supported_type in self.supported_types:+ if isinstance(m, supported_type):+ is_supported_type = True+ break+ if not is_supported_type:+ continue+ if n not in weight_config.keys():+ weight_config[n] = {}+ weight_config[n]["data_type"] = self.data_type+ weight_config[n]["bits"] = self.bits+ weight_config[n]["group_size"] = self.group_size+ weight_config[n]["scheme"] = self.scheme+ else:+ if "data_type" not in weight_config[n].keys():+ weight_config[n]["data_type"] = self.data_type+ if "bits" not in weight_config[n].keys():+ weight_config[n]["bits"] = self.bits+ if "group_size" not in weight_config[n].keys():+ weight_config[n]["group_size"] = self.group_size+ if "scheme" not in weight_config[n].keys():+ weight_config[n]["scheme"] = self.scheme+ m.data_type = weight_config[n]["data_type"]+ m.bits = weight_config[n]["bits"]+ m.group_size = weight_config[n]["group_size"]+ m.scheme = weight_config[n]["scheme"]++ def get_batch_dim(self, input_others):+ """Get the batch dimension of the input tensor.++ Args:+ input_others: A dictionary containing input data, including positional_inputs.++ Returns:+ The batch dimension of the input tensor.+ """+ dim = int(len(input_others["positional_inputs"]) > 0)+ return dim++ @torch.no_grad()+ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device, batch_dim):+ """Compute the output of a given block of the model for a given input.++ Args:+ block: The block of the model.+ input_ids: The input tensor containing tokenized input ids.+ input_others: A dictionary containing additional input data.+ bs: The batch size for computing the output.+ device: The device for computation.+ cache_device: The device for storing the output.+ batch_dim: The batch dimension of the output tensor.++ Returns:+ The output tensor of the block.+ """++ output = []+ for i in range(0, self.n_samples, bs):+ indices = torch.arange(i, i + bs).to(torch.long)+ tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen)+ tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(+ cache_device+ )+ output.append(tmp_output)+ output = torch.cat(output, dim=batch_dim)+ torch.cuda.empty_cache()+ return output++ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):+ """Quantize the weights of a given block of the model.++ Args:+ block: The block of the model to be quantized.+ input_ids: The input tensor containing tokenized input ids.+ input_others: A dictionary containing additional input data.+ q_input: The quantized input tensor.+ device: The device for quantization.++ Returns:+ Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output)+ """+ batch_dim = get_batch_dim(input_others)+ if not self.low_gpu_mem_usage and input_ids.device != device:+ input_ids = move_input_to_device(input_ids, device)+ input_others = move_input_to_device(input_others, device)+ cache_device = device+ if self.low_gpu_mem_usage:+ cache_device = "cpu"+ output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device, batch_dim)++ if q_input is not None:+ input_ids = q_input.to(cache_device)++ names = wrapper_block(block, self.enable_minmax_tuning)+ logger.info(names)++ round_params = []+ minmax_params = []+ for n, m in block.named_modules():+ if hasattr(m, "orig_layer"):+ round_params.append(m.value)+ minmax_params.append(m.min_scale)+ minmax_params.append(m.max_scale)++ if self.enable_minmax_tuning:+ optimizer = self.optimizer(+ [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0+ )+ else:+ optimizer = self.optimizer(round_params, lr=self.lr, weight_decay=0)++ if self.lr_scheduler is None:+ lr_schedule = torch.optim.lr_scheduler.LinearLR(+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters, verbose=False+ )+ else:+ lr_schedule = copy.deepcopy(self.lr_scheduler)++ pick_samples = self.train_bs+ if len(input_ids.shape) == 3:+ n_samples = input_ids.shape[batch_dim]+ else:+ n_samples = input_ids.shape[0] // self.seqlen+ if self.sampler != "rand":+ indices = torch.randperm(n_samples)[:pick_samples]+ last_best_iter = 0+ best_loss = torch.finfo(torch.float).max+ mse_loss = torch.nn.MSELoss().to(device)+ scaler = self.get_scaler() # pylint: disable=assignment-from-none+ for i in range(self.iters):+ if self.sampler == "rand":+ indices = torch.randperm(n_samples)[:pick_samples]++ total_loss = 0+ for _ in range(self.gradient_accumulate_steps):+ current_input_ids, current_input_others = sampling_inputs(+ input_ids, input_others, indices, seqlen=self.seqlen+ )+ if len(input_ids.shape) == 3:+ if batch_dim == 0:+ current_output = output[indices, :, :]+ elif batch_dim == 1:+ current_output = output[:, indices, :]+ else:+ current_output = output[:, :, indices]+ else:+ current_output = output.view(n_samples, self.seqlen, -1)+ current_output = current_output[indices, :, :]+ current_output = current_output.reshape(-1, current_output.shape[-1])+ current_output = move_input_to_device(current_output, device)++ output_q = block_forward(+ block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device+ )+ if self.amp and not check_is_cpu(device):+ with autocast(device_type="cuda", dtype=self.amp_dtype):+ loss = mse_loss(output_q, current_output)+ else:+ loss = mse_loss(output_q.to(torch.float32), current_output.to(torch.float32))++ total_loss += loss.item() / self.gradient_accumulate_steps+ self.scale_loss_and_backward(scaler, loss)++ if total_loss < best_loss:+ best_loss = total_loss+ if not self.not_use_mse:+ # print(f"get better result at iter {i}, the loss is {total_loss}", flush=True)+ best_v = collect_round_v(block)+ best_min_scale, best_max_scale = collect_minmax_scale(block)+ last_best_iter = i+ if self.not_use_mse and i == self.iters - 1:+ best_v = collect_round_v(block)+ best_min_scale, best_max_scale = collect_minmax_scale(block)++ if not self.not_use_mse:+ if self.dynamic_max_gap > 0 and i - last_best_iter >= self.dynamic_max_gap:+ break+ self.step(scaler, optimizer, lr_schedule)+ unwrapper_block(block, best_v, best_min_scale, best_max_scale)+ if self.use_quant_input:+ q_outputs = self.get_block_outputs(+ block, input_ids, input_others, self.train_bs, device, cache_device, batch_dim+ )++ return q_outputs, output++ else:+ return None, output++ def q_dq_weight_round(+ self,+ model: torch.nn.Module,+ inputs,+ block_names,+ n_blocks=1,+ device=torch.device("cpu"),+ ):+ """Quantize and dequantize the weights of the specified blocks in the model.++ Args:+ model: The PyTorch model to be quantized.+ inputs: The input data for quantization.+ block_names: The names of the blocks to be quantized and dequantized.+ n_blocks: The number of blocks to quantize and dequantize.+ device: The device for quantization and dequantization.++ Returns:+ None+ """+ q_input = None+ torch.cuda.empty_cache()+ for n, m in model.named_parameters():+ m.requires_grad_(False)+ input_ids = inputs["input_ids"]+ inputs.pop("input_ids", None)+ input_others = inputs+ torch.cuda.empty_cache()+ for i in range(0, len(block_names), n_blocks):+ if n_blocks == 1:+ n = block_names[i]+ logger.info(n)+ m = get_module(model, n)+ else:+ names = block_names[i : i + n_blocks]+ logger.info(names)+ modules = [get_module(model, n) for n in names]+ m = WrapperMultiblock(modules)++ m = m.to(device)++ q_input, input_ids = self.quant_block(+ m,+ input_ids,+ input_others,+ q_input=q_input,+ device=device,+ )+ m.to("cpu")+ torch.cuda.empty_cache()++ del q_input+ del input_ids+ del input_others+ del inputs++ torch.cuda.empty_cache()++ def quantize(self):+ """Quantize the model and return the quantized model along with weight configurations.++ Returns:+ The quantized model and weight configurations.+ """+ start_time = time.time()+ # logger.info("cache block input")+ block_names = get_block_names(self.model)+ if len(block_names) == 0:+ logger.warning("could not find blocks, exit with original model")+ return++ if self.amp:+ self.model = self.model.to(self.amp_dtype)+ if not self.low_gpu_mem_usage:+ self.model = self.model.to(self.device)++ save_input_actor = SaveInputs(self.model, self.dataloader, self.seqlen, block_names[0])+ inputs = save_input_actor.get_inputs(n_samples=self.n_samples)+ del save_input_actor+ if "input_ids" in inputs.keys():+ total_samples = inputs["input_ids"].shape[0]+ if total_samples < self.train_bs:+ self.train_bs = total_samples+ logger.warning(f"force the train batch size to {total_samples} ")+ self.model = self.model.to("cpu")+ torch.cuda.empty_cache()+ self.q_dq_weight_round(+ self.model,+ inputs,+ block_names,+ n_blocks=self.n_blocks,+ device=self.device,+ )+ for n, m in self.model.named_modules():+ if n in self.weight_config.keys():+ if hasattr(m, "scale"):+ self.weight_config[n]["scale"] = m.scale+ self.weight_config[n]["zp"] = m.zp+ delattr(m, "scale")+ delattr(m, "zp")+ else:+ self.weight_config[n]["data_type"] = "float"+ if self.amp_dtype == torch.bfloat16:+ self.weight_config[n]["data_type"] = "bfloat"+ self.weight_config[n]["bits"] = 16+ self.weight_config[n]["group_size"] = None+ self.weight_config[n]["sym"] = None++ end_time = time.time()+ cost_time = end_time - start_time+ logger.info(f"quantization runtime {cost_time}")+ return self.model, self.weight_config+++class AutoOPTRound(AutoRound):+ """Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.++ Args:+ model: The PyTorch model to be quantized.+ tokenizer: An optional tokenizer for processing input data.+ bits (int): Number of bits for quantization (default is 4).+ group_size (int): Size of the quantization group (default is 128).+ scheme (str): The quantization scheme to be used (default is "asym").+ weight_config (dict): Configuration for weight quantization (default is an empty dictionary).+ enable_full_range (bool): Whether to enable full range quantization (default is False).+ bs (int): Batch size for training (default is 8).+ amp (bool): Whether to use automatic mixed precision (default is True).+ device: The device to be used for training (default is "cuda:0").+ lr_scheduler: The learning rate scheduler to be used.+ dataloader: The dataloader for input data (to be supported in future).+ dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").+ dataset_split (str): The split of the dataset to be used (default is "train").+ use_quant_input (bool): Whether to use quantized input data (default is True).+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).+ lr (float): The learning rate (default is 0.005).+ minmax_lr (float): The learning rate for min-max tuning (default is None).+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).+ iters (int): Number of iterations (default is 200).+ seqlen (int): Length of the sequence.+ n_samples (int): Number of samples (default is 512).+ sampler (str): The sampling method (default is "rand").+ seed (int): The random seed (default is 42).+ n_blocks (int): Number of blocks (default is 1).+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).+ not_use_mse (bool): Whether to use mean squared error (default is False).+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).+ data_type (str): The data type to be used (default is "int").+ optimizer: string or object+ **kwargs: Additional keyword arguments.++ Returns:+ The quantized model.+ """++ def __init__(+ self,+ model,+ tokenizer=None,+ bits: int = 4,+ group_size: int = 128,+ scheme: str = "asym",+ weight_config: dict = {},+ enable_full_range: bool = False,+ bs: int = 8,+ amp: bool = True,+ device="cuda:0",+ lr_scheduler=None,+ dataloader=None,+ dataset_name: str = "NeelNanda/pile-10k",+ dataset_split: str = "train",+ use_quant_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = 0.005,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = True,+ iters: int = 200,+ seqlen: int = 2048,+ n_samples: int = 512,+ sampler: str = "rand",+ seed: int = 42,+ n_blocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int",+ optimizer="AdamW",+ **kwargs,+ ):+ super(AutoOPTRound, self).__init__(+ model,+ tokenizer,+ bits,+ group_size,+ scheme,+ weight_config,+ enable_full_range,+ bs,+ amp,+ device,+ lr_scheduler,+ dataloader,+ dataset_name,+ dataset_split,+ use_quant_input,+ enable_minmax_tuning,+ lr,+ minmax_lr,+ low_gpu_mem_usage,+ iters,+ seqlen,+ n_samples,+ sampler,+ seed,+ n_blocks,+ gradient_accumulate_steps,+ not_use_mse,+ dynamic_max_gap,+ data_type,+ **kwargs,+ )++ self.optimizer = self.get_optimizer(optimizer)++ def get_optimizer(self, optimizer):+ if optimizer is None:+ optimizer = torch.optim.AdamW+ elif isinstance(optimizer, str):+ optimizer = getattr(torch.optim, optimizer)+ else:+ optimizer = optimizer+ return optimizer++ def get_scaler(self):+ scaler = None+ if self.amp and not check_is_cpu(self.device):+ from torch.cuda.amp import GradScaler++ scaler = GradScaler(init_scale=1024, growth_interval=100000)+ return scaler++ def scale_loss_and_backward(self, scaler, loss):+ if scaler is not None:+ scale_loss = scaler.scale(loss)+ scale_loss.backward()+ return scale_loss+ else:+ loss.backward()+ return loss++ def step(self, scaler, optimizer, lr_schedule):+ if scaler is not None:+ scaler.step(optimizer)+ optimizer.zero_grad()+ lr_schedule.step()+ scaler.update()+ else:+ optimizer.step()+ optimizer.zero_grad()+ lr_schedule.step()+++class AutoAdamRound(AutoOPTRound):+ """Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.+ The default lr has been changed.++ Args:+ model: The PyTorch model to be quantized.+ tokenizer: An optional tokenizer for processing input data.+ bits (int): Number of bits for quantization (default is 4).+ group_size (int): Size of the quantization group (default is 128).+ scheme (str): The quantization scheme to be used (default is "asym").+ weight_config (dict): Configuration for weight quantization (default is an empty dictionary).+ enable_full_range (bool): Whether to enable full range quantization (default is False).+ bs (int): Batch size for training (default is 8).+ amp (bool): Whether to use automatic mixed precision (default is True).+ device: The device to be used for training (default is "cuda:0").+ lr_scheduler: The learning rate scheduler to be used.+ dataloader: The dataloader for input data (to be supported in future).+ dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").+ dataset_split (str): The split of the dataset to be used (default is "train").+ use_quant_input (bool): Whether to use quantized input data (default is True).+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).+ lr (float): The learning rate (default is 0.005).+ minmax_lr (float): The learning rate for min-max tuning (default is None).+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).+ iters (int): Number of iterations (default is 200).+ seqlen (int): Length of the sequence.+ n_samples (int): Number of samples (default is 512).+ sampler (str): The sampling method (default is "rand").+ seed (int): The random seed (default is 42).+ n_blocks (int): Number of blocks (default is 1).+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).+ not_use_mse (bool): Whether to use mean squared error (default is False).+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).+ data_type (str): The data type to be used (default is "int").+ optimizer: string or object+ **kwargs: Additional keyword arguments.++ Returns:+ The quantized model.+ """++ def __init__(+ self,+ model,+ tokenizer=None,+ bits: int = 4,+ group_size: int = 128,+ scheme: str = "asym",+ weight_config: dict = {},+ enable_full_range: bool = False,+ bs: int = 8,+ amp: bool = True,+ device="cuda:0",+ lr_scheduler=None,+ dataloader=None,+ dataset_name: str = "NeelNanda/pile-10k",+ dataset_split: str = "train",+ use_quant_input: bool = True,+ enable_minmax_tuning: bool = True,+ lr: float = 0.01,+ minmax_lr: float = None,+ low_gpu_mem_usage: bool = True,+ iters: int = 200,+ seqlen: int = 2048,+ n_samples: int = 512,+ sampler: str = "rand",+ seed: int = 42,+ n_blocks: int = 1,+ gradient_accumulate_steps: int = 1,+ not_use_mse: bool = False,+ dynamic_max_gap: int = -1,+ data_type: str = "int",+ optimizer="AdamW",+ **kwargs,+ ):+ super(AutoAdamRound, self).__init__(+ model,+ tokenizer,+ bits,+ group_size,+ scheme,+ weight_config,+ enable_full_range,+ bs,+ amp,+ device,+ lr_scheduler,+ dataloader,+ dataset_name,+ dataset_split,+ use_quant_input,+ enable_minmax_tuning,+ lr,+ minmax_lr,+ low_gpu_mem_usage,+ iters,+ seqlen,+ n_samples,+ sampler,+ seed,+ n_blocks,+ gradient_accumulate_steps,+ not_use_mse,+ dynamic_max_gap,+ data_type,+ optimizer,+ **kwargs,+ )diff --git a/autoround/src/sign_sgd.py b/autoround/src/sign_sgd.pynew file mode 100644index 000000000..851378a7b--- /dev/null+++ b/autoround/src/sign_sgd.py@@ -0,0 +1,389 @@+#+# -*- coding: utf-8 -*-+#+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++from typing import List, Optional++# From PyTorch:+#+# Copyright (c) 2016- Facebook, Inc (Adam Paszke)+# Copyright (c) 2014- Facebook, Inc (Soumith Chintala)+# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)+# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu)+# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)+# Copyright (c) 2011-2013 NYU (Clement Farabet)+# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)+# Copyright (c) 2006 Idiap Research Institute (Samy Bengio)+# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)+#+# From Caffe2:+#+# Copyright (c) 2016-present, Facebook Inc. All rights reserved.+#+# All contributions by Facebook:+# Copyright (c) 2016 Facebook Inc.+#+# All contributions by Google:+# Copyright (c) 2015 Google Inc.+# All rights reserved.+#+# All contributions by Yangqing Jia:+# Copyright (c) 2015 Yangqing Jia+# All rights reserved.+#+# All contributions by Kakao Brain:+# Copyright 2019-2020 Kakao Brain+#+# All contributions by Cruise LLC:+# Copyright (c) 2022 Cruise LLC.+# All rights reserved.+#+# All contributions from Caffe:+# Copyright(c) 2013, 2014, 2015, the respective contributors+# All rights reserved.+#+# All other contributions:+# Copyright(c) 2015, 2016 the respective contributors+# All rights reserved.+#+# Caffe2 uses a copyright model similar to Caffe: each contributor holds+# copyright over their contributions to Caffe2. The project versioning records+# all such contribution and copyright details. If a contributor wants to further+# mark their specific copyright on a particular contribution, they should+# indicate their copyright solely in the commit message of the change when it is+# committed.+#+# All rights reserved.+#+# Redistribution and use in source and binary forms, with or without+# modification, are permitted provided that the following conditions are met:+#+# 1. Redistributions of source code must retain the above copyright+# notice, this list of conditions and the following disclaimer.+#+# 2. Redistributions in binary form must reproduce the above copyright+# notice, this list of conditions and the following disclaimer in the+# documentation and/or other materials provided with the distribution.+#+# 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America+# and IDIAP Research Institute nor the names of its contributors may be+# used to endorse or promote products derived from this software without+# specific prior written permission.+#+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE+# POSSIBILITY OF SUCH DAMAGE.+import torch+from torch import Tensor+from torch.optim.optimizer import Optimizer++__all__ = ["SGD", "sgd"]+++class _RequiredParameter(object):+ """Singleton class representing a required parameter for an Optimizer."""++ def __repr__(self):+ return "<required parameter>"+++required = _RequiredParameter()+++def _use_grad_for_differentiable(func):+ def _use_grad(self, *args, **kwargs):+ prev_grad = torch.is_grad_enabled()+ try:+ torch.set_grad_enabled(self.defaults["differentiable"])+ ret = func(self, *args, **kwargs)+ finally:+ torch.set_grad_enabled(prev_grad)+ return ret++ return _use_grad+++class SGD(Optimizer):+ r"""Implements stochastic gradient descent (optionally with momentum).++ .. math::+ \begin{aligned}+ &\rule{110mm}{0.4pt} \\+ &\textbf{input} : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta)+ \text{ (objective)}, \: \lambda \text{ (weight decay)}, \\+ &\hspace{13mm} \:\mu \text{ (momentum)}, \:\tau \text{ (dampening)},+ \:\textit{ nesterov,}\:\textit{ maximize} \\[-1.ex]+ &\rule{110mm}{0.4pt} \\+ &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\+ &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\+ &\hspace{5mm}\textbf{if} \: \lambda \neq 0 \\+ &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\+ &\hspace{5mm}\textbf{if} \: \mu \neq 0 \\+ &\hspace{10mm}\textbf{if} \: t > 1 \\+ &\hspace{15mm} \textbf{b}_t \leftarrow \mu \textbf{b}_{t-1} + (1-\tau) g_t \\+ &\hspace{10mm}\textbf{else} \\+ &\hspace{15mm} \textbf{b}_t \leftarrow g_t \\+ &\hspace{10mm}\textbf{if} \: \textit{nesterov} \\+ &\hspace{15mm} g_t \leftarrow g_{t} + \mu \textbf{b}_t \\+ &\hspace{10mm}\textbf{else} \\[-1.ex]+ &\hspace{15mm} g_t \leftarrow \textbf{b}_t \\+ &\hspace{5mm}\textbf{if} \: \textit{maximize} \\+ &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} + \gamma g_t \\[-1.ex]+ &\hspace{5mm}\textbf{else} \\[-1.ex]+ &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - \gamma g_t \\[-1.ex]+ &\rule{110mm}{0.4pt} \\[-1.ex]+ &\bf{return} \: \theta_t \\[-1.ex]+ &\rule{110mm}{0.4pt} \\[-1.ex]+ \end{aligned}++ Nesterov momentum is based on the formula from+ `On the importance of initialization and momentum in deep learning`__.++ Args:+ params (iterable): iterable of parameters to optimize or dicts defining+ parameter groups+ lr (float): learning rate+ momentum (float, optional): momentum factor (default: 0)+ weight_decay (float, optional): weight decay (L2 penalty) (default: 0)+ dampening (float, optional): dampening for momentum (default: 0)+ nesterov (bool, optional): enables Nesterov momentum (default: False)+ maximize (bool, optional): maximize the params based on the objective, instead of+ minimizing (default: False)+ foreach (bool, optional): whether foreach implementation of optimizer+ is used (default: None)++ Example:+ >>> # xdoctest: +SKIP+ >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)+ >>> optimizer.zero_grad()+ >>> loss_fn(model(input), target).backward()+ >>> optimizer.step()++ __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf++ .. note::+ The implementation of SGD with Momentum/Nesterov subtly differs from+ Sutskever et. al. and implementations in some other frameworks.++ Considering the specific case of Momentum, the update can be written as++ .. math::+ \begin{aligned}+ v_{t+1} & = \mu * v_{t} + g_{t+1}, \\+ p_{t+1} & = p_{t} - \text{lr} * v_{t+1},+ \end{aligned}++ where :math:`p`, :math:`g`, :math:`v` and :math:`\mu` denote the+ parameters, gradient, velocity, and momentum respectively.++ This is in contrast to Sutskever et. al. and+ other frameworks which employ an update of the form++ .. math::+ \begin{aligned}+ v_{t+1} & = \mu * v_{t} + \text{lr} * g_{t+1}, \\+ p_{t+1} & = p_{t} - v_{t+1}.+ \end{aligned}++ The Nesterov version is analogously modified.+ """++ def __init__(+ self,+ params,+ lr=required,+ momentum=0,+ dampening=0,+ weight_decay=0,+ nesterov=False,+ *,+ maximize=False,+ foreach: Optional[bool] = None,+ differentiable=False+ ):+ if lr is not required and lr < 0.0:+ raise ValueError("Invalid learning rate: {}".format(lr))+ if momentum < 0.0:+ raise ValueError("Invalid momentum value: {}".format(momentum))+ if weight_decay < 0.0:+ raise ValueError("Invalid weight_decay value: {}".format(weight_decay))++ defaults = dict(+ lr=lr,+ momentum=momentum,+ dampening=dampening,+ weight_decay=weight_decay,+ nesterov=nesterov,+ maximize=maximize,+ foreach=foreach,+ differentiable=differentiable,+ )+ if nesterov and (momentum <= 0 or dampening != 0):+ raise ValueError("Nesterov momentum requires a momentum and zero dampening")+ super(SGD, self).__init__(params, defaults)++ def __setstate__(self, state):+ super().__setstate__(state)+ for group in self.param_groups:+ group.setdefault("nesterov", False)+ group.setdefault("maximize", False)+ group.setdefault("foreach", None)+ group.setdefault("differentiable", False)++ @_use_grad_for_differentiable+ def step(self, closure=None):+ """Performs a single optimization step.++ Args:+ closure (Callable, optional): A closure that reevaluates the model+ and returns the loss.+ """+ loss = None+ if closure is not None:+ with torch.enable_grad():+ loss = closure()++ for group in self.param_groups:+ params_with_grad = []+ d_p_list = []+ momentum_buffer_list = []+ has_sparse_grad = False++ for p in group["params"]:+ if p.grad is not None:+ params_with_grad.append(p)+ d_p_list.append(p.grad)+ if p.grad.is_sparse:+ has_sparse_grad = True++ state = self.state[p]+ if "momentum_buffer" not in state:+ momentum_buffer_list.append(None)+ else:+ momentum_buffer_list.append(state["momentum_buffer"])++ sgd(+ params_with_grad,+ d_p_list,+ momentum_buffer_list,+ weight_decay=group["weight_decay"],+ momentum=group["momentum"],+ lr=group["lr"],+ dampening=group["dampening"],+ nesterov=group["nesterov"],+ maximize=group["maximize"],+ has_sparse_grad=has_sparse_grad,+ foreach=group["foreach"],+ )++ # update momentum_buffers in state+ for p, momentum_buffer in zip(params_with_grad, momentum_buffer_list):+ state = self.state[p]+ state["momentum_buffer"] = momentum_buffer++ return loss+++def sgd(+ params: List[Tensor],+ d_p_list: List[Tensor],+ momentum_buffer_list: List[Optional[Tensor]],+ # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627+ # setting this as kwarg for now as functional API is compiled by torch/distributed/optim+ has_sparse_grad: bool = None,+ foreach: bool = None,+ *,+ weight_decay: float,+ momentum: float,+ lr: float,+ dampening: float,+ nesterov: bool,+ maximize: bool+):+ r"""Functional API that performs SGD algorithm computation.++ See :class:`~torch.optim.SGD` for details.+ """++ if foreach is None:+ # Placeholder for more complex foreach logic to be added when value is not set+ foreach = False++ if foreach and torch.jit.is_scripting():+ raise RuntimeError("torch.jit.script not supported with foreach optimizers")++ # if foreach and not torch.jit.is_scripting():+ # func = _multi_tensor_sgd+ # else:+ func = _single_tensor_sgd++ func(+ params,+ d_p_list,+ momentum_buffer_list,+ weight_decay=weight_decay,+ momentum=momentum,+ lr=lr,+ dampening=dampening,+ nesterov=nesterov,+ has_sparse_grad=has_sparse_grad,+ maximize=maximize,+ )+++def _single_tensor_sgd(+ params: List[Tensor],+ d_p_list: List[Tensor],+ momentum_buffer_list: List[Optional[Tensor]],+ *,+ weight_decay: float,+ momentum: float,+ lr: float,+ dampening: float,+ nesterov: bool,+ maximize: bool,+ has_sparse_grad: bool+):+ for i, param in enumerate(params):+ d_p = d_p_list[i] if not maximize else -d_p_list[i]++ if weight_decay != 0:+ d_p = d_p.add(param, alpha=weight_decay)++ if momentum != 0:+ buf = momentum_buffer_list[i]++ if buf is None:+ buf = torch.clone(d_p).detach()+ momentum_buffer_list[i] = buf+ else:+ buf.mul_(momentum).add_(d_p, alpha=1 - dampening)++ if nesterov:+ d_p = d_p.add(buf, alpha=momentum)+ else:+ d_p = buf++ param.add_(torch.sign(d_p), alpha=-lr)