Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add more files

wenhuach21b18cbcbb2e7c
diff --git a/autoround/eval/__init__.py b/autoround/eval/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/autoround/eval/eval.py b/autoround/eval/eval.py
new file mode 100644
index 000000000..695f492f7
--- /dev/null
+++ b/autoround/eval/eval.py
@@ -0,0 +1,369 @@
+import os
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+import os.path
+import torch
+import torch.nn as nn
+from parse_results import result_parser
+import pprint
+import json
+import re
+import shutil
+import transformers
+import time
+
+EXT_TASKS = ['wikitext2', 'ptb', 'c4', 'ptb-new', 'c4-new']
+fewshots_dict = {}
+fewshots_dict['paper'] = {
+ "lambada_openai": [0],
+ "hellaswag": [0],
+ "winogrande": [0],
+ "piqa": [0],
+ "hendrycksTest-*": [0],
+ "wikitext": [0],
+ "truthfulqa_mc": [0],
+ "openbookqa": [0],
+ "boolq": [0],
+ "rte": [0],
+ "arc_easy": [0],
+ "arc_challenge": [0],
+}
+fewshots_dict['leadboard'] = {
+ "hellaswag": [10],
+ "winogrande": [5],
+ "arc_easy": [25],
+ "arc_challenge": [25],
+ "hendrycksTest-*": [5],
+ "drop": [3],
+ "gsm8k": [5],
+}
+fewshots_dict['all'] = {
+ "lambada_openai": [0],
+ "hellaswag": [0, 10],
+ "winogrande": [0, 5],
+ "piqa": [0],
+ "coqa": [], ## coqa is not enabled in llamav1 models
+ "truthfulqa_mc": [0],
+ "openbookqa": [0],
+ "boolq": [0],
+ "rte": [0],
+ "arc_easy": [0, 25],
+ "arc_challenge": [0, 25],
+ "hendrycksTest-*": [0, 5],
+ "wikitext": [0],
+ "drop": [3],
+ "gsm8k": [5]
+}
+
+
+def simple_evaluate(
+ model,
+ model_args=None,
+ tasks=[],
+ num_fewshot=0,
+ batch_size=None,
+ max_batch_size=None,
+ device=None,
+ no_cache=True, ##changed by wenhua
+ limit=None,
+ bootstrap_iters=100000,
+ description_dict=None,
+ check_integrity=False,
+ decontamination_ngrams_path=None,
+ write_out=False,
+ output_base_path=None,
+ lm=None ##changed by wenhua
+):
+ """Instantiate and evaluate a model on a list of tasks.
+
+ :param model: Union[str, LM]
+ Name of model, transformers.PreTrainedModel object, or LM object, see lm_eval.models.get_model
+ :param model_args: Optional[str]
+ String arguments for each model class, see LM.create_from_arg_string.
+ Ignored if `model` argument is a LM object.
+ :param tasks: list[Union[str, Task]]
+ List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
+ :param num_fewshot: int
+ Number of examples in few-shot context
+ :param batch_size: int or str, optional
+ Batch size for model
+ :param max_batch_size: int, optional
+ Maximal batch size to try with automatic batch size detection
+ :param device: str, optional
+ PyTorch device (e.g. "cpu" or "cuda:0") for running models
+ :param no_cache: bool
+ Whether or not to cache
+ :param limit: int or float, optional
+ Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
+ :param bootstrap_iters:
+ Number of iterations for bootstrap statistics
+ :param description_dict: dict[str, str]
+ Dictionary of custom task descriptions of the form: `task_name: description`
+ :param check_integrity: bool
+ Whether to run the relevant part of the test suite for the tasks
+ :param write_out: bool
+ If True, write details about prompts and logits to json for all tasks
+ :param output_base_path: str, optional
+ Directory to which detailed eval info will be written. Defaults to present working dir.
+ :return
+ Dictionary of results
+ """
+ import collections
+ import itertools
+ import random
+
+ import lm_eval.metrics
+ import lm_eval.models
+ import lm_eval.tasks
+ import lm_eval.base
+ from lm_eval.utils import positional_deprecated, run_task_tests
+ from lm_eval.models.gpt2 import HFLM
+
+ import numpy as np
+ import transformers
+ from lm_eval.evaluator import evaluate
+ random.seed(1234)
+ np.random.seed(1234)
+
+ assert tasks != [], "No tasks specified"
+ if lm == None:
+ if isinstance(model, str):
+ if model_args is None:
+ model_args = ""
+ lm = lm_eval.models.get_model(model).create_from_arg_string(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
+ )
+ elif isinstance(model, transformers.PreTrainedModel):
+ lm = lm_eval.models.get_model("hf-causal")(
+ pretrained=model,
+ batch_size=batch_size,
+ max_batch_size=max_batch_size,
+ )
+ no_cache = True
+ else:
+ assert isinstance(model, lm_eval.base.LM)
+ lm = model
+
+ if not no_cache:
+ lm = lm_eval.base.CachingLM(
+ lm,
+ "lm_cache/"
+ + (model if isinstance(model, str) else model.model.config._name_or_path)
+ + "_"
+ + model_args.replace("=", "-").replace(",", "_").replace("/", "-")
+ + ".db",
+ )
+
+ task_dict = lm_eval.tasks.get_task_dict(tasks)
+
+ if check_integrity:
+ run_task_tests(task_list=tasks)
+
+ results = evaluate(
+ lm=lm,
+ task_dict=task_dict,
+ num_fewshot=num_fewshot,
+ limit=limit,
+ bootstrap_iters=bootstrap_iters,
+ description_dict=description_dict,
+ decontamination_ngrams_path=decontamination_ngrams_path,
+ write_out=write_out,
+ output_base_path=output_base_path,
+ )
+
+ # add info about the model and few shot config
+ model_name = None
+ if isinstance(model, str):
+ model_name = model
+ elif isinstance(model, transformers.PreTrainedModel):
+ model_name = "pretrained=" + model.config._name_or_path
+ results["config"] = {
+ "model": model_name,
+ "model_args": model_args,
+ "num_fewshot": num_fewshot,
+ "batch_size": batch_size,
+ "batch_sizes": list(lm.batch_sizes.values())
+ if hasattr(lm, "batch_sizes")
+ else [],
+ "device": device,
+ "no_cache": no_cache,
+ "limit": limit,
+ "bootstrap_iters": bootstrap_iters,
+ "description_dict": description_dict,
+ }
+
+ return results, lm
+
+
+def eval_model(output_dir=None, model=None, tokenizer=None,
+ tasks=["lambada_openai", "hellaswag", "winogrande", "piqa"],
+ eval_bs=32, use_accelerate=True, dtype="float16", limit=None,
+ device="cuda:0", seed=0, nsamples=128, eval_orig_float=False, mark="paper", excel_file="tmp.xlsx"):
+ print("evaluation with official lm-eval", flush=True)
+ try:
+ import lm_eval
+ from lm_eval import evaluator
+ from lm_eval.tasks import ALL_TASKS, get_task_dict
+ except:
+ raise ImportError("""follow requirements to install dependencies.""")
+
+ org_s = time.time()
+ ##save model
+ if output_dir is None:
+ output_dir = "./tmp_signround"
+
+ if os.path.exists(output_dir) and not eval_orig_float:
+ shutil.rmtree(output_dir)
+ if not eval_orig_float:
+ model = model.to(torch.float16)
+ model = model.to("cpu")
+ model.save_pretrained(output_dir)
+ tokenizer.save_pretrained(output_dir)
+
+ if (hasattr(model, 'config') and model.config.torch_dtype is torch.bfloat16):
+ dtype = 'bfloat16'
+
+ external_tasks = []
+ for each in EXT_TASKS:
+ if each in tasks:
+ external_tasks.append(each)
+ tasks.remove(each)
+
+ results = {}
+ model = None
+ lm = None
+ for tmp_tasks in tasks:
+ try:
+ num_fewshot = fewshots_dict[mark][tmp_tasks]
+ task_names = lm_eval.utils.pattern_match([tmp_tasks], ALL_TASKS)
+ print(f'********* {tmp_tasks} evaluate ************')
+ task_s = time.time()
+ for shot in num_fewshot:
+ if bool(re.search("chatglm", output_dir.lower())):
+ model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},trust_remote_code=True'
+ model_type = "hf-causal"
+ else:
+ model_args = f'pretrained={output_dir},tokenizer={output_dir},dtype={dtype},use_accelerate={use_accelerate},trust_remote_code=True'
+ model_type = "hf-causal-experimental"
+
+ if "wikitext" in task_names:
+ tmp_eval_bs = 1
+ else:
+ tmp_eval_bs = eval_bs
+ tmp_results, lm = simple_evaluate(model=model_type, model_args=model_args, tasks=task_names,
+ num_fewshot=shot, limit=limit, batch_size=tmp_eval_bs,
+ max_batch_size=tmp_eval_bs, lm=lm)
+ sub_name = f'{tmp_tasks} {shot}-shot'
+ print(f'{sub_name}: ')
+ pprint.pprint(tmp_results["results"])
+ print(f"\n{sub_name} cost time: {time.time() - task_s}\n")
+ results[sub_name] = tmp_results
+ except Exception as e:
+ print(f'********* {tmp_tasks} ERROR ************')
+ print(str(e))
+ continue
+
+ tokenizer = transformers.AutoTokenizer.from_pretrained(output_dir, use_fast=False, trust_remote_code=True)
+ model = lm.model
+ # for external tasks
+ # maybe adjust for specific model
+ # if hasattr(lm.model.config, "max_position_embeddings"):
+ # lm.model.seqlen = lm.model.config.max_position_embeddings
+ # else:
+ # ## for llama-1, opt
+ # lm.model.seqlen = 2048
+
+ # if "opt" in model_name:
+ # seqlen = model.config.max_position_embeddings
+ # model.seqlen = model.config.max_position_embeddings
+ # else:
+ # seqlen = 2048
+ # model.seqlen = seqlen
+
+ model.seqlen = 2048
+ from utils import get_loaders, eval_ppl_same_with_gptq
+ for dataset in external_tasks:
+ try:
+ dataloader, testloader = get_loaders(
+ dataset, nsamples=nsamples, seed=seed,
+ tokenizer=tokenizer, seqlen=model.seqlen
+ )
+ ppl = eval_ppl_same_with_gptq(model, testloader, device)
+ print(dataset, ppl)
+
+ results.update({dataset: ppl})
+ except Exception as e:
+ print(str(e))
+ continue
+
+ print(results, flush=True)
+ new_results = result_parser(results)
+ print("cost time: ", time.time() - org_s)
+ import pickle
+ from collections import OrderedDict
+ new_dict = OrderedDict()
+ new_dict["model"] = "tmp"
+ new_dict["paper-avg"] = 0
+ new_dict["leaderboard-avg"] = 0
+ new_dict["wikitext2"] = 0
+ new_dict["ptb-new"] = 0
+ new_dict["c4-new"] = 0
+ new_dict["wikitext 0-shot_word_perplexity"] = 0
+ new_dict["wikitext 0-shot_byte_perplexity"] = 0
+ new_dict["wikitext 0-shot_bits_per_byte"] = 0
+ for key in new_results.keys():
+ if key == "model" or key == "paper-avg" or key == "leaderboard-avg":
+ continue
+ data = new_results[key]
+ if not isinstance(data, dict):
+ new_dict[key] = new_results[key]
+ continue
+ for sub_key in data.keys():
+ for sub_sub_key in data[sub_key].keys():
+ if "hendry" in key:
+ new_key = key + "-" + sub_key + "-" + sub_sub_key
+ else:
+ new_key = key + "_" + sub_sub_key
+ if "std" in new_key:
+ continue
+ # if "norm" in new_key:
+ # continue
+ new_dict[new_key] = data[sub_key][sub_sub_key]
+
+ import pandas as pd
+ df = pd.DataFrame(data=new_dict, index=[0])
+ df.to_excel(excel_file)
+
+
+if __name__ == "__main__":
+ import time
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--model_name", default="/models/opt-125m"
+ )
+ parser.add_argument(
+ "--bs", default=1,
+ )
+
+ args = parser.parse_args()
+ s = time.time()
+
+ test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
+ "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte", "arc_easy",
+ "arc_challenge"]
+ test_tasks = ['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
+ ]
+ excel_name = (args.model_name).split('/')[-1] + ".xlsx"
+ eval_model(output_dir=args.model_name,
+ tasks=test_tasks,
+ eval_bs=args.bs, eval_orig_float=True, limit=None, excel_file=excel_name)
+
+ print("cost time: ", time.time() - s)
+
diff --git a/autoround/eval/parse_results.py b/autoround/eval/parse_results.py
new file mode 100644
index 000000000..a1f237392
--- /dev/null
+++ b/autoround/eval/parse_results.py
@@ -0,0 +1,140 @@
+
+import sys
+import json
+import pprint
+
+# input_file = sys.argv[1]
+
+# with open(input_file) as f:
+# data = json.load(f)
+
+# data = data["results"]
+
+task_domain = {
+ "abstract_algebra": ["STEM"],
+ "anatomy": ["Other"],
+ "astronomy": ["STEM"],
+ "business_ethics": ["Other"],
+ "clinical_knowledge": ["Other"],
+ "college_biology": ["STEM"],
+ "college_chemistry": ["STEM"],
+ "college_computer_science": ["STEM"],
+ "college_mathematics": ["STEM"],
+ "college_medicine": ["Other"],
+ "college_physics": ["STEM"],
+ "computer_security": ["STEM"],
+ "conceptual_physics": ["STEM"],
+ "econometrics": ["Social"],
+ "electrical_engineering": ["STEM"],
+ "elementary_mathematics": ["STEM"],
+ "formal_logic": ["Hums"],
+ "global_facts": ["Other"],
+ "high_school_biology": ["STEM"],
+ "high_school_chemistry": ["STEM"],
+ "high_school_computer_science": ["STEM"],
+ "high_school_european_history": ["Hums"],
+ "high_school_geography": ["Social"],
+ "high_school_government_and_politics": ["Social"],
+ "high_school_macroeconomics": ["Social"],
+ "high_school_mathematics": ["STEM"],
+ "high_school_microeconomics": ["Social"],
+ "high_school_physics": ["STEM"],
+ "high_school_psychology": ["Social"],
+ "high_school_statistics": ["STEM"],
+ "high_school_us_history": ["Hums"],
+ "high_school_world_history": ["Hums"],
+ "human_aging": ["Other"],
+ "human_sexuality": ["Social"],
+ "international_law": ["Hums"],
+ "jurisprudence": ["Hums"],
+ "logical_fallacies": ["Hums"],
+ "machine_learning": ["STEM"],
+ "management": ["Other"],
+ "marketing": ["Other"],
+ "medical_genetics": ["Other"],
+ "miscellaneous": ["Other"],
+ "moral_disputes": ["Hums"],
+ "moral_scenarios": ["Hums"],
+ "nutrition": ["Other"],
+ "philosophy": ["Hums"],
+ "prehistory": ["Hums"],
+ "professional_accounting": ["Other"],
+ "professional_law": ["Hums"],
+ "professional_medicine": ["Other"],
+ "professional_psychology": ["Social"],
+ "public_relations": ["Social"],
+ "security_studies": ["Social"],
+ "sociology": ["Social"],
+ "us_foreign_policy": ["Social"],
+ "virology": ["Other"],
+ "world_religions": ["Hums"],
+}
+
+
+def result_parser(data):
+ mmlu_results = {}
+ other_results = {}
+ for task in data.keys():
+ acc_dict = {"Hums": [0, 0., 0.], "STEM": [0, 0., 0.], "Social": [0, 0., 0.], "Other": [0, 0., 0.],
+ "Avg.": [0, 0., 0.]}
+ if isinstance(data[task], float):
+ other_results[task] = data[task]
+ else:
+ results = data[task]['results']
+ if "hendrycksTest" in task:
+ for sub_task in results.keys():
+ sub_name = sub_task.split("hendrycksTest-")[1]
+ domain = task_domain[sub_name][0]
+ domain_list = acc_dict[domain]
+ domain_list[0] += 1
+ domain_list[1] += results[sub_task]["acc"]
+ domain_list[2] += results[sub_task]["acc_norm"]
+ acc_dict['Avg.'][0] += 1
+ acc_dict['Avg.'][1] += results[sub_task]["acc"]
+ acc_dict['Avg.'][2] += results[sub_task]["acc_norm"]
+
+ mmlu_results[task] = {}
+ for key in acc_dict.keys():
+ mmlu_results[task][key] = {}
+ mmlu_results[task][key]['acc'] = acc_dict[key][1] / acc_dict[key][0]
+ mmlu_results[task][key]['acc_norm'] = acc_dict[key][2] / acc_dict[key][0]
+ else:
+ other_results[task] = results
+
+ # other_results = {}
+ # for key in results.keys():
+ # if 'hendrycksTest' in key:
+ # sub_name = key.split("hendrycksTest-")[1]
+ # domain=task_domain[sub_name][0]
+ # domain_list = acc_dict[domain]
+ # domain_list[0] += 1
+ # domain_list[1] += results[key]["acc"]
+ # domain_list[2] += results[key]["acc_norm"]
+ # acc_dict['Avg.'][0] += 1
+ # acc_dict['Avg.'][1] += results[key]["acc"]
+ # acc_dict['Avg.'][2] += results[key]["acc_norm"]
+ # else:
+ # other_results[key] = results[key]
+
+ # mmlu_results = {}
+ # for key in acc_dict.keys():
+ # mmlu_results[key] = {}
+ # mmlu_results[key]['acc'] = acc_dict[key][1]/acc_dict[key][0]
+ # mmlu_results[key]['acc_norm'] = acc_dict[key][2]/acc_dict[key][0]
+
+ print("***** MMLU RESULTS *****")
+ pprint.pprint(mmlu_results)
+ print()
+
+ print("***** OTHER RESULTS *****")
+ pprint.pprint(other_results)
+ print()
+ new_results = mmlu_results
+ new_results.update(other_results)
+ return new_results
+
+
+
+
+
+
diff --git a/autoround/eval/utils.py b/autoround/eval/utils.py
new file mode 100644
index 000000000..269b8e676
--- /dev/null
+++ b/autoround/eval/utils.py
@@ -0,0 +1,195 @@
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+def set_seed(seed):
+ np.random.seed(seed)
+ torch.random.manual_seed(seed)
+
+
+def get_wikitext2(nsamples, seed, seqlen, tokenizer):
+ from datasets import load_dataset
+ traindata = load_dataset('wikitext', 'wikitext-2-raw-v1', split='train')
+ testdata = load_dataset('wikitext', 'wikitext-2-raw-v1', split='test')
+
+ trainenc = tokenizer("\n\n".join(traindata['text']), return_tensors='pt')
+ testenc = tokenizer("\n\n".join(testdata['text']), return_tensors='pt')
+
+ import random
+ random.seed(seed)
+ trainloader = []
+ for _ in range(nsamples):
+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)
+ j = i + seqlen
+ inp = trainenc.input_ids[:, i:j]
+ tar = inp.clone()
+ tar[:, :-1] = -100
+ trainloader.append((inp, tar))
+ return trainloader, testenc
+
+def get_ptb(nsamples, seed, seqlen, tokenizer):
+ from datasets import load_dataset
+ traindata = load_dataset('ptb_text_only', 'penn_treebank', split='train')
+ valdata = load_dataset('ptb_text_only', 'penn_treebank', split='validation')
+
+ trainenc = tokenizer("\n\n".join(traindata['sentence']), return_tensors='pt')
+ testenc = tokenizer("\n\n".join(valdata['sentence']), return_tensors='pt')
+
+ import random
+ random.seed(seed)
+ trainloader = []
+ for _ in range(nsamples):
+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)
+ j = i + seqlen
+ inp = trainenc.input_ids[:, i:j]
+ tar = inp.clone()
+ tar[:, :-1] = -100
+ trainloader.append((inp, tar))
+ return trainloader, testenc
+
+def get_c4(nsamples, seed, seqlen, tokenizer):
+ from datasets import load_dataset
+ traindata = load_dataset(
+ 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'
+ )
+ valdata = load_dataset(
+ 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'
+ )
+
+
+ import random
+ random.seed(seed)
+ trainloader = []
+ for _ in range(nsamples):
+ while True:
+ i = random.randint(0, len(traindata) - 1)
+ trainenc = tokenizer(traindata[i]['text'], return_tensors='pt')
+ if trainenc.input_ids.shape[1] >= seqlen:
+ break
+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)
+ j = i + seqlen
+ inp = trainenc.input_ids[:, i:j]
+ tar = inp.clone()
+ tar[:, :-1] = -100
+ trainloader.append((inp, tar))
+
+ import random
+ random.seed(0)
+ valenc = []
+ for _ in range(256):
+ while True:
+ i = random.randint(0, len(valdata) - 1)
+ tmp = tokenizer(valdata[i]['text'], return_tensors='pt')
+ if tmp.input_ids.shape[1] >= seqlen:
+ break
+ i = random.randint(0, tmp.input_ids.shape[1] - seqlen - 1)
+ j = i + seqlen
+ valenc.append(tmp.input_ids[:, i:j])
+ valenc = torch.hstack(valenc)
+ class TokenizerWrapper:
+ def __init__(self, input_ids):
+ self.input_ids = input_ids
+ valenc = TokenizerWrapper(valenc)
+
+ return trainloader, valenc
+
+def get_ptb_new(nsamples, seed, seqlen, tokenizer):
+ from datasets import load_dataset
+ traindata = load_dataset('ptb_text_only', 'penn_treebank', split='train')
+ testdata = load_dataset('ptb_text_only', 'penn_treebank', split='test')
+
+ trainenc = tokenizer(" ".join(traindata['sentence']), return_tensors='pt')
+ testenc = tokenizer(" ".join(testdata['sentence']), return_tensors='pt')
+
+ import random
+ random.seed(seed)
+ trainloader = []
+ for _ in range(nsamples):
+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)
+ j = i + seqlen
+ inp = trainenc.input_ids[:, i:j]
+ tar = inp.clone()
+ tar[:, :-1] = -100
+ trainloader.append((inp, tar))
+ return trainloader, testenc
+
+def get_c4_new(nsamples, seed, seqlen, tokenizer):
+ from datasets import load_dataset
+ traindata = load_dataset(
+ 'allenai/c4', 'allenai--c4', data_files={'train': 'en/c4-train.00000-of-01024.json.gz'}, split='train'
+ )
+ valdata = load_dataset(
+ 'allenai/c4', 'allenai--c4', data_files={'validation': 'en/c4-validation.00000-of-00008.json.gz'}, split='validation'
+ )
+
+ import random
+ random.seed(seed)
+ trainloader = []
+ for _ in range(nsamples):
+ while True:
+ i = random.randint(0, len(traindata) - 1)
+ trainenc = tokenizer(traindata[i]['text'], return_tensors='pt')
+ if trainenc.input_ids.shape[1] >= seqlen:
+ break
+ i = random.randint(0, trainenc.input_ids.shape[1] - seqlen - 1)
+ j = i + seqlen
+ inp = trainenc.input_ids[:, i:j]
+ tar = inp.clone()
+ tar[:, :-1] = -100
+ trainloader.append((inp, tar))
+
+ valenc = tokenizer(' '.join(valdata[:1100]['text']), return_tensors='pt')
+ valenc = valenc.input_ids[:, :(256 * seqlen)]
+
+ class TokenizerWrapper:
+ def __init__(self, input_ids):
+ self.input_ids = input_ids
+ valenc = TokenizerWrapper(valenc)
+
+ return trainloader, valenc
+
+
+def get_loaders(
+ name, nsamples=128, seed=0, seqlen=2048, tokenizer=''
+):
+ if 'wikitext2' in name:
+ return get_wikitext2(nsamples, seed, seqlen, tokenizer)
+ if 'ptb' in name:
+ if 'new' in name:
+ return get_ptb_new(nsamples, seed, seqlen, tokenizer)
+ return get_ptb(nsamples, seed, seqlen, tokenizer)
+ if 'c4' in name:
+ if 'new' in name:
+ return get_c4_new(nsamples, seed, seqlen, tokenizer)
+ return get_c4(nsamples, seed, seqlen, tokenizer)
+
+
+def eval_ppl_same_with_gptq(model, testenc, dev):
+ print('Evaluating ...', flush=True)
+
+ testenc = testenc.input_ids
+ nsamples = testenc.numel() // model.seqlen
+
+ use_cache = model.config.use_cache
+ model.config.use_cache = False
+
+ testenc = testenc.to(dev)
+ nlls = []
+ for i in range(nsamples):
+ batch = testenc[:, (i * model.seqlen):((i + 1) * model.seqlen)].to(dev)
+ lm_logits = model(batch).logits
+ # to main gpu for multi-gpus
+ shift_logits = lm_logits[:, :-1, :].contiguous().to(dev)
+ shift_labels = testenc[
+ :, (i * model.seqlen):((i + 1) * model.seqlen)
+ ][:, 1:].to(dev)
+ loss_fct = nn.CrossEntropyLoss()
+ loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+ neg_log_likelihood = loss.float() * model.seqlen
+ nlls.append(neg_log_likelihood)
+ ppl = torch.exp(torch.stack(nlls).sum() / (nsamples * model.seqlen))
+
+ model.config.use_cache = use_cache
+ return ppl.item()
diff --git a/autoround/main.py b/autoround/main.py
new file mode 100644
index 000000000..4307e141e
--- /dev/null
+++ b/autoround/main.py
@@ -0,0 +1,205 @@
+import argparse
+import sys
+sys.path.insert(0, './')
+from src.autoround import AutoRound, AutoAdamRound
+
+parser = argparse.ArgumentParser()
+import torch
+import os
+
+os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+torch.use_deterministic_algorithms(True, warn_only=True)
+from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel
+
+from transformers import set_seed
+
+from eval.eval import eval_model
+
+import re
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+
+if __name__ == '__main__':
+
+ parser.add_argument(
+ "--model_name", default="/models/opt-125m"
+ )
+
+ parser.add_argument("--num_bits", default=4, type=int,
+ help="number of bits")
+
+ parser.add_argument("--group_size", default=128, type=int,
+ help="group size")
+
+ parser.add_argument("--train_bs", default=8, type=int,
+ help="train batch size")
+
+ parser.add_argument("--eval_bs", default=4, type=int,
+ help="eval batch size")
+
+ parser.add_argument("--device", default=0, type=str,
+ help="device gpu int number, or 'cpu' ")
+ #
+ parser.add_argument("--sym", action='store_true',
+ help=" sym quantization")
+
+ parser.add_argument("--iters", default=400, type=int,
+ help=" iters")
+
+ parser.add_argument("--use_quant_input", action='store_true',
+ help="whether to use the output of quantized block to tune the next block")
+
+ parser.add_argument("--lr", default=0.05, type=float,
+ help="step size")
+
+ parser.add_argument("--minmax_lr", default=None, type=float,
+ help="minmax learning rate, if None,it will set to be the same with lr")
+
+ parser.add_argument("--seed", default=42, type=int,
+ help="seed")
+
+ parser.add_argument("--eval_fp16_baseline", action='store_true',
+ help="whether to eval FP16 baseline")
+
+ parser.add_argument("--amp", action='store_true',
+ help="amp")
+
+ parser.add_argument("--adam", action='store_true',
+ help="adam")
+
+ parser.add_argument("--seqlen", default=2048, type=int,
+ help="sequence length")
+
+ parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
+
+ parser.add_argument("--n_blocks", default=1, type=int, help="num of blocks to tune together")
+
+ parser.add_argument("--n_samples", default=512, type=int,
+ help="number of samples")
+
+ parser.add_argument("--low_gpu_mem_usage", action='store_true',
+ help="low_gpu_mem_usage")
+
+ parser.add_argument("--enable_minmax_tuning", action='store_true',
+ help="whether enable weight minmax tuning")
+
+ # parser.add_argument("--tasks", default=["lambada_openai", "hellaswag", "winogrande", "piqa"],
+ # help="lm-eval tasks")
+
+ # parser.add_argument("--tasks", default=["lambada_openai"],
+ # help="lm-eval tasks")
+
+ # parser.add_argument("--tasks",
+ # default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
+ # 'coqa', 'truthfulqa_mc', 'openbookqa', 'boolq', 'rte', 'arc_easy', 'arc_challenge',
+ # 'hendrycksTest-*', 'wikitext', 'drop', 'gsm8k'],##all
+ parser.add_argument("--tasks",
+ default=['wikitext2', 'ptb-new', 'c4-new', 'lambada_openai', 'hellaswag', 'winogrande', 'piqa',
+ "hendrycksTest-*", "wikitext", "truthfulqa_mc", "openbookqa", "boolq", "rte",
+ "arc_easy", "arc_challenge"],
+ help="lm-eval tasks") # "truthfulqa_gen"
+
+ # parser.add_argument("--tasks", default=["lambada_openai"],
+ # help="lm-eval tasks")
+
+ parser.add_argument("--output_dir", default="./tmp_signround", type=str,
+ help="Where to store the final model.")
+
+ args = parser.parse_args()
+ set_seed(args.seed)
+
+ model_name = args.model_name
+ if model_name[-1] == "/":
+ model_name = model_name[:-1]
+ print(model_name, flush=True)
+
+ tasks = args.tasks
+
+ if args.device == "cpu":
+ device_str = "cpu"
+ else:
+ device_str = f"cuda:{int(args.device)}"
+ cuda_device = torch.device(device_str)
+ is_glm = bool(re.search("chatglm", model_name.lower()))
+ if is_glm:
+ model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
+ else:
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name, low_cpu_mem_usage=True, torch_dtype="auto", trust_remote_code=True
+ )
+ model = model.eval()
+ # align wigh GPTQ to eval ppl
+ if "opt" in model_name:
+ seqlen = model.config.max_position_embeddings
+ model.seqlen = model.config.max_position_embeddings
+ else:
+ seqlen = 2048
+ model.seqlen = seqlen
+ seqlen = args.seqlen
+
+ if "llama" in model_name:
+ from transformers import LlamaTokenizer
+
+ tokenizer = LlamaTokenizer.from_pretrained(model_name)
+ if tokenizer.pad_token is None:
+ tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+ else:
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ if hasattr(tokenizer, "model_max_length"):
+ if tokenizer.model_max_length < seqlen:
+ print(f"change sequence length to {tokenizer.model_max_length} due to the limitation of model_max_length",
+ flush=True)
+ seqlen = min(seqlen, tokenizer.model_max_length)
+ args.seqlen = seqlen
+
+ excel_name = f"{model_name}_{args.num_bits}_{args.group_size}"
+ if args.eval_fp16_baseline:
+ if not args.low_gpu_mem_usage:
+ model = model.to(cuda_device)
+ excel_name += "_fp16.xlsx"
+ eval_model(output_dir=model_name, model=model, tokenizer=tokenizer, tasks=args.tasks, \
+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device,
+ eval_orig_float=True, excel_file=excel_name)
+ exit()
+
+ # if args.iters <= 0:
+ # print("eval rtn", flush=True)
+ # excel_name += "_rtn.xlsx"
+ # q_dq_weight(model, num_bits=args.num_bits, group_size=args.group_size)
+ # if not args.low_gpu_mem_usage:
+ # model = model.to(cuda_device)
+ # eval_model(output_dir=args.output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \
+ # eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device,
+ # excel_file=excel_name)
+ # exit()
+
+ if not args.low_gpu_mem_usage:
+ model = model.to(cuda_device)
+
+ scheme = "asym"
+ if args.sym:
+ scheme = "sym"
+ round = AutoRound
+ if args.adam:
+ round = AutoAdamRound
+
+ autoround = round(model, tokenizer, args.num_bits, args.group_size, scheme, bs=args.train_bs,
+ seqlen=seqlen, n_blocks=args.n_blocks, iters=args.iters, lr=args.lr,
+ minmax_lr=args.minmax_lr, use_quant_input=args.use_quant_input,
+ amp=args.amp, n_samples=args.n_samples, low_gpu_mem_usage=args.low_gpu_mem_usage, seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps) ##TODO args pass
+ autoround.quantize()
+
+ torch.cuda.empty_cache()
+ model.eval()
+ output_dir = args.output_dir + "_" + args.model_name.split('/')[-1] + f"_w{args.num_bits}_g{args.group_size}"
+
+ # model.to(cuda_device)
+ # eval_model(model, model_name, tokenizer, tasks=args.tasks, eval_bs=args.eval_bs)
+ excel_name = f"{output_dir}_result.xlsx"
+ output_dir += "/"
+ print(excel_name, flush=True)
+ eval_model(output_dir=output_dir, model=model, tokenizer=tokenizer, tasks=args.tasks, \
+ eval_bs=args.eval_bs, use_accelerate=args.low_gpu_mem_usage, device=cuda_device, excel_file=excel_name,
+ limit=None)
diff --git a/autoround/requirements.txt b/autoround/requirements.txt
new file mode 100644
index 000000000..25c5f5238
--- /dev/null
+++ b/autoround/requirements.txt
@@ -0,0 +1,10 @@
+transformers==4.36.0
+torch==2.0.1
+git+https://github.com/EleutherAI/lm-evaluation-harness.git@008fc2a23245c40384f2312718433eeb1e0f87a9
+fsspec==2023.9.2
+tiktoken
+transformers_stream_generator
+peft
+sentencepiece
+einops
+
diff --git a/autoround/src/__init__.py b/autoround/src/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/autoround/src/autoround.py b/autoround/src/autoround.py
new file mode 100644
index 000000000..fa66a25e0
--- /dev/null
+++ b/autoround/src/autoround.py
@@ -0,0 +1,1667 @@
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+try:
+ from neural_compressor.utils.utility import LazyImport
+
+ torch = LazyImport("torch")
+ from neural_compressor.utils import logger
+except: # pragma: no cover
+ import logging
+
+ import torch
+
+ logger = logging.getLogger()
+
+import copy
+import time
+from collections import UserDict
+from functools import partial
+
+from torch.amp import autocast
+from torch.functional import F
+
+
+def quant_weight_asym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):
+ """Quantizes and dequantizes weight asymmetrically.
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ maxq = torch.tensor(2**num_bits - 1)
+ zeros = torch.zeros(weight.shape[0], device=weight.device)
+ if isinstance(min_scale, torch.Tensor):
+ wmin_tmp = torch.minimum(weight.min(1)[0], zeros)
+ wmax_tmp = torch.maximum(weight.max(1)[0], zeros)
+ wmin_tmp *= min_scale + 1.0
+ wmax_tmp *= max_scale + 1.0
+ wmax = torch.maximum(wmax_tmp, wmin_tmp)
+ wmin = torch.minimum(wmax_tmp, wmin_tmp)
+ else:
+ wmin = torch.minimum(weight.min(1)[0], zeros)
+ wmax = torch.maximum(weight.max(1)[0], zeros)
+ tmp = (wmin == 0) & (wmax == 0)
+ wmin[tmp] = -1
+ wmax[tmp] = +1
+ scale = (wmax - wmin) / maxq
+ zp = round_ste(-wmin / scale)
+ scale = scale.unsqueeze(dim=-1)
+ zp = zp.unsqueeze(dim=-1)
+ int_w = round_ste(weight / scale + v)
+ q = torch.clamp(int_w + zp, 0, maxq)
+ return scale * (q - zp), scale, zp
+
+
+def quant_weight_sym(weight, num_bits=4, v=0, min_scale=0, max_scale=0):
+ """Quantizes and dequantizes weight symmetrically.
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ maxq = torch.tensor(2 ** (num_bits - 1) - 1).to(weight.device)
+ minq = torch.tensor(-(2 ** (num_bits - 1))).to(weight.device)
+ if num_bits == 1:
+ maxq = torch.tensor(2 ** (num_bits - 1))
+ minq = torch.tensor(2 ** (num_bits - 1) - 1)
+
+ wmax = torch.abs(weight).max(1)[0]
+ wmax *= 1 + max_scale
+ tmp = wmax == 0
+ wmax[tmp] = +1
+ scale = wmax / ((maxq - minq) / 2)
+ scale.unsqueeze_(dim=-1)
+ q = torch.clamp(round_ste(weight / scale + v), minq, maxq)
+ return scale * q, scale, None
+
+
+def quant_weight_actor(weight, num_bits, scheme, v, min_scale, max_scale):
+ """Quantizes and dequantizes weight symmetrically or asymmetrically .
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ scheme: Sym or asym
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ assert num_bits > 0, "num_bits should be larger than 0"
+ if scheme == "sym":
+ return quant_weight_sym(weight, num_bits, v, min_scale, max_scale)
+ else:
+ return quant_weight_asym(weight, num_bits, v, min_scale, max_scale)
+
+
+def quant_weight(weight, num_bits=4, group_size=-1, scheme="asym", v=0, min_scale=0, max_scale=0):
+ """Quantizes and dequantizes weight, handing the group size issue .
+
+ Args:
+ weight: Tensor containing the weight to be quantized
+ num_bits: Number of bits for quantization (e.g., 2, 3, 4, 8)
+ group_size: The number of elements shares scale and zero point
+ scheme: Sym or asym
+ v: Rounding value perturbation
+ min_scale: Minimum scale coefficient for weight
+ max_scale: Maximum scale coefficient for weight
+
+ Returns:
+ Quantized and dequantized weight, scale, zero-point
+ """
+ if group_size == -1 or weight.shape[1] < group_size:
+ return quant_weight_actor(weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale)
+ orig_shape = weight.shape
+ if weight.shape[1] % group_size == 0:
+ weight = weight.reshape(-1, group_size)
+ if isinstance(v, torch.Tensor):
+ v = v.reshape(-1, group_size)
+
+ weight, scale, zp = quant_weight_actor(
+ weight, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale
+ )
+ weight = weight.reshape(orig_shape)
+ return weight, scale, zp
+
+ else:
+ pad_len = (weight.shape[1] + group_size - 1) // group_size * group_size - weight.shape[1]
+ weight_new = torch.nn.functional.pad(weight, (0, pad_len))
+ v = torch.nn.functional.pad(v, (0, pad_len))
+ weight_new = weight_new.reshape(-1, group_size)
+ if isinstance(v, torch.Tensor):
+ v = v.reshape(-1, group_size)
+ weight_new, scale, zp = quant_weight_actor(
+ weight_new, num_bits, scheme=scheme, v=v, min_scale=min_scale, max_scale=max_scale
+ )
+ weight_new = weight_new.reshape(orig_shape[0], -1)
+
+ weight_new = weight_new[:, :-pad_len]
+ return weight_new, scale, zp
+
+
+def round_ste(x: torch.Tensor):
+ """Straight-Through Estimator for rounding.
+ This function is adapted from omniquant.
+
+ Args:
+ x: torch.Tensor
+
+ Returns:
+ torch.Tensor
+ """
+ return (x.round() - x).detach() + x
+
+
+class SaveInputs:
+ """Cache the inputs of the first block."""
+
+ def __init__(self, model, dataloader, seqlen=256, block_name=None):
+ """Initializes the SaveInputs class.
+
+ Args:
+ model: The model to be used.
+ dataloader: The dataloader for the input data.
+ seqlen (int): The sequence length.
+ block_name (str): The name of the block.
+ """
+ self.model = model.eval()
+ self.dataloader = dataloader
+ self.inputs = {}
+ self.block_name = block_name
+ self.seqlen = seqlen
+
+ @torch.no_grad()
+ def get_forward_func(self, name):
+ """Gets the forward function.
+
+ Args:
+ name (str): The name of the function.
+
+ Returns:
+ function: The forward function.
+ """
+
+ def forward(_, hidden_states, *positional_args, **kwargs):
+ dim = int((hasattr(self.model, "config") and "chatglm" in self.model.config.model_type))
+ if name in self.inputs:
+ data = torch.cat([self.inputs[name]["input_ids"], hidden_states.to("cpu")], dim=dim)
+ self.inputs[name]["input_ids"] = data
+ else:
+ self.inputs[name] = {}
+ self.inputs[name]["input_ids"] = hidden_states.to("cpu")
+
+ if "positional_inputs" not in self.inputs[name]:
+ self.inputs[name]["positional_inputs"] = []
+ for idx, item in enumerate(positional_args):
+ self.inputs[name]["positional_inputs"] = move_input_to_device(positional_args)
+
+ for key in kwargs.keys():
+ if isinstance(kwargs[key], torch.Tensor) or isinstance(kwargs[key], list) or (key == "alibi"):
+ if "attention_mask" in key:
+ if key not in self.inputs[name].keys():
+ self.inputs[name][key] = None
+ if kwargs[key] is not None:
+ if self.inputs[name][key] is not None:
+ self.inputs[name][key] = torch.cat(
+ [self.inputs[name][key], kwargs[key].to("cpu")], dim=0
+ )
+ else:
+ self.inputs[name][key] = kwargs[key].to("cpu")
+ elif "alibi" in key:
+ if key not in self.inputs[name].keys():
+ self.inputs[name][key] = None
+ if isinstance(kwargs[key], torch.Tensor):
+ alibi = kwargs[key]
+ batch = kwargs["attention_mask"].shape[0]
+ alibi = alibi.reshape(batch, -1, alibi.shape[1], alibi.shape[2])
+ if self.inputs[name][key] is not None:
+ self.inputs[name][key] = torch.cat([self.inputs[name][key], alibi.to("cpu")], dim=0)
+ else:
+ self.inputs[name][key] = alibi.to("cpu")
+ elif key not in self.inputs[name].keys():
+ self.inputs[name][key] = move_input_to_device(kwargs[key], device=torch.device("cpu"))
+ raise NotImplementedError
+
+ return forward
+
+ @torch.no_grad()
+ def get_inputs(self, n_samples=512):
+ """Gets the inputs.
+
+ Args:
+ n_samples (int): The number of samples.
+
+ Returns:
+ dict: The inputs.
+ """
+ total_cnt = 0
+ self._replace_forward()
+ for data in self.dataloader:
+ if data is None:
+ continue
+ if isinstance(data, torch.Tensor):
+ input_ids = data.to(self.model.device)
+ else:
+ input_ids = data["input_ids"].to(self.model.device)
+ if input_ids.shape[-1] < self.seqlen:
+ continue
+ if total_cnt + input_ids.shape[0] > n_samples:
+ input_ids = input_ids[: n_samples - total_cnt, ...]
+ try:
+ self.model(input_ids)
+ except NotImplementedError:
+ pass
+ except Exception as error:
+ logger.error(error)
+ total_cnt += input_ids.shape[0]
+ if total_cnt >= n_samples:
+ break
+ self._recover_forward()
+ if total_cnt == 0:
+ logger.error(
+ f"no data has been cached, please provide more data with sequence length >={self.seqlen} in the "
+ f"dataloader or decease the sequence length"
+ )
+ exit()
+ elif total_cnt < n_samples:
+ logger.warning(
+ f"Insufficient number of samples collected may affect the quantification. "
+ f"Effective samples size:{total_cnt}, Target sample size:{n_samples}"
+ )
+ res = self.inputs[self.block_name]
+ if "input_ids" in res.keys():
+ total_samples = res["input_ids"].shape[0]
+ if total_samples < n_samples:
+ logger.warning("only cache {total_samples}")
+
+ return res
+
+ def _recover_forward(self):
+ """Recovers the forward function."""
+ for n, m in self.model.named_modules():
+ if n == self.block_name:
+ m.forward = m.orig_forward
+ delattr(m, "orig_forward")
+ break
+
+ def _replace_forward(self):
+ """Replaces the forward function."""
+ for n, m in self.model.named_modules():
+ if n == self.block_name:
+ m.orig_forward = m.forward
+ m.forward = partial(self.get_forward_func(n), m)
+ break
+
+
+def get_module(model, key):
+ """Get module from model by key name.
+
+ Args:
+ model (torch.nn.Module): original model
+ key (str): module name to be replaced
+ """
+ module = model
+ name_list = key.split(".")
+ for name in name_list:
+ if hasattr(module, name):
+ module = getattr(module, name)
+ module = module
+ return module
+
+
+def set_module(model, key, new_module):
+ """Set new module into model by key name.
+
+ Args:
+ model (torch.nn.Module): original model
+ key (str): module name to be replaced
+ new_module (torch.nn.Module): new module to be inserted
+ """
+ module = model
+ name_list = key.split(".")
+ for name in name_list[:-1]:
+ if hasattr(module, name):
+ module = getattr(module, name)
+ else:
+ module = module
+ setattr(module, name_list[-1], new_module)
+
+
+def get_scale_shape(weight, group_size):
+ """Computes the shape of the scale tensor for quantization based on the weight tensor and group size.
+
+ Args:
+ weight (torch.Tensor): The weight tensor of the layer.
+ group_size (int): The size of the groups for quantization.
+
+ Returns:
+ The shape of the scale tensor to be used for quantization.
+ """
+ if group_size == -1 or weight.shape[1] < group_size:
+ shape = weight.shape[0]
+ else:
+ shape = weight.shape[0] * ((weight.shape[1] + group_size - 1) // group_size)
+
+ return shape
+
+
+class WrapperLinear(torch.nn.Module):
+ def __init__(self, orig_layer, enable_minmax_tuning=True):
+ """A wrapper module for linear layers that enables quantization and min-max tuning of weights.
+
+ Args:
+ - orig_layer (torch.nn.Module): The original linear layer to be wrapped.
+ - enable_minmax_tuning (bool): Whether to enable min-max scaling tuning. Default is True.
+
+ Attributes:
+ - orig_layer (torch.nn.Module): The original linear layer being wrapped.
+ - num_bits (int): The number of bits for quantization.
+ - group_size (int): The size of the groups for quantization.
+ - scheme (str): The quantization scheme to use.
+ - value (torch.nn.Parameter): The learnable parameter for quantization.
+ - enable_minmax_tuning (bool): Whether min-max scaling tuning is enabled.
+ - min_scale (torch.nn.Parameter or torch.Tensor): The minimum scale for min-max tuning.
+ - max_scale (torch.nn.Parameter or torch.Tensor): The maximum scale for min-max tuning.
+ """
+ super(WrapperLinear, self).__init__()
+ self.orig_layer = orig_layer
+ self.num_bits = self.orig_layer.bits
+ self.group_size = self.orig_layer.group_size
+ self.scheme = (self.orig_layer.scheme,)
+ self.value = torch.nn.Parameter(
+ torch.zeros(self.orig_layer.weight.shape, device=self.orig_layer.weight.device), requires_grad=True
+ )
+ self.enable_minmax_tuning = enable_minmax_tuning
+ shape = get_scale_shape(self.orig_layer.weight, self.group_size)
+
+ if self.enable_minmax_tuning:
+ self.min_scale = torch.nn.Parameter(
+ torch.zeros(shape, device=self.orig_layer.weight.device), requires_grad=True
+ )
+ self.max_scale = torch.nn.Parameter(
+ torch.zeros(shape, device=self.orig_layer.weight.device), requires_grad=True
+ )
+ else:
+ self.min_scale = torch.tensor(0, device=self.orig_layer.weight.device)
+ self.max_scale = torch.tensor(0, device=self.orig_layer.weight.device)
+
+ def unwrapper(self, v, min_scale, max_scale):
+ """Unwrapper the layer to the original layer.
+
+ Args:
+ - v (torch.Tensor): The rounding v parameter for quantization.
+ - min_scale (torch.nn.Parameter or torch.Tensor): The minimum scale for min-max tuning.
+ - max_scale (torch.nn.Parameter or torch.Tensor): The maximum scale for min-max tuning.
+
+ Returns:
+ - torch.nn.Module: The original linear layer with updated weights after quantization and dequantization.
+ """
+ min_scale.clamp_(-1, 0)
+ max_scale.clamp_(-1, 0)
+
+ q_dq_weight, scale, zp = quant_weight(
+ self.orig_layer.weight,
+ self.num_bits,
+ self.group_size,
+ self.scheme,
+ v,
+ min_scale,
+ max_scale,
+ )
+ self.orig_layer.weight.data.copy_(q_dq_weight)
+ self.orig_layer.weight.grad = None ##clear grad
+ self.orig_layer.scale = scale.to("cpu")
+ self.orig_layer.zp = zp.to("cpu")
+ return self.orig_layer
+
+ def forward(self, x):
+ """Performs forward pass through the wrapped linear layer with quantized weights.
+
+ Args:
+ - x (torch.Tensor): The input tensor.
+
+ Returns:
+ - torch.Tensor: The output tensor after applying the linear transformation with quantized weights.
+ """
+ weight = self.orig_layer.weight
+ self.min_scale.data.copy_(torch.clamp(self.min_scale.data, -1, 0))
+ self.max_scale.data.copy_(torch.clamp(self.max_scale.data, -1, 0))
+ weight_q, _, _ = quant_weight(
+ weight, self.num_bits, self.group_size, self.scheme, self.value, self.min_scale, self.max_scale
+ )
+ weight_q = weight_q.to(weight.dtype)
+ return F.linear(x, weight_q, self.orig_layer.bias)
+
+
+class WrapperTransformerConv1d(torch.nn.Module):
+ def __init__(self, orig_layer, enable_minmax_tuning=True):
+ """A wrapper module for transformers 1D convolutional layers used in transformers,
+ enabling quantization and min-max tuning of weights.
+
+ Args:
+ - orig_layer (torch.nn.Module): The original 1D convolutional layer to be wrapped.
+ - num_bits (int): The number of bits for quantization.
+ - group_size (int): The size of the groups for quantization.
+ - scheme (str): The quantization scheme to use.
+ - enable_minmax_tuning (bool): Whether to enable min-max scaling tuning. Default is True.
+
+ Attributes:
+ - orig_layer (torch.nn.Module): The original 1D convolutional layer being wrapped.
+ - num_bits (int): The number of bits for quantization.
+ - group_size (int): The size of the groups for quantization.
+ - scheme (str): The quantization scheme to use.
+ - weight_t (torch.Tensor): Transposed weight tensor of the original layer.
+ - value (torch.nn.Parameter): The learnable parameter for quantization.
+ - enable_minmax_tuning (bool): Whether min-max scaling tuning is enabled.
+ - min_scale (torch.nn.Parameter or torch.Tensor): The minimum scale for min-max tuning.
+ - max_scale (torch.nn.Parameter or torch.Tensor): The maximum scale for min-max tuning.
+ """
+ super(WrapperTransformerConv1d, self).__init__()
+ self.orig_layer = orig_layer
+ self.num_bits = self.orig_layer.bits
+ self.group_size = self.orig_layer.group_size
+ self.scheme = self.orig_layer.scheme
+ device = self.orig_layer.weight.device
+ self.weight_t = self.orig_layer.weight.t()
+ self.value = torch.nn.Parameter(torch.zeros(self.weight_t.shape, device=device), requires_grad=True)
+ shape = get_scale_shape(self.weight_t, self.group_size)
+
+ if enable_minmax_tuning:
+ self.min_scale = torch.nn.Parameter(torch.zeros(shape, device=device), requires_grad=True)
+ self.max_scale = torch.nn.Parameter(torch.zeros(shape, device=device), requires_grad=True)
+ else:
+ self.min_scale = torch.tensor(0, device=device)
+ self.max_scale = torch.tensor(0, device=device)
+
+ def unwrapper(self, v, min_scale, max_scale):
+ """Unwrapper the layer to the original conv1d layer..
+
+ Args:
+ - v (torch.Tensor): The scaling parameter for quantization.
+ - min_scale (torch.nn.Parameter or torch.Tensor): The minimum scale for min-max tuning.
+ - max_scale (torch.nn.Parameter or torch.Tensor): The maximum scale for min-max tuning.
+
+ Returns:
+ - torch.nn.Module: The original 1D convolutional layer with updated weights after inverse quantization.
+ """
+ min_scale.clamp_(-1, 0)
+ max_scale.clamp_(-1, 0)
+ weight_q, scale, zp = quant_weight(
+ self.weight_t,
+ self.num_bits,
+ self.group_size,
+ self.scheme,
+ v,
+ min_scale,
+ max_scale,
+ )
+ self.orig_layer.weight.data.copy_(weight_q.t())
+ self.orig_layer.weight.grad = None
+ self.orig_layer.scale = scale.to("cpu")
+ self.orig_layer.zp = zp.to("cpu")
+ return self.orig_layer
+
+ def forward(self, x):
+ """Performs forward pass through the wrapped 1D convolutional layer with quantized weights.
+
+ Args:
+ x (torch.Tensor): The input tensor.
+
+ Returns:
+ torch.Tensor: The output tensor after applying the convolutional transformation with quantized weights.
+ """
+ with torch.no_grad():
+ self.min_scale.clamp_(-1, 0)
+ self.max_scale.clamp_(-1, 0)
+ weight_q, _, _ = quant_weight(
+ self.weight_t,
+ self.num_bits,
+ self.group_size,
+ self.scheme,
+ self.value,
+ self.min_scale,
+ self.max_scale,
+ )
+ weight_q = weight_q.to(self.weight_t.dtype)
+ size_out = x.size()[:-1] + (self.orig_layer.nf,)
+ x = torch.addmm(self.orig_layer.bias, x.view(-1, x.size(-1)), weight_q.t())
+ x = x.view(*size_out)
+ return x
+
+
+def wrapper_block(block, enable_minmax_tuning):
+ """Wraps the layers in the given block with a custom Wrapper module.
+
+ Args:
+ block: The input block containing linear and conv1d layers to be wrapped.
+ enable_minmax_tuning: A boolean indicating whether min-max tuning is enabled.
+
+ Returns:
+ list: A list of names of the wrapped layers.
+ """
+ names = []
+ for n, m in block.named_modules():
+ if isinstance(m, torch.nn.Linear):
+ if "fp" in m.data_type or "float" in m.data_type:
+ continue
+ new_m = WrapperLinear(m, enable_minmax_tuning=enable_minmax_tuning)
+ set_module(block, n, new_m)
+ names.append(n)
+
+ try:
+ import transformers
+
+ if isinstance(m, transformers.modeling_utils.Conv1D):
+ if "fp" in m.data_type or "float" in m.data_type:
+ continue
+ new_m = WrapperTransformerConv1d(m, enable_minmax_tuning=enable_minmax_tuning)
+ set_module(block, n, new_m)
+ names.append(n)
+ except:
+ pass
+ return names
+
+
+def unwrapper_block(block, vs, min_scales, max_scales):
+ """Unwraps the WrapperLinear and WrapperTransformerConv1d modules in the given block.
+
+ Args:
+ block: The input block containing wrapped modules to be unwrapped.
+ vs: A dictionary of scaling parameters for the wrapped modules.
+ min_scales: A dictionary of minimum scaling values for the wrapped modules.
+ max_scales: A dictionary of maximum scaling values for the wrapped modules.
+ """
+ for n, m in block.named_modules():
+ if hasattr(m, "orig_layer"):
+ v = 0
+ min_scale = 0
+ max_scale = 0
+ if isinstance(vs, dict):
+ v = vs[n]
+ if isinstance(min_scales, dict):
+ min_scale = min_scales[n]
+ min_scale = torch.clamp(min_scale, -1, 0)
+ if isinstance(max_scales, dict):
+ max_scale = max_scales[n]
+ max_scale = torch.clamp(max_scale, -1, 0)
+ orig_layer = m.unwrapper(v, min_scale, max_scale)
+ set_module(block, n, orig_layer)
+
+
+def sampling_inputs(input_ids, input_others, indices, seqlen):
+ """Samples inputs based on the given indices and sequence length.
+
+ Args:
+ input_ids: The input tensor containing IDs.
+ input_others: A dictionary containing other input data.
+ indices: The indices to sample from the input.
+ seqlen: The sequence length.
+
+ Returns:
+ current_input_ids: The sampled input IDs.
+ current_input_others: The sampled other input data.
+ """
+ if len(input_ids.shape) == 3:
+ if int(len(input_others["positional_inputs"]) > 0):
+ current_input_ids = input_ids[:, indices, :]
+ else:
+ current_input_ids = input_ids[indices, :, :]
+ else:
+ n_samples = input_ids.shape[0] // seqlen
+ current_input_ids = input_ids.view(n_samples, seqlen, -1)
+ current_input_ids = current_input_ids[indices, :, :]
+ current_input_ids = current_input_ids.reshape(-1, input.shape[-1])
+
+ current_input_others = {"positional_inputs": input_others["positional_inputs"]}
+ for key in input_others.keys():
+ if "attention_mask" in key or "alibi" in key:
+ current_input_others[key] = None
+ if input_others[key] is not None:
+ current_input_others[key] = input_others[key][indices, ...]
+ else:
+ current_input_others[key] = input_others[key]
+
+ return current_input_ids, current_input_others
+
+
+def move_input_to_device(input, device=torch.device("cpu")):
+ """Moves input data to the specified device.
+
+ Args:
+ input: The input data to be moved.
+ device: The target device.
+
+ Returns:
+ The input data on the specified device.
+ """
+ if isinstance(input, torch.Tensor):
+ return input.to(device)
+ if isinstance(input, dict) or isinstance(input, UserDict):
+ for inp in input.keys():
+ input[inp] = move_input_to_device(input[inp], device)
+ elif isinstance(input, list) or isinstance(input, tuple):
+ input_res = []
+ for inp in input:
+ input_res.append(move_input_to_device(inp, device))
+ input = input_res
+ return input
+
+
+def check_is_cpu(device):
+ """Check if the device is a CPU.
+
+ Args:
+ device: The device to be checked.
+
+ Returns:
+ bool: True if the device is a CPU, False otherwise.
+ """
+ return device == torch.device("cpu") or device == "cpu"
+
+
+def block_forward(block, input_ids, input_others, amp=False, amp_dtype=torch.float16, device=torch.device("cpu")):
+ """Performs a forward pass through a block with the given inputs.
+
+ Args:
+ block: The block to perform the forward pass on.
+ input_ids: The input IDs.
+ input_others: A dictionary containing other input data.
+ amp: A boolean indicating whether to use automatic mixed precision.
+ amp_dtype: The data type for automatic mixed precision.
+ device: The target device.
+
+ Returns:
+ output: The output of the forward pass.
+ """
+ if input_ids.device != device:
+ # input_ids, input_others = move_to_device(input_ids, input_others, device)
+ input_ids = move_input_to_device(input_ids, device)
+ input_others = move_input_to_device(input_others, device)
+ if "alibi" in input_others.keys():
+ attention_mask = input_others["attention_mask"]
+ alibi = input_others["alibi"]
+ if alibi is not None:
+ alibi = alibi.reshape(-1, alibi.shape[2], alibi.shape[3])
+ if amp and not check_is_cpu(device):
+ with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover
+ output = block(
+ input_ids, attention_mask=attention_mask, alibi=alibi
+ ) ##TODO is this correct for all models with alibi?
+ elif amp and check_is_cpu(device):
+ with torch.autocast(device_type="cpu", dtype=torch.bfloat16):
+ output = block(input_ids, attention_mask=attention_mask, alibi=alibi)
+ else:
+ output = block(input_ids, attention_mask=attention_mask, alibi=alibi)
+ else:
+ input_tuple = input_others.pop("positional_inputs", None)
+ if amp and not check_is_cpu(device):
+ with autocast(device_type="cuda", dtype=amp_dtype): # pragma: no cover
+ output = block.forward(input_ids, *input_tuple, **input_others)
+ elif amp and check_is_cpu(device):
+ with torch.autocast(device_type="cpu", dtype=torch.bfloat16):
+ output = block.forward(input_ids, *input_tuple, **input_others)
+ else:
+ output = block.forward(input_ids, *input_tuple, **input_others)
+ if isinstance(output, list) or isinstance(output, tuple):
+ output = output[0]
+ return output
+
+
+def collect_round_v(block):
+ """Collects the round values for wrapped linear modules in the given block.
+
+ Args:
+ block: The input block.
+
+ Returns:
+ vs: A dictionary of round values for the wrapped linear modules.
+ """
+ vs = {}
+ for n, m in block.named_modules():
+ if hasattr(m, "orig_layer"):
+ v = m.value.data
+ vs[n] = copy.deepcopy(v)
+ return vs
+
+
+def collect_minmax_scale(block):
+ """Collects the min-max scaling values for wrapped linear modules in the given block.
+
+ Args:
+ block: The input block.
+
+ Returns:
+ min_scales: A dictionary of minimum scaling values.
+ max_scales: A dictionary of maximum scaling values.
+ """
+ min_scales = {}
+ max_scales = {}
+ for n, m in block.named_modules():
+ if hasattr(m, "orig_layer"):
+ min_scales[n] = copy.deepcopy(torch.clamp(m.min_scale.data, -1, 0))
+ max_scales[n] = copy.deepcopy(torch.clamp(m.max_scale.data, -1, 0))
+ return min_scales, max_scales
+
+
+def get_batch_dim(input_others):
+ """Gets the batch dimension based on the input positional inputs.
+
+ Args:
+ input_others: A dictionary containing input data.
+
+ Returns:
+ dim: The batch dimension.
+ """
+ dim = int(len(input_others["positional_inputs"]) > 0)
+ return dim
+
+
+class WrapperMultiblock(torch.nn.Module):
+ """A wrapper for a list of modules to be act as a single block.
+
+ Args:
+ module_list: The list of modules to wrap.
+ """
+
+ def __init__(self, module_list):
+ super(WrapperMultiblock, self).__init__()
+ self.layers = torch.nn.ModuleList(module_list)
+
+ def forward(self, x, **kwargs):
+ hidden_states = x
+ for idx, decoder_layer in enumerate(self.layers):
+ layer_outputs = decoder_layer(hidden_states, **kwargs)
+ hidden_states = layer_outputs
+ if isinstance(hidden_states, tuple) or isinstance(hidden_states, list):
+ hidden_states = layer_outputs[0]
+ return hidden_states
+
+
+def get_block_names(model):
+ """Get the block names for transformers-like networks.
+
+ Args:
+ model: The model.
+
+ Returns:
+ block_names: A list of block names.
+ """
+ block_names = []
+ target_m = None
+ for n, m in model.named_modules():
+ if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
+ target_m = (n, m)
+ for n, m in target_m[1].named_children():
+ block_names.append(target_m[0] + "." + n)
+ return block_names
+
+
+def get_tokenizer_function(tokenizer, seqlen):
+ """Returns a default tokenizer function.
+
+ Args:
+ tokenizer: The tokenizer to be used for tokenization.
+ seqlen: The maximum sequence length.
+
+ Returns: A default tokenizer function that applies the provided tokenizer with truncation and a maximum length of
+ seqlen to the "text" field of examples.
+ """
+
+ def default_tokenizer_function(examples):
+ example = tokenizer(examples["text"], truncation=True, max_length=seqlen)
+ return example
+
+ return default_tokenizer_function
+
+
+def get_dataloader(tokenizer, seqlen, data_name="NeelNanda/pile-10k", split="train", seed=42, bs=4):
+ """Returns a dataloader for the specified dataset and split.
+
+ Args:
+ tokenizer: The tokenizer to be used for tokenization.
+ seqlen: The maximum sequence length.
+ data_name: The name of the dataset.
+ split: The data split to be used (e.g., "train", "test").
+ seed: The random seed for shuffling the dataset.
+ bs: The batch size for the dataloader.
+
+ Returns:
+ A dataloader for the specified dataset and split, using the provided tokenizer and sequence length.
+ """
+ from datasets import load_dataset
+ from torch.utils.data import DataLoader
+
+ tokenizer_function = get_tokenizer_function(tokenizer, seqlen)
+
+ @torch.no_grad()
+ def collate_batch(batch):
+ input_ids_new = []
+ for text in batch:
+ input_ids = text["input_ids"]
+ if input_ids.shape[0] < seqlen:
+ continue
+ input_ids = input_ids[:seqlen]
+ input_ids_list = input_ids.tolist()
+ if input_ids_list.count(input_ids_list[-1]) > seqlen // 2:
+ continue
+ input_ids_new.append(input_ids)
+ if len(input_ids_new) == 0:
+ return None
+ tmp = torch.vstack(input_ids_new)
+ res = {"input_ids": tmp}
+ return res
+
+ calib_dataset = load_dataset(data_name, split=split)
+ calib_dataset = calib_dataset.shuffle(seed=seed)
+ calib_dataset = calib_dataset.map(tokenizer_function, batched=True)
+ calib_dataset.set_format(type="torch", columns=["input_ids"])
+ calib_dataloader = DataLoader(calib_dataset, batch_size=bs, shuffle=False, collate_fn=collate_batch)
+ return calib_dataloader
+
+
+class AutoRound(object):
+ """This is Signround+ which is an advanced version of Signround. For more information,
+ please refer to Cheng, Wenhua, et al. "Optimize weight rounding via signed gradient descent
+ for the quantization of llms." arXiv preprint arXiv:2309.05516 (2023).
+
+ Args:
+ model: The PyTorch model to be quantized.
+ tokenizer: An optional tokenizer for processing input data.
+ bits (int): Number of bits for quantization (default is 4).
+ group_size (int): Size of the quantization group (default is 128).
+ scheme (str): The quantization scheme to be used (default is "asym").
+ weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
+ weight_config={
+ 'layer1':##layer_name
+ {
+ 'data_type': 'int',
+ 'bits': 4,
+ 'group_size': 32,
+ 'scheme': "asym", ## or sym
+ }
+ ...
+ }
+ enable_full_range (bool): Whether to enable full range quantization (default is False).
+ bs (int): Batch size for training (default is 8).
+ amp (bool): Whether to use automatic mixed precision (default is True).
+ device: The device to be used for training (default is "cuda:0").
+ lr_scheduler: The learning rate scheduler to be used.
+ dataloader: The dataloader for input data (to be supported in future).
+ dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
+ dataset_split (str): The split of the dataset to be used (default is "train").
+ use_quant_input (bool): Whether to use quantized input data (default is True).
+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
+ lr (float): The learning rate (default is 0.005).
+ minmax_lr (float): The learning rate for min-max tuning (default is None).
+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
+ iters (int): Number of iterations (default is 200).
+ seqlen (int): Length of the sequence.
+ n_samples (int): Number of samples (default is 512).
+ sampler (str): The sampling method (default is "rand").
+ seed (int): The random seed (default is 42).
+ n_blocks (int): Number of blocks (default is 1).
+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
+ not_use_mse (bool): Whether to use mean squared error (default is False).
+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).
+ data_type (str): The data type to be used (default is "int").
+ **kwargs: Additional keyword arguments.
+
+ Returns:
+ The quantized model.
+ """
+
+ def __init__(
+ self,
+ model,
+ tokenizer=None,
+ bits: int = 4,
+ group_size: int = 128,
+ scheme: str = "asym",
+ weight_config: dict = {},
+ enable_full_range: bool = False, ##for symmetric, TODO support later
+ bs: int = 8,
+ amp: bool = True,
+ device="cuda:0",
+ lr_scheduler=None,
+ dataloader=None, ## to support later
+ dataset_name: str = "NeelNanda/pile-10k",
+ dataset_split: str = "train",
+ use_quant_input: bool = True,
+ enable_minmax_tuning: bool = True,
+ lr: float = 0.005,
+ minmax_lr: float = None,
+ low_gpu_mem_usage: bool = True,
+ iters: int = 200,
+ seqlen: int = 2048,
+ n_samples: int = 512,
+ sampler: str = "rand",
+ seed: int = 42,
+ n_blocks: int = 1,
+ gradient_accumulate_steps: int = 1,
+ not_use_mse: bool = False,
+ dynamic_max_gap: int = -1,
+ data_type: str = "int", ##only support data_type
+ **kwargs,
+ ):
+ self.model = model
+ self.model = self.model.to("cpu")
+ self.amp = amp
+ self.use_quant_input = use_quant_input
+ self.enable_minmax_tuning = enable_minmax_tuning
+ self.n_samples = n_samples
+ self.n_blocks = n_blocks
+ self.bits = bits
+ self.group_size = group_size
+ self.scheme = scheme
+ self.low_gpu_mem_usage = low_gpu_mem_usage
+ self.data_type = data_type
+ self.supported_types = [torch.nn.Linear]
+ try:
+ import transformers
+
+ self.supported_types.append(transformers.modeling_utils.Conv1D)
+ except:
+ pass
+ self.weight_config = weight_config
+ assert dataloader is not None or tokenizer is not None
+ self.dataset_split = dataset_split
+ self.seed = seed
+ self.tokenizer = tokenizer
+ self.seqlen = seqlen
+ self.train_bs = bs
+ self.n_blocks = n_blocks
+ self.device = device
+ self.amp_dtype = torch.float16
+ if self.model.dtype != torch.float32:
+ self.amp_dtype = self.model.dtype
+ if self.device == "cpu":
+ self.amp_dtype = torch.bfloat16
+ if self.amp:
+ self.model = self.model.to(self.amp_dtype)
+ logger.info(f"using {self.amp_dtype}")
+ self.dataset_name = dataset_name
+
+ if dataloader is None:
+ self.dataloader = get_dataloader(
+ self.tokenizer,
+ self.seqlen,
+ seed=self.seed,
+ bs=self.train_bs,
+ split=self.dataset_split,
+ data_name=self.dataset_name,
+ )
+ else:
+ self.dataloader = dataloader
+ self.lr = lr
+ self.minmax_lr = minmax_lr
+ if self.minmax_lr is None:
+ self.minmax_lr = self.lr
+ self.iters = iters
+ self.sampler = sampler
+ self.gradient_accumulate_steps = gradient_accumulate_steps
+ self.not_use_mse = not_use_mse
+ self.dynamic_max_gap = dynamic_max_gap
+ self.enable_full_range = enable_full_range
+ assert self.enable_full_range is False, "only support enable_full_range=False currently"
+ self.lr_scheduler = lr_scheduler
+ self.set_layerwise_config(self.weight_config)
+ self.optimizer = self.get_optimizer(None)
+ self.check_configs()
+
+ def get_optimizer(self, optimizer):
+ """Returns the specified optimizer. In SignRound, we fix the optimizer.
+
+ Args:
+ optimizer: The optimizer to be used.
+
+ Returns:
+ The specified optimizer.
+ """
+ from .sign_sgd import SGD
+
+ return SGD
+
+ def get_scaler(self):
+ """Returns scaler, in SignRound, no need to use scaler."""
+ return None
+
+ def scale_loss_and_backward(self, scaler, loss):
+ """Scales the loss and performs backward pass.
+
+ Args:
+ scaler: The scaler to be used.
+ loss: The loss to be scaled.
+
+ Returns:
+ The scaled loss.
+ """
+ scale_loss = loss * 1000
+ scale_loss.backward()
+ return scale_loss
+
+ def step(self, scaler, optimizer, lr_schedule):
+ """Performs a step in the optimization process.
+
+ Args:
+ scaler: The scaler to be used.
+ optimizer: The optimizer for the step.
+ lr_schedule: The learning rate schedule.
+
+ Returns:
+ None
+ """
+ optimizer.step()
+ optimizer.zero_grad()
+ lr_schedule.step()
+
+ def check_configs(self):
+ """Checks if the configurations are valid.
+
+ Raises:
+ AssertionError: If any of the configurations are invalid.
+ """
+ assert isinstance(self.model, torch.nn.Module)
+ assert self.bits > 0, "bits must be positive"
+ assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"
+ assert self.train_bs > 0, "batch size must be positive"
+ assert self.iters > 0, "iters must be positive"
+ assert self.seqlen > 0, "seqlen must be positive"
+ assert self.n_blocks > 0, "n_blocks must be positive"
+ assert self.gradient_accumulate_steps > 0, "gradient accumulate step must be positive"
+
+ def set_layerwise_config(self, weight_config):
+ """Sets the layer-wise configuration based on the provided weight_config.
+
+ Args:
+ weight_config: The weight configuration.
+
+ Returns:
+ None
+ """
+ for n, m in self.model.named_modules():
+ is_supported_type = False
+ for supported_type in self.supported_types:
+ if isinstance(m, supported_type):
+ is_supported_type = True
+ break
+ if not is_supported_type:
+ continue
+ if n not in weight_config.keys():
+ weight_config[n] = {}
+ weight_config[n]["data_type"] = self.data_type
+ weight_config[n]["bits"] = self.bits
+ weight_config[n]["group_size"] = self.group_size
+ weight_config[n]["scheme"] = self.scheme
+ else:
+ if "data_type" not in weight_config[n].keys():
+ weight_config[n]["data_type"] = self.data_type
+ if "bits" not in weight_config[n].keys():
+ weight_config[n]["bits"] = self.bits
+ if "group_size" not in weight_config[n].keys():
+ weight_config[n]["group_size"] = self.group_size
+ if "scheme" not in weight_config[n].keys():
+ weight_config[n]["scheme"] = self.scheme
+ m.data_type = weight_config[n]["data_type"]
+ m.bits = weight_config[n]["bits"]
+ m.group_size = weight_config[n]["group_size"]
+ m.scheme = weight_config[n]["scheme"]
+
+ def get_batch_dim(self, input_others):
+ """Get the batch dimension of the input tensor.
+
+ Args:
+ input_others: A dictionary containing input data, including positional_inputs.
+
+ Returns:
+ The batch dimension of the input tensor.
+ """
+ dim = int(len(input_others["positional_inputs"]) > 0)
+ return dim
+
+ @torch.no_grad()
+ def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device, batch_dim):
+ """Compute the output of a given block of the model for a given input.
+
+ Args:
+ block: The block of the model.
+ input_ids: The input tensor containing tokenized input ids.
+ input_others: A dictionary containing additional input data.
+ bs: The batch size for computing the output.
+ device: The device for computation.
+ cache_device: The device for storing the output.
+ batch_dim: The batch dimension of the output tensor.
+
+ Returns:
+ The output tensor of the block.
+ """
+
+ output = []
+ for i in range(0, self.n_samples, bs):
+ indices = torch.arange(i, i + bs).to(torch.long)
+ tmp_input_ids, tmp_input_others = sampling_inputs(input_ids, input_others, indices, self.seqlen)
+ tmp_output = block_forward(block, tmp_input_ids, tmp_input_others, self.amp, self.amp_dtype, device).to(
+ cache_device
+ )
+ output.append(tmp_output)
+ output = torch.cat(output, dim=batch_dim)
+ torch.cuda.empty_cache()
+ return output
+
+ def quant_block(self, block, input_ids, input_others, q_input=None, device=torch.device("cpu")):
+ """Quantize the weights of a given block of the model.
+
+ Args:
+ block: The block of the model to be quantized.
+ input_ids: The input tensor containing tokenized input ids.
+ input_others: A dictionary containing additional input data.
+ q_input: The quantized input tensor.
+ device: The device for quantization.
+
+ Returns:
+ Tuple: (q_outputs, output) if self.use_quant_input is True, else (None, output)
+ """
+ batch_dim = get_batch_dim(input_others)
+ if not self.low_gpu_mem_usage and input_ids.device != device:
+ input_ids = move_input_to_device(input_ids, device)
+ input_others = move_input_to_device(input_others, device)
+ cache_device = device
+ if self.low_gpu_mem_usage:
+ cache_device = "cpu"
+ output = self.get_block_outputs(block, input_ids, input_others, self.train_bs, device, cache_device, batch_dim)
+
+ if q_input is not None:
+ input_ids = q_input.to(cache_device)
+
+ names = wrapper_block(block, self.enable_minmax_tuning)
+ logger.info(names)
+
+ round_params = []
+ minmax_params = []
+ for n, m in block.named_modules():
+ if hasattr(m, "orig_layer"):
+ round_params.append(m.value)
+ minmax_params.append(m.min_scale)
+ minmax_params.append(m.max_scale)
+
+ if self.enable_minmax_tuning:
+ optimizer = self.optimizer(
+ [{"params": round_params}, {"params": minmax_params, "lr": self.minmax_lr}], lr=self.lr, weight_decay=0
+ )
+ else:
+ optimizer = self.optimizer(round_params, lr=self.lr, weight_decay=0)
+
+ if self.lr_scheduler is None:
+ lr_schedule = torch.optim.lr_scheduler.LinearLR(
+ optimizer, start_factor=1.0, end_factor=0.0, total_iters=self.iters, verbose=False
+ )
+ else:
+ lr_schedule = copy.deepcopy(self.lr_scheduler)
+
+ pick_samples = self.train_bs
+ if len(input_ids.shape) == 3:
+ n_samples = input_ids.shape[batch_dim]
+ else:
+ n_samples = input_ids.shape[0] // self.seqlen
+ if self.sampler != "rand":
+ indices = torch.randperm(n_samples)[:pick_samples]
+ last_best_iter = 0
+ best_loss = torch.finfo(torch.float).max
+ mse_loss = torch.nn.MSELoss().to(device)
+ scaler = self.get_scaler() # pylint: disable=assignment-from-none
+ for i in range(self.iters):
+ if self.sampler == "rand":
+ indices = torch.randperm(n_samples)[:pick_samples]
+
+ total_loss = 0
+ for _ in range(self.gradient_accumulate_steps):
+ current_input_ids, current_input_others = sampling_inputs(
+ input_ids, input_others, indices, seqlen=self.seqlen
+ )
+ if len(input_ids.shape) == 3:
+ if batch_dim == 0:
+ current_output = output[indices, :, :]
+ elif batch_dim == 1:
+ current_output = output[:, indices, :]
+ else:
+ current_output = output[:, :, indices]
+ else:
+ current_output = output.view(n_samples, self.seqlen, -1)
+ current_output = current_output[indices, :, :]
+ current_output = current_output.reshape(-1, current_output.shape[-1])
+ current_output = move_input_to_device(current_output, device)
+
+ output_q = block_forward(
+ block, current_input_ids, current_input_others, self.amp, self.amp_dtype, device
+ )
+ if self.amp and not check_is_cpu(device):
+ with autocast(device_type="cuda", dtype=self.amp_dtype):
+ loss = mse_loss(output_q, current_output)
+ else:
+ loss = mse_loss(output_q.to(torch.float32), current_output.to(torch.float32))
+
+ total_loss += loss.item() / self.gradient_accumulate_steps
+ self.scale_loss_and_backward(scaler, loss)
+
+ if total_loss < best_loss:
+ best_loss = total_loss
+ if not self.not_use_mse:
+ # print(f"get better result at iter {i}, the loss is {total_loss}", flush=True)
+ best_v = collect_round_v(block)
+ best_min_scale, best_max_scale = collect_minmax_scale(block)
+ last_best_iter = i
+ if self.not_use_mse and i == self.iters - 1:
+ best_v = collect_round_v(block)
+ best_min_scale, best_max_scale = collect_minmax_scale(block)
+
+ if not self.not_use_mse:
+ if self.dynamic_max_gap > 0 and i - last_best_iter >= self.dynamic_max_gap:
+ break
+ self.step(scaler, optimizer, lr_schedule)
+ unwrapper_block(block, best_v, best_min_scale, best_max_scale)
+ if self.use_quant_input:
+ q_outputs = self.get_block_outputs(
+ block, input_ids, input_others, self.train_bs, device, cache_device, batch_dim
+ )
+
+ return q_outputs, output
+
+ else:
+ return None, output
+
+ def q_dq_weight_round(
+ self,
+ model: torch.nn.Module,
+ inputs,
+ block_names,
+ n_blocks=1,
+ device=torch.device("cpu"),
+ ):
+ """Quantize and dequantize the weights of the specified blocks in the model.
+
+ Args:
+ model: The PyTorch model to be quantized.
+ inputs: The input data for quantization.
+ block_names: The names of the blocks to be quantized and dequantized.
+ n_blocks: The number of blocks to quantize and dequantize.
+ device: The device for quantization and dequantization.
+
+ Returns:
+ None
+ """
+ q_input = None
+ torch.cuda.empty_cache()
+ for n, m in model.named_parameters():
+ m.requires_grad_(False)
+ input_ids = inputs["input_ids"]
+ inputs.pop("input_ids", None)
+ input_others = inputs
+ torch.cuda.empty_cache()
+ for i in range(0, len(block_names), n_blocks):
+ if n_blocks == 1:
+ n = block_names[i]
+ logger.info(n)
+ m = get_module(model, n)
+ else:
+ names = block_names[i : i + n_blocks]
+ logger.info(names)
+ modules = [get_module(model, n) for n in names]
+ m = WrapperMultiblock(modules)
+
+ m = m.to(device)
+
+ q_input, input_ids = self.quant_block(
+ m,
+ input_ids,
+ input_others,
+ q_input=q_input,
+ device=device,
+ )
+ m.to("cpu")
+ torch.cuda.empty_cache()
+
+ del q_input
+ del input_ids
+ del input_others
+ del inputs
+
+ torch.cuda.empty_cache()
+
+ def quantize(self):
+ """Quantize the model and return the quantized model along with weight configurations.
+
+ Returns:
+ The quantized model and weight configurations.
+ """
+ start_time = time.time()
+ # logger.info("cache block input")
+ block_names = get_block_names(self.model)
+ if len(block_names) == 0:
+ logger.warning("could not find blocks, exit with original model")
+ return
+
+ if self.amp:
+ self.model = self.model.to(self.amp_dtype)
+ if not self.low_gpu_mem_usage:
+ self.model = self.model.to(self.device)
+
+ save_input_actor = SaveInputs(self.model, self.dataloader, self.seqlen, block_names[0])
+ inputs = save_input_actor.get_inputs(n_samples=self.n_samples)
+ del save_input_actor
+ if "input_ids" in inputs.keys():
+ total_samples = inputs["input_ids"].shape[0]
+ if total_samples < self.train_bs:
+ self.train_bs = total_samples
+ logger.warning(f"force the train batch size to {total_samples} ")
+ self.model = self.model.to("cpu")
+ torch.cuda.empty_cache()
+ self.q_dq_weight_round(
+ self.model,
+ inputs,
+ block_names,
+ n_blocks=self.n_blocks,
+ device=self.device,
+ )
+ for n, m in self.model.named_modules():
+ if n in self.weight_config.keys():
+ if hasattr(m, "scale"):
+ self.weight_config[n]["scale"] = m.scale
+ self.weight_config[n]["zp"] = m.zp
+ delattr(m, "scale")
+ delattr(m, "zp")
+ else:
+ self.weight_config[n]["data_type"] = "float"
+ if self.amp_dtype == torch.bfloat16:
+ self.weight_config[n]["data_type"] = "bfloat"
+ self.weight_config[n]["bits"] = 16
+ self.weight_config[n]["group_size"] = None
+ self.weight_config[n]["sym"] = None
+
+ end_time = time.time()
+ cost_time = end_time - start_time
+ logger.info(f"quantization runtime {cost_time}")
+ return self.model, self.weight_config
+
+
+class AutoOPTRound(AutoRound):
+ """Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.
+
+ Args:
+ model: The PyTorch model to be quantized.
+ tokenizer: An optional tokenizer for processing input data.
+ bits (int): Number of bits for quantization (default is 4).
+ group_size (int): Size of the quantization group (default is 128).
+ scheme (str): The quantization scheme to be used (default is "asym").
+ weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
+ enable_full_range (bool): Whether to enable full range quantization (default is False).
+ bs (int): Batch size for training (default is 8).
+ amp (bool): Whether to use automatic mixed precision (default is True).
+ device: The device to be used for training (default is "cuda:0").
+ lr_scheduler: The learning rate scheduler to be used.
+ dataloader: The dataloader for input data (to be supported in future).
+ dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
+ dataset_split (str): The split of the dataset to be used (default is "train").
+ use_quant_input (bool): Whether to use quantized input data (default is True).
+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
+ lr (float): The learning rate (default is 0.005).
+ minmax_lr (float): The learning rate for min-max tuning (default is None).
+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
+ iters (int): Number of iterations (default is 200).
+ seqlen (int): Length of the sequence.
+ n_samples (int): Number of samples (default is 512).
+ sampler (str): The sampling method (default is "rand").
+ seed (int): The random seed (default is 42).
+ n_blocks (int): Number of blocks (default is 1).
+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
+ not_use_mse (bool): Whether to use mean squared error (default is False).
+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).
+ data_type (str): The data type to be used (default is "int").
+ optimizer: string or object
+ **kwargs: Additional keyword arguments.
+
+ Returns:
+ The quantized model.
+ """
+
+ def __init__(
+ self,
+ model,
+ tokenizer=None,
+ bits: int = 4,
+ group_size: int = 128,
+ scheme: str = "asym",
+ weight_config: dict = {},
+ enable_full_range: bool = False,
+ bs: int = 8,
+ amp: bool = True,
+ device="cuda:0",
+ lr_scheduler=None,
+ dataloader=None,
+ dataset_name: str = "NeelNanda/pile-10k",
+ dataset_split: str = "train",
+ use_quant_input: bool = True,
+ enable_minmax_tuning: bool = True,
+ lr: float = 0.005,
+ minmax_lr: float = None,
+ low_gpu_mem_usage: bool = True,
+ iters: int = 200,
+ seqlen: int = 2048,
+ n_samples: int = 512,
+ sampler: str = "rand",
+ seed: int = 42,
+ n_blocks: int = 1,
+ gradient_accumulate_steps: int = 1,
+ not_use_mse: bool = False,
+ dynamic_max_gap: int = -1,
+ data_type: str = "int",
+ optimizer="AdamW",
+ **kwargs,
+ ):
+ super(AutoOPTRound, self).__init__(
+ model,
+ tokenizer,
+ bits,
+ group_size,
+ scheme,
+ weight_config,
+ enable_full_range,
+ bs,
+ amp,
+ device,
+ lr_scheduler,
+ dataloader,
+ dataset_name,
+ dataset_split,
+ use_quant_input,
+ enable_minmax_tuning,
+ lr,
+ minmax_lr,
+ low_gpu_mem_usage,
+ iters,
+ seqlen,
+ n_samples,
+ sampler,
+ seed,
+ n_blocks,
+ gradient_accumulate_steps,
+ not_use_mse,
+ dynamic_max_gap,
+ data_type,
+ **kwargs,
+ )
+
+ self.optimizer = self.get_optimizer(optimizer)
+
+ def get_optimizer(self, optimizer):
+ if optimizer is None:
+ optimizer = torch.optim.AdamW
+ elif isinstance(optimizer, str):
+ optimizer = getattr(torch.optim, optimizer)
+ else:
+ optimizer = optimizer
+ return optimizer
+
+ def get_scaler(self):
+ scaler = None
+ if self.amp and not check_is_cpu(self.device):
+ from torch.cuda.amp import GradScaler
+
+ scaler = GradScaler(init_scale=1024, growth_interval=100000)
+ return scaler
+
+ def scale_loss_and_backward(self, scaler, loss):
+ if scaler is not None:
+ scale_loss = scaler.scale(loss)
+ scale_loss.backward()
+ return scale_loss
+ else:
+ loss.backward()
+ return loss
+
+ def step(self, scaler, optimizer, lr_schedule):
+ if scaler is not None:
+ scaler.step(optimizer)
+ optimizer.zero_grad()
+ lr_schedule.step()
+ scaler.update()
+ else:
+ optimizer.step()
+ optimizer.zero_grad()
+ lr_schedule.step()
+
+
+class AutoAdamRound(AutoOPTRound):
+ """Class for automatic rounding-based quantization with optimizers like adamw of a PyTorch model.
+ The default lr has been changed.
+
+ Args:
+ model: The PyTorch model to be quantized.
+ tokenizer: An optional tokenizer for processing input data.
+ bits (int): Number of bits for quantization (default is 4).
+ group_size (int): Size of the quantization group (default is 128).
+ scheme (str): The quantization scheme to be used (default is "asym").
+ weight_config (dict): Configuration for weight quantization (default is an empty dictionary).
+ enable_full_range (bool): Whether to enable full range quantization (default is False).
+ bs (int): Batch size for training (default is 8).
+ amp (bool): Whether to use automatic mixed precision (default is True).
+ device: The device to be used for training (default is "cuda:0").
+ lr_scheduler: The learning rate scheduler to be used.
+ dataloader: The dataloader for input data (to be supported in future).
+ dataset_name (str): The default dataset name (default is "NeelNanda/pile-10k").
+ dataset_split (str): The split of the dataset to be used (default is "train").
+ use_quant_input (bool): Whether to use quantized input data (default is True).
+ enable_minmax_tuning (bool): Whether to enable min-max tuning (default is True).
+ lr (float): The learning rate (default is 0.005).
+ minmax_lr (float): The learning rate for min-max tuning (default is None).
+ low_gpu_mem_usage (bool): Whether to use low GPU memory (default is True).
+ iters (int): Number of iterations (default is 200).
+ seqlen (int): Length of the sequence.
+ n_samples (int): Number of samples (default is 512).
+ sampler (str): The sampling method (default is "rand").
+ seed (int): The random seed (default is 42).
+ n_blocks (int): Number of blocks (default is 1).
+ gradient_accumulate_steps (int): Number of gradient accumulation steps (default is 1).
+ not_use_mse (bool): Whether to use mean squared error (default is False).
+ dynamic_max_gap (int): The dynamic maximum gap (default is -1).
+ data_type (str): The data type to be used (default is "int").
+ optimizer: string or object
+ **kwargs: Additional keyword arguments.
+
+ Returns:
+ The quantized model.
+ """
+
+ def __init__(
+ self,
+ model,
+ tokenizer=None,
+ bits: int = 4,
+ group_size: int = 128,
+ scheme: str = "asym",
+ weight_config: dict = {},
+ enable_full_range: bool = False,
+ bs: int = 8,
+ amp: bool = True,
+ device="cuda:0",
+ lr_scheduler=None,
+ dataloader=None,
+ dataset_name: str = "NeelNanda/pile-10k",
+ dataset_split: str = "train",
+ use_quant_input: bool = True,
+ enable_minmax_tuning: bool = True,
+ lr: float = 0.01,
+ minmax_lr: float = None,
+ low_gpu_mem_usage: bool = True,
+ iters: int = 200,
+ seqlen: int = 2048,
+ n_samples: int = 512,
+ sampler: str = "rand",
+ seed: int = 42,
+ n_blocks: int = 1,
+ gradient_accumulate_steps: int = 1,
+ not_use_mse: bool = False,
+ dynamic_max_gap: int = -1,
+ data_type: str = "int",
+ optimizer="AdamW",
+ **kwargs,
+ ):
+ super(AutoAdamRound, self).__init__(
+ model,
+ tokenizer,
+ bits,
+ group_size,
+ scheme,
+ weight_config,
+ enable_full_range,
+ bs,
+ amp,
+ device,
+ lr_scheduler,
+ dataloader,
+ dataset_name,
+ dataset_split,
+ use_quant_input,
+ enable_minmax_tuning,
+ lr,
+ minmax_lr,
+ low_gpu_mem_usage,
+ iters,
+ seqlen,
+ n_samples,
+ sampler,
+ seed,
+ n_blocks,
+ gradient_accumulate_steps,
+ not_use_mse,
+ dynamic_max_gap,
+ data_type,
+ optimizer,
+ **kwargs,
+ )
diff --git a/autoround/src/sign_sgd.py b/autoround/src/sign_sgd.py
new file mode 100644
index 000000000..851378a7b
--- /dev/null
+++ b/autoround/src/sign_sgd.py
@@ -0,0 +1,389 @@
+#
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import List, Optional
+
+# From PyTorch:
+#
+# Copyright (c) 2016- Facebook, Inc (Adam Paszke)
+# Copyright (c) 2014- Facebook, Inc (Soumith Chintala)
+# Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert)
+# Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu)
+# Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu)
+# Copyright (c) 2011-2013 NYU (Clement Farabet)
+# Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston)
+# Copyright (c) 2006 Idiap Research Institute (Samy Bengio)
+# Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz)
+#
+# From Caffe2:
+#
+# Copyright (c) 2016-present, Facebook Inc. All rights reserved.
+#
+# All contributions by Facebook:
+# Copyright (c) 2016 Facebook Inc.
+#
+# All contributions by Google:
+# Copyright (c) 2015 Google Inc.
+# All rights reserved.
+#
+# All contributions by Yangqing Jia:
+# Copyright (c) 2015 Yangqing Jia
+# All rights reserved.
+#
+# All contributions by Kakao Brain:
+# Copyright 2019-2020 Kakao Brain
+#
+# All contributions by Cruise LLC:
+# Copyright (c) 2022 Cruise LLC.
+# All rights reserved.
+#
+# All contributions from Caffe:
+# Copyright(c) 2013, 2014, 2015, the respective contributors
+# All rights reserved.
+#
+# All other contributions:
+# Copyright(c) 2015, 2016 the respective contributors
+# All rights reserved.
+#
+# Caffe2 uses a copyright model similar to Caffe: each contributor holds
+# copyright over their contributions to Caffe2. The project versioning records
+# all such contribution and copyright details. If a contributor wants to further
+# mark their specific copyright on a particular contribution, they should
+# indicate their copyright solely in the commit message of the change when it is
+# committed.
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America
+# and IDIAP Research Institute nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+import torch
+from torch import Tensor
+from torch.optim.optimizer import Optimizer
+
+__all__ = ["SGD", "sgd"]
+
+
+class _RequiredParameter(object):
+ """Singleton class representing a required parameter for an Optimizer."""
+
+ def __repr__(self):
+ return "<required parameter>"
+
+
+required = _RequiredParameter()
+
+
+def _use_grad_for_differentiable(func):
+ def _use_grad(self, *args, **kwargs):
+ prev_grad = torch.is_grad_enabled()
+ try:
+ torch.set_grad_enabled(self.defaults["differentiable"])
+ ret = func(self, *args, **kwargs)
+ finally:
+ torch.set_grad_enabled(prev_grad)
+ return ret
+
+ return _use_grad
+
+
+class SGD(Optimizer):
+ r"""Implements stochastic gradient descent (optionally with momentum).
+
+ .. math::
+ \begin{aligned}
+ &\rule{110mm}{0.4pt} \\
+ &\textbf{input} : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta)
+ \text{ (objective)}, \: \lambda \text{ (weight decay)}, \\
+ &\hspace{13mm} \:\mu \text{ (momentum)}, \:\tau \text{ (dampening)},
+ \:\textit{ nesterov,}\:\textit{ maximize} \\[-1.ex]
+ &\rule{110mm}{0.4pt} \\
+ &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\
+ &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\
+ &\hspace{5mm}\textbf{if} \: \lambda \neq 0 \\
+ &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\
+ &\hspace{5mm}\textbf{if} \: \mu \neq 0 \\
+ &\hspace{10mm}\textbf{if} \: t > 1 \\
+ &\hspace{15mm} \textbf{b}_t \leftarrow \mu \textbf{b}_{t-1} + (1-\tau) g_t \\
+ &\hspace{10mm}\textbf{else} \\
+ &\hspace{15mm} \textbf{b}_t \leftarrow g_t \\
+ &\hspace{10mm}\textbf{if} \: \textit{nesterov} \\
+ &\hspace{15mm} g_t \leftarrow g_{t} + \mu \textbf{b}_t \\
+ &\hspace{10mm}\textbf{else} \\[-1.ex]
+ &\hspace{15mm} g_t \leftarrow \textbf{b}_t \\
+ &\hspace{5mm}\textbf{if} \: \textit{maximize} \\
+ &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} + \gamma g_t \\[-1.ex]
+ &\hspace{5mm}\textbf{else} \\[-1.ex]
+ &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - \gamma g_t \\[-1.ex]
+ &\rule{110mm}{0.4pt} \\[-1.ex]
+ &\bf{return} \: \theta_t \\[-1.ex]
+ &\rule{110mm}{0.4pt} \\[-1.ex]
+ \end{aligned}
+
+ Nesterov momentum is based on the formula from
+ `On the importance of initialization and momentum in deep learning`__.
+
+ Args:
+ params (iterable): iterable of parameters to optimize or dicts defining
+ parameter groups
+ lr (float): learning rate
+ momentum (float, optional): momentum factor (default: 0)
+ weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
+ dampening (float, optional): dampening for momentum (default: 0)
+ nesterov (bool, optional): enables Nesterov momentum (default: False)
+ maximize (bool, optional): maximize the params based on the objective, instead of
+ minimizing (default: False)
+ foreach (bool, optional): whether foreach implementation of optimizer
+ is used (default: None)
+
+ Example:
+ >>> # xdoctest: +SKIP
+ >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
+ >>> optimizer.zero_grad()
+ >>> loss_fn(model(input), target).backward()
+ >>> optimizer.step()
+
+ __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf
+
+ .. note::
+ The implementation of SGD with Momentum/Nesterov subtly differs from
+ Sutskever et. al. and implementations in some other frameworks.
+
+ Considering the specific case of Momentum, the update can be written as
+
+ .. math::
+ \begin{aligned}
+ v_{t+1} & = \mu * v_{t} + g_{t+1}, \\
+ p_{t+1} & = p_{t} - \text{lr} * v_{t+1},
+ \end{aligned}
+
+ where :math:`p`, :math:`g`, :math:`v` and :math:`\mu` denote the
+ parameters, gradient, velocity, and momentum respectively.
+
+ This is in contrast to Sutskever et. al. and
+ other frameworks which employ an update of the form
+
+ .. math::
+ \begin{aligned}
+ v_{t+1} & = \mu * v_{t} + \text{lr} * g_{t+1}, \\
+ p_{t+1} & = p_{t} - v_{t+1}.
+ \end{aligned}
+
+ The Nesterov version is analogously modified.
+ """
+
+ def __init__(
+ self,
+ params,
+ lr=required,
+ momentum=0,
+ dampening=0,
+ weight_decay=0,
+ nesterov=False,
+ *,
+ maximize=False,
+ foreach: Optional[bool] = None,
+ differentiable=False
+ ):
+ if lr is not required and lr < 0.0:
+ raise ValueError("Invalid learning rate: {}".format(lr))
+ if momentum < 0.0:
+ raise ValueError("Invalid momentum value: {}".format(momentum))
+ if weight_decay < 0.0:
+ raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
+
+ defaults = dict(
+ lr=lr,
+ momentum=momentum,
+ dampening=dampening,
+ weight_decay=weight_decay,
+ nesterov=nesterov,
+ maximize=maximize,
+ foreach=foreach,
+ differentiable=differentiable,
+ )
+ if nesterov and (momentum <= 0 or dampening != 0):
+ raise ValueError("Nesterov momentum requires a momentum and zero dampening")
+ super(SGD, self).__init__(params, defaults)
+
+ def __setstate__(self, state):
+ super().__setstate__(state)
+ for group in self.param_groups:
+ group.setdefault("nesterov", False)
+ group.setdefault("maximize", False)
+ group.setdefault("foreach", None)
+ group.setdefault("differentiable", False)
+
+ @_use_grad_for_differentiable
+ def step(self, closure=None):
+ """Performs a single optimization step.
+
+ Args:
+ closure (Callable, optional): A closure that reevaluates the model
+ and returns the loss.
+ """
+ loss = None
+ if closure is not None:
+ with torch.enable_grad():
+ loss = closure()
+
+ for group in self.param_groups:
+ params_with_grad = []
+ d_p_list = []
+ momentum_buffer_list = []
+ has_sparse_grad = False
+
+ for p in group["params"]:
+ if p.grad is not None:
+ params_with_grad.append(p)
+ d_p_list.append(p.grad)
+ if p.grad.is_sparse:
+ has_sparse_grad = True
+
+ state = self.state[p]
+ if "momentum_buffer" not in state:
+ momentum_buffer_list.append(None)
+ else:
+ momentum_buffer_list.append(state["momentum_buffer"])
+
+ sgd(
+ params_with_grad,
+ d_p_list,
+ momentum_buffer_list,
+ weight_decay=group["weight_decay"],
+ momentum=group["momentum"],
+ lr=group["lr"],
+ dampening=group["dampening"],
+ nesterov=group["nesterov"],
+ maximize=group["maximize"],
+ has_sparse_grad=has_sparse_grad,
+ foreach=group["foreach"],
+ )
+
+ # update momentum_buffers in state
+ for p, momentum_buffer in zip(params_with_grad, momentum_buffer_list):
+ state = self.state[p]
+ state["momentum_buffer"] = momentum_buffer
+
+ return loss
+
+
+def sgd(
+ params: List[Tensor],
+ d_p_list: List[Tensor],
+ momentum_buffer_list: List[Optional[Tensor]],
+ # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627
+ # setting this as kwarg for now as functional API is compiled by torch/distributed/optim
+ has_sparse_grad: bool = None,
+ foreach: bool = None,
+ *,
+ weight_decay: float,
+ momentum: float,
+ lr: float,
+ dampening: float,
+ nesterov: bool,
+ maximize: bool
+):
+ r"""Functional API that performs SGD algorithm computation.
+
+ See :class:`~torch.optim.SGD` for details.
+ """
+
+ if foreach is None:
+ # Placeholder for more complex foreach logic to be added when value is not set
+ foreach = False
+
+ if foreach and torch.jit.is_scripting():
+ raise RuntimeError("torch.jit.script not supported with foreach optimizers")
+
+ # if foreach and not torch.jit.is_scripting():
+ # func = _multi_tensor_sgd
+ # else:
+ func = _single_tensor_sgd
+
+ func(
+ params,
+ d_p_list,
+ momentum_buffer_list,
+ weight_decay=weight_decay,
+ momentum=momentum,
+ lr=lr,
+ dampening=dampening,
+ nesterov=nesterov,
+ has_sparse_grad=has_sparse_grad,
+ maximize=maximize,
+ )
+
+
+def _single_tensor_sgd(
+ params: List[Tensor],
+ d_p_list: List[Tensor],
+ momentum_buffer_list: List[Optional[Tensor]],
+ *,
+ weight_decay: float,
+ momentum: float,
+ lr: float,
+ dampening: float,
+ nesterov: bool,
+ maximize: bool,
+ has_sparse_grad: bool
+):
+ for i, param in enumerate(params):
+ d_p = d_p_list[i] if not maximize else -d_p_list[i]
+
+ if weight_decay != 0:
+ d_p = d_p.add(param, alpha=weight_decay)
+
+ if momentum != 0:
+ buf = momentum_buffer_list[i]
+
+ if buf is None:
+ buf = torch.clone(d_p).detach()
+ momentum_buffer_list[i] = buf
+ else:
+ buf.mul_(momentum).add_(d_p, alpha=1 - dampening)
+
+ if nesterov:
+ d_p = d_p.add(buf, alpha=momentum)
+ else:
+ d_p = buf
+
+ param.add_(torch.sign(d_p), alpha=-lr)