PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enable phi3v tuning (#197)
WeiweiZhang1f2fef1309b57
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex a1f18cd4b..65125c7d1 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -325,7 +325,6 @@ class AutoRoundQuantizer(HfQuantizer):"""from auto_round.utils import get_layer_names_in_block- layer_names = get_layer_names_in_block(model)quantization_config = model.config.quantization_configif hasattr(quantization_config, "backend"): # pragma: no coverbackend = quantization_config.backend@@ -337,6 +336,9 @@ class AutoRoundQuantizer(HfQuantizer):data_type = quantization_config.data_type if hasattr(quantization_config, "data_type") \else "int" # pragma: no coversym = quantization_config.sym+ quant_block_list = quantization_config.quant_block_list \+ if hasattr(quantization_config, "quant_block_list") else None+ layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)extra_config = {}if hasattr(quantization_config, "extra_config"):extra_config = quantization_config.extra_config@@ -482,3 +484,4 @@ if transformers_version[0] == 4 and transformers_version[1] < 38:transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizertransformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer+diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 43766ad23..9a232dc66 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1124,6 +1124,7 @@ class AutoRound(object):"amp","nsamples","low_gpu_mem_usage",+ "quant_block_list","enable_norm_bias_tuning"]if isinstance(self.dataset, str):@@ -1544,3 +1545,4 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)+diff --git a/examples/multimodal-modeling/Phi-3-vision/README.md b/examples/multimodal-modeling/Phi-3-vision/README.mdnew file mode 100644index 000000000..3105c051b--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/README.md@@ -0,0 +1,74 @@+Step-by-Step+============+transformers>=4.41.0+This document presents step-by-step instructions for auto-round.+# Run Quantization on Phi-3-vision Models++In this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as Phi-3-vision.++## Download the calibration data++Our calibration process resembles the official visual instruction tuning process.++Please download the annotation of the final mixture our instruction tuning data [llava_v1_5_mix665k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/llava_v1_5_mix665k.json), and download the images from constituting datasets:++COCO: [train2017](http://images.cocodataset.org/zips/train2017.zip), and unzip the image folder to any directory you desire.+++## 2. Run Examples+Enter into the examples folder and install lm-eval to run the evaluation+```bash+pip install -r requirements.txt+```++- **Default Settings:**+```bash+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name microsoft/Phi-3-vision-128k-instruct --bits 4 --group_size 128+```++- **Speedup the tuning:**++disable_low_gpu_mem_usage(more gpu memory)++reduce the seqlen to 512(potential large accuracy drop)++or combine them++- **Enable quantized lm-head:**++Currently only support in Intel xpu and AutoRound format, however, we found the fake tuning could improve the accuracy is some scenarios. Disable --low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing nsamples,e.g. 128, to alleviate the issue.+```bash+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name microsoft/Phi-3-vision-128k-instruct --bits 4 --group_size 128 --quant_lm_head+```++- **Utilizing the AdamW Optimizer:**++Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.++- **Running on Intel Gaudi2**+```bash+bash run_autoround_on_gaudi.sh+```+++## 3. Environment++PyTorch 1.8 or higher version is needed+++## Reference+If you find SignRound useful for your research, please cite our paper:+```bash+@article{cheng2023optimize,+ title={Optimize Weight Rounding via Signed Gradient Descent for the Quantization of LLMs},+ author={Cheng, Wenhua and Zhang, Weiwei and Shen, Haihao and Cai, Yiyang and He, Xin and Lv, Kaokao},+ journal={arXiv preprint arXiv:2309.05516},+ year={2023}+}+```++++++diff --git a/examples/multimodal-modeling/Phi-3-vision/eval_042/__init__.py b/examples/multimodal-modeling/Phi-3-vision/eval_042/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.pynew file mode 100644index 000000000..e9e63fef9--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py@@ -0,0 +1,616 @@+import itertools+import logging+import random+import time+from collections import defaultdict+from typing import TYPE_CHECKING, List, Optional, Union++import numpy as np+import torch++import lm_eval.api.metrics+import lm_eval.api.registry+import lm_eval.models+from lm_eval.caching.cache import delete_cache+from lm_eval.evaluator_utils import (+ consolidate_results,+ get_sample_size,+ get_task_list,+ prepare_print_tasks,+ print_writeout,+ run_task_tests,+)+from lm_eval.logging_utils import add_env_info, get_git_commit_hash+from lm_eval.tasks import TaskManager, get_task_dict+from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string++if TYPE_CHECKING:+ from lm_eval.api.model import LM+ from lm_eval.tasks import Task+++@positional_deprecated+def simple_evaluate(+ model,+ model_args: Optional[Union[str, dict]] = None,+ tasks: Optional[List[Union[str, dict, object]]] = None,+ num_fewshot: Optional[int] = None,+ batch_size: Optional[int] = None,+ max_batch_size: Optional[int] = None,+ device: Optional[str] = None,+ use_cache: Optional[str] = None,+ cache_requests: bool = False,+ rewrite_requests_cache: bool = False,+ delete_requests_cache: bool = False,+ limit: Optional[Union[int, float]] = None,+ bootstrap_iters: int = 100000,+ check_integrity: bool = False,+ write_out: bool = False,+ log_samples: bool = True,+ gen_kwargs: Optional[str] = None,+ task_manager: Optional[TaskManager] = None,+ verbosity: str = "INFO",+ predict_only: bool = False,+ random_seed: int = 0,+ numpy_random_seed: int = 1234,+ torch_random_seed: int = 1234,+ user_model = None, ##user model does not support tensor parallelism+):+ """Instantiate and evaluate a model on a list of tasks.++ :param model: Union[str, LM]+ Name of model or LM object, see lm_eval.models.get_model+ :param model_args: Optional[str, dict]+ String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.+ Ignored if `model` argument is a LM object.+ :param tasks: list[Union[str, dict, Task]]+ List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.+ :param num_fewshot: int+ Number of examples in few-shot context+ :param batch_size: int or str, optional+ Batch size for model+ :param max_batch_size: int, optional+ Maximal batch size to try with automatic batch size detection+ :param device: str, optional+ PyTorch device (e.g. "cpu" or "cuda:0") for running models+ :param use_cache: str, optional+ A path to a sqlite db file for caching model responses. `None` if not caching.+ :param cache_requests: bool, optional+ Speed up evaluation by caching the building of dataset requests. `None` if not caching.+ :param rewrite_requests_cache: bool, optional+ Rewrites all of the request cache if set to `True`. `None` if not desired.+ :param delete_requests_cache: bool, optional+ Deletes all of the request cache if set to `True`. `None` if not desired.+ :param limit: int or float, optional+ Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.+ :param bootstrap_iters:+ Number of iterations for bootstrap statistics+ :param check_integrity: bool+ Whether to run the relevant part of the test suite for the tasks+ :param write_out: bool+ If True, write out an example document and model input for checking task integrity+ :param log_samples: bool+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis+ :param gen_kwargs: str+ String arguments for model generation+ Ignored for all tasks with loglikelihood output_type+ :param predict_only: bool+ If true only model outputs will be generated and returned. Metrics will not be evaluated+ :param random_seed: int+ Random seed for python's random module. If set to None, the seed will not be set.+ :param numpy_random_seed: int+ Random seed for numpy. If set to None, the seed will not be set.+ :param torch_random_seed: int+ Random seed for torch. If set to None, the seed will not be set.++ :return+ Dictionary of results+ """+ from auto_round.auto_quantizer import AutoHfQuantizer+ eval_logger.setLevel(getattr(logging, f"{verbosity}"))+ start_date = time.time()++ if delete_requests_cache:+ eval_logger.info("Deleting requests cache...")+ delete_cache()++ seed_message = []+ if random_seed is not None:+ # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412+ seed_message.append(f"Setting random seed to {random_seed}")+ random.seed(random_seed)++ if numpy_random_seed is not None:+ seed_message.append(f"Setting numpy seed to {numpy_random_seed}")+ np.random.seed(numpy_random_seed)++ if torch_random_seed is not None:+ seed_message.append(f"Setting torch manual seed to {torch_random_seed}")+ torch.manual_seed(torch_random_seed)++ if seed_message:+ eval_logger.info(" | ".join(seed_message))++ if tasks is None:+ tasks = []+ if len(tasks) == 0:+ raise ValueError(+ "No tasks specified, or no tasks found. Please verify the task names."+ )++ if gen_kwargs is not None:+ gen_kwargs = simple_parse_args_string(gen_kwargs)+ eval_logger.warning(+ "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "+ "Ensure 'do_sample=True' for non-greedy decoding!"+ )+ if gen_kwargs == "":+ gen_kwargs = None++ if isinstance(model, str):+ if model_args is None:+ model_args = ""++ if isinstance(model_args, dict):+ lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(+ model_args,+ {+ "batch_size": batch_size,+ "max_batch_size": max_batch_size,+ "device": device,+ },+ )++ else:+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(+ model_args,+ {+ "batch_size": batch_size,+ "max_batch_size": max_batch_size,+ "device": device,+ },+ )+ else:+ if not isinstance(model, lm_eval.api.model.LM):+ raise TypeError+ lm = model++ if use_cache is not None:+ eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")+ lm = lm_eval.api.model.CachingLM(+ lm,+ use_cache+ # each rank receives a different cache db.+ # necessary to avoid multiple writes to cache at once+ + "_rank"+ + str(lm.rank)+ + ".db",+ )+ if user_model is not None:+ lm._model = user_model++ if task_manager is None:+ task_manager = TaskManager(verbosity)++ task_dict = get_task_dict(tasks, task_manager)+ for task_name in task_dict.keys():+ task_obj = task_dict[task_name]+ if isinstance(task_obj, tuple):+ _, task_obj = task_obj+ if task_obj is None:+ continue++ if task_obj.get_config("output_type") == "generate_until":+ if gen_kwargs is not None:+ task_obj.set_config(+ key="generation_kwargs", value=gen_kwargs, update=True+ )++ if predict_only:+ log_samples = True+ eval_logger.info(+ f"Processing {task_name} in output-only mode. Metrics will not be calculated!"+ )+ # we have to change the class properties post-hoc. This is pretty hacky.+ task_obj.override_metric(metric_name="bypass")++ # override tasks' fewshot values to the provided num_fewshot arg value+ # except if tasks have it set to 0 manually in their configs--then we should never overwrite that+ if num_fewshot is not None:+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:+ eval_logger.info(+ f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."+ )+ else:+ eval_logger.warning(+ f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"+ )+ task_obj.set_config(key="num_fewshot", value=num_fewshot)+ else:+ # if num_fewshot not provided, and the task does not define a default one, default to 0+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None:+ task_obj.set_config(key="num_fewshot", value=0)++ if check_integrity:+ run_task_tests(task_list=tasks)++ results = evaluate(+ lm=lm,+ task_dict=task_dict,+ limit=limit,+ cache_requests=cache_requests,+ rewrite_requests_cache=rewrite_requests_cache,+ bootstrap_iters=bootstrap_iters,+ write_out=write_out,+ log_samples=log_samples,+ verbosity=verbosity,+ )++ if lm.rank == 0:+ if isinstance(model, str):+ model_name = model+ elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):+ model_name = model.config._name_or_path+ else:+ model_name = type(model).__name__++ # add info about the model and few shot config+ results["config"] = {+ "model": model_name,+ "model_args": model_args,+ "batch_size": batch_size,+ "batch_sizes": (+ list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []+ ),+ "device": device,+ "use_cache": use_cache,+ "limit": limit,+ "bootstrap_iters": bootstrap_iters,+ "gen_kwargs": gen_kwargs,+ }+ results["git_hash"] = get_git_commit_hash()+ results["date"] = start_date+ add_env_info(results) # additional environment info to results+ return results+ else:+ return None+++@positional_deprecated+def evaluate(+ lm: "LM",+ task_dict,+ limit: Optional[int] = None,+ cache_requests: bool = False,+ rewrite_requests_cache: bool = False,+ bootstrap_iters: Optional[int] = 100000,+ write_out: bool = False,+ log_samples: bool = True,+ verbosity: str = "INFO",+):+ """Instantiate and evaluate a model on a list of tasks.++ :param lm: obj+ Language Model+ :param task_dict: dict[str, Task]+ Dictionary of tasks. Tasks will be taken to have name type(task).config.task .+ :param limit: int, optional+ Limit the number of examples per task (only use this for testing)+ :param bootstrap_iters:+ Number of iterations for bootstrap statistics+ :param write_out: bool+ If True, write out an example document and model input for checking task integrity+ :param log_samples: bool+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis+ :return+ Dictionary of results+ """++ eval_logger.setLevel(getattr(logging, f"{verbosity}"))++ # tracks all Instances/requests a model must generate output on.+ requests = defaultdict(list)+ # stores the amount to pad out reqs per req. type so that+ # number of fwd passes per distributed rank is equal+ padding_requests = defaultdict(int)++ # get lists of group hierarchy and each type of request+ task_hierarchy, eval_tasks = get_task_list(task_dict)+ if not log_samples:+ if not all(+ "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()+ for task_output in eval_tasks+ ):+ raise ValueError("log_samples must be True for 'bypass' metric-only tasks")+ for task_output in eval_tasks:+ task: Task = task_output.task+ limit = get_sample_size(task, limit)+ task.build_all_requests(+ limit=limit,+ rank=lm.rank,+ world_size=lm.world_size,+ cache_requests=cache_requests,+ rewrite_requests_cache=rewrite_requests_cache,+ )+ eval_logger.debug(+ f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"+ )++ if write_out:+ print_writeout(task)+ # aggregate Instances by LM method requested to get output.+ for instance in task.instances:+ reqtype = instance.request_type+ requests[reqtype].append(instance)++ if lm.world_size > 1:+ instances_rnk = torch.tensor(len(task._instances), device=lm.device)+ gathered_item = (+ lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()+ )+ # "multiple_choice" task types dispatch (several) "loglikelihood" request types+ reqtype = (+ "loglikelihood"+ if task.OUTPUT_TYPE == "multiple_choice"+ else task.OUTPUT_TYPE+ )+ # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)+ numpad = max(gathered_item) - gathered_item[lm.rank]+ # todo: may not account for padding in cases like SquadV2 which has multiple req types+ padding_requests[reqtype] += numpad++ ### Run LM on inputs, get all outputs ###+ # execute each type of request+ for reqtype, reqs in requests.items():+ eval_logger.info(f"Running {reqtype} requests")+ # create `K` copies of each request `req` based off `K = req.repeats`+ cloned_reqs = []+ for req in reqs:+ cloned_reqs.extend([req] * req.repeats)++ if (lm.world_size > 1) and (padding_requests[reqtype] > 0):+ for _ in range(padding_requests[reqtype]):+ cloned_reqs.extend([req] * req.repeats)++ # run requests through model+ resps = getattr(lm, reqtype)(cloned_reqs)++ # put responses from model into a list of length K for each request.+ for x, req in zip(resps, cloned_reqs):+ req.resps.append(x)++ if lm.world_size > 1:+ lm.accelerator.wait_for_everyone()++ RANK = lm.rank+ WORLD_SIZE = lm.world_size+ ### Postprocess outputs ###+ # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)+ for task_output in eval_tasks:+ task = task_output.task+ task.apply_filters()++ ### Collect values of metrics on all datapoints ###+ # # unpack results and sort back in order and return control to Task+ # TODO: make it possible to use a different metric per filter+ # Pre-process task.instances to group by doc_id+ instances_by_doc_id = defaultdict(list)+ for instance in task.instances:+ instances_by_doc_id[instance.doc_id].append(instance)+ # Sort instances within each group+ for instances in instances_by_doc_id.values():+ instances.sort(key=lambda x: x.idx)+ # iterate over different filters used+ for filter_key in task.instances[0].filtered_resps.keys():+ doc_iterator = task.doc_iterator(+ rank=RANK, limit=limit, world_size=WORLD_SIZE+ )+ for doc_id, doc in doc_iterator:+ requests = instances_by_doc_id[doc_id]+ metrics = task.process_results(+ doc, [req.filtered_resps[filter_key] for req in requests]+ )+ if log_samples:+ target = task.doc_to_target(doc)+ example = {+ "doc_id": doc_id,+ "doc": doc,+ "target": target,+ "arguments": [req.args for req in requests],+ "resps": [req.resps for req in requests],+ "filtered_resps": [+ req.filtered_resps[filter_key] for req in requests+ ],+ }+ example.update(metrics)+ task_output.logged_samples.append(example)+ for metric, value in metrics.items():+ task_output.sample_metrics[(metric, filter_key)].append(value)++ if WORLD_SIZE > 1:+ # if multigpu, then gather data across all ranks to rank 0+ # first gather logged samples across all ranks+ for task_output in eval_tasks:+ if log_samples:+ # for task_name, task_samples in list(samples.items()):+ full_samples = [None] * WORLD_SIZE if RANK == 0 else None+ torch.distributed.gather_object(+ obj=task_output.logged_samples,+ object_gather_list=full_samples,+ dst=0,+ )++ if RANK == 0:+ task_output.logged_samples = list(+ itertools.chain.from_iterable(full_samples)+ )++ # then collect metrics across all ranks+ for metrics in task_output.sample_metrics:+ metric_list = [None] * WORLD_SIZE if RANK == 0 else None+ torch.distributed.gather_object(+ obj=task_output.sample_metrics[metrics],+ object_gather_list=metric_list,+ dst=0,+ )+ if RANK == 0:+ task_output.sample_metrics[metrics] = list(+ itertools.chain.from_iterable(metric_list)+ )++ if RANK == 0:+ ### Aggregate results over all datapoints ###+ # aggregate results ; run bootstrap CIs+ for task_output in eval_tasks:+ task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)+ results, samples, configs, versions, num_fewshot = consolidate_results(+ eval_tasks+ )++ ### Calculate group metrics ###+ if bool(results):+ for group, task_list in reversed(task_hierarchy.items()):+ if len(task_list) == 0:+ # task_hierarchy entries are either+ # `group_name: [subtask1, subtask2, ...]`+ # or `task_name: []`.+ # we only want to operate on groups here.+ continue+ metric_list = list(+ {+ key+ for task in task_list+ for key in results[task].keys()+ if "_stderr" not in key and key not in ["alias", "samples"]+ }+ )+ for metric in metric_list:+ stderr = "_stderr,".join(metric.split(","))++ # gather metrics, sizes, and stderrs from subtasks+ metrics = [+ results[task][metric]+ for task in task_list+ if metric in results[task]+ ] # TODO: copy?+ stderrs = [+ results[task][stderr]+ for task in task_list+ if stderr in results[task]+ ]+ sizes = [+ results[task]["samples"]+ for task in task_list+ if metric in results[task]+ ]++ # compute group's pooled metric and stderr+ results[group][+ metric+ ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes)+ # TODO: calculate grouped metric using aggregation fn+ if "N/A" in stderrs:+ results[group][stderr] = "N/A"+ else:+ results[group][+ stderr+ ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)+ # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility+ # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line:+ # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics)++ results[group]["samples"] = sum(sizes)++ results_agg = defaultdict(dict)+ groups_agg = defaultdict(dict)+ all_tasks_list = list(task_hierarchy.keys())+ while True:+ add_tasks_list = list(k for k in results_agg.keys())+ left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list)))+ if len(left_tasks_list) == 0:+ break++ _task_hierarchy = {+ k: v for k, v in task_hierarchy.items() if k in left_tasks_list+ }+ _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results)++ results_agg = {**results_agg, **_results_agg}+ groups_agg = {**groups_agg, **_groups_agg}++ for group_name, task_list in task_hierarchy.items():+ if task_list:+ num_fewshot[group_name] = num_fewshot[+ task_list[0]+ ] # TODO: validate this++ results_dict = {+ "results": dict(results_agg.items()),+ **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),+ "group_subtasks": dict(reversed(task_hierarchy.items())),+ "configs": dict(sorted(configs.items())),+ "versions": dict(sorted(versions.items())),+ "n-shot": dict(sorted(num_fewshot.items())),+ }+ if log_samples:+ results_dict["samples"] = dict(samples)++ return results_dict++ else:+ return None+++if __name__ == "__main__":++ import sys++ sys.path.insert(0, '../../../')+ import time+ import argparse++ parser = argparse.ArgumentParser()+ parser.add_argument(+ "--model_name", default="/models/opt-125m/"+ )+ parser.add_argument(+ "--eval_bs", default=1,+ )+ parser.add_argument(+ "--trust_remote_code", action='store_true',+ help="Whether to enable trust_remote_code"+ )+ parser.add_argument("--tasks",+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \+ "openbookqa,boolq,rte,arc_easy,arc_challenge",+ help="lm-eval tasks for lm_eval version 0.4.2")++ args = parser.parse_args()+ s = time.time()+ from transformers import AutoConfig++ config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)+ model_args = f"pretrained={args.model_name}"+ if hasattr(config, "quantization_config"):+ quantization_config = config.quantization_config+ if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:+ from auto_round.auto_quantizer import AutoHfQuantizer+ else:+ from phi3_v import Phi3VGPTQForCausalLM+ model_args += f",autogptq=True,gptq_use_triton=True"+ if args.trust_remote_code:+ model_args += f",trust_remote_code=True"++ test_tasks = args.tasks+ if isinstance(test_tasks, str):+ test_tasks = test_tasks.split(',')+ model_name = args.model_name.rstrip('/')+ from lm_eval.utils import make_table+ result = simple_evaluate(model="hf",+ model_args=model_args,+ tasks=test_tasks,+ batch_size=args.eval_bs)+ print(make_table(result))++ print("cost time: ", time.time() - s)+diff --git a/examples/multimodal-modeling/Phi-3-vision/eval_042/phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/eval_042/phi3_v.pynew file mode 100644index 000000000..75366ea26--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/eval_042/phi3_v.py@@ -0,0 +1,21 @@+from auto_gptq.modeling._base import BaseGPTQForCausalLM+from auto_gptq.utils.import_utils import compare_transformers_version+from auto_gptq.modeling._const import SUPPORTED_MODELS+from auto_gptq.modeling.auto import GPTQ_CAUSAL_LM_MODEL_MAP++class Phi3VGPTQForCausalLM(BaseGPTQForCausalLM):+ layer_type = "Phi3DecoderLayer"+ layers_block_name = "model.layers"+ outside_layer_modules = ["model.embed_tokens", "model.final_layernorm"] # "model.vision_embed_tokens"?+ inside_layer_modules = [+ ["self_attn.o_proj"],+ ["self_attn.qkv_proj"],+ ["mlp.gate_up_proj"],+ ["mlp.down_proj"],+ ]++__all__ = ["Phi3VGPTQForCausalLM"]++if compare_transformers_version("v4.41.0", op="ge"):+ SUPPORTED_MODELS.append("phi3_v")+GPTQ_CAUSAL_LM_MODEL_MAP["phi3_v"] = Phi3VGPTQForCausalLMdiff --git a/examples/multimodal-modeling/Phi-3-vision/main.py b/examples/multimodal-modeling/Phi-3-vision/main.pynew file mode 100644index 000000000..ad9114e6f--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/main.py@@ -0,0 +1,466 @@+import argparse+import sys+sys.path.insert(0, '../../..')+parser = argparse.ArgumentParser()+import torch+import os+os.environ["TOKENIZERS_PARALLELISM"] = "false"+os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"+torch.use_deterministic_algorithms(True, warn_only=True)+import copy+from PIL import Image+import json+from torch.utils.data import Dataset, DataLoader+import torch+from typing import Dict, Optional, List, Union, Sequence+import transformers+from model.processing_phi3_v import Phi3VProcessor+from dataclasses import dataclass, field+from transformers import AutoModelForCausalLM, AutoConfig+from transformers.trainer_pt_utils import LabelSmoother++IGNORE_TOKEN_ID = LabelSmoother.ignore_index+import subprocess+LLaVA_IMAGE_TOKEN = "<image>"+DEFAULT_IMAGE_TOKEN = "<|image_1|>"+IMAGE_TOKEN_INDEX = -200+IGNORE_INDEX = -100++@dataclass+class DataArguments:+ data_path: str = field(+ default=None, metadata={"help": "Path to the training data."}+ )+ lazy_preprocess: bool = True+ is_multimodal: bool = True+ image_folder: Optional[str] = field(default=None)+ max_seq_length: int = field(+ default=2048, metadata={"help": "Maximum sequence length."}+ )+++def llava_to_openai(data):+ role_mapping = {"human": "user", "gpt": "assistant"}++ transformed_data = []+ for entry in data:+ transformed_entry = {+ "role": role_mapping.get(entry["from"], entry["from"]),+ "content": entry["value"].replace(LLaVA_IMAGE_TOKEN, DEFAULT_IMAGE_TOKEN),+ }+ transformed_data.append(transformed_entry)++ return transformed_data+++class LazySupervisedDataset(Dataset):+ """Dataset for supervised fine-tuning."""++ def __init__(+ self,+ data_path: Union[int, str],+ processor: transformers.PreTrainedTokenizer,+ data_args: DataArguments,+ padding=True,+ ):+ super(LazySupervisedDataset, self).__init__()+ if isinstance(data_path, str):+ list_data_dict = json.load(open(data_path, "r"))+ else:+ list_data_dict = data_path++ print("Formatting inputs...Skip in lazy mode")+ self.processor = processor+ self.list_data_dict = list_data_dict+ self.data_args = data_args+ self.padding = padding+ self.max_seq_length = data_args.max_seq_length++ def __len__(self):+ return len(self.list_data_dict)++ def __getitem__(self, i) -> Dict[str, torch.Tensor]:+ sources = self.list_data_dict[i]+ if isinstance(i, int):+ sources = [sources]+ assert len(sources) == 1, "Don't know why it is wrapped to a list" # FIXME+ processor = self.processor+ if "image" in sources[0]:+ image_file = os.path.basename(self.list_data_dict[i]["image"])+ image_folder = self.data_args.image_folder+ image_file = os.path.join(image_folder, image_file)+ image = [Image.open(image_file).convert("RGB")]+ else:+ image = None+ sources = copy.deepcopy([e["conversations"] for e in sources])+ for i in range(len(sources)):+ sources[i] = llava_to_openai(sources[i])++ prompt = processor.tokenizer.apply_chat_template(+ sources[0], tokenize=False, add_generation_prompt=True+ )+ data_dict = processor(prompt, image, return_tensors="pt")++ if self.padding:+ training_length = self.max_seq_length+ if 'pixel_values' not in data_dict:+ data_dict['pixel_values'] = torch.zeros([1, 17, 3, 336, 336], dtype=torch.bfloat16)+ data_dict['image_sizes'] = torch.zeros([1, 2], dtype=torch.int64)+ data_dict = dict(+ input_ids=data_dict["input_ids"][0],+ attention_mask=data_dict["attention_mask"][0],+ pixel_values=data_dict["pixel_values"][0],+ image_sizes=data_dict["image_sizes"][0],+ labels=data_dict["labels"][0],+ )+ return data_dict++@dataclass+class DataCollatorForSupervisedDataset(object):+ """Collate examples for supervised fine-tuning."""++ tokenizer: transformers.PreTrainedTokenizer++ def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:+ input_ids, labels = tuple([instance[key] for instance in instances]+ for key in ("input_ids", "labels"))+ input_ids = torch.nn.utils.rnn.pad_sequence(+ input_ids,+ batch_first=True,+ padding_value=self.tokenizer.pad_token_id)+ labels = torch.nn.utils.rnn.pad_sequence(+ labels,+ batch_first=True,+ padding_value=IGNORE_INDEX)+ input_ids = input_ids[:, :self.tokenizer.model_max_length]+ labels = labels[:, :self.tokenizer.model_max_length]++ pixel_values = [instance["pixel_values"] for instance in instances]+ pixel_values = torch.stack(pixel_values, dim=0)+ image_sizes = [instance["image_sizes"] for instance in instances]+ image_sizes = torch.stack(image_sizes, dim=0)++ batch = dict(+ input_ids=input_ids,+ labels=labels,+ pixel_values=pixel_values,+ image_sizes=image_sizes,+ attention_mask=input_ids.ne(self.tokenizer.pad_token_id),+ )+ return batch++def create_data_loader(dataset, batch_size=1, data_collator=None):+ assert batch_size == 1, "batch_size must be 1"+ data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, collate_fn=data_collator)+ return data_loader+++if __name__ == '__main__':++ parser.add_argument(+ "--model_name", default="microsoft/Phi-3-vision-128k-instruct"+ )++ parser.add_argument("--bits", default=4, type=int,+ help="number of bits")++ parser.add_argument("--group_size", default=128, type=int,+ help="group size")++ parser.add_argument("--train_bs", default=1, type=int,+ help="train batch size")++ parser.add_argument("--eval_bs", default=4, type=int,+ help="eval batch size")++ parser.add_argument("--device", default="auto", type=str,+ help="The device to be used for tuning. The default is set to auto/None,"+ "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")++ parser.add_argument("--sym", action='store_true',+ help=" sym quantization")++ parser.add_argument("--iters", default=200, type=int,+ help=" iters")++ parser.add_argument("--lr", default=None, type=float,+ help="learning rate, if None, it will be set to 1.0/iters automatically")++ parser.add_argument("--minmax_lr", default=None, type=float,+ help="minmax learning rate, if None,it will beset to be the same with lr")++ parser.add_argument("--seed", default=42, type=int,+ help="seed")++ parser.add_argument("--eval_fp16_baseline", action='store_true',+ help="whether to eval FP16 baseline")++ parser.add_argument("--adam", action='store_true',+ help="adam")++ parser.add_argument("--seqlen", default=2048, type=int,+ help="sequence length")++ parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")++ parser.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")++ parser.add_argument("--nsamples", default=512, type=int,+ help="number of samples")++ parser.add_argument("--low_gpu_mem_usage", action='store_true',+ help="low_gpu_mem_usage is deprecated")++ parser.add_argument("--deployment_device", default=None, type=str,+ help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'gpu' 'xpu' and 'auto_round'."+ "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")++ parser.add_argument("--scale_dtype", default='fp16',+ help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")++ parser.add_argument("--output_dir", default="./tmp_autoround", type=str,+ help="Where to store the final model.")++ parser.add_argument("--disable_eval", action='store_true',+ help="Whether to do lmeval evaluation.")++ parser.add_argument("--disable_amp", action='store_true',+ help="disable amp")++ parser.add_argument("--disable_minmax_tuning", action='store_true',+ help="whether disable enable weight minmax tuning")++ parser.add_argument("--disable_trust_remote_code", action='store_true',+ help="Whether to disable trust_remote_code")++ parser.add_argument("--disable_quanted_input", action='store_true',+ help="whether to disuse the output of quantized block to tune the next block")++ parser.add_argument("--quant_lm_head", action='store_true',+ help="quant_lm_head")++ parser.add_argument("--model_dtype", default=None, type=str,+ help="force to convert the dtype, some backends supports fp16 dtype better")++ parser.add_argument("--model_max_length", default=2048, type=int,+ help="")++ parser.add_argument("--act_bits", default=32, type=int,+ help="activation bits")++ parser.add_argument("--quant_vision", action='store_true',+ help="To determine whether the quantization should handle vision component.")++ parser.add_argument("--enable_safe_serialization", action='store_true',+ help="To determine whether the save_pretrained process should use safe_serialization.")++ # ========== Calibration Datasets =============+ parser.add_argument("--image_folder", default="coco", type=str,+ help="The dataset for quantization training. It can be a custom one.")++ parser.add_argument("--question_file", default=None, type=str,+ help="The dataset for quantization training. It can be a custom one.")++ # ================= Evaluation Related =====================+ parser.add_argument("--tasks", #wikitext+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",+ help="lm-eval tasks for lm_eval version 0.4")++ parser.add_argument("--eval-dataset", type=str, default="textvqa_val")++ args = parser.parse_args()++ tasks = args.tasks++ if args.deployment_device is None:+ args.deployment_device = "auto_round"++ if args.act_bits <= 8:+ print(+ "Warning, activation quantization is an experiment feature")++ if args.act_bits <= 8 and args.deployment_device != "fake":+ assert False, "only support fake mode for activation quantization currently"++ if "marlin" in args.deployment_device and args.sym == False:+ assert False, "marlin backend only supports sym quantization, please set --sym"++ model_name = args.model_name+ if model_name[-1] == "/":+ model_name = model_name[:-1]+ print(model_name, flush=True)++ from auto_round.utils import detect_device++ device_str = detect_device(args.device)+ torch_device = torch.device(device_str)++ torch.manual_seed(1234)+ model_name = args.model_name+ seqlen = args.seqlen+ questions = json.load(open(args.question_file, "r"))+ torch_dtype = "auto"+ if "hpu" in device_str:+ torch_dtype = torch.bfloat16+ if args.model_dtype != None:+ if args.model_dtype == "float16" or args.model_dtype == "fp16":+ torch_dtype = torch.float16+ if args.model_dtype == "bfloat16" or args.model_dtype == "bfp16":+ torch_dtype = torch.bfloat16++ config = AutoConfig.from_pretrained(+ model_name,+ trust_remote_code=True,+ )+ config.use_cache = False++ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ torch_dtype=torch_dtype,+ trust_remote_code=not args.disable_trust_remote_code,+ config=config,+ )+ seqlen = args.seqlen+ processor = Phi3VProcessor.from_pretrained(model_name)+ tokenizer = processor.tokenizer+ data_args = DataArguments(+ data_path=args.question_file,+ is_multimodal=True,+ image_folder=args.image_folder,+ max_seq_length=seqlen,+ )+ dataset = LazySupervisedDataset(+ data_path=args.question_file, processor=processor, data_args=data_args+ )+ data_collator = DataCollatorForSupervisedDataset(tokenizer=processor.tokenizer)+ dataloader = create_data_loader(dataset, batch_size=args.train_bs, data_collator=data_collator)++ from auto_round import (AutoRound,+ AutoAdamRound)+ from auto_round.utils import get_multimodal_block_names++ round = AutoRound+ if args.adam:+ round = AutoAdamRound++ layer_config = {}+ for n, m in model.named_modules():+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):+ if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:+ layer_config[n] = {"bits": 32}+ print(+ f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")+ lm_head_layer_name = "lm_head"+ for n, _ in model.named_modules():+ lm_head_layer_name = n+ if args.quant_lm_head:+ from transformers import AutoConfig++ config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)+ if config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"):+ tied_keys = model._tied_weights_keys+ for item in tied_keys:+ if lm_head_layer_name in item: ##TODO extend to encoder-decoder layer, seq classification model+ args.quant_lm_head = False+ print(+ f"warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "+ f"supported currently")+ break+ if args.quant_lm_head:+ layer_config[lm_head_layer_name] = {"bits": args.bits}+ transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]+ if transformers_version[0] == 4 and transformers_version[1] < 38:+ error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."+ raise EnvironmentError(error_message)++ if args.quant_lm_head and args.low_gpu_mem_usage:+ print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "+ f"gpu")++ quant_block_list = get_multimodal_block_names(model, args.quant_vision)++ autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,+ dataset=dataloader, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,+ minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,+ amp=not args.disable_amp, nsamples=args.nsamples, layer_config=layer_config,+ low_gpu_mem_usage=args.low_gpu_mem_usage, scale_dtype=args.scale_dtype,+ seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,+ quant_block_list=quant_block_list)+ model, _ = autoround.quantize()+ model_name = args.model_name.rstrip("/")++ model.eval()+ if args.device != "cpu":+ torch.cuda.empty_cache()++ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"+ output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"++ deployment_device = args.deployment_device.split(',')+ gpu_formats = []+ for item in deployment_device:+ if "gpu" in item or "auto_gptq" in item or "auto_round" in item:+ gpu_formats.append(item)++ if 'gpu' in deployment_device:+ if lm_head_layer_name in layer_config.keys() and layer_config[lm_head_layer_name]["data_type"] == "int":+ gpu_formats.append("auto_round")+ else:+ gpu_formats.append("auto_gptq")+ gpu_formats = list(set(gpu_formats))++ inplace = True if len(deployment_device) < 2 and args.disable_eval else False+ eval_folder = None+ for gpu_format in gpu_formats:+ if "round" in gpu_format:+ eval_folder = f'{export_dir}-round'+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace,+ safe_serialization=args.enable_safe_serialization)+ elif "gptq" in gpu_format:+ eval_folder = f'{export_dir}-gpu'+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace,+ safe_serialization=args.enable_safe_serialization)++ if 'xpu' in deployment_device:+ autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,+ compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,+ device="xpu", safe_serialization=args.enable_safe_serialization)+ if "cpu" in deployment_device:+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace,+ safe_serialization=args.enable_safe_serialization)+ if "fake" in deployment_device:+ model = model.to("cpu")+ model.save_pretrained(output_dir, safe_serialization=args.enable_safe_serialization)+ tokenizer.save_pretrained(output_dir)+ if eval_folder is None:+ eval_folder = output_dir++ if isinstance(tasks, str):+ tasks = tasks.split(',')+ if 'fake' in args.deployment_device and not args.disable_eval:+ print(f"Using the latest lm_eval 0.4.2")++ if not args.disable_eval:+ from eval_042.evaluation import simple_evaluate++ if 'gpu' in deployment_device or len(gpu_formats) > 0:+ model_args = f"pretrained={eval_folder}"+ elif "fake" in deployment_device:+ model_args = f"pretrained={eval_folder}"+ else:+ exit() ## does not support cpu,xpu model eval+ model_args = model_args + f",trust_remote_code={not args.disable_trust_remote_code}"+ user_model = model+ # model = model.to("cpu")+ if args.act_bits <= 8:+ user_model = model.to(device_str)++ res = simple_evaluate(model="hf", model_args=model_args,+ tasks=tasks,+ limit=5,+ batch_size=args.eval_bs, user_model=user_model)+ from lm_eval.utils import make_table++ print(make_table(res))diff --git a/examples/multimodal-modeling/Phi-3-vision/model/__init__.py b/examples/multimodal-modeling/Phi-3-vision/model/__init__.pynew file mode 100644index 000000000..dbe21cebc--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/model/__init__.py@@ -0,0 +1,4 @@+if __name__ == "__main__":+ import sys+ sys.path.insert(0, './')+diff --git a/examples/multimodal-modeling/Phi-3-vision/model/configuration_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/configuration_phi3_v.pynew file mode 100644index 000000000..573bb8d39--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/model/configuration_phi3_v.py@@ -0,0 +1,217 @@+# coding=utf-8+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++""" Phi-3-V model configuration"""+++from transformers.configuration_utils import PretrainedConfig+from transformers.utils import logging+++logger = logging.get_logger(__name__)++PHI3V_PRETRAINED_CONFIG_ARCHIVE_MAP = {+ "microsoft/Phi-3-vision-128k-instruct": "https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/resolve/main/config.json",+}+++class Phi3VConfig(PretrainedConfig):+ r"""+ This is the configuration class to store the configuration of a [`Phi3VModel`]. It is used to instantiate a Phi-3+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the+ defaults will yield a similar configuration to that of the+ [microsoft/Phi-3-vision-128k-instruct](https://huggingface.co/microsoft/Phi-3-vision-128k-instruct).++ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the+ documentation from [`PretrainedConfig`] for more information.++ Args:+ vocab_size (`int`, *optional*, defaults to 32064):+ Vocabulary size of the Phi-3-V model. Defines the number of different tokens that can be represented by the+ `inputs_ids` passed when calling [`Phi3VModel`].+ hidden_size (`int`, *optional*, defaults to 3072):+ Dimension of the hidden representations.+ intermediate_size (`int`, *optional*, defaults to 8192):+ Dimension of the MLP representations.+ num_hidden_layers (`int`, *optional*, defaults to 32):+ Number of hidden layers in the Transformer decoder.+ num_attention_heads (`int`, *optional*, defaults to 32):+ Number of attention heads for each attention layer in the Transformer decoder.+ num_key_value_heads (`int`, *optional*):+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed+ by meanpooling all the original heads within that group. For more details checkout [this+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to+ `num_attention_heads`.+ resid_pdrop (`float`, *optional*, defaults to 0.0):+ Dropout probability for mlp outputs.+ embd_pdrop (`int`, *optional*, defaults to 0.0):+ The dropout ratio for the embeddings.+ attention_dropout (`float`, *optional*, defaults to 0.0):+ The dropout ratio after computing the attention scores.+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):+ The non-linear activation function (function or string) in the decoder.+ max_position_embeddings (`int`, *optional*, defaults to 4096):+ The maximum sequence length that this model might ever be used with.+ original_max_position_embeddings (`int`, *optional*, defaults to 4096):+ The maximum sequence length that this model was trained with. This is used to determine the size of the+ original RoPE embeddings when using long scaling.+ initializer_range (`float`, *optional*, defaults to 0.02):+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.+ rms_norm_eps (`float`, *optional*, defaults to 1e-05):+ The epsilon value used for the RMSNorm.+ use_cache (`bool`, *optional*, defaults to `True`):+ Whether or not the model should return the last key/values attentions (not used by all models). Only+ relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not.+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):+ Whether to tie weight embeddings+ rope_theta (`float`, *optional*, defaults to 10000.0):+ The base period of the RoPE embeddings.+ rope_scaling (`dict`, *optional*):+ The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must+ contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be either `su` or `yarn` and+ the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size+ divided by the number of attention heads divided by 2.+ bos_token_id (`int`, *optional*, defaults to 1):+ The id of the "beginning-of-sequence" token.+ eos_token_id (`int`, *optional*, defaults to 32000):+ The id of the "end-of-sequence" token.+ pad_token_id (`int`, *optional*, defaults to 32000):+ The id of the padding token.+ sliding_window (`int`, *optional*):+ Sliding window attention window size. If `None`, no sliding window is applied.+ embd_layer (`str`, *optional*, defaults to `"default"`):+ The embedding layer to use. Can be either `"default"` or `"image"`. "default" uses the standard embedding for text.++ Example:++ ```python+ >>> from transformers import Phi3VModel, Phi3VConfig++ >>> # Initializing a Phi-3-V style configuration+ >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-vision-128k-instruct")++ >>> # Initializing a model from the configuration+ >>> model = Phi3VModel(configuration)++ >>> # Accessing the model configuration+ >>> configuration = model.config+ ```"""++ model_type = "phi3_v"+ keys_to_ignore_at_inference = ["past_key_values"]++ def __init__(+ self,+ vocab_size=32064,+ hidden_size=3072,+ intermediate_size=8192,+ num_hidden_layers=32,+ num_attention_heads=32,+ num_key_value_heads=None,+ resid_pdrop=0.0,+ embd_pdrop=0.0,+ attention_dropout=0.0,+ hidden_act="silu",+ max_position_embeddings=4096,+ original_max_position_embeddings=4096,+ initializer_range=0.02,+ rms_norm_eps=1e-5,+ use_cache=True,+ tie_word_embeddings=False,+ rope_theta=10000.0,+ rope_scaling=None,+ bos_token_id=1,+ eos_token_id=32000,+ pad_token_id=32000,+ sliding_window=None,+ embd_layer: str = "default",+ **kwargs,+ ):+ self.vocab_size = vocab_size+ self.hidden_size = hidden_size+ self.intermediate_size = intermediate_size+ self.num_hidden_layers = num_hidden_layers+ self.num_attention_heads = num_attention_heads++ if num_key_value_heads is None:+ num_key_value_heads = num_attention_heads++ self.num_key_value_heads = num_key_value_heads+ self.resid_pdrop = resid_pdrop+ self.embd_pdrop = embd_pdrop+ self.attention_dropout = attention_dropout+ self.hidden_act = hidden_act+ self.max_position_embeddings = max_position_embeddings+ self.original_max_position_embeddings = original_max_position_embeddings+ self.initializer_range = initializer_range+ self.rms_norm_eps = rms_norm_eps+ self.use_cache = use_cache+ self.rope_theta = rope_theta+ self.rope_scaling = rope_scaling+ self._rope_scaling_validation()+ self.sliding_window = sliding_window+ self.embd_layer = embd_layer+++ super().__init__(+ bos_token_id=bos_token_id,+ eos_token_id=eos_token_id,+ pad_token_id=pad_token_id,+ tie_word_embeddings=tie_word_embeddings,+ **kwargs,+ )++ def _rope_scaling_validation(self):+ """+ Validate the `rope_scaling` configuration.+ """+ if self.rope_scaling is None:+ return++ if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3:+ raise ValueError(+ "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, "+ f"got {self.rope_scaling}"+ )+ rope_scaling_type = self.rope_scaling.get("type", None)+ rope_scaling_short_factor = self.rope_scaling.get("short_factor", None)+ rope_scaling_long_factor = self.rope_scaling.get("long_factor", None)+ if rope_scaling_type is None or rope_scaling_type not in ["su", "yarn"]:+ raise ValueError(f"`rope_scaling`'s type field must be one of ['su', 'yarn'], got {rope_scaling_type}")+ if not (+ isinstance(rope_scaling_short_factor, list)+ and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)+ ):+ raise ValueError(+ f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"+ )+ if not len(rope_scaling_short_factor) == self.hidden_size // self.num_attention_heads // 2:+ raise ValueError(+ f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}"+ )+ if not (+ isinstance(rope_scaling_long_factor, list)+ and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor)+ ):+ raise ValueError(+ f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}"+ )+ if not len(rope_scaling_long_factor) == self.hidden_size // self.num_attention_heads // 2:+ raise ValueError(+ f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}"+ )\ No newline at end of filediff --git a/examples/multimodal-modeling/Phi-3-vision/model/image_embedding_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/image_embedding_phi3_v.pynew file mode 100644index 000000000..c2994c6ca--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/model/image_embedding_phi3_v.py@@ -0,0 +1,313 @@+# coding=utf-8+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import math+import torch+import torch.nn as nn+from transformers import CLIPVisionModel, PretrainedConfig+from transformers import CLIPVisionConfig+from transformers.utils import logging+from datetime import datetime++logger = logging.get_logger(__name__)++CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(+ attention_dropout=0.0,+ dropout=0.0,+ hidden_act="quick_gelu",+ hidden_size=1024,+ image_size=336,+ initializer_factor=1.0,+ initializer_range=0.02,+ intermediate_size=4096,+ layer_norm_eps=1e-05,+ num_attention_heads=16,+ num_channels=3,+ num_hidden_layers=24,+ patch_size=14,+ projection_dim=768+)++class Phi3ImageEmbedding(nn.Module):+ """Phi3 Image embedding."""++ def __init__(self, config: PretrainedConfig, wte=None, **kwargs) -> None:+ super().__init__()++ # n_embed or hidden_size+ hidden_size = config.n_embd if hasattr(config, 'n_embd') else config.hidden_size+ if hasattr(config, 'embd_pdrop') or hasattr(config, 'embed_pdrop'):+ embd_drop = config.embd_pdrop if hasattr(config, 'embd_pdrop') else config.embed_pdrop+ self.drop = nn.Dropout(embd_drop)+ else:+ self.drop = None++ self.wte = wte++ if isinstance(config.img_processor, dict) and config.img_processor.get('name', None) == 'clip_vision_model':+ assert 'model_name' in config.img_processor, 'model_name must be provided for CLIPVisionModel'+ assert 'image_dim_out' in config.img_processor, 'image_dim_out must be provided for CLIPVisionModel'+ assert 'num_img_tokens' in config.img_processor, 'num_img_tokens must be provided for CLIPVisionModel'+ assert config.img_processor['model_name'] == 'openai/clip-vit-large-patch14-336'+ clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG+ self.img_processor = CLIPVisionModel(clip_config)+ image_dim_out = config.img_processor['image_dim_out']+ self.num_img_tokens = config.img_processor['num_img_tokens']+ else:+ raise NotImplementedError(f'img_processor = {config.img_processor}, not implemented')++ self.image_dim_out = image_dim_out+ self.img_sizes = None++ # global_gn and sub_gn for hd transform, serves as line separator+ self.use_hd_transform = kwargs.get('use_hd_transform', False)+ self.with_learnable_separator = kwargs.get('with_learnable_separator', False)+ self.hd_transform_order = kwargs.get('hd_transform_order', 'glb_sub')+ # with_hd_transform and with_learnable_separator should have same value+ assert self.use_hd_transform == self.with_learnable_separator, 'use_hd_transform and with_learnable_separator should have same value'+ if self.with_learnable_separator:+ assert self.use_hd_transform, 'learnable separator is only for hd transform'+ # 1024 * 4, merge spatial to channel dimension+ self.glb_GN = nn.Parameter(torch.zeros([1, 1, self.image_dim_out * 4]))+ self.sub_GN = nn.Parameter(torch.zeros([1, 1, 1, self.image_dim_out * 4]))+ logger.info(f'learnable separator enabled for hd transform, hd_transform_order = {self.hd_transform_order}')++ projection_cls = kwargs.get('projection_cls', 'linear')+ if projection_cls == 'linear':+ self.img_projection = nn.Linear(image_dim_out, hidden_size)+ elif projection_cls == 'mlp' and self.use_hd_transform:+ dim_projection = hidden_size+ depth = 2+ layers = [nn.Linear(image_dim_out * 4, dim_projection)]+ for _ in range(1, depth):+ layers.extend([nn.GELU(),+ nn.Linear(dim_projection, dim_projection)])+ self.img_projection = nn.Sequential(*layers)+ elif projection_cls == 'mlp':+ dim_projection = hidden_size+ depth = 2+ layers = [nn.Linear(image_dim_out, dim_projection)]+ for _ in range(1, depth):+ layers.extend([nn.GELU(),+ nn.Linear(dim_projection, dim_projection)])+ self.img_projection = nn.Sequential(*layers)+ else:+ raise NotImplementedError(f'projection_cls = {projection_cls}, not implemented')++ self.vocab_size = config.vocab_size+ self.img_features = None++ if isinstance(config.img_processor, dict):+ self.layer_idx = config.img_processor.get('layer_idx', -2)+ self.type_feature = config.img_processor.get('type_feature', 'patch')+ else:+ self.layer_idx = -2+ self.type_feature = 'patch'+++ def set_img_features(self, img_features: torch.FloatTensor) -> None:+ self.img_features = img_features++ def set_img_sizes(self, img_sizes: torch.LongTensor) -> None:+ self.img_sizes = img_sizes++ def get_img_features(self, img_embeds: torch.FloatTensor) -> torch.FloatTensor:+ LAYER_IDX = self.layer_idx+ TYPE_FEATURE = self.type_feature++ img_processor_output = self.img_processor(img_embeds, output_hidden_states=True)+ img_feature = img_processor_output.hidden_states[LAYER_IDX]++ if TYPE_FEATURE == "patch":+ patch_feature = img_feature[:, 1:]+ return patch_feature++ if TYPE_FEATURE == "cls_patch":+ return img_feature++ raise NotImplementedError++ def forward(self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, image_sizes=None) -> torch.FloatTensor:++ MAX_INPUT_ID = int(1e9)+ img_embeds = pixel_values+ img_sizes = image_sizes++ if self.img_features is not None:+ img_embeds = self.img_features.clone()+ self.img_features = None++ if self.img_sizes is not None:+ img_sizes = self.img_sizes++ input_shape = input_ids.size()+ input_ids = input_ids.view(-1, input_shape[-1])++ with torch.no_grad():+ positions = torch.nonzero((input_ids < 0) & (input_ids > -MAX_INPUT_ID), as_tuple=False)++ select = False++ if isinstance(self.img_projection, nn.Sequential):+ target_device = self.img_projection[0].bias.device+ target_dtype = self.img_projection[0].bias.dtype+ else: # It's a single nn.Linear layer+ target_device = self.img_projection.bias.device+ target_dtype = self.img_projection.bias.dtype++ if len(positions.tolist()) > 0:+ with torch.no_grad():+ g_values = abs(input_ids[positions[:, 0], positions[:, 1]])++ if self.use_hd_transform and img_sizes is not None and len(img_sizes):+ hd_transform = True+ assert img_embeds.ndim == 5, f'img_embeds size: {img_embeds.size()}, expect 5D tensor for hd transform'+ # img_embeds: (num_images, max_num_crops, 3, H, W)+ # img_sizes: (num_images, 2).view(1, -1)++ start_time = datetime.now()+ bs = img_embeds.shape[0]+ # Nx(HW)xC+ img_features = self.get_img_features(img_embeds.flatten(0, 1))+ base_feat_height = base_feat_width = int(img_features.shape[1] ** 0.5)++ assert base_feat_height == 24 and base_feat_width == 24, f'base_feat_height: {base_feat_height}, base_feat_width: {base_feat_width}, expect 24x24 features for hd transform'++ # bs x max_num_crops x (24x24) x C+ img_features = img_features.view(bs, -1, base_feat_height * base_feat_width, self.image_dim_out)+ C = self.image_dim_out+ H = base_feat_height++ output_imgs = []+ output_len = []+ # training is tensor, inference is list+ if isinstance(img_sizes, torch.Tensor):+ img_sizes = img_sizes.view(-1, 2)+ num_pure_text = 0+ for _bs in range(bs):+ h, w = img_sizes[_bs]+ h = h // 336+ w = w // 336+ B_ = h * w+ if B_ == 0:+ num_pure_text += 1+ continue++ # 1 x (24x24) x 1024+ global_img_feature = img_features[_bs, :1]++ # 1 x 12 x 12 x 4096+ glb_img = global_img_feature.reshape(1,H,H,C).reshape(1,H//2,2,H//2,2,C).contiguous().permute(0,1,3,2,4,5).reshape(1,H//2,H//2,4*C).contiguous()+ temp_glb_GN = self.sub_GN.repeat(1, H//2, 1, 1)++ # 1 x 156 x 4096+ glb_img = torch.cat([glb_img, temp_glb_GN], dim=2).reshape(1,-1,4*C)++ # (max_num_crops-1) x (12x12) x C+ sub_img = img_features[_bs, 1:]+ # 16x574x1024+ # get rid of padding sub_img+ sub_img = sub_img[:B_]++ # (num_crops, 12, 2, 12, 2, 1024) -> (num_crops, 12, 12, 2, 2, 1024) -> (num_crops, 12*12, 4*1024)+ sub_img = sub_img.reshape(B_,H,H,C).reshape(B_,H//2,2,H//2,2,C).contiguous().permute(0,1,3,2,4,5).reshape(B_,-1,4*C).contiguous()+ sub_img = sub_img.reshape(1, h, w, 12, 12, -1).permute(0,1,3,2,4,5).reshape(1,h*12,w*12,4*C)+ temp_sub_GN = self.sub_GN.repeat(1, h*12, 1, 1)+ sub_img = torch.cat([sub_img, temp_sub_GN], dim=2).reshape(1,-1,4*C)+ # (1, num_img_tokens, 1024*4)++ # glb + sub+ if self.hd_transform_order == 'glb_sub':+ output_imgs.append(torch.cat([glb_img, self.glb_GN, sub_img], dim=1))+ elif self.hd_transform_order == 'sub_glb':+ output_imgs.append(torch.cat([sub_img, self.glb_GN, glb_img], dim=1))+ else:+ raise NotImplementedError(f'hd_transform_order = {self.hd_transform_order}, not implemented')++ temp_len = int((h*w+1)*144 + 1 + (h+1)*12)+ assert temp_len == output_imgs[-1].shape[1], f'temp_len: {temp_len}, output_imgs[-1].shape[1]: {output_imgs[-1].shape[1]}'+ output_len.append(temp_len)++ num_img_tokens = output_len+ img_set_tensor = []+ for _output_img in output_imgs:+ img_feature_proj = self.img_projection(_output_img.to(target_device).to(target_dtype))+ img_set_tensor.append(img_feature_proj)+ # logger.info(f'img_embeds size: {img_embeds.size()}, image sizes: {img_sizes} loading time {datetime.now() - start_time}')+ elif img_embeds.ndim == 4:+ selected_g_values = g_values[::self.num_img_tokens]+ assert len(img_embeds) == len(selected_g_values), f'img_embeds size: {img_embeds.size()}, selected_g_values size: {len(selected_g_values)}, selected_g_value {selected_g_values}'+ start_time = datetime.now()+ tt = (+ self.get_img_features(img_embeds)+ .to(target_device)+ .to(target_dtype)+ .reshape(-1, self.image_dim_out)+ )+ # logger.info(f'img_embeds size: {img_embeds.size()}, loading time {datetime.now() - start_time}')+ img_set_tensor = self.img_projection(tt) # adapted visual features.+ elif img_embeds.ndim == 3:+ selected_g_values = g_values[::self.num_img_tokens]+ assert len(img_embeds) == len(selected_g_values), f'img_embeds size: {img_embeds.size()}, selected_g_values size: {len(selected_g_values)}, selected_g_value {selected_g_values}'+ tt = (+ img_embeds+ .to(target_device)+ .to(target_dtype)+ .view(-1, self.image_dim_out)+ )+ img_set_tensor = self.img_projection(tt) # adapted visual features.+ else:+ raise NotImplementedError+ select = True+ # It's a hacky way to walkaround the hang-out problem when deepspeed `zero3` is used+ # and the training batch is a mixture of pure text and vision-language data.+ else:+ num_pure_text = input_ids.shape[0]+ self.get_img_features(img_embeds.flatten(0, 1))+ for _ in range(num_pure_text):+ self.img_projection(torch.zeros(1, 1921, 4096, device=self.img_processor.device, dtype=self.img_processor.dtype))+ with torch.no_grad():+ input_ids.clamp_min_(0).clamp_max_(self.vocab_size)++ hidden_states = self.wte(input_ids)++ if select:+ if hd_transform:+ idx = 0+ for i, cnt in enumerate(num_img_tokens):+ # see https://github.com/GaiZhenbiao/Phi3V-Finetuning/pull/5+ hidden_states = hidden_states.clone()+ hidden_states[positions[idx, 0], positions[idx, 1] : positions[idx, 1] + cnt] = (+ img_set_tensor[i]+ .to(hidden_states.dtype)+ .to(hidden_states.device)+ )+ idx += cnt+ else:+ idx = 0+ assert len(selected_g_values) * self.num_img_tokens == len(img_set_tensor), f'len(selected_g_values) * self.num_img_tokens = {len(selected_g_values) * self.num_img_tokens}, len(img_set_tensor) = {len(img_set_tensor)}'+ for i, g in enumerate(selected_g_values):+ cnt = self.num_img_tokens+ hidden_states[positions[idx, 0], positions[idx, 1] : positions[idx, 1] + cnt] = (+ img_set_tensor[i * cnt : (i + 1) * cnt]+ .to(hidden_states.dtype)+ .to(hidden_states.device)+ )+ idx += cnt++ if self.drop is not None:+ hidden_states = self.drop(hidden_states)++ return hidden_statesdiff --git a/examples/multimodal-modeling/Phi-3-vision/model/image_processing_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/image_processing_phi3_v.pynew file mode 100644index 000000000..821096527--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/model/image_processing_phi3_v.py@@ -0,0 +1,274 @@+# coding=utf-8+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Image processor class for Phi3-V."""++from typing import List, Optional, Union++import numpy as np+from auto_round.utils import transformers+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature+from transformers.image_transforms import (+ convert_to_rgb,+)+from transformers.image_utils import (+ OPENAI_CLIP_MEAN,+ OPENAI_CLIP_STD,+ ImageInput,+ make_list_of_images,+ valid_images,+)+from transformers.utils import TensorType, is_vision_available, logging++from transformers import AutoImageProcessor++logger = logging.get_logger(__name__)+++if is_vision_available():+ from PIL import Image++import torch+import torchvision++def padding_336(b):+ width, height = b.size+ tar = int(np.ceil(height / 336) * 336)+ top_padding = int((tar - height)/2)+ bottom_padding = tar - height - top_padding+ left_padding = 0+ right_padding = 0+ b = torchvision.transforms.functional.pad(b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255,255,255])++ return b++def calc_padded_size(width, height, padding_unit=336):+ target_height = int(np.ceil(height / padding_unit) * padding_unit)+ top_padding = int((target_height - height) / 2)+ bottom_padding = target_height - height - top_padding+ left_padding = 0+ right_padding = 0+ padded_width = width + left_padding + right_padding+ padded_height = height + top_padding + bottom_padding+ return padded_width, padded_height++def HD_transform(img, hd_num=16):+ width, height = img.size+ trans = False+ if width < height:+ img = img.transpose(Image.TRANSPOSE)+ trans = True+ width, height = img.size+ ratio = (width/ height)+ scale = 1+ while scale*np.ceil(scale/ratio) <= hd_num:+ scale += 1+ scale -= 1+ new_w = int(scale * 336)+ new_h = int(new_w / ratio)++ img = torchvision.transforms.functional.resize(img, [new_h, new_w],)+ img = padding_336(img)+ width, height = img.size+ if trans:+ img = img.transpose(Image.TRANSPOSE)++ return img++def calc_hd_transform_size(width, height, hd_num=16):+ transposed = False+ if width < height:+ width, height = height, width+ transposed = True++ ratio = width / height+ scale = 1+ while scale * np.ceil(scale / ratio) <= hd_num:+ scale += 1+ scale -= 1++ new_width = int(scale * 336)+ new_height = int(new_width / ratio)++ padded_width, padded_height = calc_padded_size(new_width, new_height)++ if transposed:+ padded_width, padded_height = padded_height, padded_width++ return padded_width, padded_height++def pad_to_max_num_crops_tensor(images, max_crops=5):+ """+ images: B x 3 x H x W, B<=max_crops+ """+ B, _, H, W = images.shape+ if B < max_crops:+ pad = torch.zeros(max_crops - B, 3, H, W, dtype=images.dtype, device=images.device)+ images = torch.cat([images, pad], dim=0)+ return images+++class Phi3VImageProcessor(BaseImageProcessor):+ r"""+ Constructs a Phi3 image processor. Based on [`CLIPImageProcessor`] with incorporation of additional techniques+ for processing high resolution images as explained in the [InternLM-XComposer2-4KHD](https://arxiv.org/abs/2401.16420)++ Args:+ image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):+ Mean to use if normalizing the image. This is a float or list of floats the length of the number of+ channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method.+ image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):+ Standard deviation to use if normalizing the image. This is a float or list of floats the length of the+ number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.+ Can be overridden by the `image_std` parameter in the `preprocess` method.+ do_convert_rgb (`bool`, *optional*, defaults to `True`):+ Whether to convert the image to RGB.+ """++ model_input_names = ["pixel_values"]++ def __init__(+ self,+ num_crops: int = 1,+ image_mean: Optional[Union[float, List[float]]] = None,+ image_std: Optional[Union[float, List[float]]] = None,+ do_convert_rgb: bool = True,+ **kwargs,+ ) -> None:+ super().__init__(**kwargs)+ self.num_crops = num_crops+ self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN+ self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD+ self.do_convert_rgb = do_convert_rgb++ def calc_num_image_tokens(+ self,+ images: ImageInput+ ):+ """ Calculate the number of image tokens for each image.+ Args:+ images (`ImageInput`):+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.+ """+ images = make_list_of_images(images)++ if not valid_images(images):+ raise ValueError(+ "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "+ "torch.Tensor, tf.Tensor or jax.ndarray."+ )++ images = [image.convert('RGB') for image in images]+ # (H, W, C)+ elems = [HD_transform(im, hd_num = self.num_crops) for im in images]+ shapes = [[im.size[1], im.size[0]] for im in elems]+ num_img_tokens = [int((h//336*w//336+1)*144 + 1 + (h//336+1)*12) for h, w in shapes]+ return num_img_tokens++ def calc_num_image_tokens_from_image_size(self, width, height):+ """+ Calculate the number of image tokens for a given image size.+ Args:+ width (`int`): Width of the image.+ height (`int`): Height of the image.+ """+ new_width, new_height = calc_hd_transform_size(width, height, hd_num=self.num_crops)+ num_img_tokens = int((new_height // 336 * new_width // 336 + 1) * 144 + 1 + (new_height // 336 + 1) * 12)+ return num_img_tokens++ def preprocess(+ self,+ images: ImageInput,+ image_mean: Optional[Union[float, List[float]]] = None,+ image_std: Optional[Union[float, List[float]]] = None,+ do_convert_rgb: bool = None,+ return_tensors: Optional[Union[str, TensorType]] = None,+ ):+ """+ Args:+ images (`ImageInput`):+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.+ image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):+ Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.+ image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):+ Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to+ `True`.+ do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):+ Whether to convert the image to RGB.+ return_tensors (`str` or `TensorType`, *optional*):+ The type of tensors to return. Can be one of:+ - Unset: Return a list of `np.ndarray`.+ - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.+ - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.+ - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.+ - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.+ """+ image_mean = image_mean if image_mean is not None else self.image_mean+ image_std = image_std if image_std is not None else self.image_std+ do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb++ images = make_list_of_images(images)++ if not valid_images(images):+ raise ValueError(+ "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "+ "torch.Tensor, tf.Tensor or jax.ndarray."+ )++ if do_convert_rgb:+ images = [convert_to_rgb(image) for image in images]++ image_sizes = []+ img_processor = torchvision.transforms.Compose([+ torchvision.transforms.ToTensor(),+ torchvision.transforms.Normalize(image_mean, image_std)+ ])++ # PIL images+ # HD_transform pad images to size of multiiply of 336, 336+ # convert to RGB first+ images = [image.convert('RGB') for image in images]+ elems = [HD_transform(im, hd_num = self.num_crops) for im in images]+ # tensor transform and normalize+ hd_images = [img_processor(im) for im in elems]+ # create global image+ global_image = [torch.nn.functional.interpolate(im.unsqueeze(0).float(), size=(336, 336), mode='bicubic',).to(im.dtype) for im in hd_images]++ # [(3, h, w)], where h, w is multiple of 336+ shapes = [[im.size(1), im.size(2)] for im in hd_images]+ num_img_tokens = [int((h//336*w//336+1)*144 + 1 + (h//336+1)*12) for h, w in shapes]+ # reshape to channel dimension -> (num_images, num_crops, 3, 336, 336)+ # (1, 3, h//336, 336, w//336, 336) -> (1, h//336, w//336, 3, 336, 336) -> (h//336*w//336, 3, 336, 336)+ hd_images_reshape = [im.reshape(1, 3, h//336, 336, w//336, 336).permute(0,2,4,1,3,5).reshape(-1, 3, 336, 336).contiguous() for im, (h, w) in zip(hd_images, shapes)]+ # concat global image and local image+ hd_images_reshape = [torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape)]++ # pad to max_num_crops+ image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops+1) for im in hd_images_reshape]+ image_transformed = torch.stack(image_transformed, dim=0)+ image_sizes = [torch.LongTensor(_shapes) for _shapes in shapes]+ padded_images = image_transformed+ image_sizes = shapes++ data = {"pixel_values": padded_images,+ "image_sizes": image_sizes,+ "num_img_tokens": num_img_tokens+ }++ return BatchFeature(data=data, tensor_type=return_tensors)++AutoImageProcessor.register("Phi3VImageProcessor", Phi3VImageProcessor)\ No newline at end of filediff --git a/examples/multimodal-modeling/Phi-3-vision/model/modeling_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/modeling_phi3_v.pynew file mode 100644index 000000000..0b5357b16--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/model/modeling_phi3_v.py@@ -0,0 +1,1634 @@+# coding=utf-8+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++""" PyTorch Phi-3-V model."""++import inspect+import math+import warnings+from typing import List, Optional, Tuple, Union++import torch+import torch.nn.functional as F+import torch.utils.checkpoint+from torch import nn+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss+from .image_embedding_phi3_v import Phi3ImageEmbedding+import transformers+from transformers.activations import ACT2FN+from transformers.cache_utils import Cache, DynamicCache+from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask+from transformers.modeling_outputs import (+ BaseModelOutputWithPast,+ CausalLMOutputWithPast,+ SequenceClassifierOutputWithPast,+ TokenClassifierOutput,+)+from transformers.modeling_utils import PreTrainedModel+from transformers.utils import (+ add_code_sample_docstrings,+ add_start_docstrings,+ add_start_docstrings_to_model_forward,+ is_flash_attn_2_available,+ is_flash_attn_greater_or_equal_2_10,+ logging,+ replace_return_docstrings,+)+from .configuration_phi3_v import Phi3VConfig++++logger = logging.get_logger(__name__)++if is_flash_attn_2_available():+ from flash_attn import flash_attn_func, flash_attn_varlen_func+ from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa++ _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters)++_CHECKPOINT_FOR_DOC = "microsoft/Phi-3-vision-128k-instruct"+_CONFIG_FOR_DOC = "Phi3VConfig"++PHI3V_PRETRAINED_MODEL_ARCHIVE_LIST = [+ "microsoft/Phi-3-vision-128k-instruct",+ # See all Phi-3 models at https://huggingface.co/models?filter=Phi-3+]+++# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Phi3+class Phi3RMSNorm(nn.Module):+ def __init__(self, hidden_size, eps=1e-6):+ """+ Phi3RMSNorm is equivalent to T5LayerNorm+ """+ super().__init__()+ self.weight = nn.Parameter(torch.ones(hidden_size))+ self.variance_epsilon = eps++ def forward(self, hidden_states):+ input_dtype = hidden_states.dtype+ hidden_states = hidden_states.to(torch.float32)+ variance = hidden_states.pow(2).mean(-1, keepdim=True)+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)+ return self.weight * hidden_states.to(input_dtype)+++# Copied from transformers.models.llama.modeling_llama._get_unpad_data+def _get_unpad_data(attention_mask):+ seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)+ indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()+ max_seqlen_in_batch = seqlens_in_batch.max().item()+ cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))+ return (+ indices,+ cu_seqlens,+ max_seqlen_in_batch,+ )+++# Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding with gemma->phi3, Gemma->Phi3+class Phi3RotaryEmbedding(nn.Module):+ def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):+ super().__init__()++ self.dim = dim+ self.max_position_embeddings = max_position_embeddings+ self.base = base+ self.register_buffer("inv_freq", None, persistent=False)++ @torch.no_grad()+ def forward(self, x, position_ids, seq_len=None):+ # x: [bs, num_attention_heads, seq_len, head_size]+ if self.inv_freq is None:+ self.inv_freq = 1.0 / (+ self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim)+ )+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)+ position_ids_expanded = position_ids[:, None, :].float()+ # Force float32 since bfloat16 loses precision on long contexts+ # See https://github.com/huggingface/transformers/pull/29285+ device_type = x.device.type+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"+ with torch.autocast(device_type=device_type, enabled=False):+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)+ emb = torch.cat((freqs, freqs), dim=-1)+ cos = emb.cos()+ sin = emb.sin()+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)+++class Phi3SuScaledRotaryEmbedding(Phi3RotaryEmbedding):+ def __init__(self, dim, config, device=None):+ super().__init__(dim, config.max_position_embeddings, config.rope_theta, device)++ self.short_factor = config.rope_scaling["short_factor"]+ self.long_factor = config.rope_scaling["long_factor"]+ self.original_max_position_embeddings = config.original_max_position_embeddings++ @torch.no_grad()+ def forward(self, x, position_ids, seq_len=None):+ seq_len = torch.max(position_ids) + 1+ if seq_len > self.original_max_position_embeddings:+ ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device)+ else:+ ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device)++ inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim+ self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape)++ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)+ position_ids_expanded = position_ids[:, None, :].float()++ # Force float32 since bfloat16 loses precision on long contexts+ # See https://github.com/huggingface/transformers/pull/29285+ device_type = x.device.type+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"+ with torch.autocast(device_type=device_type, enabled=False):+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)+ emb = torch.cat((freqs, freqs), dim=-1)++ scale = self.max_position_embeddings / self.original_max_position_embeddings+ if scale <= 1.0:+ scaling_factor = 1.0+ else:+ scaling_factor = math.sqrt(1 + math.log(scale) / math.log(self.original_max_position_embeddings))++ cos = emb.cos() * scaling_factor+ sin = emb.sin() * scaling_factor+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)+++class Phi3YarnScaledRotaryEmbedding(Phi3RotaryEmbedding):+ def __init__(self, dim, config, device=None):+ super().__init__(dim, config.max_position_embeddings, config.rope_theta, device)++ self.short_factor = config.rope_scaling["short_factor"]+ self.long_factor = config.rope_scaling["long_factor"]+ self.original_max_position_embeddings = config.original_max_position_embeddings++ @torch.no_grad()+ def forward(self, x, position_ids, seq_len=None):+ seq_len = torch.max(position_ids) + 1+ if seq_len > self.original_max_position_embeddings:+ ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device)+ else:+ ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device)++ inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim+ self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape)++ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)+ position_ids_expanded = position_ids[:, None, :].float()++ # Force float32 since bfloat16 loses precision on long contexts+ # See https://github.com/huggingface/transformers/pull/29285+ device_type = x.device.type+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"+ with torch.autocast(device_type=device_type, enabled=False):+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)+ emb = torch.cat((freqs, freqs), dim=-1)++ scale = self.max_position_embeddings / self.original_max_position_embeddings+ if scale <= 1.0:+ scaling_factor = 1.0+ else:+ scaling_factor = 0.1 * math.log(scale) + 1.0++ cos = emb.cos() * scaling_factor+ sin = emb.sin() * scaling_factor+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)+++# Copied from transformers.models.llama.modeling_llama.rotate_half+def rotate_half(x):+ """Rotates half the hidden dims of the input."""+ x1 = x[..., : x.shape[-1] // 2]+ x2 = x[..., x.shape[-1] // 2 :]+ return torch.cat((-x2, x1), dim=-1)+++# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):+ """Applies Rotary Position Embedding to the query and key tensors.++ Args:+ q (`torch.Tensor`): The query tensor.+ k (`torch.Tensor`): The key tensor.+ cos (`torch.Tensor`): The cosine part of the rotary embedding.+ sin (`torch.Tensor`): The sine part of the rotary embedding.+ position_ids (`torch.Tensor`, *optional*):+ Deprecated and unused.+ unsqueeze_dim (`int`, *optional*, defaults to 1):+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.+ Returns:+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.+ """+ cos = cos.unsqueeze(unsqueeze_dim)+ sin = sin.unsqueeze(unsqueeze_dim)+ q_embed = (q * cos) + (rotate_half(q) * sin)+ k_embed = (k * cos) + (rotate_half(k) * sin)+ return q_embed, k_embed+++class Phi3MLP(nn.Module):+ def __init__(self, config):+ super().__init__()++ self.config = config+ self.gate_up_proj = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False)+ self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)++ self.activation_fn = ACT2FN[config.hidden_act]++ def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:+ up_states = self.gate_up_proj(hidden_states)++ gate, up_states = up_states.chunk(2, dim=-1)+ up_states = up_states * self.activation_fn(gate)++ return self.down_proj(up_states)+++# Copied from transformers.models.llama.modeling_llama.repeat_kv with llama->phi+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:+ """+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)+ """+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape+ if n_rep == 1:+ return hidden_states+ hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)+++class Phi3Attention(nn.Module):+ """Multi-headed attention from 'Attention Is All You Need' paper"""++ def __init__(self, config: Phi3VConfig, layer_idx: Optional[int] = None):+ super().__init__()+ self.config = config+ self.layer_idx = layer_idx+ if layer_idx is None:+ logger.warning_once(+ f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will "+ "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` "+ "when creating this class."+ )++ self.attention_dropout = config.attention_dropout+ self.hidden_size = config.hidden_size+ self.num_heads = config.num_attention_heads+ self.head_dim = self.hidden_size // self.num_heads+ self.num_key_value_heads = config.num_key_value_heads+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads+ self.max_position_embeddings = config.max_position_embeddings+ self.original_max_position_embeddings = config.original_max_position_embeddings+ self.rope_theta = config.rope_theta+ self.rope_scaling = config.rope_scaling+ self.is_causal = True++ if (self.head_dim * self.num_heads) != self.hidden_size:+ raise ValueError(+ f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"+ f" and `num_heads`: {self.num_heads})."+ )++ op_size = self.num_heads * self.head_dim + 2 * (self.num_key_value_heads * self.head_dim)+ self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)+ self.qkv_proj = nn.Linear(self.hidden_size, op_size, bias=False)+ self._init_rope()++ def _init_rope(self):+ if self.rope_scaling is None:+ self.rotary_emb = Phi3RotaryEmbedding(+ self.head_dim,+ max_position_embeddings=self.max_position_embeddings,+ base=self.rope_theta,+ )+ else:+ scaling_type = self.config.rope_scaling["type"]+ if scaling_type == "su":+ self.rotary_emb = Phi3SuScaledRotaryEmbedding(self.head_dim, self.config)+ elif scaling_type == "yarn":+ self.rotary_emb = Phi3YarnScaledRotaryEmbedding(self.head_dim, self.config)+ else:+ raise ValueError(f"Unknown RoPE scaling type {scaling_type}")++ def forward(+ self,+ hidden_states: torch.Tensor,+ attention_mask: Optional[torch.Tensor] = None,+ position_ids: Optional[torch.LongTensor] = None,+ past_key_value: Optional[Cache] = None,+ output_attentions: bool = False,+ use_cache: bool = False,+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:+ logger.warning_once("You are not running the flash-attention implementation, expect numerical differences.")++ bsz, q_len, _ = hidden_states.size()++ qkv = self.qkv_proj(hidden_states)+ query_pos = self.num_heads * self.head_dim+ query_states = qkv[..., :query_pos]+ key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]+ value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]++ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)++ kv_seq_len = key_states.shape[-2]+ if past_key_value is not None:+ if self.layer_idx is None:+ raise ValueError(+ f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "+ "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "+ "with a layer index."+ )+ kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)+ cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len)++ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)++ if past_key_value is not None:+ cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)++ # repeat k/v heads if n_kv_heads < n_heads+ key_states = repeat_kv(key_states, self.num_key_value_groups)+ value_states = repeat_kv(value_states, self.num_key_value_groups)++ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)++ if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):+ raise ValueError(+ f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"+ f" {attn_weights.size()}"+ )++ if attention_mask is not None:+ if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):+ raise ValueError(+ f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"+ )+ attn_weights = attn_weights + attention_mask++ # upcast attention to fp32+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(value_states.dtype)+ attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)++ attn_output = torch.matmul(attn_weights, value_states)++ if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):+ raise ValueError(+ f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"+ f" {attn_output.size()}"+ )++ attn_output = attn_output.transpose(1, 2).contiguous()+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)++ attn_output = self.o_proj(attn_output)++ if not output_attentions:+ attn_weights = None++ return attn_output, attn_weights, past_key_value+++class Phi3FlashAttention2(Phi3Attention):+ """+ Phi-3 flash attention module. This module inherits from `Phi3Attention` as the weights of the module stays+ untouched. The only required change would be on the forward pass where it needs to correctly call the public API of+ flash attention and deal with padding tokens in case the input contains any of them.+ """++ # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)++ # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.+ # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.+ # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).+ self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()++ def forward(+ self,+ hidden_states: torch.Tensor,+ attention_mask: Optional[torch.LongTensor] = None,+ position_ids: Optional[torch.LongTensor] = None,+ past_key_value: Optional[Cache] = None,+ output_attentions: bool = False,+ use_cache: bool = False,+ **kwargs,+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:+ # Phi3FlashAttention2 attention does not support output_attentions++ if not _flash_supports_window_size:+ logger.warning_once(+ "The current flash attention version does not support sliding window attention. Please use `attn_implementation='eager'` or upgrade flash-attn library."+ )+ raise ValueError("The current flash attention version does not support sliding window attention.")++ output_attentions = False++ if "padding_mask" in kwargs:+ warnings.warn(+ "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"+ )++ # overwrite attention_mask with padding_mask+ attention_mask = kwargs.pop("padding_mask")++ bsz, q_len, _ = hidden_states.size()++ qkv = self.qkv_proj(hidden_states)+ query_pos = self.num_heads * self.head_dim+ query_states = qkv[..., :query_pos]+ key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]+ value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]++ # Flash attention requires the input to have the shape+ # batch_size x seq_length x head_dim x hidden_dim+ # therefore we just need to keep the original shape+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)++ kv_seq_len = key_states.shape[-2]+ if past_key_value is not None:+ if self.layer_idx is None:+ raise ValueError(+ f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "+ "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "+ "with a layer index."+ )+ kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)++ # Because the input can be padded, the absolute sequence length depends on the max position id.+ rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1+ cos, sin = self.rotary_emb(value_states, position_ids, seq_len=rotary_seq_len)++ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)++ use_sliding_windows = (+ _flash_supports_window_size+ and getattr(self.config, "sliding_window", None) is not None+ and kv_seq_len > self.config.sliding_window+ )++ if past_key_value is not None:+ # Activate slicing cache only if the config has a value `sliding_windows` attribute+ cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0+ if (+ getattr(self.config, "sliding_window", None) is not None+ and kv_seq_len > self.config.sliding_window+ and cache_has_contents+ ):+ slicing_tokens = 1 - self.config.sliding_window++ past_key = past_key_value[self.layer_idx][0]+ past_value = past_key_value[self.layer_idx][1]++ past_key = past_key[:, :, slicing_tokens:, :].contiguous()+ past_value = past_value[:, :, slicing_tokens:, :].contiguous()++ if past_key.shape[-2] != self.config.sliding_window - 1:+ raise ValueError(+ f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got"+ f" {past_key.shape}"+ )++ if attention_mask is not None:+ attention_mask = attention_mask[:, slicing_tokens:]+ attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1)++ cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)++ # repeat k/v heads if n_kv_heads < n_heads+ key_states = repeat_kv(key_states, self.num_key_value_groups)+ value_states = repeat_kv(value_states, self.num_key_value_groups)++ attn_dropout = self.attention_dropout if self.training else 0.0++ # In PEFT, usually we cast the layer norms in float32 for training stability reasons+ # therefore the input hidden states gets silently casted in float32. Hence, we need+ # cast them back in the correct dtype just to be sure everything works as expected.+ # This might slowdown training & inference so it is recommended to not cast the LayerNorms+ # in fp32.++ if query_states.dtype == torch.float32:+ if torch.is_autocast_enabled():+ target_dtype = torch.get_autocast_gpu_dtype()+ # Handle the case where the model is quantized+ elif hasattr(self.config, "_pre_quantization_dtype"):+ target_dtype = self.config._pre_quantization_dtype+ else:+ target_dtype = self.qkv_proj.weight.dtype++ logger.warning_once(+ f"The input hidden states seems to be silently casted in float32, this might be related to"+ f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"+ f" {target_dtype}."+ )++ query_states = query_states.to(target_dtype)+ key_states = key_states.to(target_dtype)+ value_states = value_states.to(target_dtype)++ # Reashape to the expected shape for Flash Attention+ query_states = query_states.transpose(1, 2)+ key_states = key_states.transpose(1, 2)+ value_states = value_states.transpose(1, 2)++ attn_output = self._flash_attention_forward(+ query_states,+ key_states,+ value_states,+ attention_mask,+ q_len,+ dropout=attn_dropout,+ use_sliding_windows=use_sliding_windows,+ )++ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()+ attn_output = self.o_proj(attn_output)++ if not output_attentions:+ attn_weights = None++ return attn_output, attn_weights, past_key_value++ # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._flash_attention_forward+ def _flash_attention_forward(+ self,+ query_states,+ key_states,+ value_states,+ attention_mask,+ query_length,+ dropout=0.0,+ softmax_scale=None,+ use_sliding_windows=False,+ ):+ """+ Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token+ first unpad the input, then computes the attention scores and pad the final attention scores.++ Args:+ query_states (`torch.Tensor`):+ Input query states to be passed to Flash Attention API+ key_states (`torch.Tensor`):+ Input key states to be passed to Flash Attention API+ value_states (`torch.Tensor`):+ Input value states to be passed to Flash Attention API+ attention_mask (`torch.Tensor`):+ The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the+ position of padding tokens and 1 for the position of non-padding tokens.+ dropout (`float`):+ Attention dropout+ softmax_scale (`float`, *optional*):+ The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)+ use_sliding_windows (`bool`, *optional*):+ Whether to activate sliding window attention.+ """+ if not self._flash_attn_uses_top_left_mask:+ causal = self.is_causal+ else:+ # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.+ causal = self.is_causal and query_length != 1++ # Contains at least one padding token in the sequence+ if attention_mask is not None:+ batch_size = query_states.shape[0]+ query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(+ query_states, key_states, value_states, attention_mask, query_length+ )++ cu_seqlens_q, cu_seqlens_k = cu_seq_lens+ max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens++ if not use_sliding_windows:+ attn_output_unpad = flash_attn_varlen_func(+ query_states,+ key_states,+ value_states,+ cu_seqlens_q=cu_seqlens_q,+ cu_seqlens_k=cu_seqlens_k,+ max_seqlen_q=max_seqlen_in_batch_q,+ max_seqlen_k=max_seqlen_in_batch_k,+ dropout_p=dropout,+ softmax_scale=softmax_scale,+ causal=causal,+ )+ else:+ attn_output_unpad = flash_attn_varlen_func(+ query_states,+ key_states,+ value_states,+ cu_seqlens_q=cu_seqlens_q,+ cu_seqlens_k=cu_seqlens_k,+ max_seqlen_q=max_seqlen_in_batch_q,+ max_seqlen_k=max_seqlen_in_batch_k,+ dropout_p=dropout,+ softmax_scale=softmax_scale,+ causal=causal,+ window_size=(self.config.sliding_window, self.config.sliding_window),+ )++ attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)+ else:+ if not use_sliding_windows:+ attn_output = flash_attn_func(+ query_states,+ key_states,+ value_states,+ dropout,+ softmax_scale=softmax_scale,+ causal=causal,+ )+ else:+ attn_output = flash_attn_func(+ query_states,+ key_states,+ value_states,+ dropout,+ softmax_scale=softmax_scale,+ causal=causal,+ window_size=(self.config.sliding_window, self.config.sliding_window),+ )++ return attn_output++ # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._upad_input+ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):+ batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape++ # On the first iteration we need to properly re-create the padding mask+ # by slicing it on the proper place+ if kv_seq_len != attention_mask.shape[-1]:+ attention_mask_num_tokens = attention_mask.shape[-1]+ attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :]++ indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)++ key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)+ value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)++ if query_length == kv_seq_len:+ query_layer = index_first_axis(+ query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k+ )+ cu_seqlens_q = cu_seqlens_k+ max_seqlen_in_batch_q = max_seqlen_in_batch_k+ indices_q = indices_k+ elif query_length == 1:+ max_seqlen_in_batch_q = 1+ cu_seqlens_q = torch.arange(+ batch_size + 1, dtype=torch.int32, device=query_layer.device+ ) # There is a memcpy here, that is very bad.+ indices_q = cu_seqlens_q[:-1]+ query_layer = query_layer.squeeze(1)+ else:+ # The -q_len: slice assumes left padding.+ attention_mask = attention_mask[:, -query_length:]+ query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)++ return (+ query_layer,+ key_layer,+ value_layer,+ indices_q,+ (cu_seqlens_q, cu_seqlens_k),+ (max_seqlen_in_batch_q, max_seqlen_in_batch_k),+ )+++# copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Phi3+# TODO @Arthur no longer copied from LLama after static cache+class Phi3SdpaAttention(Phi3Attention):+ """+ Phi3 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from+ `Phi3Attention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to+ SDPA API.+ """++ # Adapted from Phi3Attention.forward+ def forward(+ self,+ hidden_states: torch.Tensor,+ attention_mask: Optional[torch.Tensor] = None,+ position_ids: Optional[torch.LongTensor] = None,+ past_key_value: Optional[Cache] = None,+ output_attentions: bool = False,+ use_cache: bool = False,+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:+ if output_attentions:+ # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.+ logger.warning_once(+ "Phi3Model is using Phi3SdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "+ 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'+ )+ return super().forward(+ hidden_states=hidden_states,+ attention_mask=attention_mask,+ position_ids=position_ids,+ past_key_value=past_key_value,+ output_attentions=output_attentions,+ use_cache=use_cache,+ )++ bsz, q_len, _ = hidden_states.size()++ qkv = self.qkv_proj(hidden_states)+ query_pos = self.num_heads * self.head_dim+ query_states = qkv[..., :query_pos]+ key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]+ value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]++ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)++ kv_seq_len = key_states.shape[-2]+ if past_key_value is not None:+ kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)+ cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len)++ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)++ if past_key_value is not None:+ cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)++ key_states = repeat_kv(key_states, self.num_key_value_groups)+ value_states = repeat_kv(value_states, self.num_key_value_groups)++ if attention_mask is not None:+ if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):+ raise ValueError(+ f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"+ )++ # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,+ # Reference: https://github.com/pytorch/pytorch/issues/112577.+ if query_states.device.type == "cuda" and attention_mask is not None:+ query_states = query_states.contiguous()+ key_states = key_states.contiguous()+ value_states = value_states.contiguous()++ attn_output = torch.nn.functional.scaled_dot_product_attention(+ query_states,+ key_states,+ value_states,+ attn_mask=attention_mask,+ dropout_p=self.attention_dropout if self.training else 0.0,+ # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1.+ is_causal=self.is_causal and attention_mask is None and q_len > 1,+ )++ attn_output = attn_output.transpose(1, 2).contiguous()+ attn_output = attn_output.view(bsz, q_len, self.hidden_size)++ attn_output = self.o_proj(attn_output)++ return attn_output, None, past_key_value+++PHI3_ATTENTION_CLASSES = {+ "eager": Phi3Attention,+ "flash_attention_2": Phi3FlashAttention2,+ "sdpa": Phi3SdpaAttention,+}+++class Phi3DecoderLayer(nn.Module):+ def __init__(self, config: Phi3VConfig, layer_idx: int):+ super().__init__()++ self.config = config+ self.self_attn = PHI3_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx)++ self.mlp = Phi3MLP(config)+ self.input_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)++ self.resid_attn_dropout = nn.Dropout(config.resid_pdrop)+ self.resid_mlp_dropout = nn.Dropout(config.resid_pdrop)+ self.post_attention_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)++ def forward(+ self,+ hidden_states: torch.Tensor,+ attention_mask: Optional[torch.Tensor] = None,+ position_ids: Optional[torch.LongTensor] = None,+ past_key_value: Optional[Tuple[torch.Tensor]] = None,+ output_attentions: Optional[bool] = False,+ use_cache: Optional[bool] = False,+ **kwargs,+ ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:+ if "padding_mask" in kwargs:+ warnings.warn(+ "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"+ )+ """+ Args:+ hidden_states (`torch.FloatTensor`):+ input to the layer of shape `(batch, seq_len, embed_dim)`+ attention_mask (`torch.FloatTensor`, *optional*): attention mask of size+ `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range+ `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)+ output_attentions (`bool`, *optional*):+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under+ returned tensors for more detail.+ use_cache (`bool`, *optional*):+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding+ (see `past_key_values`).+ past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states+ """++ residual = hidden_states++ hidden_states = self.input_layernorm(hidden_states)++ # Self Attention+ attn_outputs, self_attn_weights, present_key_value = self.self_attn(+ hidden_states=hidden_states,+ attention_mask=attention_mask,+ position_ids=position_ids,+ past_key_value=past_key_value,+ output_attentions=output_attentions,+ use_cache=use_cache,+ )++ hidden_states = residual + self.resid_attn_dropout(attn_outputs)++ residual = hidden_states+ hidden_states = self.post_attention_layernorm(hidden_states)+ hidden_states = self.mlp(hidden_states)+ hidden_states = residual + self.resid_mlp_dropout(hidden_states)++ outputs = (hidden_states,)++ if output_attentions:+ outputs += (self_attn_weights,)++ if use_cache:+ outputs += (present_key_value,)++ return outputs+++PHI3V_START_DOCSTRING = r"""+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads+ etc.)++ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage+ and behavior.++ Parameters:+ config ([`Phi3VConfig`]):+ Model configuration class with all the parameters of the model. Initializing with a config file does not+ load the weights associated with the model, only the configuration. Check out the+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.+"""+++@add_start_docstrings(+ "The bare Phi-3-V model outputting raw hidden-states without any specific head on top.",+ PHI3V_START_DOCSTRING,+)+class Phi3VPreTrainedModel(PreTrainedModel):+ config_class = Phi3VConfig+ base_model_prefix = "model"+ supports_gradient_checkpointing = True+ _no_split_modules = ["Phi3DecoderLayer"]+ _skip_keys_device_placement = "past_key_values"+ _supports_flash_attn_2 = True+ _supports_sdpa = False+ _supports_cache_class = True++ _version = "0.0.5"++ def _init_weights(self, module):+ std = self.config.initializer_range+ if isinstance(module, nn.Linear):+ module.weight.data.normal_(mean=0.0, std=std)+ if module.bias is not None:+ module.bias.data.zero_()+ elif isinstance(module, nn.Embedding):+ module.weight.data.normal_(mean=0.0, std=std)+ if module.padding_idx is not None:+ module.weight.data[module.padding_idx].zero_()+++PHI3V_INPUTS_DOCSTRING = r"""+ Args:+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide+ it.++ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and+ [`PreTrainedTokenizer.__call__`] for details.++ [What are input IDs?](../glossary#input-ids)+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:++ - 1 for tokens that are **not masked**,+ - 0 for tokens that are **masked**.++ [What are attention masks?](../glossary#attention-mask)++ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and+ [`PreTrainedTokenizer.__call__`] for details.++ If `past_key_values` is used, optionally only the last `input_ids` have to be input (see+ `past_key_values`).++ If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]+ and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more+ information on the default strategy.++ - 1 indicates the head is **not masked**,+ - 0 indicates the head is **masked**.+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,+ config.n_positions - 1]`.++ [What are position IDs?](../glossary#position-ids)+ past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):+ Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention+ blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`+ returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.++ Two formats are allowed:+ - a [`~cache_utils.Cache`] instance;+ - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy+ cache format.++ The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the+ legacy cache format will be returned.++ If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't+ have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`+ of shape `(batch_size, sequence_length)`.+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the+ model's internal embedding lookup matrix.+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):+ The tensors corresponding to the input images. Pixel values can be obtained using [`AutoImageProcessor`].+ See [`Phi3ImageProcessor.__call__`] for details.+ image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*):+ The sizes of the images in the batch, being (height, width) for each image.+ use_cache (`bool`, *optional*):+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see+ `past_key_values`).+ output_attentions (`bool`, *optional*):+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned+ tensors for more detail.+ output_hidden_states (`bool`, *optional*):+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for+ more detail.+ return_dict (`bool`, *optional*):+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.+"""+++@add_start_docstrings(+ "The bare Phi-3-V model outputting raw hidden-states without any specific head on top.",+ PHI3V_START_DOCSTRING,+)+class Phi3VModel(Phi3VPreTrainedModel):+ """+ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi3DecoderLayer`]++ Args:+ config: Phi3Config+ """++ def __init__(self, config: Phi3VConfig):+ super().__init__(config)+ self.padding_idx = config.pad_token_id+ self.vocab_size = config.vocab_size++ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)+ self.embed_dropout = nn.Dropout(config.embd_pdrop)++ self.vision_embed_tokens = None+ if isinstance(config.embd_layer, dict):+ # vision embedding layer+ embedding_config = {+ 'embedding_cls': config.embd_layer['embedding_cls'],+ **config.embd_layer+ }+ self.vision_embed_tokens = Phi3ImageEmbedding(config, wte=self.embed_tokens, **embedding_config)+ # # set wte the same for vision embedding+ # self.vision_embed_tokens.wte.weight = self.embed_tokens.weight++ self.layers = nn.ModuleList(+ [Phi3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]+ )+ self._attn_implementation = config._attn_implementation+ self.norm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)++ self.gradient_checkpointing = False+ # Initialize weights and apply final processing+ self.post_init()++ def get_input_embeddings(self):+ return self.embed_tokens++ def set_input_embeddings(self, value):+ self.embed_tokens = value++ @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING)+ def forward(+ self,+ input_ids: torch.LongTensor = None,+ attention_mask: Optional[torch.Tensor] = None,+ position_ids: Optional[torch.LongTensor] = None,+ past_key_values: Optional[List[torch.FloatTensor]] = None,+ inputs_embeds: Optional[torch.FloatTensor] = None,+ pixel_values: Optional[torch.FloatTensor] = None,+ image_sizes: Optional[torch.LongTensor] = None,+ use_cache: Optional[bool] = None,+ output_attentions: Optional[bool] = None,+ output_hidden_states: Optional[bool] = None,+ return_dict: Optional[bool] = None,+ ) -> Union[Tuple, BaseModelOutputWithPast]:+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions+ output_hidden_states = (+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states+ )+ use_cache = use_cache if use_cache is not None else self.config.use_cache++ return_dict = return_dict if return_dict is not None else self.config.use_return_dict++ # retrieve input_ids and inputs_embeds+ if input_ids is not None and inputs_embeds is not None:+ raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")+ elif input_ids is not None:+ batch_size, seq_length = input_ids.shape[:2]+ elif inputs_embeds is not None:+ batch_size, seq_length = inputs_embeds.shape[:2]+ else:+ raise ValueError("You have to specify either input_ids or inputs_embeds")++ past_key_values_length = 0++ if self.gradient_checkpointing and self.training:+ if use_cache:+ logger.warning_once(+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."+ )+ use_cache = False++ if use_cache:+ use_legacy_cache = not isinstance(past_key_values, Cache)+ if use_legacy_cache:+ past_key_values = DynamicCache.from_legacy_cache(past_key_values)+ past_key_values_length = past_key_values.get_usable_length(seq_length)++ if position_ids is None:+ device = input_ids.device if input_ids is not None else inputs_embeds.device+ position_ids = torch.arange(+ past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device+ )+ position_ids = position_ids.unsqueeze(0).view(-1, seq_length)+ else:+ position_ids = position_ids.view(-1, seq_length).long()++ if inputs_embeds is None:+ if pixel_values is not None and image_sizes is not None:+ assert self.vision_embed_tokens is not None, "Vision embedding layer is not defined"+ inputs_embeds = self.vision_embed_tokens(input_ids, pixel_values=pixel_values, image_sizes=image_sizes)+ else:+ inputs_embeds = self.embed_tokens(input_ids)++ if attention_mask is not None and self._attn_implementation == "flash_attention_2" and use_cache:+ is_padding_right = attention_mask[:, -1].sum().item() != batch_size+ if is_padding_right:+ raise ValueError(+ "You are attempting to perform batched generation with padding_side='right'"+ " this may lead to unexpected behaviour for Flash Attention version of Phi3. Make sure to "+ " call `tokenizer.padding_side = 'left'` before tokenizing the input. "+ )++ if self._attn_implementation == "flash_attention_2":+ # 2d mask is passed through the layers+ attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None+ else:+ # 4d mask is passed through the layers+ attention_mask = _prepare_4d_causal_attention_mask(+ attention_mask,+ (batch_size, seq_length),+ inputs_embeds,+ past_key_values_length,+ sliding_window=self.config.sliding_window,+ )++ hidden_states = inputs_embeds++ # decoder layers+ all_hidden_states = () if output_hidden_states else None+ all_self_attns = () if output_attentions else None+ next_decoder_cache = None++ for decoder_layer in self.layers:+ if output_hidden_states:+ all_hidden_states += (hidden_states,)++ if self.gradient_checkpointing and self.training:+ layer_outputs = torch.utils.checkpoint.checkpoint(+ decoder_layer.__call__,+ hidden_states,+ attention_mask,+ position_ids,+ past_key_values,+ output_attentions,+ use_cache,+ use_reentrant=False+ )+ else:+ layer_outputs = decoder_layer(+ hidden_states,+ attention_mask=attention_mask,+ position_ids=position_ids,+ past_key_value=past_key_values,+ output_attentions=output_attentions,+ use_cache=use_cache,+ )++ hidden_states = layer_outputs[0]++ if use_cache:+ next_decoder_cache = layer_outputs[2 if output_attentions else 1]++ if output_attentions:+ all_self_attns += (layer_outputs[1],)++ hidden_states = self.norm(hidden_states)++ # add hidden states from the last decoder layer+ if output_hidden_states:+ all_hidden_states += (hidden_states,)++ next_cache = None+ if use_cache:+ next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache+ if not return_dict:+ return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)+ return BaseModelOutputWithPast(+ last_hidden_state=hidden_states,+ past_key_values=next_cache,+ hidden_states=all_hidden_states,+ attentions=all_self_attns,+ )+++class Phi3VForCausalLM(Phi3VPreTrainedModel):+ _tied_weights_keys = ["lm_head.weight"]++ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.__init__ with Llama->Phi3+ def __init__(self, config):+ super().__init__(config)+ self.model = Phi3VModel(config)+ self.vocab_size = config.vocab_size+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)++ # Initialize weights and apply final processing+ self.post_init()++ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_input_embeddings+ def get_input_embeddings(self):+ return self.model.embed_tokens++ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_input_embeddings+ def set_input_embeddings(self, value):+ self.model.embed_tokens = value++ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_output_embeddings+ def get_output_embeddings(self):+ return self.lm_head++ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_output_embeddings+ def set_output_embeddings(self, new_embeddings):+ self.lm_head = new_embeddings++ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_decoder+ def set_decoder(self, decoder):+ self.model = decoder++ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_decoder+ def get_decoder(self):+ return self.model++ # Ignore copy+ @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING)+ @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)+ def forward(+ self,+ input_ids: torch.LongTensor = None,+ attention_mask: Optional[torch.Tensor] = None,+ position_ids: Optional[torch.LongTensor] = None,+ past_key_values: Optional[List[torch.FloatTensor]] = None,+ inputs_embeds: Optional[torch.FloatTensor] = None,+ pixel_values: Optional[torch.FloatTensor] = None,+ image_sizes: Optional[torch.LongTensor] = None,+ labels: Optional[torch.LongTensor] = None,+ use_cache: Optional[bool] = None,+ output_attentions: Optional[bool] = None,+ output_hidden_states: Optional[bool] = None,+ return_dict: Optional[bool] = None,+ ) -> Union[Tuple, CausalLMOutputWithPast]:+ r"""+ Args:+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.++ Returns:++ Example:++ ```python+ >>> from transformers import AutoTokenizer, Phi3ForCausalLM++ >>> model = Phi3ForCausalLM.from_pretrained("microsoft/phi-3-mini-4k-instruct")+ >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-3-mini-4k-instruct")++ >>> prompt = "This is an example script ."+ >>> inputs = tokenizer(prompt, return_tensors="pt")++ >>> # Generate+ >>> generate_ids = model.generate(inputs.input_ids, max_length=30)+ >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]+ 'This is an example script .\n Certainly! Below is a sample script that demonstrates a simple task, such as calculating the sum'+ ```"""++ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions+ output_hidden_states = (+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states+ )+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict++ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)+ outputs = self.model(+ input_ids=input_ids,+ attention_mask=attention_mask,+ position_ids=position_ids,+ past_key_values=past_key_values,+ inputs_embeds=inputs_embeds,+ pixel_values=pixel_values,+ image_sizes=image_sizes,+ use_cache=use_cache,+ output_attentions=output_attentions,+ output_hidden_states=output_hidden_states,+ return_dict=return_dict,+ )++ hidden_states = outputs[0]+ logits = self.lm_head(hidden_states)+ logits = logits.float()++ loss = None+ if labels is not None:+ # Shift so that tokens < n predict n+ shift_logits = logits[..., :-1, :].contiguous()+ shift_labels = labels[..., 1:].contiguous()+ # Flatten the tokens+ loss_fct = CrossEntropyLoss()+ shift_logits = shift_logits.view(-1, self.config.vocab_size)+ shift_labels = shift_labels.view(-1)+ # Enable model parallelism+ shift_labels = shift_labels.to(shift_logits.device)+ loss = loss_fct(shift_logits, shift_labels)++ if not return_dict:+ output = (logits,) + outputs[1:]+ return (loss,) + output if loss is not None else output++ return CausalLMOutputWithPast(+ loss=loss,+ logits=logits,+ past_key_values=outputs.past_key_values,+ hidden_states=outputs.hidden_states,+ attentions=outputs.attentions,+ )++ # Copied from transformers.models.persimmon.modeling_persimmon.PersimmonForCausalLM.prepare_inputs_for_generation+ def prepare_inputs_for_generation(+ self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, pixel_values=None, image_sizes=None, **kwargs+ ):+ if past_key_values is not None:+ if isinstance(past_key_values, Cache):+ cache_length = past_key_values.get_seq_length()+ past_length = past_key_values.seen_tokens+ max_cache_length = past_key_values.get_max_length()+ else:+ cache_length = past_length = past_key_values[0][0].shape[2]+ max_cache_length = None++ # Keep only the unprocessed tokens:+ # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where+ # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as+ # input)+ if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:+ input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]+ # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard+ # input_ids based on the past_length.+ elif past_length < input_ids.shape[1]:+ input_ids = input_ids[:, past_length:]+ # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.++ # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.+ if (+ max_cache_length is not None+ and attention_mask is not None+ and cache_length + input_ids.shape[1] > max_cache_length+ ):+ attention_mask = attention_mask[:, -max_cache_length:]++ position_ids = kwargs.get("position_ids", None)+ if attention_mask is not None and position_ids is None:+ # create position_ids on the fly for batch generation+ position_ids = attention_mask.long().cumsum(-1) - 1+ position_ids.masked_fill_(attention_mask == 0, 1)+ if past_key_values:+ position_ids = position_ids[:, -input_ids.shape[1] :]++ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step+ if inputs_embeds is not None and past_key_values is None:+ model_inputs = {"inputs_embeds": inputs_embeds}+ else:+ model_inputs = {"input_ids": input_ids}++ model_inputs.update(+ {+ "position_ids": position_ids,+ "past_key_values": past_key_values,+ "use_cache": kwargs.get("use_cache"),+ "attention_mask": attention_mask,+ "pixel_values": pixel_values,+ "image_sizes": image_sizes,+ }+ )+ return model_inputs++ @staticmethod+ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM._reorder_cache+ def _reorder_cache(past_key_values, beam_idx):+ reordered_past = ()+ for layer_past in past_key_values:+ reordered_past += (+ tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),+ )+ return reordered_past+++@add_start_docstrings(+ """+ The [`Phi3VModel`] with a sequence classification head on top (linear layer).++ [`Phi3VForSequenceClassification`] uses the last token in order to do the classification, as other causal models+ (e.g. GPT-2) do.++ Since it does classification on the last token, it requires to know the position of the last token. If a+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in+ each row of the batch).+ """,+ PHI3V_START_DOCSTRING,+)+# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Phi3, LLAMA->PHI3, self.transformer->self.model, transformer_outputs->model_outputs+class Phi3VForSequenceClassification(Phi3VPreTrainedModel):+ def __init__(self, config):+ super().__init__(config)+ self.num_labels = config.num_labels+ self.model = Phi3VModel(config)+ self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)++ # Initialize weights and apply final processing+ self.post_init()++ def get_input_embeddings(self):+ return self.model.embed_tokens++ def set_input_embeddings(self, value):+ self.model.embed_tokens = value++ @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING)+ def forward(+ self,+ input_ids: torch.LongTensor = None,+ attention_mask: Optional[torch.Tensor] = None,+ position_ids: Optional[torch.LongTensor] = None,+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,+ inputs_embeds: Optional[torch.FloatTensor] = None,+ pixel_values: Optional[torch.FloatTensor] = None,+ image_sizes: Optional[torch.LongTensor] = None,+ labels: Optional[torch.LongTensor] = None,+ use_cache: Optional[bool] = None,+ output_attentions: Optional[bool] = None,+ output_hidden_states: Optional[bool] = None,+ return_dict: Optional[bool] = None,+ ) -> Union[Tuple, SequenceClassifierOutputWithPast]:+ r"""+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).+ """+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict++ model_outputs = self.model(+ input_ids,+ attention_mask=attention_mask,+ position_ids=position_ids,+ past_key_values=past_key_values,+ inputs_embeds=inputs_embeds,+ pixel_values=pixel_values,+ image_sizes=image_sizes,+ use_cache=use_cache,+ output_attentions=output_attentions,+ output_hidden_states=output_hidden_states,+ return_dict=return_dict,+ )+ hidden_states = model_outputs[0]+ logits = self.score(hidden_states)++ if input_ids is not None:+ batch_size = input_ids.shape[0]+ else:+ batch_size = inputs_embeds.shape[0]++ if self.config.pad_token_id is None and batch_size != 1:+ raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")+ if self.config.pad_token_id is None:+ sequence_lengths = -1+ else:+ if input_ids is not None:+ # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility+ sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1+ sequence_lengths = sequence_lengths % input_ids.shape[-1]+ sequence_lengths = sequence_lengths.to(logits.device)+ else:+ sequence_lengths = -1++ pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]++ loss = None+ if labels is not None:+ labels = labels.to(logits.device)+ if self.config.problem_type is None:+ if self.num_labels == 1:+ self.config.problem_type = "regression"+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):+ self.config.problem_type = "single_label_classification"+ else:+ self.config.problem_type = "multi_label_classification"++ if self.config.problem_type == "regression":+ loss_fct = MSELoss()+ if self.num_labels == 1:+ loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())+ else:+ loss = loss_fct(pooled_logits, labels)+ elif self.config.problem_type == "single_label_classification":+ loss_fct = CrossEntropyLoss()+ loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))+ elif self.config.problem_type == "multi_label_classification":+ loss_fct = BCEWithLogitsLoss()+ loss = loss_fct(pooled_logits, labels)+ if not return_dict:+ output = (pooled_logits,) + model_outputs[1:]+ return ((loss,) + output) if loss is not None else output++ return SequenceClassifierOutputWithPast(+ loss=loss,+ logits=pooled_logits,+ past_key_values=model_outputs.past_key_values,+ hidden_states=model_outputs.hidden_states,+ attentions=model_outputs.attentions,+ )+++@add_start_docstrings(+ """+ [`Phi3VModel`] with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for+ Named-Entity-Recognition (NER) tasks.+ """,+ PHI3V_START_DOCSTRING,+)+# Copied from transformers.models.mpt.modeling_mpt.MptForTokenClassification with Mpt->Phi3,MPT->PHI3,self.transformer->self.model,transformer_outputs->model_outputs+class Phi3VForTokenClassification(Phi3VPreTrainedModel):+ def __init__(self, config: Phi3VConfig):+ super().__init__(config)+ self.num_labels = config.num_labels++ self.model = Phi3VModel(config)+ if hasattr(config, "classifier_dropout") and config.classifier_dropout is not None:+ classifier_dropout = config.classifier_dropout+ elif hasattr(config, "hidden_dropout") and config.hidden_dropout is not None:+ classifier_dropout = config.hidden_dropout+ else:+ classifier_dropout = 0.1+ self.dropout = nn.Dropout(classifier_dropout)+ self.classifier = nn.Linear(config.hidden_size, config.num_labels)++ # Initialize weights and apply final processing+ self.post_init()++ @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING)+ @add_code_sample_docstrings(+ checkpoint=_CHECKPOINT_FOR_DOC,+ output_type=TokenClassifierOutput,+ config_class=_CONFIG_FOR_DOC,+ )+ def forward(+ self,+ input_ids: Optional[torch.LongTensor] = None,+ past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,+ attention_mask: Optional[torch.Tensor] = None,+ inputs_embeds: Optional[torch.Tensor] = None,+ pixel_values: Optional[torch.FloatTensor] = None,+ image_sizes: Optional[torch.LongTensor] = None,+ labels: Optional[torch.Tensor] = None,+ use_cache: Optional[bool] = None,+ output_attentions: Optional[bool] = None,+ output_hidden_states: Optional[bool] = None,+ return_dict: Optional[bool] = None,+ **deprecated_arguments,+ ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]:+ r"""+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).+ """+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict++ model_outputs = self.model(+ input_ids,+ past_key_values=past_key_values,+ attention_mask=attention_mask,+ inputs_embeds=inputs_embeds,+ pixel_values=pixel_values,+ image_sizes=image_sizes,+ use_cache=use_cache,+ output_attentions=output_attentions,+ output_hidden_states=output_hidden_states,+ return_dict=return_dict,+ )++ hidden_states = model_outputs[0]+ hidden_states = self.dropout(hidden_states)+ logits = self.classifier(hidden_states)++ loss = None+ if labels is not None:+ # move labels to correct device to enable model parallelism+ labels = labels.to(logits.device)+ batch_size, seq_length = labels.shape+ loss_fct = CrossEntropyLoss()+ loss = loss_fct(+ logits.view(batch_size * seq_length, self.num_labels), labels.view(batch_size * seq_length)+ )++ if not return_dict:+ output = (logits,) + model_outputs[2:]+ return ((loss,) + output) if loss is not None else output++ return TokenClassifierOutput(+ loss=loss,+ logits=logits,+ hidden_states=model_outputs.hidden_states,+ attentions=model_outputs.attentions,+ )\ No newline at end of filediff --git a/examples/multimodal-modeling/Phi-3-vision/model/processing_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/processing_phi3_v.pynew file mode 100644index 000000000..1263845d1--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/model/processing_phi3_v.py@@ -0,0 +1,296 @@+# coding=utf-8+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# Modified by Github@GaiZhenbiao++"""+Processor class for Phi3-V.+"""+import re+from typing import List, Optional, Union++import torch+from .image_processing_phi3_v import Phi3VImageProcessor+from transformers.feature_extraction_utils import BatchFeature+from transformers.image_utils import ImageInput+from transformers.processing_utils import ProcessorMixin+from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy+from transformers.utils import TensorType+import transformers+transformers.Phi3VImageProcessor = Phi3VImageProcessor++class Phi3VProcessor(ProcessorMixin):+ r"""+ Constructs a Phi3-V processor which wraps a Phi3-V image processor and a LLaMa tokenizer into a single processor.++ [`Phi3VProcessor`] offers all the functionalities of [`Phi3VImageProcessor`] and [`LlamaTokenizerFast`]. See the+ [`~Phi3VProcessor.__call__`] and [`~Phi3VProcessor.decode`] for more information.++ Args:+ image_processor ([`Phi3VImageProcessor`], *optional*):+ The image processor is a required input.+ tokenizer ([`LlamaTokenizerFast`], *optional*):+ The tokenizer is a required input.+ """++ attributes = ["image_processor", "tokenizer"]+ image_processor_class = "Phi3VImageProcessor"+ tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast")+ special_image_token = "<|image|>"++ def __init__(self, image_processor, tokenizer):+ self.image_processor = image_processor+ self.tokenizer = tokenizer+ self.num_img_tokens = image_processor.num_img_tokens+ self.img_tokens = [f"<|image_{i+1}|>" for i in range(1000000)]++ def __call__(+ self,+ text: Union[TextInput, List[TextInput]],+ images: ImageInput = None,+ padding: Union[bool, str, PaddingStrategy] = False,+ truncation: Union[bool, str, TruncationStrategy] = None,+ max_length=None,+ return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,+ ) -> BatchFeature:+ """+ Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`+ and `kwargs` arguments to LlamaTokenizerFast's [`~LlamaTokenizerFast.__call__`] if `text` is not `None` to encode+ the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to+ Phi3ImageProcessor's [`~Phi3ImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring+ of the above two methods for more information.++ Args:+ text (`str`, `List[str]`, `List[List[str]]`):+ The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings+ (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set+ `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).+ images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):+ The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch+ tensor. Both channels-first and channels-last formats are supported.+ padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):+ Select a strategy to pad the returned sequences (according to the model's padding side and padding+ index) among:+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single+ sequence if provided).+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum+ acceptable input length for the model if that argument is not provided.+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different+ lengths).+ max_length (`int`, *optional*):+ Maximum length of the returned list and optionally padding length (see above).+ truncation (`bool`, *optional*):+ Activates truncation to cut input sequences longer than `max_length` to `max_length`.+ return_tensors (`str` or [`~utils.TensorType`], *optional*):+ If set, will return tensors of a particular framework. Acceptable values are:++ - `'tf'`: Return TensorFlow `tf.constant` objects.+ - `'pt'`: Return PyTorch `torch.Tensor` objects.+ - `'np'`: Return NumPy `np.ndarray` objects.+ - `'jax'`: Return JAX `jnp.ndarray` objects.++ Returns:+ [`BatchFeature`]: A [`BatchFeature`] with the following fields:++ - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.+ - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when+ `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not+ `None`).+ - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.+ """+ if images is not None:+ image_inputs = self.image_processor(images, return_tensors=return_tensors)+ else:+ image_inputs = {}+ inputs = self._convert_images_texts_to_inputs(image_inputs, text, padding=padding, truncation=truncation, max_length=max_length, return_tensors=return_tensors)+ return inputs++ def calc_num_image_tokens(self, images: ImageInput):+ """ Calculate the number of image tokens for each image.+ Args:+ images (`ImageInput`):+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.+ """+ return self.image_processor.calc_num_image_tokens(images)++ def calc_num_image_tokens_from_image_size(self, width, height):+ """ Calculate the number of image token for an image with given width and height.+ Args:+ width (`int`):+ Width of the image.+ height (`int`):+ Height of the image.+ """+ return self.image_processor.calc_num_image_tokens_from_image_size(width, height)+++ @property+ def special_image_token_id(self):+ return self.tokenizer.convert_tokens_to_ids(self.special_image_token)++ def get_special_image_token_id(self):+ return self.tokenizer.convert_tokens_to_ids(self.special_image_token)++ def _convert_images_texts_to_inputs(self, images, texts, padding=False, truncation=None, max_length=None, return_tensors=None):++ def split_with_separators(s, separators):+ parts = []+ start = 0+ sep_len = {sep: len(sep) for sep in separators}+ while start < len(s):+ index = min((s.find(sep, start), sep) for sep in separators if s.find(sep, start) != -1)+ if index[0] == -1:+ parts.append(s[start:])+ break+ if s[start:index[0]]:+ parts.append(s[start:index[0]])+ parts.append(index[1])+ start = index[0] + sep_len[index[1]]+ return parts++ def split_with_roles(input_text):+ parts = split_with_separators(input_text, ["<|user|>\n", "<|end|>\n", "<|assistant|>\n", "<|image_1|>"])+ new_parts = []+ current_role = None+ for p in parts:+ if p in ["<|user|>\n", "<|assistant|>\n", "<|end|>\n"]:+ if p == "<|user|>\n":+ current_role = "user"+ elif p == "<|assistant|>\n":+ current_role = "assistant"+ _type = ["<|user|>\n", "<|assistant|>\n", "<|end|>\n"].index(p) + 1+ new_parts.append({"role": current_role, "content": p, "type": _type})+ else:+ new_parts.append({"role": current_role, "content": p, "type": 0})+ return new_parts++ if not len(images):+ model_inputs = self.tokenizer(texts, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length)+ # prompt_chunks = []+ label_prompt_chunks = []+ # the behavior of the tokenizer is very very weird, what I observed is concluded by the following:+ # 1. "<|user|>\n" is encoded as 3 tokens, while "<|assistant|>\n" is encoded as 1 tokens+ # 2. tokenizing "I am here" and "\nI am here", the tokens of "I" in these two cases are different ("I" can be any word and is used as an example here)+ # 3. when tokenizing "<|user|>\nI am here", the tokens of "I" follow the tokenization of "I" in "\nI am here"+ # 4. when tokenizing "<|assistant|>\nI am here", the tokens of "I" follow the tokenization of "I" in "I am here"+ # [Edited by zhenwei - 2024-06-01 22:25]+ for chunk in split_with_roles(texts):+ if chunk["role"] == "assistant" and chunk['type'] in [0, 3]:+ tmp_input_ids = self.tokenizer(chunk["content"], add_special_tokens=False).input_ids+ # prompt_chunks.append(tmp_input_ids)+ label_prompt_chunks.append(tmp_input_ids)+ else:+ tmp_input_ids = self.tokenizer('\n' + chunk["content"], add_special_tokens=False).input_ids[2:]+ # prompt_chunks.append(tmp_input_ids)+ label_prompt_chunks.append([-100 for _ in range(len(tmp_input_ids))])++ labels = [-100]+ for chunk in label_prompt_chunks:+ labels.extend(chunk)+ # input_ids = [1]+ # for chunk in prompt_chunks:+ # input_ids.extend(chunk)++ labels = torch.tensor(labels, dtype=torch.long).unsqueeze(0)+ # with open('tmp/input_ids.txt', 'w') as f:+ # print(texts, file=f)+ # print(split_with_roles(texts), file=f)+ # print("input_ids_before", file=f)+ # print(model_inputs['input_ids'][0].tolist(), file=f)+ # print("input_ids", file=f)+ # print(input_ids, file=f)+ assert labels.shape[1] == model_inputs['input_ids'].shape[1], f"labels length: {labels.shape[1]}, input_ids length: {model_inputs['input_ids'].shape[1]}"+ return BatchFeature(data={**model_inputs, "labels": labels})+++ if 'num_img_tokens' in images:+ num_img_tokens = images['num_img_tokens']+ else:+ assert 'num_crops' in images, 'num_crops must be provided in images if num_img_tokens is not provided'+ num_crops = images['num_crops']+ num_img_tokens = [_num_crops * self.num_img_tokens for _num_crops in num_crops]++ images, image_sizes = images['pixel_values'], images['image_sizes']++ pattern = r"<\|image_\d+\|>"+ # image_tags needs to start from 1 to n+ image_tags = re.findall(pattern, texts)+ # image_ids = [int(s.split("|")[1].split("_")[-1]) * -1 for s in image_tags]+ # image_ids_pad = [[iid]*num_img_tokens[i] for i, iid in enumerate(image_ids)]+ image_ids = [int(s.split("|")[1].split("_")[-1]) for s in image_tags]+ unique_image_ids = sorted(list(set(image_ids)))+ # image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be [1, 4, 5]+ # check the condition+ assert unique_image_ids == list(range(1, len(unique_image_ids)+1)), f"image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be {unique_image_ids}"+ # total images must be the same as the number of image tags+ assert len(unique_image_ids) == len(images), f"total images must be the same as the number of image tags, got {len(unique_image_ids)} image tags and {len(images)} images"++ image_ids_pad = [[-iid]*num_img_tokens[iid-1] for iid in image_ids]++ prompt_chunks = []+ label_prompt_chunks = []+ for chunk in split_with_roles(texts):+ if chunk["role"] == "assistant" and chunk['type'] in [0, 3]:+ tmp_input_ids = self.tokenizer(chunk["content"], add_special_tokens=False).input_ids+ prompt_chunks.append(tmp_input_ids)+ label_prompt_chunks.append(tmp_input_ids)+ else:+ if chunk["content"] == "<|image_1|>":+ tmp_input_ids = image_ids_pad.pop(0)+ else:+ tmp_input_ids = self.tokenizer('\n' + chunk["content"], add_special_tokens=False).input_ids[2:]+ prompt_chunks.append(tmp_input_ids)+ label_prompt_chunks.append([-100 for _ in range(len(tmp_input_ids))])++ input_ids = [1]+ labels = [-100]+ for chunk in prompt_chunks:+ input_ids.extend(chunk)+ for chunk in label_prompt_chunks:+ labels.extend(chunk)+ input_ids = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0)+ labels = torch.tensor(labels, dtype=torch.long).unsqueeze(0)+ attention_mask = (input_ids > -1000000).to(torch.long)++ return BatchFeature(data={"input_ids": input_ids,+ "attention_mask": attention_mask,+ "pixel_values": images,+ "image_sizes": image_sizes,+ "labels": labels})+++ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama+ def batch_decode(self, *args, **kwargs):+ """+ This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please+ refer to the docstring of this method for more information.+ """+ return self.tokenizer.batch_decode(*args, **kwargs)++ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama+ def decode(self, *args, **kwargs):+ """+ This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to+ the docstring of this method for more information.+ """+ return self.tokenizer.decode(*args, **kwargs)++ @property+ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names+ def model_input_names(self):+ tokenizer_input_names = self.tokenizer.model_input_names+ image_processor_input_names = self.image_processor.model_input_names+ return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))\ No newline at end of filediff --git a/examples/multimodal-modeling/Phi-3-vision/requirements.txt b/examples/multimodal-modeling/Phi-3-vision/requirements.txtnew file mode 100644index 000000000..43e29fe2f--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/requirements.txt@@ -0,0 +1,18 @@+transformers==4.41.0+torch+tiktoken+transformers_stream_generator+peft+sentencepiece+einops+accelerate+datasets+protobuf+auto-gptq+openpyxl+wandb+py-cpuinfo+Pillow+torchvision+lm-eval==0.4.2+setuptools==69.5.1\ No newline at end of filediff --git a/examples/multimodal-modeling/Phi-3-vision/run_autoround.sh b/examples/multimodal-modeling/Phi-3-vision/run_autoround.shnew file mode 100644index 000000000..3583704bc--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/run_autoround.sh@@ -0,0 +1,11 @@+#!/bin/bash+set -x+device=0+model_name=microsoft/Phi-3-vision-128k-instruct+CUDA_VISIBLE_DEVICES=$device \+python3 main.py \+--model_name=$model_name \+--deployment_device 'auto_round' \+--image_folder /PATH/TO/coco/images/train2017 \+--question_file /PATH/TO/llava_v1_5_mix665k.json \+--output_dir "./tmp_autoround"diff --git a/examples/multimodal-modeling/Phi-3-vision/run_autoround_on_gaudi.sh b/examples/multimodal-modeling/Phi-3-vision/run_autoround_on_gaudi.shnew file mode 100644index 000000000..90764afbd--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/run_autoround_on_gaudi.sh@@ -0,0 +1,10 @@+#!/bin/bash+set -x+model_name=microsoft/Phi-3-vision-128k-instruct++python3 main.py \+ --model_name $model_name \+ --group_size 128 \+ --bits 4 \+ --deployment_device "fake" \+ --output_dir "./tmp_autoround"\ No newline at end of filediff --git a/examples/multimodal-modeling/Phi-3-vision/run_eval.sh b/examples/multimodal-modeling/Phi-3-vision/run_eval.shnew file mode 100644index 000000000..da14a0213--- /dev/null+++ b/examples/multimodal-modeling/Phi-3-vision/run_eval.sh@@ -0,0 +1,48 @@+export https_proxy=http://proxy.ims.intel.com:911+export http_proxy=http://proxy.ims.intel.com:911+export HF_HOME=/home/weiweiz1/.cache/++# Mistral-7B-Instruct-v0.2+# device=3+# Baichuan2-7B-Chat Phi-3-mini-4k-instruct+# Llama-2-7b-chat-hf+# lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu,+# ceval-valid,cmmlu+# dir=/data5/zww/test_faster/+# dir=/models+# for model in Phi-3-mini-4k-instruct Meta-Llama-3-8B-Instruct+# do+# echo ${model}/default+# CUDA_VISIBLE_DEVICES=$device \+# python3 eval_042/evaluation.py --model_name ${dir}${model}_default/$model-autoround-w4g128-gpu \+# --trust_remote_code \+# --eval_bs 16 --tasks gsm8k,ceval-valid,cmmlu \+# 2>&1| tee -a /data4/zww/test_faster/rounding_${model}_rtn.txt+# echo ${model}/rtn+# done&++device=2+dir=/data4/zww/tmp/+# dir=/data5/models/+for model in Phi-3-vision-128k-instruct+do+ echo ${model}+ CUDA_VISIBLE_DEVICES=$device \+ python3 eval_042/evaluation.py --model_name ${dir}/$model-autoround-w4g128-round \+ --trust_remote_code \+ --eval_bs 16 --tasks lambada_openai \+ 2>&1| tee -a /data4/zww/test_faster/rounding_${model}.txt+ echo ${model}+done+# dir=/data5/zww/test_faster/+# for model in Phi-3-mini-4k-instruct Mistral-7B-Instruct-v0.2+# do+# echo ${model}/rtn+# CUDA_VISIBLE_DEVICES=$device \+# python3 eval_042/evaluation.py --model_name ${dir}${model}_rtn/$model-autoround-w4g128-gpu \+# --trust_remote_code \+# --eval_bs 16 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu,gsm8k \+# 2>&1| tee -a /data4/zww/test_faster/rounding_${model}_rtn.txt+# echo ${model}/rtn+# done+diff --git a/test/test_block_names.py b/test/test_block_names.pyindex 5b6218a51..bee555a22 100644--- a/test/test_block_names.py+++ b/test/test_block_names.py@@ -168,8 +168,18 @@ class TestQuantizationBlocks(unittest.TestCase):import auto_gptqexcept:return- autoround.save_quantized("./saved", inplace=False, safe_serialization=False)- autoround.save_quantized(output_dir="./saved", inplace=False, format="auto_round")+ quantized_model_path = "./saved"+ autoround.save_quantized(quantized_model_path, inplace=False, safe_serialization=False, format="auto_round")++ from auto_round.auto_quantizer import AutoHfQuantizer+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree("./saved", ignore_errors=True)+ quant_config = model.config.quantization_config+ assert quant_config.quant_block_list is not None@@ -177,3 +187,4 @@ class TestQuantizationBlocks(unittest.TestCase):if __name__ == "__main__":unittest.main()+