Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enable phi3v tuning (#197)

WeiweiZhang1f2fef1309b57
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index a1f18cd4b..65125c7d1 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -325,7 +325,6 @@ class AutoRoundQuantizer(HfQuantizer):
"""
from auto_round.utils import get_layer_names_in_block
- layer_names = get_layer_names_in_block(model)
quantization_config = model.config.quantization_config
if hasattr(quantization_config, "backend"): # pragma: no cover
backend = quantization_config.backend
@@ -337,6 +336,9 @@ class AutoRoundQuantizer(HfQuantizer):
data_type = quantization_config.data_type if hasattr(quantization_config, "data_type") \
else "int" # pragma: no cover
sym = quantization_config.sym
+ quant_block_list = quantization_config.quant_block_list \
+ if hasattr(quantization_config, "quant_block_list") else None
+ layer_names = get_layer_names_in_block(model, quant_block_list=quant_block_list)
extra_config = {}
if hasattr(quantization_config, "extra_config"):
extra_config = quantization_config.extra_config
@@ -482,3 +484,4 @@ if transformers_version[0] == 4 and transformers_version[1] < 38:
transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizer
transformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer
+
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 43766ad23..9a232dc66 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1124,6 +1124,7 @@ class AutoRound(object):
"amp",
"nsamples",
"low_gpu_mem_usage",
+ "quant_block_list",
"enable_norm_bias_tuning"
]
if isinstance(self.dataset, str):
@@ -1544,3 +1545,4 @@ class AutoAdamRound(AutoOPTRound):
optimizer,
**kwargs,
)
+
diff --git a/examples/multimodal-modeling/Phi-3-vision/README.md b/examples/multimodal-modeling/Phi-3-vision/README.md
new file mode 100644
index 000000000..3105c051b
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/README.md
@@ -0,0 +1,74 @@
+Step-by-Step
+============
+transformers>=4.41.0
+This document presents step-by-step instructions for auto-round.
+# Run Quantization on Phi-3-vision Models
+
+In this example, we introduce an straight-forward way to execute quantization on some popular multimodal models such as Phi-3-vision.
+
+## Download the calibration data
+
+Our calibration process resembles the official visual instruction tuning process.
+
+Please download the annotation of the final mixture our instruction tuning data [llava_v1_5_mix665k.json](https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/blob/main/llava_v1_5_mix665k.json), and download the images from constituting datasets:
+
+COCO: [train2017](http://images.cocodataset.org/zips/train2017.zip), and unzip the image folder to any directory you desire.
+
+
+## 2. Run Examples
+Enter into the examples folder and install lm-eval to run the evaluation
+```bash
+pip install -r requirements.txt
+```
+
+- **Default Settings:**
+```bash
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name microsoft/Phi-3-vision-128k-instruct --bits 4 --group_size 128
+```
+
+- **Speedup the tuning:**
+
+disable_low_gpu_mem_usage(more gpu memory)
+
+reduce the seqlen to 512(potential large accuracy drop)
+
+or combine them
+
+- **Enable quantized lm-head:**
+
+Currently only support in Intel xpu and AutoRound format, however, we found the fake tuning could improve the accuracy is some scenarios. Disable --low_gpu_mem_usage is strongly recommended if the whole model could be loaded to the device, otherwise it will be quite slow to cache the inputs of lm-head. Another way is reducing nsamples,e.g. 128, to alleviate the issue.
+```bash
+CUDA_VISIBLE_DEVICES=0 python3 main.py --model_name microsoft/Phi-3-vision-128k-instruct --bits 4 --group_size 128 --quant_lm_head
+```
+
+- **Utilizing the AdamW Optimizer:**
+
+Include the flag `--adam`. Note that AdamW is less effective than sign gradient descent in many scenarios we tested.
+
+- **Running on Intel Gaudi2**
+```bash
+bash run_autoround_on_gaudi.sh
+```
+
+
+## 3. Environment
+
+PyTorch 1.8 or higher version is needed
+
+
+## Reference
+If you find SignRound useful for your research, please cite our paper:
+```bash
+@article{cheng2023optimize,
+ title={Optimize Weight Rounding via Signed Gradient Descent for the Quantization of LLMs},
+ author={Cheng, Wenhua and Zhang, Weiwei and Shen, Haihao and Cai, Yiyang and He, Xin and Lv, Kaokao},
+ journal={arXiv preprint arXiv:2309.05516},
+ year={2023}
+}
+```
+
+
+
+
+
+
diff --git a/examples/multimodal-modeling/Phi-3-vision/eval_042/__init__.py b/examples/multimodal-modeling/Phi-3-vision/eval_042/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py
new file mode 100644
index 000000000..e9e63fef9
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/eval_042/evaluation.py
@@ -0,0 +1,616 @@
+import itertools
+import logging
+import random
+import time
+from collections import defaultdict
+from typing import TYPE_CHECKING, List, Optional, Union
+
+import numpy as np
+import torch
+
+import lm_eval.api.metrics
+import lm_eval.api.registry
+import lm_eval.models
+from lm_eval.caching.cache import delete_cache
+from lm_eval.evaluator_utils import (
+ consolidate_results,
+ get_sample_size,
+ get_task_list,
+ prepare_print_tasks,
+ print_writeout,
+ run_task_tests,
+)
+from lm_eval.logging_utils import add_env_info, get_git_commit_hash
+from lm_eval.tasks import TaskManager, get_task_dict
+from lm_eval.utils import eval_logger, positional_deprecated, simple_parse_args_string
+
+if TYPE_CHECKING:
+ from lm_eval.api.model import LM
+ from lm_eval.tasks import Task
+
+
+@positional_deprecated
+def simple_evaluate(
+ model,
+ model_args: Optional[Union[str, dict]] = None,
+ tasks: Optional[List[Union[str, dict, object]]] = None,
+ num_fewshot: Optional[int] = None,
+ batch_size: Optional[int] = None,
+ max_batch_size: Optional[int] = None,
+ device: Optional[str] = None,
+ use_cache: Optional[str] = None,
+ cache_requests: bool = False,
+ rewrite_requests_cache: bool = False,
+ delete_requests_cache: bool = False,
+ limit: Optional[Union[int, float]] = None,
+ bootstrap_iters: int = 100000,
+ check_integrity: bool = False,
+ write_out: bool = False,
+ log_samples: bool = True,
+ gen_kwargs: Optional[str] = None,
+ task_manager: Optional[TaskManager] = None,
+ verbosity: str = "INFO",
+ predict_only: bool = False,
+ random_seed: int = 0,
+ numpy_random_seed: int = 1234,
+ torch_random_seed: int = 1234,
+ user_model = None, ##user model does not support tensor parallelism
+):
+ """Instantiate and evaluate a model on a list of tasks.
+
+ :param model: Union[str, LM]
+ Name of model or LM object, see lm_eval.models.get_model
+ :param model_args: Optional[str, dict]
+ String or dict arguments for each model class, see LM.create_from_arg_string and LM.create_from_arg_object.
+ Ignored if `model` argument is a LM object.
+ :param tasks: list[Union[str, dict, Task]]
+ List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise.
+ :param num_fewshot: int
+ Number of examples in few-shot context
+ :param batch_size: int or str, optional
+ Batch size for model
+ :param max_batch_size: int, optional
+ Maximal batch size to try with automatic batch size detection
+ :param device: str, optional
+ PyTorch device (e.g. "cpu" or "cuda:0") for running models
+ :param use_cache: str, optional
+ A path to a sqlite db file for caching model responses. `None` if not caching.
+ :param cache_requests: bool, optional
+ Speed up evaluation by caching the building of dataset requests. `None` if not caching.
+ :param rewrite_requests_cache: bool, optional
+ Rewrites all of the request cache if set to `True`. `None` if not desired.
+ :param delete_requests_cache: bool, optional
+ Deletes all of the request cache if set to `True`. `None` if not desired.
+ :param limit: int or float, optional
+ Limit the number of examples per task (only use this for testing), If <1, limit is a percentage of the total number of examples.
+ :param bootstrap_iters:
+ Number of iterations for bootstrap statistics
+ :param check_integrity: bool
+ Whether to run the relevant part of the test suite for the tasks
+ :param write_out: bool
+ If True, write out an example document and model input for checking task integrity
+ :param log_samples: bool
+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
+ :param gen_kwargs: str
+ String arguments for model generation
+ Ignored for all tasks with loglikelihood output_type
+ :param predict_only: bool
+ If true only model outputs will be generated and returned. Metrics will not be evaluated
+ :param random_seed: int
+ Random seed for python's random module. If set to None, the seed will not be set.
+ :param numpy_random_seed: int
+ Random seed for numpy. If set to None, the seed will not be set.
+ :param torch_random_seed: int
+ Random seed for torch. If set to None, the seed will not be set.
+
+ :return
+ Dictionary of results
+ """
+ from auto_round.auto_quantizer import AutoHfQuantizer
+ eval_logger.setLevel(getattr(logging, f"{verbosity}"))
+ start_date = time.time()
+
+ if delete_requests_cache:
+ eval_logger.info("Deleting requests cache...")
+ delete_cache()
+
+ seed_message = []
+ if random_seed is not None:
+ # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1412
+ seed_message.append(f"Setting random seed to {random_seed}")
+ random.seed(random_seed)
+
+ if numpy_random_seed is not None:
+ seed_message.append(f"Setting numpy seed to {numpy_random_seed}")
+ np.random.seed(numpy_random_seed)
+
+ if torch_random_seed is not None:
+ seed_message.append(f"Setting torch manual seed to {torch_random_seed}")
+ torch.manual_seed(torch_random_seed)
+
+ if seed_message:
+ eval_logger.info(" | ".join(seed_message))
+
+ if tasks is None:
+ tasks = []
+ if len(tasks) == 0:
+ raise ValueError(
+ "No tasks specified, or no tasks found. Please verify the task names."
+ )
+
+ if gen_kwargs is not None:
+ gen_kwargs = simple_parse_args_string(gen_kwargs)
+ eval_logger.warning(
+ "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. "
+ "Ensure 'do_sample=True' for non-greedy decoding!"
+ )
+ if gen_kwargs == "":
+ gen_kwargs = None
+
+ if isinstance(model, str):
+ if model_args is None:
+ model_args = ""
+
+ if isinstance(model_args, dict):
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_obj(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
+ )
+
+ else:
+ lm = lm_eval.api.registry.get_model(model).create_from_arg_string(
+ model_args,
+ {
+ "batch_size": batch_size,
+ "max_batch_size": max_batch_size,
+ "device": device,
+ },
+ )
+ else:
+ if not isinstance(model, lm_eval.api.model.LM):
+ raise TypeError
+ lm = model
+
+ if use_cache is not None:
+ eval_logger.info(f"Using cache at {use_cache + '_rank' + str(lm.rank) + '.db'}")
+ lm = lm_eval.api.model.CachingLM(
+ lm,
+ use_cache
+ # each rank receives a different cache db.
+ # necessary to avoid multiple writes to cache at once
+ + "_rank"
+ + str(lm.rank)
+ + ".db",
+ )
+ if user_model is not None:
+ lm._model = user_model
+
+ if task_manager is None:
+ task_manager = TaskManager(verbosity)
+
+ task_dict = get_task_dict(tasks, task_manager)
+ for task_name in task_dict.keys():
+ task_obj = task_dict[task_name]
+ if isinstance(task_obj, tuple):
+ _, task_obj = task_obj
+ if task_obj is None:
+ continue
+
+ if task_obj.get_config("output_type") == "generate_until":
+ if gen_kwargs is not None:
+ task_obj.set_config(
+ key="generation_kwargs", value=gen_kwargs, update=True
+ )
+
+ if predict_only:
+ log_samples = True
+ eval_logger.info(
+ f"Processing {task_name} in output-only mode. Metrics will not be calculated!"
+ )
+ # we have to change the class properties post-hoc. This is pretty hacky.
+ task_obj.override_metric(metric_name="bypass")
+
+ # override tasks' fewshot values to the provided num_fewshot arg value
+ # except if tasks have it set to 0 manually in their configs--then we should never overwrite that
+ if num_fewshot is not None:
+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0:
+ eval_logger.info(
+ f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored."
+ )
+ else:
+ eval_logger.warning(
+ f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}"
+ )
+ task_obj.set_config(key="num_fewshot", value=num_fewshot)
+ else:
+ # if num_fewshot not provided, and the task does not define a default one, default to 0
+ if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None:
+ task_obj.set_config(key="num_fewshot", value=0)
+
+ if check_integrity:
+ run_task_tests(task_list=tasks)
+
+ results = evaluate(
+ lm=lm,
+ task_dict=task_dict,
+ limit=limit,
+ cache_requests=cache_requests,
+ rewrite_requests_cache=rewrite_requests_cache,
+ bootstrap_iters=bootstrap_iters,
+ write_out=write_out,
+ log_samples=log_samples,
+ verbosity=verbosity,
+ )
+
+ if lm.rank == 0:
+ if isinstance(model, str):
+ model_name = model
+ elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"):
+ model_name = model.config._name_or_path
+ else:
+ model_name = type(model).__name__
+
+ # add info about the model and few shot config
+ results["config"] = {
+ "model": model_name,
+ "model_args": model_args,
+ "batch_size": batch_size,
+ "batch_sizes": (
+ list(lm.batch_sizes.values()) if hasattr(lm, "batch_sizes") else []
+ ),
+ "device": device,
+ "use_cache": use_cache,
+ "limit": limit,
+ "bootstrap_iters": bootstrap_iters,
+ "gen_kwargs": gen_kwargs,
+ }
+ results["git_hash"] = get_git_commit_hash()
+ results["date"] = start_date
+ add_env_info(results) # additional environment info to results
+ return results
+ else:
+ return None
+
+
+@positional_deprecated
+def evaluate(
+ lm: "LM",
+ task_dict,
+ limit: Optional[int] = None,
+ cache_requests: bool = False,
+ rewrite_requests_cache: bool = False,
+ bootstrap_iters: Optional[int] = 100000,
+ write_out: bool = False,
+ log_samples: bool = True,
+ verbosity: str = "INFO",
+):
+ """Instantiate and evaluate a model on a list of tasks.
+
+ :param lm: obj
+ Language Model
+ :param task_dict: dict[str, Task]
+ Dictionary of tasks. Tasks will be taken to have name type(task).config.task .
+ :param limit: int, optional
+ Limit the number of examples per task (only use this for testing)
+ :param bootstrap_iters:
+ Number of iterations for bootstrap statistics
+ :param write_out: bool
+ If True, write out an example document and model input for checking task integrity
+ :param log_samples: bool
+ If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis
+ :return
+ Dictionary of results
+ """
+
+ eval_logger.setLevel(getattr(logging, f"{verbosity}"))
+
+ # tracks all Instances/requests a model must generate output on.
+ requests = defaultdict(list)
+ # stores the amount to pad out reqs per req. type so that
+ # number of fwd passes per distributed rank is equal
+ padding_requests = defaultdict(int)
+
+ # get lists of group hierarchy and each type of request
+ task_hierarchy, eval_tasks = get_task_list(task_dict)
+ if not log_samples:
+ if not all(
+ "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys()
+ for task_output in eval_tasks
+ ):
+ raise ValueError("log_samples must be True for 'bypass' metric-only tasks")
+ for task_output in eval_tasks:
+ task: Task = task_output.task
+ limit = get_sample_size(task, limit)
+ task.build_all_requests(
+ limit=limit,
+ rank=lm.rank,
+ world_size=lm.world_size,
+ cache_requests=cache_requests,
+ rewrite_requests_cache=rewrite_requests_cache,
+ )
+ eval_logger.debug(
+ f"Task: {task_output.task_name}; number of requests on this rank: {len(task.instances)}"
+ )
+
+ if write_out:
+ print_writeout(task)
+ # aggregate Instances by LM method requested to get output.
+ for instance in task.instances:
+ reqtype = instance.request_type
+ requests[reqtype].append(instance)
+
+ if lm.world_size > 1:
+ instances_rnk = torch.tensor(len(task._instances), device=lm.device)
+ gathered_item = (
+ lm.accelerator.gather(instances_rnk).cpu().detach().numpy().tolist()
+ )
+ # "multiple_choice" task types dispatch (several) "loglikelihood" request types
+ reqtype = (
+ "loglikelihood"
+ if task.OUTPUT_TYPE == "multiple_choice"
+ else task.OUTPUT_TYPE
+ )
+ # compute number of pseudo-batches to pad with (FSDP/DDP require even batches among ranks)
+ numpad = max(gathered_item) - gathered_item[lm.rank]
+ # todo: may not account for padding in cases like SquadV2 which has multiple req types
+ padding_requests[reqtype] += numpad
+
+ ### Run LM on inputs, get all outputs ###
+ # execute each type of request
+ for reqtype, reqs in requests.items():
+ eval_logger.info(f"Running {reqtype} requests")
+ # create `K` copies of each request `req` based off `K = req.repeats`
+ cloned_reqs = []
+ for req in reqs:
+ cloned_reqs.extend([req] * req.repeats)
+
+ if (lm.world_size > 1) and (padding_requests[reqtype] > 0):
+ for _ in range(padding_requests[reqtype]):
+ cloned_reqs.extend([req] * req.repeats)
+
+ # run requests through model
+ resps = getattr(lm, reqtype)(cloned_reqs)
+
+ # put responses from model into a list of length K for each request.
+ for x, req in zip(resps, cloned_reqs):
+ req.resps.append(x)
+
+ if lm.world_size > 1:
+ lm.accelerator.wait_for_everyone()
+
+ RANK = lm.rank
+ WORLD_SIZE = lm.world_size
+ ### Postprocess outputs ###
+ # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
+ for task_output in eval_tasks:
+ task = task_output.task
+ task.apply_filters()
+
+ ### Collect values of metrics on all datapoints ###
+ # # unpack results and sort back in order and return control to Task
+ # TODO: make it possible to use a different metric per filter
+ # Pre-process task.instances to group by doc_id
+ instances_by_doc_id = defaultdict(list)
+ for instance in task.instances:
+ instances_by_doc_id[instance.doc_id].append(instance)
+ # Sort instances within each group
+ for instances in instances_by_doc_id.values():
+ instances.sort(key=lambda x: x.idx)
+ # iterate over different filters used
+ for filter_key in task.instances[0].filtered_resps.keys():
+ doc_iterator = task.doc_iterator(
+ rank=RANK, limit=limit, world_size=WORLD_SIZE
+ )
+ for doc_id, doc in doc_iterator:
+ requests = instances_by_doc_id[doc_id]
+ metrics = task.process_results(
+ doc, [req.filtered_resps[filter_key] for req in requests]
+ )
+ if log_samples:
+ target = task.doc_to_target(doc)
+ example = {
+ "doc_id": doc_id,
+ "doc": doc,
+ "target": target,
+ "arguments": [req.args for req in requests],
+ "resps": [req.resps for req in requests],
+ "filtered_resps": [
+ req.filtered_resps[filter_key] for req in requests
+ ],
+ }
+ example.update(metrics)
+ task_output.logged_samples.append(example)
+ for metric, value in metrics.items():
+ task_output.sample_metrics[(metric, filter_key)].append(value)
+
+ if WORLD_SIZE > 1:
+ # if multigpu, then gather data across all ranks to rank 0
+ # first gather logged samples across all ranks
+ for task_output in eval_tasks:
+ if log_samples:
+ # for task_name, task_samples in list(samples.items()):
+ full_samples = [None] * WORLD_SIZE if RANK == 0 else None
+ torch.distributed.gather_object(
+ obj=task_output.logged_samples,
+ object_gather_list=full_samples,
+ dst=0,
+ )
+
+ if RANK == 0:
+ task_output.logged_samples = list(
+ itertools.chain.from_iterable(full_samples)
+ )
+
+ # then collect metrics across all ranks
+ for metrics in task_output.sample_metrics:
+ metric_list = [None] * WORLD_SIZE if RANK == 0 else None
+ torch.distributed.gather_object(
+ obj=task_output.sample_metrics[metrics],
+ object_gather_list=metric_list,
+ dst=0,
+ )
+ if RANK == 0:
+ task_output.sample_metrics[metrics] = list(
+ itertools.chain.from_iterable(metric_list)
+ )
+
+ if RANK == 0:
+ ### Aggregate results over all datapoints ###
+ # aggregate results ; run bootstrap CIs
+ for task_output in eval_tasks:
+ task_output.calculate_aggregate_metric(bootstrap_iters=bootstrap_iters)
+ results, samples, configs, versions, num_fewshot = consolidate_results(
+ eval_tasks
+ )
+
+ ### Calculate group metrics ###
+ if bool(results):
+ for group, task_list in reversed(task_hierarchy.items()):
+ if len(task_list) == 0:
+ # task_hierarchy entries are either
+ # `group_name: [subtask1, subtask2, ...]`
+ # or `task_name: []`.
+ # we only want to operate on groups here.
+ continue
+ metric_list = list(
+ {
+ key
+ for task in task_list
+ for key in results[task].keys()
+ if "_stderr" not in key and key not in ["alias", "samples"]
+ }
+ )
+ for metric in metric_list:
+ stderr = "_stderr,".join(metric.split(","))
+
+ # gather metrics, sizes, and stderrs from subtasks
+ metrics = [
+ results[task][metric]
+ for task in task_list
+ if metric in results[task]
+ ] # TODO: copy?
+ stderrs = [
+ results[task][stderr]
+ for task in task_list
+ if stderr in results[task]
+ ]
+ sizes = [
+ results[task]["samples"]
+ for task in task_list
+ if metric in results[task]
+ ]
+
+ # compute group's pooled metric and stderr
+ results[group][
+ metric
+ ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes)
+ # TODO: calculate grouped metric using aggregation fn
+ if "N/A" in stderrs:
+ results[group][stderr] = "N/A"
+ else:
+ results[group][
+ stderr
+ ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes)
+ # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility
+ # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line:
+ # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics)
+
+ results[group]["samples"] = sum(sizes)
+
+ results_agg = defaultdict(dict)
+ groups_agg = defaultdict(dict)
+ all_tasks_list = list(task_hierarchy.keys())
+ while True:
+ add_tasks_list = list(k for k in results_agg.keys())
+ left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list)))
+ if len(left_tasks_list) == 0:
+ break
+
+ _task_hierarchy = {
+ k: v for k, v in task_hierarchy.items() if k in left_tasks_list
+ }
+ _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results)
+
+ results_agg = {**results_agg, **_results_agg}
+ groups_agg = {**groups_agg, **_groups_agg}
+
+ for group_name, task_list in task_hierarchy.items():
+ if task_list:
+ num_fewshot[group_name] = num_fewshot[
+ task_list[0]
+ ] # TODO: validate this
+
+ results_dict = {
+ "results": dict(results_agg.items()),
+ **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}),
+ "group_subtasks": dict(reversed(task_hierarchy.items())),
+ "configs": dict(sorted(configs.items())),
+ "versions": dict(sorted(versions.items())),
+ "n-shot": dict(sorted(num_fewshot.items())),
+ }
+ if log_samples:
+ results_dict["samples"] = dict(samples)
+
+ return results_dict
+
+ else:
+ return None
+
+
+if __name__ == "__main__":
+
+ import sys
+
+ sys.path.insert(0, '../../../')
+ import time
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--model_name", default="/models/opt-125m/"
+ )
+ parser.add_argument(
+ "--eval_bs", default=1,
+ )
+ parser.add_argument(
+ "--trust_remote_code", action='store_true',
+ help="Whether to enable trust_remote_code"
+ )
+ parser.add_argument("--tasks",
+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \
+ "openbookqa,boolq,rte,arc_easy,arc_challenge",
+ help="lm-eval tasks for lm_eval version 0.4.2")
+
+ args = parser.parse_args()
+ s = time.time()
+ from transformers import AutoConfig
+
+ config = AutoConfig.from_pretrained(args.model_name, trust_remote_code=args.trust_remote_code)
+ model_args = f"pretrained={args.model_name}"
+ if hasattr(config, "quantization_config"):
+ quantization_config = config.quantization_config
+ if "quant_method" in quantization_config and "auto-round" in quantization_config["quant_method"]:
+ from auto_round.auto_quantizer import AutoHfQuantizer
+ else:
+ from phi3_v import Phi3VGPTQForCausalLM
+ model_args += f",autogptq=True,gptq_use_triton=True"
+ if args.trust_remote_code:
+ model_args += f",trust_remote_code=True"
+
+ test_tasks = args.tasks
+ if isinstance(test_tasks, str):
+ test_tasks = test_tasks.split(',')
+ model_name = args.model_name.rstrip('/')
+ from lm_eval.utils import make_table
+ result = simple_evaluate(model="hf",
+ model_args=model_args,
+ tasks=test_tasks,
+ batch_size=args.eval_bs)
+ print(make_table(result))
+
+ print("cost time: ", time.time() - s)
+
diff --git a/examples/multimodal-modeling/Phi-3-vision/eval_042/phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/eval_042/phi3_v.py
new file mode 100644
index 000000000..75366ea26
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/eval_042/phi3_v.py
@@ -0,0 +1,21 @@
+from auto_gptq.modeling._base import BaseGPTQForCausalLM
+from auto_gptq.utils.import_utils import compare_transformers_version
+from auto_gptq.modeling._const import SUPPORTED_MODELS
+from auto_gptq.modeling.auto import GPTQ_CAUSAL_LM_MODEL_MAP
+
+class Phi3VGPTQForCausalLM(BaseGPTQForCausalLM):
+ layer_type = "Phi3DecoderLayer"
+ layers_block_name = "model.layers"
+ outside_layer_modules = ["model.embed_tokens", "model.final_layernorm"] # "model.vision_embed_tokens"?
+ inside_layer_modules = [
+ ["self_attn.o_proj"],
+ ["self_attn.qkv_proj"],
+ ["mlp.gate_up_proj"],
+ ["mlp.down_proj"],
+ ]
+
+__all__ = ["Phi3VGPTQForCausalLM"]
+
+if compare_transformers_version("v4.41.0", op="ge"):
+ SUPPORTED_MODELS.append("phi3_v")
+GPTQ_CAUSAL_LM_MODEL_MAP["phi3_v"] = Phi3VGPTQForCausalLM
diff --git a/examples/multimodal-modeling/Phi-3-vision/main.py b/examples/multimodal-modeling/Phi-3-vision/main.py
new file mode 100644
index 000000000..ad9114e6f
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/main.py
@@ -0,0 +1,466 @@
+import argparse
+import sys
+sys.path.insert(0, '../../..')
+parser = argparse.ArgumentParser()
+import torch
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+torch.use_deterministic_algorithms(True, warn_only=True)
+import copy
+from PIL import Image
+import json
+from torch.utils.data import Dataset, DataLoader
+import torch
+from typing import Dict, Optional, List, Union, Sequence
+import transformers
+from model.processing_phi3_v import Phi3VProcessor
+from dataclasses import dataclass, field
+from transformers import AutoModelForCausalLM, AutoConfig
+from transformers.trainer_pt_utils import LabelSmoother
+
+IGNORE_TOKEN_ID = LabelSmoother.ignore_index
+import subprocess
+LLaVA_IMAGE_TOKEN = "<image>"
+DEFAULT_IMAGE_TOKEN = "<|image_1|>"
+IMAGE_TOKEN_INDEX = -200
+IGNORE_INDEX = -100
+
+@dataclass
+class DataArguments:
+ data_path: str = field(
+ default=None, metadata={"help": "Path to the training data."}
+ )
+ lazy_preprocess: bool = True
+ is_multimodal: bool = True
+ image_folder: Optional[str] = field(default=None)
+ max_seq_length: int = field(
+ default=2048, metadata={"help": "Maximum sequence length."}
+ )
+
+
+def llava_to_openai(data):
+ role_mapping = {"human": "user", "gpt": "assistant"}
+
+ transformed_data = []
+ for entry in data:
+ transformed_entry = {
+ "role": role_mapping.get(entry["from"], entry["from"]),
+ "content": entry["value"].replace(LLaVA_IMAGE_TOKEN, DEFAULT_IMAGE_TOKEN),
+ }
+ transformed_data.append(transformed_entry)
+
+ return transformed_data
+
+
+class LazySupervisedDataset(Dataset):
+ """Dataset for supervised fine-tuning."""
+
+ def __init__(
+ self,
+ data_path: Union[int, str],
+ processor: transformers.PreTrainedTokenizer,
+ data_args: DataArguments,
+ padding=True,
+ ):
+ super(LazySupervisedDataset, self).__init__()
+ if isinstance(data_path, str):
+ list_data_dict = json.load(open(data_path, "r"))
+ else:
+ list_data_dict = data_path
+
+ print("Formatting inputs...Skip in lazy mode")
+ self.processor = processor
+ self.list_data_dict = list_data_dict
+ self.data_args = data_args
+ self.padding = padding
+ self.max_seq_length = data_args.max_seq_length
+
+ def __len__(self):
+ return len(self.list_data_dict)
+
+ def __getitem__(self, i) -> Dict[str, torch.Tensor]:
+ sources = self.list_data_dict[i]
+ if isinstance(i, int):
+ sources = [sources]
+ assert len(sources) == 1, "Don't know why it is wrapped to a list" # FIXME
+ processor = self.processor
+ if "image" in sources[0]:
+ image_file = os.path.basename(self.list_data_dict[i]["image"])
+ image_folder = self.data_args.image_folder
+ image_file = os.path.join(image_folder, image_file)
+ image = [Image.open(image_file).convert("RGB")]
+ else:
+ image = None
+ sources = copy.deepcopy([e["conversations"] for e in sources])
+ for i in range(len(sources)):
+ sources[i] = llava_to_openai(sources[i])
+
+ prompt = processor.tokenizer.apply_chat_template(
+ sources[0], tokenize=False, add_generation_prompt=True
+ )
+ data_dict = processor(prompt, image, return_tensors="pt")
+
+ if self.padding:
+ training_length = self.max_seq_length
+ if 'pixel_values' not in data_dict:
+ data_dict['pixel_values'] = torch.zeros([1, 17, 3, 336, 336], dtype=torch.bfloat16)
+ data_dict['image_sizes'] = torch.zeros([1, 2], dtype=torch.int64)
+ data_dict = dict(
+ input_ids=data_dict["input_ids"][0],
+ attention_mask=data_dict["attention_mask"][0],
+ pixel_values=data_dict["pixel_values"][0],
+ image_sizes=data_dict["image_sizes"][0],
+ labels=data_dict["labels"][0],
+ )
+ return data_dict
+
+@dataclass
+class DataCollatorForSupervisedDataset(object):
+ """Collate examples for supervised fine-tuning."""
+
+ tokenizer: transformers.PreTrainedTokenizer
+
+ def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
+ input_ids, labels = tuple([instance[key] for instance in instances]
+ for key in ("input_ids", "labels"))
+ input_ids = torch.nn.utils.rnn.pad_sequence(
+ input_ids,
+ batch_first=True,
+ padding_value=self.tokenizer.pad_token_id)
+ labels = torch.nn.utils.rnn.pad_sequence(
+ labels,
+ batch_first=True,
+ padding_value=IGNORE_INDEX)
+ input_ids = input_ids[:, :self.tokenizer.model_max_length]
+ labels = labels[:, :self.tokenizer.model_max_length]
+
+ pixel_values = [instance["pixel_values"] for instance in instances]
+ pixel_values = torch.stack(pixel_values, dim=0)
+ image_sizes = [instance["image_sizes"] for instance in instances]
+ image_sizes = torch.stack(image_sizes, dim=0)
+
+ batch = dict(
+ input_ids=input_ids,
+ labels=labels,
+ pixel_values=pixel_values,
+ image_sizes=image_sizes,
+ attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
+ )
+ return batch
+
+def create_data_loader(dataset, batch_size=1, data_collator=None):
+ assert batch_size == 1, "batch_size must be 1"
+ data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, collate_fn=data_collator)
+ return data_loader
+
+
+if __name__ == '__main__':
+
+ parser.add_argument(
+ "--model_name", default="microsoft/Phi-3-vision-128k-instruct"
+ )
+
+ parser.add_argument("--bits", default=4, type=int,
+ help="number of bits")
+
+ parser.add_argument("--group_size", default=128, type=int,
+ help="group size")
+
+ parser.add_argument("--train_bs", default=1, type=int,
+ help="train batch size")
+
+ parser.add_argument("--eval_bs", default=4, type=int,
+ help="eval batch size")
+
+ parser.add_argument("--device", default="auto", type=str,
+ help="The device to be used for tuning. The default is set to auto/None,"
+ "allowing for automatic detection. Currently, device settings support CPU, GPU, and HPU.")
+
+ parser.add_argument("--sym", action='store_true',
+ help=" sym quantization")
+
+ parser.add_argument("--iters", default=200, type=int,
+ help=" iters")
+
+ parser.add_argument("--lr", default=None, type=float,
+ help="learning rate, if None, it will be set to 1.0/iters automatically")
+
+ parser.add_argument("--minmax_lr", default=None, type=float,
+ help="minmax learning rate, if None,it will beset to be the same with lr")
+
+ parser.add_argument("--seed", default=42, type=int,
+ help="seed")
+
+ parser.add_argument("--eval_fp16_baseline", action='store_true',
+ help="whether to eval FP16 baseline")
+
+ parser.add_argument("--adam", action='store_true',
+ help="adam")
+
+ parser.add_argument("--seqlen", default=2048, type=int,
+ help="sequence length")
+
+ parser.add_argument("--gradient_accumulate_steps", default=1, type=int, help="gradient accumulate steps")
+
+ parser.add_argument("--nblocks", default=1, type=int, help="num of blocks to tune together")
+
+ parser.add_argument("--nsamples", default=512, type=int,
+ help="number of samples")
+
+ parser.add_argument("--low_gpu_mem_usage", action='store_true',
+ help="low_gpu_mem_usage is deprecated")
+
+ parser.add_argument("--deployment_device", default=None, type=str,
+ help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'gpu' 'xpu' and 'auto_round'."
+ "default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")
+
+ parser.add_argument("--scale_dtype", default='fp16',
+ help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")
+
+ parser.add_argument("--output_dir", default="./tmp_autoround", type=str,
+ help="Where to store the final model.")
+
+ parser.add_argument("--disable_eval", action='store_true',
+ help="Whether to do lmeval evaluation.")
+
+ parser.add_argument("--disable_amp", action='store_true',
+ help="disable amp")
+
+ parser.add_argument("--disable_minmax_tuning", action='store_true',
+ help="whether disable enable weight minmax tuning")
+
+ parser.add_argument("--disable_trust_remote_code", action='store_true',
+ help="Whether to disable trust_remote_code")
+
+ parser.add_argument("--disable_quanted_input", action='store_true',
+ help="whether to disuse the output of quantized block to tune the next block")
+
+ parser.add_argument("--quant_lm_head", action='store_true',
+ help="quant_lm_head")
+
+ parser.add_argument("--model_dtype", default=None, type=str,
+ help="force to convert the dtype, some backends supports fp16 dtype better")
+
+ parser.add_argument("--model_max_length", default=2048, type=int,
+ help="")
+
+ parser.add_argument("--act_bits", default=32, type=int,
+ help="activation bits")
+
+ parser.add_argument("--quant_vision", action='store_true',
+ help="To determine whether the quantization should handle vision component.")
+
+ parser.add_argument("--enable_safe_serialization", action='store_true',
+ help="To determine whether the save_pretrained process should use safe_serialization.")
+
+ # ========== Calibration Datasets =============
+ parser.add_argument("--image_folder", default="coco", type=str,
+ help="The dataset for quantization training. It can be a custom one.")
+
+ parser.add_argument("--question_file", default=None, type=str,
+ help="The dataset for quantization training. It can be a custom one.")
+
+ # ================= Evaluation Related =====================
+ parser.add_argument("--tasks", #wikitext
+ default="lambada_openai,hellaswag,winogrande,piqa,mmlu,truthfulqa_mc1," \
+ "truthfulqa_mc2,openbookqa,boolq,rte,arc_easy,arc_challenge",
+ help="lm-eval tasks for lm_eval version 0.4")
+
+ parser.add_argument("--eval-dataset", type=str, default="textvqa_val")
+
+ args = parser.parse_args()
+
+ tasks = args.tasks
+
+ if args.deployment_device is None:
+ args.deployment_device = "auto_round"
+
+ if args.act_bits <= 8:
+ print(
+ "Warning, activation quantization is an experiment feature")
+
+ if args.act_bits <= 8 and args.deployment_device != "fake":
+ assert False, "only support fake mode for activation quantization currently"
+
+ if "marlin" in args.deployment_device and args.sym == False:
+ assert False, "marlin backend only supports sym quantization, please set --sym"
+
+ model_name = args.model_name
+ if model_name[-1] == "/":
+ model_name = model_name[:-1]
+ print(model_name, flush=True)
+
+ from auto_round.utils import detect_device
+
+ device_str = detect_device(args.device)
+ torch_device = torch.device(device_str)
+
+ torch.manual_seed(1234)
+ model_name = args.model_name
+ seqlen = args.seqlen
+ questions = json.load(open(args.question_file, "r"))
+ torch_dtype = "auto"
+ if "hpu" in device_str:
+ torch_dtype = torch.bfloat16
+ if args.model_dtype != None:
+ if args.model_dtype == "float16" or args.model_dtype == "fp16":
+ torch_dtype = torch.float16
+ if args.model_dtype == "bfloat16" or args.model_dtype == "bfp16":
+ torch_dtype = torch.bfloat16
+
+ config = AutoConfig.from_pretrained(
+ model_name,
+ trust_remote_code=True,
+ )
+ config.use_cache = False
+
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ torch_dtype=torch_dtype,
+ trust_remote_code=not args.disable_trust_remote_code,
+ config=config,
+ )
+ seqlen = args.seqlen
+ processor = Phi3VProcessor.from_pretrained(model_name)
+ tokenizer = processor.tokenizer
+ data_args = DataArguments(
+ data_path=args.question_file,
+ is_multimodal=True,
+ image_folder=args.image_folder,
+ max_seq_length=seqlen,
+ )
+ dataset = LazySupervisedDataset(
+ data_path=args.question_file, processor=processor, data_args=data_args
+ )
+ data_collator = DataCollatorForSupervisedDataset(tokenizer=processor.tokenizer)
+ dataloader = create_data_loader(dataset, batch_size=args.train_bs, data_collator=data_collator)
+
+ from auto_round import (AutoRound,
+ AutoAdamRound)
+ from auto_round.utils import get_multimodal_block_names
+
+ round = AutoRound
+ if args.adam:
+ round = AutoAdamRound
+
+ layer_config = {}
+ for n, m in model.named_modules():
+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):
+ if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:
+ layer_config[n] = {"bits": 32}
+ print(
+ f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq")
+ lm_head_layer_name = "lm_head"
+ for n, _ in model.named_modules():
+ lm_head_layer_name = n
+ if args.quant_lm_head:
+ from transformers import AutoConfig
+
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
+ if config.tie_word_embeddings and hasattr(model, "_tied_weights_keys"):
+ tied_keys = model._tied_weights_keys
+ for item in tied_keys:
+ if lm_head_layer_name in item: ##TODO extend to encoder-decoder layer, seq classification model
+ args.quant_lm_head = False
+ print(
+ f"warning, disable quant_lm_head as quantizing lm_head with tied weights has not been "
+ f"supported currently")
+ break
+ if args.quant_lm_head:
+ layer_config[lm_head_layer_name] = {"bits": args.bits}
+ transformers_version = [int(item) for item in transformers.__version__.split('.')[:2]]
+ if transformers_version[0] == 4 and transformers_version[1] < 38:
+ error_message = "Please upgrade transformers>=4.38.0 to support lm-head quantization."
+ raise EnvironmentError(error_message)
+
+ if args.quant_lm_head and args.low_gpu_mem_usage:
+ print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "
+ f"gpu")
+
+ quant_block_list = get_multimodal_block_names(model, args.quant_vision)
+
+ autoround = round(model, tokenizer, args.bits, args.group_size, sym=args.sym, batch_size=args.train_bs,
+ dataset=dataloader, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters, lr=args.lr,
+ minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input, device=device_str,
+ amp=not args.disable_amp, nsamples=args.nsamples, layer_config=layer_config,
+ low_gpu_mem_usage=args.low_gpu_mem_usage, scale_dtype=args.scale_dtype,
+ seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,
+ enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,
+ quant_block_list=quant_block_list)
+ model, _ = autoround.quantize()
+ model_name = args.model_name.rstrip("/")
+
+ model.eval()
+ if args.device != "cpu":
+ torch.cuda.empty_cache()
+
+ export_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}"
+ output_dir = args.output_dir + "/" + model_name.split('/')[-1] + f"-autoround-w{args.bits}g{args.group_size}-qdq"
+
+ deployment_device = args.deployment_device.split(',')
+ gpu_formats = []
+ for item in deployment_device:
+ if "gpu" in item or "auto_gptq" in item or "auto_round" in item:
+ gpu_formats.append(item)
+
+ if 'gpu' in deployment_device:
+ if lm_head_layer_name in layer_config.keys() and layer_config[lm_head_layer_name]["data_type"] == "int":
+ gpu_formats.append("auto_round")
+ else:
+ gpu_formats.append("auto_gptq")
+ gpu_formats = list(set(gpu_formats))
+
+ inplace = True if len(deployment_device) < 2 and args.disable_eval else False
+ eval_folder = None
+ for gpu_format in gpu_formats:
+ if "round" in gpu_format:
+ eval_folder = f'{export_dir}-round'
+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace,
+ safe_serialization=args.enable_safe_serialization)
+ elif "gptq" in gpu_format:
+ eval_folder = f'{export_dir}-gpu'
+ autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace,
+ safe_serialization=args.enable_safe_serialization)
+
+ if 'xpu' in deployment_device:
+ autoround.save_quantized(f'{export_dir}-xpu', format="itrex_xpu", use_triton=True, inplace=inplace,
+ compression_dtype=torch.int8, compression_dim=0, use_optimum_format=False,
+ device="xpu", safe_serialization=args.enable_safe_serialization)
+ if "cpu" in deployment_device:
+ autoround.save_quantized(output_dir=f'{export_dir}-cpu', format='itrex', inplace=inplace,
+ safe_serialization=args.enable_safe_serialization)
+ if "fake" in deployment_device:
+ model = model.to("cpu")
+ model.save_pretrained(output_dir, safe_serialization=args.enable_safe_serialization)
+ tokenizer.save_pretrained(output_dir)
+ if eval_folder is None:
+ eval_folder = output_dir
+
+ if isinstance(tasks, str):
+ tasks = tasks.split(',')
+ if 'fake' in args.deployment_device and not args.disable_eval:
+ print(f"Using the latest lm_eval 0.4.2")
+
+ if not args.disable_eval:
+ from eval_042.evaluation import simple_evaluate
+
+ if 'gpu' in deployment_device or len(gpu_formats) > 0:
+ model_args = f"pretrained={eval_folder}"
+ elif "fake" in deployment_device:
+ model_args = f"pretrained={eval_folder}"
+ else:
+ exit() ## does not support cpu,xpu model eval
+ model_args = model_args + f",trust_remote_code={not args.disable_trust_remote_code}"
+ user_model = model
+ # model = model.to("cpu")
+ if args.act_bits <= 8:
+ user_model = model.to(device_str)
+
+ res = simple_evaluate(model="hf", model_args=model_args,
+ tasks=tasks,
+ limit=5,
+ batch_size=args.eval_bs, user_model=user_model)
+ from lm_eval.utils import make_table
+
+ print(make_table(res))
diff --git a/examples/multimodal-modeling/Phi-3-vision/model/__init__.py b/examples/multimodal-modeling/Phi-3-vision/model/__init__.py
new file mode 100644
index 000000000..dbe21cebc
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/model/__init__.py
@@ -0,0 +1,4 @@
+if __name__ == "__main__":
+ import sys
+ sys.path.insert(0, './')
+
diff --git a/examples/multimodal-modeling/Phi-3-vision/model/configuration_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/configuration_phi3_v.py
new file mode 100644
index 000000000..573bb8d39
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/model/configuration_phi3_v.py
@@ -0,0 +1,217 @@
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Phi-3-V model configuration"""
+
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+PHI3V_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+ "microsoft/Phi-3-vision-128k-instruct": "https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/resolve/main/config.json",
+}
+
+
+class Phi3VConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`Phi3VModel`]. It is used to instantiate a Phi-3
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the
+ [microsoft/Phi-3-vision-128k-instruct](https://huggingface.co/microsoft/Phi-3-vision-128k-instruct).
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 32064):
+ Vocabulary size of the Phi-3-V model. Defines the number of different tokens that can be represented by the
+ `inputs_ids` passed when calling [`Phi3VModel`].
+ hidden_size (`int`, *optional*, defaults to 3072):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 8192):
+ Dimension of the MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 32):
+ Number of hidden layers in the Transformer decoder.
+ num_attention_heads (`int`, *optional*, defaults to 32):
+ Number of attention heads for each attention layer in the Transformer decoder.
+ num_key_value_heads (`int`, *optional*):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+ `num_attention_heads`.
+ resid_pdrop (`float`, *optional*, defaults to 0.0):
+ Dropout probability for mlp outputs.
+ embd_pdrop (`int`, *optional*, defaults to 0.0):
+ The dropout ratio for the embeddings.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio after computing the attention scores.
+ hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+ The non-linear activation function (function or string) in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 4096):
+ The maximum sequence length that this model might ever be used with.
+ original_max_position_embeddings (`int`, *optional*, defaults to 4096):
+ The maximum sequence length that this model was trained with. This is used to determine the size of the
+ original RoPE embeddings when using long scaling.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+ The epsilon value used for the RMSNorm.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether to tie weight embeddings
+ rope_theta (`float`, *optional*, defaults to 10000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`dict`, *optional*):
+ The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must
+ contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be either `su` or `yarn` and
+ the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size
+ divided by the number of attention heads divided by 2.
+ bos_token_id (`int`, *optional*, defaults to 1):
+ The id of the "beginning-of-sequence" token.
+ eos_token_id (`int`, *optional*, defaults to 32000):
+ The id of the "end-of-sequence" token.
+ pad_token_id (`int`, *optional*, defaults to 32000):
+ The id of the padding token.
+ sliding_window (`int`, *optional*):
+ Sliding window attention window size. If `None`, no sliding window is applied.
+ embd_layer (`str`, *optional*, defaults to `"default"`):
+ The embedding layer to use. Can be either `"default"` or `"image"`. "default" uses the standard embedding for text.
+
+ Example:
+
+ ```python
+ >>> from transformers import Phi3VModel, Phi3VConfig
+
+ >>> # Initializing a Phi-3-V style configuration
+ >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-vision-128k-instruct")
+
+ >>> # Initializing a model from the configuration
+ >>> model = Phi3VModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```"""
+
+ model_type = "phi3_v"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ def __init__(
+ self,
+ vocab_size=32064,
+ hidden_size=3072,
+ intermediate_size=8192,
+ num_hidden_layers=32,
+ num_attention_heads=32,
+ num_key_value_heads=None,
+ resid_pdrop=0.0,
+ embd_pdrop=0.0,
+ attention_dropout=0.0,
+ hidden_act="silu",
+ max_position_embeddings=4096,
+ original_max_position_embeddings=4096,
+ initializer_range=0.02,
+ rms_norm_eps=1e-5,
+ use_cache=True,
+ tie_word_embeddings=False,
+ rope_theta=10000.0,
+ rope_scaling=None,
+ bos_token_id=1,
+ eos_token_id=32000,
+ pad_token_id=32000,
+ sliding_window=None,
+ embd_layer: str = "default",
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+
+ if num_key_value_heads is None:
+ num_key_value_heads = num_attention_heads
+
+ self.num_key_value_heads = num_key_value_heads
+ self.resid_pdrop = resid_pdrop
+ self.embd_pdrop = embd_pdrop
+ self.attention_dropout = attention_dropout
+ self.hidden_act = hidden_act
+ self.max_position_embeddings = max_position_embeddings
+ self.original_max_position_embeddings = original_max_position_embeddings
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self._rope_scaling_validation()
+ self.sliding_window = sliding_window
+ self.embd_layer = embd_layer
+
+
+ super().__init__(
+ bos_token_id=bos_token_id,
+ eos_token_id=eos_token_id,
+ pad_token_id=pad_token_id,
+ tie_word_embeddings=tie_word_embeddings,
+ **kwargs,
+ )
+
+ def _rope_scaling_validation(self):
+ """
+ Validate the `rope_scaling` configuration.
+ """
+ if self.rope_scaling is None:
+ return
+
+ if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3:
+ raise ValueError(
+ "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, "
+ f"got {self.rope_scaling}"
+ )
+ rope_scaling_type = self.rope_scaling.get("type", None)
+ rope_scaling_short_factor = self.rope_scaling.get("short_factor", None)
+ rope_scaling_long_factor = self.rope_scaling.get("long_factor", None)
+ if rope_scaling_type is None or rope_scaling_type not in ["su", "yarn"]:
+ raise ValueError(f"`rope_scaling`'s type field must be one of ['su', 'yarn'], got {rope_scaling_type}")
+ if not (
+ isinstance(rope_scaling_short_factor, list)
+ and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"
+ )
+ if not len(rope_scaling_short_factor) == self.hidden_size // self.num_attention_heads // 2:
+ raise ValueError(
+ f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}"
+ )
+ if not (
+ isinstance(rope_scaling_long_factor, list)
+ and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor)
+ ):
+ raise ValueError(
+ f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}"
+ )
+ if not len(rope_scaling_long_factor) == self.hidden_size // self.num_attention_heads // 2:
+ raise ValueError(
+ f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}"
+ )
\ No newline at end of file
diff --git a/examples/multimodal-modeling/Phi-3-vision/model/image_embedding_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/image_embedding_phi3_v.py
new file mode 100644
index 000000000..c2994c6ca
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/model/image_embedding_phi3_v.py
@@ -0,0 +1,313 @@
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import torch
+import torch.nn as nn
+from transformers import CLIPVisionModel, PretrainedConfig
+from transformers import CLIPVisionConfig
+from transformers.utils import logging
+from datetime import datetime
+
+logger = logging.get_logger(__name__)
+
+CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(
+ attention_dropout=0.0,
+ dropout=0.0,
+ hidden_act="quick_gelu",
+ hidden_size=1024,
+ image_size=336,
+ initializer_factor=1.0,
+ initializer_range=0.02,
+ intermediate_size=4096,
+ layer_norm_eps=1e-05,
+ num_attention_heads=16,
+ num_channels=3,
+ num_hidden_layers=24,
+ patch_size=14,
+ projection_dim=768
+)
+
+class Phi3ImageEmbedding(nn.Module):
+ """Phi3 Image embedding."""
+
+ def __init__(self, config: PretrainedConfig, wte=None, **kwargs) -> None:
+ super().__init__()
+
+ # n_embed or hidden_size
+ hidden_size = config.n_embd if hasattr(config, 'n_embd') else config.hidden_size
+ if hasattr(config, 'embd_pdrop') or hasattr(config, 'embed_pdrop'):
+ embd_drop = config.embd_pdrop if hasattr(config, 'embd_pdrop') else config.embed_pdrop
+ self.drop = nn.Dropout(embd_drop)
+ else:
+ self.drop = None
+
+ self.wte = wte
+
+ if isinstance(config.img_processor, dict) and config.img_processor.get('name', None) == 'clip_vision_model':
+ assert 'model_name' in config.img_processor, 'model_name must be provided for CLIPVisionModel'
+ assert 'image_dim_out' in config.img_processor, 'image_dim_out must be provided for CLIPVisionModel'
+ assert 'num_img_tokens' in config.img_processor, 'num_img_tokens must be provided for CLIPVisionModel'
+ assert config.img_processor['model_name'] == 'openai/clip-vit-large-patch14-336'
+ clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
+ self.img_processor = CLIPVisionModel(clip_config)
+ image_dim_out = config.img_processor['image_dim_out']
+ self.num_img_tokens = config.img_processor['num_img_tokens']
+ else:
+ raise NotImplementedError(f'img_processor = {config.img_processor}, not implemented')
+
+ self.image_dim_out = image_dim_out
+ self.img_sizes = None
+
+ # global_gn and sub_gn for hd transform, serves as line separator
+ self.use_hd_transform = kwargs.get('use_hd_transform', False)
+ self.with_learnable_separator = kwargs.get('with_learnable_separator', False)
+ self.hd_transform_order = kwargs.get('hd_transform_order', 'glb_sub')
+ # with_hd_transform and with_learnable_separator should have same value
+ assert self.use_hd_transform == self.with_learnable_separator, 'use_hd_transform and with_learnable_separator should have same value'
+ if self.with_learnable_separator:
+ assert self.use_hd_transform, 'learnable separator is only for hd transform'
+ # 1024 * 4, merge spatial to channel dimension
+ self.glb_GN = nn.Parameter(torch.zeros([1, 1, self.image_dim_out * 4]))
+ self.sub_GN = nn.Parameter(torch.zeros([1, 1, 1, self.image_dim_out * 4]))
+ logger.info(f'learnable separator enabled for hd transform, hd_transform_order = {self.hd_transform_order}')
+
+ projection_cls = kwargs.get('projection_cls', 'linear')
+ if projection_cls == 'linear':
+ self.img_projection = nn.Linear(image_dim_out, hidden_size)
+ elif projection_cls == 'mlp' and self.use_hd_transform:
+ dim_projection = hidden_size
+ depth = 2
+ layers = [nn.Linear(image_dim_out * 4, dim_projection)]
+ for _ in range(1, depth):
+ layers.extend([nn.GELU(),
+ nn.Linear(dim_projection, dim_projection)])
+ self.img_projection = nn.Sequential(*layers)
+ elif projection_cls == 'mlp':
+ dim_projection = hidden_size
+ depth = 2
+ layers = [nn.Linear(image_dim_out, dim_projection)]
+ for _ in range(1, depth):
+ layers.extend([nn.GELU(),
+ nn.Linear(dim_projection, dim_projection)])
+ self.img_projection = nn.Sequential(*layers)
+ else:
+ raise NotImplementedError(f'projection_cls = {projection_cls}, not implemented')
+
+ self.vocab_size = config.vocab_size
+ self.img_features = None
+
+ if isinstance(config.img_processor, dict):
+ self.layer_idx = config.img_processor.get('layer_idx', -2)
+ self.type_feature = config.img_processor.get('type_feature', 'patch')
+ else:
+ self.layer_idx = -2
+ self.type_feature = 'patch'
+
+
+ def set_img_features(self, img_features: torch.FloatTensor) -> None:
+ self.img_features = img_features
+
+ def set_img_sizes(self, img_sizes: torch.LongTensor) -> None:
+ self.img_sizes = img_sizes
+
+ def get_img_features(self, img_embeds: torch.FloatTensor) -> torch.FloatTensor:
+ LAYER_IDX = self.layer_idx
+ TYPE_FEATURE = self.type_feature
+
+ img_processor_output = self.img_processor(img_embeds, output_hidden_states=True)
+ img_feature = img_processor_output.hidden_states[LAYER_IDX]
+
+ if TYPE_FEATURE == "patch":
+ patch_feature = img_feature[:, 1:]
+ return patch_feature
+
+ if TYPE_FEATURE == "cls_patch":
+ return img_feature
+
+ raise NotImplementedError
+
+ def forward(self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, image_sizes=None) -> torch.FloatTensor:
+
+ MAX_INPUT_ID = int(1e9)
+ img_embeds = pixel_values
+ img_sizes = image_sizes
+
+ if self.img_features is not None:
+ img_embeds = self.img_features.clone()
+ self.img_features = None
+
+ if self.img_sizes is not None:
+ img_sizes = self.img_sizes
+
+ input_shape = input_ids.size()
+ input_ids = input_ids.view(-1, input_shape[-1])
+
+ with torch.no_grad():
+ positions = torch.nonzero((input_ids < 0) & (input_ids > -MAX_INPUT_ID), as_tuple=False)
+
+ select = False
+
+ if isinstance(self.img_projection, nn.Sequential):
+ target_device = self.img_projection[0].bias.device
+ target_dtype = self.img_projection[0].bias.dtype
+ else: # It's a single nn.Linear layer
+ target_device = self.img_projection.bias.device
+ target_dtype = self.img_projection.bias.dtype
+
+ if len(positions.tolist()) > 0:
+ with torch.no_grad():
+ g_values = abs(input_ids[positions[:, 0], positions[:, 1]])
+
+ if self.use_hd_transform and img_sizes is not None and len(img_sizes):
+ hd_transform = True
+ assert img_embeds.ndim == 5, f'img_embeds size: {img_embeds.size()}, expect 5D tensor for hd transform'
+ # img_embeds: (num_images, max_num_crops, 3, H, W)
+ # img_sizes: (num_images, 2).view(1, -1)
+
+ start_time = datetime.now()
+ bs = img_embeds.shape[0]
+ # Nx(HW)xC
+ img_features = self.get_img_features(img_embeds.flatten(0, 1))
+ base_feat_height = base_feat_width = int(img_features.shape[1] ** 0.5)
+
+ assert base_feat_height == 24 and base_feat_width == 24, f'base_feat_height: {base_feat_height}, base_feat_width: {base_feat_width}, expect 24x24 features for hd transform'
+
+ # bs x max_num_crops x (24x24) x C
+ img_features = img_features.view(bs, -1, base_feat_height * base_feat_width, self.image_dim_out)
+ C = self.image_dim_out
+ H = base_feat_height
+
+ output_imgs = []
+ output_len = []
+ # training is tensor, inference is list
+ if isinstance(img_sizes, torch.Tensor):
+ img_sizes = img_sizes.view(-1, 2)
+ num_pure_text = 0
+ for _bs in range(bs):
+ h, w = img_sizes[_bs]
+ h = h // 336
+ w = w // 336
+ B_ = h * w
+ if B_ == 0:
+ num_pure_text += 1
+ continue
+
+ # 1 x (24x24) x 1024
+ global_img_feature = img_features[_bs, :1]
+
+ # 1 x 12 x 12 x 4096
+ glb_img = global_img_feature.reshape(1,H,H,C).reshape(1,H//2,2,H//2,2,C).contiguous().permute(0,1,3,2,4,5).reshape(1,H//2,H//2,4*C).contiguous()
+ temp_glb_GN = self.sub_GN.repeat(1, H//2, 1, 1)
+
+ # 1 x 156 x 4096
+ glb_img = torch.cat([glb_img, temp_glb_GN], dim=2).reshape(1,-1,4*C)
+
+ # (max_num_crops-1) x (12x12) x C
+ sub_img = img_features[_bs, 1:]
+ # 16x574x1024
+ # get rid of padding sub_img
+ sub_img = sub_img[:B_]
+
+ # (num_crops, 12, 2, 12, 2, 1024) -> (num_crops, 12, 12, 2, 2, 1024) -> (num_crops, 12*12, 4*1024)
+ sub_img = sub_img.reshape(B_,H,H,C).reshape(B_,H//2,2,H//2,2,C).contiguous().permute(0,1,3,2,4,5).reshape(B_,-1,4*C).contiguous()
+ sub_img = sub_img.reshape(1, h, w, 12, 12, -1).permute(0,1,3,2,4,5).reshape(1,h*12,w*12,4*C)
+ temp_sub_GN = self.sub_GN.repeat(1, h*12, 1, 1)
+ sub_img = torch.cat([sub_img, temp_sub_GN], dim=2).reshape(1,-1,4*C)
+ # (1, num_img_tokens, 1024*4)
+
+ # glb + sub
+ if self.hd_transform_order == 'glb_sub':
+ output_imgs.append(torch.cat([glb_img, self.glb_GN, sub_img], dim=1))
+ elif self.hd_transform_order == 'sub_glb':
+ output_imgs.append(torch.cat([sub_img, self.glb_GN, glb_img], dim=1))
+ else:
+ raise NotImplementedError(f'hd_transform_order = {self.hd_transform_order}, not implemented')
+
+ temp_len = int((h*w+1)*144 + 1 + (h+1)*12)
+ assert temp_len == output_imgs[-1].shape[1], f'temp_len: {temp_len}, output_imgs[-1].shape[1]: {output_imgs[-1].shape[1]}'
+ output_len.append(temp_len)
+
+ num_img_tokens = output_len
+ img_set_tensor = []
+ for _output_img in output_imgs:
+ img_feature_proj = self.img_projection(_output_img.to(target_device).to(target_dtype))
+ img_set_tensor.append(img_feature_proj)
+ # logger.info(f'img_embeds size: {img_embeds.size()}, image sizes: {img_sizes} loading time {datetime.now() - start_time}')
+ elif img_embeds.ndim == 4:
+ selected_g_values = g_values[::self.num_img_tokens]
+ assert len(img_embeds) == len(selected_g_values), f'img_embeds size: {img_embeds.size()}, selected_g_values size: {len(selected_g_values)}, selected_g_value {selected_g_values}'
+ start_time = datetime.now()
+ tt = (
+ self.get_img_features(img_embeds)
+ .to(target_device)
+ .to(target_dtype)
+ .reshape(-1, self.image_dim_out)
+ )
+ # logger.info(f'img_embeds size: {img_embeds.size()}, loading time {datetime.now() - start_time}')
+ img_set_tensor = self.img_projection(tt) # adapted visual features.
+ elif img_embeds.ndim == 3:
+ selected_g_values = g_values[::self.num_img_tokens]
+ assert len(img_embeds) == len(selected_g_values), f'img_embeds size: {img_embeds.size()}, selected_g_values size: {len(selected_g_values)}, selected_g_value {selected_g_values}'
+ tt = (
+ img_embeds
+ .to(target_device)
+ .to(target_dtype)
+ .view(-1, self.image_dim_out)
+ )
+ img_set_tensor = self.img_projection(tt) # adapted visual features.
+ else:
+ raise NotImplementedError
+ select = True
+ # It's a hacky way to walkaround the hang-out problem when deepspeed `zero3` is used
+ # and the training batch is a mixture of pure text and vision-language data.
+ else:
+ num_pure_text = input_ids.shape[0]
+ self.get_img_features(img_embeds.flatten(0, 1))
+ for _ in range(num_pure_text):
+ self.img_projection(torch.zeros(1, 1921, 4096, device=self.img_processor.device, dtype=self.img_processor.dtype))
+ with torch.no_grad():
+ input_ids.clamp_min_(0).clamp_max_(self.vocab_size)
+
+ hidden_states = self.wte(input_ids)
+
+ if select:
+ if hd_transform:
+ idx = 0
+ for i, cnt in enumerate(num_img_tokens):
+ # see https://github.com/GaiZhenbiao/Phi3V-Finetuning/pull/5
+ hidden_states = hidden_states.clone()
+ hidden_states[positions[idx, 0], positions[idx, 1] : positions[idx, 1] + cnt] = (
+ img_set_tensor[i]
+ .to(hidden_states.dtype)
+ .to(hidden_states.device)
+ )
+ idx += cnt
+ else:
+ idx = 0
+ assert len(selected_g_values) * self.num_img_tokens == len(img_set_tensor), f'len(selected_g_values) * self.num_img_tokens = {len(selected_g_values) * self.num_img_tokens}, len(img_set_tensor) = {len(img_set_tensor)}'
+ for i, g in enumerate(selected_g_values):
+ cnt = self.num_img_tokens
+ hidden_states[positions[idx, 0], positions[idx, 1] : positions[idx, 1] + cnt] = (
+ img_set_tensor[i * cnt : (i + 1) * cnt]
+ .to(hidden_states.dtype)
+ .to(hidden_states.device)
+ )
+ idx += cnt
+
+ if self.drop is not None:
+ hidden_states = self.drop(hidden_states)
+
+ return hidden_states
diff --git a/examples/multimodal-modeling/Phi-3-vision/model/image_processing_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/image_processing_phi3_v.py
new file mode 100644
index 000000000..821096527
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/model/image_processing_phi3_v.py
@@ -0,0 +1,274 @@
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Image processor class for Phi3-V."""
+
+from typing import List, Optional, Union
+
+import numpy as np
+from auto_round.utils import transformers
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+from transformers.image_transforms import (
+ convert_to_rgb,
+)
+from transformers.image_utils import (
+ OPENAI_CLIP_MEAN,
+ OPENAI_CLIP_STD,
+ ImageInput,
+ make_list_of_images,
+ valid_images,
+)
+from transformers.utils import TensorType, is_vision_available, logging
+
+from transformers import AutoImageProcessor
+
+logger = logging.get_logger(__name__)
+
+
+if is_vision_available():
+ from PIL import Image
+
+import torch
+import torchvision
+
+def padding_336(b):
+ width, height = b.size
+ tar = int(np.ceil(height / 336) * 336)
+ top_padding = int((tar - height)/2)
+ bottom_padding = tar - height - top_padding
+ left_padding = 0
+ right_padding = 0
+ b = torchvision.transforms.functional.pad(b, [left_padding, top_padding, right_padding, bottom_padding], fill=[255,255,255])
+
+ return b
+
+def calc_padded_size(width, height, padding_unit=336):
+ target_height = int(np.ceil(height / padding_unit) * padding_unit)
+ top_padding = int((target_height - height) / 2)
+ bottom_padding = target_height - height - top_padding
+ left_padding = 0
+ right_padding = 0
+ padded_width = width + left_padding + right_padding
+ padded_height = height + top_padding + bottom_padding
+ return padded_width, padded_height
+
+def HD_transform(img, hd_num=16):
+ width, height = img.size
+ trans = False
+ if width < height:
+ img = img.transpose(Image.TRANSPOSE)
+ trans = True
+ width, height = img.size
+ ratio = (width/ height)
+ scale = 1
+ while scale*np.ceil(scale/ratio) <= hd_num:
+ scale += 1
+ scale -= 1
+ new_w = int(scale * 336)
+ new_h = int(new_w / ratio)
+
+ img = torchvision.transforms.functional.resize(img, [new_h, new_w],)
+ img = padding_336(img)
+ width, height = img.size
+ if trans:
+ img = img.transpose(Image.TRANSPOSE)
+
+ return img
+
+def calc_hd_transform_size(width, height, hd_num=16):
+ transposed = False
+ if width < height:
+ width, height = height, width
+ transposed = True
+
+ ratio = width / height
+ scale = 1
+ while scale * np.ceil(scale / ratio) <= hd_num:
+ scale += 1
+ scale -= 1
+
+ new_width = int(scale * 336)
+ new_height = int(new_width / ratio)
+
+ padded_width, padded_height = calc_padded_size(new_width, new_height)
+
+ if transposed:
+ padded_width, padded_height = padded_height, padded_width
+
+ return padded_width, padded_height
+
+def pad_to_max_num_crops_tensor(images, max_crops=5):
+ """
+ images: B x 3 x H x W, B<=max_crops
+ """
+ B, _, H, W = images.shape
+ if B < max_crops:
+ pad = torch.zeros(max_crops - B, 3, H, W, dtype=images.dtype, device=images.device)
+ images = torch.cat([images, pad], dim=0)
+ return images
+
+
+class Phi3VImageProcessor(BaseImageProcessor):
+ r"""
+ Constructs a Phi3 image processor. Based on [`CLIPImageProcessor`] with incorporation of additional techniques
+ for processing high resolution images as explained in the [InternLM-XComposer2-4KHD](https://arxiv.org/abs/2401.16420)
+
+ Args:
+ image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
+ Mean to use if normalizing the image. This is a float or list of floats the length of the number of
+ channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method.
+ image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
+ Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
+ number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.
+ Can be overridden by the `image_std` parameter in the `preprocess` method.
+ do_convert_rgb (`bool`, *optional*, defaults to `True`):
+ Whether to convert the image to RGB.
+ """
+
+ model_input_names = ["pixel_values"]
+
+ def __init__(
+ self,
+ num_crops: int = 1,
+ image_mean: Optional[Union[float, List[float]]] = None,
+ image_std: Optional[Union[float, List[float]]] = None,
+ do_convert_rgb: bool = True,
+ **kwargs,
+ ) -> None:
+ super().__init__(**kwargs)
+ self.num_crops = num_crops
+ self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
+ self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
+ self.do_convert_rgb = do_convert_rgb
+
+ def calc_num_image_tokens(
+ self,
+ images: ImageInput
+ ):
+ """ Calculate the number of image tokens for each image.
+ Args:
+ images (`ImageInput`):
+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+ """
+ images = make_list_of_images(images)
+
+ if not valid_images(images):
+ raise ValueError(
+ "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
+ "torch.Tensor, tf.Tensor or jax.ndarray."
+ )
+
+ images = [image.convert('RGB') for image in images]
+ # (H, W, C)
+ elems = [HD_transform(im, hd_num = self.num_crops) for im in images]
+ shapes = [[im.size[1], im.size[0]] for im in elems]
+ num_img_tokens = [int((h//336*w//336+1)*144 + 1 + (h//336+1)*12) for h, w in shapes]
+ return num_img_tokens
+
+ def calc_num_image_tokens_from_image_size(self, width, height):
+ """
+ Calculate the number of image tokens for a given image size.
+ Args:
+ width (`int`): Width of the image.
+ height (`int`): Height of the image.
+ """
+ new_width, new_height = calc_hd_transform_size(width, height, hd_num=self.num_crops)
+ num_img_tokens = int((new_height // 336 * new_width // 336 + 1) * 144 + 1 + (new_height // 336 + 1) * 12)
+ return num_img_tokens
+
+ def preprocess(
+ self,
+ images: ImageInput,
+ image_mean: Optional[Union[float, List[float]]] = None,
+ image_std: Optional[Union[float, List[float]]] = None,
+ do_convert_rgb: bool = None,
+ return_tensors: Optional[Union[str, TensorType]] = None,
+ ):
+ """
+ Args:
+ images (`ImageInput`):
+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+ image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+ Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+ image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+ Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+ `True`.
+ do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+ Whether to convert the image to RGB.
+ return_tensors (`str` or `TensorType`, *optional*):
+ The type of tensors to return. Can be one of:
+ - Unset: Return a list of `np.ndarray`.
+ - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
+ - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
+ - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
+ - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
+ """
+ image_mean = image_mean if image_mean is not None else self.image_mean
+ image_std = image_std if image_std is not None else self.image_std
+ do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
+
+ images = make_list_of_images(images)
+
+ if not valid_images(images):
+ raise ValueError(
+ "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
+ "torch.Tensor, tf.Tensor or jax.ndarray."
+ )
+
+ if do_convert_rgb:
+ images = [convert_to_rgb(image) for image in images]
+
+ image_sizes = []
+ img_processor = torchvision.transforms.Compose([
+ torchvision.transforms.ToTensor(),
+ torchvision.transforms.Normalize(image_mean, image_std)
+ ])
+
+ # PIL images
+ # HD_transform pad images to size of multiiply of 336, 336
+ # convert to RGB first
+ images = [image.convert('RGB') for image in images]
+ elems = [HD_transform(im, hd_num = self.num_crops) for im in images]
+ # tensor transform and normalize
+ hd_images = [img_processor(im) for im in elems]
+ # create global image
+ global_image = [torch.nn.functional.interpolate(im.unsqueeze(0).float(), size=(336, 336), mode='bicubic',).to(im.dtype) for im in hd_images]
+
+ # [(3, h, w)], where h, w is multiple of 336
+ shapes = [[im.size(1), im.size(2)] for im in hd_images]
+ num_img_tokens = [int((h//336*w//336+1)*144 + 1 + (h//336+1)*12) for h, w in shapes]
+ # reshape to channel dimension -> (num_images, num_crops, 3, 336, 336)
+ # (1, 3, h//336, 336, w//336, 336) -> (1, h//336, w//336, 3, 336, 336) -> (h//336*w//336, 3, 336, 336)
+ hd_images_reshape = [im.reshape(1, 3, h//336, 336, w//336, 336).permute(0,2,4,1,3,5).reshape(-1, 3, 336, 336).contiguous() for im, (h, w) in zip(hd_images, shapes)]
+ # concat global image and local image
+ hd_images_reshape = [torch.cat([_global_image] + [_im], dim=0) for _global_image, _im in zip(global_image, hd_images_reshape)]
+
+ # pad to max_num_crops
+ image_transformed = [pad_to_max_num_crops_tensor(im, self.num_crops+1) for im in hd_images_reshape]
+ image_transformed = torch.stack(image_transformed, dim=0)
+ image_sizes = [torch.LongTensor(_shapes) for _shapes in shapes]
+ padded_images = image_transformed
+ image_sizes = shapes
+
+ data = {"pixel_values": padded_images,
+ "image_sizes": image_sizes,
+ "num_img_tokens": num_img_tokens
+ }
+
+ return BatchFeature(data=data, tensor_type=return_tensors)
+
+AutoImageProcessor.register("Phi3VImageProcessor", Phi3VImageProcessor)
\ No newline at end of file
diff --git a/examples/multimodal-modeling/Phi-3-vision/model/modeling_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/modeling_phi3_v.py
new file mode 100644
index 000000000..0b5357b16
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/model/modeling_phi3_v.py
@@ -0,0 +1,1634 @@
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" PyTorch Phi-3-V model."""
+
+import inspect
+import math
+import warnings
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+from .image_embedding_phi3_v import Phi3ImageEmbedding
+import transformers
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask
+from transformers.modeling_outputs import (
+ BaseModelOutputWithPast,
+ CausalLMOutputWithPast,
+ SequenceClassifierOutputWithPast,
+ TokenClassifierOutput,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import (
+ add_code_sample_docstrings,
+ add_start_docstrings,
+ add_start_docstrings_to_model_forward,
+ is_flash_attn_2_available,
+ is_flash_attn_greater_or_equal_2_10,
+ logging,
+ replace_return_docstrings,
+)
+from .configuration_phi3_v import Phi3VConfig
+
+
+
+logger = logging.get_logger(__name__)
+
+if is_flash_attn_2_available():
+ from flash_attn import flash_attn_func, flash_attn_varlen_func
+ from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input # noqa
+
+ _flash_supports_window_size = "window_size" in list(inspect.signature(flash_attn_func).parameters)
+
+_CHECKPOINT_FOR_DOC = "microsoft/Phi-3-vision-128k-instruct"
+_CONFIG_FOR_DOC = "Phi3VConfig"
+
+PHI3V_PRETRAINED_MODEL_ARCHIVE_LIST = [
+ "microsoft/Phi-3-vision-128k-instruct",
+ # See all Phi-3 models at https://huggingface.co/models?filter=Phi-3
+]
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Phi3
+class Phi3RMSNorm(nn.Module):
+ def __init__(self, hidden_size, eps=1e-6):
+ """
+ Phi3RMSNorm is equivalent to T5LayerNorm
+ """
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(hidden_size))
+ self.variance_epsilon = eps
+
+ def forward(self, hidden_states):
+ input_dtype = hidden_states.dtype
+ hidden_states = hidden_states.to(torch.float32)
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
+ hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+ return self.weight * hidden_states.to(input_dtype)
+
+
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
+def _get_unpad_data(attention_mask):
+ seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+ indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+ max_seqlen_in_batch = seqlens_in_batch.max().item()
+ cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
+ return (
+ indices,
+ cu_seqlens,
+ max_seqlen_in_batch,
+ )
+
+
+# Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding with gemma->phi3, Gemma->Phi3
+class Phi3RotaryEmbedding(nn.Module):
+ def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
+ super().__init__()
+
+ self.dim = dim
+ self.max_position_embeddings = max_position_embeddings
+ self.base = base
+ self.register_buffer("inv_freq", None, persistent=False)
+
+ @torch.no_grad()
+ def forward(self, x, position_ids, seq_len=None):
+ # x: [bs, num_attention_heads, seq_len, head_size]
+ if self.inv_freq is None:
+ self.inv_freq = 1.0 / (
+ self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim)
+ )
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
+ position_ids_expanded = position_ids[:, None, :].float()
+ # Force float32 since bfloat16 loses precision on long contexts
+ # See https://github.com/huggingface/transformers/pull/29285
+ device_type = x.device.type
+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
+ with torch.autocast(device_type=device_type, enabled=False):
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+ emb = torch.cat((freqs, freqs), dim=-1)
+ cos = emb.cos()
+ sin = emb.sin()
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+class Phi3SuScaledRotaryEmbedding(Phi3RotaryEmbedding):
+ def __init__(self, dim, config, device=None):
+ super().__init__(dim, config.max_position_embeddings, config.rope_theta, device)
+
+ self.short_factor = config.rope_scaling["short_factor"]
+ self.long_factor = config.rope_scaling["long_factor"]
+ self.original_max_position_embeddings = config.original_max_position_embeddings
+
+ @torch.no_grad()
+ def forward(self, x, position_ids, seq_len=None):
+ seq_len = torch.max(position_ids) + 1
+ if seq_len > self.original_max_position_embeddings:
+ ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device)
+ else:
+ ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device)
+
+ inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim
+ self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape)
+
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
+ position_ids_expanded = position_ids[:, None, :].float()
+
+ # Force float32 since bfloat16 loses precision on long contexts
+ # See https://github.com/huggingface/transformers/pull/29285
+ device_type = x.device.type
+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
+ with torch.autocast(device_type=device_type, enabled=False):
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+ emb = torch.cat((freqs, freqs), dim=-1)
+
+ scale = self.max_position_embeddings / self.original_max_position_embeddings
+ if scale <= 1.0:
+ scaling_factor = 1.0
+ else:
+ scaling_factor = math.sqrt(1 + math.log(scale) / math.log(self.original_max_position_embeddings))
+
+ cos = emb.cos() * scaling_factor
+ sin = emb.sin() * scaling_factor
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+class Phi3YarnScaledRotaryEmbedding(Phi3RotaryEmbedding):
+ def __init__(self, dim, config, device=None):
+ super().__init__(dim, config.max_position_embeddings, config.rope_theta, device)
+
+ self.short_factor = config.rope_scaling["short_factor"]
+ self.long_factor = config.rope_scaling["long_factor"]
+ self.original_max_position_embeddings = config.original_max_position_embeddings
+
+ @torch.no_grad()
+ def forward(self, x, position_ids, seq_len=None):
+ seq_len = torch.max(position_ids) + 1
+ if seq_len > self.original_max_position_embeddings:
+ ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device)
+ else:
+ ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device)
+
+ inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim
+ self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape)
+
+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
+ position_ids_expanded = position_ids[:, None, :].float()
+
+ # Force float32 since bfloat16 loses precision on long contexts
+ # See https://github.com/huggingface/transformers/pull/29285
+ device_type = x.device.type
+ device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
+ with torch.autocast(device_type=device_type, enabled=False):
+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
+ emb = torch.cat((freqs, freqs), dim=-1)
+
+ scale = self.max_position_embeddings / self.original_max_position_embeddings
+ if scale <= 1.0:
+ scaling_factor = 1.0
+ else:
+ scaling_factor = 0.1 * math.log(scale) + 1.0
+
+ cos = emb.cos() * scaling_factor
+ sin = emb.sin() * scaling_factor
+ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+# Copied from transformers.models.llama.modeling_llama.rotate_half
+def rotate_half(x):
+ """Rotates half the hidden dims of the input."""
+ x1 = x[..., : x.shape[-1] // 2]
+ x2 = x[..., x.shape[-1] // 2 :]
+ return torch.cat((-x2, x1), dim=-1)
+
+
+# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
+ """Applies Rotary Position Embedding to the query and key tensors.
+
+ Args:
+ q (`torch.Tensor`): The query tensor.
+ k (`torch.Tensor`): The key tensor.
+ cos (`torch.Tensor`): The cosine part of the rotary embedding.
+ sin (`torch.Tensor`): The sine part of the rotary embedding.
+ position_ids (`torch.Tensor`, *optional*):
+ Deprecated and unused.
+ unsqueeze_dim (`int`, *optional*, defaults to 1):
+ The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+ sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+ that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+ k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+ cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+ the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+ Returns:
+ `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+ """
+ cos = cos.unsqueeze(unsqueeze_dim)
+ sin = sin.unsqueeze(unsqueeze_dim)
+ q_embed = (q * cos) + (rotate_half(q) * sin)
+ k_embed = (k * cos) + (rotate_half(k) * sin)
+ return q_embed, k_embed
+
+
+class Phi3MLP(nn.Module):
+ def __init__(self, config):
+ super().__init__()
+
+ self.config = config
+ self.gate_up_proj = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False)
+ self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
+
+ self.activation_fn = ACT2FN[config.hidden_act]
+
+ def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
+ up_states = self.gate_up_proj(hidden_states)
+
+ gate, up_states = up_states.chunk(2, dim=-1)
+ up_states = up_states * self.activation_fn(gate)
+
+ return self.down_proj(up_states)
+
+
+# Copied from transformers.models.llama.modeling_llama.repeat_kv with llama->phi
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+ """
+ This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+ num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+ """
+ batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+ if n_rep == 1:
+ return hidden_states
+ hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+ return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+class Phi3Attention(nn.Module):
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+ def __init__(self, config: Phi3VConfig, layer_idx: Optional[int] = None):
+ super().__init__()
+ self.config = config
+ self.layer_idx = layer_idx
+ if layer_idx is None:
+ logger.warning_once(
+ f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will "
+ "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` "
+ "when creating this class."
+ )
+
+ self.attention_dropout = config.attention_dropout
+ self.hidden_size = config.hidden_size
+ self.num_heads = config.num_attention_heads
+ self.head_dim = self.hidden_size // self.num_heads
+ self.num_key_value_heads = config.num_key_value_heads
+ self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+ self.max_position_embeddings = config.max_position_embeddings
+ self.original_max_position_embeddings = config.original_max_position_embeddings
+ self.rope_theta = config.rope_theta
+ self.rope_scaling = config.rope_scaling
+ self.is_causal = True
+
+ if (self.head_dim * self.num_heads) != self.hidden_size:
+ raise ValueError(
+ f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+ f" and `num_heads`: {self.num_heads})."
+ )
+
+ op_size = self.num_heads * self.head_dim + 2 * (self.num_key_value_heads * self.head_dim)
+ self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
+ self.qkv_proj = nn.Linear(self.hidden_size, op_size, bias=False)
+ self._init_rope()
+
+ def _init_rope(self):
+ if self.rope_scaling is None:
+ self.rotary_emb = Phi3RotaryEmbedding(
+ self.head_dim,
+ max_position_embeddings=self.max_position_embeddings,
+ base=self.rope_theta,
+ )
+ else:
+ scaling_type = self.config.rope_scaling["type"]
+ if scaling_type == "su":
+ self.rotary_emb = Phi3SuScaledRotaryEmbedding(self.head_dim, self.config)
+ elif scaling_type == "yarn":
+ self.rotary_emb = Phi3YarnScaledRotaryEmbedding(self.head_dim, self.config)
+ else:
+ raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Cache] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ logger.warning_once("You are not running the flash-attention implementation, expect numerical differences.")
+
+ bsz, q_len, _ = hidden_states.size()
+
+ qkv = self.qkv_proj(hidden_states)
+ query_pos = self.num_heads * self.head_dim
+ query_states = qkv[..., :query_pos]
+ key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]
+ value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]
+
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+ kv_seq_len = key_states.shape[-2]
+ if past_key_value is not None:
+ if self.layer_idx is None:
+ raise ValueError(
+ f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
+ "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+ "with a layer index."
+ )
+ kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+ cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len)
+
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+ if past_key_value is not None:
+ cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+ # repeat k/v heads if n_kv_heads < n_heads
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+ attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+ if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+ raise ValueError(
+ f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+ f" {attn_weights.size()}"
+ )
+
+ if attention_mask is not None:
+ if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+ raise ValueError(
+ f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+ )
+ attn_weights = attn_weights + attention_mask
+
+ # upcast attention to fp32
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(value_states.dtype)
+ attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+
+ attn_output = torch.matmul(attn_weights, value_states)
+
+ if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+ raise ValueError(
+ f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+ f" {attn_output.size()}"
+ )
+
+ attn_output = attn_output.transpose(1, 2).contiguous()
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+ attn_output = self.o_proj(attn_output)
+
+ if not output_attentions:
+ attn_weights = None
+
+ return attn_output, attn_weights, past_key_value
+
+
+class Phi3FlashAttention2(Phi3Attention):
+ """
+ Phi-3 flash attention module. This module inherits from `Phi3Attention` as the weights of the module stays
+ untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
+ flash attention and deal with padding tokens in case the input contains any of them.
+ """
+
+ # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
+ # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignment, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
+ # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
+ self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.LongTensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Cache] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ **kwargs,
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ # Phi3FlashAttention2 attention does not support output_attentions
+
+ if not _flash_supports_window_size:
+ logger.warning_once(
+ "The current flash attention version does not support sliding window attention. Please use `attn_implementation='eager'` or upgrade flash-attn library."
+ )
+ raise ValueError("The current flash attention version does not support sliding window attention.")
+
+ output_attentions = False
+
+ if "padding_mask" in kwargs:
+ warnings.warn(
+ "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
+ )
+
+ # overwrite attention_mask with padding_mask
+ attention_mask = kwargs.pop("padding_mask")
+
+ bsz, q_len, _ = hidden_states.size()
+
+ qkv = self.qkv_proj(hidden_states)
+ query_pos = self.num_heads * self.head_dim
+ query_states = qkv[..., :query_pos]
+ key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]
+ value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]
+
+ # Flash attention requires the input to have the shape
+ # batch_size x seq_length x head_dim x hidden_dim
+ # therefore we just need to keep the original shape
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+ kv_seq_len = key_states.shape[-2]
+ if past_key_value is not None:
+ if self.layer_idx is None:
+ raise ValueError(
+ f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
+ "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+ "with a layer index."
+ )
+ kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+
+ # Because the input can be padded, the absolute sequence length depends on the max position id.
+ rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1
+ cos, sin = self.rotary_emb(value_states, position_ids, seq_len=rotary_seq_len)
+
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+ use_sliding_windows = (
+ _flash_supports_window_size
+ and getattr(self.config, "sliding_window", None) is not None
+ and kv_seq_len > self.config.sliding_window
+ )
+
+ if past_key_value is not None:
+ # Activate slicing cache only if the config has a value `sliding_windows` attribute
+ cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0
+ if (
+ getattr(self.config, "sliding_window", None) is not None
+ and kv_seq_len > self.config.sliding_window
+ and cache_has_contents
+ ):
+ slicing_tokens = 1 - self.config.sliding_window
+
+ past_key = past_key_value[self.layer_idx][0]
+ past_value = past_key_value[self.layer_idx][1]
+
+ past_key = past_key[:, :, slicing_tokens:, :].contiguous()
+ past_value = past_value[:, :, slicing_tokens:, :].contiguous()
+
+ if past_key.shape[-2] != self.config.sliding_window - 1:
+ raise ValueError(
+ f"past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got"
+ f" {past_key.shape}"
+ )
+
+ if attention_mask is not None:
+ attention_mask = attention_mask[:, slicing_tokens:]
+ attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1)
+
+ cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+ # repeat k/v heads if n_kv_heads < n_heads
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+ attn_dropout = self.attention_dropout if self.training else 0.0
+
+ # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+ # therefore the input hidden states gets silently casted in float32. Hence, we need
+ # cast them back in the correct dtype just to be sure everything works as expected.
+ # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+ # in fp32.
+
+ if query_states.dtype == torch.float32:
+ if torch.is_autocast_enabled():
+ target_dtype = torch.get_autocast_gpu_dtype()
+ # Handle the case where the model is quantized
+ elif hasattr(self.config, "_pre_quantization_dtype"):
+ target_dtype = self.config._pre_quantization_dtype
+ else:
+ target_dtype = self.qkv_proj.weight.dtype
+
+ logger.warning_once(
+ f"The input hidden states seems to be silently casted in float32, this might be related to"
+ f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+ f" {target_dtype}."
+ )
+
+ query_states = query_states.to(target_dtype)
+ key_states = key_states.to(target_dtype)
+ value_states = value_states.to(target_dtype)
+
+ # Reashape to the expected shape for Flash Attention
+ query_states = query_states.transpose(1, 2)
+ key_states = key_states.transpose(1, 2)
+ value_states = value_states.transpose(1, 2)
+
+ attn_output = self._flash_attention_forward(
+ query_states,
+ key_states,
+ value_states,
+ attention_mask,
+ q_len,
+ dropout=attn_dropout,
+ use_sliding_windows=use_sliding_windows,
+ )
+
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
+ attn_output = self.o_proj(attn_output)
+
+ if not output_attentions:
+ attn_weights = None
+
+ return attn_output, attn_weights, past_key_value
+
+ # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._flash_attention_forward
+ def _flash_attention_forward(
+ self,
+ query_states,
+ key_states,
+ value_states,
+ attention_mask,
+ query_length,
+ dropout=0.0,
+ softmax_scale=None,
+ use_sliding_windows=False,
+ ):
+ """
+ Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+ first unpad the input, then computes the attention scores and pad the final attention scores.
+
+ Args:
+ query_states (`torch.Tensor`):
+ Input query states to be passed to Flash Attention API
+ key_states (`torch.Tensor`):
+ Input key states to be passed to Flash Attention API
+ value_states (`torch.Tensor`):
+ Input value states to be passed to Flash Attention API
+ attention_mask (`torch.Tensor`):
+ The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+ position of padding tokens and 1 for the position of non-padding tokens.
+ dropout (`float`):
+ Attention dropout
+ softmax_scale (`float`, *optional*):
+ The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+ use_sliding_windows (`bool`, *optional*):
+ Whether to activate sliding window attention.
+ """
+ if not self._flash_attn_uses_top_left_mask:
+ causal = self.is_causal
+ else:
+ # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
+ causal = self.is_causal and query_length != 1
+
+ # Contains at least one padding token in the sequence
+ if attention_mask is not None:
+ batch_size = query_states.shape[0]
+ query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+ query_states, key_states, value_states, attention_mask, query_length
+ )
+
+ cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+ max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+ if not use_sliding_windows:
+ attn_output_unpad = flash_attn_varlen_func(
+ query_states,
+ key_states,
+ value_states,
+ cu_seqlens_q=cu_seqlens_q,
+ cu_seqlens_k=cu_seqlens_k,
+ max_seqlen_q=max_seqlen_in_batch_q,
+ max_seqlen_k=max_seqlen_in_batch_k,
+ dropout_p=dropout,
+ softmax_scale=softmax_scale,
+ causal=causal,
+ )
+ else:
+ attn_output_unpad = flash_attn_varlen_func(
+ query_states,
+ key_states,
+ value_states,
+ cu_seqlens_q=cu_seqlens_q,
+ cu_seqlens_k=cu_seqlens_k,
+ max_seqlen_q=max_seqlen_in_batch_q,
+ max_seqlen_k=max_seqlen_in_batch_k,
+ dropout_p=dropout,
+ softmax_scale=softmax_scale,
+ causal=causal,
+ window_size=(self.config.sliding_window, self.config.sliding_window),
+ )
+
+ attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+ else:
+ if not use_sliding_windows:
+ attn_output = flash_attn_func(
+ query_states,
+ key_states,
+ value_states,
+ dropout,
+ softmax_scale=softmax_scale,
+ causal=causal,
+ )
+ else:
+ attn_output = flash_attn_func(
+ query_states,
+ key_states,
+ value_states,
+ dropout,
+ softmax_scale=softmax_scale,
+ causal=causal,
+ window_size=(self.config.sliding_window, self.config.sliding_window),
+ )
+
+ return attn_output
+
+ # Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._upad_input
+ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+ batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape
+
+ # On the first iteration we need to properly re-create the padding mask
+ # by slicing it on the proper place
+ if kv_seq_len != attention_mask.shape[-1]:
+ attention_mask_num_tokens = attention_mask.shape[-1]
+ attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :]
+
+ indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+
+ key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
+ value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
+
+ if query_length == kv_seq_len:
+ query_layer = index_first_axis(
+ query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k
+ )
+ cu_seqlens_q = cu_seqlens_k
+ max_seqlen_in_batch_q = max_seqlen_in_batch_k
+ indices_q = indices_k
+ elif query_length == 1:
+ max_seqlen_in_batch_q = 1
+ cu_seqlens_q = torch.arange(
+ batch_size + 1, dtype=torch.int32, device=query_layer.device
+ ) # There is a memcpy here, that is very bad.
+ indices_q = cu_seqlens_q[:-1]
+ query_layer = query_layer.squeeze(1)
+ else:
+ # The -q_len: slice assumes left padding.
+ attention_mask = attention_mask[:, -query_length:]
+ query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+ return (
+ query_layer,
+ key_layer,
+ value_layer,
+ indices_q,
+ (cu_seqlens_q, cu_seqlens_k),
+ (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+ )
+
+
+# copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Phi3
+# TODO @Arthur no longer copied from LLama after static cache
+class Phi3SdpaAttention(Phi3Attention):
+ """
+ Phi3 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
+ `Phi3Attention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
+ SDPA API.
+ """
+
+ # Adapted from Phi3Attention.forward
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Cache] = None,
+ output_attentions: bool = False,
+ use_cache: bool = False,
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+ if output_attentions:
+ # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
+ logger.warning_once(
+ "Phi3Model is using Phi3SdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, "
+ 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+ )
+ return super().forward(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ )
+
+ bsz, q_len, _ = hidden_states.size()
+
+ qkv = self.qkv_proj(hidden_states)
+ query_pos = self.num_heads * self.head_dim
+ query_states = qkv[..., :query_pos]
+ key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]
+ value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]
+
+ query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+ key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+ value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+ kv_seq_len = key_states.shape[-2]
+ if past_key_value is not None:
+ kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+ cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len)
+
+ query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
+
+ if past_key_value is not None:
+ cache_kwargs = {"sin": sin, "cos": cos} # Specific to RoPE models
+ key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+ key_states = repeat_kv(key_states, self.num_key_value_groups)
+ value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+ if attention_mask is not None:
+ if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+ raise ValueError(
+ f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+ )
+
+ # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
+ # Reference: https://github.com/pytorch/pytorch/issues/112577.
+ if query_states.device.type == "cuda" and attention_mask is not None:
+ query_states = query_states.contiguous()
+ key_states = key_states.contiguous()
+ value_states = value_states.contiguous()
+
+ attn_output = torch.nn.functional.scaled_dot_product_attention(
+ query_states,
+ key_states,
+ value_states,
+ attn_mask=attention_mask,
+ dropout_p=self.attention_dropout if self.training else 0.0,
+ # The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1.
+ is_causal=self.is_causal and attention_mask is None and q_len > 1,
+ )
+
+ attn_output = attn_output.transpose(1, 2).contiguous()
+ attn_output = attn_output.view(bsz, q_len, self.hidden_size)
+
+ attn_output = self.o_proj(attn_output)
+
+ return attn_output, None, past_key_value
+
+
+PHI3_ATTENTION_CLASSES = {
+ "eager": Phi3Attention,
+ "flash_attention_2": Phi3FlashAttention2,
+ "sdpa": Phi3SdpaAttention,
+}
+
+
+class Phi3DecoderLayer(nn.Module):
+ def __init__(self, config: Phi3VConfig, layer_idx: int):
+ super().__init__()
+
+ self.config = config
+ self.self_attn = PHI3_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx)
+
+ self.mlp = Phi3MLP(config)
+ self.input_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+ self.resid_attn_dropout = nn.Dropout(config.resid_pdrop)
+ self.resid_mlp_dropout = nn.Dropout(config.resid_pdrop)
+ self.post_attention_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
+ output_attentions: Optional[bool] = False,
+ use_cache: Optional[bool] = False,
+ **kwargs,
+ ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+ if "padding_mask" in kwargs:
+ warnings.warn(
+ "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
+ )
+ """
+ Args:
+ hidden_states (`torch.FloatTensor`):
+ input to the layer of shape `(batch, seq_len, embed_dim)`
+ attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
+ `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+ position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range
+ `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+ returned tensors for more detail.
+ use_cache (`bool`, *optional*):
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
+ past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+ """
+
+ residual = hidden_states
+
+ hidden_states = self.input_layernorm(hidden_states)
+
+ # Self Attention
+ attn_outputs, self_attn_weights, present_key_value = self.self_attn(
+ hidden_states=hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_value,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ )
+
+ hidden_states = residual + self.resid_attn_dropout(attn_outputs)
+
+ residual = hidden_states
+ hidden_states = self.post_attention_layernorm(hidden_states)
+ hidden_states = self.mlp(hidden_states)
+ hidden_states = residual + self.resid_mlp_dropout(hidden_states)
+
+ outputs = (hidden_states,)
+
+ if output_attentions:
+ outputs += (self_attn_weights,)
+
+ if use_cache:
+ outputs += (present_key_value,)
+
+ return outputs
+
+
+PHI3V_START_DOCSTRING = r"""
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
+
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
+
+ Parameters:
+ config ([`Phi3VConfig`]):
+ Model configuration class with all the parameters of the model. Initializing with a config file does not
+ load the weights associated with the model, only the configuration. Check out the
+ [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+ "The bare Phi-3-V model outputting raw hidden-states without any specific head on top.",
+ PHI3V_START_DOCSTRING,
+)
+class Phi3VPreTrainedModel(PreTrainedModel):
+ config_class = Phi3VConfig
+ base_model_prefix = "model"
+ supports_gradient_checkpointing = True
+ _no_split_modules = ["Phi3DecoderLayer"]
+ _skip_keys_device_placement = "past_key_values"
+ _supports_flash_attn_2 = True
+ _supports_sdpa = False
+ _supports_cache_class = True
+
+ _version = "0.0.5"
+
+ def _init_weights(self, module):
+ std = self.config.initializer_range
+ if isinstance(module, nn.Linear):
+ module.weight.data.normal_(mean=0.0, std=std)
+ if module.bias is not None:
+ module.bias.data.zero_()
+ elif isinstance(module, nn.Embedding):
+ module.weight.data.normal_(mean=0.0, std=std)
+ if module.padding_idx is not None:
+ module.weight.data[module.padding_idx].zero_()
+
+
+PHI3V_INPUTS_DOCSTRING = r"""
+ Args:
+ input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+ Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+ it.
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ [What are input IDs?](../glossary#input-ids)
+ attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+
+ - 1 for tokens that are **not masked**,
+ - 0 for tokens that are **masked**.
+
+ [What are attention masks?](../glossary#attention-mask)
+
+ Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
+
+ If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
+ `past_key_values`).
+
+ If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+ and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+ information on the default strategy.
+
+ - 1 indicates the head is **not masked**,
+ - 0 indicates the head is **masked**.
+ position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.n_positions - 1]`.
+
+ [What are position IDs?](../glossary#position-ids)
+ past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
+ Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+ blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
+ returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
+
+ Two formats are allowed:
+ - a [`~cache_utils.Cache`] instance;
+ - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
+ cache format.
+
+ The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
+ legacy cache format will be returned.
+
+ If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
+ have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
+ of shape `(batch_size, sequence_length)`.
+ inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
+ pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, image_size, image_size)):
+ The tensors corresponding to the input images. Pixel values can be obtained using [`AutoImageProcessor`].
+ See [`Phi3ImageProcessor.__call__`] for details.
+ image_sizes (`torch.LongTensor` of shape `(batch_size, 2)`, *optional*):
+ The sizes of the images in the batch, being (height, width) for each image.
+ use_cache (`bool`, *optional*):
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
+ output_attentions (`bool`, *optional*):
+ Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+ tensors for more detail.
+ output_hidden_states (`bool`, *optional*):
+ Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+ more detail.
+ return_dict (`bool`, *optional*):
+ Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+@add_start_docstrings(
+ "The bare Phi-3-V model outputting raw hidden-states without any specific head on top.",
+ PHI3V_START_DOCSTRING,
+)
+class Phi3VModel(Phi3VPreTrainedModel):
+ """
+ Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi3DecoderLayer`]
+
+ Args:
+ config: Phi3Config
+ """
+
+ def __init__(self, config: Phi3VConfig):
+ super().__init__(config)
+ self.padding_idx = config.pad_token_id
+ self.vocab_size = config.vocab_size
+
+ self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+ self.embed_dropout = nn.Dropout(config.embd_pdrop)
+
+ self.vision_embed_tokens = None
+ if isinstance(config.embd_layer, dict):
+ # vision embedding layer
+ embedding_config = {
+ 'embedding_cls': config.embd_layer['embedding_cls'],
+ **config.embd_layer
+ }
+ self.vision_embed_tokens = Phi3ImageEmbedding(config, wte=self.embed_tokens, **embedding_config)
+ # # set wte the same for vision embedding
+ # self.vision_embed_tokens.wte.weight = self.embed_tokens.weight
+
+ self.layers = nn.ModuleList(
+ [Phi3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+ )
+ self._attn_implementation = config._attn_implementation
+ self.norm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+ self.gradient_checkpointing = False
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.embed_tokens = value
+
+ @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING)
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ image_sizes: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ # retrieve input_ids and inputs_embeds
+ if input_ids is not None and inputs_embeds is not None:
+ raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+ elif input_ids is not None:
+ batch_size, seq_length = input_ids.shape[:2]
+ elif inputs_embeds is not None:
+ batch_size, seq_length = inputs_embeds.shape[:2]
+ else:
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+ past_key_values_length = 0
+
+ if self.gradient_checkpointing and self.training:
+ if use_cache:
+ logger.warning_once(
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+ )
+ use_cache = False
+
+ if use_cache:
+ use_legacy_cache = not isinstance(past_key_values, Cache)
+ if use_legacy_cache:
+ past_key_values = DynamicCache.from_legacy_cache(past_key_values)
+ past_key_values_length = past_key_values.get_usable_length(seq_length)
+
+ if position_ids is None:
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
+ position_ids = torch.arange(
+ past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device
+ )
+ position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
+ else:
+ position_ids = position_ids.view(-1, seq_length).long()
+
+ if inputs_embeds is None:
+ if pixel_values is not None and image_sizes is not None:
+ assert self.vision_embed_tokens is not None, "Vision embedding layer is not defined"
+ inputs_embeds = self.vision_embed_tokens(input_ids, pixel_values=pixel_values, image_sizes=image_sizes)
+ else:
+ inputs_embeds = self.embed_tokens(input_ids)
+
+ if attention_mask is not None and self._attn_implementation == "flash_attention_2" and use_cache:
+ is_padding_right = attention_mask[:, -1].sum().item() != batch_size
+ if is_padding_right:
+ raise ValueError(
+ "You are attempting to perform batched generation with padding_side='right'"
+ " this may lead to unexpected behaviour for Flash Attention version of Phi3. Make sure to "
+ " call `tokenizer.padding_side = 'left'` before tokenizing the input. "
+ )
+
+ if self._attn_implementation == "flash_attention_2":
+ # 2d mask is passed through the layers
+ attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
+ else:
+ # 4d mask is passed through the layers
+ attention_mask = _prepare_4d_causal_attention_mask(
+ attention_mask,
+ (batch_size, seq_length),
+ inputs_embeds,
+ past_key_values_length,
+ sliding_window=self.config.sliding_window,
+ )
+
+ hidden_states = inputs_embeds
+
+ # decoder layers
+ all_hidden_states = () if output_hidden_states else None
+ all_self_attns = () if output_attentions else None
+ next_decoder_cache = None
+
+ for decoder_layer in self.layers:
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ if self.gradient_checkpointing and self.training:
+ layer_outputs = torch.utils.checkpoint.checkpoint(
+ decoder_layer.__call__,
+ hidden_states,
+ attention_mask,
+ position_ids,
+ past_key_values,
+ output_attentions,
+ use_cache,
+ use_reentrant=False
+ )
+ else:
+ layer_outputs = decoder_layer(
+ hidden_states,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_value=past_key_values,
+ output_attentions=output_attentions,
+ use_cache=use_cache,
+ )
+
+ hidden_states = layer_outputs[0]
+
+ if use_cache:
+ next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+
+ if output_attentions:
+ all_self_attns += (layer_outputs[1],)
+
+ hidden_states = self.norm(hidden_states)
+
+ # add hidden states from the last decoder layer
+ if output_hidden_states:
+ all_hidden_states += (hidden_states,)
+
+ next_cache = None
+ if use_cache:
+ next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache
+ if not return_dict:
+ return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
+ return BaseModelOutputWithPast(
+ last_hidden_state=hidden_states,
+ past_key_values=next_cache,
+ hidden_states=all_hidden_states,
+ attentions=all_self_attns,
+ )
+
+
+class Phi3VForCausalLM(Phi3VPreTrainedModel):
+ _tied_weights_keys = ["lm_head.weight"]
+
+ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.__init__ with Llama->Phi3
+ def __init__(self, config):
+ super().__init__(config)
+ self.model = Phi3VModel(config)
+ self.vocab_size = config.vocab_size
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_input_embeddings
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_input_embeddings
+ def set_input_embeddings(self, value):
+ self.model.embed_tokens = value
+
+ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_output_embeddings
+ def get_output_embeddings(self):
+ return self.lm_head
+
+ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_output_embeddings
+ def set_output_embeddings(self, new_embeddings):
+ self.lm_head = new_embeddings
+
+ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_decoder
+ def set_decoder(self, decoder):
+ self.model = decoder
+
+ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_decoder
+ def get_decoder(self):
+ return self.model
+
+ # Ignore copy
+ @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING)
+ @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ image_sizes: Optional[torch.LongTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
+ r"""
+ Args:
+ labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+
+ Returns:
+
+ Example:
+
+ ```python
+ >>> from transformers import AutoTokenizer, Phi3ForCausalLM
+
+ >>> model = Phi3ForCausalLM.from_pretrained("microsoft/phi-3-mini-4k-instruct")
+ >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-3-mini-4k-instruct")
+
+ >>> prompt = "This is an example script ."
+ >>> inputs = tokenizer(prompt, return_tensors="pt")
+
+ >>> # Generate
+ >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
+ >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+ 'This is an example script .\n Certainly! Below is a sample script that demonstrates a simple task, such as calculating the sum'
+ ```"""
+
+ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+ output_hidden_states = (
+ output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+ )
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+ outputs = self.model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ pixel_values=pixel_values,
+ image_sizes=image_sizes,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ hidden_states = outputs[0]
+ logits = self.lm_head(hidden_states)
+ logits = logits.float()
+
+ loss = None
+ if labels is not None:
+ # Shift so that tokens < n predict n
+ shift_logits = logits[..., :-1, :].contiguous()
+ shift_labels = labels[..., 1:].contiguous()
+ # Flatten the tokens
+ loss_fct = CrossEntropyLoss()
+ shift_logits = shift_logits.view(-1, self.config.vocab_size)
+ shift_labels = shift_labels.view(-1)
+ # Enable model parallelism
+ shift_labels = shift_labels.to(shift_logits.device)
+ loss = loss_fct(shift_logits, shift_labels)
+
+ if not return_dict:
+ output = (logits,) + outputs[1:]
+ return (loss,) + output if loss is not None else output
+
+ return CausalLMOutputWithPast(
+ loss=loss,
+ logits=logits,
+ past_key_values=outputs.past_key_values,
+ hidden_states=outputs.hidden_states,
+ attentions=outputs.attentions,
+ )
+
+ # Copied from transformers.models.persimmon.modeling_persimmon.PersimmonForCausalLM.prepare_inputs_for_generation
+ def prepare_inputs_for_generation(
+ self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, pixel_values=None, image_sizes=None, **kwargs
+ ):
+ if past_key_values is not None:
+ if isinstance(past_key_values, Cache):
+ cache_length = past_key_values.get_seq_length()
+ past_length = past_key_values.seen_tokens
+ max_cache_length = past_key_values.get_max_length()
+ else:
+ cache_length = past_length = past_key_values[0][0].shape[2]
+ max_cache_length = None
+
+ # Keep only the unprocessed tokens:
+ # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+ # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+ # input)
+ if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+ input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+ # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+ # input_ids based on the past_length.
+ elif past_length < input_ids.shape[1]:
+ input_ids = input_ids[:, past_length:]
+ # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+
+ # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
+ if (
+ max_cache_length is not None
+ and attention_mask is not None
+ and cache_length + input_ids.shape[1] > max_cache_length
+ ):
+ attention_mask = attention_mask[:, -max_cache_length:]
+
+ position_ids = kwargs.get("position_ids", None)
+ if attention_mask is not None and position_ids is None:
+ # create position_ids on the fly for batch generation
+ position_ids = attention_mask.long().cumsum(-1) - 1
+ position_ids.masked_fill_(attention_mask == 0, 1)
+ if past_key_values:
+ position_ids = position_ids[:, -input_ids.shape[1] :]
+
+ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+ if inputs_embeds is not None and past_key_values is None:
+ model_inputs = {"inputs_embeds": inputs_embeds}
+ else:
+ model_inputs = {"input_ids": input_ids}
+
+ model_inputs.update(
+ {
+ "position_ids": position_ids,
+ "past_key_values": past_key_values,
+ "use_cache": kwargs.get("use_cache"),
+ "attention_mask": attention_mask,
+ "pixel_values": pixel_values,
+ "image_sizes": image_sizes,
+ }
+ )
+ return model_inputs
+
+ @staticmethod
+ # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM._reorder_cache
+ def _reorder_cache(past_key_values, beam_idx):
+ reordered_past = ()
+ for layer_past in past_key_values:
+ reordered_past += (
+ tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
+ )
+ return reordered_past
+
+
+@add_start_docstrings(
+ """
+ The [`Phi3VModel`] with a sequence classification head on top (linear layer).
+
+ [`Phi3VForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+ (e.g. GPT-2) do.
+
+ Since it does classification on the last token, it requires to know the position of the last token. If a
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
+ """,
+ PHI3V_START_DOCSTRING,
+)
+# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Phi3, LLAMA->PHI3, self.transformer->self.model, transformer_outputs->model_outputs
+class Phi3VForSequenceClassification(Phi3VPreTrainedModel):
+ def __init__(self, config):
+ super().__init__(config)
+ self.num_labels = config.num_labels
+ self.model = Phi3VModel(config)
+ self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ def get_input_embeddings(self):
+ return self.model.embed_tokens
+
+ def set_input_embeddings(self, value):
+ self.model.embed_tokens = value
+
+ @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING)
+ def forward(
+ self,
+ input_ids: torch.LongTensor = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ position_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None,
+ inputs_embeds: Optional[torch.FloatTensor] = None,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ image_sizes: Optional[torch.LongTensor] = None,
+ labels: Optional[torch.LongTensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+ r"""
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ """
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ model_outputs = self.model(
+ input_ids,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ pixel_values=pixel_values,
+ image_sizes=image_sizes,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+ hidden_states = model_outputs[0]
+ logits = self.score(hidden_states)
+
+ if input_ids is not None:
+ batch_size = input_ids.shape[0]
+ else:
+ batch_size = inputs_embeds.shape[0]
+
+ if self.config.pad_token_id is None and batch_size != 1:
+ raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
+ if self.config.pad_token_id is None:
+ sequence_lengths = -1
+ else:
+ if input_ids is not None:
+ # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
+ sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
+ sequence_lengths = sequence_lengths % input_ids.shape[-1]
+ sequence_lengths = sequence_lengths.to(logits.device)
+ else:
+ sequence_lengths = -1
+
+ pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+
+ loss = None
+ if labels is not None:
+ labels = labels.to(logits.device)
+ if self.config.problem_type is None:
+ if self.num_labels == 1:
+ self.config.problem_type = "regression"
+ elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+ self.config.problem_type = "single_label_classification"
+ else:
+ self.config.problem_type = "multi_label_classification"
+
+ if self.config.problem_type == "regression":
+ loss_fct = MSELoss()
+ if self.num_labels == 1:
+ loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
+ else:
+ loss = loss_fct(pooled_logits, labels)
+ elif self.config.problem_type == "single_label_classification":
+ loss_fct = CrossEntropyLoss()
+ loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+ elif self.config.problem_type == "multi_label_classification":
+ loss_fct = BCEWithLogitsLoss()
+ loss = loss_fct(pooled_logits, labels)
+ if not return_dict:
+ output = (pooled_logits,) + model_outputs[1:]
+ return ((loss,) + output) if loss is not None else output
+
+ return SequenceClassifierOutputWithPast(
+ loss=loss,
+ logits=pooled_logits,
+ past_key_values=model_outputs.past_key_values,
+ hidden_states=model_outputs.hidden_states,
+ attentions=model_outputs.attentions,
+ )
+
+
+@add_start_docstrings(
+ """
+ [`Phi3VModel`] with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
+ Named-Entity-Recognition (NER) tasks.
+ """,
+ PHI3V_START_DOCSTRING,
+)
+# Copied from transformers.models.mpt.modeling_mpt.MptForTokenClassification with Mpt->Phi3,MPT->PHI3,self.transformer->self.model,transformer_outputs->model_outputs
+class Phi3VForTokenClassification(Phi3VPreTrainedModel):
+ def __init__(self, config: Phi3VConfig):
+ super().__init__(config)
+ self.num_labels = config.num_labels
+
+ self.model = Phi3VModel(config)
+ if hasattr(config, "classifier_dropout") and config.classifier_dropout is not None:
+ classifier_dropout = config.classifier_dropout
+ elif hasattr(config, "hidden_dropout") and config.hidden_dropout is not None:
+ classifier_dropout = config.hidden_dropout
+ else:
+ classifier_dropout = 0.1
+ self.dropout = nn.Dropout(classifier_dropout)
+ self.classifier = nn.Linear(config.hidden_size, config.num_labels)
+
+ # Initialize weights and apply final processing
+ self.post_init()
+
+ @add_start_docstrings_to_model_forward(PHI3V_INPUTS_DOCSTRING)
+ @add_code_sample_docstrings(
+ checkpoint=_CHECKPOINT_FOR_DOC,
+ output_type=TokenClassifierOutput,
+ config_class=_CONFIG_FOR_DOC,
+ )
+ def forward(
+ self,
+ input_ids: Optional[torch.LongTensor] = None,
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ inputs_embeds: Optional[torch.Tensor] = None,
+ pixel_values: Optional[torch.FloatTensor] = None,
+ image_sizes: Optional[torch.LongTensor] = None,
+ labels: Optional[torch.Tensor] = None,
+ use_cache: Optional[bool] = None,
+ output_attentions: Optional[bool] = None,
+ output_hidden_states: Optional[bool] = None,
+ return_dict: Optional[bool] = None,
+ **deprecated_arguments,
+ ) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]:
+ r"""
+ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ """
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+ model_outputs = self.model(
+ input_ids,
+ past_key_values=past_key_values,
+ attention_mask=attention_mask,
+ inputs_embeds=inputs_embeds,
+ pixel_values=pixel_values,
+ image_sizes=image_sizes,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ )
+
+ hidden_states = model_outputs[0]
+ hidden_states = self.dropout(hidden_states)
+ logits = self.classifier(hidden_states)
+
+ loss = None
+ if labels is not None:
+ # move labels to correct device to enable model parallelism
+ labels = labels.to(logits.device)
+ batch_size, seq_length = labels.shape
+ loss_fct = CrossEntropyLoss()
+ loss = loss_fct(
+ logits.view(batch_size * seq_length, self.num_labels), labels.view(batch_size * seq_length)
+ )
+
+ if not return_dict:
+ output = (logits,) + model_outputs[2:]
+ return ((loss,) + output) if loss is not None else output
+
+ return TokenClassifierOutput(
+ loss=loss,
+ logits=logits,
+ hidden_states=model_outputs.hidden_states,
+ attentions=model_outputs.attentions,
+ )
\ No newline at end of file
diff --git a/examples/multimodal-modeling/Phi-3-vision/model/processing_phi3_v.py b/examples/multimodal-modeling/Phi-3-vision/model/processing_phi3_v.py
new file mode 100644
index 000000000..1263845d1
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/model/processing_phi3_v.py
@@ -0,0 +1,296 @@
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Modified by Github@GaiZhenbiao
+
+"""
+Processor class for Phi3-V.
+"""
+import re
+from typing import List, Optional, Union
+
+import torch
+from .image_processing_phi3_v import Phi3VImageProcessor
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ImageInput
+from transformers.processing_utils import ProcessorMixin
+from transformers.tokenization_utils_base import PaddingStrategy, TextInput, TruncationStrategy
+from transformers.utils import TensorType
+import transformers
+transformers.Phi3VImageProcessor = Phi3VImageProcessor
+
+class Phi3VProcessor(ProcessorMixin):
+ r"""
+ Constructs a Phi3-V processor which wraps a Phi3-V image processor and a LLaMa tokenizer into a single processor.
+
+ [`Phi3VProcessor`] offers all the functionalities of [`Phi3VImageProcessor`] and [`LlamaTokenizerFast`]. See the
+ [`~Phi3VProcessor.__call__`] and [`~Phi3VProcessor.decode`] for more information.
+
+ Args:
+ image_processor ([`Phi3VImageProcessor`], *optional*):
+ The image processor is a required input.
+ tokenizer ([`LlamaTokenizerFast`], *optional*):
+ The tokenizer is a required input.
+ """
+
+ attributes = ["image_processor", "tokenizer"]
+ image_processor_class = "Phi3VImageProcessor"
+ tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast")
+ special_image_token = "<|image|>"
+
+ def __init__(self, image_processor, tokenizer):
+ self.image_processor = image_processor
+ self.tokenizer = tokenizer
+ self.num_img_tokens = image_processor.num_img_tokens
+ self.img_tokens = [f"<|image_{i+1}|>" for i in range(1000000)]
+
+ def __call__(
+ self,
+ text: Union[TextInput, List[TextInput]],
+ images: ImageInput = None,
+ padding: Union[bool, str, PaddingStrategy] = False,
+ truncation: Union[bool, str, TruncationStrategy] = None,
+ max_length=None,
+ return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
+ ) -> BatchFeature:
+ """
+ Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
+ and `kwargs` arguments to LlamaTokenizerFast's [`~LlamaTokenizerFast.__call__`] if `text` is not `None` to encode
+ the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to
+ Phi3ImageProcessor's [`~Phi3ImageProcessor.__call__`] if `images` is not `None`. Please refer to the doctsring
+ of the above two methods for more information.
+
+ Args:
+ text (`str`, `List[str]`, `List[List[str]]`):
+ The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
+ (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
+ `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
+ images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
+ The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
+ tensor. Both channels-first and channels-last formats are supported.
+ padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
+ Select a strategy to pad the returned sequences (according to the model's padding side and padding
+ index) among:
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
+ max_length (`int`, *optional*):
+ Maximum length of the returned list and optionally padding length (see above).
+ truncation (`bool`, *optional*):
+ Activates truncation to cut input sequences longer than `max_length` to `max_length`.
+ return_tensors (`str` or [`~utils.TensorType`], *optional*):
+ If set, will return tensors of a particular framework. Acceptable values are:
+
+ - `'tf'`: Return TensorFlow `tf.constant` objects.
+ - `'pt'`: Return PyTorch `torch.Tensor` objects.
+ - `'np'`: Return NumPy `np.ndarray` objects.
+ - `'jax'`: Return JAX `jnp.ndarray` objects.
+
+ Returns:
+ [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+ - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
+ - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+ `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+ `None`).
+ - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
+ """
+ if images is not None:
+ image_inputs = self.image_processor(images, return_tensors=return_tensors)
+ else:
+ image_inputs = {}
+ inputs = self._convert_images_texts_to_inputs(image_inputs, text, padding=padding, truncation=truncation, max_length=max_length, return_tensors=return_tensors)
+ return inputs
+
+ def calc_num_image_tokens(self, images: ImageInput):
+ """ Calculate the number of image tokens for each image.
+ Args:
+ images (`ImageInput`):
+ Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
+ passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+ """
+ return self.image_processor.calc_num_image_tokens(images)
+
+ def calc_num_image_tokens_from_image_size(self, width, height):
+ """ Calculate the number of image token for an image with given width and height.
+ Args:
+ width (`int`):
+ Width of the image.
+ height (`int`):
+ Height of the image.
+ """
+ return self.image_processor.calc_num_image_tokens_from_image_size(width, height)
+
+
+ @property
+ def special_image_token_id(self):
+ return self.tokenizer.convert_tokens_to_ids(self.special_image_token)
+
+ def get_special_image_token_id(self):
+ return self.tokenizer.convert_tokens_to_ids(self.special_image_token)
+
+ def _convert_images_texts_to_inputs(self, images, texts, padding=False, truncation=None, max_length=None, return_tensors=None):
+
+ def split_with_separators(s, separators):
+ parts = []
+ start = 0
+ sep_len = {sep: len(sep) for sep in separators}
+ while start < len(s):
+ index = min((s.find(sep, start), sep) for sep in separators if s.find(sep, start) != -1)
+ if index[0] == -1:
+ parts.append(s[start:])
+ break
+ if s[start:index[0]]:
+ parts.append(s[start:index[0]])
+ parts.append(index[1])
+ start = index[0] + sep_len[index[1]]
+ return parts
+
+ def split_with_roles(input_text):
+ parts = split_with_separators(input_text, ["<|user|>\n", "<|end|>\n", "<|assistant|>\n", "<|image_1|>"])
+ new_parts = []
+ current_role = None
+ for p in parts:
+ if p in ["<|user|>\n", "<|assistant|>\n", "<|end|>\n"]:
+ if p == "<|user|>\n":
+ current_role = "user"
+ elif p == "<|assistant|>\n":
+ current_role = "assistant"
+ _type = ["<|user|>\n", "<|assistant|>\n", "<|end|>\n"].index(p) + 1
+ new_parts.append({"role": current_role, "content": p, "type": _type})
+ else:
+ new_parts.append({"role": current_role, "content": p, "type": 0})
+ return new_parts
+
+ if not len(images):
+ model_inputs = self.tokenizer(texts, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length)
+ # prompt_chunks = []
+ label_prompt_chunks = []
+ # the behavior of the tokenizer is very very weird, what I observed is concluded by the following:
+ # 1. "<|user|>\n" is encoded as 3 tokens, while "<|assistant|>\n" is encoded as 1 tokens
+ # 2. tokenizing "I am here" and "\nI am here", the tokens of "I" in these two cases are different ("I" can be any word and is used as an example here)
+ # 3. when tokenizing "<|user|>\nI am here", the tokens of "I" follow the tokenization of "I" in "\nI am here"
+ # 4. when tokenizing "<|assistant|>\nI am here", the tokens of "I" follow the tokenization of "I" in "I am here"
+ # [Edited by zhenwei - 2024-06-01 22:25]
+ for chunk in split_with_roles(texts):
+ if chunk["role"] == "assistant" and chunk['type'] in [0, 3]:
+ tmp_input_ids = self.tokenizer(chunk["content"], add_special_tokens=False).input_ids
+ # prompt_chunks.append(tmp_input_ids)
+ label_prompt_chunks.append(tmp_input_ids)
+ else:
+ tmp_input_ids = self.tokenizer('\n' + chunk["content"], add_special_tokens=False).input_ids[2:]
+ # prompt_chunks.append(tmp_input_ids)
+ label_prompt_chunks.append([-100 for _ in range(len(tmp_input_ids))])
+
+ labels = [-100]
+ for chunk in label_prompt_chunks:
+ labels.extend(chunk)
+ # input_ids = [1]
+ # for chunk in prompt_chunks:
+ # input_ids.extend(chunk)
+
+ labels = torch.tensor(labels, dtype=torch.long).unsqueeze(0)
+ # with open('tmp/input_ids.txt', 'w') as f:
+ # print(texts, file=f)
+ # print(split_with_roles(texts), file=f)
+ # print("input_ids_before", file=f)
+ # print(model_inputs['input_ids'][0].tolist(), file=f)
+ # print("input_ids", file=f)
+ # print(input_ids, file=f)
+ assert labels.shape[1] == model_inputs['input_ids'].shape[1], f"labels length: {labels.shape[1]}, input_ids length: {model_inputs['input_ids'].shape[1]}"
+ return BatchFeature(data={**model_inputs, "labels": labels})
+
+
+ if 'num_img_tokens' in images:
+ num_img_tokens = images['num_img_tokens']
+ else:
+ assert 'num_crops' in images, 'num_crops must be provided in images if num_img_tokens is not provided'
+ num_crops = images['num_crops']
+ num_img_tokens = [_num_crops * self.num_img_tokens for _num_crops in num_crops]
+
+ images, image_sizes = images['pixel_values'], images['image_sizes']
+
+ pattern = r"<\|image_\d+\|>"
+ # image_tags needs to start from 1 to n
+ image_tags = re.findall(pattern, texts)
+ # image_ids = [int(s.split("|")[1].split("_")[-1]) * -1 for s in image_tags]
+ # image_ids_pad = [[iid]*num_img_tokens[i] for i, iid in enumerate(image_ids)]
+ image_ids = [int(s.split("|")[1].split("_")[-1]) for s in image_tags]
+ unique_image_ids = sorted(list(set(image_ids)))
+ # image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be [1, 4, 5]
+ # check the condition
+ assert unique_image_ids == list(range(1, len(unique_image_ids)+1)), f"image_ids must start from 1, and must be continuous int, e.g. [1, 2, 3], cannot be {unique_image_ids}"
+ # total images must be the same as the number of image tags
+ assert len(unique_image_ids) == len(images), f"total images must be the same as the number of image tags, got {len(unique_image_ids)} image tags and {len(images)} images"
+
+ image_ids_pad = [[-iid]*num_img_tokens[iid-1] for iid in image_ids]
+
+ prompt_chunks = []
+ label_prompt_chunks = []
+ for chunk in split_with_roles(texts):
+ if chunk["role"] == "assistant" and chunk['type'] in [0, 3]:
+ tmp_input_ids = self.tokenizer(chunk["content"], add_special_tokens=False).input_ids
+ prompt_chunks.append(tmp_input_ids)
+ label_prompt_chunks.append(tmp_input_ids)
+ else:
+ if chunk["content"] == "<|image_1|>":
+ tmp_input_ids = image_ids_pad.pop(0)
+ else:
+ tmp_input_ids = self.tokenizer('\n' + chunk["content"], add_special_tokens=False).input_ids[2:]
+ prompt_chunks.append(tmp_input_ids)
+ label_prompt_chunks.append([-100 for _ in range(len(tmp_input_ids))])
+
+ input_ids = [1]
+ labels = [-100]
+ for chunk in prompt_chunks:
+ input_ids.extend(chunk)
+ for chunk in label_prompt_chunks:
+ labels.extend(chunk)
+ input_ids = torch.tensor(input_ids, dtype=torch.long).unsqueeze(0)
+ labels = torch.tensor(labels, dtype=torch.long).unsqueeze(0)
+ attention_mask = (input_ids > -1000000).to(torch.long)
+
+ return BatchFeature(data={"input_ids": input_ids,
+ "attention_mask": attention_mask,
+ "pixel_values": images,
+ "image_sizes": image_sizes,
+ "labels": labels})
+
+
+ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama
+ def batch_decode(self, *args, **kwargs):
+ """
+ This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+ refer to the docstring of this method for more information.
+ """
+ return self.tokenizer.batch_decode(*args, **kwargs)
+
+ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama
+ def decode(self, *args, **kwargs):
+ """
+ This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+ the docstring of this method for more information.
+ """
+ return self.tokenizer.decode(*args, **kwargs)
+
+ @property
+ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
+ def model_input_names(self):
+ tokenizer_input_names = self.tokenizer.model_input_names
+ image_processor_input_names = self.image_processor.model_input_names
+ return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
\ No newline at end of file
diff --git a/examples/multimodal-modeling/Phi-3-vision/requirements.txt b/examples/multimodal-modeling/Phi-3-vision/requirements.txt
new file mode 100644
index 000000000..43e29fe2f
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/requirements.txt
@@ -0,0 +1,18 @@
+transformers==4.41.0
+torch
+tiktoken
+transformers_stream_generator
+peft
+sentencepiece
+einops
+accelerate
+datasets
+protobuf
+auto-gptq
+openpyxl
+wandb
+py-cpuinfo
+Pillow
+torchvision
+lm-eval==0.4.2
+setuptools==69.5.1
\ No newline at end of file
diff --git a/examples/multimodal-modeling/Phi-3-vision/run_autoround.sh b/examples/multimodal-modeling/Phi-3-vision/run_autoround.sh
new file mode 100644
index 000000000..3583704bc
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/run_autoround.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -x
+device=0
+model_name=microsoft/Phi-3-vision-128k-instruct
+CUDA_VISIBLE_DEVICES=$device \
+python3 main.py \
+--model_name=$model_name \
+--deployment_device 'auto_round' \
+--image_folder /PATH/TO/coco/images/train2017 \
+--question_file /PATH/TO/llava_v1_5_mix665k.json \
+--output_dir "./tmp_autoround"
diff --git a/examples/multimodal-modeling/Phi-3-vision/run_autoround_on_gaudi.sh b/examples/multimodal-modeling/Phi-3-vision/run_autoround_on_gaudi.sh
new file mode 100644
index 000000000..90764afbd
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/run_autoround_on_gaudi.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -x
+model_name=microsoft/Phi-3-vision-128k-instruct
+
+python3 main.py \
+ --model_name $model_name \
+ --group_size 128 \
+ --bits 4 \
+ --deployment_device "fake" \
+ --output_dir "./tmp_autoround"
\ No newline at end of file
diff --git a/examples/multimodal-modeling/Phi-3-vision/run_eval.sh b/examples/multimodal-modeling/Phi-3-vision/run_eval.sh
new file mode 100644
index 000000000..da14a0213
--- /dev/null
+++ b/examples/multimodal-modeling/Phi-3-vision/run_eval.sh
@@ -0,0 +1,48 @@
+export https_proxy=http://proxy.ims.intel.com:911
+export http_proxy=http://proxy.ims.intel.com:911
+export HF_HOME=/home/weiweiz1/.cache/
+
+# Mistral-7B-Instruct-v0.2
+# device=3
+# Baichuan2-7B-Chat Phi-3-mini-4k-instruct
+# Llama-2-7b-chat-hf
+# lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu,
+# ceval-valid,cmmlu
+# dir=/data5/zww/test_faster/
+# dir=/models
+# for model in Phi-3-mini-4k-instruct Meta-Llama-3-8B-Instruct
+# do
+# echo ${model}/default
+# CUDA_VISIBLE_DEVICES=$device \
+# python3 eval_042/evaluation.py --model_name ${dir}${model}_default/$model-autoround-w4g128-gpu \
+# --trust_remote_code \
+# --eval_bs 16 --tasks gsm8k,ceval-valid,cmmlu \
+# 2>&1| tee -a /data4/zww/test_faster/rounding_${model}_rtn.txt
+# echo ${model}/rtn
+# done&
+
+device=2
+dir=/data4/zww/tmp/
+# dir=/data5/models/
+for model in Phi-3-vision-128k-instruct
+do
+ echo ${model}
+ CUDA_VISIBLE_DEVICES=$device \
+ python3 eval_042/evaluation.py --model_name ${dir}/$model-autoround-w4g128-round \
+ --trust_remote_code \
+ --eval_bs 16 --tasks lambada_openai \
+ 2>&1| tee -a /data4/zww/test_faster/rounding_${model}.txt
+ echo ${model}
+done
+# dir=/data5/zww/test_faster/
+# for model in Phi-3-mini-4k-instruct Mistral-7B-Instruct-v0.2
+# do
+# echo ${model}/rtn
+# CUDA_VISIBLE_DEVICES=$device \
+# python3 eval_042/evaluation.py --model_name ${dir}${model}_rtn/$model-autoround-w4g128-gpu \
+# --trust_remote_code \
+# --eval_bs 16 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu,gsm8k \
+# 2>&1| tee -a /data4/zww/test_faster/rounding_${model}_rtn.txt
+# echo ${model}/rtn
+# done
+
diff --git a/test/test_block_names.py b/test/test_block_names.py
index 5b6218a51..bee555a22 100644
--- a/test/test_block_names.py
+++ b/test/test_block_names.py
@@ -168,8 +168,18 @@ class TestQuantizationBlocks(unittest.TestCase):
import auto_gptq
except:
return
- autoround.save_quantized("./saved", inplace=False, safe_serialization=False)
- autoround.save_quantized(output_dir="./saved", inplace=False, format="auto_round")
+ quantized_model_path = "./saved"
+ autoround.save_quantized(quantized_model_path, inplace=False, safe_serialization=False, format="auto_round")
+
+ from auto_round.auto_quantizer import AutoHfQuantizer
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
+ quant_config = model.config.quantization_config
+ assert quant_config.quant_block_list is not None
@@ -177,3 +187,4 @@ class TestQuantizationBlocks(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+