Open-Sora: Democratizing Efficient Video Production for All
Python
added support for llava mistral (#44)
Frank Lee65cb8eb67cb8
diff --git a/tools/caption/acceleration/llava/policies/__init__.py b/tools/caption/acceleration/llava/policies/__init__.pynew file mode 100644index 0000000..35998d4--- /dev/null+++ b/tools/caption/acceleration/llava/policies/__init__.py@@ -0,0 +1,2 @@+from .llama import LlavaLlamaForCausalLMPolicy+from .mistral import LlavaMistralForCausalLMPolicydiff --git a/tools/caption/acceleration/llava/policy.py b/tools/caption/acceleration/llava/policies/llama.pysimilarity index 83%rename from tools/caption/acceleration/llava/policy.pyrename to tools/caption/acceleration/llava/policies/llama.pyindex 235d904..dff8f01 100644--- a/tools/caption/acceleration/llava/policy.py+++ b/tools/caption/acceleration/llava/policies/llama.py@@ -1,26 +1,25 @@-import warningsfrom typing import Dict, Unionimport torch.nn as nnfrom colossalai.shardformer.layer import Linear1D_Col, Linear1D_Rowfrom colossalai.shardformer.policies.base_policy import ModulePolicyDescription, Policy, SubModuleReplacementDescription-__all__ = ["LlavaPolicy", "LlavaForCausalLMPolicy"]+__all__ = ["LlavaLlamaPolicy", "LlavaLlamaForCausalLMPolicy"]-class LlavaPolicy(Policy):+class LlavaLlamaPolicy(Policy):def config_sanity_check(self):passdef preprocess(self):if self.shard_config.enable_tensor_parallelism:# Resize embedding- vocab_size = self.model.config.vocab_size- world_size = self.shard_config.tensor_parallel_size+ self.model.config.vocab_size+ self.shard_config.tensor_parallel_size- if vocab_size % world_size != 0:- new_vocab_size = vocab_size + world_size - vocab_size % world_size- self.model.resize_token_embeddings(new_vocab_size)+ # if vocab_size % world_size != 0:+ # new_vocab_size = vocab_size + world_size - vocab_size % world_size+ # self.model.resize_token_embeddings(new_vocab_size)return self.model@@ -29,10 +28,6 @@ class LlavaPolicy(Policy):policy = {}- if self.shard_config.enable_sequence_parallelism:- self.shard_config.enable_sequence_parallelism = False- warnings.warn("Llama doesn't support sequence parallelism now, will ignore the sequence parallelism flag.")-if self.shard_config.enable_tensor_parallelism:decoder_attribute_replacement = {"self_attn.hidden_size": self.model.config.hidden_size // self.shard_config.tensor_parallel_size,@@ -83,7 +78,7 @@ class LlavaPolicy(Policy):return self.model-class LlavaForCausalLMPolicy(LlavaPolicy):+class LlavaLlamaForCausalLMPolicy(LlavaLlamaPolicy):def module_policy(self):from transformers import LlamaForCausalLMdiff --git a/tools/caption/acceleration/llava/policies/mistral.py b/tools/caption/acceleration/llava/policies/mistral.pynew file mode 100644index 0000000..0afea57--- /dev/null+++ b/tools/caption/acceleration/llava/policies/mistral.py@@ -0,0 +1,113 @@+import warnings+from typing import Dict, Union++import torch.nn as nn+from colossalai.shardformer.layer import Linear1D_Col, Linear1D_Row, VocabParallelEmbedding1D+from colossalai.shardformer.policies.base_policy import ModulePolicyDescription, Policy, SubModuleReplacementDescription++__all__ = ["LlavaMistralPolicy", "LlavaMistralForCausalLMPolicy"]+++class LlavaMistralPolicy(Policy):+ def config_sanity_check(self):+ pass++ def preprocess(self):+ if self.shard_config.enable_tensor_parallelism:+ # Resize embedding+ vocab_size = self.model.config.vocab_size+ world_size = self.shard_config.tensor_parallel_size++ if vocab_size % world_size != 0:+ new_vocab_size = vocab_size + world_size - vocab_size % world_size+ self.model.resize_token_embeddings(new_vocab_size)++ return self.model++ def module_policy(self) -> Dict[Union[str, nn.Module], ModulePolicyDescription]:+ from transformers.models.mistral.modeling_mistral import MistralDecoderLayer, MistralModel++ policy = {}++ if self.shard_config.enable_sequence_parallelism:+ self.shard_config.enable_sequence_parallelism = False+ warnings.warn(+ "Mistral doesn't support sequence parallelism now, will ignore the sequence parallelism flag."+ )++ if self.shard_config.enable_tensor_parallelism:+ decoder_attribute_replacement = {+ "self_attn.hidden_size": self.model.config.hidden_size // self.shard_config.tensor_parallel_size,+ "self_attn.num_heads": self.model.config.num_attention_heads // self.shard_config.tensor_parallel_size,+ "self_attn.num_key_value_heads": self.model.config.num_key_value_heads+ // self.shard_config.tensor_parallel_size,+ }++ policy[MistralDecoderLayer] = ModulePolicyDescription(+ attribute_replacement=decoder_attribute_replacement,+ sub_module_replacement=[+ SubModuleReplacementDescription(+ suffix="self_attn.q_proj",+ target_module=Linear1D_Col,+ ),+ SubModuleReplacementDescription(+ suffix="self_attn.k_proj",+ target_module=Linear1D_Col,+ ),+ SubModuleReplacementDescription(+ suffix="self_attn.v_proj",+ target_module=Linear1D_Col,+ ),+ SubModuleReplacementDescription(+ suffix="self_attn.o_proj",+ target_module=Linear1D_Row,+ ),+ SubModuleReplacementDescription(+ suffix="mlp.gate_proj",+ target_module=Linear1D_Col,+ ),+ SubModuleReplacementDescription(+ suffix="mlp.up_proj",+ target_module=Linear1D_Col,+ ),+ SubModuleReplacementDescription(+ suffix="mlp.down_proj",+ target_module=Linear1D_Row,+ ),+ ],+ )++ self.append_or_create_submodule_replacement(+ description=SubModuleReplacementDescription(+ suffix="embed_tokens",+ target_module=VocabParallelEmbedding1D,+ ),+ policy=policy,+ target_key=MistralModel,+ )++ return policy++ def postprocess(self):+ return self.model+++class LlavaMistralForCausalLMPolicy(LlavaMistralPolicy):+ def module_policy(self):+ from transformers import MistralForCausalLM++ policy = super().module_policy()++ if self.shard_config.enable_tensor_parallelism:+ # add a new item for casual lm+ new_item = {+ MistralForCausalLM: ModulePolicyDescription(+ sub_module_replacement=[+ SubModuleReplacementDescription(+ suffix="lm_head", target_module=Linear1D_Col, kwargs=dict(gather_output=True)+ )+ ]+ )+ }+ policy.update(new_item)+ return policydiff --git a/tools/caption/caption_llava.py b/tools/caption/caption_llava.pyindex 2f75595..abf12e1 100644--- a/tools/caption/caption_llava.py+++ b/tools/caption/caption_llava.py@@ -17,7 +17,7 @@ from llava.utils import disable_torch_initfrom torch.utils.data.distributed import DistributedSamplerfrom tqdm import tqdm-from .acceleration.llava.policy import LlavaForCausalLMPolicy+from .acceleration.llava.policies import LlavaLlamaForCausalLMPolicy, LlavaMistralForCausalLMPolicyfrom .utils import IMG_EXTENSIONS, PROMPTS, VID_EXTENSIONS, Timer, VideoTextDataset, collate_fndisable_torch_init()@@ -93,14 +93,22 @@ def main(args):# ======================================================# 3. Apply system optimization# ======================================================- # create huggingface model as normaltp_size = dist.get_world_size(tp_group)shard_config = ShardConfig(tensor_parallel_process_group=tp_group if tp_size > 1 else None,enable_tensor_parallelism=True if tp_size > 1 else False,)shard_former = ShardFormer(shard_config=shard_config)- model = shard_former.optimize(model, policy=LlavaForCausalLMPolicy())[0].cuda()++ # check the model type+ model_name = model.__class__.__name__+ print(model_name)+ if model_name == "LlavaLlamaForCausalLM":+ model = shard_former.optimize(model, policy=LlavaLlamaForCausalLMPolicy())[0].cuda()+ elif model_name == "LlavaMistralForCausalLM":+ model = shard_former.optimize(model, policy=LlavaMistralForCausalLMPolicy())[0].cuda()+ else:+ print(f"The shardformer policy for {model_name} is not implemented, skip")torch.cuda.empty_cache()# ======================================================@@ -209,7 +217,6 @@ def main(args):if args.profile:encode_time = []- frame_extraction_time = []generate_time = []output_length = []total_time = []@@ -292,7 +299,6 @@ def main(args):print(output_length)num_samples_after_warmup = total_num_videos - args.bs * args.profile_warmup * dp_sizeprint(f"throughput (samples/s): {num_samples_after_warmup / sum(total_time)}")- print(f"average frame extraction time per sample: {sum(frame_extraction_time) / num_samples_after_warmup}")print(f"average encode time per sample: {sum(encode_time) / num_samples_after_warmup}")print(f"average generate time per sample: {sum(generate_time) / num_samples_after_warmup}")print(f"average number of tokens characters per sample: {sum(output_length) / num_samples_after_warmup}")