Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

added support for llava mistral (#44)

Frank Lee65cb8eb67cb8
diff --git a/tools/caption/acceleration/llava/policies/__init__.py b/tools/caption/acceleration/llava/policies/__init__.py
new file mode 100644
index 0000000..35998d4
--- /dev/null
+++ b/tools/caption/acceleration/llava/policies/__init__.py
@@ -0,0 +1,2 @@
+from .llama import LlavaLlamaForCausalLMPolicy
+from .mistral import LlavaMistralForCausalLMPolicy
diff --git a/tools/caption/acceleration/llava/policy.py b/tools/caption/acceleration/llava/policies/llama.py
similarity index 83%
rename from tools/caption/acceleration/llava/policy.py
rename to tools/caption/acceleration/llava/policies/llama.py
index 235d904..dff8f01 100644
--- a/tools/caption/acceleration/llava/policy.py
+++ b/tools/caption/acceleration/llava/policies/llama.py
@@ -1,26 +1,25 @@
-import warnings
from typing import Dict, Union
import torch.nn as nn
from colossalai.shardformer.layer import Linear1D_Col, Linear1D_Row
from colossalai.shardformer.policies.base_policy import ModulePolicyDescription, Policy, SubModuleReplacementDescription
-__all__ = ["LlavaPolicy", "LlavaForCausalLMPolicy"]
+__all__ = ["LlavaLlamaPolicy", "LlavaLlamaForCausalLMPolicy"]
-class LlavaPolicy(Policy):
+class LlavaLlamaPolicy(Policy):
def config_sanity_check(self):
pass
def preprocess(self):
if self.shard_config.enable_tensor_parallelism:
# Resize embedding
- vocab_size = self.model.config.vocab_size
- world_size = self.shard_config.tensor_parallel_size
+ self.model.config.vocab_size
+ self.shard_config.tensor_parallel_size
- if vocab_size % world_size != 0:
- new_vocab_size = vocab_size + world_size - vocab_size % world_size
- self.model.resize_token_embeddings(new_vocab_size)
+ # if vocab_size % world_size != 0:
+ # new_vocab_size = vocab_size + world_size - vocab_size % world_size
+ # self.model.resize_token_embeddings(new_vocab_size)
return self.model
@@ -29,10 +28,6 @@ class LlavaPolicy(Policy):
policy = {}
- if self.shard_config.enable_sequence_parallelism:
- self.shard_config.enable_sequence_parallelism = False
- warnings.warn("Llama doesn't support sequence parallelism now, will ignore the sequence parallelism flag.")
-
if self.shard_config.enable_tensor_parallelism:
decoder_attribute_replacement = {
"self_attn.hidden_size": self.model.config.hidden_size // self.shard_config.tensor_parallel_size,
@@ -83,7 +78,7 @@ class LlavaPolicy(Policy):
return self.model
-class LlavaForCausalLMPolicy(LlavaPolicy):
+class LlavaLlamaForCausalLMPolicy(LlavaLlamaPolicy):
def module_policy(self):
from transformers import LlamaForCausalLM
diff --git a/tools/caption/acceleration/llava/policies/mistral.py b/tools/caption/acceleration/llava/policies/mistral.py
new file mode 100644
index 0000000..0afea57
--- /dev/null
+++ b/tools/caption/acceleration/llava/policies/mistral.py
@@ -0,0 +1,113 @@
+import warnings
+from typing import Dict, Union
+
+import torch.nn as nn
+from colossalai.shardformer.layer import Linear1D_Col, Linear1D_Row, VocabParallelEmbedding1D
+from colossalai.shardformer.policies.base_policy import ModulePolicyDescription, Policy, SubModuleReplacementDescription
+
+__all__ = ["LlavaMistralPolicy", "LlavaMistralForCausalLMPolicy"]
+
+
+class LlavaMistralPolicy(Policy):
+ def config_sanity_check(self):
+ pass
+
+ def preprocess(self):
+ if self.shard_config.enable_tensor_parallelism:
+ # Resize embedding
+ vocab_size = self.model.config.vocab_size
+ world_size = self.shard_config.tensor_parallel_size
+
+ if vocab_size % world_size != 0:
+ new_vocab_size = vocab_size + world_size - vocab_size % world_size
+ self.model.resize_token_embeddings(new_vocab_size)
+
+ return self.model
+
+ def module_policy(self) -> Dict[Union[str, nn.Module], ModulePolicyDescription]:
+ from transformers.models.mistral.modeling_mistral import MistralDecoderLayer, MistralModel
+
+ policy = {}
+
+ if self.shard_config.enable_sequence_parallelism:
+ self.shard_config.enable_sequence_parallelism = False
+ warnings.warn(
+ "Mistral doesn't support sequence parallelism now, will ignore the sequence parallelism flag."
+ )
+
+ if self.shard_config.enable_tensor_parallelism:
+ decoder_attribute_replacement = {
+ "self_attn.hidden_size": self.model.config.hidden_size // self.shard_config.tensor_parallel_size,
+ "self_attn.num_heads": self.model.config.num_attention_heads // self.shard_config.tensor_parallel_size,
+ "self_attn.num_key_value_heads": self.model.config.num_key_value_heads
+ // self.shard_config.tensor_parallel_size,
+ }
+
+ policy[MistralDecoderLayer] = ModulePolicyDescription(
+ attribute_replacement=decoder_attribute_replacement,
+ sub_module_replacement=[
+ SubModuleReplacementDescription(
+ suffix="self_attn.q_proj",
+ target_module=Linear1D_Col,
+ ),
+ SubModuleReplacementDescription(
+ suffix="self_attn.k_proj",
+ target_module=Linear1D_Col,
+ ),
+ SubModuleReplacementDescription(
+ suffix="self_attn.v_proj",
+ target_module=Linear1D_Col,
+ ),
+ SubModuleReplacementDescription(
+ suffix="self_attn.o_proj",
+ target_module=Linear1D_Row,
+ ),
+ SubModuleReplacementDescription(
+ suffix="mlp.gate_proj",
+ target_module=Linear1D_Col,
+ ),
+ SubModuleReplacementDescription(
+ suffix="mlp.up_proj",
+ target_module=Linear1D_Col,
+ ),
+ SubModuleReplacementDescription(
+ suffix="mlp.down_proj",
+ target_module=Linear1D_Row,
+ ),
+ ],
+ )
+
+ self.append_or_create_submodule_replacement(
+ description=SubModuleReplacementDescription(
+ suffix="embed_tokens",
+ target_module=VocabParallelEmbedding1D,
+ ),
+ policy=policy,
+ target_key=MistralModel,
+ )
+
+ return policy
+
+ def postprocess(self):
+ return self.model
+
+
+class LlavaMistralForCausalLMPolicy(LlavaMistralPolicy):
+ def module_policy(self):
+ from transformers import MistralForCausalLM
+
+ policy = super().module_policy()
+
+ if self.shard_config.enable_tensor_parallelism:
+ # add a new item for casual lm
+ new_item = {
+ MistralForCausalLM: ModulePolicyDescription(
+ sub_module_replacement=[
+ SubModuleReplacementDescription(
+ suffix="lm_head", target_module=Linear1D_Col, kwargs=dict(gather_output=True)
+ )
+ ]
+ )
+ }
+ policy.update(new_item)
+ return policy
diff --git a/tools/caption/caption_llava.py b/tools/caption/caption_llava.py
index 2f75595..abf12e1 100644
--- a/tools/caption/caption_llava.py
+++ b/tools/caption/caption_llava.py
@@ -17,7 +17,7 @@ from llava.utils import disable_torch_init
from torch.utils.data.distributed import DistributedSampler
from tqdm import tqdm
-from .acceleration.llava.policy import LlavaForCausalLMPolicy
+from .acceleration.llava.policies import LlavaLlamaForCausalLMPolicy, LlavaMistralForCausalLMPolicy
from .utils import IMG_EXTENSIONS, PROMPTS, VID_EXTENSIONS, Timer, VideoTextDataset, collate_fn
disable_torch_init()
@@ -93,14 +93,22 @@ def main(args):
# ======================================================
# 3. Apply system optimization
# ======================================================
- # create huggingface model as normal
tp_size = dist.get_world_size(tp_group)
shard_config = ShardConfig(
tensor_parallel_process_group=tp_group if tp_size > 1 else None,
enable_tensor_parallelism=True if tp_size > 1 else False,
)
shard_former = ShardFormer(shard_config=shard_config)
- model = shard_former.optimize(model, policy=LlavaForCausalLMPolicy())[0].cuda()
+
+ # check the model type
+ model_name = model.__class__.__name__
+ print(model_name)
+ if model_name == "LlavaLlamaForCausalLM":
+ model = shard_former.optimize(model, policy=LlavaLlamaForCausalLMPolicy())[0].cuda()
+ elif model_name == "LlavaMistralForCausalLM":
+ model = shard_former.optimize(model, policy=LlavaMistralForCausalLMPolicy())[0].cuda()
+ else:
+ print(f"The shardformer policy for {model_name} is not implemented, skip")
torch.cuda.empty_cache()
# ======================================================
@@ -209,7 +217,6 @@ def main(args):
if args.profile:
encode_time = []
- frame_extraction_time = []
generate_time = []
output_length = []
total_time = []
@@ -292,7 +299,6 @@ def main(args):
print(output_length)
num_samples_after_warmup = total_num_videos - args.bs * args.profile_warmup * dp_size
print(f"throughput (samples/s): {num_samples_after_warmup / sum(total_time)}")
- print(f"average frame extraction time per sample: {sum(frame_extraction_time) / num_samples_after_warmup}")
print(f"average encode time per sample: {sum(encode_time) / num_samples_after_warmup}")
print(f"average generate time per sample: {sum(generate_time) / num_samples_after_warmup}")
print(f"average number of tokens characters per sample: {sum(output_length) / num_samples_after_warmup}")