Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

reduce flux CUDA CI tuning memory from 30G to 5GB (#1694)

Xin He751216a44108
diff --git a/test/fixtures.py b/test/fixtures.py
index fe74030f6..54bb8abdc 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -118,6 +118,15 @@ def tiny_flux_model_path():
num_layers=1,
is_diffusion=True,
from_config=True,
+ config_overrides={
+ "num_attention_heads": 2,
+ "attention_head_dim": 128,
+ "joint_attention_dim": 256,
+ "pooled_projection_dim": 256,
+ "hidden_size": 256,
+ "max_position_embeddings": 128,
+ "intermediate_size": 256,
+ },
)
yield tiny_model_path
shutil.rmtree(tiny_model_path, ignore_errors=True)
diff --git a/test/helpers.py b/test/helpers.py
index a45c953f5..739b8925f 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -222,8 +222,27 @@ def _reduce_config_layers(config, num_layers, num_experts=None):
_apply(inner_cfg)
+def _apply_config_overrides(config, overrides):
+ """Apply arbitrary key-value overrides to a config (dict or object)."""
+ if not overrides:
+ return
+ for key, value in overrides.items():
+ if isinstance(config, dict):
+ if key in config:
+ config[key] = value
+ elif hasattr(config, key):
+ setattr(config, key, value)
+
+
def get_tiny_model(
- model_name_or_path, num_layers=2, num_experts=None, is_mllm=False, from_config=False, is_diffusion=False, **kwargs
+ model_name_or_path,
+ num_layers=2,
+ num_experts=None,
+ is_mllm=False,
+ from_config=False,
+ is_diffusion=False,
+ config_overrides=None,
+ **kwargs,
):
"""Generate a tiny model by slicing layers from the original model.
@@ -266,12 +285,14 @@ def get_tiny_model(
with open(os.path.join(local_dir, folder_name, "config.json"), "r", encoding="utf-8") as f:
config = json.load(f)
_reduce_config_layers(config, num_layers, num_experts)
+ _apply_config_overrides(config, config_overrides)
return getattr(getattr(diffusers_module, mod_name), "from_config")(config)
else:
config = transformers.AutoConfig.from_pretrained(
os.path.join(local_dir, folder_name, "config.json")
)
_reduce_config_layers(config, num_layers, num_experts)
+ _apply_config_overrides(config, config_overrides)
return getattr(getattr(transformers_module, mod_name), "_from_config")(config)
with open(os.path.join(local_dir, "model_index.json"), "r", encoding="utf-8") as f:
@@ -370,6 +391,7 @@ def save_tiny_model(
force_untie=False,
from_config=False,
is_diffusion=False,
+ config_overrides=None,
**kwargs,
):
"""Generate a tiny model and save to the specified path.
@@ -392,6 +414,7 @@ def save_tiny_model(
is_mllm=is_mllm,
from_config=from_config,
is_diffusion=is_diffusion,
+ config_overrides=config_overrides,
**kwargs,
)
if force_untie: